oxedyne/fe2o3 — fe2o3_text/tests/annealer_corpus/nu

oxedyne/fe2o3/fe2o3_text/tests/annealer_corpus/nu_parser.rs

274 KiB, 1 run

created by r1870400018:11760, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1	#![allow(clippy::byte_char_slices)]
2
3	use crate::{
4	Token, TokenContents,
5	lex::{LexState, is_assignment_operator, lex, lex_n_tokens, lex_signature},
6	lite_parser::{LiteCommand, LitePipeline, LiteRedirection, LiteRedirectionTarget, lite_parse},
7	parse_keywords::*,
8	parse_patterns::parse_pattern,
9	parse_shape_specs::{parse_completer, parse_shape_name, parse_type},
10	type_check::{self, check_range_types, math_result_type, type_compatible},
11	};
12	use itertools::Itertools;
13	use log::trace;
14	use nu_engine::DIR_VAR_PARSER_INFO;
15	use nu_protocol::{
16	BlockId, DeclId, DidYouMean, ENV_VARIABLE_ID, FilesizeUnit, Flag, IN_VARIABLE_ID, ParseError,
17	PositionalArg, ShellError, Signature, Span, Spanned, SyntaxShape, Type, Value, VarId,
18	ast::*,
19	casing::Casing,
20	did_you_mean,
21	engine::{CommandType, StateWorkingSet},
22	eval_const::eval_constant,
23	};
24	use std::{
25	collections::{HashMap, HashSet},
26	str,
27	sync::Arc,
28	};
29
30	pub(crate) const PERCENT_FORCED_BUILTIN_PARSER_INFO: &str = "percent_forced_builtin";
31
32	pub fn garbage(working_set: &mut StateWorkingSet, span: Span) -> Expression {
33	Expression::garbage(working_set, span)
34	}
35
36	pub fn garbage_pipeline(working_set: &mut StateWorkingSet, spans: &[Span]) -> Pipeline {
37	Pipeline::from_vec(vec![garbage(working_set, Span::concat(spans))])
38	}
39
40	fn is_identifier_byte(b: u8) -> bool {
41	b != b'.'
42	&& b != b'['
43	&& b != b'('
44	&& b != b'{'
45	&& b != b'+'
46	&& b != b'-'
47	&& b != b'*'
48	&& b != b'^'
49	&& b != b'%'
50	&& b != b'/'
51	&& b != b'='
52	&& b != b'!'
53	&& b != b'<'
54	&& b != b'>'
55	&& b != b'&'
56	&& b != b'\|'
57	}
58
59	pub fn is_math_expression_like(working_set: &mut StateWorkingSet, span: Span) -> bool {
60	let bytes = working_set.get_span_contents(span);
61	if bytes.is_empty() {
62	return false;
63	}
64
65	if bytes == b"true"
66	\|\| bytes == b"false"
67	\|\| bytes == b"null"
68	\|\| bytes == b"not"
69	\|\| bytes == b"if"
70	\|\| bytes == b"match"
71	{
72	return true;
73	}
74
75	let b = bytes[0];
76
77	// check for raw string
78	if bytes.starts_with(b"r#") {
79	return true;
80	}
81
82	if b == b'(' \|\| b == b'{' \|\| b == b'[' \|\| b == b'$' \|\| b == b'"' \|\| b == b'\'' \|\| b == b'-' {
83	return true;
84	}
85
86	let starting_error_count = working_set.parse_errors.len();
87
88	// Number
89	parse_number(working_set, span);
90	if working_set.parse_errors.len() == starting_error_count {
91	return true;
92	}
93	working_set.parse_errors.truncate(starting_error_count);
94
95	// Filesize
96	parse_filesize(working_set, span);
97	if working_set.parse_errors.len() == starting_error_count {
98	return true;
99	}
100	working_set.parse_errors.truncate(starting_error_count);
101
102	parse_duration(working_set, span);
103	if working_set.parse_errors.len() == starting_error_count {
104	return true;
105	}
106	working_set.parse_errors.truncate(starting_error_count);
107
108	parse_datetime(working_set, span);
109	if working_set.parse_errors.len() == starting_error_count {
110	return true;
111	}
112	working_set.parse_errors.truncate(starting_error_count);
113
114	parse_binary(working_set, span);
115	// We need an additional negate match to check if the last error was unexpected
116	// or more specifically, if it was `ParseError::InvalidBinaryString`.
117	// If so, we suppress the error and stop parsing to the next (which is `parse_range()`).
118	if working_set.parse_errors.len() == starting_error_count {
119	return true;
120	} else if !matches!(
121	working_set.parse_errors.last(),
122	Some(ParseError::Expected(_, _))
123	) {
124	working_set.parse_errors.truncate(starting_error_count);
125	return true;
126	}
127	working_set.parse_errors.truncate(starting_error_count);
128
129	let is_range = parse_range(working_set, span).is_some();
130	working_set.parse_errors.truncate(starting_error_count);
131	is_range
132	}
133
134	fn is_env_variable_name(bytes: &[u8]) -> bool {
135	if bytes.is_empty() {
136	return false;
137	}
138
139	let first = bytes[0];
140	if !first.is_ascii_alphabetic() && first != b'_' {
141	return false;
142	}
143
144	bytes
145	.iter()
146	.skip(1)
147	.all(\|&b\| b.is_ascii_alphanumeric() \|\| b == b'_')
148	}
149
150	fn is_identifier(bytes: &[u8]) -> bool {
151	bytes.iter().all(\|x\| is_identifier_byte(*x))
152	}
153
154	pub fn is_variable(bytes: &[u8]) -> bool {
155	if bytes.len() > 1 && bytes[0] == b'$' {
156	is_identifier(&bytes[1..])
157	} else {
158	is_identifier(bytes)
159	}
160	}
161
162	pub fn trim_quotes(bytes: &[u8]) -> &[u8] {
163	if (bytes.starts_with(b"\"") && bytes.ends_with(b"\"") && bytes.len() > 1)
164	\|\| (bytes.starts_with(b"\'") && bytes.ends_with(b"\'") && bytes.len() > 1)
165	\|\| (bytes.starts_with(b"`") && bytes.ends_with(b"`") && bytes.len() > 1)
166	{
167	&bytes[1..(bytes.len() - 1)]
168	} else {
169	bytes
170	}
171	}
172
173	pub fn trim_quotes_str(s: &str) -> &str {
174	if (s.starts_with('"') && s.ends_with('"') && s.len() > 1)
175	\|\| (s.starts_with('\'') && s.ends_with('\'') && s.len() > 1)
176	\|\| (s.starts_with('`') && s.ends_with('`') && s.len() > 1)
177	{
178	&s[1..(s.len() - 1)]
179	} else {
180	s
181	}
182	}
183
184	/// Return type of `check_call`
185	#[derive(Debug, PartialEq, Eq)]
186	pub enum CallKind {
187	Help,
188	Valid,
189	Invalid,
190	}
191
192	pub(crate) fn check_call(
193	working_set: &mut StateWorkingSet,
194	command: Span,
195	sig: &Signature,
196	call: &Call,
197	) -> CallKind {
198	// Allow the call to pass if they pass in the help flag
199	if call.named_iter().any(\|(n, _, _)\| n.item == "help") {
200	return CallKind::Help;
201	}
202
203	if call.positional_iter().count() < sig.required_positional.len() {
204	let end_offset = call
205	.positional_iter()
206	.last()
207	.map(\|last\| last.span.end)
208	.unwrap_or(command.end);
209	// Comparing the types of all signature positional arguments against the parsed
210	// expressions found in the call. If one type is not found then it could be assumed
211	// that that positional argument is missing from the parsed call
212	for argument in &sig.required_positional {
213	let found = call.positional_iter().fold(false, \|ac, expr\| {
214	if argument.shape.to_type() == expr.ty \|\| argument.shape == SyntaxShape::Any {
215	true
216	} else {
217	ac
218	}
219	});
220	if !found {
221	working_set.error(ParseError::MissingPositional(
222	argument.name.clone(),
223	Span::new(end_offset, end_offset),
224	sig.call_signature(),
225	));
226	return CallKind::Invalid;
227	}
228	}
229
230	let missing = &sig.required_positional[call.positional_iter().count()];
231	working_set.error(ParseError::MissingPositional(
232	missing.name.clone(),
233	Span::new(end_offset, end_offset),
234	sig.call_signature(),
235	));
236	return CallKind::Invalid;
237	} else {
238	for req_flag in sig.named.iter().filter(\|x\| x.required) {
239	if call.named_iter().all(\|(n, _, _)\| n.item != req_flag.long) {
240	working_set.error(ParseError::MissingRequiredFlag(
241	req_flag.long.clone(),
242	command,
243	));
244	return CallKind::Invalid;
245	}
246	}
247	}
248	CallKind::Valid
249	}
250
251	/// Parses an unknown argument for the given signature. This handles the parsing as appropriate to
252	/// the rest type of the command.
253	fn parse_unknown_arg(
254	working_set: &mut StateWorkingSet,
255	span: Span,
256	signature: &Signature,
257	) -> Expression {
258	let shape = signature
259	.rest_positional
260	.as_ref()
261	.map(\|arg\| arg.shape.clone())
262	.unwrap_or(SyntaxShape::Any);
263
264	parse_value(working_set, span, &shape)
265	}
266
267	/// Parses a string in the arg or head position of an external call.
268	///
269	/// If the string begins with `r#`, it is parsed as a raw string. If it doesn't contain any quotes
270	/// or parentheses, it is parsed as a glob pattern so that tilde and glob expansion can be handled
271	/// by `run-external`. Otherwise, we use a custom state machine to put together an interpolated
272	/// string, where each balanced pair of quotes is parsed as a separate part of the string, and then
273	/// concatenated together.
274	///
275	/// For example, `-foo="bar\nbaz"` becomes `$"-foo=bar\nbaz"`
276	fn parse_external_string(working_set: &mut StateWorkingSet, span: Span) -> Expression {
277	let contents = working_set.get_span_contents(span);
278
279	if contents.starts_with(b"r#") {
280	parse_raw_string(working_set, span)
281	} else if contents
282	.iter()
283	.any(\|b\| matches!(b, b'"' \| b'\'' \| b'(' \| b')' \| b'`'))
284	{
285	enum State {
286	Bare {
287	from: usize,
288	},
289	BackTickQuote {
290	from: usize,
291	},
292	Quote {
293	from: usize,
294	quote_char: u8,
295	escaped: bool,
296	},
297	Parenthesized {
298	from: usize,
299	depth: usize,
300	},
301	}
302	// Find the spans of parts of the string that can be parsed as their own strings for
303	// concatenation.
304	//
305	// By passing each of these parts to `parse_string()`, we can eliminate the quotes and also
306	// handle string interpolation.
307	let make_span = \|from: usize, index: usize\| Span {
308	start: span.start + from,
309	end: span.start + index,
310	};
311	let mut spans = vec![];
312	let mut state = State::Bare { from: 0 };
313	let mut index = 0;
314	while index < contents.len() {
315	let ch = contents[index];
316	match &mut state {
317	State::Bare { from } => match ch {
318	b'"' \| b'\'' => {
319	// Push bare string
320	if index != *from {
321	spans.push(make_span(*from, index));
322	}
323	// then transition to other state
324	state = State::Quote {
325	from: index,
326	quote_char: ch,
327	escaped: false,
328	};
329	}
330	b'$' => {
331	if let Some(&quote_char @ (b'"' \| b'\'')) = contents.get(index + 1) {
332	// Start a dollar quote (interpolated string)
333	if index != *from {
334	spans.push(make_span(*from, index));
335	}
336	state = State::Quote {
337	from: index,
338	quote_char,
339	escaped: false,
340	};
341	// Skip over two chars (the dollar sign and the quote)
342	index += 2;
343	continue;
344	}
345	}
346	b'`' => {
347	if index != *from {
348	spans.push(make_span(*from, index))
349	}
350	state = State::BackTickQuote { from: index }
351	}
352	b'(' => {
353	if index != *from {
354	spans.push(make_span(*from, index))
355	}
356	state = State::Parenthesized {
357	from: index,
358	depth: 1,
359	}
360	}
361	// Continue to consume
362	_ => (),
363	},
364	State::Quote {
365	from,
366	quote_char,
367	escaped,
368	} => match ch {
369	ch if ch == quote_char && !escaped => {
370	// quoted string ended, just make a new span for it.
371	spans.push(make_span(*from, index + 1));
372	// go back to Bare state.
373	state = State::Bare { from: index + 1 };
374	}
375	b'\\' if !escaped && quote_char == b'"' => {
376	// The next token is escaped so it doesn't count (only for double quote)
377	*escaped = true;
378	}
379	_ => {
380	*escaped = false;
381	}
382	},
383	State::BackTickQuote { from } => {
384	if ch == b'`' {
385	spans.push(make_span(*from, index + 1));
386	state = State::Bare { from: index + 1 };
387	}
388	}
389	State::Parenthesized { from, depth } => {
390	if ch == b')' {
391	if *depth == 1 {
392	spans.push(make_span(*from, index + 1));
393	state = State::Bare { from: index + 1 };
394	} else {
395	*depth -= 1;
396	}
397	} else if ch == b'(' {
398	*depth += 1;
399	}
400	}
401	}
402	index += 1;
403	}
404
405	// Add the final span
406	match state {
407	State::Bare { from }
408	\| State::Quote { from, .. }
409	\| State::Parenthesized { from, .. }
410	\| State::BackTickQuote { from, .. } => {
411	if from < contents.len() {
412	spans.push(make_span(from, contents.len()));
413	}
414	}
415	}
416
417	// Log the spans that will be parsed
418	if log::log_enabled!(log::Level::Trace) {
419	let contents = spans
420	.iter()
421	.map(\|span\| String::from_utf8_lossy(working_set.get_span_contents(*span)))
422	.collect::<Vec<_>>();
423
424	trace!("parsing: external string, parts: {contents:?}")
425	}
426
427	// Check if the whole thing is quoted. If not, it should be a glob
428	let quoted =
429	(contents.len() >= 3 && contents.starts_with(b"$\"") && contents.ends_with(b"\""))
430	\|\| is_quoted(contents);
431
432	// Parse each as its own string
433	let exprs: Vec<Expression> = spans
434	.into_iter()
435	.map(\|span\| parse_string(working_set, span))
436	.collect();
437
438	if exprs
439	.iter()
440	.all(\|expr\| matches!(expr.expr, Expr::String(..)))
441	{
442	// If the exprs are all strings anyway, just collapse into a single string.
443	let string = exprs
444	.into_iter()
445	.map(\|expr\| {
446	let Expr::String(contents) = expr.expr else {
447	unreachable!("already checked that this was a String")
448	};
449	contents
450	})
451	.collect::<String>();
452	if quoted {
453	Expression::new(working_set, Expr::String(string), span, Type::String)
454	} else {
455	Expression::new(
456	working_set,
457	Expr::GlobPattern(string, false),
458	span,
459	Type::Glob,
460	)
461	}
462	} else {
463	// Flatten any string interpolations contained with the exprs.
464	let exprs = exprs
465	.into_iter()
466	.flat_map(\|expr\| match expr.expr {
467	Expr::StringInterpolation(subexprs) => subexprs,
468	_ => vec![expr],
469	})
470	.collect();
471	// Make an interpolation out of the expressions. Use `GlobInterpolation` if it's a bare
472	// word, so that the unquoted state can get passed through to `run-external`.
473	if quoted {
474	Expression::new(
475	working_set,
476	Expr::StringInterpolation(exprs),
477	span,
478	Type::String,
479	)
480	} else {
481	Expression::new(
482	working_set,
483	Expr::GlobInterpolation(exprs, false),
484	span,
485	Type::Glob,
486	)
487	}
488	}
489	} else {
490	parse_glob_pattern(working_set, span)
491	}
492	}
493
494	fn parse_external_arg(working_set: &mut StateWorkingSet, span: Span) -> ExternalArgument {
495	let contents = working_set.get_span_contents(span);
496
497	if contents.len() > 3
498	&& contents.starts_with(b"...")
499	&& (contents[3] == b'$' \|\| contents[3] == b'[' \|\| contents[3] == b'(')
500	{
501	ExternalArgument::Spread(parse_value(
502	working_set,
503	Span::new(span.start + 3, span.end),
504	&SyntaxShape::List(Box::new(SyntaxShape::Any)),
505	))
506	} else {
507	ExternalArgument::Regular(parse_regular_external_arg(working_set, span))
508	}
509	}
510
511	fn parse_regular_external_arg(working_set: &mut StateWorkingSet, span: Span) -> Expression {
512	let contents = working_set.get_span_contents(span);
513
514	if contents.starts_with(b"$") {
515	parse_dollar_expr(working_set, span)
516	} else if contents.starts_with(b"(") {
517	parse_paren_expr(working_set, span, &SyntaxShape::Any)
518	} else if contents.starts_with(b"[") {
519	parse_list_expression(working_set, span, &SyntaxShape::Any)
520	} else {
521	parse_external_string(working_set, span)
522	}
523	}
524
525	pub fn parse_external_call(
526	working_set: &mut StateWorkingSet,
527	spans: &[Span],
528	call_span: Span,
529	) -> Expression {
530	trace!("parse external");
531
532	let head_span = spans[0];
533
534	let head_contents = working_set.get_span_contents(head_span).to_vec();
535
536	let head = if head_contents.starts_with(b"$") \|\| head_contents.starts_with(b"(") {
537	// the expression is inside external_call, so it's a subexpression
538	let arg = parse_expression(working_set, &[head_span]);
539	Box::new(arg)
540	} else {
541	Box::new(parse_external_string(working_set, head_span))
542	};
543
544	let args = spans[1..]
545	.iter()
546	.map(\|&span\| parse_external_arg(working_set, span))
547	.collect();
548
549	Expression::new(
550	working_set,
551	Expr::ExternalCall(head, args),
552	call_span,
553	Type::Any,
554	)
555	}
556
557	fn ensure_flag_arg_type(
558	working_set: &mut StateWorkingSet,
559	arg_name: String,
560	arg: Expression,
561	arg_shape: &SyntaxShape,
562	long_name_span: Span,
563	) -> (Spanned<String>, Expression) {
564	if !type_compatible(&arg.ty, &arg_shape.to_type()) {
565	working_set.error(ParseError::TypeMismatch(
566	arg_shape.to_type(),
567	arg.ty,
568	arg.span,
569	));
570	(
571	Spanned {
572	item: arg_name,
573	span: long_name_span,
574	},
575	Expression::garbage(working_set, arg.span),
576	)
577	} else {
578	(
579	Spanned {
580	item: arg_name,
581	span: long_name_span,
582	},
583	arg,
584	)
585	}
586	}
587
588	fn parse_long_flag(
589	working_set: &mut StateWorkingSet,
590	spans: &[Span],
591	spans_idx: &mut usize,
592	sig: &Signature,
593	) -> (Option<Spanned<String>>, Option<Expression>) {
594	let arg_span = spans[*spans_idx];
595	let arg_contents = working_set.get_span_contents(arg_span);
596
597	if arg_contents.starts_with(b"--") {
598	// FIXME: only use the first flag you find?
599	let split: Vec<_> = arg_contents.split(\|x\| *x == b'=').collect();
600	let long_name = String::from_utf8(split[0].into());
601	if let Ok(long_name) = long_name {
602	let long_name = long_name[2..].to_string();
603	if let Some(flag) = sig.get_long_flag(&long_name) {
604	if let Some(arg_shape) = &flag.arg {
605	if split.len() > 1 {
606	// and we also have the argument
607	let long_name_len = long_name.len();
608	let mut span = arg_span;
609	span.start += long_name_len + 3; //offset by long flag and '='
610
611	let arg = parse_value(working_set, span, arg_shape);
612	let (arg_name, val_expression) = ensure_flag_arg_type(
613	working_set,
614	long_name,
615	arg,
616	arg_shape,
617	Span::new(arg_span.start, arg_span.start + long_name_len + 2),
618	);
619	(Some(arg_name), Some(val_expression))
620	} else if let Some(arg) = spans.get(*spans_idx + 1) {
621	let arg = parse_value(working_set, *arg, arg_shape);
622
623	*spans_idx += 1;
624	let (arg_name, val_expression) =
625	ensure_flag_arg_type(working_set, long_name, arg, arg_shape, arg_span);
626	(Some(arg_name), Some(val_expression))
627	} else {
628	working_set.error(ParseError::MissingFlagParam(
629	arg_shape.to_string(),
630	arg_span,
631	));
632	// NOTE: still need to cover this incomplete flag in the final expression
633	// see https://github.com/nushell/nushell/issues/16375
634	(
635	Some(Spanned {
636	item: long_name,
637	span: arg_span,
638	}),
639	None,
640	)
641	}
642	} else {
643	// A flag with no argument
644	// It can also takes a boolean value like --x=true
645	if split.len() > 1 {
646	// and we also have the argument
647	let long_name_len = long_name.len();
648	let mut span = arg_span;
649	span.start += long_name_len + 3; //offset by long flag and '='
650
651	let arg = parse_value(working_set, span, &SyntaxShape::Boolean);
652
653	let (arg_name, val_expression) = ensure_flag_arg_type(
654	working_set,
655	long_name,
656	arg,
657	&SyntaxShape::Boolean,
658	Span::new(arg_span.start, arg_span.start + long_name_len + 2),
659	);
660	(Some(arg_name), Some(val_expression))
661	} else {
662	(
663	Some(Spanned {
664	item: long_name,
665	span: arg_span,
666	}),
667	None,
668	)
669	}
670	}
671	} else {
672	let suggestion = did_you_mean(sig.get_names(), &long_name)
673	.map(\|name\| format!("Did you mean: `--{name}`?"))
674	.unwrap_or("Use `--help` to see available flags".to_owned());
675	working_set.error(ParseError::UnknownFlag(
676	sig.name.clone(),
677	long_name.clone(),
678	arg_span,
679	suggestion,
680	));
681	(
682	Some(Spanned {
683	item: long_name.clone(),
684	span: arg_span,
685	}),
686	None,
687	)
688	}
689	} else {
690	working_set.error(ParseError::NonUtf8(arg_span));
691	(
692	Some(Spanned {
693	item: "--".into(),
694	span: arg_span,
695	}),
696	None,
697	)
698	}
699	} else {
700	(None, None)
701	}
702	}
703
704	fn parse_short_flags(
705	working_set: &mut StateWorkingSet,
706	spans: &[Span],
707	spans_idx: &mut usize,
708	positional_idx: usize,
709	sig: &Signature,
710	) -> Option<Vec<Flag>> {
711	let arg_span = spans[*spans_idx];
712
713	let arg_contents = working_set.get_span_contents(arg_span);
714
715	if let Ok(arg_contents_uft8_ref) = str::from_utf8(arg_contents) {
716	if arg_contents_uft8_ref.starts_with('-') && arg_contents_uft8_ref.len() > 1 {
717	let short_flags = &arg_contents_uft8_ref[1..];
718	let num_chars = short_flags.chars().count();
719	let mut found_short_flags = vec![];
720	let mut unmatched_short_flags = vec![];
721	for (offset, short_flag) in short_flags.char_indices() {
722	let short_flag_span = Span::new(
723	arg_span.start + 1 + offset,
724	arg_span.start + 1 + offset + short_flag.len_utf8(),
725	);
726	if let Some(flag) = sig.get_short_flag(short_flag) {
727	// Allow args in short flag batches as long as it is the last flag.
728	if flag.arg.is_some() && offset < num_chars - 1 {
729	working_set
730	.error(ParseError::OnlyLastFlagInBatchCanTakeArg(short_flag_span));
731	break;
732	}
733	found_short_flags.push(flag);
734	} else {
735	unmatched_short_flags.push(short_flag_span);
736	}
737	}
738
739	if found_short_flags.is_empty()
740	// check to see if we have a negative number
741	&& matches!(
742	sig.get_positional(positional_idx),
743	Some(PositionalArg {
744	shape: SyntaxShape::Int \| SyntaxShape::Number \| SyntaxShape::Float,
745	..
746	})
747	)
748	&& String::from_utf8_lossy(working_set.get_span_contents(arg_span))
749	.parse::<f64>()
750	.is_ok()
751	{
752	return None;
753	} else if let Some(first) = unmatched_short_flags.first() {
754	let contents = working_set.get_span_contents(*first);
755	working_set.error(ParseError::UnknownFlag(
756	sig.name.clone(),
757	format!("-{}", String::from_utf8_lossy(contents)),
758	*first,
759	"Use `--help` to see available flags".to_owned(),
760	));
761	}
762
763	Some(found_short_flags)
764	} else {
765	None
766	}
767	} else {
768	working_set.error(ParseError::NonUtf8(arg_span));
769	None
770	}
771	}
772
773	fn first_kw_idx(
774	working_set: &StateWorkingSet,
775	signature: &Signature,
776	spans: &[Span],
777	spans_idx: usize,
778	positional_idx: usize,
779	) -> (Option<usize>, usize) {
780	for idx in (positional_idx + 1)..signature.num_positionals() {
781	if let Some(PositionalArg {
782	shape: SyntaxShape::Keyword(kw, ..),
783	..
784	}) = signature.get_positional(idx)
785	{
786	for (span_idx, &span) in spans.iter().enumerate().skip(spans_idx) {
787	let contents = working_set.get_span_contents(span);
788
789	if contents == kw {
790	return (Some(idx), span_idx);
791	}
792	}
793	}
794	}
795	(None, spans.len())
796	}
797
798	fn calculate_end_span(
799	working_set: &StateWorkingSet,
800	signature: &Signature,
801	spans: &[Span],
802	spans_idx: usize,
803	positional_idx: usize,
804	) -> usize {
805	if signature.rest_positional.is_some() {
806	spans.len()
807	} else {
808	let (kw_pos, kw_idx) =
809	first_kw_idx(working_set, signature, spans, spans_idx, positional_idx);
810
811	if let Some(kw_pos) = kw_pos {
812	// We found a keyword. Keywords, once found, create a guidepost to
813	// show us where the positionals will lay into the arguments. Because they're
814	// keywords, they get to set this by being present
815
816	let positionals_between = kw_pos - positional_idx - 1;
817	if positionals_between >= (kw_idx - spans_idx) {
818	kw_idx
819	} else {
820	kw_idx - positionals_between
821	}
822	} else {
823	// Make space for the remaining require positionals, if we can
824	// spans_idx < spans.len() is an invariant
825	let remaining_spans = spans.len() - (spans_idx + 1);
826	// positional_idx can be larger than required_positional.len() if we have optional args
827	let remaining_positional = signature
828	.required_positional
829	.len()
830	.saturating_sub(positional_idx + 1);
831	// Saturates to 0 when we have too few args
832	let extra_spans = remaining_spans.saturating_sub(remaining_positional);
833	spans_idx + 1 + extra_spans
834	}
835	}
836	}
837
838	fn parse_oneof(
839	working_set: &mut StateWorkingSet,
840	spans: &[Span],
841	spans_idx: &mut usize,
842	possible_shapes: &Vec<SyntaxShape>,
843	multispan: bool,
844	) -> Expression {
845	let starting_spans_idx = *spans_idx;
846	let mut best_guess = None;
847	let mut best_guess_errors = Vec::new();
848	let mut max_first_error_offset = 0;
849	let mut propagate_error = false;
850	for shape in possible_shapes {
851	let starting_error_count = working_set.parse_errors.len();
852	*spans_idx = starting_spans_idx;
853	let value = match multispan {
854	true => parse_multispan_value(working_set, spans, spans_idx, shape),
855	false => parse_value(working_set, spans[*spans_idx], shape),
856	};
857
858	let new_errors = working_set.parse_errors[starting_error_count..].to_vec();
859	// no new errors found means success
860	let Some(first_error_offset) = new_errors.iter().map(\|e\| e.span().start).min() else {
861	return value;
862	};
863
864	if first_error_offset > max_first_error_offset {
865	// while trying the possible shapes, ignore Expected type errors
866	// unless they're inside a block, closure, or expression
867	propagate_error = match working_set.parse_errors.last() {
868	Some(ParseError::Expected(_, error_span))
869	\| Some(ParseError::ExpectedWithStringMsg(_, error_span)) => {
870	matches!(
871	shape,
872	SyntaxShape::Block \| SyntaxShape::Closure(_) \| SyntaxShape::Expression
873	) && error_span != spans[spans_idx]
874	}
875	_ => true,
876	};
877	max_first_error_offset = first_error_offset;
878	best_guess = Some(value);
879	best_guess_errors = new_errors;
880	}
881	working_set.parse_errors.truncate(starting_error_count);
882	}
883
884	// if best_guess results in new errors further than current span, then accept it
885	// or propagate_error is marked as true for it
886	if max_first_error_offset > spans[starting_spans_idx].start \|\| propagate_error {
887	working_set.parse_errors.extend(best_guess_errors);
888	best_guess.expect("best_guess should not be None here!")
889	} else {
890	working_set.error(ParseError::ExpectedWithStringMsg(
891	format!("one of a list of accepted shapes: {possible_shapes:?}"),
892	spans[starting_spans_idx],
893	));
894	Expression::garbage(working_set, spans[starting_spans_idx])
895	}
896	}
897
898	pub fn parse_multispan_value(
899	working_set: &mut StateWorkingSet,
900	spans: &[Span],
901	spans_idx: &mut usize,
902	shape: &SyntaxShape,
903	) -> Expression {
904	trace!("parse multispan value");
905	match shape {
906	SyntaxShape::VarWithOptType => {
907	trace!("parsing: var with opt type");
908
909	parse_var_with_opt_type(working_set, spans, spans_idx, false).0
910	}
911	SyntaxShape::RowCondition => {
912	trace!("parsing: row condition");
913	let arg = parse_row_condition(working_set, &spans[*spans_idx..]);
914	*spans_idx = spans.len() - 1;
915
916	arg
917	}
918	SyntaxShape::MathExpression => {
919	trace!("parsing: math expression");
920
921	let arg = parse_math_expression(working_set, &spans[*spans_idx..], None);
922	*spans_idx = spans.len() - 1;
923
924	arg
925	}
926	SyntaxShape::OneOf(possible_shapes) => {
927	parse_oneof(working_set, spans, spans_idx, possible_shapes, true)
928	}
929
930	SyntaxShape::Expression => {
931	trace!("parsing: expression");
932
933	// is it subexpression?
934	// Not sure, but let's make it not, so the behavior is the same as previous version of nushell.
935	let arg = parse_expression(working_set, &spans[*spans_idx..]);
936	*spans_idx = spans.len().saturating_sub(1);
937
938	arg
939	}
940	SyntaxShape::Signature => {
941	trace!("parsing: signature");
942
943	let sig = parse_full_signature(working_set, &spans[*spans_idx..], false);
944	*spans_idx = spans.len().saturating_sub(1);
945
946	sig
947	}
948	SyntaxShape::ExternalSignature => {
949	trace!("parsing: external signature");
950
951	let sig = parse_full_signature(working_set, &spans[*spans_idx..], true);
952	*spans_idx = spans.len().saturating_sub(1);
953
954	sig
955	}
956	SyntaxShape::Keyword(keyword, arg) => {
957	trace!(
958	"parsing: keyword({}) {:?}",
959	String::from_utf8_lossy(keyword),
960	arg
961	);
962	let arg_span = spans[*spans_idx];
963
964	let arg_contents = working_set.get_span_contents(arg_span);
965
966	if arg_contents != keyword {
967	// When keywords mismatch, this is a strong indicator of something going wrong.
968	// We won't often override the current error, but as this is a strong indicator
969	// go ahead and override the current error and tell the user about the missing
970	// keyword/literal.
971	working_set.error(ParseError::ExpectedKeyword(
972	String::from_utf8_lossy(keyword).into(),
973	arg_span,
974	))
975	}
976
977	*spans_idx += 1;
978	if *spans_idx >= spans.len() {
979	working_set.error(ParseError::KeywordMissingArgument(
980	arg.to_string(),
981	String::from_utf8_lossy(keyword).into(),
982	Span::new(spans[spans_idx - 1].end, spans[spans_idx - 1].end),
983	));
984	let keyword = Keyword {
985	keyword: keyword.as_slice().into(),
986	span: spans[*spans_idx - 1],
987	expr: Expression::garbage(working_set, arg_span),
988	};
989	return Expression::new(
990	working_set,
991	Expr::Keyword(Box::new(keyword)),
992	arg_span,
993	Type::Any,
994	);
995	}
996
997	let keyword = Keyword {
998	keyword: keyword.as_slice().into(),
999	span: spans[*spans_idx - 1],
1000	expr: parse_multispan_value(working_set, spans, spans_idx, arg),
1001	};
1002
1003	Expression::new(
1004	working_set,
1005	Expr::Keyword(Box::new(keyword.clone())),
1006	keyword.span.merge(keyword.expr.span),
1007	keyword.expr.ty,
1008	)
1009	}
1010	_ => {
1011	// All other cases are single-span values
1012	let arg_span = spans[*spans_idx];
1013
1014	parse_value(working_set, arg_span, shape)
1015	}
1016	}
1017	}
1018
1019	pub struct ParsedInternalCall {
1020	pub call: Box<Call>,
1021	pub output: Type,
1022	pub call_kind: CallKind,
1023	}
1024
1025	/// Sometimes the arguments of an internal command need to be parsed in dedicated functions, e.g. `parse_module`.
1026	/// If so, `parse_internal_call` should be called with the appropriate parsing level to avoid repetition.
1027	///
1028	/// Defaults to `ArgumentParsingLevel::Full`
1029	#[derive(Default)]
1030	pub enum ArgumentParsingLevel {
1031	#[default]
1032	Full,
1033	/// Parse only the first `k` arguments
1034	FirstK { k: usize },
1035	}
1036
1037	pub fn parse_internal_call(
1038	working_set: &mut StateWorkingSet,
1039	command_span: Span,
1040	spans: &[Span],
1041	decl_id: DeclId,
1042	arg_parsing_level: ArgumentParsingLevel,
1043	) -> ParsedInternalCall {
1044	trace!("parsing: internal call (decl id: {})", decl_id.get());
1045
1046	let mut call = Call::new(command_span);
1047	call.decl_id = decl_id;
1048	call.head = command_span;
1049	let _ = working_set.add_span(call.head);
1050
1051	let decl = working_set.get_decl(decl_id);
1052	let signature = working_set.get_signature(decl);
1053	let output = signature.get_output_type();
1054
1055	let deprecation = decl.deprecation_info();
1056
1057	// storing the var ID for later due to borrowing issues
1058	let lib_dirs_var_id = match decl.name() {
1059	"use" \| "overlay use" \| "source-env" if decl.is_keyword() => {
1060	find_dirs_var(working_set, LIB_DIRS_VAR)
1061	}
1062	"nu-check" if decl.is_builtin() => find_dirs_var(working_set, LIB_DIRS_VAR),
1063	_ => None,
1064	};
1065
1066	// The index into the positional parameter in the definition
1067	let mut positional_idx = 0;
1068
1069	// The index into the spans of argument data given to parse
1070	// Starting at the first argument
1071	let mut spans_idx = 0;
1072
1073	if let Some(alias) = decl.as_alias() {
1074	if let Expression {
1075	expr: Expr::Call(wrapped_call),
1076	..
1077	} = &alias.wrapped_call
1078	{
1079	// Replace this command's call with the aliased call, but keep the alias name
1080	call = *wrapped_call.clone();
1081	call.head = command_span;
1082	// Skip positionals passed to aliased call
1083	positional_idx = call.positional_iter().count();
1084	} else {
1085	working_set.error(ParseError::UnknownState(
1086	"Alias does not point to internal call.".to_string(),
1087	command_span,
1088	));
1089	return ParsedInternalCall {
1090	call: Box::new(call),
1091	output: Type::Any,
1092	call_kind: CallKind::Invalid,
1093	};
1094	}
1095	}
1096
1097	if let Some(var_id) = lib_dirs_var_id {
1098	call.set_parser_info(
1099	DIR_VAR_PARSER_INFO.to_owned(),
1100	Expression::new(working_set, Expr::Var(var_id), call.head, Type::Any),
1101	);
1102	}
1103
1104	if signature.creates_scope {
1105	working_set.enter_scope();
1106	}
1107
1108	while spans_idx < spans.len() {
1109	let arg_span = spans[spans_idx];
1110
1111	let starting_error_count = working_set.parse_errors.len();
1112	// Check if we're on a long flag, if so, parse
1113	let (long_name, arg) = parse_long_flag(working_set, spans, &mut spans_idx, &signature);
1114
1115	if let Some(long_name) = long_name {
1116	// We found a long flag, like --bar
1117	if working_set.parse_errors[starting_error_count..]
1118	.iter()
1119	.any(\|x\| matches!(x, ParseError::UnknownFlag(_, _, _, _)))
1120	&& signature.allows_unknown_args
1121	{
1122	working_set.parse_errors.truncate(starting_error_count);
1123	let arg = parse_unknown_arg(working_set, arg_span, &signature);
1124
1125	call.add_unknown(arg);
1126	} else {
1127	call.add_named((long_name, None, arg));
1128	}
1129
1130	spans_idx += 1;
1131	continue;
1132	}
1133
1134	let starting_error_count = working_set.parse_errors.len();
1135
1136	// Check if we're on a short flag or group of short flags, if so, parse
1137	let short_flags = parse_short_flags(
1138	working_set,
1139	spans,
1140	&mut spans_idx,
1141	positional_idx,
1142	&signature,
1143	);
1144
1145	if let Some(mut short_flags) = short_flags {
1146	if short_flags.is_empty() {
1147	// workaround for completions (PR #6067)
1148	short_flags.push(Flag {
1149	long: "".to_string(),
1150	short: Some('a'),
1151	arg: None,
1152	required: false,
1153	desc: "".to_string(),
1154	var_id: None,
1155	default_value: None,
1156	completion: None,
1157	})
1158	}
1159
1160	if working_set.parse_errors[starting_error_count..]
1161	.iter()
1162	.any(\|x\| matches!(x, ParseError::UnknownFlag(_, _, _, _)))
1163	&& signature.allows_unknown_args
1164	{
1165	working_set.parse_errors.truncate(starting_error_count);
1166	let arg = parse_unknown_arg(working_set, arg_span, &signature);
1167
1168	call.add_unknown(arg);
1169	} else {
1170	for flag in short_flags {
1171	let _ = working_set.add_span(spans[spans_idx]);
1172
1173	if let Some(arg_shape) = flag.arg {
1174	if let Some(arg) = spans.get(spans_idx + 1) {
1175	let arg = parse_value(working_set, *arg, &arg_shape);
1176	let (arg_name, val_expression) = ensure_flag_arg_type(
1177	working_set,
1178	flag.long.clone(),
1179	arg.clone(),
1180	&arg_shape,
1181	spans[spans_idx],
1182	);
1183
1184	if flag.long.is_empty() {
1185	if let Some(short) = flag.short {
1186	call.add_named((
1187	arg_name,
1188	Some(Spanned {
1189	item: short.to_string(),
1190	span: spans[spans_idx],
1191	}),
1192	Some(val_expression),
1193	));
1194	}
1195	} else {
1196	call.add_named((arg_name, None, Some(val_expression)));
1197	}
1198	spans_idx += 1;
1199	} else {
1200	working_set.error(ParseError::MissingFlagParam(
1201	arg_shape.to_string(),
1202	arg_span,
1203	));
1204	// NOTE: still need to cover this incomplete flag in the final expression
1205	// see https://github.com/nushell/nushell/issues/16375
1206	call.add_named((
1207	Spanned {
1208	item: String::new(),
1209	span: spans[spans_idx],
1210	},
1211	None,
1212	None,
1213	));
1214	}
1215	} else if flag.long.is_empty() {
1216	if let Some(short) = flag.short {
1217	call.add_named((
1218	Spanned {
1219	item: String::new(),
1220	span: spans[spans_idx],
1221	},
1222	Some(Spanned {
1223	item: short.to_string(),
1224	span: spans[spans_idx],
1225	}),
1226	None,
1227	));
1228	}
1229	} else {
1230	call.add_named((
1231	Spanned {
1232	item: flag.long.clone(),
1233	span: spans[spans_idx],
1234	},
1235	None,
1236	None,
1237	));
1238	}
1239	}
1240	}
1241
1242	spans_idx += 1;
1243	continue;
1244	}
1245
1246	{
1247	let contents = working_set.get_span_contents(spans[spans_idx]);
1248
1249	if contents.len() > 3
1250	&& contents.starts_with(b"...")
1251	&& (contents[3] == b'$' \|\| contents[3] == b'[' \|\| contents[3] == b'(')
1252	{
1253	if signature.rest_positional.is_none() && !signature.allows_unknown_args {
1254	working_set.error(ParseError::UnexpectedSpreadArg(
1255	signature.call_signature(),
1256	arg_span,
1257	));
1258	call.add_positional(Expression::garbage(working_set, arg_span));
1259	} else if positional_idx < signature.required_positional.len() {
1260	working_set.error(ParseError::MissingPositional(
1261	signature.required_positional[positional_idx].name.clone(),
1262	Span::new(spans[spans_idx].start, spans[spans_idx].start),
1263	signature.call_signature(),
1264	));
1265	call.add_positional(Expression::garbage(working_set, arg_span));
1266	} else {
1267	let rest_shape = match &signature.rest_positional {
1268	Some(arg) if matches!(arg.shape, SyntaxShape::ExternalArgument) => {
1269	// External args aren't parsed inside lists in spread position.
1270	SyntaxShape::Any
1271	}
1272	Some(arg) => arg.shape.clone(),
1273	None => SyntaxShape::Any,
1274	};
1275	// Parse list of arguments to be spread
1276	let args = parse_value(
1277	working_set,
1278	Span::new(arg_span.start + 3, arg_span.end),
1279	&SyntaxShape::List(Box::new(rest_shape)),
1280	);
1281
1282	call.add_spread(args);
1283	// Let the parser know that it's parsing rest arguments now
1284	positional_idx =
1285	signature.required_positional.len() + signature.optional_positional.len();
1286	}
1287
1288	spans_idx += 1;
1289	continue;
1290	}
1291	}
1292
1293	// Parse a positional arg if there is one
1294	if let Some(positional) = signature.get_positional(positional_idx) {
1295	let end = calculate_end_span(working_set, &signature, spans, spans_idx, positional_idx);
1296
1297	// Missing arguments before next keyword
1298	if end == spans_idx {
1299	let prev_span = if spans_idx == 0 {
1300	command_span
1301	} else {
1302	spans[spans_idx - 1]
1303	};
1304	let whitespace_span = Span::new(prev_span.end, spans[spans_idx].start);
1305	working_set.error(ParseError::MissingPositional(
1306	positional.name.clone(),
1307	whitespace_span,
1308	signature.call_signature(),
1309	));
1310	call.add_positional(Expression::garbage(working_set, whitespace_span));
1311	positional_idx += 1;
1312	continue;
1313	}
1314	debug_assert!(end <= spans.len());
1315
1316	if spans[..end].is_empty() \|\| spans_idx == end {
1317	working_set.error(ParseError::MissingPositional(
1318	positional.name.clone(),
1319	Span::new(spans[spans_idx].end, spans[spans_idx].end),
1320	signature.call_signature(),
1321	));
1322	positional_idx += 1;
1323	continue;
1324	}
1325
1326	let compile_error_count = working_set.compile_errors.len();
1327
1328	// HACK: avoid repeated parsing of argument values in special cases
1329	// see https://github.com/nushell/nushell/issues/16398
1330	let arg = match arg_parsing_level {
1331	ArgumentParsingLevel::FirstK { k } if k <= positional_idx => {
1332	Expression::garbage(working_set, spans[spans_idx])
1333	}
1334	_ => parse_multispan_value(
1335	working_set,
1336	&spans[..end],
1337	&mut spans_idx,
1338	&positional.shape,
1339	),
1340	};
1341
1342	// HACK: try-catch's signature defines the catch block as a Closure, even though it's
1343	// used like a Block. Because closures are compiled eagerly, this ends up making the
1344	// following code technically invalid:
1345	// ```nu
1346	// loop { try { } catch {\|e\| break } }
1347	// ```
1348	// Thus, we discard the compilation error here
1349	if let SyntaxShape::OneOf(ref shapes) = positional.shape {
1350	for one_shape in shapes {
1351	if let SyntaxShape::Keyword(keyword, ..) = one_shape
1352	&& keyword == b"catch"
1353	&& let [nu_protocol::CompileError::NotInALoop { .. }] =
1354	&working_set.compile_errors[compile_error_count..]
1355	{
1356	working_set.compile_errors.truncate(compile_error_count);
1357	}
1358	}
1359	}
1360
1361	let arg = if !type_compatible(&positional.shape.to_type(), &arg.ty) {
1362	working_set.error(ParseError::TypeMismatch(
1363	positional.shape.to_type(),
1364	arg.ty,
1365	arg.span,
1366	));
1367	Expression::garbage(working_set, arg.span)
1368	} else {
1369	arg
1370	};
1371
1372	call.add_positional(arg);
1373	positional_idx += 1;
1374	} else if signature.allows_unknown_args {
1375	let arg = parse_unknown_arg(working_set, arg_span, &signature);
1376
1377	call.add_unknown(arg);
1378	} else {
1379	call.add_positional(Expression::garbage(working_set, arg_span));
1380	working_set.error(ParseError::ExtraPositional(
1381	signature.call_signature(),
1382	arg_span,
1383	))
1384	}
1385
1386	spans_idx += 1;
1387	}
1388
1389	// TODO: Inline `check_call`,
1390	// move missing positional checking into the while loop above with two pointers.
1391	// Maybe more `CallKind::Invalid` if errors found during argument parsing.
1392	let call_kind = check_call(working_set, command_span, &signature, &call);
1393
1394	deprecation
1395	.into_iter()
1396	.filter_map(\|entry\| entry.parse_warning(&signature.name, &call))
1397	.for_each(\|warning\| {
1398	// FIXME: if two flags are deprecated and both are used in one command,
1399	// the second flag's deprecation won't show until the first flag is removed
1400	// (but it won't be flagged as reported until it is actually reported)
1401	working_set.warning(warning);
1402	});
1403
1404	if signature.creates_scope {
1405	working_set.exit_scope();
1406	}
1407
1408	ParsedInternalCall {
1409	call: Box::new(call),
1410	output,
1411	call_kind,
1412	}
1413	}
1414
1415	pub fn parse_call(working_set: &mut StateWorkingSet, spans: &[Span], head: Span) -> Expression {
1416	trace!("parsing: call");
1417	let call_span = Span::concat(spans);
1418
1419	if spans.is_empty() {
1420	working_set.error(ParseError::UnknownState(
1421	"Encountered command with zero spans".into(),
1422	call_span,
1423	));
1424	return garbage(working_set, head);
1425	}
1426
1427	let call_sigil = match working_set.get_span_contents(spans[0]).first() {
1428	Some(b'^') => Some(b'^'),
1429	Some(b'%') => Some(b'%'),
1430	_ => None,
1431	};
1432
1433	let mut adjusted_spans = Vec::new();
1434	let resolution_spans = match call_sigil {
1435	Some(b'^') \| Some(b'%') => {
1436	adjusted_spans.reserve(spans.len());
1437	adjusted_spans.push(Span::new(spans[0].start + 1, spans[0].end));
1438	adjusted_spans.extend_from_slice(&spans[1..]);
1439	adjusted_spans.as_slice()
1440	}
1441	_ => spans,
1442	};
1443
1444	// `^` always forces external command parsing and must bypass declaration
1445	// resolution, even when an internal command with the same name exists.
1446	if call_sigil == Some(b'^') {
1447	trace!("parsing: forced external call");
1448	return parse_external_call(working_set, resolution_spans, call_span);
1449	}
1450
1451	// Check if we have a percent sigil with a dynamic head (variable or expression).
1452	// Supports two token layouts:
1453	// - single token: `%$cmd` or `%($cmd)` — stripping `%` leaves `$cmd` / `($cmd)` in [0]
1454	// - two tokens: `%` and `($cmd)` — stripping `%` leaves an empty span in [0]; head is [1]
1455	// If so, defer builtin validation to runtime (the IR compiler will rewrite to `run-internal`).
1456	if call_sigil == Some(b'%') && !resolution_spans.is_empty() {
1457	// Locate the actual head span, skipping an empty leading span.
1458	let (head_idx, head_span) = {
1459	let first = working_set.get_span_contents(resolution_spans[0]);
1460	if first.is_empty() && resolution_spans.len() > 1 {
1461	(1, resolution_spans[1])
1462	} else {
1463	(0, resolution_spans[0])
1464	}
1465	};
1466
1467	let dynamic_head_contents = working_set.get_span_contents(head_span);
1468	let is_dynamic_head = !dynamic_head_contents.is_empty()
1469	&& (dynamic_head_contents[0] == b'$' \|\| dynamic_head_contents[0] == b'(');
1470
1471	if is_dynamic_head {
1472	trace!("parsing: dynamic percent builtin dispatch");
1473
1474	let head_expr = parse_expression(working_set, &[head_span]);
1475
1476	// Create a placeholder call; the IR compiler will rewrite this to `run-internal`.
1477	let mut call = Call::new(call_span);
1478	call.decl_id = DeclId::new(0);
1479
1480	// Store the head expression for the IR compiler to pick up.
1481	call.set_parser_info(PERCENT_FORCED_BUILTIN_PARSER_INFO.to_string(), head_expr);
1482
1483	// Mirror the dynamic external-call path by preserving `...expr` as an explicit spread
1484	// argument so runtime dispatch can forward it without flattening first.
1485	for arg_span in resolution_spans.iter().skip(head_idx + 1) {
1486	let contents = working_set.get_span_contents(*arg_span);
1487	if contents.len() > 3
1488	&& contents.starts_with(b"...")
1489	&& (contents[3] == b'$' \|\| contents[3] == b'[' \|\| contents[3] == b'(')
1490	{
1491	let spread_expr = parse_value(
1492	working_set,
1493	Span::new(arg_span.start + 3, arg_span.end),
1494	&SyntaxShape::List(Box::new(SyntaxShape::Any)),
1495	);
1496	call.arguments.push(Argument::Spread(spread_expr));
1497	} else {
1498	let arg_expr = parse_value(working_set, *arg_span, &SyntaxShape::Any);
1499	call.arguments.push(Argument::Positional(arg_expr));
1500	}
1501	}
1502
1503	return Expression::new(
1504	working_set,
1505	Expr::Call(Box::new(call)),
1506	call_span,
1507	Type::Any,
1508	);
1509	}
1510	}
1511
1512	let (cmd_start, pos, _name, maybe_decl_id) = if call_sigil == Some(b'%') {
1513	find_longest_decl_with_command_type(working_set, resolution_spans, CommandType::Builtin)
1514	} else {
1515	find_longest_decl(working_set, resolution_spans)
1516	};
1517
1518	if let Some(decl_id) = maybe_decl_id {
1519	// Before the internal parsing we check if there is no let or alias declarations
1520	// that are missing their name, e.g.: let = 1 or alias = 2
1521	if resolution_spans.len() > 1 {
1522	let test_equal = working_set.get_span_contents(resolution_spans[1]);
1523
1524	if test_equal == [b'='] {
1525	trace!("incomplete statement");
1526
1527	working_set.error(ParseError::UnknownState(
1528	"Incomplete statement".into(),
1529	call_span,
1530	));
1531	return garbage(working_set, call_span);
1532	}
1533	}
1534
1535	let decl = working_set.get_decl(decl_id);
1536
1537	let parsed_call = if let Some(alias) = decl.as_alias() {
1538	if let Expression {
1539	expr: Expr::ExternalCall(head, args),
1540	span: _,
1541	span_id: _,
1542	ty,
1543	} = &alias.clone().wrapped_call
1544	{
1545	trace!("parsing: alias of external call");
1546
1547	let mut head = head.clone();
1548	head.span = Span::concat(&resolution_spans[cmd_start..pos]); // replacing the spans preserves syntax highlighting
1549
1550	let mut final_args = args.clone().into_vec();
1551	for arg_span in &resolution_spans[pos..] {
1552	let arg = parse_external_arg(working_set, *arg_span);
1553	final_args.push(arg);
1554	}
1555
1556	let expression = Expression::new(
1557	working_set,
1558	Expr::ExternalCall(head, final_args.into()),
1559	Span::concat(spans),
1560	ty.clone(),
1561	);
1562
1563	return expression;
1564	} else {
1565	trace!("parsing: alias of internal call");
1566	parse_internal_call(
1567	working_set,
1568	Span::concat(&resolution_spans[cmd_start..pos]),
1569	&resolution_spans[pos..],
1570	decl_id,
1571	ArgumentParsingLevel::Full,
1572	)
1573	}
1574	} else {
1575	trace!("parsing: internal call");
1576	parse_internal_call(
1577	working_set,
1578	Span::concat(&resolution_spans[cmd_start..pos]),
1579	&resolution_spans[pos..],
1580	decl_id,
1581	ArgumentParsingLevel::Full,
1582	)
1583	};
1584
1585	Expression::new(
1586	working_set,
1587	Expr::Call(parsed_call.call),
1588	call_span,
1589	parsed_call.output,
1590	)
1591	} else {
1592	if call_sigil == Some(b'%') {
1593	working_set.error(ParseError::LabeledErrorWithHelp {
1594	error: "percent sigil requires a built-in command".into(),
1595	label: "unknown built-in command".into(),
1596	help:
1597	"remove `%` to use normal resolution, or use `^` to run an external command explicitly".into(),
1598	span: resolution_spans[0],
1599	});
1600
1601	// Preserve expression shape for features like completion while retaining the parse error.
1602	return parse_external_call(working_set, spans, call_span);
1603	}
1604
1605	// We might be parsing left-unbounded range ("..10")
1606	let bytes = working_set.get_span_contents(spans[0]);
1607	trace!("parsing: range {bytes:?}");
1608	if let (Some(b'.'), Some(b'.')) = (bytes.first(), bytes.get(1)) {
1609	trace!("-- found leading range indicator");
1610	let starting_error_count = working_set.parse_errors.len();
1611
1612	if let Some(range_expr) = parse_range(working_set, spans[0]) {
1613	trace!("-- successfully parsed range");
1614	return range_expr;
1615	}
1616	working_set.parse_errors.truncate(starting_error_count);
1617	}
1618	trace!("parsing: external call");
1619
1620	// Otherwise, try external command
1621	parse_external_call(working_set, spans, call_span)
1622	}
1623	}
1624
1625	fn find_decl_with_command_type(
1626	working_set: &StateWorkingSet<'_>,
1627	name: &[u8],
1628	command_type: CommandType,
1629	) -> Option<DeclId> {
1630	// Search all known declarations so `%cmd` can still resolve a built-in even when
1631	// a custom command with the same name shadows it in normal visibility lookup.
1632	for idx in (0..working_set.num_decls()).rev() {
1633	let decl_id = DeclId::new(idx);
1634	let decl = working_set.get_decl(decl_id);
1635	if decl.command_type() == command_type && decl.name().as_bytes() == name {
1636	return Some(decl_id);
1637	}
1638	}
1639
1640	None
1641	}
1642
1643	// Build a command name from spaced spans, preserving the existing parser command-name behavior.
1644	fn command_name_from_spans(
1645	working_set: &StateWorkingSet<'_>,
1646	spans: &[Span],
1647	prefix: &[u8],
1648	) -> Vec<u8> {
1649	let mut name = Vec::with_capacity(prefix.len() + spans.len() * 2);
1650	name.extend(prefix);
1651
1652	for span in spans {
1653	let name_part = working_set.get_span_contents(*span);
1654	if name.is_empty() {
1655	name.extend(name_part);
1656	} else {
1657	name.push(b' ');
1658	name.extend(name_part);
1659	}
1660	}
1661
1662	name
1663	}
1664
1665	// Variant of `find_longest_decl` that constrains matches to a specific command type.
1666	fn find_longest_decl_with_command_type(
1667	working_set: &StateWorkingSet<'_>,
1668	spans: &[Span],
1669	command_type: CommandType,
1670	) -> (
1671	usize,
1672	usize,
1673	Vec<u8>,
1674	Option<nu_protocol::Id<nu_protocol::marker::Decl>>,
1675	) {
1676	let mut pos = spans.len();
1677	let cmd_start = 0;
1678	let mut name_spans = spans.to_vec();
1679
1680	let mut name = command_name_from_spans(working_set, &name_spans, b"");
1681
1682	let mut maybe_decl_id = find_decl_with_command_type(working_set, &name, command_type);
1683
1684	while maybe_decl_id.is_none() {
1685	if name_spans.len() <= 1 {
1686	break;
1687	}
1688
1689	name_spans.pop();
1690	pos -= 1;
1691
1692	name = command_name_from_spans(working_set, &name_spans, b"");
1693
1694	maybe_decl_id = find_decl_with_command_type(working_set, &name, command_type);
1695	}
1696
1697	(cmd_start, pos, name, maybe_decl_id)
1698	}
1699
1700	pub fn find_longest_decl(
1701	working_set: &mut StateWorkingSet<'_>,
1702	spans: &[Span],
1703	) -> (
1704	usize,
1705	usize,
1706	Vec<u8>,
1707	Option<nu_protocol::Id<nu_protocol::marker::Decl>>,
1708	) {
1709	find_longest_decl_with_prefix(working_set, spans, b"")
1710	}
1711
1712	pub fn find_longest_decl_with_prefix(
1713	working_set: &mut StateWorkingSet<'_>,
1714	spans: &[Span],
1715	prefix: &[u8],
1716	) -> (
1717	usize,
1718	usize,
1719	Vec<u8>,
1720	Option<nu_protocol::Id<nu_protocol::marker::Decl>>,
1721	) {
1722	let mut pos = 0;
1723	let cmd_start = pos;
1724	let mut name_spans = vec![];
1725
1726	for word_span in spans[cmd_start..].iter() {
1727	// Find the longest group of words that could form a command
1728
1729	name_spans.push(*word_span);
1730
1731	pos += 1;
1732	}
1733
1734	let mut name = command_name_from_spans(working_set, &name_spans, prefix);
1735
1736	let mut maybe_decl_id = working_set.find_decl(&name);
1737
1738	while maybe_decl_id.is_none() {
1739	// Find the longest command match
1740	if name_spans.len() <= 1 {
1741	// Keep the first word even if it does not match -- could be external command
1742	break;
1743	}
1744
1745	name_spans.pop();
1746	pos -= 1;
1747
1748	name = command_name_from_spans(working_set, &name_spans, prefix);
1749	maybe_decl_id = working_set.find_decl(&name);
1750	}
1751
1752	// If there is a declaration and there are remaining spans, check if it's an alias.
1753	// If it is, try to see if there are sub commands
1754	if let Some(decl_id) = maybe_decl_id
1755	&& pos < spans.len()
1756	{
1757	let decl = working_set.get_decl(decl_id);
1758	if let Some(alias) = decl.as_alias() {
1759	// Extract the command name from the alias
1760	// The wrapped_call should be a Call expression for internal commands
1761	if let Expression {
1762	expr: Expr::Call(call),
1763	..
1764	} = &alias.wrapped_call
1765	{
1766	let aliased_decl_id = call.decl_id;
1767	let aliased_name = working_set.get_decl(aliased_decl_id).name().to_string();
1768
1769	// Try to find a longer match using the aliased command name with remaining spans
1770	let (_, new_pos, new_name, new_decl_id) = find_longest_decl_with_prefix(
1771	working_set,
1772	&spans[pos..],
1773	aliased_name.as_bytes(),
1774	);
1775
1776	// If we find a sub command, use it instead.
1777	if new_decl_id.is_some() && new_pos > 0 {
1778	let total_pos = pos + new_pos;
1779	return (cmd_start, total_pos, new_name, new_decl_id);
1780	}
1781	}
1782	}
1783	}
1784
1785	(cmd_start, pos, name, maybe_decl_id)
1786	}
1787
1788	pub fn parse_attribute(
1789	working_set: &mut StateWorkingSet,
1790	lite_command: &LiteCommand,
1791	) -> (Attribute, Option<String>) {
1792	let _ = lite_command
1793	.parts
1794	.first()
1795	.filter(\|s\| working_set.get_span_contents(**s).starts_with(b"@"))
1796	.expect("Attributes always start with an `@`");
1797
1798	assert!(
1799	lite_command.attribute_idx.is_empty(),
1800	"attributes can't have attributes"
1801	);
1802
1803	let mut spans = lite_command.parts.clone();
1804	if let Some(first) = spans.first_mut() {
1805	first.start += 1;
1806	}
1807	let spans = spans.as_slice();
1808	let attr_span = Span::concat(spans);
1809
1810	let (cmd_start, cmd_end, mut name, decl_id) =
1811	find_longest_decl_with_prefix(working_set, spans, b"attr");
1812
1813	debug_assert!(name.starts_with(b"attr "));
1814	let _ = name.drain(..(b"attr ".len()));
1815
1816	let name_span = Span::concat(&spans[cmd_start..cmd_end]);
1817
1818	let Ok(name) = String::from_utf8(name) else {
1819	working_set.error(ParseError::NonUtf8(name_span));
1820	return (
1821	Attribute {
1822	expr: garbage(working_set, attr_span),
1823	},
1824	None,
1825	);
1826	};
1827
1828	let Some(decl_id) = decl_id else {
1829	working_set.error(ParseError::UnknownCommand(name_span));
1830	return (
1831	Attribute {
1832	expr: garbage(working_set, attr_span),
1833	},
1834	None,
1835	);
1836	};
1837
1838	let decl = working_set.get_decl(decl_id);
1839
1840	let parsed_call = match decl.as_alias() {
1841	// TODO: Once `const def` is available, we should either disallow aliases as attributes OR
1842	// allow them but rather than using the aliases' name, use the name of the aliased command
1843	Some(alias) => match &alias.clone().wrapped_call {
1844	Expression {
1845	expr: Expr::ExternalCall(..),
1846	..
1847	} => {
1848	let shell_error = ShellError::NotAConstCommand { span: name_span };
1849	working_set.error(shell_error.wrap(working_set, attr_span));
1850	return (
1851	Attribute {
1852	expr: garbage(working_set, Span::concat(spans)),
1853	},
1854	None,
1855	);
1856	}
1857	_ => {
1858	trace!("parsing: alias of internal call");
1859	parse_internal_call(
1860	working_set,
1861	name_span,
1862	&spans[cmd_end..],
1863	decl_id,
1864	ArgumentParsingLevel::Full,
1865	)
1866	}
1867	},
1868	None => {
1869	trace!("parsing: internal call");
1870	parse_internal_call(
1871	working_set,
1872	name_span,
1873	&spans[cmd_end..],
1874	decl_id,
1875	ArgumentParsingLevel::Full,
1876	)
1877	}
1878	};
1879
1880	(
1881	Attribute {
1882	expr: Expression::new(
1883	working_set,
1884	Expr::Call(parsed_call.call),
1885	Span::concat(spans),
1886	parsed_call.output,
1887	),
1888	},
1889	Some(name),
1890	)
1891	}
1892
1893	pub fn parse_binary(working_set: &mut StateWorkingSet, span: Span) -> Expression {
1894	trace!("parsing: binary");
1895	let contents = working_set.get_span_contents(span);
1896	if contents.starts_with(b"0x[") {
1897	parse_binary_with_base(working_set, span, 16, 2, b"0x[", b"]")
1898	} else if contents.starts_with(b"0o[") {
1899	parse_binary_with_base(working_set, span, 8, 3, b"0o[", b"]")
1900	} else if contents.starts_with(b"0b[") {
1901	parse_binary_with_base(working_set, span, 2, 8, b"0b[", b"]")
1902	} else {
1903	working_set.error(ParseError::Expected("binary", span));
1904	garbage(working_set, span)
1905	}
1906	}
1907
1908	fn parse_binary_with_base(
1909	working_set: &mut StateWorkingSet,
1910	span: Span,
1911	base: u32,
1912	min_digits_per_byte: usize,
1913	prefix: &[u8],
1914	suffix: &[u8],
1915	) -> Expression {
1916	let token = working_set.get_span_contents(span);
1917
1918	if let Some(token) = token.strip_prefix(prefix)
1919	&& let Some(token) = token.strip_suffix(suffix)
1920	{
1921	let (lexed, err) = lex(
1922	token,
1923	span.start + prefix.len(),
1924	&[b',', b'\r', b'\n'],
1925	&[],
1926	true,
1927	);
1928	if let Some(err) = err {
1929	working_set.error(err);
1930	}
1931
1932	let mut binary_value = vec![];
1933	for token in lexed {
1934	match token.contents {
1935	TokenContents::Item => {
1936	let contents = working_set.get_span_contents(token.span);
1937
1938	binary_value.extend_from_slice(contents);
1939	}
1940	TokenContents::Pipe
1941	\| TokenContents::PipePipe
1942	\| TokenContents::ErrGreaterPipe
1943	\| TokenContents::OutGreaterThan
1944	\| TokenContents::OutErrGreaterPipe
1945	\| TokenContents::OutGreaterGreaterThan
1946	\| TokenContents::ErrGreaterThan
1947	\| TokenContents::ErrGreaterGreaterThan
1948	\| TokenContents::OutErrGreaterThan
1949	\| TokenContents::OutErrGreaterGreaterThan
1950	\| TokenContents::AssignmentOperator => {
1951	working_set.error(ParseError::Expected("binary", span));
1952	return garbage(working_set, span);
1953	}
1954	TokenContents::Comment \| TokenContents::Semicolon \| TokenContents::Eol => {}
1955	}
1956	}
1957
1958	let required_padding =
1959	(min_digits_per_byte - binary_value.len() % min_digits_per_byte) % min_digits_per_byte;
1960
1961	if required_padding != 0 {
1962	binary_value = {
1963	let mut tail = binary_value;
1964	let mut binary_value: Vec<u8> = vec![b'0'; required_padding];
1965	binary_value.append(&mut tail);
1966	binary_value
1967	};
1968	}
1969
1970	let str = String::from_utf8_lossy(&binary_value).to_string();
1971
1972	match decode_with_base(&str, base, min_digits_per_byte) {
1973	Ok(v) => return Expression::new(working_set, Expr::Binary(v), span, Type::Binary),
1974	Err(help) => {
1975	working_set.error(ParseError::InvalidBinaryString(span, help.to_string()));
1976	return garbage(working_set, span);
1977	}
1978	}
1979	}
1980
1981	working_set.error(ParseError::Expected("binary", span));
1982	garbage(working_set, span)
1983	}
1984
1985	fn decode_with_base(s: &str, base: u32, digits_per_byte: usize) -> Result<Vec<u8>, &str> {
1986	s.chars()
1987	.chunks(digits_per_byte)
1988	.into_iter()
1989	.map(\|chunk\| {
1990	let str: String = chunk.collect();
1991	u8::from_str_radix(&str, base).map_err(\|_\| match base {
1992	2 => "binary strings may contain only 0 or 1.",
1993	8 => "octal strings must have a length that is a multiple of three and contain values between 0o000 and 0o377.",
1994	16 => "hexadecimal strings may contain only the characters 0–9 and A–F.",
1995	_ => "internal error: radix other than 2, 8, or 16 is not allowed."
1996	})
1997	})
1998	.collect()
1999	}
2000
2001	fn strip_underscores(token: &[u8]) -> String {
2002	String::from_utf8_lossy(token)
2003	.chars()
2004	.filter(\|c\| *c != '_')
2005	.collect()
2006	}
2007
2008	pub fn parse_int(working_set: &mut StateWorkingSet, span: Span) -> Expression {
2009	let token = working_set.get_span_contents(span);
2010
2011	fn extract_int(
2012	working_set: &mut StateWorkingSet,
2013	token: &str,
2014	span: Span,
2015	radix: u32,
2016	) -> Expression {
2017	// Parse as a u64, then cast to i64, otherwise, for numbers like "0xffffffffffffffef",
2018	// you'll get `Error parsing hex string: number too large to fit in target type`.
2019	if let Ok(num) = u64::from_str_radix(token, radix).map(\|val\| val as i64) {
2020	Expression::new(working_set, Expr::Int(num), span, Type::Int)
2021	} else {
2022	working_set.error(ParseError::InvalidLiteral(
2023	format!("invalid digits for radix {radix}"),
2024	"int".into(),
2025	span,
2026	));
2027
2028	garbage(working_set, span)
2029	}
2030	}
2031
2032	let token = strip_underscores(token);
2033
2034	if token.is_empty() {
2035	working_set.error(ParseError::Expected("int", span));
2036	return garbage(working_set, span);
2037	}
2038
2039	if let Some(num) = token.strip_prefix("0b") {
2040	extract_int(working_set, num, span, 2)
2041	} else if let Some(num) = token.strip_prefix("0o") {
2042	extract_int(working_set, num, span, 8)
2043	} else if let Some(num) = token.strip_prefix("0x") {
2044	extract_int(working_set, num, span, 16)
2045	} else if let Ok(num) = token.parse::<i64>() {
2046	Expression::new(working_set, Expr::Int(num), span, Type::Int)
2047	} else {
2048	working_set.error(ParseError::Expected("int", span));
2049	garbage(working_set, span)
2050	}
2051	}
2052
2053	pub fn parse_float(working_set: &mut StateWorkingSet, span: Span) -> Expression {
2054	let token = working_set.get_span_contents(span);
2055	let token = strip_underscores(token);
2056
2057	if let Ok(x) = token.parse::<f64>() {
2058	Expression::new(working_set, Expr::Float(x), span, Type::Float)
2059	} else {
2060	working_set.error(ParseError::Expected("float", span));
2061
2062	garbage(working_set, span)
2063	}
2064	}
2065
2066	pub fn parse_number(working_set: &mut StateWorkingSet, span: Span) -> Expression {
2067	let starting_error_count = working_set.parse_errors.len();
2068
2069	let result = parse_int(working_set, span);
2070	if starting_error_count == working_set.parse_errors.len() {
2071	return result;
2072	} else if let Some(ParseError::Expected(_, _)) = working_set.parse_errors.last() {
2073	working_set.parse_errors.truncate(starting_error_count);
2074	}
2075
2076	let result = parse_float(working_set, span);
2077
2078	if starting_error_count == working_set.parse_errors.len() {
2079	return result;
2080	}
2081	working_set.parse_errors.truncate(starting_error_count);
2082
2083	working_set.error(ParseError::Expected("number", span));
2084	garbage(working_set, span)
2085	}
2086
2087	pub fn parse_range(working_set: &mut StateWorkingSet, span: Span) -> Option<Expression> {
2088	trace!("parsing: range");
2089	let starting_error_count = working_set.parse_errors.len();
2090
2091	// Range follows the following syntax: [<from>][<next_operator><next>]<range_operator>[<to>]
2092	// where <next_operator> is ".."
2093	// and <range_operator> is "..", "..=" or "..<"
2094	// and one of the <from> or <to> bounds must be present (just '..' is not allowed since it
2095	// looks like parent directory)
2096	//bugbug range cannot be [..] because that looks like parent directory
2097
2098	let contents = working_set.get_span_contents(span);
2099
2100	let token = if let Ok(s) = String::from_utf8(contents.into()) {
2101	s
2102	} else {
2103	working_set.error(ParseError::NonUtf8(span));
2104	return None;
2105	};
2106
2107	if token.starts_with("...") {
2108	working_set.error(ParseError::Expected(
2109	"range operator ('..'), got spread ('...')",
2110	span,
2111	));
2112	return None;
2113	}
2114
2115	if !token.contains("..") {
2116	working_set.error(ParseError::Expected("at least one range bound set", span));
2117	return None;
2118	}
2119
2120	let dotdot_pos: Vec<_> = token
2121	.match_indices("..")
2122	.filter_map(\|(pos, _)\| {
2123	// paren_depth = count of unclosed parens prior to pos
2124	let before = &token[..pos];
2125	let paren_depth = before
2126	.chars()
2127	.filter(\|&c\| c == '(')
2128	.count()
2129	.checked_sub(before.chars().filter(\|&c\| c == ')').count());
2130	paren_depth.and_then(\|d\| (d == 0).then_some(pos))
2131	})
2132	.collect();
2133
2134	let (next_op_pos, range_op_pos) = match dotdot_pos.len() {
2135	1 => (None, dotdot_pos[0]),
2136	2 => (Some(dotdot_pos[0]), dotdot_pos[1]),
2137	_ => {
2138	working_set.error(ParseError::Expected(
2139	"one range operator ('..' or '..<') and optionally one next operator ('..')",
2140	span,
2141	));
2142	return None;
2143	}
2144	};
2145	// Avoid calling sub-parsers on unmatched parens, to prevent quadratic time on things like ((((1..2))))
2146	// No need to call the expensive parse_value on "((((1"
2147	if dotdot_pos[0] > 0 {
2148	let (_tokens, err) = lex(
2149	&contents[..dotdot_pos[0]],
2150	span.start,
2151	&[],
2152	&[b'.', b'?', b'!'],
2153	true,
2154	);
2155	if let Some(_err) = err {
2156	working_set.error(ParseError::Expected("Valid expression before ..", span));
2157	return None;
2158	}
2159	}
2160
2161	let (inclusion, range_op_str, range_op_span) = if let Some(pos) = token.find("..<") {
2162	if pos == range_op_pos {
2163	let op_str = "..<";
2164	let op_span = Span::new(
2165	span.start + range_op_pos,
2166	span.start + range_op_pos + op_str.len(),
2167	);
2168	(RangeInclusion::RightExclusive, "..<", op_span)
2169	} else {
2170	working_set.error(ParseError::Expected(
2171	"inclusive operator preceding second range bound",
2172	span,
2173	));
2174	return None;
2175	}
2176	} else {
2177	let op_str = if token[range_op_pos..].starts_with("..=") {
2178	"..="
2179	} else {
2180	".."
2181	};
2182
2183	let op_span = Span::new(
2184	span.start + range_op_pos,
2185	span.start + range_op_pos + op_str.len(),
2186	);
2187	(RangeInclusion::Inclusive, op_str, op_span)
2188	};
2189
2190	// Now, based on the operator positions, figure out where the bounds & next are located and
2191	// parse them
2192	// TODO: Actually parse the next number in the range
2193	let from = if token.starts_with("..") {
2194	// token starts with either next operator, or range operator -- we don't care which one
2195	None
2196	} else {
2197	let from_span = Span::new(span.start, span.start + dotdot_pos[0]);
2198	Some(parse_value(working_set, from_span, &SyntaxShape::Number))
2199	};
2200
2201	let to = if token.ends_with(range_op_str) {
2202	None
2203	} else {
2204	let to_span = Span::new(range_op_span.end, span.end);
2205	Some(parse_value(working_set, to_span, &SyntaxShape::Number))
2206	};
2207
2208	trace!("-- from: {from:?} to: {to:?}");
2209
2210	if let (None, None) = (&from, &to) {
2211	working_set.error(ParseError::Expected("at least one range bound set", span));
2212	return None;
2213	}
2214
2215	let (next, next_op_span) = if let Some(pos) = next_op_pos {
2216	let next_op_span = Span::new(span.start + pos, span.start + pos + "..".len());
2217	let next_span = Span::new(next_op_span.end, range_op_span.start);
2218
2219	(
2220	Some(parse_value(working_set, next_span, &SyntaxShape::Number)),
2221	next_op_span,
2222	)
2223	} else {
2224	(None, span)
2225	};
2226
2227	if working_set.parse_errors.len() != starting_error_count {
2228	return None;
2229	}
2230
2231	let operator = RangeOperator {
2232	inclusion,
2233	span: range_op_span,
2234	next_op_span,
2235	};
2236
2237	let mut range = Range {
2238	from,
2239	next,
2240	to,
2241	operator,
2242	};
2243
2244	check_range_types(working_set, &mut range);
2245
2246	Some(Expression::new(
2247	working_set,
2248	Expr::Range(Box::new(range)),
2249	span,
2250	Type::Range,
2251	))
2252	}
2253
2254	pub(crate) fn parse_dollar_expr(working_set: &mut StateWorkingSet, span: Span) -> Expression {
2255	trace!("parsing: dollar expression");
2256	let contents = working_set.get_span_contents(span);
2257
2258	if contents.starts_with(b"$\"") \|\| contents.starts_with(b"$'") {
2259	parse_string_interpolation(working_set, span)
2260	} else if contents.starts_with(b"$.") {
2261	parse_simple_cell_path(working_set, Span::new(span.start + 2, span.end))
2262	} else {
2263	let starting_error_count = working_set.parse_errors.len();
2264
2265	if let Some(expr) = parse_range(working_set, span) {
2266	expr
2267	} else {
2268	working_set.parse_errors.truncate(starting_error_count);
2269	parse_full_cell_path(working_set, None, span)
2270	}
2271	}
2272	}
2273
2274	pub fn parse_raw_string(working_set: &mut StateWorkingSet, span: Span) -> Expression {
2275	trace!("parsing: raw-string, with required delimiters");
2276
2277	let bytes = working_set.get_span_contents(span);
2278
2279	let prefix_sharp_cnt = if bytes.starts_with(b"r#") {
2280	// actually `sharp_cnt` is always `index - 1`
2281	// but create a variable here to make it clearer.
2282	let mut sharp_cnt = 1;
2283	let mut index = 2;
2284	while index < bytes.len() && bytes[index] == b'#' {
2285	index += 1;
2286	sharp_cnt += 1;
2287	}
2288	sharp_cnt
2289	} else {
2290	working_set.error(ParseError::Expected("r#", span));
2291	return garbage(working_set, span);
2292	};
2293	let expect_postfix_sharp_cnt = prefix_sharp_cnt;
2294	// check the length of whole raw string.
2295	// the whole raw string should contains at least
2296	// 1(r) + prefix_sharp_cnt + 1(') + 1(') + postfix_sharp characters
2297	if bytes.len() < prefix_sharp_cnt + expect_postfix_sharp_cnt + 3 {
2298	working_set.error(ParseError::Unclosed('\''.into(), span));
2299	return garbage(working_set, span);
2300	}
2301
2302	// check for unbalanced # and single quotes.
2303	let postfix_bytes = &bytes[bytes.len() - expect_postfix_sharp_cnt..bytes.len()];
2304	if postfix_bytes.iter().any(\|b\| *b != b'#') {
2305	working_set.error(ParseError::Unbalanced(
2306	"prefix #".to_string(),
2307	"postfix #".to_string(),
2308	span,
2309	));
2310	return garbage(working_set, span);
2311	}
2312	// check for unblanaced single quotes.
2313	if bytes[1 + prefix_sharp_cnt] != b'\''
2314	\|\| bytes[bytes.len() - expect_postfix_sharp_cnt - 1] != b'\''
2315	{
2316	working_set.error(ParseError::Unclosed('\''.into(), span));
2317	return garbage(working_set, span);
2318	}
2319
2320	let bytes = &bytes[prefix_sharp_cnt + 1 + 1..bytes.len() - 1 - prefix_sharp_cnt];
2321	if let Ok(token) = String::from_utf8(bytes.into()) {
2322	Expression::new(working_set, Expr::RawString(token), span, Type::String)
2323	} else {
2324	working_set.error(ParseError::Expected("utf8 raw-string", span));
2325	garbage(working_set, span)
2326	}
2327	}
2328
2329	pub fn parse_paren_expr(
2330	working_set: &mut StateWorkingSet,
2331	span: Span,
2332	shape: &SyntaxShape,
2333	) -> Expression {
2334	let starting_error_count = working_set.parse_errors.len();
2335
2336	if let Some(expr) = parse_range(working_set, span) {
2337	return expr;
2338	}
2339
2340	working_set.parse_errors.truncate(starting_error_count);
2341
2342	if let SyntaxShape::Signature = shape {
2343	return parse_signature(working_set, span, false);
2344	}
2345
2346	if let SyntaxShape::ExternalSignature = shape {
2347	return parse_signature(working_set, span, true);
2348	}
2349
2350	let fcp_expr = parse_full_cell_path(working_set, None, span);
2351	let fcp_error_count = working_set.parse_errors.len();
2352	if fcp_error_count > starting_error_count {
2353	let malformed_subexpr = working_set.parse_errors[starting_error_count..]
2354	.first()
2355	.is_some_and(\|e\| match e {
2356	ParseError::Unclosed(right, _) if (right == ")") => true,
2357	ParseError::Unbalanced(left, right, _) if left == "(" && right == ")" => true,
2358	_ => false,
2359	});
2360	if malformed_subexpr {
2361	working_set.parse_errors.truncate(starting_error_count);
2362	parse_string_interpolation(working_set, span)
2363	} else {
2364	fcp_expr
2365	}
2366	} else {
2367	fcp_expr
2368	}
2369	}
2370
2371	pub fn parse_brace_expr(
2372	working_set: &mut StateWorkingSet,
2373	span: Span,
2374	shape: &SyntaxShape,
2375	) -> Expression {
2376	// Try to detect what kind of value we're about to parse
2377	// FIXME: In the future, we should work over the token stream so we only have to do this once
2378	// before parsing begins
2379
2380	// FIXME: we're still using the shape because we rely on it to know how to handle syntax where
2381	// the parse is ambiguous. We'll need to update the parts of the grammar where this is ambiguous
2382	// and then revisit the parsing.
2383
2384	if span.end <= (span.start + 1) {
2385	working_set.error(ParseError::ExpectedWithStringMsg(
2386	format!("non-block value: {shape}"),
2387	span,
2388	));
2389	return Expression::garbage(working_set, span);
2390	}
2391	let bytes = working_set.get_span_contents(Span::new(span.start + 1, span.end - 1));
2392	let (tokens, _) = lex(bytes, span.start + 1, &[b'\r', b'\n', b'\t'], &[b':'], true);
2393
2394	match tokens.as_slice() {
2395	// If we're empty, that means an empty record or closure
2396	[] => match shape {
2397	SyntaxShape::Closure(_) => parse_closure_expression(working_set, shape, span),
2398	SyntaxShape::Block => parse_block_expression(working_set, span),
2399	SyntaxShape::MatchBlock => parse_match_block_expression(working_set, span),
2400	_ => parse_record(working_set, span),
2401	},
2402	[
2403	Token {
2404	contents: TokenContents::Pipe \| TokenContents::PipePipe,
2405	..
2406	},
2407	..,
2408	] => {
2409	if let SyntaxShape::Block = shape {
2410	working_set.error(ParseError::Mismatch("block".into(), "closure".into(), span));
2411	return Expression::garbage(working_set, span);
2412	}
2413	parse_closure_expression(working_set, shape, span)
2414	}
2415	[_, third, ..] if working_set.get_span_contents(third.span) == b":" => {
2416	parse_full_cell_path(working_set, None, span)
2417	}
2418	[second, ..] => {
2419	let second_bytes = working_set.get_span_contents(second.span);
2420	match shape {
2421	SyntaxShape::Closure(_) => parse_closure_expression(working_set, shape, span),
2422	SyntaxShape::Block => parse_block_expression(working_set, span),
2423	SyntaxShape::MatchBlock => parse_match_block_expression(working_set, span),
2424	// For edge case of `{}.foo?`, #17896
2425	_ if second_bytes == b"}" => parse_full_cell_path(working_set, None, span),
2426	_ if second_bytes.starts_with(b"...")
2427	&& second_bytes.get(3).is_some_and(\|c\| b"${(".contains(c)) =>
2428	{
2429	parse_record(working_set, span)
2430	}
2431	SyntaxShape::Any => parse_closure_expression(working_set, shape, span),
2432	_ => {
2433	working_set.error(ParseError::ExpectedWithStringMsg(
2434	format!("non-block value: {shape}"),
2435	span,
2436	));
2437
2438	Expression::garbage(working_set, span)
2439	}
2440	}
2441	}
2442	}
2443	}
2444
2445	pub fn parse_string_interpolation(working_set: &mut StateWorkingSet, span: Span) -> Expression {
2446	#[derive(PartialEq, Eq, Debug)]
2447	enum InterpolationMode {
2448	String,
2449	Expression,
2450	}
2451
2452	let contents = working_set.get_span_contents(span);
2453
2454	let mut double_quote = false;
2455
2456	let (start, end) = if contents.starts_with(b"$\"") {
2457	double_quote = true;
2458
2459	if let Err(err) = check_string_no_trailing_tokens(contents, span, 1, b'\"') {
2460	working_set.error(err);
2461	return garbage(working_set, span);
2462	}
2463
2464	let end = if contents.ends_with(b"\"") && contents.len() > 2 {
2465	span.end - 1
2466	} else {
2467	span.end
2468	};
2469	(span.start + 2, end)
2470	} else if contents.starts_with(b"$'") {
2471	if let Err(err) = check_string_no_trailing_tokens(contents, span, 1, b'\'') {
2472	working_set.error(err);
2473	return garbage(working_set, span);
2474	}
2475
2476	let end = if contents.ends_with(b"'") && contents.len() > 2 {
2477	span.end - 1
2478	} else {
2479	span.end
2480	};
2481	(span.start + 2, end)
2482	} else {
2483	(span.start, span.end)
2484	};
2485
2486	let inner_span = Span::new(start, end);
2487	let contents = working_set.get_span_contents(inner_span).to_vec();
2488
2489	let mut output = vec![];
2490	let mut mode = InterpolationMode::String;
2491	let mut token_start = start;
2492
2493	#[repr(u8)]
2494	#[derive(Clone, Copy, PartialEq, Eq)]
2495	enum Delimiter {
2496	SingleQuote = b'\'',
2497	DoubleQuote = b'"',
2498	Backtick = b'`',
2499	ParenLeft = b'(',
2500	ParenRight = b')',
2501	}
2502
2503	impl Delimiter {
2504	const fn from_u8(b: u8) -> Option<Self> {
2505	Some(match b {
2506	b'\'' => Self::SingleQuote,
2507	b'"' => Self::DoubleQuote,
2508	b'`' => Self::Backtick,
2509	b'(' => Self::ParenLeft,
2510	b')' => Self::ParenRight,
2511	_ => return None,
2512	})
2513	}
2514	const fn is_paren(self) -> bool {
2515	matches!(self, Self::ParenLeft \| Self::ParenRight)
2516	}
2517	const fn pair(self) -> Self {
2518	match self {
2519	Self::ParenLeft => Self::ParenRight,
2520	Self::ParenRight => Self::ParenLeft,
2521	_ => self,
2522	}
2523	}
2524	}
2525	let mut delimiter_stack: Vec<Delimiter> = vec![];
2526
2527	let mut consecutive_backslashes: usize = 0;
2528
2529	let mut b = start;
2530
2531	while b != end {
2532	let current_byte = contents[b - start];
2533
2534	if mode == InterpolationMode::String {
2535	let preceding_consecutive_backslashes = consecutive_backslashes;
2536
2537	let is_backslash = current_byte == b'\\';
2538	consecutive_backslashes = if is_backslash {
2539	preceding_consecutive_backslashes + 1
2540	} else {
2541	0
2542	};
2543
2544	if current_byte == b'('
2545	&& (!double_quote \|\| preceding_consecutive_backslashes.is_multiple_of(2))
2546	{
2547	mode = InterpolationMode::Expression;
2548	if token_start < b {
2549	let span = Span::new(token_start, b);
2550	let str_contents = working_set.get_span_contents(span);
2551
2552	let (str_contents, err) = if double_quote {
2553	unescape_string(str_contents, span)
2554	} else {
2555	(str_contents.to_vec(), None)
2556	};
2557	if let Some(err) = err {
2558	working_set.error(err);
2559	}
2560
2561	output.push(Expression::new(
2562	working_set,
2563	Expr::String(String::from_utf8_lossy(&str_contents).to_string()),
2564	span,
2565	Type::String,
2566	));
2567	token_start = b;
2568	}
2569	}
2570	}
2571
2572	if mode == InterpolationMode::Expression {
2573	let byte = Delimiter::from_u8(current_byte);
2574	match (delimiter_stack.last().copied(), byte) {
2575	(Some(d), Some(byte)) if !d.is_paren() => {
2576	if byte == d {
2577	delimiter_stack.pop();
2578	}
2579	}
2580	(_, Some(byte)) if byte != Delimiter::ParenRight => {
2581	delimiter_stack.push(byte.pair())
2582	}
2583	(d, Some(Delimiter::ParenRight)) => {
2584	if let Some(Delimiter::ParenRight) = d {
2585	delimiter_stack.pop();
2586	}
2587	if delimiter_stack.is_empty() {
2588	mode = InterpolationMode::String;
2589
2590	if token_start < b {
2591	let span = Span::new(token_start, b + 1);
2592
2593	let expr = parse_full_cell_path(working_set, None, span);
2594	output.push(expr);
2595	}
2596
2597	token_start = b + 1;
2598	continue;
2599	}
2600	}
2601	_ => (),
2602	}
2603	}
2604	b += 1;
2605	}
2606
2607	match mode {
2608	InterpolationMode::String => {
2609	if token_start < end {
2610	let span = Span::new(token_start, end);
2611	let str_contents = working_set.get_span_contents(span);
2612
2613	let (str_contents, err) = if double_quote {
2614	unescape_string(str_contents, span)
2615	} else {
2616	(str_contents.to_vec(), None)
2617	};
2618	if let Some(err) = err {
2619	working_set.error(err);
2620	}
2621
2622	output.push(Expression::new(
2623	working_set,
2624	Expr::String(String::from_utf8_lossy(&str_contents).to_string()),
2625	span,
2626	Type::String,
2627	));
2628	}
2629	}
2630	InterpolationMode::Expression => {
2631	if token_start < end {
2632	let span = Span::new(token_start, end);
2633	let expr = parse_full_cell_path(working_set, None, span);
2634	output.push(expr);
2635	}
2636	}
2637	}
2638
2639	Expression::new(
2640	working_set,
2641	Expr::StringInterpolation(output),
2642	span,
2643	Type::String,
2644	)
2645	}
2646
2647	pub fn parse_variable_expr(working_set: &mut StateWorkingSet, span: Span) -> Expression {
2648	let contents = working_set.get_span_contents(span);
2649
2650	if contents == b"$nu" {
2651	return Expression::new(
2652	working_set,
2653	Expr::Var(nu_protocol::NU_VARIABLE_ID),
2654	span,
2655	Type::Any,
2656	);
2657	} else if contents == b"$in" {
2658	return Expression::new(
2659	working_set,
2660	Expr::Var(nu_protocol::IN_VARIABLE_ID),
2661	span,
2662	Type::Any,
2663	);
2664	} else if contents == b"$env" {
2665	return Expression::new(
2666	working_set,
2667	Expr::Var(nu_protocol::ENV_VARIABLE_ID),
2668	span,
2669	Type::Any,
2670	);
2671	}
2672
2673	let name = if contents.starts_with(b"$") {
2674	String::from_utf8_lossy(&contents[1..]).to_string()
2675	} else {
2676	String::from_utf8_lossy(contents).to_string()
2677	};
2678
2679	let bytes = working_set.get_span_contents(span);
2680	let suggestion = \|\| {
2681	DidYouMean::new(
2682	&working_set.list_variables(),
2683	working_set.get_span_contents(span),
2684	)
2685	};
2686	if !is_variable(bytes) {
2687	working_set.error(ParseError::ExpectedWithDidYouMean(
2688	"valid variable name",
2689	suggestion(),
2690	span,
2691	));
2692	garbage(working_set, span)
2693	} else if let Some(id) = working_set.find_variable(bytes) {
2694	Expression::new(
2695	working_set,
2696	Expr::Var(id),
2697	span,
2698	working_set.get_variable(id).ty.clone(),
2699	)
2700	} else if working_set.get_env_var(&name).is_some() {
2701	working_set.error(ParseError::EnvVarNotVar(name, span));
2702	garbage(working_set, span)
2703	} else {
2704	working_set.error(ParseError::VariableNotFound(suggestion(), span));
2705	garbage(working_set, span)
2706	}
2707	}
2708
2709	pub fn parse_cell_path(
2710	working_set: &mut StateWorkingSet,
2711	tokens: impl Iterator<Item = Token>,
2712	expect_dot: bool,
2713	) -> Vec<PathMember> {
2714	enum TokenType {
2715	Dot, // .
2716	DotOrSign, // . or ? or !
2717	DotOrExclamation, // . or !
2718	DotOrQuestion, // . or ?
2719	PathMember, // an int or string, like `1` or `foo`
2720	}
2721
2722	enum ModifyMember {
2723	No,
2724	Optional,
2725	Insensitive,
2726	}
2727
2728	impl TokenType {
2729	fn expect(&mut self, byte: u8) -> Result<ModifyMember, &'static str> {
2730	match (&*self, byte) {
2731	(Self::PathMember, _) => {
2732	*self = Self::DotOrSign;
2733	Ok(ModifyMember::No)
2734	}
2735	(
2736	Self::Dot \| Self::DotOrSign \| Self::DotOrExclamation \| Self::DotOrQuestion,
2737	b'.',
2738	) => {
2739	*self = Self::PathMember;
2740	Ok(ModifyMember::No)
2741	}
2742	(Self::DotOrSign, b'!') => {
2743	*self = Self::DotOrQuestion;
2744	Ok(ModifyMember::Insensitive)
2745	}
2746	(Self::DotOrSign, b'?') => {
2747	*self = Self::DotOrExclamation;
2748	Ok(ModifyMember::Optional)
2749	}
2750	(Self::DotOrSign, _) => Err(". or ! or ?"),
2751	(Self::DotOrExclamation, b'!') => {
2752	*self = Self::Dot;
2753	Ok(ModifyMember::Insensitive)
2754	}
2755	(Self::DotOrExclamation, _) => Err(". or !"),
2756	(Self::DotOrQuestion, b'?') => {
2757	*self = Self::Dot;
2758	Ok(ModifyMember::Optional)
2759	}
2760	(Self::DotOrQuestion, _) => Err(". or ?"),
2761	(Self::Dot, _) => Err("."),
2762	}
2763	}
2764	}
2765
2766	// Parsing a cell path is essentially a state machine, and this is the state
2767	let mut expected_token = if expect_dot {
2768	TokenType::Dot
2769	} else {
2770	TokenType::PathMember
2771	};
2772
2773	let mut tail = vec![];
2774
2775	for path_element in tokens {
2776	let bytes = working_set.get_span_contents(path_element.span);
2777
2778	// both parse_int and parse_string require their source to be non-empty
2779	// all cases where `bytes` is empty is an error
2780	let Some((&first, rest)) = bytes.split_first() else {
2781	working_set.error(ParseError::Expected("string", path_element.span));
2782	return tail;
2783	};
2784	let single_char = rest.is_empty();
2785
2786	if let TokenType::PathMember = expected_token {
2787	let starting_error_count = working_set.parse_errors.len();
2788
2789	let expr = parse_int(working_set, path_element.span);
2790	working_set.parse_errors.truncate(starting_error_count);
2791
2792	match expr {
2793	Expression {
2794	expr: Expr::Int(val),
2795	span,
2796	..
2797	} => tail.push(PathMember::Int {
2798	val: val as usize,
2799	span,
2800	optional: false,
2801	}),
2802	_ => {
2803	let result = parse_string(working_set, path_element.span);
2804	match result {
2805	Expression {
2806	expr: Expr::String(string),
2807	span,
2808	..
2809	} => {
2810	tail.push(PathMember::String {
2811	val: string,
2812	span,
2813	optional: false,
2814	casing: Casing::Sensitive,
2815	});
2816	}
2817	_ => {
2818	working_set.error(ParseError::Expected("string", path_element.span));
2819	return tail;
2820	}
2821	}
2822	}
2823	}
2824	expected_token = TokenType::DotOrSign;
2825	} else {
2826	match expected_token.expect(if single_char { first } else { b' ' }) {
2827	Ok(modify) => {
2828	if let Some(last) = tail.last_mut() {
2829	match modify {
2830	ModifyMember::No => {}
2831	ModifyMember::Optional => last.make_optional(),
2832	ModifyMember::Insensitive => last.make_insensitive(),
2833	}
2834	};
2835	}
2836	Err(expected) => {
2837	working_set.error(ParseError::Expected(expected, path_element.span));
2838	return tail;
2839	}
2840	}
2841	}
2842	}
2843
2844	tail
2845	}
2846
2847	pub fn parse_simple_cell_path(working_set: &mut StateWorkingSet, span: Span) -> Expression {
2848	let source = working_set.get_span_contents(span);
2849
2850	let (tokens, err) = lex(
2851	source,
2852	span.start,
2853	&[b'\n', b'\r'],
2854	&[b'.', b'?', b'!'],
2855	true,
2856	);
2857	if let Some(err) = err {
2858	working_set.error(err)
2859	}
2860
2861	let tokens = tokens.into_iter().peekable();
2862
2863	let cell_path = parse_cell_path(working_set, tokens, false);
2864
2865	Expression::new(
2866	working_set,
2867	Expr::CellPath(CellPath { members: cell_path }),
2868	span,
2869	Type::CellPath,
2870	)
2871	}
2872
2873	pub fn parse_full_cell_path(
2874	working_set: &mut StateWorkingSet,
2875	implicit_head: Option<VarId>,
2876	span: Span,
2877	) -> Expression {
2878	trace!("parsing: full cell path");
2879	let full_cell_span = span;
2880	let source = working_set.get_span_contents(span);
2881
2882	let (tokens, err) = lex(
2883	source,
2884	span.start,
2885	&[b'\n', b'\r'],
2886	&[b'.', b'?', b'!'],
2887	true,
2888	);
2889	if let Some(err) = err {
2890	working_set.error(err)
2891	}
2892
2893	let mut tokens = tokens.into_iter().peekable();
2894	if let Some(head) = tokens.peek() {
2895	let bytes = working_set.get_span_contents(head.span);
2896	let (head, expect_dot) = if bytes.starts_with(b"(") {
2897	trace!("parsing: paren-head of full cell path");
2898
2899	let head_span = head.span;
2900	let mut start = head.span.start;
2901	let mut end = head.span.end;
2902	let mut is_closed = true;
2903
2904	if bytes.starts_with(b"(") {
2905	start += 1;
2906	}
2907	if bytes.ends_with(b")") {
2908	end -= 1;
2909	} else {
2910	working_set.error(ParseError::Unclosed(")".into(), Span::new(end, end)));
2911	is_closed = false;
2912	}
2913
2914	let span = Span::new(start, end);
2915
2916	let source = working_set.get_span_contents(span);
2917
2918	let (output, err) = lex(source, span.start, &[b'\n', b'\r'], &[], true);
2919	if let Some(err) = err {
2920	working_set.error(err)
2921	}
2922
2923	// Creating a Type scope to parse the new block. This will keep track of
2924	// the previous input type found in that block
2925	let output = parse_block(working_set, &output, span, is_closed, true);
2926
2927	let ty = output.output_type();
2928
2929	let block_id = working_set.add_block(Arc::new(output));
2930	tokens.next();
2931
2932	(
2933	Expression::new(working_set, Expr::Subexpression(block_id), head_span, ty),
2934	true,
2935	)
2936	} else if bytes.starts_with(b"[") {
2937	trace!("parsing: table head of full cell path");
2938
2939	let output = parse_table_expression(working_set, head.span, &SyntaxShape::Any);
2940
2941	tokens.next();
2942
2943	(output, true)
2944	} else if bytes.starts_with(b"{") {
2945	trace!("parsing: record head of full cell path");
2946	let output = parse_record(working_set, head.span);
2947
2948	tokens.next();
2949
2950	(output, true)
2951	} else if bytes.starts_with(b"$") {
2952	trace!("parsing: $variable head of full cell path");
2953
2954	let out = parse_variable_expr(working_set, head.span);
2955
2956	tokens.next();
2957
2958	(out, true)
2959	} else if let Some(var_id) = implicit_head {
2960	trace!("parsing: implicit head of full cell path");
2961	(
2962	Expression::new(working_set, Expr::Var(var_id), head.span, Type::Any),
2963	false,
2964	)
2965	} else {
2966	working_set.error(ParseError::Mismatch(
2967	"variable or subexpression".into(),
2968	String::from_utf8_lossy(bytes).to_string(),
2969	span,
2970	));
2971	return garbage(working_set, span);
2972	};
2973
2974	let tail = parse_cell_path(working_set, tokens, expect_dot);
2975	let ty = if !tail.is_empty() {
2976	if nu_experimental::CELL_PATH_TYPES.get() {
2977	head.ty
2978	.follow_cell_path(&tail)
2979	.map(\|ty\| ty.into_owned())
2980	.unwrap_or(Type::Any)
2981	} else {
2982	Type::Any
2983	}
2984	} else {
2985	head.ty.clone()
2986	};
2987
2988	Expression::new(
2989	working_set,
2990	Expr::FullCellPath(Box::new(FullCellPath { head, tail })),
2991	full_cell_span,
2992	ty,
2993	)
2994	} else {
2995	garbage(working_set, span)
2996	}
2997	}
2998
2999	pub fn parse_directory(working_set: &mut StateWorkingSet, span: Span) -> Expression {
3000	let bytes = working_set.get_span_contents(span);
3001	trace!("parsing: directory");
3002
3003	// Check for bare word interpolation
3004	if !bytes.is_empty()
3005	&& bytes[0] != b'\''
3006	&& bytes[0] != b'"'
3007	&& bytes[0] != b'`'
3008	&& bytes.contains(&b'(')
3009	{
3010	return parse_string_interpolation(working_set, span);
3011	}
3012
3013	let quoted = is_quoted(bytes);
3014	let (token, err) = unescape_unquote_string(bytes, span);
3015
3016	if err.is_none() {
3017	trace!("-- found {token}");
3018
3019	Expression::new(
3020	working_set,
3021	Expr::Directory(token, quoted),
3022	span,
3023	Type::String,
3024	)
3025	} else {
3026	working_set.error(ParseError::Expected("directory", span));
3027
3028	garbage(working_set, span)
3029	}
3030	}
3031
3032	pub fn parse_filepath(working_set: &mut StateWorkingSet, span: Span) -> Expression {
3033	let bytes = working_set.get_span_contents(span);
3034	trace!("parsing: filepath");
3035
3036	// Check for bare word interpolation
3037	if !bytes.is_empty()
3038	&& bytes[0] != b'\''
3039	&& bytes[0] != b'"'
3040	&& bytes[0] != b'`'
3041	&& bytes.contains(&b'(')
3042	{
3043	return parse_string_interpolation(working_set, span);
3044	}
3045
3046	let quoted = is_quoted(bytes);
3047	let (token, err) = unescape_unquote_string(bytes, span);
3048
3049	if err.is_none() {
3050	trace!("-- found {token}");
3051
3052	Expression::new(
3053	working_set,
3054	Expr::Filepath(token, quoted),
3055	span,
3056	Type::String,
3057	)
3058	} else {
3059	working_set.error(ParseError::Expected("filepath", span));
3060
3061	garbage(working_set, span)
3062	}
3063	}
3064
3065	/// Parse a datetime type, eg '2022-02-02'
3066	pub fn parse_datetime(working_set: &mut StateWorkingSet, span: Span) -> Expression {
3067	trace!("parsing: datetime");
3068
3069	let bytes = working_set.get_span_contents(span);
3070
3071	if bytes.len() < 6
3072	\|\| !bytes[0].is_ascii_digit()
3073	\|\| !bytes[1].is_ascii_digit()
3074	\|\| !bytes[2].is_ascii_digit()
3075	\|\| !bytes[3].is_ascii_digit()
3076	\|\| bytes[4] != b'-'
3077	{
3078	working_set.error(ParseError::Expected("datetime", span));
3079	return garbage(working_set, span);
3080	}
3081
3082	let token = String::from_utf8_lossy(bytes).to_string();
3083
3084	if let Ok(datetime) = chrono::DateTime::parse_from_rfc3339(&token) {
3085	return Expression::new(working_set, Expr::DateTime(datetime), span, Type::Date);
3086	}
3087
3088	// Just the date
3089	let just_date = token.clone() + "T00:00:00+00:00";
3090	if let Ok(datetime) = chrono::DateTime::parse_from_rfc3339(&just_date) {
3091	return Expression::new(working_set, Expr::DateTime(datetime), span, Type::Date);
3092	}
3093
3094	// Date and time, assume UTC
3095	let datetime = token + "+00:00";
3096	if let Ok(datetime) = chrono::DateTime::parse_from_rfc3339(&datetime) {
3097	return Expression::new(working_set, Expr::DateTime(datetime), span, Type::Date);
3098	}
3099
3100	working_set.error(ParseError::Expected("datetime", span));
3101
3102	garbage(working_set, span)
3103	}
3104
3105	/// Parse a duration type, eg '10day'
3106	pub fn parse_duration(working_set: &mut StateWorkingSet, span: Span) -> Expression {
3107	trace!("parsing: duration");
3108
3109	let bytes = working_set.get_span_contents(span);
3110
3111	match parse_unit_value(bytes, span, DURATION_UNIT_GROUPS, Type::Duration, \|x\| x) {
3112	Some(Ok(expr)) => {
3113	let span_id = working_set.add_span(span);
3114	expr.with_span_id(span_id)
3115	}
3116	Some(Err(mk_err_for)) => {
3117	working_set.error(mk_err_for("duration"));
3118	garbage(working_set, span)
3119	}
3120	None => {
3121	working_set.error(ParseError::Expected("duration with valid units", span));
3122	garbage(working_set, span)
3123	}
3124	}
3125	}
3126
3127	/// Parse a unit type, eg '10kb'
3128	pub fn parse_filesize(working_set: &mut StateWorkingSet, span: Span) -> Expression {
3129	trace!("parsing: filesize");
3130
3131	let bytes = working_set.get_span_contents(span);
3132
3133	// the hex digit `b` might be mistaken for the unit `b`, so check that first
3134	if bytes.starts_with(b"0x") {
3135	working_set.error(ParseError::Expected("filesize with valid units", span));
3136	return garbage(working_set, span);
3137	}
3138
3139	match parse_unit_value(bytes, span, FILESIZE_UNIT_GROUPS, Type::Filesize, \|x\| {
3140	x.to_ascii_uppercase()
3141	}) {
3142	Some(Ok(expr)) => {
3143	let span_id = working_set.add_span(span);
3144	expr.with_span_id(span_id)
3145	}
3146	Some(Err(mk_err_for)) => {
3147	working_set.error(mk_err_for("filesize"));
3148	garbage(working_set, span)
3149	}
3150	None => {
3151	working_set.error(ParseError::Expected("filesize with valid units", span));
3152	garbage(working_set, span)
3153	}
3154	}
3155	}
3156
3157	type ParseUnitResult<'res> = Result<Expression, Box<dyn Fn(&'res str) -> ParseError>>;
3158	type UnitGroup<'unit> = (Unit, &'unit str, Option<(Unit, i64)>);
3159
3160	pub fn parse_unit_value<'res>(
3161	bytes: &[u8],
3162	span: Span,
3163	unit_groups: &[UnitGroup],
3164	ty: Type,
3165	transform: fn(String) -> String,
3166	) -> Option<ParseUnitResult<'res>> {
3167	if bytes.len() < 2
3168	\|\| !(bytes[0].is_ascii_digit()
3169	\|\| bytes[0] == b'.'
3170	\|\| (bytes[0] == b'-' && bytes[1].is_ascii_digit()))
3171	{
3172	return None;
3173	}
3174
3175	// Bail if not UTF-8
3176	let value = transform(str::from_utf8(bytes).ok()?.into());
3177
3178	if let Some((unit, name, convert)) = unit_groups.iter().find(\|x\| value.ends_with(x.1)) {
3179	let lhs_len = value.len() - name.len();
3180	let lhs = strip_underscores(&value.as_bytes()[..lhs_len]);
3181	let lhs_span = Span::new(span.start, span.start + lhs_len);
3182	let unit_span = Span::new(span.start + lhs_len, span.end);
3183	if lhs.ends_with('$') {
3184	// If `parse_unit_value` has higher precedence over `parse_range`,
3185	// a variable with the name of a unit could otherwise not be used as the end of a range.
3186	return None;
3187	}
3188
3189	let (decimal_part, number_part) = modf(match lhs.parse::<f64>() {
3190	Ok(it) => it,
3191	Err(_) => {
3192	let mk_err = move \|name\| {
3193	ParseError::LabeledError(
3194	format!("{name} value must be a number"),
3195	"not a number".into(),
3196	lhs_span,
3197	)
3198	};
3199	return Some(Err(Box::new(mk_err)));
3200	}
3201	});
3202
3203	let mut unit = match convert {
3204	Some(convert_to) => convert_to.0,
3205	None => *unit,
3206	};
3207
3208	let num_float = match convert {
3209	Some(convert_to) => {
3210	(number_part * convert_to.1 as f64) + (decimal_part * convert_to.1 as f64)
3211	}
3212	None => number_part,
3213	};
3214
3215	// Convert all durations to nanoseconds, and filesizes to bytes,
3216	// to minimize loss of precision
3217	let factor = match ty {
3218	Type::Filesize => unit_to_byte_factor(&unit),
3219	Type::Duration => unit_to_ns_factor(&unit),
3220	_ => None,
3221	};
3222
3223	let num = match factor {
3224	Some(factor) => {
3225	let num_base = num_float * factor;
3226	if i64::MIN as f64 <= num_base && num_base <= i64::MAX as f64 {
3227	unit = if ty == Type::Filesize {
3228	Unit::Filesize(FilesizeUnit::B)
3229	} else {
3230	Unit::Nanosecond
3231	};
3232	num_base as i64
3233	} else {
3234	// not safe to convert, because of the overflow
3235	num_float as i64
3236	}
3237	}
3238	None => num_float as i64,
3239	};
3240
3241	trace!("-- found {num} {unit:?}");
3242	let value = ValueWithUnit {
3243	expr: Expression::new_unknown(Expr::Int(num), lhs_span, Type::Number),
3244	unit: Spanned {
3245	item: unit,
3246	span: unit_span,
3247	},
3248	};
3249	let expr = Expression::new_unknown(Expr::ValueWithUnit(Box::new(value)), span, ty);
3250
3251	Some(Ok(expr))
3252	} else {
3253	None
3254	}
3255	}
3256
3257	pub const FILESIZE_UNIT_GROUPS: &[UnitGroup] = &[
3258	(
3259	Unit::Filesize(FilesizeUnit::KB),
3260	"KB",
3261	Some((Unit::Filesize(FilesizeUnit::B), 1000)),
3262	),
3263	(
3264	Unit::Filesize(FilesizeUnit::MB),
3265	"MB",
3266	Some((Unit::Filesize(FilesizeUnit::KB), 1000)),
3267	),
3268	(
3269	Unit::Filesize(FilesizeUnit::GB),
3270	"GB",
3271	Some((Unit::Filesize(FilesizeUnit::MB), 1000)),
3272	),
3273	(
3274	Unit::Filesize(FilesizeUnit::TB),
3275	"TB",
3276	Some((Unit::Filesize(FilesizeUnit::GB), 1000)),
3277	),
3278	(
3279	Unit::Filesize(FilesizeUnit::PB),
3280	"PB",
3281	Some((Unit::Filesize(FilesizeUnit::TB), 1000)),
3282	),
3283	(
3284	Unit::Filesize(FilesizeUnit::EB),
3285	"EB",
3286	Some((Unit::Filesize(FilesizeUnit::PB), 1000)),
3287	),
3288	(
3289	Unit::Filesize(FilesizeUnit::KiB),
3290	"KIB",
3291	Some((Unit::Filesize(FilesizeUnit::B), 1024)),
3292	),
3293	(
3294	Unit::Filesize(FilesizeUnit::MiB),
3295	"MIB",
3296	Some((Unit::Filesize(FilesizeUnit::KiB), 1024)),
3297	),
3298	(
3299	Unit::Filesize(FilesizeUnit::GiB),
3300	"GIB",
3301	Some((Unit::Filesize(FilesizeUnit::MiB), 1024)),
3302	),
3303	(
3304	Unit::Filesize(FilesizeUnit::TiB),
3305	"TIB",
3306	Some((Unit::Filesize(FilesizeUnit::GiB), 1024)),
3307	),
3308	(
3309	Unit::Filesize(FilesizeUnit::PiB),
3310	"PIB",
3311	Some((Unit::Filesize(FilesizeUnit::TiB), 1024)),
3312	),
3313	(
3314	Unit::Filesize(FilesizeUnit::EiB),
3315	"EIB",
3316	Some((Unit::Filesize(FilesizeUnit::PiB), 1024)),
3317	),
3318	(Unit::Filesize(FilesizeUnit::B), "B", None),
3319	];
3320
3321	pub const DURATION_UNIT_GROUPS: &[UnitGroup] = &[
3322	(Unit::Nanosecond, "ns", None),
3323	// todo start adding aliases for duration units here
3324	(Unit::Microsecond, "us", Some((Unit::Nanosecond, 1000))),
3325	(
3326	// µ Micro Sign
3327	Unit::Microsecond,
3328	"\u{00B5}s",
3329	Some((Unit::Nanosecond, 1000)),
3330	),
3331	(
3332	// μ Greek small letter Mu
3333	Unit::Microsecond,
3334	"\u{03BC}s",
3335	Some((Unit::Nanosecond, 1000)),
3336	),
3337	(Unit::Millisecond, "ms", Some((Unit::Microsecond, 1000))),
3338	(Unit::Second, "sec", Some((Unit::Millisecond, 1000))),
3339	(Unit::Minute, "min", Some((Unit::Second, 60))),
3340	(Unit::Hour, "hr", Some((Unit::Minute, 60))),
3341	(Unit::Day, "day", Some((Unit::Minute, 1440))),
3342	(Unit::Week, "wk", Some((Unit::Day, 7))),
3343	];
3344
3345	fn unit_to_ns_factor(unit: &Unit) -> Option<f64> {
3346	match unit {
3347	Unit::Nanosecond => Some(1.0),
3348	Unit::Microsecond => Some(1_000.0),
3349	Unit::Millisecond => Some(1_000_000.0),
3350	Unit::Second => Some(1_000_000_000.0),
3351	Unit::Minute => Some(60.0 * 1_000_000_000.0),
3352	Unit::Hour => Some(60.0 * 60.0 * 1_000_000_000.0),
3353	Unit::Day => Some(24.0 * 60.0 * 60.0 * 1_000_000_000.0),
3354	Unit::Week => Some(7.0 * 24.0 * 60.0 * 60.0 * 1_000_000_000.0),
3355	_ => None,
3356	}
3357	}
3358
3359	fn unit_to_byte_factor(unit: &Unit) -> Option<f64> {
3360	match unit {
3361	Unit::Filesize(FilesizeUnit::B) => Some(1.0),
3362	Unit::Filesize(FilesizeUnit::KB) => Some(1_000.0),
3363	Unit::Filesize(FilesizeUnit::MB) => Some(1_000_000.0),
3364	Unit::Filesize(FilesizeUnit::GB) => Some(1_000_000_000.0),
3365	Unit::Filesize(FilesizeUnit::TB) => Some(1_000_000_000_000.0),
3366	Unit::Filesize(FilesizeUnit::PB) => Some(1_000_000_000_000_000.0),
3367	Unit::Filesize(FilesizeUnit::EB) => Some(1_000_000_000_000_000_000.0),
3368	Unit::Filesize(FilesizeUnit::KiB) => Some(1024.0),
3369	Unit::Filesize(FilesizeUnit::MiB) => Some(1024.0 * 1024.0),
3370	Unit::Filesize(FilesizeUnit::GiB) => Some(1024.0 * 1024.0 * 1024.0),
3371	Unit::Filesize(FilesizeUnit::TiB) => Some(1024.0 * 1024.0 * 1024.0 * 1024.0),
3372	Unit::Filesize(FilesizeUnit::PiB) => Some(1024.0 * 1024.0 * 1024.0 * 1024.0 * 1024.0),
3373	Unit::Filesize(FilesizeUnit::EiB) => {
3374	Some(1024.0 * 1024.0 * 1024.0 * 1024.0 * 1024.0 * 1024.0)
3375	}
3376	_ => None,
3377	}
3378	}
3379
3380	// Borrowed from libm at https://github.com/rust-lang/libm/blob/master/src/math/modf.rs
3381	fn modf(x: f64) -> (f64, f64) {
3382	let rv2: f64;
3383	let mut u = x.to_bits();
3384	let e = (((u >> 52) & 0x7ff) as i32) - 0x3ff;
3385
3386	/* no fractional part */
3387	if e >= 52 {
3388	rv2 = x;
3389	if e == 0x400 && (u << 12) != 0 {
3390	/* nan */
3391	return (x, rv2);
3392	}
3393	u &= 1 << 63;
3394	return (f64::from_bits(u), rv2);
3395	}
3396
3397	/* no integral part*/
3398	if e < 0 {
3399	u &= 1 << 63;
3400	rv2 = f64::from_bits(u);
3401	return (x, rv2);
3402	}
3403
3404	let mask = ((!0) >> 12) >> e;
3405	if (u & mask) == 0 {
3406	rv2 = x;
3407	u &= 1 << 63;
3408	return (f64::from_bits(u), rv2);
3409	}
3410	u &= !mask;
3411	rv2 = f64::from_bits(u);
3412	(x - rv2, rv2)
3413	}
3414
3415	pub fn parse_glob_pattern(working_set: &mut StateWorkingSet, span: Span) -> Expression {
3416	let bytes = working_set.get_span_contents(span);
3417	let quoted = is_quoted(bytes);
3418	trace!("parsing: glob pattern");
3419
3420	// Check for bare word interpolation
3421	if !bytes.is_empty()
3422	&& bytes[0] != b'\''
3423	&& bytes[0] != b'"'
3424	&& bytes[0] != b'`'
3425	&& bytes.contains(&b'(')
3426	{
3427	let interpolation_expr = parse_string_interpolation(working_set, span);
3428
3429	// Convert StringInterpolation to GlobInterpolation
3430	if let Expr::StringInterpolation(exprs) = interpolation_expr.expr {
3431	return Expression::new(
3432	working_set,
3433	Expr::GlobInterpolation(exprs, quoted),
3434	span,
3435	Type::Glob,
3436	);
3437	}
3438
3439	return interpolation_expr;
3440	}
3441
3442	let (token, err) = unescape_unquote_string(bytes, span);
3443
3444	if err.is_none() {
3445	trace!("-- found {token}");
3446
3447	Expression::new(
3448	working_set,
3449	Expr::GlobPattern(token, quoted),
3450	span,
3451	Type::Glob,
3452	)
3453	} else {
3454	working_set.error(ParseError::Expected("glob pattern string", span));
3455
3456	garbage(working_set, span)
3457	}
3458	}
3459
3460	pub fn unescape_string(bytes: &[u8], span: Span) -> (Vec<u8>, Option<ParseError>) {
3461	let mut output = Vec::new();
3462	let mut error = None;
3463
3464	let mut idx = 0;
3465
3466	if !bytes.contains(&b'\\') {
3467	return (bytes.to_vec(), None);
3468	}
3469
3470	'us_loop: while idx < bytes.len() {
3471	if bytes[idx] == b'\\' {
3472	// We're in an escape
3473	idx += 1;
3474
3475	match bytes.get(idx) {
3476	Some(b'"') => {
3477	output.push(b'"');
3478	idx += 1;
3479	}
3480	Some(b'\'') => {
3481	output.push(b'\'');
3482	idx += 1;
3483	}
3484	Some(b'\\') => {
3485	output.push(b'\\');
3486	idx += 1;
3487	}
3488	Some(b'/') => {
3489	output.push(b'/');
3490	idx += 1;
3491	}
3492	Some(b'(') => {
3493	output.push(b'(');
3494	idx += 1;
3495	}
3496	Some(b')') => {
3497	output.push(b')');
3498	idx += 1;
3499	}
3500	Some(b'{') => {
3501	output.push(b'{');
3502	idx += 1;
3503	}
3504	Some(b'}') => {
3505	output.push(b'}');
3506	idx += 1;
3507	}
3508	Some(b'$') => {
3509	output.push(b'$');
3510	idx += 1;
3511	}
3512	Some(b'^') => {
3513	output.push(b'^');
3514	idx += 1;
3515	}
3516	Some(b'#') => {
3517	output.push(b'#');
3518	idx += 1;
3519	}
3520	Some(b'\|') => {
3521	output.push(b'\|');
3522	idx += 1;
3523	}
3524	Some(b'~') => {
3525	output.push(b'~');
3526	idx += 1;
3527	}
3528	Some(b'a') => {
3529	output.push(0x7);
3530	idx += 1;
3531	}
3532	Some(b'b') => {
3533	output.push(0x8);
3534	idx += 1;
3535	}
3536	Some(b'e') => {
3537	output.push(0x1b);
3538	idx += 1;
3539	}
3540	Some(b'f') => {
3541	output.push(0xc);
3542	idx += 1;
3543	}
3544	Some(b'n') => {
3545	output.push(b'\n');
3546	idx += 1;
3547	}
3548	Some(b'r') => {
3549	output.push(b'\r');
3550	idx += 1;
3551	}
3552	Some(b't') => {
3553	output.push(b'\t');
3554	idx += 1;
3555	}
3556	Some(b'u') => {
3557	let mut digits = String::with_capacity(10);
3558	let mut cur_idx = idx + 1; // index of first beyond current end of token
3559
3560	if let Some(b'{') = bytes.get(idx + 1) {
3561	cur_idx = idx + 2;
3562	loop {
3563	match bytes.get(cur_idx) {
3564	Some(b'}') => {
3565	cur_idx += 1;
3566	break;
3567	}
3568	Some(c) => {
3569	digits.push(*c as char);
3570	cur_idx += 1;
3571	}
3572	_ => {
3573	error = error.or(Some(ParseError::InvalidLiteral(
3574	"missing '}' for unicode escape '\\u{X...}'".into(),
3575	"string".into(),
3576	Span::new(span.start + idx, span.end),
3577	)));
3578	break 'us_loop;
3579	}
3580	}
3581	}
3582	}
3583
3584	if (1..=6).contains(&digits.len()) {
3585	let int = u32::from_str_radix(&digits, 16);
3586
3587	if let Ok(int) = int
3588	&& int <= 0x10ffff
3589	{
3590	let result = char::from_u32(int);
3591
3592	if let Some(result) = result {
3593	let mut buffer = [0; 4];
3594	let result = result.encode_utf8(&mut buffer);
3595
3596	for elem in result.bytes() {
3597	output.push(elem);
3598	}
3599
3600	idx = cur_idx;
3601	continue 'us_loop;
3602	}
3603	}
3604	}
3605	// fall through -- escape not accepted above, must be error.
3606	error = error.or(Some(ParseError::InvalidLiteral(
3607	"invalid unicode escape '\\u{X...}', must be 1-6 hex digits, max value 10FFFF".into(),
3608	"string".into(),
3609	Span::new(span.start + idx, span.end),
3610	)));
3611	break 'us_loop;
3612	}
3613
3614	_ => {
3615	error = error.or(Some(ParseError::InvalidLiteral(
3616	"unrecognized escape after '\\'".into(),
3617	"string".into(),
3618	Span::new(span.start + idx, span.end),
3619	)));
3620	break 'us_loop;
3621	}
3622	}
3623	} else {
3624	output.push(bytes[idx]);
3625	idx += 1;
3626	}
3627	}
3628
3629	(output, error)
3630	}
3631
3632	pub fn unescape_unquote_string(bytes: &[u8], span: Span) -> (String, Option<ParseError>) {
3633	if bytes.starts_with(b"\"") {
3634	// Needs unescaping
3635	let bytes = trim_quotes(bytes);
3636
3637	let (bytes, err) = unescape_string(bytes, span);
3638
3639	if let Ok(token) = String::from_utf8(bytes) {
3640	(token, err)
3641	} else {
3642	(String::new(), Some(ParseError::Expected("string", span)))
3643	}
3644	} else {
3645	let bytes = trim_quotes(bytes);
3646
3647	if let Ok(token) = String::from_utf8(bytes.into()) {
3648	(token, None)
3649	} else {
3650	(String::new(), Some(ParseError::Expected("string", span)))
3651	}
3652	}
3653	}
3654
3655	fn check_string_no_trailing_tokens(
3656	bytes: &[u8],
3657	span: Span,
3658	opening_quote_pos: usize,
3659	quote: u8,
3660	) -> Result<(), ParseError> {
3661	let pos = bytes
3662	.iter()
3663	.rposition(\|ch\| *ch == quote)
3664	.expect("string begins with quote");
3665	if pos == bytes.len() - 1 {
3666	Ok(())
3667	} else if pos == opening_quote_pos {
3668	// this may look like an error, but it's not:
3669	// some code, like completions, requires allowing
3670	// unterminated strings at this stage.
3671	Ok(())
3672	} else {
3673	let span = Span::new(span.start + pos + 1, span.end);
3674	Err(ParseError::ExtraTokensAfterClosingDelimiter(span))
3675	}
3676	}
3677
3678	pub fn parse_string(working_set: &mut StateWorkingSet, span: Span) -> Expression {
3679	trace!("parsing: string");
3680
3681	let bytes = working_set.get_span_contents(span);
3682
3683	if bytes.is_empty() {
3684	working_set.error(ParseError::Expected("String", span));
3685	return Expression::garbage(working_set, span);
3686	}
3687
3688	// Check for bare word interpolation
3689	if bytes[0] != b'\'' && bytes[0] != b'"' && bytes[0] != b'`' && bytes.contains(&b'(') {
3690	return parse_string_interpolation(working_set, span);
3691	}
3692
3693	// Check for unbalanced quotes:
3694	for quote in [b'\"', b'\''] {
3695	if bytes[0] == quote
3696	&& let Err(err) = check_string_no_trailing_tokens(bytes, span, 0, quote)
3697	{
3698	working_set.error(err);
3699	return garbage(working_set, span);
3700	}
3701	}
3702
3703	let (s, err) = unescape_unquote_string(bytes, span);
3704	if let Some(err) = err {
3705	working_set.error(err);
3706	}
3707
3708	Expression::new(working_set, Expr::String(s), span, Type::String)
3709	}
3710
3711	fn is_quoted(bytes: &[u8]) -> bool {
3712	(bytes.starts_with(b"\"") && bytes.ends_with(b"\"") && bytes.len() > 1)
3713	\|\| (bytes.starts_with(b"\'") && bytes.ends_with(b"\'") && bytes.len() > 1)
3714	}
3715
3716	pub fn parse_string_strict(working_set: &mut StateWorkingSet, span: Span) -> Expression {
3717	trace!("parsing: string, with required delimiters");
3718
3719	let bytes = working_set.get_span_contents(span);
3720
3721	// Check for unbalanced quotes:
3722	{
3723	let bytes = if bytes.starts_with(b"$") {
3724	&bytes[1..]
3725	} else {
3726	bytes
3727	};
3728	if bytes.starts_with(b"\"") && (bytes.len() == 1 \|\| !bytes.ends_with(b"\"")) {
3729	working_set.error(ParseError::Unclosed("\"".into(), span));
3730	return garbage(working_set, span);
3731	}
3732	if bytes.starts_with(b"\'") && (bytes.len() == 1 \|\| !bytes.ends_with(b"\'")) {
3733	working_set.error(ParseError::Unclosed("\'".into(), span));
3734	return garbage(working_set, span);
3735	}
3736	if bytes.starts_with(b"r#") && (bytes.len() == 1 \|\| !bytes.ends_with(b"#")) {
3737	working_set.error(ParseError::Unclosed("r#".into(), span));
3738	return garbage(working_set, span);
3739	}
3740	}
3741
3742	let (bytes, quoted) = if (bytes.starts_with(b"\"") && bytes.ends_with(b"\"") && bytes.len() > 1)
3743	\|\| (bytes.starts_with(b"\'") && bytes.ends_with(b"\'") && bytes.len() > 1)
3744	{
3745	(&bytes[1..(bytes.len() - 1)], true)
3746	} else if (bytes.starts_with(b"$\"") && bytes.ends_with(b"\"") && bytes.len() > 2)
3747	\|\| (bytes.starts_with(b"$\'") && bytes.ends_with(b"\'") && bytes.len() > 2)
3748	{
3749	(&bytes[2..(bytes.len() - 1)], true)
3750	} else {
3751	(bytes, false)
3752	};
3753
3754	if let Ok(token) = String::from_utf8(bytes.into()) {
3755	trace!("-- found {token}");
3756
3757	if quoted {
3758	Expression::new(working_set, Expr::String(token), span, Type::String)
3759	} else if token.contains(' ') {
3760	working_set.error(ParseError::Expected("string", span));
3761
3762	garbage(working_set, span)
3763	} else {
3764	Expression::new(working_set, Expr::String(token), span, Type::String)
3765	}
3766	} else {
3767	working_set.error(ParseError::Expected("string", span));
3768	garbage(working_set, span)
3769	}
3770	}
3771
3772	pub fn parse_import_pattern<'a>(
3773	working_set: &mut StateWorkingSet,
3774	mut arg_iter: impl Iterator<Item = &'a Expression>,
3775	spans: &[Span],
3776	) -> Expression {
3777	let Some(head_expr) = arg_iter.next() else {
3778	working_set.error(ParseError::WrongImportPattern(
3779	"needs at least one component of import pattern".to_string(),
3780	Span::concat(spans),
3781	));
3782	return garbage(working_set, Span::concat(spans));
3783	};
3784
3785	let (maybe_module_id, head_name) = match eval_constant(working_set, head_expr) {
3786	Ok(Value::Nothing { .. }) => {
3787	return Expression::new(
3788	working_set,
3789	Expr::Nothing,
3790	Span::concat(spans),
3791	Type::Nothing,
3792	);
3793	}
3794	Ok(val) => match val.coerce_into_string() {
3795	Ok(s) => (working_set.find_module(s.as_bytes()), s.into_bytes()),
3796	Err(err) => {
3797	working_set.error(err.wrap(working_set, Span::concat(spans)));
3798	return garbage(working_set, Span::concat(spans));
3799	}
3800	},
3801	Err(err) => {
3802	working_set.error(err.wrap(working_set, Span::concat(spans)));
3803	return garbage(working_set, Span::concat(spans));
3804	}
3805	};
3806
3807	let mut import_pattern = ImportPattern {
3808	head: ImportPatternHead {
3809	name: head_name,
3810	id: maybe_module_id,
3811	span: head_expr.span,
3812	},
3813	members: vec![],
3814	hidden: HashSet::new(),
3815	constants: vec![],
3816	};
3817
3818	let mut leaf_member_expr: Option<(&str, Span)> = None;
3819
3820	// TODO: box pattern syntax is experimental @rust v1.89.0
3821	let handle_list_items =
3822	\|items: &Vec<ListItem>,
3823	span,
3824	working_set: &mut StateWorkingSet<'_>,
3825	import_pattern: &mut ImportPattern,
3826	leaf_member_expr: &mut Option<(&str, Span)>\| {
3827	let mut output = vec![];
3828
3829	for item in items.iter() {
3830	match item {
3831	ListItem::Item(expr) => {
3832	if let Some(name) = expr.as_string() {
3833	output.push((name.as_bytes().to_vec(), expr.span));
3834	}
3835	}
3836	ListItem::Spread(_, spread) => {
3837	working_set.error(ParseError::WrongImportPattern(
3838	"cannot spread in an import pattern".into(),
3839	spread.span,
3840	))
3841	}
3842	}
3843	}
3844
3845	import_pattern
3846	.members
3847	.push(ImportPatternMember::List { names: output });
3848
3849	*leaf_member_expr = Some(("list", span));
3850	};
3851
3852	for tail_expr in arg_iter {
3853	if let Some((what, prev_span)) = leaf_member_expr {
3854	working_set.error(ParseError::WrongImportPattern(
3855	format!("{what} member can be only at the end of an import pattern"),
3856	prev_span,
3857	));
3858	return Expression::new(
3859	working_set,
3860	Expr::ImportPattern(Box::new(import_pattern)),
3861	prev_span,
3862	Type::List(Box::new(Type::String)),
3863	);
3864	}
3865
3866	match &tail_expr.expr {
3867	Expr::String(name) => {
3868	let span = tail_expr.span;
3869	if name == "*" {
3870	import_pattern
3871	.members
3872	.push(ImportPatternMember::Glob { span });
3873
3874	leaf_member_expr = Some(("glob", span));
3875	} else {
3876	import_pattern.members.push(ImportPatternMember::Name {
3877	name: name.as_bytes().to_vec(),
3878	span,
3879	});
3880	}
3881	}
3882	Expr::FullCellPath(fcp) => {
3883	if let Expr::List(items) = &fcp.head.expr {
3884	handle_list_items(
3885	items,
3886	fcp.head.span,
3887	working_set,
3888	&mut import_pattern,
3889	&mut leaf_member_expr,
3890	);
3891	}
3892	}
3893	Expr::List(items) => {
3894	handle_list_items(
3895	items,
3896	tail_expr.span,
3897	working_set,
3898	&mut import_pattern,
3899	&mut leaf_member_expr,
3900	);
3901	}
3902	_ => {
3903	working_set.error(ParseError::WrongImportPattern(
3904	"Wrong type of import pattern, only String and List<String> are allowed."
3905	.into(),
3906	tail_expr.span,
3907	));
3908	}
3909	};
3910	}
3911
3912	Expression::new(
3913	working_set,
3914	Expr::ImportPattern(Box::new(import_pattern)),
3915	Span::concat(&spans[1..]),
3916	Type::List(Box::new(Type::String)),
3917	)
3918	}
3919
3920	/// Parse `spans[spans_idx..]` into a variable, with optional type annotation.
3921	/// If the name of the variable ends with a colon (no space in-between allowed), then a type annotation
3922	/// can appear after the variable, in which case the colon is stripped from the name of the variable.
3923	/// `spans_idx` is updated to point to the last span that has been parsed.
3924	pub fn parse_var_with_opt_type(
3925	working_set: &mut StateWorkingSet,
3926	spans: &[Span],
3927	spans_idx: &mut usize,
3928	mutable: bool,
3929	) -> (Expression, Option<Type>) {
3930	let name_span = spans[*spans_idx];
3931	let bytes = working_set.get_span_contents(name_span).to_vec();
3932
3933	if bytes.contains(&b' ')
3934	\|\| bytes.contains(&b'"')
3935	\|\| bytes.contains(&b'\'')
3936	\|\| bytes.contains(&b'`')
3937	{
3938	working_set.error(ParseError::VariableNotValid(spans[*spans_idx]));
3939	return (garbage(working_set, spans[*spans_idx]), None);
3940	}
3941
3942	if bytes.ends_with(b":") {
3943	let name_span = Span::new(name_span.start, name_span.end - 1);
3944	let var_name = bytes[0..(bytes.len() - 1)].to_vec();
3945
3946	// We end with colon, so the next span should be the type
3947	if *spans_idx + 1 < spans.len() {
3948	*spans_idx += 1;
3949	// signature like record<a: int b: int> is broken into multiple spans due to
3950	// whitespaces. Collect the rest into one span and work on it
3951	let full_span = Span::concat(&spans[*spans_idx..]);
3952	let type_bytes = working_set.get_span_contents(full_span).to_vec();
3953
3954	let (tokens, parse_error) =
3955	lex_signature(&type_bytes, full_span.start, &[], &[b','], true);
3956
3957	if let Some(parse_error) = parse_error {
3958	working_set.error(parse_error);
3959	}
3960
3961	let ty = parse_type(working_set, &type_bytes, tokens[0].span);
3962	*spans_idx = spans.len() - 1;
3963
3964	if !is_variable(&var_name) {
3965	working_set.error(ParseError::Expected(
3966	"valid variable name",
3967	spans[*spans_idx - 1],
3968	));
3969	return (garbage(working_set, spans[*spans_idx - 1]), None);
3970	}
3971
3972	ensure_not_reserved_variable_name(working_set, &var_name, name_span);
3973
3974	let id = working_set.add_variable(var_name, spans[*spans_idx - 1], ty.clone(), mutable);
3975
3976	(
3977	Expression::new(working_set, Expr::VarDecl(id), name_span, ty.clone()),
3978	Some(ty),
3979	)
3980	} else {
3981	if !is_variable(&var_name) {
3982	working_set.error(ParseError::Expected(
3983	"valid variable name",
3984	spans[*spans_idx],
3985	));
3986	return (garbage(working_set, spans[*spans_idx]), None);
3987	}
3988
3989	ensure_not_reserved_variable_name(working_set, &var_name, name_span);
3990
3991	let id = working_set.add_variable(var_name, spans[*spans_idx], Type::Any, mutable);
3992
3993	working_set.error(ParseError::MissingType(spans[*spans_idx]));
3994	(
3995	Expression::new(working_set, Expr::VarDecl(id), spans[*spans_idx], Type::Any),
3996	None,
3997	)
3998	}
3999	} else {
4000	let var_name = bytes;
4001
4002	if !is_variable(&var_name) {
4003	working_set.error(ParseError::Expected(
4004	"valid variable name",
4005	spans[*spans_idx],
4006	));
4007	return (garbage(working_set, spans[*spans_idx]), None);
4008	}
4009
4010	ensure_not_reserved_variable_name(working_set, &var_name, name_span);
4011
4012	let id = working_set.add_variable(
4013	var_name,
4014	Span::concat(&spans[spans_idx..spans_idx + 1]),
4015	Type::Any,
4016	mutable,
4017	);
4018
4019	(
4020	Expression::new(working_set, Expr::VarDecl(id), spans[*spans_idx], Type::Any),
4021	None,
4022	)
4023	}
4024	}
4025
4026	const RESERVED_VARIABLE_NAMES: [&[u8]; 3] = [b"in", b"nu", b"env"];
4027
4028	pub(crate) fn ensure_not_reserved_variable_name(
4029	working_set: &mut StateWorkingSet,
4030	name: &[u8],
4031	span: Span,
4032	) {
4033	let var_name = name.strip_prefix(b"$").unwrap_or(name);
4034
4035	if RESERVED_VARIABLE_NAMES.contains(&var_name) {
4036	working_set.error(ParseError::NameIsBuiltinVar(
4037	String::from_utf8_lossy(var_name).to_string(),
4038	span,
4039	))
4040	}
4041	}
4042
4043	pub fn expand_to_cell_path(
4044	working_set: &mut StateWorkingSet,
4045	expression: &mut Expression,
4046	var_id: VarId,
4047	) {
4048	trace!("parsing: expanding to cell path");
4049	if let Expression {
4050	expr: Expr::String(_),
4051	span,
4052	..
4053	} = expression
4054	{
4055	// Re-parse the string as if it were a cell-path
4056	let new_expression = parse_full_cell_path(working_set, Some(var_id), *span);
4057
4058	*expression = new_expression;
4059	}
4060
4061	if let Expression {
4062	expr: Expr::UnaryNot(inner),
4063	..
4064	} = expression
4065	{
4066	expand_to_cell_path(working_set, inner, var_id);
4067	}
4068	}
4069
4070	pub fn parse_input_output_types(
4071	working_set: &mut StateWorkingSet,
4072	spans: &[Span],
4073	) -> Vec<(Type, Type)> {
4074	let mut full_span = Span::concat(spans);
4075
4076	let mut bytes = working_set.get_span_contents(full_span);
4077
4078	if bytes.starts_with(b"[") {
4079	bytes = &bytes[1..];
4080	full_span.start += 1;
4081	}
4082
4083	if bytes.ends_with(b"]") {
4084	bytes = &bytes[..(bytes.len() - 1)];
4085	full_span.end -= 1;
4086	}
4087
4088	let (tokens, parse_error) =
4089	lex_signature(bytes, full_span.start, &[b'\n', b'\r', b','], &[], true);
4090
4091	if let Some(parse_error) = parse_error {
4092	working_set.error(parse_error);
4093	}
4094
4095	let mut output = vec![];
4096
4097	let mut idx = 0;
4098	while idx < tokens.len() {
4099	let type_bytes = working_set.get_span_contents(tokens[idx].span).to_vec();
4100	let input_type = parse_type(working_set, &type_bytes, tokens[idx].span);
4101
4102	idx += 1;
4103	if idx >= tokens.len() {
4104	working_set.error(ParseError::Expected(
4105	"arrow (->)",
4106	Span::new(tokens[idx - 1].span.end, tokens[idx - 1].span.end),
4107	));
4108	break;
4109	}
4110
4111	let arrow = working_set.get_span_contents(tokens[idx].span);
4112	if arrow != b"->" {
4113	working_set.error(ParseError::Expected("arrow (->)", tokens[idx].span));
4114	}
4115
4116	idx += 1;
4117	if idx >= tokens.len() {
4118	working_set.error(ParseError::MissingType(Span::new(
4119	tokens[idx - 1].span.end,
4120	tokens[idx - 1].span.end,
4121	)));
4122	break;
4123	}
4124
4125	let type_bytes = working_set.get_span_contents(tokens[idx].span).to_vec();
4126	let output_type = parse_type(working_set, &type_bytes, tokens[idx].span);
4127
4128	output.push((input_type, output_type));
4129
4130	idx += 1;
4131	}
4132
4133	output
4134	}
4135
4136	pub fn parse_full_signature(
4137	working_set: &mut StateWorkingSet,
4138	spans: &[Span],
4139	is_external: bool,
4140	) -> Expression {
4141	match spans.len() {
4142	// This case should never happen. It corresponds to declarations like `def foo {}`,
4143	// which should throw a 'Missing required positional argument.' before getting to this point
4144	0 => {
4145	working_set.error(ParseError::InternalError(
4146	"failed to catch missing positional arguments".to_string(),
4147	Span::concat(spans),
4148	));
4149	garbage(working_set, Span::concat(spans))
4150	}
4151
4152	// e.g. `[ b"[foo: string]" ]`
4153	1 => parse_signature(working_set, spans[0], is_external),
4154
4155	// This case is needed to distinguish between e.g.
4156	// `[ b"[]", b"{ true }" ]` vs `[ b"[]:", b"int" ]`
4157	2 if working_set.get_span_contents(spans[1]).starts_with(b"{") => {
4158	parse_signature(working_set, spans[0], is_external)
4159	}
4160
4161	// This should handle every other case, e.g.
4162	// `[ b"[]:", b"int" ]`
4163	// `[ b"[]", b":", b"int" ]`
4164	// `[ b"[]", b":", b"int", b"->", b"bool" ]`
4165	_ => {
4166	let (mut arg_signature, input_output_types_pos) =
4167	if working_set.get_span_contents(spans[0]).ends_with(b":") {
4168	(
4169	parse_signature(
4170	working_set,
4171	Span::new(spans[0].start, spans[0].end.saturating_sub(1)),
4172	is_external,
4173	),
4174	1,
4175	)
4176	} else if working_set.get_span_contents(spans[1]) == b":" {
4177	(parse_signature(working_set, spans[0], is_external), 2)
4178	} else {
4179	// This should be an error case, but we call parse_signature anyway
4180	// so it can handle the various possible errors
4181	// e.g. `[ b"[]", b"int" ]` or `[
4182	working_set.error(ParseError::Expected(
4183	"colon (:) before type signature",
4184	Span::concat(&spans[1..]),
4185	));
4186	// (garbage(working_set, Span::concat(spans)), 1)
4187
4188	(parse_signature(working_set, spans[0], is_external), 1)
4189	};
4190
4191	let input_output_types =
4192	parse_input_output_types(working_set, &spans[input_output_types_pos..]);
4193
4194	if let Expression {
4195	expr: Expr::Signature(sig),
4196	span: expr_span,
4197	..
4198	} = &mut arg_signature
4199	{
4200	sig.input_output_types = input_output_types;
4201	expr_span.end = Span::concat(&spans[input_output_types_pos..]).end;
4202	}
4203	arg_signature
4204	}
4205	}
4206	}
4207
4208	pub fn parse_row_condition(working_set: &mut StateWorkingSet, spans: &[Span]) -> Expression {
4209	let pos = spans.first().map(\|s\| s.start).unwrap_or(0);
4210	// New scope in case where there's already a variable named `$it`
4211	working_set.enter_scope();
4212	let var_id = working_set.add_variable(b"$it".to_vec(), Span::new(pos, pos), Type::Any, false);
4213	let expression = parse_math_expression(working_set, spans, Some(var_id));
4214	let span = Span::concat(spans);
4215
4216	let block_id = match expression.expr {
4217	Expr::Block(block_id) => block_id,
4218	Expr::Closure(block_id) => block_id,
4219	Expr::FullCellPath(ref box_fcp) if box_fcp.head.as_var().is_some_and(\|id\| id != var_id) => {
4220	let mut expression = expression;
4221	expression.ty = Type::Any;
4222	working_set.exit_scope();
4223	return expression;
4224	}
4225	Expr::Var(arg_var_id) if arg_var_id != var_id => {
4226	let mut expression = expression;
4227	expression.ty = Type::Any;
4228	working_set.exit_scope();
4229	return expression;
4230	}
4231	_ => {
4232	// We have an expression, check that it's compatible with bool
4233	if !type_compatible(&Type::Bool, &expression.ty) {
4234	working_set.error(ParseError::TypeMismatch(
4235	Type::Bool,
4236	expression.ty.clone(),
4237	expression.span,
4238	));
4239	working_set.exit_scope();
4240	return Expression::garbage(working_set, expression.span);
4241	}
4242
4243	// Convert this expression into a block.
4244	let mut block = Block::new();
4245	let mut pipeline = Pipeline::new();
4246	pipeline.elements.push(PipelineElement {
4247	pipe: None,
4248	expr: expression,
4249	redirection: None,
4250	});
4251
4252	block.pipelines.push(pipeline);
4253
4254	block.signature.required_positional.push(PositionalArg {
4255	name: "$it".into(),
4256	desc: "row condition".into(),
4257	shape: SyntaxShape::Any,
4258	var_id: Some(var_id),
4259	default_value: None,
4260	completion: None,
4261	});
4262
4263	compile_block(working_set, &mut block);
4264
4265	working_set.add_block(Arc::new(block))
4266	}
4267	};
4268	working_set.exit_scope();
4269
4270	Expression::new(working_set, Expr::RowCondition(block_id), span, Type::Bool)
4271	}
4272
4273	pub fn parse_signature(
4274	working_set: &mut StateWorkingSet,
4275	span: Span,
4276	is_external: bool,
4277	) -> Expression {
4278	let bytes = working_set.get_span_contents(span);
4279
4280	let mut start = span.start;
4281	let mut end = span.end;
4282
4283	let mut has_paren = false;
4284
4285	if bytes.starts_with(b"[") {
4286	start += 1;
4287	} else if bytes.starts_with(b"(") {
4288	has_paren = true;
4289	start += 1;
4290	} else {
4291	working_set.error(ParseError::Expected("[ or (", Span::new(start, start + 1)));
4292	return garbage(working_set, span);
4293	}
4294
4295	if (has_paren && bytes.ends_with(b")")) \|\| (!has_paren && bytes.ends_with(b"]")) {
4296	end -= 1;
4297	} else {
4298	working_set.error(ParseError::Unclosed("] or )".into(), Span::new(end, end)));
4299	}
4300
4301	let sig = parse_signature_helper(working_set, Span::new(start, end), is_external);
4302
4303	Expression::new(working_set, Expr::Signature(sig), span, Type::Any)
4304	}
4305
4306	pub fn parse_signature_helper(
4307	working_set: &mut StateWorkingSet,
4308	span: Span,
4309	is_external: bool,
4310	) -> Box<Signature> {
4311	enum ParseMode {
4312	Arg,
4313	AfterCommaArg,
4314	Type,
4315	AfterType,
4316	DefaultValue,
4317	}
4318
4319	#[derive(Debug)]
4320	enum Arg {
4321	Positional {
4322	arg: PositionalArg,
4323	required: bool,
4324	type_annotated: bool,
4325	},
4326	RestPositional(PositionalArg),
4327	Flag {
4328	flag: Flag,
4329	type_annotated: bool,
4330	},
4331	}
4332
4333	let source = working_set.get_span_contents(span);
4334
4335	let (output, err) = lex_signature(
4336	source,
4337	span.start,
4338	&[b'\n', b'\r'],
4339	&[b':', b'=', b','],
4340	false,
4341	);
4342	if let Some(err) = err {
4343	working_set.error(err);
4344	}
4345
4346	let mut args: Vec<Arg> = vec![];
4347	let mut parse_mode = ParseMode::Arg;
4348	// Track variables whose name→VarId mappings have not yet been inserted
4349	// into the overlay scope
4350	//
4351	// We defer all insertions until the entire signature is parsed so that
4352	// default value expressions always resolve to outer scope variables,
4353	// not to sibling parameters
4354	//
4355	// See #15306
4356	let mut pending_scope_inserts: Vec<(Vec<u8>, VarId)> = vec![];
4357
4358	for (index, token) in output.iter().enumerate() {
4359	let last_token = index == output.len() - 1;
4360
4361	match token {
4362	Token {
4363	contents: crate::TokenContents::Item \| crate::TokenContents::AssignmentOperator,
4364	span,
4365	} => {
4366	let span = *span;
4367	let contents = working_set.get_span_contents(span).to_vec();
4368
4369	// The : symbol separates types
4370	if contents == b":" {
4371	match parse_mode {
4372	ParseMode::Arg if last_token => working_set
4373	.error(ParseError::Expected("type", Span::new(span.end, span.end))),
4374	ParseMode::Arg => {
4375	parse_mode = ParseMode::Type;
4376	}
4377	ParseMode::AfterCommaArg \| ParseMode::AfterType => {
4378	working_set.error(ParseError::Expected("parameter or flag", span));
4379	}
4380	ParseMode::Type \| ParseMode::DefaultValue => {
4381	// We're seeing two types for the same thing for some reason, error
4382	working_set.error(ParseError::Expected("type", span));
4383	}
4384	}
4385	}
4386	// The = symbol separates a variable from its default value
4387	else if contents == b"=" {
4388	match parse_mode {
4389	ParseMode::Arg \| ParseMode::AfterType if last_token => working_set.error(
4390	ParseError::Expected("default value", Span::new(span.end, span.end)),
4391	),
4392	ParseMode::Arg \| ParseMode::AfterType => {
4393	parse_mode = ParseMode::DefaultValue;
4394	}
4395	ParseMode::Type => {
4396	working_set.error(ParseError::Expected("type", span));
4397	}
4398	ParseMode::AfterCommaArg => {
4399	working_set.error(ParseError::Expected("parameter or flag", span));
4400	}
4401	ParseMode::DefaultValue => {
4402	// We're seeing two default values for some reason, error
4403	working_set.error(ParseError::Expected("default value", span));
4404	}
4405	}
4406	}
4407	// The , symbol separates params only
4408	else if contents == b"," {
4409	match parse_mode {
4410	ParseMode::Arg \| ParseMode::AfterType => {
4411	parse_mode = ParseMode::AfterCommaArg
4412	}
4413	ParseMode::AfterCommaArg => {
4414	working_set.error(ParseError::Expected("parameter or flag", span));
4415	}
4416	ParseMode::Type => {
4417	working_set.error(ParseError::Expected("type", span));
4418	}
4419	ParseMode::DefaultValue => {
4420	working_set.error(ParseError::Expected("default value", span));
4421	}
4422	}
4423	} else {
4424	let mut check_and_add_variable =
4425	\|working_set: &mut StateWorkingSet,
4426	var_name: Vec<u8>,
4427	ty: Type,
4428	span: Span\| {
4429	if is_external {
4430	None
4431	} else {
4432	ensure_not_reserved_variable_name(working_set, &var_name, span);
4433	let var_id =
4434	working_set.add_variable_without_scope(span, ty, false);
4435	pending_scope_inserts.push((var_name, var_id));
4436	Some(var_id)
4437	}
4438	};
4439
4440	match parse_mode {
4441	ParseMode::Arg \| ParseMode::AfterCommaArg \| ParseMode::AfterType => {
4442	// Long flag with optional short form following with no whitespace, e.g. --output, --age(-a)
4443	if contents.starts_with(b"--") && contents.len() > 2 {
4444	// Split the long flag from the short flag with the ( character as delimiter.
4445	// The trailing ) is removed further down.
4446	let flags: Vec<_> = contents.split(\|x\| x == &b'(').collect();
4447
4448	let long = String::from_utf8_lossy(&flags[0][2..]).to_string();
4449	let mut variable_name = flags[0][2..].to_vec();
4450	// Replace the '-' in a variable name with '_'
4451	for byte in variable_name.iter_mut() {
4452	if *byte == b'-' {
4453	*byte = b'_';
4454	}
4455	}
4456
4457	if !is_variable(&variable_name) {
4458	working_set.error(ParseError::Expected(
4459	"valid variable name for this long flag",
4460	span,
4461	))
4462	}
4463
4464	let var_id = check_and_add_variable(
4465	working_set,
4466	variable_name,
4467	Type::Bool,
4468	span,
4469	);
4470
4471	// If there's no short flag, exit now. Otherwise, parse it.
4472	if flags.len() == 1 {
4473	args.push(Arg::Flag {
4474	flag: Flag {
4475	arg: None,
4476	desc: String::new(),
4477	long,
4478	short: None,
4479	required: false,
4480	var_id,
4481	default_value: None,
4482	completion: None,
4483	},
4484	type_annotated: false,
4485	});
4486	} else if flags.len() >= 3 {
4487	working_set.error(ParseError::Expected(
4488	"only one short flag alternative",
4489	span,
4490	));
4491	} else {
4492	let short_flag = &flags[1];
4493	let short_flag = if !short_flag.starts_with(b"-")
4494	\|\| !short_flag.ends_with(b")")
4495	{
4496	working_set.error(ParseError::Expected(
4497	"short flag alternative for the long flag",
4498	span,
4499	));
4500	short_flag
4501	} else {
4502	// Obtain the flag's name by removing the starting - and trailing )
4503	&short_flag[1..(short_flag.len() - 1)]
4504	};
4505	// Note that it is currently possible to make a short flag with non-alphanumeric characters,
4506	// like -).
4507
4508	let short_flag =
4509	String::from_utf8_lossy(short_flag).to_string();
4510	let chars: Vec<char> = short_flag.chars().collect();
4511
4512	if chars.len() == 1 {
4513	args.push(Arg::Flag {
4514	flag: Flag {
4515	arg: None,
4516	desc: String::new(),
4517	long,
4518	short: Some(chars[0]),
4519	required: false,
4520	var_id,
4521	default_value: None,
4522	completion: None,
4523	},
4524	type_annotated: false,
4525	});
4526	} else {
4527	working_set.error(ParseError::Expected("short flag", span));
4528	}
4529	}
4530	parse_mode = ParseMode::Arg;
4531	}
4532	// Mandatory short flag, e.g. -e (must be one character)
4533	else if contents.starts_with(b"-") && contents.len() > 1 {
4534	let short_flag = &contents[1..];
4535	let short_flag = String::from_utf8_lossy(short_flag).to_string();
4536	let chars: Vec<char> = short_flag.chars().collect();
4537
4538	if chars.len() > 1 {
4539	working_set.error(ParseError::Expected("short flag", span));
4540	}
4541
4542	let mut encoded_var_name = [0u8; 4];
4543	let len = chars[0].encode_utf8(&mut encoded_var_name).len();
4544	let variable_name = encoded_var_name[0..len].to_vec();
4545
4546	if !is_variable(&variable_name) {
4547	working_set.error(ParseError::Expected(
4548	"valid variable name for this short flag",
4549	span,
4550	))
4551	}
4552
4553	let var_id = check_and_add_variable(
4554	working_set,
4555	variable_name,
4556	Type::Bool,
4557	span,
4558	);
4559
4560	args.push(Arg::Flag {
4561	flag: Flag {
4562	arg: None,
4563	desc: String::new(),
4564	long: String::new(),
4565	short: Some(chars[0]),
4566	required: false,
4567	var_id,
4568	default_value: None,
4569	completion: None,
4570	},
4571	type_annotated: false,
4572	});
4573	parse_mode = ParseMode::Arg;
4574	}
4575	// Short flag alias for long flag, e.g. --b (-a)
4576	// This is the same as the short flag in --b(-a)
4577	else if let Some(short_flag) = contents.strip_prefix(b"(-") {
4578	if let ParseMode::AfterCommaArg = parse_mode {
4579	working_set
4580	.error(ParseError::Expected("parameter or flag", span));
4581	}
4582
4583	let short_flag = if !short_flag.ends_with(b")") {
4584	working_set.error(ParseError::Expected("short flag", span));
4585	short_flag
4586	} else {
4587	&short_flag[..(short_flag.len() - 1)]
4588	};
4589
4590	let short_flag = String::from_utf8_lossy(short_flag).to_string();
4591	let chars: Vec<char> = short_flag.chars().collect();
4592
4593	if chars.len() == 1 {
4594	match args.last_mut() {
4595	Some(Arg::Flag { flag, .. }) => {
4596	if flag.short.is_some() {
4597	working_set.error(ParseError::Expected(
4598	"one short flag",
4599	span,
4600	));
4601	} else {
4602	flag.short = Some(chars[0]);
4603	}
4604	}
4605	_ => {
4606	working_set
4607	.error(ParseError::Expected("unknown flag", span));
4608	}
4609	}
4610	} else {
4611	working_set.error(ParseError::Expected("short flag", span));
4612	}
4613	}
4614	// Positional arg, optional
4615	else if let Some(optional_param) = contents.strip_suffix(b"?") {
4616	let name = String::from_utf8_lossy(optional_param).to_string();
4617
4618	if !is_variable(optional_param) {
4619	working_set.error(ParseError::Expected(
4620	"valid variable name for this optional parameter",
4621	span,
4622	))
4623	}
4624
4625	let var_id = check_and_add_variable(
4626	working_set,
4627	optional_param.to_vec(),
4628	Type::Any,
4629	span,
4630	);
4631
4632	args.push(Arg::Positional {
4633	arg: PositionalArg {
4634	desc: String::new(),
4635	name,
4636	shape: SyntaxShape::Any,
4637	var_id,
4638	default_value: None,
4639	completion: None,
4640	},
4641	required: false,
4642	type_annotated: false,
4643	});
4644	parse_mode = ParseMode::Arg;
4645	}
4646	// Rest param
4647	else if let Some(contents) = contents.strip_prefix(b"...") {
4648	let name = String::from_utf8_lossy(contents).to_string();
4649	let contents_vec: Vec<u8> = contents.to_vec();
4650
4651	if !is_variable(&contents_vec) {
4652	working_set.error(ParseError::Expected(
4653	"valid variable name for this rest parameter",
4654	span,
4655	))
4656	}
4657
4658	let var_id = check_and_add_variable(
4659	working_set,
4660	contents_vec,
4661	Type::Any,
4662	span,
4663	);
4664
4665	args.push(Arg::RestPositional(PositionalArg {
4666	desc: String::new(),
4667	name,
4668	shape: SyntaxShape::Any,
4669	var_id,
4670	default_value: None,
4671	completion: None,
4672	}));
4673	parse_mode = ParseMode::Arg;
4674	}
4675	// Normal param
4676	else {
4677	let name = String::from_utf8_lossy(&contents).to_string();
4678	let contents_vec = contents.to_vec();
4679
4680	if !is_variable(&contents_vec) {
4681	working_set.error(ParseError::Expected(
4682	"valid variable name for this parameter",
4683	span,
4684	))
4685	}
4686
4687	let var_id = check_and_add_variable(
4688	working_set,
4689	contents_vec,
4690	Type::Any,
4691	span,
4692	);
4693
4694	// Positional arg, required
4695	args.push(Arg::Positional {
4696	arg: PositionalArg {
4697	desc: String::new(),
4698	name,
4699	shape: SyntaxShape::Any,
4700	var_id,
4701	default_value: None,
4702	completion: None,
4703	},
4704	required: true,
4705	type_annotated: false,
4706	});
4707	parse_mode = ParseMode::Arg;
4708	}
4709	}
4710	ParseMode::Type => {
4711	if let Some(last) = args.last_mut() {
4712	let (syntax_shape, completer) = contents
4713	.iter()
4714	.position(\|b\| *b == b'@')
4715	.and_then(\|idx\| {
4716	let (shape, completer) = contents.split_at_checked(idx)?;
4717	let (shape_span, completer_span) = span.split_at(idx)?;
4718
4719	let completer = completer.strip_prefix(b"@")?;
4720	let (_, completer_span) = completer_span.split_at(1)?;
4721
4722	Some((
4723	parse_shape_name(working_set, shape, shape_span),
4724	parse_completer(working_set, completer, completer_span),
4725	))
4726	})
4727	.unwrap_or_else(\|\| {
4728	(parse_shape_name(working_set, &contents, span), None)
4729	});
4730
4731	//TODO check if we're replacing a custom parameter already
4732	match last {
4733	Arg::Positional {
4734	arg:
4735	PositionalArg {
4736	shape,
4737	var_id,
4738	completion,
4739	..
4740	},
4741	required: _,
4742	type_annotated,
4743	} => {
4744	if !is_external {
4745	working_set.set_variable_type(
4746	var_id.expect(
4747	"internal error: all custom parameters must have \
4748	var_ids",
4749	),
4750	syntax_shape.to_type(),
4751	);
4752	}
4753	*completion = completer;
4754	*shape = syntax_shape;
4755	*type_annotated = true;
4756	}
4757	Arg::RestPositional(PositionalArg {
4758	shape,
4759	var_id,
4760	completion,
4761	..
4762	}) => {
4763	if !is_external {
4764	working_set.set_variable_type(
4765	var_id.expect(
4766	"internal error: all custom parameters must have \
4767	var_ids",
4768	),
4769	Type::List(Box::new(syntax_shape.to_type())),
4770	);
4771	}
4772	*completion = completer;
4773	*shape = syntax_shape;
4774	}
4775	Arg::Flag {
4776	flag:
4777	Flag {
4778	arg,
4779	var_id,
4780	completion,
4781	..
4782	},
4783	type_annotated,
4784	} => {
4785	if !is_external {
4786	working_set.set_variable_type(var_id.expect("internal error: all custom parameters must have var_ids"), syntax_shape.to_type());
4787	}
4788	if syntax_shape == SyntaxShape::Boolean {
4789	working_set.error(ParseError::LabeledError(
4790	"Type annotations are not allowed for boolean switches.".to_string(),
4791	"Remove the `: bool` type annotation.".to_string(),
4792	span,
4793	));
4794	}
4795	*completion = completer;
4796	*arg = Some(syntax_shape);
4797	*type_annotated = true;
4798	}
4799	}
4800	}
4801	parse_mode = ParseMode::AfterType;
4802	}
4803	ParseMode::DefaultValue => {
4804	if !is_external && let Some(last) = args.last_mut() {
4805	let expression = parse_value(working_set, span, &SyntaxShape::Any);
4806
4807	//TODO check if we're replacing a custom parameter already
4808	match last {
4809	Arg::Positional {
4810	arg:
4811	PositionalArg {
4812	shape,
4813	var_id,
4814	default_value,
4815	..
4816	},
4817	required,
4818	type_annotated,
4819	} => {
4820	let var_id = var_id.expect("internal error: all custom parameters must have var_ids");
4821	let var_type = &working_set.get_variable(var_id).ty;
4822	match var_type {
4823	Type::Any => {
4824	if !*type_annotated {
4825	working_set.set_variable_type(
4826	var_id,
4827	expression.ty.clone(),
4828	);
4829	}
4830	}
4831	_ => {
4832	if !type_compatible(var_type, &expression.ty) {
4833	working_set.error(
4834	ParseError::AssignmentMismatch(
4835	"Default value wrong type".into(),
4836	format!(
4837	"expected default value to be `{var_type}`"
4838	),
4839	expression.span,
4840	),
4841	)
4842	}
4843	}
4844	}
4845
4846	*default_value = if let Ok(constant) =
4847	eval_constant(working_set, &expression)
4848	{
4849	Some(constant)
4850	} else {
4851	working_set.error(ParseError::NonConstantDefaultValue(
4852	expression.span,
4853	));
4854	None
4855	};
4856
4857	if !*type_annotated {
4858	*shape = expression.ty.to_shape();
4859	}
4860	*required = false;
4861	}
4862	Arg::RestPositional(..) => {
4863	working_set.error(ParseError::AssignmentMismatch(
4864	"Rest parameter was given a default value".into(),
4865	"can't have default value".into(),
4866	expression.span,
4867	))
4868	}
4869	Arg::Flag {
4870	flag:
4871	Flag {
4872	arg,
4873	var_id,
4874	default_value,
4875	..
4876	},
4877	type_annotated,
4878	} => {
4879	let expression_span = expression.span;
4880
4881	*default_value = if let Ok(value) =
4882	eval_constant(working_set, &expression)
4883	{
4884	Some(value)
4885	} else {
4886	working_set.error(ParseError::NonConstantDefaultValue(
4887	expression_span,
4888	));
4889	None
4890	};
4891
4892	let var_id = var_id.expect("internal error: all custom parameters must have var_ids");
4893	let var_type = &working_set.get_variable(var_id).ty;
4894	let expression_ty = expression.ty.clone();
4895
4896	// Flags without type annotations are present/not-present
4897	// switches except when they have a default value
4898	// assigned. In that case they are regular flags and take
4899	// on the type of their default value.
4900	if !*type_annotated {
4901	*arg = Some(expression_ty.to_shape());
4902	working_set.set_variable_type(var_id, expression_ty);
4903	} else if !type_compatible(var_type, &expression_ty) {
4904	working_set.error(ParseError::AssignmentMismatch(
4905	"Default value is the wrong type".into(),
4906	format!(
4907	"expected default value to be `{var_type}`"
4908	),
4909	expression_span,
4910	))
4911	}
4912	}
4913	}
4914	}
4915	parse_mode = ParseMode::Arg;
4916	}
4917	}
4918	}
4919	}
4920	Token {
4921	contents: crate::TokenContents::Comment,
4922	span,
4923	} => {
4924	let contents = working_set.get_span_contents(Span::new(span.start + 1, span.end));
4925
4926	let mut contents = String::from_utf8_lossy(contents).to_string();
4927	contents = contents.trim().into();
4928
4929	if let Some(last) = args.last_mut() {
4930	match last {
4931	Arg::Flag { flag, .. } => {
4932	if !flag.desc.is_empty() {
4933	flag.desc.push('\n');
4934	}
4935	flag.desc.push_str(&contents);
4936	}
4937	Arg::Positional {
4938	arg: positional, ..
4939	} => {
4940	if !positional.desc.is_empty() {
4941	positional.desc.push('\n');
4942	}
4943	positional.desc.push_str(&contents);
4944	}
4945	Arg::RestPositional(positional) => {
4946	if !positional.desc.is_empty() {
4947	positional.desc.push('\n');
4948	}
4949	positional.desc.push_str(&contents);
4950	}
4951	}
4952	}
4953	}
4954	_ => {}
4955	}
4956	}
4957
4958	for (name, var_id) in pending_scope_inserts {
4959	working_set.insert_variable_into_scope(name, var_id);
4960	}
4961
4962	let mut sig = Signature::new(String::new());
4963
4964	for arg in args {
4965	match arg {
4966	Arg::Positional {
4967	arg: positional,
4968	required,
4969	..
4970	} => {
4971	if required {
4972	if !sig.optional_positional.is_empty() {
4973	working_set.error(ParseError::RequiredAfterOptional(
4974	positional.name.clone(),
4975	span,
4976	))
4977	}
4978	sig.required_positional.push(positional)
4979	} else {
4980	sig.optional_positional.push(positional)
4981	}
4982	}
4983	Arg::Flag { flag, .. } => sig.named.push(flag),
4984	Arg::RestPositional(positional) => {
4985	if positional.name.is_empty() {
4986	working_set.error(ParseError::RestNeedsName(span))
4987	} else if sig.rest_positional.is_none() {
4988	sig.rest_positional = Some(PositionalArg {
4989	name: positional.name,
4990	..positional
4991	})
4992	} else {
4993	// Too many rest params
4994	working_set.error(ParseError::MultipleRestParams(span))
4995	}
4996	}
4997	}
4998	}
4999
5000	Box::new(sig)
5001	}
5002
5003	pub fn parse_list_expression(
5004	working_set: &mut StateWorkingSet,
5005	span: Span,
5006	element_shape: &SyntaxShape,
5007	) -> Expression {
5008	let bytes = working_set.get_span_contents(span);
5009
5010	let mut start = span.start;
5011	let mut end = span.end;
5012
5013	if bytes.starts_with(b"[") {
5014	start += 1;
5015	}
5016	if bytes.ends_with(b"]") {
5017	end -= 1;
5018	} else {
5019	working_set.error(ParseError::Unclosed("]".into(), Span::new(end, end)));
5020	}
5021
5022	let inner_span = Span::new(start, end);
5023	let source = working_set.get_span_contents(inner_span);
5024
5025	let (output, err) = lex(source, inner_span.start, &[b'\n', b'\r', b','], &[], true);
5026	if let Some(err) = err {
5027	working_set.error(err)
5028	}
5029
5030	let (mut output, err) = lite_parse(&output, working_set);
5031	if let Some(err) = err {
5032	working_set.error(err)
5033	}
5034
5035	let mut args = vec![];
5036
5037	let mut contained_type: Option<Type> = None;
5038
5039	if !output.block.is_empty() {
5040	for mut command in output.block.remove(0).commands {
5041	let mut spans_idx = 0;
5042
5043	while spans_idx < command.parts.len() {
5044	let curr_span = command.parts[spans_idx];
5045	let curr_tok = working_set.get_span_contents(curr_span);
5046	let (arg, ty) = if curr_tok.starts_with(b"...")
5047	&& curr_tok.len() > 3
5048	&& (curr_tok[3] == b'$' \|\| curr_tok[3] == b'[' \|\| curr_tok[3] == b'(')
5049	{
5050	// Parse the spread operator
5051	// Remove "..." before parsing argument to spread operator
5052	command.parts[spans_idx] = Span::new(curr_span.start + 3, curr_span.end);
5053	let spread_arg = parse_multispan_value(
5054	working_set,
5055	&command.parts,
5056	&mut spans_idx,
5057	&SyntaxShape::List(Box::new(element_shape.clone())),
5058	);
5059	let elem_ty = match &spread_arg.ty {
5060	Type::List(elem_ty) => *elem_ty.clone(),
5061	_ => Type::Any,
5062	};
5063	let span = Span::new(curr_span.start, curr_span.start + 3);
5064	(ListItem::Spread(span, spread_arg), elem_ty)
5065	} else {
5066	let arg = parse_multispan_value(
5067	working_set,
5068	&command.parts,
5069	&mut spans_idx,
5070	element_shape,
5071	);
5072	let ty = arg.ty.clone();
5073	(ListItem::Item(arg), ty)
5074	};
5075
5076	contained_type = match contained_type {
5077	Some(ctype) => Some(ctype.widen(ty)),
5078	None => Some(ty),
5079	};
5080
5081	args.push(arg);
5082
5083	spans_idx += 1;
5084	}
5085	}
5086	}
5087
5088	Expression::new(
5089	working_set,
5090	Expr::List(args),
5091	span,
5092	Type::List(Box::new(if let Some(ty) = contained_type {
5093	ty
5094	} else {
5095	Type::Any
5096	})),
5097	)
5098	}
5099
5100	fn parse_table_row(
5101	working_set: &mut StateWorkingSet,
5102	span: Span,
5103	) -> Result<(Vec<Expression>, Span), Span> {
5104	let list = parse_list_expression(working_set, span, &SyntaxShape::Any);
5105	let Expression {
5106	expr: Expr::List(list),
5107	span,
5108	..
5109	} = list
5110	else {
5111	unreachable!("the item must be a list")
5112	};
5113
5114	list.into_iter()
5115	.map(\|item\| match item {
5116	ListItem::Item(expr) => Ok(expr),
5117	ListItem::Spread(_, spread) => Err(spread.span),
5118	})
5119	.collect::<Result<_, _>>()
5120	.map(\|exprs\| (exprs, span))
5121	}
5122
5123	fn parse_table_expression(
5124	working_set: &mut StateWorkingSet,
5125	span: Span,
5126	list_element_shape: &SyntaxShape,
5127	) -> Expression {
5128	let bytes = working_set.get_span_contents(span);
5129	let inner_span = {
5130	let start = if bytes.starts_with(b"[") {
5131	span.start + 1
5132	} else {
5133	span.start
5134	};
5135
5136	let end = if bytes.ends_with(b"]") {
5137	span.end - 1
5138	} else {
5139	let end = span.end;
5140	working_set.error(ParseError::Unclosed("]".into(), Span::new(end, end)));
5141	span.end
5142	};
5143
5144	Span::new(start, end)
5145	};
5146
5147	let source = working_set.get_span_contents(inner_span);
5148	let (tokens, err) = lex(source, inner_span.start, &[b'\n', b'\r', b','], &[], true);
5149	if let Some(err) = err {
5150	working_set.error(err);
5151	}
5152
5153	// Check that we have all arguments first, before trying to parse the first
5154	// in order to avoid exponential parsing time
5155	let [first, second, rest @ ..] = &tokens[..] else {
5156	return parse_list_expression(working_set, span, list_element_shape);
5157	};
5158	if !working_set.get_span_contents(first.span).starts_with(b"[")
5159	\|\| second.contents != TokenContents::Semicolon
5160	\|\| rest.is_empty()
5161	{
5162	return parse_list_expression(working_set, span, list_element_shape);
5163	};
5164	let head = parse_table_row(working_set, first.span);
5165
5166	let errors = working_set.parse_errors.len();
5167
5168	let (head, rows) = match head {
5169	Ok((head, _)) => {
5170	let rows = rest
5171	.iter()
5172	.filter_map(\|it\| {
5173	use std::cmp::Ordering;
5174
5175	match working_set.get_span_contents(it.span) {
5176	b"," => None,
5177	text if !text.starts_with(b"[") => {
5178	let err = ParseError::LabeledErrorWithHelp {
5179	error: String::from("Table item not list"),
5180	label: String::from("not a list"),
5181	span: it.span,
5182	help: String::from("All table items must be lists"),
5183	};
5184	working_set.error(err);
5185	None
5186	}
5187	_ => match parse_table_row(working_set, it.span) {
5188	Ok((list, span)) => {
5189	match list.len().cmp(&head.len()) {
5190	Ordering::Less => {
5191	let err = ParseError::MissingColumns(head.len(), span);
5192	working_set.error(err);
5193	}
5194	Ordering::Greater => {
5195	let span = {
5196	let start = list[head.len()].span.start;
5197	let end = span.end;
5198	Span::new(start, end)
5199	};
5200	let err = ParseError::ExtraColumns(head.len(), span);
5201	working_set.error(err);
5202	}
5203	Ordering::Equal => {}
5204	}
5205	Some(list)
5206	}
5207	Err(span) => {
5208	let err = ParseError::LabeledError(
5209	String::from("Cannot spread in a table row"),
5210	String::from("invalid spread here"),
5211	span,
5212	);
5213	working_set.error(err);
5214	None
5215	}
5216	},
5217	}
5218	})
5219	.collect();
5220
5221	(head, rows)
5222	}
5223	Err(span) => {
5224	let err = ParseError::LabeledError(
5225	String::from("Cannot spread in a table row"),
5226	String::from("invalid spread here"),
5227	span,
5228	);
5229	working_set.error(err);
5230	(Vec::new(), Vec::new())
5231	}
5232	};
5233
5234	let ty = if working_set.parse_errors.len() == errors {
5235	let (ty, errs) = table_type(&head, &rows);
5236	working_set.parse_errors.extend(errs);
5237	ty
5238	} else {
5239	Type::table()
5240	};
5241
5242	let table = Table {
5243	columns: head.into(),
5244	rows: rows.into_iter().map(Into::into).collect(),
5245	};
5246
5247	Expression::new(working_set, Expr::Table(table), span, ty)
5248	}
5249
5250	fn table_type(head: &[Expression], rows: &[Vec<Expression>]) -> (Type, Vec<ParseError>) {
5251	let mut errors = vec![];
5252	let mut rows: Vec<_> = rows.iter().map(\|row\| row.iter()).collect();
5253
5254	let column_types = std::iter::from_fn(move \|\| {
5255	let column = rows
5256	.iter_mut()
5257	.filter_map(\|row\| row.next())
5258	.map(\|col\| col.ty.clone());
5259	Some(Type::supertype_of(column).unwrap_or(Type::Any))
5260	});
5261
5262	let mk_error = \|span\| ParseError::LabeledErrorWithHelp {
5263	error: "Table column name not string".into(),
5264	label: "must be a string".into(),
5265	help: "Table column names should be able to be converted into strings".into(),
5266	span,
5267	};
5268
5269	let ty: Box<[(String, Type)]> = head
5270	.iter()
5271	.zip(column_types)
5272	.filter_map(\|(expr, col_ty)\| {
5273	if !Type::String.is_subtype_of(&expr.ty) {
5274	errors.push(mk_error(expr.span));
5275	None
5276	} else {
5277	expr.as_string().zip(Some(col_ty))
5278	}
5279	})
5280	.collect();
5281
5282	(Type::Table(ty), errors)
5283	}
5284
5285	pub fn parse_block_expression(working_set: &mut StateWorkingSet, span: Span) -> Expression {
5286	trace!("parsing: block expression");
5287
5288	let bytes = working_set.get_span_contents(span);
5289
5290	let mut start = span.start;
5291	let mut end = span.end;
5292	let mut is_closed = true;
5293
5294	if bytes.starts_with(b"{") {
5295	start += 1;
5296	} else {
5297	working_set.error(ParseError::Expected("block", span));
5298	return garbage(working_set, span);
5299	}
5300	if bytes.ends_with(b"}") {
5301	end -= 1;
5302	} else {
5303	working_set.error(ParseError::Unclosed("}".into(), Span::new(end, end)));
5304	is_closed = false;
5305	}
5306
5307	let inner_span = Span::new(start, end);
5308
5309	let source = working_set.get_span_contents(inner_span);
5310
5311	let (output, err) = lex(source, start, &[], &[], false);
5312	if let Some(err) = err {
5313	working_set.error(err);
5314	}
5315
5316	working_set.enter_scope();
5317
5318	// Check to see if we have parameters
5319	let (signature, amt_to_skip): (Option<(Box<Signature>, Span)>, usize) = match output.first() {
5320	Some(Token {
5321	contents: TokenContents::Pipe,
5322	span,
5323	}) => {
5324	working_set.error(ParseError::Expected("block but found closure", *span));
5325	(None, 0)
5326	}
5327	_ => (None, 0),
5328	};
5329
5330	let mut output = parse_block(working_set, &output[amt_to_skip..], span, false, false);
5331
5332	if let Some(signature) = signature {
5333	output.signature = signature.0;
5334	}
5335
5336	output.span = Some(span);
5337
5338	if is_closed {
5339	working_set.exit_scope();
5340	}
5341
5342	let block_id = working_set.add_block(Arc::new(output));
5343
5344	Expression::new(working_set, Expr::Block(block_id), span, Type::Block)
5345	}
5346
5347	pub fn parse_match_block_expression(working_set: &mut StateWorkingSet, span: Span) -> Expression {
5348	let bytes = working_set.get_span_contents(span);
5349
5350	let mut start = span.start;
5351	let mut end = span.end;
5352	let mut is_closed = true;
5353
5354	if bytes.starts_with(b"{") {
5355	start += 1;
5356	} else {
5357	working_set.error(ParseError::Expected("closure", span));
5358	return garbage(working_set, span);
5359	}
5360	if bytes.ends_with(b"}") {
5361	end -= 1;
5362	} else {
5363	working_set.error(ParseError::Unclosed("}".into(), Span::new(end, end)));
5364	is_closed = false;
5365	}
5366
5367	let inner_span = Span::new(start, end);
5368
5369	let source = working_set.get_span_contents(inner_span);
5370
5371	let (output, err) = lex(source, start, &[b' ', b'\r', b'\n', b',', b'\|'], &[], true);
5372	if let Some(err) = err {
5373	working_set.error(err);
5374	}
5375
5376	let mut position = 0;
5377
5378	let mut output_matches = vec![];
5379
5380	while position < output.len() {
5381	// Each match gets its own scope
5382
5383	working_set.enter_scope();
5384
5385	// First parse the pattern
5386	let mut pattern = parse_pattern(working_set, output[position].span);
5387
5388	position += 1;
5389
5390	if position >= output.len() {
5391	working_set.error(ParseError::Mismatch(
5392	"=>".into(),
5393	"end of input".into(),
5394	Span::new(output[position - 1].span.end, output[position - 1].span.end),
5395	));
5396
5397	working_set.exit_scope();
5398	break;
5399	}
5400
5401	let mut connector = working_set.get_span_contents(output[position].span);
5402
5403	// Multiple patterns connected by '\|'
5404	if connector == b"\|" && position < output.len() {
5405	let mut or_pattern = vec![pattern];
5406
5407	while connector == b"\|" && position < output.len() {
5408	connector = b"";
5409
5410	position += 1;
5411
5412	if position >= output.len() {
5413	working_set.error(ParseError::Mismatch(
5414	"pattern".into(),
5415	"end of input".into(),
5416	Span::new(output[position - 1].span.end, output[position - 1].span.end),
5417	));
5418	break;
5419	}
5420
5421	let pattern = parse_pattern(working_set, output[position].span);
5422	or_pattern.push(pattern);
5423
5424	position += 1;
5425	if position >= output.len() {
5426	working_set.error(ParseError::Mismatch(
5427	"=>".into(),
5428	"end of input".into(),
5429	Span::new(output[position - 1].span.end, output[position - 1].span.end),
5430	));
5431	break;
5432	} else {
5433	connector = working_set.get_span_contents(output[position].span);
5434	}
5435	}
5436
5437	let start = or_pattern
5438	.first()
5439	.expect("internal error: unexpected state of or-pattern")
5440	.span
5441	.start;
5442	let end = or_pattern
5443	.last()
5444	.expect("internal error: unexpected state of or-pattern")
5445	.span
5446	.end;
5447
5448	pattern = MatchPattern {
5449	pattern: Pattern::Or(or_pattern),
5450	guard: None,
5451	span: Span::new(start, end),
5452	}
5453	}
5454	// A match guard
5455	if connector == b"if" {
5456	let if_end = {
5457	let end = output[position].span.end;
5458	Span::new(end, end)
5459	};
5460
5461	position += 1;
5462
5463	let mk_err = \|\| ParseError::LabeledErrorWithHelp {
5464	error: "Match guard without an expression".into(),
5465	label: "expected an expression".into(),
5466	help: "The `if` keyword must be followed with an expression".into(),
5467	span: if_end,
5468	};
5469
5470	if output.get(position).is_none() {
5471	working_set.error(mk_err());
5472	return garbage(working_set, span);
5473	};
5474
5475	let (tokens, found) = if let Some((pos, _)) = output[position..]
5476	.iter()
5477	.find_position(\|t\| working_set.get_span_contents(t.span) == b"=>")
5478	{
5479	if position + pos == position {
5480	working_set.error(mk_err());
5481	return garbage(working_set, span);
5482	}
5483
5484	(&output[position..position + pos], true)
5485	} else {
5486	(&output[position..], false)
5487	};
5488
5489	let mut start = 0;
5490	let guard = parse_multispan_value(
5491	working_set,
5492	&tokens.iter().map(\|tok\| tok.span).collect_vec(),
5493	&mut start,
5494	&SyntaxShape::MathExpression,
5495	);
5496
5497	pattern.guard = Some(Box::new(guard));
5498	position += if found { start + 1 } else { start };
5499	connector = working_set.get_span_contents(output[position].span);
5500	}
5501	// Then the `=>` arrow
5502	if connector != b"=>" {
5503	working_set.error(ParseError::Mismatch(
5504	"=>".into(),
5505	"end of input".into(),
5506	Span::new(output[position - 1].span.end, output[position - 1].span.end),
5507	));
5508	} else {
5509	position += 1;
5510	}
5511
5512	// Finally, the value/expression/block that we will run to produce the result
5513	if position >= output.len() {
5514	working_set.error(ParseError::Mismatch(
5515	"match result".into(),
5516	"end of input".into(),
5517	Span::new(output[position - 1].span.end, output[position - 1].span.end),
5518	));
5519
5520	working_set.exit_scope();
5521	break;
5522	}
5523
5524	let result = parse_multispan_value(
5525	working_set,
5526	&[output[position].span],
5527	&mut 0,
5528	&SyntaxShape::OneOf(vec![SyntaxShape::Block, SyntaxShape::Expression]),
5529	);
5530	position += 1;
5531	if is_closed {
5532	working_set.exit_scope();
5533	}
5534
5535	output_matches.push((pattern, result));
5536	}
5537
5538	Expression::new(
5539	working_set,
5540	Expr::MatchBlock(output_matches),
5541	span,
5542	Type::Any,
5543	)
5544	}
5545
5546	pub fn parse_closure_expression(
5547	working_set: &mut StateWorkingSet,
5548	shape: &SyntaxShape,
5549	span: Span,
5550	) -> Expression {
5551	trace!("parsing: closure expression");
5552
5553	let bytes = working_set.get_span_contents(span);
5554
5555	let mut start = span.start;
5556	let mut end = span.end;
5557	let mut is_closed = true;
5558
5559	if bytes.starts_with(b"{") {
5560	start += 1;
5561	} else {
5562	working_set.error(ParseError::Expected("closure", span));
5563	return garbage(working_set, span);
5564	}
5565	if bytes.ends_with(b"}") {
5566	end -= 1;
5567	} else {
5568	working_set.error(ParseError::Unclosed("}".into(), Span::new(end, end)));
5569	is_closed = false;
5570	}
5571
5572	let inner_span = Span::new(start, end);
5573
5574	let source = working_set.get_span_contents(inner_span);
5575
5576	let (output, err) = lex(source, start, &[], &[], false);
5577	if let Some(err) = err {
5578	working_set.error(err);
5579	}
5580
5581	working_set.enter_scope();
5582
5583	// Check to see if we have parameters
5584	let (signature, amt_to_skip): (Option<(Box<Signature>, Span)>, usize) = match output.first() {
5585	Some(Token {
5586	contents: TokenContents::Pipe,
5587	span,
5588	}) => {
5589	// We've found a parameter list
5590	let start_point = span.start;
5591	let mut token_iter = output.iter().enumerate().skip(1);
5592	let mut end_span = None;
5593	let mut amt_to_skip = 1;
5594
5595	for token in &mut token_iter {
5596	if let Token {
5597	contents: TokenContents::Pipe,
5598	span,
5599	} = token.1
5600	{
5601	end_span = Some(span);
5602	amt_to_skip += token.0;
5603	break;
5604	}
5605	}
5606
5607	let end_point = if let Some(span) = end_span {
5608	span.end
5609	} else {
5610	working_set.error(ParseError::Unclosed("\|".into(), Span::new(end, end)));
5611	end
5612	};
5613
5614	let signature_span = Span::new(start_point, end_point);
5615	let signature = parse_signature_helper(working_set, signature_span, false);
5616
5617	(Some((signature, signature_span)), amt_to_skip)
5618	}
5619	Some(Token {
5620	contents: TokenContents::PipePipe,
5621	span,
5622	}) => (
5623	Some((Box::new(Signature::new("closure".to_string())), *span)),
5624	1,
5625	),
5626	_ => (None, 0),
5627	};
5628
5629	// TODO: Finish this
5630	if let SyntaxShape::Closure(Some(v)) = shape
5631	&& let Some((sig, sig_span)) = &signature
5632	{
5633	if sig.num_positionals() > v.len() {
5634	working_set.error(ParseError::ExpectedWithStringMsg(
5635	format!(
5636	"{} closure parameter{}",
5637	v.len(),
5638	if v.len() > 1 { "s" } else { "" }
5639	),
5640	*sig_span,
5641	));
5642	}
5643
5644	for (expected, PositionalArg { name, shape, .. }) in
5645	v.iter().zip(sig.required_positional.iter())
5646	{
5647	if expected != shape && *shape != SyntaxShape::Any {
5648	working_set.error(ParseError::ParameterMismatchType(
5649	name.to_owned(),
5650	expected.to_string(),
5651	shape.to_string(),
5652	*sig_span,
5653	));
5654	}
5655	}
5656	}
5657
5658	let mut output = parse_block(working_set, &output[amt_to_skip..], span, false, false);
5659
5660	// NOTE: closures need to be compiled eagerly due to these reasons:
5661	// - their `Block`s (which contains their `IrBlock`) are stored in the working_set
5662	// - Ir compiler does not have mutable access to the working_set and can't attach `IrBlock`s
5663	// to existing `Block`s
5664	// so they can't be compiled as part of their parent `Block`'s compilation
5665	//
5666	// If the compiler used a mechanism similar to the `EngineState`/`StateWorkingSet` divide, we
5667	// could defer all compilation and apply the generated delta to `StateWorkingSet` afterwards.
5668	if working_set.parse_errors.is_empty() {
5669	compile_block(working_set, &mut output);
5670	}
5671
5672	if let Some(signature) = signature {
5673	output.signature = signature.0;
5674	}
5675
5676	output.span = Some(span);
5677
5678	if is_closed {
5679	working_set.exit_scope();
5680	}
5681
5682	let block_id = working_set.add_block(Arc::new(output));
5683
5684	Expression::new(working_set, Expr::Closure(block_id), span, Type::Closure)
5685	}
5686
5687	pub fn parse_value(
5688	working_set: &mut StateWorkingSet,
5689	span: Span,
5690	shape: &SyntaxShape,
5691	) -> Expression {
5692	trace!("parsing: value: {shape}");
5693
5694	let bytes = working_set.get_span_contents(span);
5695
5696	if bytes.is_empty() {
5697	working_set.error(ParseError::IncompleteParser(span));
5698	return garbage(working_set, span);
5699	}
5700
5701	match bytes[0] {
5702	b'$' => return parse_dollar_expr(working_set, span),
5703	b'(' => return parse_paren_expr(working_set, span, shape),
5704	b'{' => return parse_brace_expr(working_set, span, shape),
5705	b'[' => match shape {
5706	SyntaxShape::Any
5707	\| SyntaxShape::List(_)
5708	\| SyntaxShape::Table(_)
5709	\| SyntaxShape::Signature
5710	\| SyntaxShape::ExternalSignature
5711	\| SyntaxShape::Filepath
5712	\| SyntaxShape::String
5713	\| SyntaxShape::GlobPattern
5714	\| SyntaxShape::ExternalArgument => {}
5715	SyntaxShape::OneOf(possible_shapes) => {
5716	if !possible_shapes
5717	.iter()
5718	.any(\|s\| matches!(s, SyntaxShape::List(_)))
5719	{
5720	working_set.error(ParseError::ExpectedWithStringMsg(shape.to_string(), span));
5721	return Expression::garbage(working_set, span);
5722	}
5723	}
5724	_ => {
5725	working_set.error(ParseError::ExpectedWithStringMsg(shape.to_string(), span));
5726	return Expression::garbage(working_set, span);
5727	}
5728	},
5729	b'r' if bytes.len() > 1 && bytes[1] == b'#' => {
5730	return parse_raw_string(working_set, span);
5731	}
5732	_ => {}
5733	}
5734
5735	match shape {
5736	SyntaxShape::Number => parse_number(working_set, span),
5737	SyntaxShape::Float => parse_float(working_set, span),
5738	SyntaxShape::Int => parse_int(working_set, span),
5739	SyntaxShape::Duration => parse_duration(working_set, span),
5740	SyntaxShape::DateTime => parse_datetime(working_set, span),
5741	SyntaxShape::Filesize => parse_filesize(working_set, span),
5742	SyntaxShape::Range => {
5743	parse_range(working_set, span).unwrap_or_else(\|\| garbage(working_set, span))
5744	}
5745	// Check for reserved keyword values
5746	SyntaxShape::Nothing \| SyntaxShape::Any if bytes == b"null" => {
5747	Expression::new(working_set, Expr::Nothing, span, Type::Nothing)
5748	}
5749	SyntaxShape::Boolean \| SyntaxShape::Any if bytes == b"true" => {
5750	Expression::new(working_set, Expr::Bool(true), span, Type::Bool)
5751	}
5752	SyntaxShape::Boolean \| SyntaxShape::Any if bytes == b"false" => {
5753	Expression::new(working_set, Expr::Bool(false), span, Type::Bool)
5754	}
5755	SyntaxShape::Filepath
5756	\| SyntaxShape::Directory
5757	\| SyntaxShape::GlobPattern
5758	// TODO: this serves for backward compatibility.
5759	// As a consequence, for commands like `def foo [foo: string] {}`,
5760	// it forbids usage like `foo true`, have to call it explicitly with `foo "true"`.
5761	// On the other hand, given current `SyntaxShape` based `parse_value`, `foo 10.0` doesn't raise any error.
5762	// We want to fix this discrepancy in the future.
5763	\| SyntaxShape::String
5764	if matches!(bytes, b"true" \| b"false" \| b"null") =>
5765	{
5766	working_set.error(ParseError::ExpectedWithStringMsg(shape.to_string(), span));
5767	garbage(working_set, span)
5768	}
5769	SyntaxShape::Filepath => parse_filepath(working_set, span),
5770	SyntaxShape::Directory => parse_directory(working_set, span),
5771	SyntaxShape::GlobPattern => parse_glob_pattern(working_set, span),
5772	SyntaxShape::String => parse_string(working_set, span),
5773	SyntaxShape::Binary => parse_binary(working_set, span),
5774	SyntaxShape::Signature if bytes.starts_with(b"[") => parse_signature(working_set, span, false),
5775	SyntaxShape::ExternalSignature if bytes.starts_with(b"[") => parse_signature(working_set, span, true),
5776	SyntaxShape::List(elem) if bytes.starts_with(b"[") => {
5777	parse_table_expression(working_set, span, elem)
5778	}
5779	SyntaxShape::Table(_) if bytes.starts_with(b"[") => {
5780	parse_table_expression(working_set, span, &SyntaxShape::Any)
5781	}
5782	SyntaxShape::CellPath => parse_simple_cell_path(working_set, span),
5783
5784	// Be sure to return ParseError::Expected(..) if invoked for one of these shapes, but lex
5785	// stream doesn't start with '{'} -- parsing in SyntaxShape::Any arm depends on this error variant.
5786	SyntaxShape::Block \| SyntaxShape::Closure(..) \| SyntaxShape::Record(_) => {
5787	working_set.error(ParseError::Expected("block, closure or record", span));
5788
5789	Expression::garbage(working_set, span)
5790	}
5791
5792	SyntaxShape::ExternalArgument => parse_regular_external_arg(working_set, span),
5793	SyntaxShape::OneOf(possible_shapes) => {
5794	parse_oneof(working_set, &[span], &mut 0, possible_shapes, false)
5795	}
5796
5797	SyntaxShape::Any => {
5798	if bytes.starts_with(b"[") {
5799	//parse_value(working_set, span, &SyntaxShape::Table)
5800	parse_full_cell_path(working_set, None, span)
5801	} else {
5802	let shapes = [
5803	SyntaxShape::Binary,
5804	SyntaxShape::Range,
5805	SyntaxShape::Filesize,
5806	SyntaxShape::Duration,
5807	SyntaxShape::DateTime,
5808	SyntaxShape::Int,
5809	SyntaxShape::Number,
5810	SyntaxShape::String,
5811	];
5812	for shape in shapes.iter() {
5813	let starting_error_count = working_set.parse_errors.len();
5814
5815	let s = parse_value(working_set, span, shape);
5816
5817	if starting_error_count == working_set.parse_errors.len() {
5818	return s;
5819	} else {
5820	match working_set.parse_errors.get(starting_error_count) {
5821	Some(
5822	ParseError::Expected(_, _)
5823	\| ParseError::ExpectedWithStringMsg(_, _),
5824	) => {
5825	working_set.parse_errors.truncate(starting_error_count);
5826	continue;
5827	}
5828	_ => {
5829	return s;
5830	}
5831	}
5832	}
5833	}
5834	working_set.error(ParseError::Expected("any shape", span));
5835	garbage(working_set, span)
5836	}
5837	}
5838	_ => {
5839	working_set.error(ParseError::ExpectedWithStringMsg(shape.to_string(), span));
5840	garbage(working_set, span)
5841	}
5842	}
5843	}
5844
5845	pub fn parse_assignment_operator(working_set: &mut StateWorkingSet, span: Span) -> Expression {
5846	let contents = working_set.get_span_contents(span);
5847
5848	let operator = match contents {
5849	b"=" => Operator::Assignment(Assignment::Assign),
5850	b"+=" => Operator::Assignment(Assignment::AddAssign),
5851	b"-=" => Operator::Assignment(Assignment::SubtractAssign),
5852	b"*=" => Operator::Assignment(Assignment::MultiplyAssign),
5853	b"/=" => Operator::Assignment(Assignment::DivideAssign),
5854	b"++=" => Operator::Assignment(Assignment::ConcatenateAssign),
5855	_ => {
5856	working_set.error(ParseError::Expected("assignment operator", span));
5857	return garbage(working_set, span);
5858	}
5859	};
5860
5861	Expression::new(working_set, Expr::Operator(operator), span, Type::Any)
5862	}
5863
5864	pub fn parse_assignment_expression(
5865	working_set: &mut StateWorkingSet,
5866	spans: &[Span],
5867	) -> Expression {
5868	trace!("parsing: assignment expression");
5869	let expr_span = Span::concat(spans);
5870
5871	// Assignment always has the most precedence, and its right-hand side can be a pipeline
5872	let Some(op_index) = spans
5873	.iter()
5874	.position(\|span\| is_assignment_operator(working_set.get_span_contents(*span)))
5875	else {
5876	working_set.error(ParseError::Expected("assignment expression", expr_span));
5877	return garbage(working_set, expr_span);
5878	};
5879
5880	let lhs_spans = &spans[0..op_index];
5881	let op_span = spans[op_index];
5882	let rhs_spans = &spans[(op_index + 1)..];
5883
5884	if lhs_spans.is_empty() {
5885	working_set.error(ParseError::Expected(
5886	"left hand side of assignment",
5887	op_span,
5888	));
5889	return garbage(working_set, expr_span);
5890	}
5891
5892	if rhs_spans.is_empty() {
5893	working_set.error(ParseError::Expected(
5894	"right hand side of assignment",
5895	op_span,
5896	));
5897	return garbage(working_set, expr_span);
5898	}
5899
5900	// Parse the lhs and operator as usual for a math expression
5901	let mut lhs = parse_expression(working_set, lhs_spans);
5902	// make sure that lhs is a mutable variable.
5903	match &lhs.expr {
5904	Expr::FullCellPath(p) => {
5905	if let Expr::Var(var_id) = p.head.expr
5906	&& var_id != nu_protocol::ENV_VARIABLE_ID
5907	&& !working_set.get_variable(var_id).mutable
5908	{
5909	working_set.error(ParseError::AssignmentRequiresMutableVar(lhs.span))
5910	}
5911	}
5912	_ => working_set.error(ParseError::AssignmentRequiresVar(lhs.span)),
5913	}
5914
5915	let mut operator = parse_assignment_operator(working_set, op_span);
5916
5917	// Re-parse the right-hand side as a subexpression
5918	let rhs_span = Span::concat(rhs_spans);
5919
5920	let (rhs_tokens, rhs_error) = lex(
5921	working_set.get_span_contents(rhs_span),
5922	rhs_span.start,
5923	&[],
5924	&[],
5925	false,
5926	);
5927	working_set.parse_errors.extend(rhs_error);
5928
5929	trace!("parsing: assignment right-hand side subexpression");
5930	let rhs_block = parse_block(working_set, &rhs_tokens, rhs_span, false, true);
5931	let rhs_ty = rhs_block.output_type();
5932
5933	// TEMP: double-check that if the RHS block starts with an external call, it must start with a
5934	// caret. This is to mitigate the change in assignment parsing introduced in 0.97.0 which could
5935	// result in unintentional execution of commands.
5936	if let Some(Expr::ExternalCall(head, ..)) = rhs_block
5937	.pipelines
5938	.first()
5939	.and_then(\|pipeline\| pipeline.elements.first())
5940	.map(\|element\| &element.expr.expr)
5941	{
5942	let contents = working_set.get_span_contents(Span {
5943	start: head.span.start - 1,
5944	end: head.span.end,
5945	});
5946	if !contents.starts_with(b"^") {
5947	working_set.parse_errors.push(ParseError::LabeledErrorWithHelp {
5948	error: "External command calls must be explicit in assignments".into(),
5949	label: "add a caret (^) before the command name if you intended to run and capture its output".into(),
5950	help: "the parsing of assignments was changed in 0.97.0, and this would have previously been treated as a string. Alternatively, quote the string with single or double quotes to avoid it being interpreted as a command name. This restriction may be removed in a future release.".into(),
5951	span: head.span,
5952	});
5953	}
5954	}
5955
5956	let rhs_block_id = working_set.add_block(Arc::new(rhs_block));
5957	let mut rhs = Expression::new(
5958	working_set,
5959	Expr::Subexpression(rhs_block_id),
5960	rhs_span,
5961	rhs_ty,
5962	);
5963
5964	let (result_ty, err) = math_result_type(working_set, &mut lhs, &mut operator, &mut rhs);
5965	if let Some(err) = err {
5966	working_set.parse_errors.push(err);
5967	}
5968
5969	Expression::new(
5970	working_set,
5971	Expr::BinaryOp(Box::new(lhs), Box::new(operator), Box::new(rhs)),
5972	expr_span,
5973	result_ty,
5974	)
5975	}
5976
5977	pub fn parse_operator(working_set: &mut StateWorkingSet, span: Span) -> Expression {
5978	let contents = working_set.get_span_contents(span);
5979
5980	let operator = match contents {
5981	b"==" => Operator::Comparison(Comparison::Equal),
5982	b"!=" => Operator::Comparison(Comparison::NotEqual),
5983	b"<" => Operator::Comparison(Comparison::LessThan),
5984	b"<=" => Operator::Comparison(Comparison::LessThanOrEqual),
5985	b">" => Operator::Comparison(Comparison::GreaterThan),
5986	b">=" => Operator::Comparison(Comparison::GreaterThanOrEqual),
5987	b"=~" \| b"like" => Operator::Comparison(Comparison::RegexMatch),
5988	b"!~" \| b"not-like" => Operator::Comparison(Comparison::NotRegexMatch),
5989	b"in" => Operator::Comparison(Comparison::In),
5990	b"not-in" => Operator::Comparison(Comparison::NotIn),
5991	b"has" => Operator::Comparison(Comparison::Has),
5992	b"not-has" => Operator::Comparison(Comparison::NotHas),
5993	b"starts-with" => Operator::Comparison(Comparison::StartsWith),
5994	b"not-starts-with" => Operator::Comparison(Comparison::NotStartsWith),
5995	b"ends-with" => Operator::Comparison(Comparison::EndsWith),
5996	b"not-ends-with" => Operator::Comparison(Comparison::NotEndsWith),
5997	b"+" => Operator::Math(Math::Add),
5998	b"-" => Operator::Math(Math::Subtract),
5999	b"*" => Operator::Math(Math::Multiply),
6000	b"/" => Operator::Math(Math::Divide),
6001	b"//" => Operator::Math(Math::FloorDivide),
6002	b"mod" => Operator::Math(Math::Modulo),
6003	b"**" => Operator::Math(Math::Pow),
6004	b"++" => Operator::Math(Math::Concatenate),
6005	b"bit-or" => Operator::Bits(Bits::BitOr),
6006	b"bit-xor" => Operator::Bits(Bits::BitXor),
6007	b"bit-and" => Operator::Bits(Bits::BitAnd),
6008	b"bit-shl" => Operator::Bits(Bits::ShiftLeft),
6009	b"bit-shr" => Operator::Bits(Bits::ShiftRight),
6010	b"or" => Operator::Boolean(Boolean::Or),
6011	b"xor" => Operator::Boolean(Boolean::Xor),
6012	b"and" => Operator::Boolean(Boolean::And),
6013	// WARNING: not actual operators below! Error handling only
6014	pow @ (b"^" \| b"pow") => {
6015	working_set.error(ParseError::UnknownOperator(
6016	match pow {
6017	b"^" => "^",
6018	b"pow" => "pow",
6019	_ => unreachable!(),
6020	},
6021	"Use '**' for exponentiation or 'bit-xor' for bitwise XOR.",
6022	span,
6023	));
6024	return garbage(working_set, span);
6025	}
6026	equality @ (b"is" \| b"===") => {
6027	working_set.error(ParseError::UnknownOperator(
6028	match equality {
6029	b"is" => "is",
6030	b"===" => "===",
6031	_ => unreachable!(),
6032	},
6033	"Did you mean '=='?",
6034	span,
6035	));
6036	return garbage(working_set, span);
6037	}
6038	b"contains" => {
6039	working_set.error(ParseError::UnknownOperator(
6040	"contains",
6041	"Did you mean 'has'?",
6042	span,
6043	));
6044	return garbage(working_set, span);
6045	}
6046	b"%" => {
6047	working_set.error(ParseError::UnknownOperator(
6048	"%",
6049	"Did you mean 'mod'?",
6050	span,
6051	));
6052	return garbage(working_set, span);
6053	}
6054	b"&" => {
6055	working_set.error(ParseError::UnknownOperator(
6056	"&",
6057	"Did you mean 'bit-and'?",
6058	span,
6059	));
6060	return garbage(working_set, span);
6061	}
6062	b"<<" => {
6063	working_set.error(ParseError::UnknownOperator(
6064	"<<",
6065	"Did you mean 'bit-shl'?",
6066	span,
6067	));
6068	return garbage(working_set, span);
6069	}
6070	b">>" => {
6071	working_set.error(ParseError::UnknownOperator(
6072	">>",
6073	"Did you mean 'bit-shr'?",
6074	span,
6075	));
6076	return garbage(working_set, span);
6077	}
6078	bits @ (b"bits-and" \| b"bits-xor" \| b"bits-or" \| b"bits-shl" \| b"bits-shr") => {
6079	working_set.error(ParseError::UnknownOperator(
6080	match bits {
6081	b"bits-and" => "bits-and",
6082	b"bits-xor" => "bits-xor",
6083	b"bits-or" => "bits-or",
6084	b"bits-shl" => "bits-shl",
6085	b"bits-shr" => "bits-shr",
6086	_ => unreachable!(),
6087	},
6088	match bits {
6089	b"bits-and" => "Did you mean 'bit-and'?",
6090	b"bits-xor" => "Did you mean 'bit-xor'?",
6091	b"bits-or" => "Did you mean 'bit-or'?",
6092	b"bits-shl" => "Did you mean 'bit-shl'?",
6093	b"bits-shr" => "Did you mean 'bit-shr'?",
6094	_ => unreachable!(),
6095	},
6096	span,
6097	));
6098	return garbage(working_set, span);
6099	}
6100	op if is_assignment_operator(op) => {
6101	working_set.error(ParseError::Expected("a non-assignment operator", span));
6102	return garbage(working_set, span);
6103	}
6104	_ => {
6105	working_set.error(ParseError::Expected("operator", span));
6106	return garbage(working_set, span);
6107	}
6108	};
6109
6110	Expression::new(working_set, Expr::Operator(operator), span, Type::Any)
6111	}
6112
6113	pub fn parse_math_expression(
6114	working_set: &mut StateWorkingSet,
6115	spans: &[Span],
6116	lhs_row_var_id: Option<VarId>,
6117	) -> Expression {
6118	trace!("parsing: math expression");
6119
6120	// As the expr_stack grows, we increase the required precedence to grow larger
6121	// If, at any time, the operator we're looking at is the same or lower precedence
6122	// of what is in the expression stack, we collapse the expression stack.
6123	//
6124	// This leads to an expression stack that grows under increasing precedence and collapses
6125	// under decreasing/sustained precedence
6126	//
6127	// The end result is a stack that we can fold into binary operations as right associations
6128	// safely.
6129
6130	let mut expr_stack: Vec<Expression> = vec![];
6131
6132	let mut idx = 0;
6133	let mut last_prec = u8::MAX;
6134
6135	let first_span = working_set.get_span_contents(spans[0]);
6136
6137	let mut not_start_spans = vec![];
6138
6139	if first_span == b"if" \|\| first_span == b"match" {
6140	// If expression
6141	if spans.len() > 1 {
6142	return parse_call(working_set, spans, spans[0]);
6143	} else {
6144	working_set.error(ParseError::Expected(
6145	"expression",
6146	Span::new(spans[0].end, spans[0].end),
6147	));
6148	return garbage(working_set, spans[0]);
6149	}
6150	} else if first_span == b"not" {
6151	not_start_spans.push(spans[idx].start);
6152	idx += 1;
6153	while idx < spans.len() {
6154	let next_value = working_set.get_span_contents(spans[idx]);
6155
6156	if next_value == b"not" {
6157	not_start_spans.push(spans[idx].start);
6158	idx += 1;
6159	} else {
6160	break;
6161	}
6162	}
6163
6164	if idx == spans.len() {
6165	working_set.error(ParseError::Expected(
6166	"expression",
6167	Span::new(spans[idx - 1].end, spans[idx - 1].end),
6168	));
6169	return garbage(working_set, spans[idx - 1]);
6170	}
6171	}
6172
6173	let mut lhs = parse_value(working_set, spans[idx], &SyntaxShape::Any);
6174
6175	for not_start_span in not_start_spans.iter().rev() {
6176	lhs = Expression::new(
6177	working_set,
6178	Expr::UnaryNot(Box::new(lhs)),
6179	Span::new(*not_start_span, spans[idx].end),
6180	Type::Bool,
6181	);
6182	}
6183	not_start_spans.clear();
6184
6185	idx += 1;
6186
6187	if idx >= spans.len() {
6188	// We already found the one part of our expression, so let's expand
6189	if let Some(row_var_id) = lhs_row_var_id {
6190	expand_to_cell_path(working_set, &mut lhs, row_var_id);
6191	}
6192	}
6193
6194	expr_stack.push(lhs);
6195
6196	while idx < spans.len() {
6197	let op = parse_operator(working_set, spans[idx]);
6198
6199	let op_prec = op.precedence();
6200
6201	idx += 1;
6202
6203	if idx == spans.len() {
6204	// Handle broken math expr `1 +` etc
6205	working_set.error(ParseError::IncompleteMathExpression(spans[idx - 1]));
6206
6207	expr_stack.push(Expression::garbage(working_set, spans[idx - 1]));
6208	let missing_span = Span::new(spans[idx - 1].end, spans[idx - 1].end);
6209	expr_stack.push(Expression::garbage(working_set, missing_span));
6210
6211	break;
6212	}
6213
6214	let content = working_set.get_span_contents(spans[idx]);
6215	// allow `if` to be a special value for assignment.
6216
6217	if content == b"if" \|\| content == b"match" {
6218	let rhs = parse_call(working_set, &spans[idx..], spans[0]);
6219	expr_stack.push(op);
6220	expr_stack.push(rhs);
6221	break;
6222	} else if content == b"not" {
6223	not_start_spans.push(spans[idx].start);
6224	idx += 1;
6225	while idx < spans.len() {
6226	let next_value = working_set.get_span_contents(spans[idx]);
6227
6228	if next_value == b"not" {
6229	not_start_spans.push(spans[idx].start);
6230	idx += 1;
6231	} else {
6232	break;
6233	}
6234	}
6235
6236	if idx == spans.len() {
6237	working_set.error(ParseError::Expected(
6238	"expression",
6239	Span::new(spans[idx - 1].end, spans[idx - 1].end),
6240	));
6241	return garbage(working_set, spans[idx - 1]);
6242	}
6243	}
6244	let mut rhs = parse_value(working_set, spans[idx], &SyntaxShape::Any);
6245
6246	for not_start_span in not_start_spans.iter().rev() {
6247	rhs = Expression::new(
6248	working_set,
6249	Expr::UnaryNot(Box::new(rhs)),
6250	Span::new(*not_start_span, spans[idx].end),
6251	Type::Bool,
6252	);
6253	}
6254	not_start_spans.clear();
6255
6256	// Parsing power must be right-associative unlike most operations which are left
6257	// Hence, we should not collapse if the last and current operations are both power
6258	let is_left_associative =
6259	op.expr != Expr::Operator(Operator::Math(Math::Pow)) && op_prec <= last_prec;
6260
6261	while is_left_associative && expr_stack.len() > 1 {
6262	// Collapse the right associated operations first
6263	// so that we can get back to a stack with a lower precedence
6264	let mut rhs = expr_stack
6265	.pop()
6266	.expect("internal error: expression stack empty");
6267	let mut op = expr_stack
6268	.pop()
6269	.expect("internal error: expression stack empty");
6270
6271	last_prec = op.precedence();
6272
6273	if last_prec < op_prec {
6274	expr_stack.push(op);
6275	expr_stack.push(rhs);
6276	break;
6277	}
6278
6279	let mut lhs = expr_stack
6280	.pop()
6281	.expect("internal error: expression stack empty");
6282
6283	if let Some(row_var_id) = lhs_row_var_id {
6284	expand_to_cell_path(working_set, &mut lhs, row_var_id);
6285	}
6286
6287	let (result_ty, err) = math_result_type(working_set, &mut lhs, &mut op, &mut rhs);
6288	if let Some(err) = err {
6289	working_set.error(err);
6290	}
6291
6292	let op_span = Span::append(lhs.span, rhs.span);
6293	expr_stack.push(Expression::new(
6294	working_set,
6295	Expr::BinaryOp(Box::new(lhs), Box::new(op), Box::new(rhs)),
6296	op_span,
6297	result_ty,
6298	));
6299	}
6300	expr_stack.push(op);
6301	expr_stack.push(rhs);
6302
6303	last_prec = op_prec;
6304
6305	idx += 1;
6306	}
6307
6308	while expr_stack.len() != 1 {
6309	let mut rhs = expr_stack
6310	.pop()
6311	.expect("internal error: expression stack empty");
6312	let mut op = expr_stack
6313	.pop()
6314	.expect("internal error: expression stack empty");
6315	let mut lhs = expr_stack
6316	.pop()
6317	.expect("internal error: expression stack empty");
6318
6319	if let Some(row_var_id) = lhs_row_var_id {
6320	expand_to_cell_path(working_set, &mut lhs, row_var_id);
6321	}
6322
6323	let (result_ty, err) = math_result_type(working_set, &mut lhs, &mut op, &mut rhs);
6324	if let Some(err) = err {
6325	working_set.error(err)
6326	}
6327
6328	let binary_op_span = Span::append(lhs.span, rhs.span);
6329	expr_stack.push(Expression::new(
6330	working_set,
6331	Expr::BinaryOp(Box::new(lhs), Box::new(op), Box::new(rhs)),
6332	binary_op_span,
6333	result_ty,
6334	));
6335	}
6336
6337	expr_stack
6338	.pop()
6339	.expect("internal error: expression stack empty")
6340	}
6341
6342	pub fn parse_expression(working_set: &mut StateWorkingSet, spans: &[Span]) -> Expression {
6343	trace!("parsing: expression");
6344
6345	let mut pos = 0;
6346	let mut shorthand = vec![];
6347
6348	while pos < spans.len() {
6349	// Check if there is any environment shorthand
6350	let name = working_set.get_span_contents(spans[pos]);
6351
6352	let split: Vec<_> = name.splitn(2, \|x\| *x == b'=').collect();
6353	if split.len() != 2 \|\| !is_env_variable_name(split[0]) {
6354	break;
6355	}
6356
6357	let point = split[0].len() + 1;
6358	let starting_error_count = working_set.parse_errors.len();
6359
6360	let rhs = if spans[pos].start + point < spans[pos].end {
6361	let rhs_span = Span::new(spans[pos].start + point, spans[pos].end);
6362	if split[1].starts_with(b"$") {
6363	parse_dollar_expr(working_set, rhs_span)
6364	} else {
6365	parse_string_strict(working_set, rhs_span)
6366	}
6367	} else {
6368	Expression::new(
6369	working_set,
6370	Expr::String(String::new()),
6371	Span::unknown(),
6372	Type::Nothing,
6373	)
6374	};
6375
6376	let lhs_span = Span::new(spans[pos].start, spans[pos].start + point - 1);
6377	let lhs = parse_string_strict(working_set, lhs_span);
6378
6379	if starting_error_count == working_set.parse_errors.len() {
6380	shorthand.push((lhs, rhs));
6381	pos += 1;
6382	} else {
6383	working_set.parse_errors.truncate(starting_error_count);
6384	break;
6385	}
6386	}
6387
6388	if pos == spans.len() {
6389	working_set.error(ParseError::UnknownCommand(spans[0]));
6390	return garbage(working_set, Span::concat(spans));
6391	}
6392
6393	let output = if spans[pos..]
6394	.iter()
6395	.any(\|span\| is_assignment_operator(working_set.get_span_contents(*span)))
6396	{
6397	parse_assignment_expression(working_set, &spans[pos..])
6398	} else if is_math_expression_like(working_set, spans[pos]) {
6399	parse_math_expression(working_set, &spans[pos..], None)
6400	} else {
6401	let bytes = working_set.get_span_contents(spans[pos]).to_vec();
6402
6403	// For now, check for special parses of certain keywords
6404	match bytes.as_slice() {
6405	b"def" \| b"extern" \| b"for" \| b"module" \| b"use" \| b"source" \| b"alias" \| b"export"
6406	\| b"export-env" \| b"hide" => {
6407	working_set.error(ParseError::BuiltinCommandInPipeline(
6408	String::from_utf8(bytes)
6409	.expect("builtin commands bytes should be able to convert to string"),
6410	spans[0],
6411	));
6412
6413	parse_call(working_set, &spans[pos..], spans[0])
6414	}
6415	b"const" \| b"mut" => {
6416	working_set.error(ParseError::AssignInPipeline(
6417	String::from_utf8(bytes)
6418	.expect("builtin commands bytes should be able to convert to string"),
6419	String::from_utf8_lossy(match spans.len() {
6420	1..=3 => b"value",
6421	_ => working_set.get_span_contents(spans[3]),
6422	})
6423	.to_string(),
6424	String::from_utf8_lossy(match spans.len() {
6425	1 => b"variable",
6426	_ => working_set.get_span_contents(spans[1]),
6427	})
6428	.to_string(),
6429	spans[0],
6430	));
6431	parse_call(working_set, &spans[pos..], spans[0])
6432	}
6433	b"overlay" => {
6434	if spans.len() > 1 && working_set.get_span_contents(spans[1]) == b"list" {
6435	// whitelist 'overlay list'
6436	parse_call(working_set, &spans[pos..], spans[0])
6437	} else {
6438	working_set.error(ParseError::BuiltinCommandInPipeline(
6439	"overlay".into(),
6440	spans[0],
6441	));
6442
6443	parse_call(working_set, &spans[pos..], spans[0])
6444	}
6445	}
6446	b"where" => parse_where_expr(working_set, &spans[pos..]),
6447	#[cfg(feature = "plugin")]
6448	b"plugin" => {
6449	if spans.len() > 1 && working_set.get_span_contents(spans[1]) == b"use" {
6450	// only 'plugin use' is banned
6451	working_set.error(ParseError::BuiltinCommandInPipeline(
6452	"plugin use".into(),
6453	spans[0],
6454	));
6455	}
6456
6457	parse_call(working_set, &spans[pos..], spans[0])
6458	}
6459
6460	_ => parse_call(working_set, &spans[pos..], spans[0]),
6461	}
6462	};
6463
6464	if !shorthand.is_empty() {
6465	let with_env = working_set.find_decl(b"with-env");
6466	if let Some(decl_id) = with_env {
6467	let mut block = Block::default();
6468	let ty = output.ty.clone();
6469	block.pipelines = vec![Pipeline::from_vec(vec![output])];
6470	block.span = Some(Span::concat(spans));
6471
6472	compile_block(working_set, &mut block);
6473
6474	let block_id = working_set.add_block(Arc::new(block));
6475
6476	let mut env_vars = vec![];
6477	for sh in shorthand {
6478	env_vars.push(RecordItem::Pair(sh.0, sh.1));
6479	}
6480
6481	let arguments = vec![
6482	Argument::Positional(Expression::new(
6483	working_set,
6484	Expr::Record(env_vars),
6485	Span::concat(&spans[..pos]),
6486	Type::Any,
6487	)),
6488	Argument::Positional(Expression::new(
6489	working_set,
6490	Expr::Closure(block_id),
6491	Span::concat(&spans[pos..]),
6492	Type::Closure,
6493	)),
6494	];
6495
6496	let expr = Expr::Call(Box::new(Call {
6497	head: Span::unknown(),
6498	decl_id,
6499	arguments,
6500	parser_info: HashMap::new(),
6501	}));
6502
6503	Expression::new(working_set, expr, Span::concat(spans), ty)
6504	} else {
6505	output
6506	}
6507	} else {
6508	output
6509	}
6510	}
6511
6512	pub fn parse_builtin_commands(
6513	working_set: &mut StateWorkingSet,
6514	lite_command: &LiteCommand,
6515	) -> Pipeline {
6516	trace!("parsing: builtin commands");
6517	if !is_math_expression_like(working_set, lite_command.parts[0])
6518	&& !is_unaliasable_parser_keyword(working_set, &lite_command.parts)
6519	{
6520	trace!("parsing: not math expression or unaliasable parser keyword");
6521	let name = working_set.get_span_contents(lite_command.parts[0]);
6522	if let Some(decl_id) = working_set.find_decl(name) {
6523	let cmd = working_set.get_decl(decl_id);
6524	if cmd.is_alias() {
6525	// Parse keywords that can be aliased. Note that we check for "unaliasable" keywords
6526	// because alias can have any name, therefore, we can't check for "aliasable" keywords.
6527	let call_expr = parse_call(working_set, &lite_command.parts, lite_command.parts[0]);
6528
6529	if let Expression {
6530	expr: Expr::Call(call),
6531	..
6532	} = call_expr
6533	&& !call
6534	.parser_info
6535	.contains_key(PERCENT_FORCED_BUILTIN_PARSER_INFO)
6536	{
6537	// Apply parse keyword side effects
6538	let cmd = working_set.get_decl(call.decl_id);
6539	match cmd.name() {
6540	"overlay hide" => return parse_overlay_hide(working_set, call),
6541	"overlay new" => return parse_overlay_new(working_set, call),
6542	"overlay use" => return parse_overlay_use(working_set, call),
6543	_ => { /* this alias is not a parser keyword */ }
6544	}
6545	}
6546	}
6547	}
6548	}
6549
6550	trace!("parsing: checking for keywords");
6551	let name = lite_command
6552	.command_parts()
6553	.first()
6554	.map(\|s\| working_set.get_span_contents(*s))
6555	.unwrap_or(b"");
6556
6557	match name {
6558	// `parse_def` and `parse_extern` work both with and without attributes
6559	b"def" => parse_def(working_set, lite_command, None).0,
6560	b"extern" => parse_extern(working_set, lite_command, None),
6561	// `parse_export_in_block` also handles attributes by itself
6562	b"export" => parse_export_in_block(working_set, lite_command),
6563	b"export-env" => parse_export_env(working_set, &lite_command.parts).0,
6564	// Other definitions can't have attributes, so we handle attributes here with parse_attribute_block
6565	_ if lite_command.has_attributes() => parse_attribute_block(working_set, lite_command),
6566	b"let" => parse_let(
6567	working_set,
6568	&lite_command
6569	.parts_including_redirection()
6570	.collect::<Vec<Span>>(),
6571	),
6572	b"const" => parse_const(working_set, &lite_command.parts).0,
6573	b"mut" => parse_mut(
6574	working_set,
6575	&lite_command
6576	.parts_including_redirection()
6577	.collect::<Vec<Span>>(),
6578	),
6579	b"for" => {
6580	let expr = parse_for(working_set, lite_command);
6581	Pipeline::from_vec(vec![expr])
6582	}
6583	b"alias" => parse_alias(working_set, lite_command, None),
6584	b"module" => parse_module(working_set, lite_command, None).0,
6585	b"use" => parse_use(working_set, lite_command, None).0,
6586	b"overlay" => {
6587	if let Some(redirection) = lite_command.redirection.as_ref() {
6588	working_set.error(redirecting_builtin_error("overlay", redirection));
6589	return garbage_pipeline(working_set, &lite_command.parts);
6590	}
6591	parse_keyword(working_set, lite_command)
6592	}
6593	b"source" \| b"source-env" => parse_source(working_set, lite_command),
6594	b"hide" => parse_hide(working_set, lite_command),
6595	b"where" => parse_where(working_set, lite_command),
6596	// Only "plugin use" is a keyword
6597	#[cfg(feature = "plugin")]
6598	b"plugin"
6599	if lite_command
6600	.parts
6601	.get(1)
6602	.is_some_and(\|span\| working_set.get_span_contents(*span) == b"use") =>
6603	{
6604	if let Some(redirection) = lite_command.redirection.as_ref() {
6605	working_set.error(redirecting_builtin_error("plugin use", redirection));
6606	return garbage_pipeline(working_set, &lite_command.parts);
6607	}
6608	parse_keyword(working_set, lite_command)
6609	}
6610	_ => {
6611	let element = parse_pipeline_element(working_set, lite_command);
6612
6613	// There is still a chance to make `parse_pipeline_element` parse into
6614	// some keyword that should apply side effects first, Example:
6615	//
6616	// module a { export alias b = overlay use first.nu };
6617	// use a
6618	// a b
6619	//
6620	// In this case, `a b` will be parsed as a pipeline element, which leads
6621	// to the `overlay use` command.
6622	// In this case, we need to ensure that the side effects of these keywords
6623	// are applied.
6624	if let Expression {
6625	expr: Expr::Call(call),
6626	..
6627	} = &element.expr
6628	{
6629	// Dynamic percent dispatch stores a placeholder call plus parser
6630	// metadata for later IR rewrite. Skip parser-keyword side-effects lookup here,
6631	// because there is no declaration to resolve yet.
6632	if call
6633	.parser_info
6634	.contains_key(PERCENT_FORCED_BUILTIN_PARSER_INFO)
6635	{
6636	return Pipeline {
6637	elements: vec![element],
6638	};
6639	}
6640
6641	// Apply parse keyword side effects
6642	let cmd = working_set.get_decl(call.decl_id);
6643	match cmd.name() {
6644	"overlay hide" => return parse_overlay_hide(working_set, call.clone()),
6645	"overlay new" => return parse_overlay_new(working_set, call.clone()),
6646	"overlay use" => return parse_overlay_use(working_set, call.clone()),
6647	_ => { /* this alias is not a parser keyword */ }
6648	}
6649	}
6650	Pipeline {
6651	elements: vec![element],
6652	}
6653	}
6654	}
6655	}
6656
6657	fn check_record_key_or_value(
6658	working_set: &StateWorkingSet,
6659	expr: &Expression,
6660	position: &str,
6661	) -> Option<ParseError> {
6662	let bareword_error = \|string_value: &Expression\| {
6663	working_set
6664	.get_span_contents(string_value.span)
6665	.iter()
6666	.find_position(\|b\| **b == b':')
6667	.map(\|(i, _)\| {
6668	let colon_position = i + string_value.span.start;
6669	ParseError::InvalidLiteral(
6670	"colon".to_string(),
6671	format!("bare word specifying record {position}"),
6672	Span::new(colon_position, colon_position + 1),
6673	)
6674	})
6675	};
6676	let value_span = working_set.get_span_contents(expr.span);
6677	match expr.expr {
6678	Expr::String(_) => {
6679	if ![b'"', b'\'', b'`'].contains(&value_span[0]) {
6680	bareword_error(expr)
6681	} else {
6682	None
6683	}
6684	}
6685	Expr::StringInterpolation(ref expressions) => {
6686	if value_span[0] != b'$' {
6687	expressions
6688	.iter()
6689	.filter(\|expr\| matches!(expr.expr, Expr::String(_)))
6690	.filter_map(bareword_error)
6691	.next()
6692	} else {
6693	None
6694	}
6695	}
6696	_ => None,
6697	}
6698	}
6699
6700	pub fn parse_record(working_set: &mut StateWorkingSet, span: Span) -> Expression {
6701	let bytes = working_set.get_span_contents(span);
6702
6703	let mut start = span.start;
6704	let mut end = span.end;
6705
6706	if bytes.starts_with(b"{") {
6707	start += 1;
6708	} else {
6709	working_set.error(ParseError::Expected("{", Span::new(start, start + 1)));
6710	return garbage(working_set, span);
6711	}
6712
6713	let mut unclosed = false;
6714	let mut extra_tokens = false;
6715	if bytes.ends_with(b"}") {
6716	end -= 1;
6717	} else {
6718	unclosed = true;
6719	}
6720
6721	let inner_span = Span::new(start, end);
6722
6723	let mut lex_state = LexState {
6724	input: working_set.get_span_contents(inner_span),
6725	output: Vec::new(),
6726	error: None,
6727	span_offset: start,
6728	};
6729	while !lex_state.input.is_empty() {
6730	if let Some(ParseError::Unbalanced(left, right, _)) = lex_state.error.as_ref()
6731	&& left == "{"
6732	&& right == "}"
6733	{
6734	extra_tokens = true;
6735	unclosed = false;
6736	break;
6737	}
6738	let additional_whitespace = &[b'\n', b'\r', b','];
6739	if lex_n_tokens(&mut lex_state, additional_whitespace, &[b':'], true, 1) < 1 {
6740	break;
6741	};
6742	let span = lex_state
6743	.output
6744	.last()
6745	.expect("should have gotten 1 token")
6746	.span;
6747	let contents = working_set.get_span_contents(span);
6748	if contents.len() > 3
6749	&& contents.starts_with(b"...")
6750	&& (contents[3] == b'$' \|\| contents[3] == b'{' \|\| contents[3] == b'(')
6751	{
6752	// This was a spread operator, so there's no value
6753	continue;
6754	}
6755	// Get token for colon
6756	if lex_n_tokens(&mut lex_state, additional_whitespace, &[b':'], true, 1) < 1 {
6757	break;
6758	};
6759	// Get token for value
6760	if lex_n_tokens(&mut lex_state, additional_whitespace, &[], true, 1) < 1 {
6761	break;
6762	};
6763	}
6764	let (tokens, err) = (lex_state.output, lex_state.error);
6765
6766	if unclosed {
6767	working_set.error(ParseError::Unclosed("}".into(), Span::new(end, end)));
6768	} else if extra_tokens {
6769	working_set.error(ParseError::ExtraTokensAfterClosingDelimiter(Span::new(
6770	lex_state.span_offset,
6771	end,
6772	)));
6773	}
6774
6775	if let Some(err) = err {
6776	working_set.error(err);
6777	}
6778
6779	let mut output = vec![];
6780	let mut idx = 0;
6781
6782	let mut field_types = Some(vec![]);
6783	while idx < tokens.len() {
6784	let curr_span = tokens[idx].span;
6785	let curr_tok = working_set.get_span_contents(curr_span);
6786	if curr_tok.starts_with(b"...")
6787	&& curr_tok.len() > 3
6788	&& (curr_tok[3] == b'$' \|\| curr_tok[3] == b'{' \|\| curr_tok[3] == b'(')
6789	{
6790	// Parse spread operator
6791	let inner = parse_value(
6792	working_set,
6793	Span::new(curr_span.start + 3, curr_span.end),
6794	&SyntaxShape::Record(vec![]),
6795	);
6796	idx += 1;
6797
6798	match &inner.ty {
6799	Type::Record(inner_fields) => {
6800	if let Some(fields) = &mut field_types {
6801	for (field, ty) in inner_fields.as_ref() {
6802	fields.push((field.clone(), ty.clone()));
6803	}
6804	}
6805	}
6806	_ => {
6807	// We can't properly see all the field types
6808	// so fall back to the Any type later
6809	field_types = None;
6810	}
6811	}
6812	output.push(RecordItem::Spread(
6813	Span::new(curr_span.start, curr_span.start + 3),
6814	inner,
6815	));
6816	} else {
6817	// Normal key-value pair
6818	let field_token = &tokens[idx];
6819	let field = if field_token.contents != TokenContents::Item {
6820	working_set.error(ParseError::Expected(
6821	"item in record key position",
6822	Span::new(field_token.span.start, field_token.span.end),
6823	));
6824	garbage(working_set, curr_span)
6825	} else {
6826	let field = parse_value(working_set, curr_span, &SyntaxShape::String);
6827	if let Some(error) = check_record_key_or_value(working_set, &field, "key") {
6828	working_set.error(error);
6829	garbage(working_set, field.span)
6830	} else {
6831	field
6832	}
6833	};
6834
6835	idx += 1;
6836	if idx == tokens.len() {
6837	working_set.error(ParseError::Expected(
6838	"':'",
6839	Span::new(curr_span.end, curr_span.end),
6840	));
6841	output.push(RecordItem::Pair(
6842	garbage(working_set, curr_span),
6843	garbage(working_set, Span::new(curr_span.end, curr_span.end)),
6844	));
6845	break;
6846	}
6847	let colon_span = tokens[idx].span;
6848	let colon = working_set.get_span_contents(colon_span);
6849	idx += 1;
6850	if colon != b":" {
6851	working_set.error(ParseError::Expected(
6852	"':'",
6853	Span::new(colon_span.start, colon_span.start),
6854	));
6855	output.push(RecordItem::Pair(
6856	field,
6857	garbage(
6858	working_set,
6859	Span::new(colon_span.start, tokens[tokens.len() - 1].span.end),
6860	),
6861	));
6862	break;
6863	}
6864	if idx == tokens.len() {
6865	working_set.error(ParseError::Expected(
6866	"value for record field",
6867	Span::new(colon_span.end, colon_span.end),
6868	));
6869	output.push(RecordItem::Pair(
6870	garbage(working_set, Span::new(curr_span.start, colon_span.end)),
6871	garbage(
6872	working_set,
6873	Span::new(colon_span.end, tokens[tokens.len() - 1].span.end),
6874	),
6875	));
6876	break;
6877	}
6878
6879	let value_token = &tokens[idx];
6880	let value = if value_token.contents != TokenContents::Item {
6881	working_set.error(ParseError::Expected(
6882	"item in record value position",
6883	Span::new(value_token.span.start, value_token.span.end),
6884	));
6885	garbage(
6886	working_set,
6887	Span::new(value_token.span.start, value_token.span.end),
6888	)
6889	} else {
6890	let value = parse_value(working_set, tokens[idx].span, &SyntaxShape::Any);
6891	if let Some(parse_error) = check_record_key_or_value(working_set, &value, "value") {
6892	working_set.error(parse_error);
6893	garbage(working_set, value.span)
6894	} else {
6895	value
6896	}
6897	};
6898	idx += 1;
6899
6900	if let Some(field) = field.as_string() {
6901	if let Some(fields) = &mut field_types {
6902	fields.push((field, value.ty.clone()));
6903	}
6904	} else {
6905	// We can't properly see all the field types
6906	// so fall back to the Any type later
6907	field_types = None;
6908	}
6909	output.push(RecordItem::Pair(field, value));
6910	}
6911	}
6912
6913	Expression::new(
6914	working_set,
6915	Expr::Record(output),
6916	span,
6917	if let Some(fields) = field_types {
6918	Type::Record(fields.into())
6919	} else {
6920	Type::Any
6921	},
6922	)
6923	}
6924
6925	fn parse_redirection_target(
6926	working_set: &mut StateWorkingSet,
6927	target: &LiteRedirectionTarget,
6928	) -> RedirectionTarget {
6929	match target {
6930	LiteRedirectionTarget::File {
6931	connector,
6932	file,
6933	append,
6934	} => RedirectionTarget::File {
6935	expr: parse_value(working_set, *file, &SyntaxShape::Any),
6936	append: *append,
6937	span: *connector,
6938	},
6939	LiteRedirectionTarget::Pipe { connector } => RedirectionTarget::Pipe { span: *connector },
6940	}
6941	}
6942
6943	pub(crate) fn parse_redirection(
6944	working_set: &mut StateWorkingSet,
6945	target: &LiteRedirection,
6946	) -> PipelineRedirection {
6947	match target {
6948	LiteRedirection::Single { source, target } => PipelineRedirection::Single {
6949	source: *source,
6950	target: parse_redirection_target(working_set, target),
6951	},
6952	LiteRedirection::Separate { out, err } => PipelineRedirection::Separate {
6953	out: parse_redirection_target(working_set, out),
6954	err: parse_redirection_target(working_set, err),
6955	},
6956	}
6957	}
6958
6959	fn parse_pipeline_element(
6960	working_set: &mut StateWorkingSet,
6961	command: &LiteCommand,
6962	) -> PipelineElement {
6963	trace!("parsing: pipeline element");
6964
6965	let expr = parse_expression(working_set, &command.parts);
6966
6967	let redirection = command
6968	.redirection
6969	.as_ref()
6970	.map(\|r\| parse_redirection(working_set, r));
6971
6972	PipelineElement {
6973	pipe: command.pipe,
6974	expr,
6975	redirection,
6976	}
6977	}
6978
6979	pub(crate) fn redirecting_builtin_error(
6980	name: &'static str,
6981	redirection: &LiteRedirection,
6982	) -> ParseError {
6983	match redirection {
6984	LiteRedirection::Single { target, .. } => {
6985	ParseError::RedirectingBuiltinCommand(name, target.connector(), None)
6986	}
6987	LiteRedirection::Separate { out, err } => ParseError::RedirectingBuiltinCommand(
6988	name,
6989	out.connector().min(err.connector()),
6990	Some(out.connector().max(err.connector())),
6991	),
6992	}
6993	}
6994
6995	pub fn parse_pipeline(working_set: &mut StateWorkingSet, pipeline: &LitePipeline) -> Pipeline {
6996	if pipeline.commands.len() > 1 {
6997	// Parse a normal multi command pipeline
6998	let elements: Vec<_> = pipeline
6999	.commands
7000	.iter()
7001	.enumerate()
7002	.map(\|(index, element)\| {
7003	let element = parse_pipeline_element(working_set, element);
7004	// Handle $in for pipeline elements beyond the first one
7005	if index > 0 && element.has_in_variable(working_set) {
7006	wrap_element_with_collect(working_set, element.clone())
7007	} else {
7008	element
7009	}
7010	})
7011	.collect();
7012
7013	Pipeline { elements }
7014	} else {
7015	// If there's only one command in the pipeline, this could be a builtin command
7016	parse_builtin_commands(working_set, &pipeline.commands[0])
7017	}
7018	}
7019
7020	pub fn parse_block(
7021	working_set: &mut StateWorkingSet,
7022	tokens: &[Token],
7023	span: Span,
7024	scoped: bool,
7025	is_subexpression: bool,
7026	) -> Block {
7027	let (lite_block, err) = lite_parse(tokens, working_set);
7028	if let Some(err) = err {
7029	working_set.error(err);
7030	}
7031
7032	trace!("parsing block: {lite_block:?}");
7033
7034	if scoped {
7035	working_set.enter_scope();
7036	}
7037
7038	// Pre-declare any definition so that definitions
7039	// that share the same block can see each other
7040	for pipeline in &lite_block.block {
7041	if pipeline.commands.len() == 1 {
7042	parse_def_predecl(working_set, pipeline.commands[0].command_parts())
7043	}
7044	}
7045
7046	let mut block = Block::new_with_capacity(lite_block.block.len());
7047	block.span = Some(span);
7048
7049	for lite_pipeline in &lite_block.block {
7050	let pipeline = parse_pipeline(working_set, lite_pipeline);
7051	block.pipelines.push(pipeline);
7052	}
7053
7054	// If this is not a subexpression and there are any pipelines where the first element has $in,
7055	// we can wrap the whole block in collect so that they all reference the same $in
7056	if !is_subexpression
7057	&& block
7058	.pipelines
7059	.iter()
7060	.flat_map(\|pipeline\| pipeline.elements.first())
7061	.any(\|element\| element.has_in_variable(working_set))
7062	{
7063	// Move the block out to prepare it to become a subexpression
7064	let inner_block = std::mem::take(&mut block);
7065	block.span = inner_block.span;
7066	let ty = inner_block.output_type();
7067	let block_id = working_set.add_block(Arc::new(inner_block));
7068
7069	// Now wrap it in a Collect expression, and put it in the block as the only pipeline
7070	let subexpression = Expression::new(working_set, Expr::Subexpression(block_id), span, ty);
7071	let collect = wrap_expr_with_collect(working_set, subexpression);
7072
7073	block.pipelines.push(Pipeline {
7074	elements: vec![PipelineElement {
7075	pipe: None,
7076	expr: collect,
7077	redirection: None,
7078	}],
7079	});
7080	}
7081
7082	if scoped {
7083	working_set.exit_scope();
7084	}
7085
7086	let errors = type_check::check_block_input_output(working_set, &block);
7087	if !errors.is_empty() {
7088	working_set.parse_errors.extend_from_slice(&errors);
7089	}
7090
7091	block
7092	}
7093
7094	/// Compile an [IrBlock][nu_protocol::ir::IrBlock] for the [Block], adding a compile error on
7095	/// failure.
7096	///
7097	/// To compile a block that's already in the [StateWorkingSet] use [compile_block_with_id]
7098	pub fn compile_block(working_set: &mut StateWorkingSet<'_>, block: &mut Block) {
7099	if !working_set.parse_errors.is_empty() {
7100	// This means there might be a bug in the parser, since calling this function while parse
7101	// errors are present is a logic error. However, it's not fatal and it's best to continue
7102	// without doing anything.
7103	log::error!("compile_block called with parse errors");
7104	return;
7105	}
7106
7107	match nu_engine::compile(working_set, block) {
7108	Ok(ir_block) => {
7109	block.ir_block = Some(ir_block);
7110	}
7111	Err(err) => working_set.compile_errors.push(err),
7112	}
7113	}
7114
7115	/// Compile an [IrBlock][nu_protocol::ir::IrBlock] for a [Block] that's already in the
7116	/// [StateWorkingSet] using its id, adding a compile error on failure.
7117	pub fn compile_block_with_id(working_set: &mut StateWorkingSet<'_>, block_id: BlockId) {
7118	if !working_set.parse_errors.is_empty() {
7119	// This means there might be a bug in the parser, since calling this function while parse
7120	// errors are present is a logic error. However, it's not fatal and it's best to continue
7121	// without doing anything.
7122	log::error!("compile_block_with_id called with parse errors");
7123	return;
7124	}
7125
7126	match nu_engine::compile(working_set, working_set.get_block(block_id)) {
7127	Ok(ir_block) => {
7128	working_set.get_block_mut(block_id).ir_block = Some(ir_block);
7129	}
7130	Err(err) => {
7131	working_set.compile_errors.push(err);
7132	}
7133	};
7134	}
7135
7136	pub fn discover_captures_in_closure(
7137	working_set: &StateWorkingSet,
7138	block: &Block,
7139	seen: &mut Vec<VarId>,
7140	seen_blocks: &mut HashMap<BlockId, Vec<(VarId, Span)>>,
7141	output: &mut Vec<(VarId, Span)>,
7142	) -> Result<(), ParseError> {
7143	for flag in &block.signature.named {
7144	if let Some(var_id) = flag.var_id {
7145	seen.push(var_id);
7146	}
7147	}
7148
7149	for positional in &block.signature.required_positional {
7150	if let Some(var_id) = positional.var_id {
7151	seen.push(var_id);
7152	}
7153	}
7154	for positional in &block.signature.optional_positional {
7155	if let Some(var_id) = positional.var_id {
7156	seen.push(var_id);
7157	}
7158	}
7159	if let Some(positional) = &block.signature.rest_positional
7160	&& let Some(var_id) = positional.var_id
7161	{
7162	seen.push(var_id);
7163	}
7164
7165	for pipeline in &block.pipelines {
7166	discover_captures_in_pipeline(working_set, pipeline, seen, seen_blocks, output)?;
7167	}
7168
7169	Ok(())
7170	}
7171
7172	fn discover_captures_in_pipeline(
7173	working_set: &StateWorkingSet,
7174	pipeline: &Pipeline,
7175	seen: &mut Vec<VarId>,
7176	seen_blocks: &mut HashMap<BlockId, Vec<(VarId, Span)>>,
7177	output: &mut Vec<(VarId, Span)>,
7178	) -> Result<(), ParseError> {
7179	for element in &pipeline.elements {
7180	discover_captures_in_pipeline_element(working_set, element, seen, seen_blocks, output)?;
7181	}
7182
7183	Ok(())
7184	}
7185
7186	// Closes over captured variables
7187	pub fn discover_captures_in_pipeline_element(
7188	working_set: &StateWorkingSet,
7189	element: &PipelineElement,
7190	seen: &mut Vec<VarId>,
7191	seen_blocks: &mut HashMap<BlockId, Vec<(VarId, Span)>>,
7192	output: &mut Vec<(VarId, Span)>,
7193	) -> Result<(), ParseError> {
7194	discover_captures_in_expr(working_set, &element.expr, seen, seen_blocks, output)?;
7195
7196	if let Some(redirection) = element.redirection.as_ref() {
7197	match redirection {
7198	PipelineRedirection::Single { target, .. } => {
7199	if let Some(expr) = target.expr() {
7200	discover_captures_in_expr(working_set, expr, seen, seen_blocks, output)?;
7201	}
7202	}
7203	PipelineRedirection::Separate { out, err } => {
7204	if let Some(expr) = out.expr() {
7205	discover_captures_in_expr(working_set, expr, seen, seen_blocks, output)?;
7206	}
7207	if let Some(expr) = err.expr() {
7208	discover_captures_in_expr(working_set, expr, seen, seen_blocks, output)?;
7209	}
7210	}
7211	}
7212	}
7213
7214	Ok(())
7215	}
7216
7217	pub fn discover_captures_in_pattern(pattern: &MatchPattern, seen: &mut Vec<VarId>) {
7218	match &pattern.pattern {
7219	Pattern::Variable(var_id) => seen.push(*var_id),
7220	Pattern::List(items) => {
7221	for item in items {
7222	discover_captures_in_pattern(item, seen)
7223	}
7224	}
7225	Pattern::Record(items) => {
7226	for item in items {
7227	discover_captures_in_pattern(&item.1, seen)
7228	}
7229	}
7230	Pattern::Or(patterns) => {
7231	for pattern in patterns {
7232	discover_captures_in_pattern(pattern, seen)
7233	}
7234	}
7235	Pattern::Rest(var_id) => seen.push(*var_id),
7236	Pattern::Expression(_)
7237	\| Pattern::Value(_)
7238	\| Pattern::IgnoreValue
7239	\| Pattern::IgnoreRest
7240	\| Pattern::Garbage => {}
7241	}
7242	}
7243
7244	// Closes over captured variables
7245	pub fn discover_captures_in_expr(
7246	working_set: &StateWorkingSet,
7247	expr: &Expression,
7248	seen: &mut Vec<VarId>,
7249	seen_blocks: &mut HashMap<BlockId, Vec<(VarId, Span)>>,
7250	output: &mut Vec<(VarId, Span)>,
7251	) -> Result<(), ParseError> {
7252	match &expr.expr {
7253	Expr::AttributeBlock(ab) => {
7254	discover_captures_in_expr(working_set, &ab.item, seen, seen_blocks, output)?;
7255	}
7256	Expr::BinaryOp(lhs, _, rhs) => {
7257	discover_captures_in_expr(working_set, lhs, seen, seen_blocks, output)?;
7258	discover_captures_in_expr(working_set, rhs, seen, seen_blocks, output)?;
7259	}
7260	Expr::UnaryNot(expr) => {
7261	discover_captures_in_expr(working_set, expr, seen, seen_blocks, output)?;
7262	}
7263	Expr::Closure(block_id) => {
7264	let block = working_set.get_block(*block_id);
7265	let results = {
7266	let mut seen = vec![];
7267	let mut results = vec![];
7268
7269	discover_captures_in_closure(
7270	working_set,
7271	block,
7272	&mut seen,
7273	seen_blocks,
7274	&mut results,
7275	)?;
7276
7277	for (var_id, span) in results.iter() {
7278	if !seen.contains(var_id)
7279	&& let Some(variable) = working_set.get_variable_if_possible(*var_id)
7280	&& variable.mutable
7281	{
7282	return Err(ParseError::CaptureOfMutableVar(*span));
7283	}
7284	}
7285
7286	results
7287	};
7288	seen_blocks.insert(*block_id, results.clone());
7289	for (var_id, span) in results.into_iter() {
7290	if !seen.contains(&var_id) {
7291	output.push((var_id, span))
7292	}
7293	}
7294	}
7295	Expr::Block(block_id) => {
7296	let block = working_set.get_block(*block_id);
7297	// FIXME: is this correct?
7298	let results = {
7299	let mut seen = vec![];
7300	let mut results = vec![];
7301	discover_captures_in_closure(
7302	working_set,
7303	block,
7304	&mut seen,
7305	seen_blocks,
7306	&mut results,
7307	)?;
7308	results
7309	};
7310
7311	seen_blocks.insert(*block_id, results.clone());
7312	for (var_id, span) in results.into_iter() {
7313	if !seen.contains(&var_id) {
7314	output.push((var_id, span))
7315	}
7316	}
7317	}
7318	Expr::Binary(_) => {}
7319	Expr::Bool(_) => {}
7320	Expr::Call(call) => {
7321	if let Some(head_expr) = call.parser_info.get(PERCENT_FORCED_BUILTIN_PARSER_INFO) {
7322	discover_captures_in_expr(working_set, head_expr, seen, seen_blocks, output)?;
7323	} else {
7324	let decl = working_set.get_decl(call.decl_id);
7325	if let Some(block_id) = decl.block_id() {
7326	match seen_blocks.get(&block_id) {
7327	Some(capture_list) => {
7328	// Push captures onto the outer closure that aren't created by that outer closure
7329	for capture in capture_list {
7330	if !seen.contains(&capture.0) {
7331	output.push(*capture);
7332	}
7333	}
7334	}
7335	None => {
7336	let block = working_set.get_block(block_id);
7337	if !block.captures.is_empty() {
7338	for (capture, span) in &block.captures {
7339	if !seen.contains(capture) {
7340	output.push((capture, span));
7341	}
7342	}
7343	} else {
7344	let result = {
7345	let mut seen = vec![];
7346	seen_blocks.insert(block_id, vec![]);
7347
7348	let mut result = vec![];
7349	discover_captures_in_closure(
7350	working_set,
7351	block,
7352	&mut seen,
7353	seen_blocks,
7354	&mut result,
7355	)?;
7356
7357	result
7358	};
7359	// Push captures onto the outer closure that aren't created by that outer closure
7360	for capture in &result {
7361	if !seen.contains(&capture.0) {
7362	output.push(*capture);
7363	}
7364	}
7365
7366	seen_blocks.insert(block_id, result);
7367	}
7368	}
7369	}
7370	}
7371	}
7372
7373	for arg in &call.arguments {
7374	match arg {
7375	Argument::Named(named) => {
7376	if let Some(arg) = &named.2 {
7377	discover_captures_in_expr(working_set, arg, seen, seen_blocks, output)?;
7378	}
7379	}
7380	Argument::Positional(expr)
7381	\| Argument::Unknown(expr)
7382	\| Argument::Spread(expr) => {
7383	discover_captures_in_expr(working_set, expr, seen, seen_blocks, output)?;
7384	}
7385	}
7386	}
7387	}
7388	Expr::CellPath(_) => {}
7389	Expr::DateTime(_) => {}
7390	Expr::ExternalCall(head, args) => {
7391	discover_captures_in_expr(working_set, head, seen, seen_blocks, output)?;
7392
7393	for ExternalArgument::Regular(expr) \| ExternalArgument::Spread(expr) in args.as_ref() {
7394	discover_captures_in_expr(working_set, expr, seen, seen_blocks, output)?;
7395	}
7396	}
7397	Expr::Filepath(_, _) => {}
7398	Expr::Directory(_, _) => {}
7399	Expr::Float(_) => {}
7400	Expr::FullCellPath(cell_path) => {
7401	discover_captures_in_expr(working_set, &cell_path.head, seen, seen_blocks, output)?;
7402	}
7403	Expr::ImportPattern(_) => {}
7404	Expr::Overlay(_) => {}
7405	Expr::Garbage => {}
7406	Expr::Nothing => {}
7407	Expr::GlobPattern(_, _) => {}
7408	Expr::Int(_) => {}
7409	Expr::Keyword(kw) => {
7410	discover_captures_in_expr(working_set, &kw.expr, seen, seen_blocks, output)?;
7411	}
7412	Expr::List(list) => {
7413	for item in list {
7414	discover_captures_in_expr(working_set, item.expr(), seen, seen_blocks, output)?;
7415	}
7416	}
7417	Expr::Operator(_) => {}
7418	Expr::Range(range) => {
7419	if let Some(from) = &range.from {
7420	discover_captures_in_expr(working_set, from, seen, seen_blocks, output)?;
7421	}
7422	if let Some(next) = &range.next {
7423	discover_captures_in_expr(working_set, next, seen, seen_blocks, output)?;
7424	}
7425	if let Some(to) = &range.to {
7426	discover_captures_in_expr(working_set, to, seen, seen_blocks, output)?;
7427	}
7428	}
7429	Expr::Record(items) => {
7430	for item in items {
7431	match item {
7432	RecordItem::Pair(field_name, field_value) => {
7433	discover_captures_in_expr(
7434	working_set,
7435	field_name,
7436	seen,
7437	seen_blocks,
7438	output,
7439	)?;
7440	discover_captures_in_expr(
7441	working_set,
7442	field_value,
7443	seen,
7444	seen_blocks,
7445	output,
7446	)?;
7447	}
7448	RecordItem::Spread(_, record) => {
7449	discover_captures_in_expr(working_set, record, seen, seen_blocks, output)?;
7450	}
7451	}
7452	}
7453	}
7454	Expr::Signature(sig) => {
7455	// Something with a declaration, similar to a var decl, will introduce more VarIds into the stack at eval
7456	for pos in &sig.required_positional {
7457	if let Some(var_id) = pos.var_id {
7458	seen.push(var_id);
7459	}
7460	}
7461	for pos in &sig.optional_positional {
7462	if let Some(var_id) = pos.var_id {
7463	seen.push(var_id);
7464	}
7465	}
7466	if let Some(rest) = &sig.rest_positional
7467	&& let Some(var_id) = rest.var_id
7468	{
7469	seen.push(var_id);
7470	}
7471	for named in &sig.named {
7472	if let Some(var_id) = named.var_id {
7473	seen.push(var_id);
7474	}
7475	}
7476	}
7477	Expr::String(_) => {}
7478	Expr::RawString(_) => {}
7479	Expr::StringInterpolation(exprs) \| Expr::GlobInterpolation(exprs, _) => {
7480	for expr in exprs {
7481	discover_captures_in_expr(working_set, expr, seen, seen_blocks, output)?;
7482	}
7483	}
7484	Expr::MatchBlock(match_block) => {
7485	for match_ in match_block {
7486	discover_captures_in_pattern(&match_.0, seen);
7487	discover_captures_in_expr(working_set, &match_.1, seen, seen_blocks, output)?;
7488	}
7489	}
7490	Expr::Collect(var_id, expr) => {
7491	seen.push(*var_id);
7492	discover_captures_in_expr(working_set, expr, seen, seen_blocks, output)?
7493	}
7494	Expr::RowCondition(block_id) \| Expr::Subexpression(block_id) => {
7495	let block = working_set.get_block(*block_id);
7496
7497	let results = {
7498	let mut results = vec![];
7499	let mut seen = vec![];
7500	discover_captures_in_closure(
7501	working_set,
7502	block,
7503	&mut seen,
7504	seen_blocks,
7505	&mut results,
7506	)?;
7507	results
7508	};
7509
7510	seen_blocks.insert(*block_id, results.clone());
7511	for (var_id, span) in results.into_iter() {
7512	if !seen.contains(&var_id) {
7513	output.push((var_id, span))
7514	}
7515	}
7516	}
7517	Expr::Table(table) => {
7518	for header in table.columns.as_ref() {
7519	discover_captures_in_expr(working_set, header, seen, seen_blocks, output)?;
7520	}
7521	for row in table.rows.as_ref() {
7522	for cell in row.as_ref() {
7523	discover_captures_in_expr(working_set, cell, seen, seen_blocks, output)?;
7524	}
7525	}
7526	}
7527	Expr::ValueWithUnit(value) => {
7528	discover_captures_in_expr(working_set, &value.expr, seen, seen_blocks, output)?;
7529	}
7530	Expr::Var(var_id) => {
7531	if (var_id > ENV_VARIABLE_ID \|\| var_id == IN_VARIABLE_ID) && !seen.contains(var_id) {
7532	output.push((*var_id, expr.span));
7533	}
7534	}
7535	Expr::VarDecl(var_id) => {
7536	seen.push(*var_id);
7537	}
7538	}
7539	Ok(())
7540	}
7541
7542	fn wrap_redirection_with_collect(
7543	working_set: &mut StateWorkingSet,
7544	target: RedirectionTarget,
7545	) -> RedirectionTarget {
7546	match target {
7547	RedirectionTarget::File { expr, append, span } => RedirectionTarget::File {
7548	expr: wrap_expr_with_collect(working_set, expr),
7549	span,
7550	append,
7551	},
7552	RedirectionTarget::Pipe { span } => RedirectionTarget::Pipe { span },
7553	}
7554	}
7555
7556	fn wrap_element_with_collect(
7557	working_set: &mut StateWorkingSet,
7558	element: PipelineElement,
7559	) -> PipelineElement {
7560	PipelineElement {
7561	pipe: element.pipe,
7562	expr: wrap_expr_with_collect(working_set, element.expr),
7563	redirection: element.redirection.map(\|r\| match r {
7564	PipelineRedirection::Single { source, target } => PipelineRedirection::Single {
7565	source,
7566	target: wrap_redirection_with_collect(working_set, target),
7567	},
7568	PipelineRedirection::Separate { out, err } => PipelineRedirection::Separate {
7569	out: wrap_redirection_with_collect(working_set, out),
7570	err: wrap_redirection_with_collect(working_set, err),
7571	},
7572	}),
7573	}
7574	}
7575
7576	fn wrap_expr_with_collect(working_set: &mut StateWorkingSet, expr: Expression) -> Expression {
7577	let span = expr.span;
7578
7579	// IN_VARIABLE_ID should get replaced with a unique variable, so that we don't have to
7580	// execute as a closure
7581	let var_id = working_set.add_variable(
7582	b"$in".into(),
7583	Span::new(span.start, span.start),
7584	Type::Any,
7585	false,
7586	);
7587	let mut expr = expr.clone();
7588	expr.replace_in_variable(working_set, var_id);
7589
7590	// Bind the custom `$in` variable for that particular expression
7591	let ty = expr.ty.clone();
7592	Expression::new(
7593	working_set,
7594	Expr::Collect(var_id, Box::new(expr)),
7595	span,
7596	// We can expect it to have the same result type
7597	ty,
7598	)
7599	}
7600
7601	// Parses a vector of u8 to create an AST Block. If a file name is given, then
7602	// the name is stored in the working set. When parsing a source without a file
7603	// name, the source of bytes is stored as "source"
7604	pub fn parse(
7605	working_set: &mut StateWorkingSet,
7606	fname: Option<&str>,
7607	contents: &[u8],
7608	scoped: bool,
7609	) -> Arc<Block> {
7610	trace!("parse");
7611
7612	let file_id = {
7613	let fname = fname.map(nu_path::expand_to_real_path);
7614	let fname = fname.as_deref().map(\|p\| p.to_string_lossy());
7615	let name = fname.as_deref().unwrap_or("source");
7616	working_set.add_file(name, contents)
7617	};
7618
7619	let new_span = working_set.get_span_for_file(file_id);
7620
7621	let previously_parsed_block = working_set.find_block_by_span(new_span);
7622
7623	let mut output = {
7624	if let Some(block) = previously_parsed_block {
7625	return block;
7626	} else {
7627	let (output, err) = lex(contents, new_span.start, &[], &[], false);
7628	if let Some(err) = err {
7629	working_set.error(err)
7630	}
7631
7632	Arc::new(parse_block(working_set, &output, new_span, scoped, false))
7633	}
7634	};
7635
7636	// Top level `Block`s are compiled eagerly, as they don't have a parent which would cause them
7637	// to be compiled later.
7638	if working_set.parse_errors.is_empty() {
7639	compile_block(working_set, Arc::make_mut(&mut output));
7640	}
7641
7642	let mut seen = vec![];
7643	let mut seen_blocks = HashMap::new();
7644
7645	let mut captures = vec![];
7646	match discover_captures_in_closure(
7647	working_set,
7648	&output,
7649	&mut seen,
7650	&mut seen_blocks,
7651	&mut captures,
7652	) {
7653	Ok(_) => {
7654	Arc::make_mut(&mut output).captures = captures;
7655	}
7656	Err(err) => working_set.error(err),
7657	}
7658
7659	// Also check other blocks that might have been imported
7660	let mut errors = vec![];
7661	for (block_idx, block) in working_set.delta.blocks.iter().enumerate() {
7662	let block_id = block_idx + working_set.permanent_state.num_blocks();
7663	let block_id = BlockId::new(block_id);
7664
7665	if !seen_blocks.contains_key(&block_id) {
7666	let mut captures = vec![];
7667
7668	match discover_captures_in_closure(
7669	working_set,
7670	block,
7671	&mut seen,
7672	&mut seen_blocks,
7673	&mut captures,
7674	) {
7675	Ok(_) => {
7676	seen_blocks.insert(block_id, captures);
7677	}
7678	Err(err) => {
7679	errors.push(err);
7680	}
7681	}
7682	}
7683	}
7684	for err in errors {
7685	working_set.error(err)
7686	}
7687
7688	for (block_id, captures) in seen_blocks.into_iter() {
7689	// In theory, we should only be updating captures where we have new information
7690	// the only place where this is possible would be blocks that are newly created
7691	// by our working set delta. If we ever tried to modify the permanent state, we'd
7692	// panic (again, in theory, this shouldn't be possible)
7693	let block = working_set.get_block(block_id);
7694	let block_captures_empty = block.captures.is_empty();
7695	// need to check block_id >= working_set.permanent_state.num_blocks()
7696	// to avoid mutate a block that is in the permanent state.
7697	// this can happened if user defines a function with recursive call
7698	// and pipe a variable to the command, e.g:
7699	// def px [] { if true { 42 } else { px } }; # the block px is saved in permanent state.
7700	// let x = 3
7701	// $x \| px
7702	// If we don't guard for `block_id`, it will change captures of `px`, which is
7703	// already saved in permanent state
7704	if !captures.is_empty()
7705	&& block_captures_empty
7706	&& block_id.get() >= working_set.permanent_state.num_blocks()
7707	{
7708	let block = working_set.get_block_mut(block_id);
7709	block.captures = captures;
7710	}
7711	}
7712
7713	output
7714	}