arcayr/saphyr-serde
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Format with rustfmt 0.99.4

Browse source
This commit is contained in:
David Tolnay 2018-09-15 09:49:04 -07:00
parent 124d237be0
commit 5039af6862
8 changed files with 733 additions and 584 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
parser/examples/dump_yaml.rs
View file

@ -17,14 +17,14 @@ fn dump_node(doc: &yaml::Yaml, indent: usize) {
for x in v { for x in v {
dump_node(x, indent + 1); dump_node(x, indent + 1);
} }
}, }
yaml::Yaml::Hash(ref h) => { yaml::Yaml::Hash(ref h) => {
for (k, v) in h { for (k, v) in h {
print_indent(indent); print_indent(indent);
println!("{:?}:", k); println!("{:?}:", k);
dump_node(v, indent + 1); dump_node(v, indent + 1);
} }
}, }
_ => { _ => {
print_indent(indent); print_indent(indent);
println!("{:?}", doc); println!("{:?}", doc);

92
parser/src/emitter.rs
View file

@ -1,9 +1,8 @@
use std::fmt::{self, Display};
use std::convert::From; use std::convert::From;
use std::error::Error; use std::error::Error;
use std::fmt::{self, Display};
use yaml::{Hash, Yaml}; use yaml::{Hash, Yaml};
#[derive(Copy, Clone, Debug)] #[derive(Copy, Clone, Debug)]
pub enum EmitError { pub enum EmitError {
FmtError(fmt::Error), FmtError(fmt::Error),
@ -91,7 +90,7 @@ fn escape_str(wr: &mut fmt::Write, v: &str) -> Result<(), fmt::Error> {
b'\x1e' => "\\u001e", b'\x1e' => "\\u001e",
b'\x1f' => "\\u001f", b'\x1f' => "\\u001f",
b'\x7f' => "\\u007f", b'\x7f' => "\\u007f",
_ => { continue; } _ => continue,
}; };
if start < i { if start < i {
@ -118,7 +117,7 @@ impl<'a> YamlEmitter<'a> {
best_indent: 2, best_indent: 2,
compact: true, compact: true,
level: -1 level: -1,
} }
} }
@ -147,7 +146,9 @@ impl<'a> YamlEmitter<'a> {
} }
fn write_indent(&mut self) -> EmitResult { fn write_indent(&mut self) -> EmitResult {
if self.level <= 0 { return Ok(()); } if self.level <= 0 {
return Ok(());
}
for _ in 0..self.level { for _ in 0..self.level {
for _ in 0..self.best_indent { for _ in 0..self.best_indent {
try!(write!(self.writer, " ")); try!(write!(self.writer, " "));
@ -163,12 +164,11 @@ impl<'a> YamlEmitter<'a> {
Yaml::String(ref v) => { Yaml::String(ref v) => {
if need_quotes(v) { if need_quotes(v) {
try!(escape_str(self.writer, v)); try!(escape_str(self.writer, v));
} } else {
else {
try!(write!(self.writer, "{}", v)); try!(write!(self.writer, "{}", v));
} }
Ok(()) Ok(())
}, }
Yaml::Boolean(v) => { Yaml::Boolean(v) => {
if v { if v {
try!(self.writer.write_str("true")); try!(self.writer.write_str("true"));
@ -176,21 +176,21 @@ impl<'a> YamlEmitter<'a> {
try!(self.writer.write_str("false")); try!(self.writer.write_str("false"));
} }
Ok(()) Ok(())
}, }
Yaml::Integer(v) => { Yaml::Integer(v) => {
try!(write!(self.writer, "{}", v)); try!(write!(self.writer, "{}", v));
Ok(()) Ok(())
}, }
Yaml::Real(ref v) => { Yaml::Real(ref v) => {
try!(write!(self.writer, "{}", v)); try!(write!(self.writer, "{}", v));
Ok(()) Ok(())
}, }
Yaml::Null | Yaml::BadValue => { Yaml::Null | Yaml::BadValue => {
try!(write!(self.writer, "~")); try!(write!(self.writer, "~"));
Ok(()) Ok(())
}, }
// XXX(chenyh) Alias // XXX(chenyh) Alias
_ => { Ok(()) } _ => Ok(()),
} }
} }
@ -260,7 +260,7 @@ impl<'a> YamlEmitter<'a> {
self.level -= 1; self.level -= 1;
} }
self.emit_array(v) self.emit_array(v)
}, }
Yaml::Hash(ref h) => { Yaml::Hash(ref h) => {
if (inline && self.compact) || h.is_empty() { if (inline && self.compact) || h.is_empty() {
try!(write!(self.writer, " ")); try!(write!(self.writer, " "));
@ -271,7 +271,7 @@ impl<'a> YamlEmitter<'a> {
self.level -= 1; self.level -= 1;
} }
self.emit_hash(h) self.emit_hash(h)
}, }
_ => { _ => {
try!(write!(self.writer, " ")); try!(write!(self.writer, " "));
self.emit_node(val) self.emit_node(val)
@ -296,34 +296,45 @@ impl<'a> YamlEmitter<'a> {
/// * When the string looks like a date (e.g. 2014-12-31) (otherwise it would be automatically converted into a Unix timestamp). /// * When the string looks like a date (e.g. 2014-12-31) (otherwise it would be automatically converted into a Unix timestamp).
fn need_quotes(string: &str) -> bool { fn need_quotes(string: &str) -> bool {
fn need_quotes_spaces(string: &str) -> bool { fn need_quotes_spaces(string: &str) -> bool {
string.starts_with(' ') string.starts_with(' ') || string.ends_with(' ')
|| string.ends_with(' ')
} }
string == "" string == ""
|| need_quotes_spaces(string) || need_quotes_spaces(string)
|| string.starts_with(|character: char| { || string.starts_with(|character: char| match character {
match character {
':' | '&' | '*' | '?' | '|' | '-' | '<' | '>' | '=' | '!' | '%' | '@' => true, ':' | '&' | '*' | '?' | '|' | '-' | '<' | '>' | '=' | '!' | '%' | '@' => true,
_ => false, _ => false,
}
}) })
|| string.contains(|character: char| { || string.contains(|character: char| match character {
match character { '{'
'{' | '}' | '[' | ']' | ',' | '#' | '`' | '\"' | '\'' | '\\' | '\0' ... '\x06' | '\t' | '\n' | '\r' | '\x0e' ... '\x1a' | '\x1c' ... '\x1f' => true, | '}'
| '['
| ']'
| ','
| '#'
| '`'
| '\"'
| '\''
| '\\'
| '\0'...'\x06'
| '\t'
| '\n'
| '\r'
| '\x0e'...'\x1a'
| '\x1c'...'\x1f' => true,
_ => false, _ => false,
}
}) })
|| [// http://yaml.org/type/bool.html || [
// http://yaml.org/type/bool.html
// Note: 'y', 'Y', 'n', 'N', is not quoted deliberately, as in libyaml. PyYAML also parse // Note: 'y', 'Y', 'n', 'N', is not quoted deliberately, as in libyaml. PyYAML also parse
// them as string, not booleans, although it is volating the YAML 1.1 specification. // them as string, not booleans, although it is volating the YAML 1.1 specification.
// See https://github.com/dtolnay/serde-yaml/pull/83#discussion_r152628088. // See https://github.com/dtolnay/serde-yaml/pull/83#discussion_r152628088.
"yes","Yes","YES","no","No","NO", "yes", "Yes", "YES", "no", "No", "NO", "True", "TRUE", "true", "False", "FALSE",
"True", "TRUE", "true", "False", "FALSE", "false", "false", "on", "On", "ON", "off", "Off", "OFF",
"on","On","ON","off","Off","OFF",
// http://yaml.org/type/null.html // http://yaml.org/type/null.html
"null","Null","NULL", "~" "null", "Null", "NULL", "~",
].contains(&string) ]
.contains(&string)
|| string.starts_with('.') || string.starts_with('.')
|| string.parse::<i64>().is_ok() || string.parse::<i64>().is_ok()
|| string.parse::<f64>().is_ok() || string.parse::<f64>().is_ok()
@ -349,7 +360,6 @@ a4:
- 2 - 2
"; ";
let docs = YamlLoader::load_from_str(&s).unwrap(); let docs = YamlLoader::load_from_str(&s).unwrap();
let doc = &docs[0]; let doc = &docs[0];
let mut writer = String::new(); let mut writer = String::new();
@ -361,7 +371,7 @@ a4:
println!("emitted:\n{}", writer); println!("emitted:\n{}", writer);
let docs_new = match YamlLoader::load_from_str(&writer) { let docs_new = match YamlLoader::load_from_str(&writer) {
Ok(y) => y, Ok(y) => y,
Err(e) => panic!(format!("{}", e)) Err(e) => panic!(format!("{}", e)),
}; };
let doc_new = &docs_new[0]; let doc_new = &docs_new[0];
@ -398,7 +408,7 @@ products:
} }
let docs_new = match YamlLoader::load_from_str(&writer) { let docs_new = match YamlLoader::load_from_str(&writer) {
Ok(y) => y, Ok(y) => y,
Err(e) => panic!(format!("{}", e)) Err(e) => panic!(format!("{}", e)),
}; };
let doc_new = &docs_new[0]; let doc_new = &docs_new[0];
assert_eq!(doc, doc_new); assert_eq!(doc, doc_new);
@ -506,7 +516,11 @@ bool1: false"#;
emitter.dump(doc).unwrap(); emitter.dump(doc).unwrap();
} }
assert_eq!(expected, writer, "expected:\n{}\nactual:\n{}\n", expected, writer); assert_eq!(
expected, writer,
"expected:\n{}\nactual:\n{}\n",
expected, writer
);
} }
#[test] #[test]
@ -520,7 +534,8 @@ bool1: false"#;
} }
fn test_empty_and_nested_flag(compact: bool) { fn test_empty_and_nested_flag(compact: bool) {
let s = if compact { r#"--- let s = if compact {
r#"---
a: a:
b: b:
c: hello c: hello
@ -528,7 +543,9 @@ a:
e: e:
- f - f
- g - g
- h: []"# } else { r#"--- - h: []"#
} else {
r#"---
a: a:
b: b:
c: hello c: hello
@ -537,7 +554,8 @@ e:
- f - f
- g - g
- -
h: []"# }; h: []"#
};
let docs = YamlLoader::load_from_str(&s).unwrap(); let docs = YamlLoader::load_from_str(&s).unwrap();
let doc = &docs[0]; let doc = &docs[0];

16
parser/src/lib.rs
View file

@ -41,16 +41,16 @@
extern crate linked_hash_map; extern crate linked_hash_map;
pub mod yaml;
pub mod scanner;
pub mod parser;
pub mod emitter; pub mod emitter;
pub mod parser;
pub mod scanner;
pub mod yaml;
// reexport key APIs // reexport key APIs
pub use scanner::ScanError; pub use emitter::{EmitError, YamlEmitter};
pub use parser::Event; pub use parser::Event;
pub use scanner::ScanError;
pub use yaml::{Yaml, YamlLoader}; pub use yaml::{Yaml, YamlLoader};
pub use emitter::{YamlEmitter, EmitError};
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
@ -58,8 +58,7 @@ mod tests {
#[test] #[test]
fn test_api() { fn test_api() {
let s = let s = "
"
# from yaml-cpp example # from yaml-cpp example
- name: Ogre - name: Ogre
position: [0, 5, 0] position: [0, 5, 0]
@ -104,8 +103,7 @@ mod tests {
#[test] #[test]
fn test_fail() { fn test_fail() {
let s = let s = "
"
# syntax error # syntax error
scalar scalar
key: [1, 2]] key: [1, 2]]

200
parser/src/parser.rs
View file

@ -26,7 +26,7 @@ enum State {
FlowMappingKey, FlowMappingKey,
FlowMappingValue, FlowMappingValue,
FlowMappingEmptyValue, FlowMappingEmptyValue,
End End,
} }
/// `Event` is used with the low-level event base parsing API, /// `Event` is used with the low-level event base parsing API,
@ -48,7 +48,7 @@ pub enum Event {
SequenceEnd, SequenceEnd,
/// Anchor ID /// Anchor ID
MappingStart(usize), MappingStart(usize),
MappingEnd MappingEnd,
} }
impl Event { impl Event {
@ -74,12 +74,10 @@ pub struct Parser<T> {
anchor_id: usize, anchor_id: usize,
} }
pub trait EventReceiver { pub trait EventReceiver {
fn on_event(&mut self, ev: Event); fn on_event(&mut self, ev: Event);
} }
pub trait MarkedEventReceiver { pub trait MarkedEventReceiver {
fn on_event(&mut self, ev: Event, _mark: Marker); fn on_event(&mut self, ev: Event, _mark: Marker);
} }
@ -121,9 +119,7 @@ impl<T: Iterator<Item=char>> Parser<T> {
pub fn next(&mut self) -> ParseResult { pub fn next(&mut self) -> ParseResult {
match self.current { match self.current {
None => self.parse(), None => self.parse(),
Some(_) => { Some(_) => Ok(self.current.take().unwrap()),
Ok(self.current.take().unwrap())
}
} }
} }
@ -132,28 +128,28 @@ impl<T: Iterator<Item=char>> Parser<T> {
None => { None => {
self.token = Some(try!(self.scan_next_token())); self.token = Some(try!(self.scan_next_token()));
Ok(self.token.as_ref().unwrap()) Ok(self.token.as_ref().unwrap())
}, }
Some(ref tok) => Ok(tok) Some(ref tok) => Ok(tok),
} }
} }
fn scan_next_token(&mut self) -> Result<Token, ScanError> { fn scan_next_token(&mut self) -> Result<Token, ScanError> {
let token = self.scanner.next(); let token = self.scanner.next();
match token { match token {
None => None => match self.scanner.get_error() {
match self.scanner.get_error() {
None => Err(ScanError::new(self.scanner.mark(), "unexpected eof")), None => Err(ScanError::new(self.scanner.mark(), "unexpected eof")),
Some(e) => Err(e), Some(e) => Err(e),
}, },
Some(tok) => Ok(tok) Some(tok) => Ok(tok),
} }
} }
fn fetch_token(&mut self) -> Token { fn fetch_token(&mut self) -> Token {
self.token.take().expect("fetch_token needs to be preceded by peek_token") self.token
.take()
.expect("fetch_token needs to be preceded by peek_token")
} }
fn skip(&mut self) { fn skip(&mut self) {
self.token = None; self.token = None;
//self.peek_token(); //self.peek_token();
@ -174,8 +170,11 @@ impl<T: Iterator<Item=char>> Parser<T> {
Ok((ev, mark)) Ok((ev, mark))
} }
pub fn load<R: MarkedEventReceiver>(&mut self, recv: &mut R, multi: bool) pub fn load<R: MarkedEventReceiver>(
-> Result<(), ScanError> { &mut self,
recv: &mut R,
multi: bool,
) -> Result<(), ScanError> {
if !self.scanner.stream_started() { if !self.scanner.stream_started() {
let (ev, mark) = try!(self.next()); let (ev, mark) = try!(self.next());
assert_eq!(ev, Event::StreamStart); assert_eq!(ev, Event::StreamStart);
@ -203,8 +202,12 @@ impl<T: Iterator<Item=char>> Parser<T> {
Ok(()) Ok(())
} }
fn load_document<R: MarkedEventReceiver>(&mut self, first_ev: Event, mark: Marker, recv: &mut R) fn load_document<R: MarkedEventReceiver>(
-> Result<(), ScanError> { &mut self,
first_ev: Event,
mark: Marker,
recv: &mut R,
) -> Result<(), ScanError> {
assert_eq!(first_ev, Event::DocumentStart); assert_eq!(first_ev, Event::DocumentStart);
recv.on_event(first_ev, mark); recv.on_event(first_ev, mark);
@ -219,28 +222,33 @@ impl<T: Iterator<Item=char>> Parser<T> {
Ok(()) Ok(())
} }
fn load_node<R: MarkedEventReceiver>(&mut self, first_ev: Event, mark: Marker, recv: &mut R) fn load_node<R: MarkedEventReceiver>(
-> Result<(), ScanError> { &mut self,
first_ev: Event,
mark: Marker,
recv: &mut R,
) -> Result<(), ScanError> {
match first_ev { match first_ev {
Event::Alias(..) | Event::Scalar(..) => { Event::Alias(..) | Event::Scalar(..) => {
recv.on_event(first_ev, mark); recv.on_event(first_ev, mark);
Ok(()) Ok(())
}, }
Event::SequenceStart(_) => { Event::SequenceStart(_) => {
recv.on_event(first_ev, mark); recv.on_event(first_ev, mark);
self.load_sequence(recv) self.load_sequence(recv)
}, }
Event::MappingStart(_) => { Event::MappingStart(_) => {
recv.on_event(first_ev, mark); recv.on_event(first_ev, mark);
self.load_mapping(recv) self.load_mapping(recv)
}, }
_ => { println!("UNREACHABLE EVENT: {:?}", first_ev); _ => {
unreachable!(); } println!("UNREACHABLE EVENT: {:?}", first_ev);
unreachable!();
}
} }
} }
fn load_mapping<R: MarkedEventReceiver>(&mut self, recv: &mut R) fn load_mapping<R: MarkedEventReceiver>(&mut self, recv: &mut R) -> Result<(), ScanError> {
-> Result<(), ScanError> {
let (mut key_ev, mut key_mark) = try!(self.next()); let (mut key_ev, mut key_mark) = try!(self.next());
while key_ev != Event::MappingEnd { while key_ev != Event::MappingEnd {
// key // key
@ -254,14 +262,12 @@ impl<T: Iterator<Item=char>> Parser<T> {
let (ev, mark) = try!(self.next()); let (ev, mark) = try!(self.next());
key_ev = ev; key_ev = ev;
key_mark = mark; key_mark = mark;
} }
recv.on_event(key_ev, key_mark); recv.on_event(key_ev, key_mark);
Ok(()) Ok(())
} }
fn load_sequence<R: MarkedEventReceiver>(&mut self, recv: &mut R) fn load_sequence<R: MarkedEventReceiver>(&mut self, recv: &mut R) -> Result<(), ScanError> {
-> Result<(), ScanError> {
let (mut ev, mut mark) = try!(self.next()); let (mut ev, mut mark) = try!(self.next());
while ev != Event::SequenceEnd { while ev != Event::SequenceEnd {
try!(self.load_node(ev, mark, recv)); try!(self.load_node(ev, mark, recv));
@ -289,7 +295,6 @@ impl<T: Iterator<Item=char>> Parser<T> {
State::BlockNode => self.parse_node(true, false), State::BlockNode => self.parse_node(true, false),
// State::BlockNodeOrIndentlessSequence => self.parse_node(true, true), // State::BlockNodeOrIndentlessSequence => self.parse_node(true, true),
// State::FlowNode => self.parse_node(false, false), // State::FlowNode => self.parse_node(false, false),
State::BlockMappingFirstKey => self.block_mapping_key(true), State::BlockMappingFirstKey => self.block_mapping_key(true),
State::BlockMappingKey => self.block_mapping_key(false), State::BlockMappingKey => self.block_mapping_key(false),
State::BlockMappingValue => self.block_mapping_value(), State::BlockMappingValue => self.block_mapping_value(),
@ -322,9 +327,8 @@ impl<T: Iterator<Item=char>> Parser<T> {
self.state = State::ImplicitDocumentStart; self.state = State::ImplicitDocumentStart;
self.skip(); self.skip();
Ok((Event::StreamStart, mark)) Ok((Event::StreamStart, mark))
}, }
Token(mark, _) => Err(ScanError::new(mark, Token(mark, _) => Err(ScanError::new(mark, "did not find expected <stream-start>")),
"did not find expected <stream-start>")),
} }
} }
@ -340,19 +344,19 @@ impl<T: Iterator<Item=char>> Parser<T> {
self.state = State::End; self.state = State::End;
self.skip(); self.skip();
Ok((Event::StreamEnd, mark)) Ok((Event::StreamEnd, mark))
}, }
Token(_, TokenType::VersionDirective(..)) Token(_, TokenType::VersionDirective(..))
| Token(_, TokenType::TagDirective(..)) | Token(_, TokenType::TagDirective(..))
| Token(_, TokenType::DocumentStart) => { | Token(_, TokenType::DocumentStart) => {
// explicit document // explicit document
self._explict_document_start() self._explict_document_start()
}, }
Token(mark, _) if implicit => { Token(mark, _) if implicit => {
try!(self.parser_process_directives()); try!(self.parser_process_directives());
self.push_state(State::DocumentEnd); self.push_state(State::DocumentEnd);
self.state = State::BlockNode; self.state = State::BlockNode;
Ok((Event::DocumentStart, mark)) Ok((Event::DocumentStart, mark))
}, }
_ => { _ => {
// explicit document // explicit document
self._explict_document_start() self._explict_document_start()
@ -369,11 +373,11 @@ impl<T: Iterator<Item=char>> Parser<T> {
// return Err(ScanError::new(tok.0, // return Err(ScanError::new(tok.0,
// "found incompatible YAML document")); // "found incompatible YAML document"));
//} //}
}, }
TokenType::TagDirective(..) => { TokenType::TagDirective(..) => {
// TODO add tag directive // TODO add tag directive
}, }
_ => break _ => break,
} }
self.skip(); self.skip();
} }
@ -390,7 +394,10 @@ impl<T: Iterator<Item=char>> Parser<T> {
self.skip(); self.skip();
Ok((Event::DocumentStart, mark)) Ok((Event::DocumentStart, mark))
} }
Token(mark, _) => Err(ScanError::new(mark, "did not find expected <document start>")) Token(mark, _) => Err(ScanError::new(
mark,
"did not find expected <document start>",
)),
} }
} }
@ -404,10 +411,8 @@ impl<T: Iterator<Item=char>> Parser<T> {
self.pop_state(); self.pop_state();
// empty scalar // empty scalar
Ok((Event::empty_scalar(), mark)) Ok((Event::empty_scalar(), mark))
},
_ => {
self.parse_node(true, false)
} }
_ => self.parse_node(true, false),
} }
} }
@ -418,8 +423,8 @@ impl<T: Iterator<Item=char>> Parser<T> {
self.skip(); self.skip();
_implicit = false; _implicit = false;
mark mark
}, }
Token(mark, _) => mark Token(mark, _) => mark,
}; };
// TODO tag handling // TODO tag handling
@ -447,13 +452,18 @@ impl<T: Iterator<Item=char>> Parser<T> {
self.pop_state(); self.pop_state();
if let Token(mark, TokenType::Alias(name)) = self.fetch_token() { if let Token(mark, TokenType::Alias(name)) = self.fetch_token() {
match self.anchors.get(&name) { match self.anchors.get(&name) {
None => return Err(ScanError::new(mark, "while parsing node, found unknown anchor")), None => {
Some(id) => return Ok((Event::Alias(*id), mark)) return Err(ScanError::new(
mark,
"while parsing node, found unknown anchor",
))
}
Some(id) => return Ok((Event::Alias(*id), mark)),
} }
} else { } else {
unreachable!() unreachable!()
} }
}, }
Token(_, TokenType::Anchor(_)) => { Token(_, TokenType::Anchor(_)) => {
if let Token(mark, TokenType::Anchor(name)) = self.fetch_token() { if let Token(mark, TokenType::Anchor(name)) = self.fetch_token() {
anchor_id = try!(self.register_anchor(name, &mark)); anchor_id = try!(self.register_anchor(name, &mark));
@ -467,7 +477,7 @@ impl<T: Iterator<Item=char>> Parser<T> {
} else { } else {
unreachable!() unreachable!()
} }
}, }
Token(_, TokenType::Tag(..)) => { Token(_, TokenType::Tag(..)) => {
if let tg @ TokenType::Tag(..) = self.fetch_token().1 { if let tg @ TokenType::Tag(..) = self.fetch_token().1 {
tag = Some(tg); tag = Some(tg);
@ -481,14 +491,14 @@ impl<T: Iterator<Item=char>> Parser<T> {
} else { } else {
unreachable!() unreachable!()
} }
}, }
_ => {} _ => {}
} }
match *try!(self.peek_token()) { match *try!(self.peek_token()) {
Token(mark, TokenType::BlockEntry) if indentless_sequence => { Token(mark, TokenType::BlockEntry) if indentless_sequence => {
self.state = State::IndentlessSequenceEntry; self.state = State::IndentlessSequenceEntry;
Ok((Event::SequenceStart(anchor_id), mark)) Ok((Event::SequenceStart(anchor_id), mark))
}, }
Token(_, TokenType::Scalar(..)) => { Token(_, TokenType::Scalar(..)) => {
self.pop_state(); self.pop_state();
if let Token(mark, TokenType::Scalar(style, v)) = self.fetch_token() { if let Token(mark, TokenType::Scalar(style, v)) = self.fetch_token() {
@ -496,29 +506,32 @@ impl<T: Iterator<Item=char>> Parser<T> {
} else { } else {
unreachable!() unreachable!()
} }
}, }
Token(mark, TokenType::FlowSequenceStart) => { Token(mark, TokenType::FlowSequenceStart) => {
self.state = State::FlowSequenceFirstEntry; self.state = State::FlowSequenceFirstEntry;
Ok((Event::SequenceStart(anchor_id), mark)) Ok((Event::SequenceStart(anchor_id), mark))
}, }
Token(mark, TokenType::FlowMappingStart) => { Token(mark, TokenType::FlowMappingStart) => {
self.state = State::FlowMappingFirstKey; self.state = State::FlowMappingFirstKey;
Ok((Event::MappingStart(anchor_id), mark)) Ok((Event::MappingStart(anchor_id), mark))
}, }
Token(mark, TokenType::BlockSequenceStart) if block => { Token(mark, TokenType::BlockSequenceStart) if block => {
self.state = State::BlockSequenceFirstEntry; self.state = State::BlockSequenceFirstEntry;
Ok((Event::SequenceStart(anchor_id), mark)) Ok((Event::SequenceStart(anchor_id), mark))
}, }
Token(mark, TokenType::BlockMappingStart) if block => { Token(mark, TokenType::BlockMappingStart) if block => {
self.state = State::BlockMappingFirstKey; self.state = State::BlockMappingFirstKey;
Ok((Event::MappingStart(anchor_id), mark)) Ok((Event::MappingStart(anchor_id), mark))
}, }
// ex 7.2, an empty scalar can follow a secondary tag // ex 7.2, an empty scalar can follow a secondary tag
Token(mark, _) if tag.is_some() || anchor_id > 0 => { Token(mark, _) if tag.is_some() || anchor_id > 0 => {
self.pop_state(); self.pop_state();
Ok((Event::empty_scalar_with_anchor(anchor_id, tag), mark)) Ok((Event::empty_scalar_with_anchor(anchor_id, tag), mark))
}, }
Token(mark, _) => { Err(ScanError::new(mark, "while parsing a node, did not find expected node content")) } Token(mark, _) => Err(ScanError::new(
mark,
"while parsing a node, did not find expected node content",
)),
} }
} }
@ -545,20 +558,21 @@ impl<T: Iterator<Item=char>> Parser<T> {
self.parse_node(true, true) self.parse_node(true, true)
} }
} }
}, }
// XXX(chenyh): libyaml failed to parse spec 1.2, ex8.18 // XXX(chenyh): libyaml failed to parse spec 1.2, ex8.18
Token(mark, TokenType::Value) => { Token(mark, TokenType::Value) => {
self.state = State::BlockMappingValue; self.state = State::BlockMappingValue;
Ok((Event::empty_scalar(), mark)) Ok((Event::empty_scalar(), mark))
}, }
Token(mark, TokenType::BlockEnd) => { Token(mark, TokenType::BlockEnd) => {
self.pop_state(); self.pop_state();
self.skip(); self.skip();
Ok((Event::MappingEnd, mark)) Ok((Event::MappingEnd, mark))
},
Token(mark, _) => {
Err(ScanError::new(mark, "while parsing a block mapping, did not find expected key"))
} }
Token(mark, _) => Err(ScanError::new(
mark,
"while parsing a block mapping, did not find expected key",
)),
} }
} }
@ -573,13 +587,13 @@ impl<T: Iterator<Item=char>> Parser<T> {
self.state = State::BlockMappingKey; self.state = State::BlockMappingKey;
// empty scalar // empty scalar
Ok((Event::empty_scalar(), mark)) Ok((Event::empty_scalar(), mark))
}, }
_ => { _ => {
self.push_state(State::BlockMappingKey); self.push_state(State::BlockMappingKey);
self.parse_node(true, true) self.parse_node(true, true)
} }
} }
}, }
Token(mark, _) => { Token(mark, _) => {
self.state = State::BlockMappingKey; self.state = State::BlockMappingKey;
// empty scalar // empty scalar
@ -593,7 +607,8 @@ impl<T: Iterator<Item=char>> Parser<T> {
let _ = try!(self.peek_token()); let _ = try!(self.peek_token());
self.skip(); self.skip();
} }
let marker: Marker = { let marker: Marker =
{
match *try!(self.peek_token()) { match *try!(self.peek_token()) {
Token(mark, TokenType::FlowMappingEnd) => mark, Token(mark, TokenType::FlowMappingEnd) => mark,
Token(mark, _) => { Token(mark, _) => {
@ -614,17 +629,17 @@ impl<T: Iterator<Item=char>> Parser<T> {
| Token(mark, TokenType::FlowMappingEnd) => { | Token(mark, TokenType::FlowMappingEnd) => {
self.state = State::FlowMappingValue; self.state = State::FlowMappingValue;
return Ok((Event::empty_scalar(), mark)); return Ok((Event::empty_scalar(), mark));
}, }
_ => { _ => {
self.push_state(State::FlowMappingValue); self.push_state(State::FlowMappingValue);
return self.parse_node(false, false); return self.parse_node(false, false);
} }
} }
}, }
Token(marker, TokenType::Value) => { Token(marker, TokenType::Value) => {
self.state = State::FlowMappingValue; self.state = State::FlowMappingValue;
return Ok((Event::empty_scalar(), marker)); return Ok((Event::empty_scalar(), marker));
}, }
Token(_, TokenType::FlowMappingEnd) => (), Token(_, TokenType::FlowMappingEnd) => (),
_ => { _ => {
self.push_state(State::FlowMappingEmptyValue); self.push_state(State::FlowMappingEmptyValue);
@ -653,16 +668,15 @@ impl<T: Iterator<Item=char>> Parser<T> {
Token(marker, TokenType::Value) => { Token(marker, TokenType::Value) => {
self.skip(); self.skip();
match try!(self.peek_token()).1 { match try!(self.peek_token()).1 {
TokenType::FlowEntry TokenType::FlowEntry | TokenType::FlowMappingEnd => {}
| TokenType::FlowMappingEnd => { },
_ => { _ => {
self.push_state(State::FlowMappingKey); self.push_state(State::FlowMappingKey);
return self.parse_node(false, false); return self.parse_node(false, false);
} }
} }
marker marker
}, }
Token(marker, _) => marker Token(marker, _) => marker,
} }
} }
}; };
@ -683,13 +697,15 @@ impl<T: Iterator<Item=char>> Parser<T> {
self.pop_state(); self.pop_state();
self.skip(); self.skip();
return Ok((Event::SequenceEnd, mark)); return Ok((Event::SequenceEnd, mark));
}, }
Token(_, TokenType::FlowEntry) if !first => { Token(_, TokenType::FlowEntry) if !first => {
self.skip(); self.skip();
}, }
Token(mark, _) if !first => { Token(mark, _) if !first => {
return Err(ScanError::new(mark, return Err(ScanError::new(
"while parsing a flow sequence, expectd ',' or ']'")); mark,
"while parsing a flow sequence, expectd ',' or ']'",
));
} }
_ => { /* next */ } _ => { /* next */ }
} }
@ -698,7 +714,7 @@ impl<T: Iterator<Item=char>> Parser<T> {
self.pop_state(); self.pop_state();
self.skip(); self.skip();
Ok((Event::SequenceEnd, mark)) Ok((Event::SequenceEnd, mark))
}, }
Token(mark, TokenType::Key) => { Token(mark, TokenType::Key) => {
self.state = State::FlowSequenceEntryMappingKey; self.state = State::FlowSequenceEntryMappingKey;
self.skip(); self.skip();
@ -727,7 +743,7 @@ impl<T: Iterator<Item=char>> Parser<T> {
| Token(mark, TokenType::BlockEnd) => { | Token(mark, TokenType::BlockEnd) => {
self.state = State::IndentlessSequenceEntry; self.state = State::IndentlessSequenceEntry;
Ok((Event::empty_scalar(), mark)) Ok((Event::empty_scalar(), mark))
}, }
_ => { _ => {
self.push_state(State::IndentlessSequenceEntry); self.push_state(State::IndentlessSequenceEntry);
self.parse_node(true, false) self.parse_node(true, false)
@ -747,25 +763,24 @@ impl<T: Iterator<Item=char>> Parser<T> {
self.pop_state(); self.pop_state();
self.skip(); self.skip();
Ok((Event::SequenceEnd, mark)) Ok((Event::SequenceEnd, mark))
}, }
Token(_, TokenType::BlockEntry) => { Token(_, TokenType::BlockEntry) => {
self.skip(); self.skip();
match *try!(self.peek_token()) { match *try!(self.peek_token()) {
Token(mark, TokenType::BlockEntry) Token(mark, TokenType::BlockEntry) | Token(mark, TokenType::BlockEnd) => {
| Token(mark, TokenType::BlockEnd) => {
self.state = State::BlockSequenceEntry; self.state = State::BlockSequenceEntry;
Ok((Event::empty_scalar(), mark)) Ok((Event::empty_scalar(), mark))
}, }
_ => { _ => {
self.push_state(State::BlockSequenceEntry); self.push_state(State::BlockSequenceEntry);
self.parse_node(true, false) self.parse_node(true, false)
} }
} }
},
Token(mark, _) => {
Err(ScanError::new(mark,
"while parsing a block collection, did not find expected '-' indicator"))
} }
Token(mark, _) => Err(ScanError::new(
mark,
"while parsing a block collection, did not find expected '-' indicator",
)),
} }
} }
@ -777,7 +792,7 @@ impl<T: Iterator<Item=char>> Parser<T> {
self.skip(); self.skip();
self.state = State::FlowSequenceEntryMappingValue; self.state = State::FlowSequenceEntryMappingValue;
Ok((Event::empty_scalar(), mark)) Ok((Event::empty_scalar(), mark))
}, }
_ => { _ => {
self.push_state(State::FlowSequenceEntryMappingValue); self.push_state(State::FlowSequenceEntryMappingValue);
self.parse_node(false, false) self.parse_node(false, false)
@ -791,17 +806,16 @@ impl<T: Iterator<Item=char>> Parser<T> {
self.skip(); self.skip();
self.state = State::FlowSequenceEntryMappingValue; self.state = State::FlowSequenceEntryMappingValue;
match *try!(self.peek_token()) { match *try!(self.peek_token()) {
Token(mark, TokenType::FlowEntry) Token(mark, TokenType::FlowEntry) | Token(mark, TokenType::FlowSequenceEnd) => {
| Token(mark, TokenType::FlowSequenceEnd) => {
self.state = State::FlowSequenceEntryMappingEnd; self.state = State::FlowSequenceEntryMappingEnd;
Ok((Event::empty_scalar(), mark)) Ok((Event::empty_scalar(), mark))
}, }
_ => { _ => {
self.push_state(State::FlowSequenceEntryMappingEnd); self.push_state(State::FlowSequenceEntryMappingEnd);
self.parse_node(false, false) self.parse_node(false, false)
} }
} }
}, }
Token(mark, _) => { Token(mark, _) => {
self.state = State::FlowSequenceEntryMappingEnd; self.state = State::FlowSequenceEntryMappingEnd;
Ok((Event::empty_scalar(), mark)) Ok((Event::empty_scalar(), mark))

462
parser/src/scanner.rs
View file

@ -1,10 +1,10 @@
use std::collections::VecDeque; use std::collections::VecDeque;
use std::{char, fmt};
use std::error::Error; use std::error::Error;
use std::{char, fmt};
#[derive(Clone, Copy, PartialEq, Debug, Eq)] #[derive(Clone, Copy, PartialEq, Debug, Eq)]
pub enum TEncoding { pub enum TEncoding {
Utf8 Utf8,
} }
#[derive(Clone, Copy, PartialEq, Debug, Eq)] #[derive(Clone, Copy, PartialEq, Debug, Eq)]
@ -15,7 +15,7 @@ pub enum TScalarStyle {
DoubleQuoted, DoubleQuoted,
Literal, Literal,
Foled Foled,
} }
#[derive(Clone, Copy, PartialEq, Debug, Eq)] #[derive(Clone, Copy, PartialEq, Debug, Eq)]
@ -30,7 +30,7 @@ impl Marker {
Marker { Marker {
index: index, index: index,
line: line, line: line,
col: col col: col,
} }
} }
@ -57,7 +57,7 @@ impl ScanError {
pub fn new(loc: Marker, info: &str) -> ScanError { pub fn new(loc: Marker, info: &str) -> ScanError {
ScanError { ScanError {
mark: loc, mark: loc,
info: info.to_owned() info: info.to_owned(),
} }
} }
@ -79,8 +79,13 @@ impl Error for ScanError {
impl fmt::Display for ScanError { impl fmt::Display for ScanError {
// col starts from 0 // col starts from 0
fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
write!(formatter, "{} at line {} column {}", self.info, write!(
self.mark.line, self.mark.col + 1) formatter,
"{} at line {} column {}",
self.info,
self.mark.line,
self.mark.col + 1
)
} }
} }
@ -110,7 +115,7 @@ pub enum TokenType {
Anchor(String), Anchor(String),
/// handle, suffix /// handle, suffix
Tag(String, String), Tag(String, String),
Scalar(TScalarStyle, String) Scalar(TScalarStyle, String),
} }
#[derive(Clone, PartialEq, Debug, Eq)] #[derive(Clone, PartialEq, Debug, Eq)]
@ -199,14 +204,12 @@ fn is_alpha(c: char) -> bool {
match c { match c {
'0'...'9' | 'a'...'z' | 'A'...'Z' => true, '0'...'9' | 'a'...'z' | 'A'...'Z' => true,
'_' | '-' => true, '_' | '-' => true,
_ => false _ => false,
} }
} }
#[inline] #[inline]
fn is_hex(c: char) -> bool { fn is_hex(c: char) -> bool {
(c >= '0' && c <= '9') (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')
|| (c >= 'a' && c <= 'f')
|| (c >= 'A' && c <= 'F')
} }
#[inline] #[inline]
fn as_hex(c: char) -> u32 { fn as_hex(c: char) -> u32 {
@ -214,7 +217,7 @@ fn as_hex(c: char) -> u32 {
'0'...'9' => (c as u32) - ('0' as u32), '0'...'9' => (c as u32) - ('0' as u32),
'a'...'f' => (c as u32) - ('a' as u32) + 10, 'a'...'f' => (c as u32) - ('a' as u32) + 10,
'A'...'F' => (c as u32) - ('A' as u32) + 10, 'A'...'F' => (c as u32) - ('A' as u32) + 10,
_ => unreachable!() _ => unreachable!(),
} }
} }
@ -363,7 +366,8 @@ impl<T: Iterator<Item=char>> Scanner<T> {
&& self.buffer[0] == '-' && self.buffer[0] == '-'
&& self.buffer[1] == '-' && self.buffer[1] == '-'
&& self.buffer[2] == '-' && self.buffer[2] == '-'
&& is_blankz(self.buffer[3]) { && is_blankz(self.buffer[3])
{
try!(self.fetch_document_indicator(TokenType::DocumentStart)); try!(self.fetch_document_indicator(TokenType::DocumentStart));
return Ok(()); return Ok(());
} }
@ -372,7 +376,8 @@ impl<T: Iterator<Item=char>> Scanner<T> {
&& self.buffer[0] == '.' && self.buffer[0] == '.'
&& self.buffer[1] == '.' && self.buffer[1] == '.'
&& self.buffer[2] == '.' && self.buffer[2] == '.'
&& is_blankz(self.buffer[3]) { && is_blankz(self.buffer[3])
{
try!(self.fetch_document_indicator(TokenType::DocumentEnd)); try!(self.fetch_document_indicator(TokenType::DocumentEnd));
return Ok(()); return Ok(());
} }
@ -402,8 +407,10 @@ impl<T: Iterator<Item=char>> Scanner<T> {
// plain scalar // plain scalar
'-' if !is_blankz(nc) => self.fetch_plain_scalar(), '-' if !is_blankz(nc) => self.fetch_plain_scalar(),
':' | '?' if !is_blankz(nc) && self.flow_level == 0 => self.fetch_plain_scalar(), ':' | '?' if !is_blankz(nc) && self.flow_level == 0 => self.fetch_plain_scalar(),
'%' | '@' | '`' => Err(ScanError::new(self.mark, '%' | '@' | '`' => Err(ScanError::new(
&format!("unexpected character: `{}'", c))), self.mark,
&format!("unexpected character: `{}'", c),
)),
_ => self.fetch_plain_scalar(), _ => self.fetch_plain_scalar(),
} }
} }
@ -442,7 +449,9 @@ impl<T: Iterator<Item=char>> Scanner<T> {
} }
} }
if !need_more { break; } if !need_more {
break;
}
try!(self.fetch_next_token()); try!(self.fetch_next_token());
} }
self.token_available = true; self.token_available = true;
@ -452,8 +461,9 @@ impl<T: Iterator<Item=char>> Scanner<T> {
fn stale_simple_keys(&mut self) -> ScanResult { fn stale_simple_keys(&mut self) -> ScanResult {
for sk in &mut self.simple_keys { for sk in &mut self.simple_keys {
if sk.possible && (sk.mark.line < self.mark.line if sk.possible
|| sk.mark.index + 1024 < self.mark.index) { && (sk.mark.line < self.mark.line || sk.mark.index + 1024 < self.mark.index)
{
if sk.required { if sk.required {
return Err(ScanError::new(self.mark, "simple key expect ':'")); return Err(ScanError::new(self.mark, "simple key expect ':'"));
} }
@ -476,9 +486,12 @@ impl<T: Iterator<Item=char>> Scanner<T> {
if self.flow_level == 0 { if self.flow_level == 0 {
self.allow_simple_key(); self.allow_simple_key();
} }
}
'#' => while !is_breakz(self.ch()) {
self.skip();
self.lookahead(1);
}, },
'#' => while !is_breakz(self.ch()) { self.skip(); self.lookahead(1); }, _ => break,
_ => break
} }
} }
} }
@ -488,7 +501,8 @@ impl<T: Iterator<Item=char>> Scanner<T> {
self.indent = -1; self.indent = -1;
self.stream_start_produced = true; self.stream_start_produced = true;
self.allow_simple_key(); self.allow_simple_key();
self.tokens.push_back(Token(mark, TokenType::StreamStart(TEncoding::Utf8))); self.tokens
.push_back(Token(mark, TokenType::StreamStart(TEncoding::Utf8)));
self.simple_keys.push(SimpleKey::new(Marker::new(0, 0, 0))); self.simple_keys.push(SimpleKey::new(Marker::new(0, 0, 0)));
} }
@ -503,7 +517,8 @@ impl<T: Iterator<Item=char>> Scanner<T> {
try!(self.remove_simple_key()); try!(self.remove_simple_key());
self.disallow_simple_key(); self.disallow_simple_key();
self.tokens.push_back(Token(self.mark, TokenType::StreamEnd)); self.tokens
.push_back(Token(self.mark, TokenType::StreamEnd));
Ok(()) Ok(())
} }
@ -526,12 +541,8 @@ impl<T: Iterator<Item=char>> Scanner<T> {
let name = try!(self.scan_directive_name()); let name = try!(self.scan_directive_name());
let tok = match name.as_ref() { let tok = match name.as_ref() {
"YAML" => { "YAML" => try!(self.scan_version_directive_value(&start_mark)),
try!(self.scan_version_directive_value(&start_mark)) "TAG" => try!(self.scan_tag_directive_value(&start_mark)),
},
"TAG" => {
try!(self.scan_tag_directive_value(&start_mark))
},
// XXX This should be a warning instead of an error // XXX This should be a warning instead of an error
_ => { _ => {
// skip current line // skip current line
@ -541,7 +552,10 @@ impl<T: Iterator<Item=char>> Scanner<T> {
self.lookahead(1); self.lookahead(1);
} }
// XXX return an empty TagDirective token // XXX return an empty TagDirective token
Token(start_mark, TokenType::TagDirective(String::new(), String::new())) Token(
start_mark,
TokenType::TagDirective(String::new(), String::new()),
)
// return Err(ScanError::new(start_mark, // return Err(ScanError::new(start_mark,
// "while scanning a directive, found unknown directive name")) // "while scanning a directive, found unknown directive name"))
} }
@ -561,8 +575,10 @@ impl<T: Iterator<Item=char>> Scanner<T> {
} }
if !is_breakz(self.ch()) { if !is_breakz(self.ch()) {
return Err(ScanError::new(start_mark, return Err(ScanError::new(
"while scanning a directive, did not find expected comment or line break")); start_mark,
"while scanning a directive, did not find expected comment or line break",
));
} }
// Eat a line break // Eat a line break
@ -585,8 +601,10 @@ impl<T: Iterator<Item=char>> Scanner<T> {
let major = try!(self.scan_version_directive_number(mark)); let major = try!(self.scan_version_directive_number(mark));
if self.ch() != '.' { if self.ch() != '.' {
return Err(ScanError::new(*mark, return Err(ScanError::new(
"while scanning a YAML directive, did not find expected digit or '.' character")); *mark,
"while scanning a YAML directive, did not find expected digit or '.' character",
));
} }
self.skip(); self.skip();
@ -607,13 +625,17 @@ impl<T: Iterator<Item=char>> Scanner<T> {
} }
if string.is_empty() { if string.is_empty() {
return Err(ScanError::new(start_mark, return Err(ScanError::new(
"while scanning a directive, could not find expected directive name")); start_mark,
"while scanning a directive, could not find expected directive name",
));
} }
if !is_blankz(self.ch()) { if !is_blankz(self.ch()) {
return Err(ScanError::new(start_mark, return Err(ScanError::new(
"while scanning a directive, found unexpected non-alphabetical character")); start_mark,
"while scanning a directive, found unexpected non-alphabetical character",
));
} }
Ok(string) Ok(string)
@ -625,8 +647,10 @@ impl<T: Iterator<Item=char>> Scanner<T> {
self.lookahead(1); self.lookahead(1);
while is_digit(self.ch()) { while is_digit(self.ch()) {
if length + 1 > 9 { if length + 1 > 9 {
return Err(ScanError::new(*mark, return Err(ScanError::new(
"while scanning a YAML directive, found extremely long version number")); *mark,
"while scanning a YAML directive, found extremely long version number",
));
} }
length += 1; length += 1;
val = val * 10 + ((self.ch() as u32) - ('0' as u32)); val = val * 10 + ((self.ch() as u32) - ('0' as u32));
@ -635,8 +659,10 @@ impl<T: Iterator<Item=char>> Scanner<T> {
} }
if length == 0 { if length == 0 {
return Err(ScanError::new(*mark, return Err(ScanError::new(
"while scanning a YAML directive, did not find expected version number")); *mark,
"while scanning a YAML directive, did not find expected version number",
));
} }
Ok(val) Ok(val)
@ -666,8 +692,10 @@ impl<T: Iterator<Item=char>> Scanner<T> {
if is_blankz(self.ch()) { if is_blankz(self.ch()) {
Ok(Token(*mark, TokenType::TagDirective(handle, prefix))) Ok(Token(*mark, TokenType::TagDirective(handle, prefix)))
} else { } else {
Err(ScanError::new(*mark, Err(ScanError::new(
"while scanning TAG, did not find expected whitespace or line break")) *mark,
"while scanning TAG, did not find expected whitespace or line break",
))
} }
} }
@ -696,8 +724,10 @@ impl<T: Iterator<Item=char>> Scanner<T> {
suffix = try!(self.scan_tag_uri(false, false, &String::new(), &start_mark)); suffix = try!(self.scan_tag_uri(false, false, &String::new(), &start_mark));
if self.ch() != '>' { if self.ch() != '>' {
return Err(ScanError::new(start_mark, return Err(ScanError::new(
"while scanning a tag, did not find the expected '>'")); start_mark,
"while scanning a tag, did not find the expected '>'",
));
} }
self.skip(); self.skip();
@ -727,8 +757,10 @@ impl<T: Iterator<Item=char>> Scanner<T> {
// XXX: ex 7.2, an empty scalar can follow a secondary tag // XXX: ex 7.2, an empty scalar can follow a secondary tag
Ok(Token(start_mark, TokenType::Tag(handle, suffix))) Ok(Token(start_mark, TokenType::Tag(handle, suffix)))
} else { } else {
Err(ScanError::new(start_mark, Err(ScanError::new(
"while scanning a tag, did not find expected whitespace or line break")) start_mark,
"while scanning a tag, did not find expected whitespace or line break",
))
} }
} }
@ -736,8 +768,10 @@ impl<T: Iterator<Item=char>> Scanner<T> {
let mut string = String::new(); let mut string = String::new();
self.lookahead(1); self.lookahead(1);
if self.ch() != '!' { if self.ch() != '!' {
return Err(ScanError::new(*mark, return Err(ScanError::new(
"while scanning a tag, did not find expected '!'")); *mark,
"while scanning a tag, did not find expected '!'",
));
} }
string.push(self.ch()); string.push(self.ch());
@ -758,14 +792,21 @@ impl<T: Iterator<Item=char>> Scanner<T> {
// It's either the '!' tag or not really a tag handle. If it's a %TAG // It's either the '!' tag or not really a tag handle. If it's a %TAG
// directive, it's an error. If it's a tag token, it must be a part of // directive, it's an error. If it's a tag token, it must be a part of
// URI. // URI.
return Err(ScanError::new(*mark, return Err(ScanError::new(
"while parsing a tag directive, did not find expected '!'")); *mark,
"while parsing a tag directive, did not find expected '!'",
));
} }
Ok(string) Ok(string)
} }
fn scan_tag_uri(&mut self, directive: bool, _is_secondary: bool, fn scan_tag_uri(
head: &str, mark: &Marker) -> Result<String, ScanError> { &mut self,
directive: bool,
_is_secondary: bool,
head: &str,
mark: &Marker,
) -> Result<String, ScanError> {
let mut length = head.len(); let mut length = head.len();
let mut string = String::new(); let mut string = String::new();
@ -788,7 +829,7 @@ impl<T: Iterator<Item=char>> Scanner<T> {
'=' | '+' | '$' | ',' | '.' | '!' | '~' | '*' | '\'' | '(' | ')' | '[' | ']' => true, '=' | '+' | '$' | ',' | '.' | '!' | '~' | '*' | '\'' | '(' | ')' | '[' | ']' => true,
'%' => true, '%' => true,
c if is_alpha(c) => true, c if is_alpha(c) => true,
_ => false _ => false,
} { } {
// Check if it is a URI-escape sequence. // Check if it is a URI-escape sequence.
if self.ch() == '%' { if self.ch() == '%' {
@ -803,25 +844,26 @@ impl<T: Iterator<Item=char>> Scanner<T> {
} }
if length == 0 { if length == 0 {
return Err(ScanError::new(*mark, return Err(ScanError::new(
"while parsing a tag, did not find expected tag URI")); *mark,
"while parsing a tag, did not find expected tag URI",
));
} }
Ok(string) Ok(string)
} }
fn scan_uri_escapes(&mut self, _directive: bool, mark: &Marker) fn scan_uri_escapes(&mut self, _directive: bool, mark: &Marker) -> Result<char, ScanError> {
-> Result<char, ScanError> {
let mut width = 0usize; let mut width = 0usize;
let mut code = 0u32; let mut code = 0u32;
loop { loop {
self.lookahead(3); self.lookahead(3);
if !(self.ch() == '%' if !(self.ch() == '%' && is_hex(self.buffer[1]) && is_hex(self.buffer[2])) {
&& is_hex(self.buffer[1]) return Err(ScanError::new(
&& is_hex(self.buffer[2])) { *mark,
return Err(ScanError::new(*mark, "while parsing a tag, did not find URI escaped octet",
"while parsing a tag, did not find URI escaped octet")); ));
} }
let octet = (as_hex(self.buffer[1]) << 4) + as_hex(self.buffer[2]); let octet = (as_hex(self.buffer[1]) << 4) + as_hex(self.buffer[2]);
@ -832,15 +874,19 @@ impl<T: Iterator<Item=char>> Scanner<T> {
_ if octet & 0xF0 == 0xE0 => 3, _ if octet & 0xF0 == 0xE0 => 3,
_ if octet & 0xF8 == 0xF0 => 4, _ if octet & 0xF8 == 0xF0 => 4,
_ => { _ => {
return Err(ScanError::new(*mark, return Err(ScanError::new(
"while parsing a tag, found an incorrect leading UTF-8 octet")); *mark,
"while parsing a tag, found an incorrect leading UTF-8 octet",
));
} }
}; };
code = octet; code = octet;
} else { } else {
if octet & 0xc0 != 0x80 { if octet & 0xc0 != 0x80 {
return Err(ScanError::new(*mark, return Err(ScanError::new(
"while parsing a tag, found an incorrect trailing UTF-8 octet")); *mark,
"while parsing a tag, found an incorrect trailing UTF-8 octet",
));
} }
code = (code << 8) + octet; code = (code << 8) + octet;
} }
@ -857,8 +903,10 @@ impl<T: Iterator<Item=char>> Scanner<T> {
match char::from_u32(code) { match char::from_u32(code) {
Some(ch) => Ok(ch), Some(ch) => Ok(ch),
None => Err(ScanError::new(*mark, None => Err(ScanError::new(
"while parsing a tag, found an invalid UTF-8 codepoint")) *mark,
"while parsing a tag, found an invalid UTF-8 codepoint",
)),
} }
} }
@ -873,8 +921,7 @@ impl<T: Iterator<Item=char>> Scanner<T> {
Ok(()) Ok(())
} }
fn scan_anchor(&mut self, alias: bool) fn scan_anchor(&mut self, alias: bool) -> Result<Token, ScanError> {
-> Result<Token, ScanError> {
let mut string = String::new(); let mut string = String::new();
let start_mark = self.mark; let start_mark = self.mark;
@ -886,11 +933,10 @@ impl<T: Iterator<Item=char>> Scanner<T> {
self.lookahead(1); self.lookahead(1);
} }
if string.is_empty() if string.is_empty() || match self.ch() {
|| match self.ch() {
c if is_blankz(c) => false, c if is_blankz(c) => false,
'?' | ':' | ',' | ']' | '}' | '%' | '@' | '`' => false, '?' | ':' | ',' | ']' | '}' | '%' | '@' | '`' => false,
_ => true _ => true,
} { } {
return Err(ScanError::new(start_mark, "while scanning an anchor or alias, did not find expected alphabetic or numeric character")); return Err(ScanError::new(start_mark, "while scanning an anchor or alias, did not find expected alphabetic or numeric character"));
} }
@ -937,13 +983,16 @@ impl<T: Iterator<Item=char>> Scanner<T> {
let start_mark = self.mark; let start_mark = self.mark;
self.skip(); self.skip();
self.tokens.push_back(Token(start_mark, TokenType::FlowEntry)); self.tokens
.push_back(Token(start_mark, TokenType::FlowEntry));
Ok(()) Ok(())
} }
fn increase_flow_level(&mut self) -> ScanResult { fn increase_flow_level(&mut self) -> ScanResult {
self.simple_keys.push(SimpleKey::new(Marker::new(0, 0, 0))); self.simple_keys.push(SimpleKey::new(Marker::new(0, 0, 0)));
self.flow_level = self.flow_level.checked_add(1) self.flow_level = self
.flow_level
.checked_add(1)
.ok_or_else(|| ScanError::new(self.mark, "Recursion limit exceeded"))?; .ok_or_else(|| ScanError::new(self.mark, "Recursion limit exceeded"))?;
Ok(()) Ok(())
} }
@ -958,8 +1007,10 @@ impl<T: Iterator<Item=char>> Scanner<T> {
if self.flow_level == 0 { if self.flow_level == 0 {
// Check if we are allowed to start a new entry. // Check if we are allowed to start a new entry.
if !self.simple_key_allowed { if !self.simple_key_allowed {
return Err(ScanError::new(self.mark, return Err(ScanError::new(
"block sequence entries are not allowed in this context")); self.mark,
"block sequence entries are not allowed in this context",
));
} }
let mark = self.mark; let mark = self.mark;
@ -967,7 +1018,10 @@ impl<T: Iterator<Item=char>> Scanner<T> {
self.roll_indent(mark.col, None, TokenType::BlockSequenceStart, mark); self.roll_indent(mark.col, None, TokenType::BlockSequenceStart, mark);
} else { } else {
// - * only allowed in block // - * only allowed in block
return Err(ScanError::new(self.mark, r#""-" is only valid inside a block"#)) return Err(ScanError::new(
self.mark,
r#""-" is only valid inside a block"#,
));
} }
try!(self.remove_simple_key()); try!(self.remove_simple_key());
self.allow_simple_key(); self.allow_simple_key();
@ -975,7 +1029,8 @@ impl<T: Iterator<Item=char>> Scanner<T> {
let start_mark = self.mark; let start_mark = self.mark;
self.skip(); self.skip();
self.tokens.push_back(Token(start_mark, TokenType::BlockEntry)); self.tokens
.push_back(Token(start_mark, TokenType::BlockEntry));
Ok(()) Ok(())
} }
@ -1029,16 +1084,20 @@ impl<T: Iterator<Item=char>> Scanner<T> {
self.lookahead(1); self.lookahead(1);
if is_digit(self.ch()) { if is_digit(self.ch()) {
if self.ch() == '0' { if self.ch() == '0' {
return Err(ScanError::new(start_mark, return Err(ScanError::new(
"while scanning a block scalar, found an intendation indicator equal to 0")); start_mark,
"while scanning a block scalar, found an intendation indicator equal to 0",
));
} }
increment = (self.ch() as usize) - ('0' as usize); increment = (self.ch() as usize) - ('0' as usize);
self.skip(); self.skip();
} }
} else if is_digit(self.ch()) { } else if is_digit(self.ch()) {
if self.ch() == '0' { if self.ch() == '0' {
return Err(ScanError::new(start_mark, return Err(ScanError::new(
"while scanning a block scalar, found an intendation indicator equal to 0")); start_mark,
"while scanning a block scalar, found an intendation indicator equal to 0",
));
} }
increment = (self.ch() as usize) - ('0' as usize); increment = (self.ch() as usize) - ('0' as usize);
@ -1071,8 +1130,10 @@ impl<T: Iterator<Item=char>> Scanner<T> {
// Check if we are at the end of the line. // Check if we are at the end of the line.
if !is_breakz(self.ch()) { if !is_breakz(self.ch()) {
return Err(ScanError::new(start_mark, return Err(ScanError::new(
"while scanning a block scalar, did not find expected comment or line break")); start_mark,
"while scanning a block scalar, did not find expected comment or line break",
));
} }
if is_break(self.ch()) { if is_break(self.ch()) {
@ -1081,7 +1142,11 @@ impl<T: Iterator<Item=char>> Scanner<T> {
} }
if increment > 0 { if increment > 0 {
indent = if self.indent >= 0 { (self.indent + increment as isize) as usize } else { increment } indent = if self.indent >= 0 {
(self.indent + increment as isize) as usize
} else {
increment
}
} }
// Scan the leading line breaks and determine the indentation level if needed. // Scan the leading line breaks and determine the indentation level if needed.
try!(self.block_scalar_breaks(&mut indent, &mut trailing_breaks)); try!(self.block_scalar_breaks(&mut indent, &mut trailing_breaks));
@ -1093,8 +1158,7 @@ impl<T: Iterator<Item=char>> Scanner<T> {
while self.mark.col == indent && !is_z(self.ch()) { while self.mark.col == indent && !is_z(self.ch()) {
// We are at the beginning of a non-empty line. // We are at the beginning of a non-empty line.
trailing_blank = is_blank(self.ch()); trailing_blank = is_blank(self.ch());
if !literal && !leading_break.is_empty() if !literal && !leading_break.is_empty() && !leading_blank && !trailing_blank {
&& !leading_blank && !trailing_blank {
if trailing_breaks.is_empty() { if trailing_breaks.is_empty() {
string.push(' '); string.push(' ');
} }
@ -1115,7 +1179,9 @@ impl<T: Iterator<Item=char>> Scanner<T> {
self.lookahead(1); self.lookahead(1);
} }
// break on EOF // break on EOF
if is_z(self.ch()) { break; } if is_z(self.ch()) {
break;
}
self.lookahead(2); self.lookahead(2);
self.read_break(&mut leading_break); self.read_break(&mut leading_break);
@ -1134,9 +1200,15 @@ impl<T: Iterator<Item=char>> Scanner<T> {
} }
if literal { if literal {
Ok(Token(start_mark, TokenType::Scalar(TScalarStyle::Literal, string))) Ok(Token(
start_mark,
TokenType::Scalar(TScalarStyle::Literal, string),
))
} else { } else {
Ok(Token(start_mark, TokenType::Scalar(TScalarStyle::Foled, string))) Ok(Token(
start_mark,
TokenType::Scalar(TScalarStyle::Foled, string),
))
} }
} }
@ -1144,8 +1216,7 @@ impl<T: Iterator<Item=char>> Scanner<T> {
let mut max_indent = 0; let mut max_indent = 0;
loop { loop {
self.lookahead(1); self.lookahead(1);
while (*indent == 0 || self.mark.col < *indent) while (*indent == 0 || self.mark.col < *indent) && self.buffer[0] == ' ' {
&& self.buffer[0] == ' ' {
self.skip(); self.skip();
self.lookahead(1); self.lookahead(1);
} }
@ -1155,8 +1226,7 @@ impl<T: Iterator<Item=char>> Scanner<T> {
} }
// Check for a tab character messing the intendation. // Check for a tab character messing the intendation.
if (*indent == 0 || self.mark.col < *indent) if (*indent == 0 || self.mark.col < *indent) && self.buffer[0] == '\t' {
&& self.buffer[0] == '\t' {
return Err(ScanError::new(self.mark, return Err(ScanError::new(self.mark,
"while scanning a block scalar, found a tab character where an intendation space is expected")); "while scanning a block scalar, found a tab character where an intendation space is expected"));
} }
@ -1208,21 +1278,24 @@ impl<T: Iterator<Item=char>> Scanner<T> {
/* Check for a document indicator. */ /* Check for a document indicator. */
self.lookahead(4); self.lookahead(4);
if self.mark.col == 0 && if self.mark.col == 0
(((self.buffer[0] == '-') && && (((self.buffer[0] == '-') && (self.buffer[1] == '-') && (self.buffer[2] == '-'))
(self.buffer[1] == '-') && || ((self.buffer[0] == '.')
(self.buffer[2] == '-')) || && (self.buffer[1] == '.')
((self.buffer[0] == '.') && && (self.buffer[2] == '.')))
(self.buffer[1] == '.') && && is_blankz(self.buffer[3])
(self.buffer[2] == '.'))) && {
is_blankz(self.buffer[3]) { return Err(ScanError::new(
return Err(ScanError::new(start_mark, start_mark,
"while scanning a quoted scalar, found unexpected document indicator")); "while scanning a quoted scalar, found unexpected document indicator",
));
} }
if is_z(self.ch()) { if is_z(self.ch()) {
return Err(ScanError::new(start_mark, return Err(ScanError::new(
"while scanning a quoted scalar, found unexpected end of stream")); start_mark,
"while scanning a quoted scalar, found unexpected end of stream",
));
} }
self.lookahead(2); self.lookahead(2);
@ -1237,10 +1310,10 @@ impl<T: Iterator<Item=char>> Scanner<T> {
string.push('\''); string.push('\'');
self.skip(); self.skip();
self.skip(); self.skip();
}, }
// Check for the right quote. // Check for the right quote.
'\'' if single => { break; }, '\'' if single => break,
'"' if !single => { break; }, '"' if !single => break,
// Check for an escaped line break. // Check for an escaped line break.
'\\' if !single && is_break(self.buffer[1]) => { '\\' if !single && is_break(self.buffer[1]) => {
self.lookahead(3); self.lookahead(3);
@ -1277,8 +1350,12 @@ impl<T: Iterator<Item=char>> Scanner<T> {
'x' => code_length = 2, 'x' => code_length = 2,
'u' => code_length = 4, 'u' => code_length = 4,
'U' => code_length = 8, 'U' => code_length = 8,
_ => return Err(ScanError::new(start_mark, _ => {
"while parsing a quoted scalar, found unknown escape character")) return Err(ScanError::new(
start_mark,
"while parsing a quoted scalar, found unknown escape character",
))
}
} }
self.skip(); self.skip();
self.skip(); self.skip();
@ -1307,15 +1384,18 @@ impl<T: Iterator<Item=char>> Scanner<T> {
self.skip(); self.skip();
} }
} }
}, }
c => { string.push(c); self.skip(); } c => {
string.push(c);
self.skip();
}
} }
self.lookahead(2); self.lookahead(2);
} }
self.lookahead(1); self.lookahead(1);
match self.ch() { match self.ch() {
'\'' if single => { break; }, '\'' if single => break,
'"' if !single => { break; }, '"' if !single => break,
_ => {} _ => {}
} }
@ -1368,9 +1448,15 @@ impl<T: Iterator<Item=char>> Scanner<T> {
self.skip(); self.skip();
if single { if single {
Ok(Token(start_mark, TokenType::Scalar(TScalarStyle::SingleQuoted, string))) Ok(Token(
start_mark,
TokenType::Scalar(TScalarStyle::SingleQuoted, string),
))
} else { } else {
Ok(Token(start_mark, TokenType::Scalar(TScalarStyle::DoubleQuoted, string))) Ok(Token(
start_mark,
TokenType::Scalar(TScalarStyle::DoubleQuoted, string),
))
} }
} }
@ -1398,23 +1484,25 @@ impl<T: Iterator<Item=char>> Scanner<T> {
/* Check for a document indicator. */ /* Check for a document indicator. */
self.lookahead(4); self.lookahead(4);
if self.mark.col == 0 && if self.mark.col == 0
(((self.buffer[0] == '-') && && (((self.buffer[0] == '-') && (self.buffer[1] == '-') && (self.buffer[2] == '-'))
(self.buffer[1] == '-') && || ((self.buffer[0] == '.')
(self.buffer[2] == '-')) || && (self.buffer[1] == '.')
((self.buffer[0] == '.') && && (self.buffer[2] == '.')))
(self.buffer[1] == '.') && && is_blankz(self.buffer[3])
(self.buffer[2] == '.'))) && {
is_blankz(self.buffer[3]) {
break; break;
} }
if self.ch() == '#' { break; } if self.ch() == '#' {
break;
}
while !is_blankz(self.ch()) { while !is_blankz(self.ch()) {
if self.flow_level > 0 && self.ch() == ':' if self.flow_level > 0 && self.ch() == ':' && is_blankz(self.ch()) {
&& is_blankz(self.ch()) { return Err(ScanError::new(
return Err(ScanError::new(start_mark, start_mark,
"while scanning a plain scalar, found unexpected ':'")); "while scanning a plain scalar, found unexpected ':'",
));
} }
// indicators ends a plain scalar // indicators ends a plain scalar
match self.ch() { match self.ch() {
@ -1438,7 +1526,6 @@ impl<T: Iterator<Item=char>> Scanner<T> {
trailing_breaks.clear(); trailing_breaks.clear();
} }
leading_break.clear(); leading_break.clear();
} }
leading_blanks = false; leading_blanks = false;
} else { } else {
@ -1452,15 +1539,18 @@ impl<T: Iterator<Item=char>> Scanner<T> {
self.lookahead(2); self.lookahead(2);
} }
// is the end? // is the end?
if !(is_blank(self.ch()) || is_break(self.ch())) { break; } if !(is_blank(self.ch()) || is_break(self.ch())) {
break;
}
self.lookahead(1); self.lookahead(1);
while is_blank(self.ch()) || is_break(self.ch()) { while is_blank(self.ch()) || is_break(self.ch()) {
if is_blank(self.ch()) { if is_blank(self.ch()) {
if leading_blanks && (self.mark.col as isize) < indent if leading_blanks && (self.mark.col as isize) < indent && self.ch() == '\t' {
&& self.ch() == '\t' { return Err(ScanError::new(
return Err(ScanError::new(start_mark, start_mark,
"while scanning a plain scalar, found a tab")); "while scanning a plain scalar, found a tab",
));
} }
if leading_blanks { if leading_blanks {
@ -1493,7 +1583,10 @@ impl<T: Iterator<Item=char>> Scanner<T> {
self.allow_simple_key(); self.allow_simple_key();
} }
Ok(Token(start_mark, TokenType::Scalar(TScalarStyle::Plain, string))) Ok(Token(
start_mark,
TokenType::Scalar(TScalarStyle::Plain, string),
))
} }
fn fetch_key(&mut self) -> ScanResult { fn fetch_key(&mut self) -> ScanResult {
@ -1501,10 +1594,17 @@ impl<T: Iterator<Item=char>> Scanner<T> {
if self.flow_level == 0 { if self.flow_level == 0 {
// Check if we are allowed to start a new key (not nessesary simple). // Check if we are allowed to start a new key (not nessesary simple).
if !self.simple_key_allowed { if !self.simple_key_allowed {
return Err(ScanError::new(self.mark, "mapping keys are not allowed in this context")); return Err(ScanError::new(
self.mark,
"mapping keys are not allowed in this context",
));
} }
self.roll_indent(start_mark.col, None, self.roll_indent(
TokenType::BlockMappingStart, start_mark); start_mark.col,
None,
TokenType::BlockMappingStart,
start_mark,
);
} }
try!(self.remove_simple_key()); try!(self.remove_simple_key());
@ -1530,8 +1630,12 @@ impl<T: Iterator<Item=char>> Scanner<T> {
self.insert_token(sk.token_number - tokens_parsed, tok); self.insert_token(sk.token_number - tokens_parsed, tok);
// Add the BLOCK-MAPPING-START token if needed. // Add the BLOCK-MAPPING-START token if needed.
self.roll_indent(sk.mark.col, Some(sk.token_number), self.roll_indent(
TokenType::BlockMappingStart, start_mark); sk.mark.col,
Some(sk.token_number),
TokenType::BlockMappingStart,
start_mark,
);
self.simple_keys.last_mut().unwrap().possible = false; self.simple_keys.last_mut().unwrap().possible = false;
self.disallow_simple_key(); self.disallow_simple_key();
@ -1539,12 +1643,18 @@ impl<T: Iterator<Item=char>> Scanner<T> {
// The ':' indicator follows a complex key. // The ':' indicator follows a complex key.
if self.flow_level == 0 { if self.flow_level == 0 {
if !self.simple_key_allowed { if !self.simple_key_allowed {
return Err(ScanError::new(start_mark, return Err(ScanError::new(
"mapping values are not allowed in this context")); start_mark,
"mapping values are not allowed in this context",
));
} }
self.roll_indent(start_mark.col, None, self.roll_indent(
TokenType::BlockMappingStart, start_mark); start_mark.col,
None,
TokenType::BlockMappingStart,
start_mark,
);
} }
if self.flow_level == 0 { if self.flow_level == 0 {
@ -1559,8 +1669,7 @@ impl<T: Iterator<Item=char>> Scanner<T> {
Ok(()) Ok(())
} }
fn roll_indent(&mut self, col: usize, number: Option<usize>, fn roll_indent(&mut self, col: usize, number: Option<usize>, tok: TokenType, mark: Marker) {
tok: TokenType, mark: Marker) {
if self.flow_level > 0 { if self.flow_level > 0 {
return; return;
} }
@ -1571,7 +1680,7 @@ impl<T: Iterator<Item=char>> Scanner<T> {
let tokens_parsed = self.tokens_parsed; let tokens_parsed = self.tokens_parsed;
match number { match number {
Some(n) => self.insert_token(n - tokens_parsed, Token(mark, tok)), Some(n) => self.insert_token(n - tokens_parsed, Token(mark, tok)),
None => self.tokens.push_back(Token(mark, tok)) None => self.tokens.push_back(Token(mark, tok)),
} }
} }
} }
@ -1611,23 +1720,21 @@ impl<T: Iterator<Item=char>> Scanner<T> {
last.possible = false; last.possible = false;
Ok(()) Ok(())
} }
} }
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use super::*;
use super::TokenType::*; use super::TokenType::*;
use super::*;
macro_rules! next { macro_rules! next {
($p:ident, $tk:pat) => {{ ($p:ident, $tk:pat) => {{
let tok = $p.next().unwrap(); let tok = $p.next().unwrap();
match tok.1 { match tok.1 {
$tk => {}, $tk => {}
_ => { panic!("unexpected token: {:?}", _ => panic!("unexpected token: {:?}", tok),
tok) }
} }
}} }};
} }
macro_rules! next_scalar { macro_rules! next_scalar {
@ -1637,17 +1744,16 @@ macro_rules! next_scalar {
Scalar(style, ref v) => { Scalar(style, ref v) => {
assert_eq!(style, $tk); assert_eq!(style, $tk);
assert_eq!(v, $v); assert_eq!(v, $v);
},
_ => { panic!("unexpected token: {:?}",
tok) }
} }
}} _ => panic!("unexpected token: {:?}", tok),
}
}};
} }
macro_rules! end { macro_rules! end {
($p:ident) => {{ ($p:ident) => {{
assert_eq!($p.next(), None); assert_eq!($p.next(), None);
}} }};
} }
/// test cases in libyaml scanner.c /// test cases in libyaml scanner.c
#[test] #[test]
@ -1671,8 +1777,7 @@ macro_rules! end {
#[test] #[test]
fn test_explicit_scalar() { fn test_explicit_scalar() {
let s = let s = "---
"---
'a scalar' 'a scalar'
... ...
"; ";
@ -1687,8 +1792,7 @@ macro_rules! end {
#[test] #[test]
fn test_multiple_documents() { fn test_multiple_documents() {
let s = let s = "
"
'a scalar' 'a scalar'
--- ---
'a scalar' 'a scalar'
@ -1724,8 +1828,7 @@ macro_rules! end {
#[test] #[test]
fn test_a_flow_mapping() { fn test_a_flow_mapping() {
let s = let s = "
"
{ {
a simple key: a value, # Note that the KEY token is produced. a simple key: a value, # Note that the KEY token is produced.
? a complex key: another value, ? a complex key: another value,
@ -1751,8 +1854,7 @@ macro_rules! end {
#[test] #[test]
fn test_block_sequences() { fn test_block_sequences() {
let s = let s = "
"
- item 1 - item 1
- item 2 - item 2
- -
@ -1794,8 +1896,7 @@ macro_rules! end {
#[test] #[test]
fn test_block_mappings() { fn test_block_mappings() {
let s = let s = "
"
a simple key: a value # The KEY token is produced here. a simple key: a value # The KEY token is produced here.
? a complex key ? a complex key
: another value : another value
@ -1842,13 +1943,11 @@ a sequence:
next!(p, BlockEnd); next!(p, BlockEnd);
next!(p, StreamEnd); next!(p, StreamEnd);
end!(p); end!(p);
} }
#[test] #[test]
fn test_no_block_sequence_start() { fn test_no_block_sequence_start() {
let s = let s = "
"
key: key:
- item 1 - item 1
- item 2 - item 2
@ -1870,8 +1969,7 @@ key:
#[test] #[test]
fn test_collections_in_sequence() { fn test_collections_in_sequence() {
let s = let s = "
"
- - item 1 - - item 1
- item 2 - item 2
- key 1: value 1 - key 1: value 1
@ -1914,8 +2012,7 @@ key:
#[test] #[test]
fn test_collections_in_mapping() { fn test_collections_in_mapping() {
let s = let s = "
"
? a sequence ? a sequence
: - item 1 : - item 1
- item 2 - item 2
@ -1955,8 +2052,7 @@ key:
#[test] #[test]
fn test_spec_ex7_3() { fn test_spec_ex7_3() {
let s = let s = "
"
{ {
? foo :, ? foo :,
: bar, : bar,

154
parser/src/yaml.rs
View file

@ -1,13 +1,13 @@
use linked_hash_map::LinkedHashMap;
use parser::*;
use scanner::{Marker, ScanError, TScalarStyle, TokenType};
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::f64;
use std::i64;
use std::mem;
use std::ops::Index; use std::ops::Index;
use std::string; use std::string;
use std::i64;
use std::f64;
use std::mem;
use std::vec; use std::vec;
use parser::*;
use scanner::{TScalarStyle, ScanError, TokenType, Marker};
use linked_hash_map::LinkedHashMap;
/// A YAML node is stored as this `Yaml` enumeration, which provides an easy way to /// A YAML node is stored as this `Yaml` enumeration, which provides an easy way to
/// access your YAML document. /// access your YAML document.
@ -62,7 +62,7 @@ fn parse_f64(v: &str) -> Option<f64> {
".inf" | ".Inf" | ".INF" | "+.inf" | "+.Inf" | "+.INF" => Some(f64::INFINITY), ".inf" | ".Inf" | ".INF" | "+.inf" | "+.Inf" | "+.INF" => Some(f64::INFINITY),
"-.inf" | "-.Inf" | "-.INF" => Some(f64::NEG_INFINITY), "-.inf" | "-.Inf" | "-.INF" => Some(f64::NEG_INFINITY),
".nan" | "NaN" | ".NAN" => Some(f64::NAN), ".nan" | "NaN" | ".NAN" => Some(f64::NAN),
_ => v.parse::<f64>().ok() _ => v.parse::<f64>().ok(),
} }
} }
@ -81,31 +81,31 @@ impl MarkedEventReceiver for YamlLoader {
match ev { match ev {
Event::DocumentStart => { Event::DocumentStart => {
// do nothing // do nothing
}, }
Event::DocumentEnd => { Event::DocumentEnd => {
match self.doc_stack.len() { match self.doc_stack.len() {
// empty document // empty document
0 => self.docs.push(Yaml::BadValue), 0 => self.docs.push(Yaml::BadValue),
1 => self.docs.push(self.doc_stack.pop().unwrap().0), 1 => self.docs.push(self.doc_stack.pop().unwrap().0),
_ => unreachable!() _ => unreachable!(),
}
} }
},
Event::SequenceStart(aid) => { Event::SequenceStart(aid) => {
self.doc_stack.push((Yaml::Array(Vec::new()), aid)); self.doc_stack.push((Yaml::Array(Vec::new()), aid));
}, }
Event::SequenceEnd => { Event::SequenceEnd => {
let node = self.doc_stack.pop().unwrap(); let node = self.doc_stack.pop().unwrap();
self.insert_new_node(node); self.insert_new_node(node);
}, }
Event::MappingStart(aid) => { Event::MappingStart(aid) => {
self.doc_stack.push((Yaml::Hash(Hash::new()), aid)); self.doc_stack.push((Yaml::Hash(Hash::new()), aid));
self.key_stack.push(Yaml::BadValue); self.key_stack.push(Yaml::BadValue);
}, }
Event::MappingEnd => { Event::MappingEnd => {
self.key_stack.pop().unwrap(); self.key_stack.pop().unwrap();
let node = self.doc_stack.pop().unwrap(); let node = self.doc_stack.pop().unwrap();
self.insert_new_node(node); self.insert_new_node(node);
}, }
Event::Scalar(v, style, aid, tag) => { Event::Scalar(v, style, aid, tag) => {
let node = if style != TScalarStyle::Plain { let node = if style != TScalarStyle::Plain {
Yaml::String(v) Yaml::String(v)
@ -117,27 +117,21 @@ impl MarkedEventReceiver for YamlLoader {
// "true" or "false" // "true" or "false"
match v.parse::<bool>() { match v.parse::<bool>() {
Err(_) => Yaml::BadValue, Err(_) => Yaml::BadValue,
Ok(v) => Yaml::Boolean(v) Ok(v) => Yaml::Boolean(v),
} }
}, }
"int" => { "int" => match v.parse::<i64>() {
match v.parse::<i64>() {
Err(_) => Yaml::BadValue, Err(_) => Yaml::BadValue,
Ok(v) => Yaml::Integer(v) Ok(v) => Yaml::Integer(v),
}
}, },
"float" => { "float" => match parse_f64(&v) {
match parse_f64(&v) {
Some(_) => Yaml::Real(v), Some(_) => Yaml::Real(v),
None => Yaml::BadValue, None => Yaml::BadValue,
}
}, },
"null" => { "null" => match v.as_ref() {
match v.as_ref() {
"~" | "null" => Yaml::Null, "~" | "null" => Yaml::Null,
_ => Yaml::BadValue, _ => Yaml::BadValue,
} },
}
_ => Yaml::String(v), _ => Yaml::String(v),
} }
} else { } else {
@ -149,7 +143,7 @@ impl MarkedEventReceiver for YamlLoader {
}; };
self.insert_new_node((node, aid)); self.insert_new_node((node, aid));
}, }
Event::Alias(id) => { Event::Alias(id) => {
let n = match self.anchor_map.get(&id) { let n = match self.anchor_map.get(&id) {
Some(v) => v.clone(), Some(v) => v.clone(),
@ -186,7 +180,7 @@ impl YamlLoader {
mem::swap(&mut newkey, cur_key); mem::swap(&mut newkey, cur_key);
h.insert(newkey, node.0); h.insert(newkey, node.0);
} }
}, }
_ => unreachable!(), _ => unreachable!(),
} }
} }
@ -255,35 +249,35 @@ impl Yaml {
pub fn is_null(&self) -> bool { pub fn is_null(&self) -> bool {
match *self { match *self {
Yaml::Null => true, Yaml::Null => true,
_ => false _ => false,
} }
} }
pub fn is_badvalue(&self) -> bool { pub fn is_badvalue(&self) -> bool {
match *self { match *self {
Yaml::BadValue => true, Yaml::BadValue => true,
_ => false _ => false,
} }
} }
pub fn is_array(&self) -> bool { pub fn is_array(&self) -> bool {
match *self { match *self {
Yaml::Array(_) => true, Yaml::Array(_) => true,
_ => false _ => false,
} }
} }
pub fn as_f64(&self) -> Option<f64> { pub fn as_f64(&self) -> Option<f64> {
match *self { match *self {
Yaml::Real(ref v) => parse_f64(v), Yaml::Real(ref v) => parse_f64(v),
_ => None _ => None,
} }
} }
pub fn into_f64(self) -> Option<f64> { pub fn into_f64(self) -> Option<f64> {
match self { match self {
Yaml::Real(ref v) => parse_f64(v), Yaml::Real(ref v) => parse_f64(v),
_ => None _ => None,
} }
} }
} }
@ -315,7 +309,7 @@ impl Yaml {
_ if v.parse::<i64>().is_ok() => Yaml::Integer(v.parse::<i64>().unwrap()), _ if v.parse::<i64>().is_ok() => Yaml::Integer(v.parse::<i64>().unwrap()),
// try parsing as f64 // try parsing as f64
_ if parse_f64(v).is_some() => Yaml::Real(v.to_owned()), _ if parse_f64(v).is_some() => Yaml::Real(v.to_owned()),
_ => Yaml::String(v.to_owned()) _ => Yaml::String(v.to_owned()),
} }
} }
} }
@ -328,7 +322,7 @@ impl<'a> Index<&'a str> for Yaml {
let key = Yaml::String(idx.to_owned()); let key = Yaml::String(idx.to_owned());
match self.as_hash() { match self.as_hash() {
Some(h) => h.get(&key).unwrap_or(&BAD_VALUE), Some(h) => h.get(&key).unwrap_or(&BAD_VALUE),
None => &BAD_VALUE None => &BAD_VALUE,
} }
} }
} }
@ -354,8 +348,7 @@ impl IntoIterator for Yaml {
fn into_iter(self) -> Self::IntoIter { fn into_iter(self) -> Self::IntoIter {
YamlIter { YamlIter {
yaml: self.into_vec() yaml: self.into_vec().unwrap_or_else(Vec::new).into_iter(),
.unwrap_or_else(Vec::new).into_iter()
} }
} }
} }
@ -374,8 +367,8 @@ impl Iterator for YamlIter {
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use yaml::*;
use std::f64; use std::f64;
use yaml::*;
#[test] #[test]
fn test_coerce() { fn test_coerce() {
let s = "--- let s = "---
@ -424,8 +417,7 @@ a7: 你好
#[test] #[test]
fn test_multi_doc() { fn test_multi_doc() {
let s = let s = "
"
'a scalar' 'a scalar'
--- ---
'a scalar' 'a scalar'
@ -438,8 +430,7 @@ a7: 你好
#[test] #[test]
fn test_anchor() { fn test_anchor() {
let s = let s = "
"
a1: &DEFAULT a1: &DEFAULT
b1: 4 b1: 4
b2: d b2: d
@ -452,8 +443,7 @@ a2: *DEFAULT
#[test] #[test]
fn test_bad_anchor() { fn test_bad_anchor() {
let s = let s = "
"
a1: &DEFAULT a1: &DEFAULT
b1: 4 b1: 4
b2: *DEFAULT b2: *DEFAULT
@ -461,7 +451,6 @@ a1: &DEFAULT
let out = YamlLoader::load_from_str(&s).unwrap(); let out = YamlLoader::load_from_str(&s).unwrap();
let doc = &out[0]; let doc = &out[0];
assert_eq!(doc["a1"]["b2"], Yaml::BadValue); assert_eq!(doc["a1"]["b2"], Yaml::BadValue);
} }
#[test] #[test]
@ -475,8 +464,7 @@ a1: &DEFAULT
#[test] #[test]
fn test_plain_datatype() { fn test_plain_datatype() {
let s = let s = "
"
- 'string' - 'string'
- \"string\" - \"string\"
- string - string
@ -555,15 +543,23 @@ a1: &DEFAULT
#[test] #[test]
fn test_bad_docstart() { fn test_bad_docstart() {
assert!(YamlLoader::load_from_str("---This used to cause an infinite loop").is_ok()); assert!(YamlLoader::load_from_str("---This used to cause an infinite loop").is_ok());
assert_eq!(YamlLoader::load_from_str("----"), Ok(vec![Yaml::String(String::from("----"))])); assert_eq!(
assert_eq!(YamlLoader::load_from_str("--- #here goes a comment"), Ok(vec![Yaml::Null])); YamlLoader::load_from_str("----"),
assert_eq!(YamlLoader::load_from_str("---- #here goes a comment"), Ok(vec![Yaml::String(String::from("----"))])); Ok(vec![Yaml::String(String::from("----"))])
);
assert_eq!(
YamlLoader::load_from_str("--- #here goes a comment"),
Ok(vec![Yaml::Null])
);
assert_eq!(
YamlLoader::load_from_str("---- #here goes a comment"),
Ok(vec![Yaml::String(String::from("----"))])
);
} }
#[test] #[test]
fn test_plain_datatype_with_into_methods() { fn test_plain_datatype_with_into_methods() {
let s = let s = "
"
- 'string' - 'string'
- \"string\" - \"string\"
- string - string
@ -620,9 +616,18 @@ c: ~
let out = YamlLoader::load_from_str(&s).unwrap(); let out = YamlLoader::load_from_str(&s).unwrap();
let first = out.into_iter().next().unwrap(); let first = out.into_iter().next().unwrap();
let mut iter = first.into_hash().unwrap().into_iter(); let mut iter = first.into_hash().unwrap().into_iter();
assert_eq!(Some((Yaml::String("b".to_owned()), Yaml::Null)), iter.next()); assert_eq!(
assert_eq!(Some((Yaml::String("a".to_owned()), Yaml::Null)), iter.next()); Some((Yaml::String("b".to_owned()), Yaml::Null)),
assert_eq!(Some((Yaml::String("c".to_owned()), Yaml::Null)), iter.next()); iter.next()
);
assert_eq!(
Some((Yaml::String("a".to_owned()), Yaml::Null)),
iter.next()
);
assert_eq!(
Some((Yaml::String("c".to_owned()), Yaml::Null)),
iter.next()
);
assert_eq!(None, iter.next()); assert_eq!(None, iter.next());
} }
@ -641,30 +646,49 @@ c: ~
#[test] #[test]
fn test_indentation_equality() { fn test_indentation_equality() {
let four_spaces = YamlLoader::load_from_str(
let four_spaces = YamlLoader::load_from_str(r#" r#"
hash: hash:
with: with:
indentations indentations
"#).unwrap().into_iter().next().unwrap(); "#,
).unwrap()
.into_iter()
.next()
.unwrap();
let two_spaces = YamlLoader::load_from_str(r#" let two_spaces = YamlLoader::load_from_str(
r#"
hash: hash:
with: with:
indentations indentations
"#).unwrap().into_iter().next().unwrap(); "#,
).unwrap()
.into_iter()
.next()
.unwrap();
let one_space = YamlLoader::load_from_str(r#" let one_space = YamlLoader::load_from_str(
r#"
hash: hash:
with: with:
indentations indentations
"#).unwrap().into_iter().next().unwrap(); "#,
).unwrap()
.into_iter()
.next()
.unwrap();
let mixed_spaces = YamlLoader::load_from_str(r#" let mixed_spaces = YamlLoader::load_from_str(
r#"
hash: hash:
with: with:
indentations indentations
"#).unwrap().into_iter().next().unwrap(); "#,
).unwrap()
.into_iter()
.next()
.unwrap();
assert_eq!(four_spaces, two_spaces); assert_eq!(four_spaces, two_spaces);
assert_eq!(two_spaces, one_space); assert_eq!(two_spaces, one_space);

2
parser/tests/quickcheck.rs
View file

@ -3,8 +3,8 @@ extern crate yaml_rust;
extern crate quickcheck; extern crate quickcheck;
use quickcheck::TestResult; use quickcheck::TestResult;
use yaml_rust::{Yaml, YamlLoader, YamlEmitter};
use std::error::Error; use std::error::Error;
use yaml_rust::{Yaml, YamlEmitter, YamlLoader};
quickcheck! { quickcheck! {
fn test_check_weird_keys(xs: Vec<String>) -> TestResult { fn test_check_weird_keys(xs: Vec<String>) -> TestResult {

25
parser/tests/spec_test.rs
View file

@ -2,7 +2,7 @@
#![allow(non_upper_case_globals)] #![allow(non_upper_case_globals)]
extern crate yaml_rust; extern crate yaml_rust;
use yaml_rust::parser::{Parser, EventReceiver, Event}; use yaml_rust::parser::{Event, EventReceiver, Parser};
use yaml_rust::scanner::TScalarStyle; use yaml_rust::scanner::TScalarStyle;
// These names match the names used in the C++ test suite. // These names match the names used in the C++ test suite.
@ -21,7 +21,7 @@ enum TestEvent {
} }
struct YamlChecker { struct YamlChecker {
pub evs: Vec<TestEvent> pub evs: Vec<TestEvent>,
} }
impl EventReceiver for YamlChecker { impl EventReceiver for YamlChecker {
@ -39,30 +39,30 @@ impl EventReceiver for YamlChecker {
} else { } else {
TestEvent::OnScalar TestEvent::OnScalar
} }
}, }
Event::Alias(_) => TestEvent::OnAlias, Event::Alias(_) => TestEvent::OnAlias,
_ => { return } // ignore other events _ => return, // ignore other events
}; };
self.evs.push(tev); self.evs.push(tev);
} }
} }
fn str_to_test_events(docs: &str) -> Vec<TestEvent> { fn str_to_test_events(docs: &str) -> Vec<TestEvent> {
let mut p = YamlChecker { let mut p = YamlChecker { evs: Vec::new() };
evs: Vec::new()
};
let mut parser = Parser::new(docs.chars()); let mut parser = Parser::new(docs.chars());
parser.load(&mut p, true).unwrap(); parser.load(&mut p, true).unwrap();
p.evs p.evs
} }
macro_rules! assert_next { macro_rules! assert_next {
($v:expr, $p:pat) => ( ($v:expr, $p:pat) => {
match $v.next().unwrap() { match $v.next().unwrap() {
$p => {}, $p => {}
e => { panic!("unexpected event: {:?}", e); } e => {
panic!("unexpected event: {:?}", e);
} }
) }
};
} }
// auto generated from handler_spec_test.cpp // auto generated from handler_spec_test.cpp
@ -77,7 +77,7 @@ include!("spec_test.rs.inc");
#[test] #[test]
fn test_mapvec_legal() { fn test_mapvec_legal() {
use yaml_rust::yaml::{Array, Hash, Yaml}; use yaml_rust::yaml::{Array, Hash, Yaml};
use yaml_rust::{YamlLoader, YamlEmitter}; use yaml_rust::{YamlEmitter, YamlLoader};
// Emitting a `map<map<seq<_>>, _>` should result in legal yaml that // Emitting a `map<map<seq<_>>, _>` should result in legal yaml that
// we can parse. // we can parse.
@ -138,4 +138,3 @@ fn test_mapvec_legal() {
YamlLoader::load_from_str(&out_str).unwrap(); YamlLoader::load_from_str(&out_str).unwrap();
} }