Propagate tag to MappingStart event.

parser/src/parser.rs

@@ -51,7 +51,7 @@ pub enum Event {
         usize,
     ),
     /// Value, style, anchor_id, tag
-    Scalar(String, TScalarStyle, usize, Option<TokenType>),
+    Scalar(String, TScalarStyle, usize, Option<Tag>),
     SequenceStart(
         /// The anchor ID of the start of the squence.
         usize,
@@ -60,10 +60,21 @@ pub enum Event {
     MappingStart(
         /// The anchor ID of the start of the mapping.
         usize,
+        /// An optional tag
+        Option<Tag>,
     ),
     MappingEnd,
 }
 
+/// A YAML tag.
+#[derive(Clone, PartialEq, Debug, Eq)]
+pub struct Tag {
+    /// Handle of the tag (`!` included).
+    pub handle: String,
+    /// The suffix of the tag.
+    pub suffix: String,
+}
+
 impl Event {
     /// Create an empty scalar.
     fn empty_scalar() -> Event {
@@ -72,7 +83,7 @@ impl Event {
     }
 
     /// Create an empty scalar with the given anchor.
-    fn empty_scalar_with_anchor(anchor: usize, tag: Option<TokenType>) -> Event {
+    fn empty_scalar_with_anchor(anchor: usize, tag: Option<Tag>) -> Event {
         Event::Scalar(String::new(), TScalarStyle::Plain, anchor, tag)
     }
 }
@@ -340,7 +351,7 @@ impl<T: Iterator<Item = char>> Parser<T> {
                 recv.on_event(first_ev, mark);
                 self.load_sequence(recv)
             }
-            Event::MappingStart(_) => {
+            Event::MappingStart(..) => {
                 recv.on_event(first_ev, mark);
                 self.load_mapping(recv)
             }
@@ -575,8 +586,8 @@ impl<T: Iterator<Item = char>> Parser<T> {
                 if let Token(mark, TokenType::Anchor(name)) = self.fetch_token() {
                     anchor_id = self.register_anchor(name, &mark);
                     if let TokenType::Tag(..) = self.peek_token()?.1 {
-                        if let tg @ TokenType::Tag(..) = self.fetch_token().1 {
+                        if let TokenType::Tag(handle, suffix) = self.fetch_token().1 {
-                            tag = Some(tg);
+                            tag = Some(Tag { handle, suffix });
                         } else {
                             unreachable!()
                         }
@@ -586,9 +597,9 @@ impl<T: Iterator<Item = char>> Parser<T> {
                 }
             }
             Token(_, TokenType::Tag(..)) => {
-                if let tg @ TokenType::Tag(..) = self.fetch_token().1 {
+                if let TokenType::Tag(handle, suffix) = self.fetch_token().1 {
-                    tag = Some(tg);
+                    tag = Some(Tag { handle, suffix });
-                    if let TokenType::Anchor(_) = self.peek_token()?.1 {
+                    if let TokenType::Anchor(_) = &self.peek_token()?.1 {
                         if let Token(mark, TokenType::Anchor(name)) = self.fetch_token() {
                             anchor_id = self.register_anchor(name, &mark);
                         } else {
@@ -620,7 +631,7 @@ impl<T: Iterator<Item = char>> Parser<T> {
             }
             Token(mark, TokenType::FlowMappingStart) => {
                 self.state = State::FlowMappingFirstKey;
-                Ok((Event::MappingStart(anchor_id), mark))
+                Ok((Event::MappingStart(anchor_id, tag), mark))
             }
             Token(mark, TokenType::BlockSequenceStart) if block => {
                 self.state = State::BlockSequenceFirstEntry;
@@ -628,7 +639,7 @@ impl<T: Iterator<Item = char>> Parser<T> {
             }
             Token(mark, TokenType::BlockMappingStart) if block => {
                 self.state = State::BlockMappingFirstKey;
-                Ok((Event::MappingStart(anchor_id), mark))
+                Ok((Event::MappingStart(anchor_id, tag), mark))
             }
             // ex 7.2, an empty scalar can follow a secondary tag
             Token(mark, _) if tag.is_some() || anchor_id > 0 => {
@@ -819,7 +830,7 @@ impl<T: Iterator<Item = char>> Parser<T> {
             Token(mark, TokenType::Key) => {
                 self.state = State::FlowSequenceEntryMappingKey;
                 self.skip();
-                Ok((Event::MappingStart(0), mark))
+                Ok((Event::MappingStart(0, None), mark))
             }
             _ => {
                 self.push_state(State::FlowSequenceEntry);

parser/src/scanner.rs

@@ -109,26 +109,43 @@ impl fmt::Display for ScanError {
 #[derive(Clone, PartialEq, Debug, Eq)]
 pub enum TokenType {
     NoToken,
+    /// The start of the stream. Sent first, before even [`DocumentStart`].
     StreamStart(TEncoding),
+    /// The end of the stream, EOF.
     StreamEnd,
-    /// major, minor
+    VersionDirective(
-    VersionDirective(u32, u32),
+        /// Major
-    /// handle, prefix
+        u32,
-    TagDirective(String, String),
+        /// Minor
+        u32,
+    ),
+    TagDirective(
+        /// Handle
+        String,
+        /// Prefix
+        String,
+    ),
+    /// The start of a YAML document (`---`).
     DocumentStart,
+    /// The end of a YAML document (`...`).
     DocumentEnd,
     BlockSequenceStart,
     BlockMappingStart,
     BlockEnd,
+    /// Start of an inline array (`[ a, b ]`).
     FlowSequenceStart,
+    /// End of an inline array.
     FlowSequenceEnd,
+    /// Start of an inline mapping (`{ a: b, c: d }`).
     FlowMappingStart,
+    /// End of an inline mapping.
     FlowMappingEnd,
     BlockEntry,
     FlowEntry,
     Key,
     Value,
     Alias(String),
+    /// A YAML anchor (`&`/`*`).
     Anchor(String),
     /// handle, suffix
     Tag(String, String),

parser/src/yaml.rs

@@ -1,6 +1,6 @@
 #![allow(clippy::module_name_repetitions)]
 
-use crate::parser::{Event, MarkedEventReceiver, Parser};
+use crate::parser::{Event, MarkedEventReceiver, Parser, Tag};
 use crate::scanner::{Marker, ScanError, TScalarStyle, TokenType};
 use linked_hash_map::LinkedHashMap;
 use std::collections::BTreeMap;
@@ -98,7 +98,7 @@ impl MarkedEventReceiver for YamlLoader {
                 let node = self.doc_stack.pop().unwrap();
                 self.insert_new_node(node);
             }
-            Event::MappingStart(aid) => {
+            Event::MappingStart(aid, _) => {
                 self.doc_stack.push((Yaml::Hash(Hash::new()), aid));
                 self.key_stack.push(Yaml::BadValue);
             }
@@ -110,7 +110,7 @@ impl MarkedEventReceiver for YamlLoader {
             Event::Scalar(v, style, aid, tag) => {
                 let node = if style != TScalarStyle::Plain {
                     Yaml::String(v)
-                } else if let Some(TokenType::Tag(ref handle, ref suffix)) = tag {
+                } else if let Some(Tag { ref handle, ref suffix }) = tag {
                     // XXX tag:yaml.org,2002:
                     if handle == "!!" {
                         match suffix.as_ref() {

parser/tests/yaml-test-suite.rs

@@ -3,8 +3,8 @@ use std::fs::{self, DirEntry};
 use libtest_mimic::{run_tests, Arguments, Outcome, Test};
 
 use yaml_rust::{
-    parser::{Event, EventReceiver, Parser},
+    parser::{Event, EventReceiver, Parser, Tag},
-    scanner::{TScalarStyle, TokenType},
+    scanner::{TScalarStyle},
     yaml, ScanError, Yaml, YamlLoader,
 };
 
@@ -148,7 +148,9 @@ impl EventReceiver for EventReporter {
             Event::SequenceStart(idx) => format!("+SEQ{}", format_index(idx)),
             Event::SequenceEnd => "-SEQ".into(),
 
-            Event::MappingStart(idx) => format!("+MAP{}", format_index(idx)),
+            Event::MappingStart(idx, tag) => {
+                format!("+MAP{}{}", format_index(idx), format_tag(&tag))
+            }
             Event::MappingEnd => "-MAP".into(),
 
             Event::Scalar(ref text, style, idx, ref tag) => {
@@ -197,13 +199,13 @@ fn escape_text(text: &str) -> String {
     text
 }
 
-fn format_tag(tag: &Option<TokenType>) -> String {
+fn format_tag(tag: &Option<Tag>) -> String {
-    if let Some(TokenType::Tag(ns, tag)) = tag {
+    if let Some(tag) = tag {
-        let ns = match ns.as_str() {
+        let ns = match tag.handle.as_str() {
             "!!" => "tag:yaml.org,2002:", // Wrong if this ns is overridden
             other => other,
         };
-        format!(" <{}{}>", ns, tag)
+        format!(" <{}{}>", ns, tag.suffix)
     } else {
         "".into()
     }
@@ -302,13 +304,9 @@ static EXPECTED_FAILURES: &[&str] = &[
     "35KP",
     "57H4",
     "6JWB",
-    "735Y",
-    "9KAX",
-    "BU8L",
     "C4HZ",
     "EHF6",
     "J7PZ",
-    "UGM3",
     // Cannot resolve tag namespaces
     "5TYM",
     "6CK3",