mirror of
https://chromium.googlesource.com/crosvm/crosvm
synced 2024-12-25 04:14:06 +00:00
poll_token: Use syn to simplify poll token derive
This CL removes 300 lines of parsing code and 200 lines of tests of parsing code by using the parsers provided by Syn, which we already use in implementing our other custom derives. TEST=cargo test poll_token_derive TEST=cargo check crosvm Change-Id: Ie2743b1bbb1b374326f9845fc37fc578b178c53d Reviewed-on: https://chromium-review.googlesource.com/1365112 Commit-Ready: ChromeOS CL Exonerator Bot <chromiumos-cl-exonerator@appspot.gserviceaccount.com> Tested-by: David Tolnay <dtolnay@chromium.org> Reviewed-by: Zach Reizner <zachr@chromium.org>
This commit is contained in:
David Tolnay
chrome-bot
parent
caf32ee5bb
commit
d4d9c26f04
4 changed files with 194 additions and 684 deletions
5
Cargo.lock
generated
5
Cargo.lock
generated
|
|
@ -317,6 +317,11 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "poll_token_derive"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"proc-macro2 0.4.24 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"quote 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"syn 0.15.21 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
|
|
|
|||
|
|
@ -6,3 +6,8 @@ authors = ["The Chromium OS Authors"]
|
|||
[lib]
|
||||
proc-macro = true
|
||||
path = "poll_token_derive.rs"
|
||||
|
||||
[dependencies]
|
||||
syn = "0.15"
|
||||
quote = "0.6"
|
||||
proc-macro2 = "0.4"
|
||||
|
|
|
|||
|
|
@ -2,312 +2,21 @@
|
|||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
extern crate proc_macro;
|
||||
#![recursion_limit = "128"]
|
||||
|
||||
use proc_macro::TokenStream;
|
||||
use std::fmt::Write;
|
||||
use std::mem;
|
||||
use std::str::FromStr;
|
||||
extern crate proc_macro;
|
||||
extern crate proc_macro2;
|
||||
extern crate quote;
|
||||
extern crate syn;
|
||||
|
||||
use proc_macro2::{Ident, TokenStream};
|
||||
use quote::quote;
|
||||
use syn::{parse_macro_input, Data, DeriveInput, Field, Fields, Index, Member, Variant};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
// This file is meant to be read from top to bottom to reflect how this code processes and generates
|
||||
// Rust enum definitions and implementations. The algorithm overview:
|
||||
// 1) Split the rust source by whitespace (`str::split_whitespace`).
|
||||
// 2a) Attempt to tokenize each piece (see: `Tokenized`, all functions starting with `matches`).
|
||||
// 2b) Feed the token to the `ParseState` (see `ParseState::handle_token`).
|
||||
// 3) After the source is fully processed, the `ParseState` has an `EnumModel` representing the
|
||||
// input enum.
|
||||
// 4) Glue together an implementation of PollToken using a template.
|
||||
|
||||
// A single token after splitting input source by white space and simple stateless matching.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
enum Tokenized {
|
||||
// `enum`
|
||||
Enum,
|
||||
|
||||
// `pub' | `pub(crate)`
|
||||
Visiblity,
|
||||
|
||||
// `Hello`, `index`, `data,`
|
||||
Ident(String),
|
||||
|
||||
// `index:`, 'first:`
|
||||
FieldIdent(String),
|
||||
|
||||
// `Socket(u32)`, `Client(usize),`,
|
||||
IdentAndType(String, String),
|
||||
|
||||
// `{`
|
||||
OpenBrace,
|
||||
|
||||
// `}`, `},`
|
||||
CloseBrace,
|
||||
}
|
||||
|
||||
// Attempts to match strings of the form "identifier" with optional trailing comma.
|
||||
fn matches_ident(s: &str) -> Option<String> {
|
||||
let ident = s.trim_right_matches(',');
|
||||
if !ident.is_empty() && ident.chars().all(char::is_alphanumeric) {
|
||||
Some(ident.to_owned())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
// Attempts to match strings of the form "Identifier(Type)" with optional trailing comma. If the
|
||||
// given string matches, the identifier and type are returned as a 2-tuple receptively.
|
||||
fn matches_ident_and_type(s: &str) -> Option<(String, String)> {
|
||||
let mut buffer = String::new();
|
||||
let mut ident = String::new();
|
||||
let mut type_ = String::new();
|
||||
let mut brace_depth = 0;
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'(' if brace_depth == 0 && !buffer.is_empty() && ident.is_empty() => {
|
||||
mem::swap(&mut ident, &mut buffer);
|
||||
brace_depth += 1;
|
||||
}
|
||||
')' if brace_depth == 1 && !buffer.is_empty() && type_.is_empty() => {
|
||||
mem::swap(&mut type_, &mut buffer);
|
||||
brace_depth -= 1;
|
||||
}
|
||||
',' => {}
|
||||
c if c.is_alphanumeric() => buffer.push(c),
|
||||
_ => return None,
|
||||
}
|
||||
}
|
||||
if !ident.is_empty() && !type_.is_empty() {
|
||||
Some((ident, type_))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
// Attempts to match strings of the form "identifier:".
|
||||
fn matches_field_ident(s: &str) -> Option<String> {
|
||||
let field_ident = s.trim_right_matches(':');
|
||||
if s.ends_with(':') && field_ident.chars().all(char::is_alphanumeric) {
|
||||
Some(field_ident.to_owned())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
impl Tokenized {
|
||||
fn from_str(s: &str) -> Tokenized {
|
||||
if s.starts_with("pub(") {
|
||||
return Tokenized::Visiblity;
|
||||
}
|
||||
match s {
|
||||
"enum" => Tokenized::Enum,
|
||||
"pub" => Tokenized::Visiblity,
|
||||
"{" => Tokenized::OpenBrace,
|
||||
"}" | "}," => Tokenized::CloseBrace,
|
||||
_ => {
|
||||
// Try to match from most specific to least specific.
|
||||
if let Some(ident) = matches_field_ident(s) {
|
||||
Tokenized::FieldIdent(ident)
|
||||
} else if let Some((ident, type_)) = matches_ident_and_type(s) {
|
||||
Tokenized::IdentAndType(ident, type_)
|
||||
} else if let Some(ident) = matches_ident(s) {
|
||||
Tokenized::Ident(ident)
|
||||
} else {
|
||||
panic!("unable to parse token: {}", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Data field for an enum, with possible field name.
|
||||
#[derive(Debug, PartialEq)]
|
||||
struct EnumVariantData {
|
||||
type_: String,
|
||||
name: Option<String>,
|
||||
}
|
||||
|
||||
// Data for one variant of an enum, with optional single data field.
|
||||
#[derive(Debug, PartialEq)]
|
||||
struct EnumVariant {
|
||||
name: String,
|
||||
data: Option<EnumVariantData>,
|
||||
}
|
||||
|
||||
// Data for an entire enum type.
|
||||
#[derive(Debug, Default, PartialEq)]
|
||||
struct EnumModel {
|
||||
name: String,
|
||||
variants: Vec<EnumVariant>,
|
||||
}
|
||||
// Note: impl for EnumModel is below the parsing code and definitions because all of the methods are
|
||||
// for generating the PollToken impl.
|
||||
|
||||
// Labels for each of the states in the parsing state machine. The '->` symbol means that the given
|
||||
// state may transition to the state pointed to.
|
||||
#[derive(PartialEq, Debug)]
|
||||
enum States {
|
||||
// Initial state, expecting to see visibility rules (e.g. `pub`) or `enum` keyword.
|
||||
Start, // -> Ident
|
||||
|
||||
// Expect to see the name of the enum field.
|
||||
Ident, // -> Brace
|
||||
|
||||
// Expect to see an opening brace.
|
||||
Brace, // -> VariantIdent, -> End
|
||||
|
||||
// Expect to see a variant's name.
|
||||
VariantIdent, // -> VariantIdent, -> VariantData, -> End
|
||||
|
||||
// Expect to see the field name of a variant's data.
|
||||
VariantData, // -> VariantIdent, -> VariantDataType
|
||||
|
||||
// Expect to see the tye name of a variant's data.
|
||||
VariantDataType, // -> VariantData
|
||||
|
||||
// Expect to see no more tokens.
|
||||
End,
|
||||
}
|
||||
|
||||
// The state machine for parsing a stream of `Tokenized`. After the States::End state is reached, a
|
||||
// complete `EnumModel` is ready to be used for generating an implementation.
|
||||
struct ParseState {
|
||||
current_state: States,
|
||||
current_variant: Option<EnumVariant>,
|
||||
model: EnumModel,
|
||||
}
|
||||
|
||||
impl ParseState {
|
||||
fn new() -> ParseState {
|
||||
ParseState {
|
||||
current_state: States::Start,
|
||||
current_variant: Default::default(),
|
||||
model: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
// Handles the next token in the stream of tokens.
|
||||
fn handle_token(&mut self, tok: Tokenized) {
|
||||
match self.current_state {
|
||||
States::Start => self.handle_start(tok),
|
||||
States::Ident => self.handle_ident(tok),
|
||||
States::Brace => self.handle_brace(tok),
|
||||
States::VariantIdent => self.handle_variant_ident(tok),
|
||||
States::VariantData => self.handle_variant_data(tok),
|
||||
States::VariantDataType => self.handle_variant_data_type(tok),
|
||||
States::End => self.handle_end(tok),
|
||||
}
|
||||
}
|
||||
|
||||
// All the following are handlers name after the current state that handle the next token.
|
||||
|
||||
fn handle_start(&mut self, tok: Tokenized) {
|
||||
self.current_state = match tok {
|
||||
Tokenized::Enum => States::Ident,
|
||||
Tokenized::Visiblity => States::Start,
|
||||
_ => panic!("derives for enum types only"),
|
||||
};
|
||||
}
|
||||
|
||||
fn handle_ident(&mut self, tok: Tokenized) {
|
||||
self.current_state = match tok {
|
||||
Tokenized::Ident(ident) => {
|
||||
self.model.name = ident;
|
||||
States::Brace
|
||||
}
|
||||
_ => panic!("unexpected token: {:?}", tok),
|
||||
};
|
||||
}
|
||||
|
||||
fn handle_brace(&mut self, tok: Tokenized) {
|
||||
self.current_state = match tok {
|
||||
Tokenized::OpenBrace => States::VariantIdent,
|
||||
Tokenized::CloseBrace => States::End,
|
||||
_ => panic!("unexpected token: {:?}", tok),
|
||||
};
|
||||
}
|
||||
|
||||
fn handle_variant_ident(&mut self, tok: Tokenized) {
|
||||
// This handler is the most complex because it has the most branches for the new
|
||||
// `current_state`. Adding to that complexity is that many branches indicate a new variant
|
||||
// is being handled, which means the old `current_variant` needs to be added to `variants`
|
||||
// and a fresh one needs to be started with the fresh data embedded in the token.
|
||||
self.current_state = match tok {
|
||||
Tokenized::Ident(ident) => {
|
||||
let mut variant = Some(EnumVariant {
|
||||
name: ident,
|
||||
data: None,
|
||||
});
|
||||
mem::swap(&mut variant, &mut self.current_variant);
|
||||
if let Some(variant) = variant {
|
||||
self.model.variants.push(variant);
|
||||
}
|
||||
States::VariantIdent
|
||||
}
|
||||
Tokenized::IdentAndType(ident, type_) => {
|
||||
let variant_data = EnumVariantData { type_, name: None };
|
||||
let mut variant = Some(EnumVariant {
|
||||
name: ident,
|
||||
data: Some(variant_data),
|
||||
});
|
||||
mem::swap(&mut variant, &mut self.current_variant);
|
||||
if let Some(variant) = variant {
|
||||
self.model.variants.push(variant);
|
||||
}
|
||||
States::VariantIdent
|
||||
}
|
||||
Tokenized::OpenBrace => States::VariantData,
|
||||
Tokenized::CloseBrace => {
|
||||
let mut variant = Default::default();
|
||||
mem::swap(&mut variant, &mut self.current_variant);
|
||||
if let Some(variant) = variant {
|
||||
self.model.variants.push(variant);
|
||||
}
|
||||
States::End
|
||||
}
|
||||
_ => panic!("unexpected token: {:?}", tok),
|
||||
};
|
||||
}
|
||||
|
||||
fn handle_variant_data(&mut self, tok: Tokenized) {
|
||||
let variant = self.current_variant.as_mut().unwrap();
|
||||
self.current_state = match tok {
|
||||
Tokenized::FieldIdent(ident) => {
|
||||
assert!(
|
||||
variant.data.is_none(),
|
||||
"enum variant can only have one field"
|
||||
);
|
||||
variant.data = Some(EnumVariantData {
|
||||
type_: "".to_owned(),
|
||||
name: Some(ident),
|
||||
});
|
||||
States::VariantDataType
|
||||
}
|
||||
Tokenized::CloseBrace => States::VariantIdent,
|
||||
_ => panic!("unexpected token: {:?}", tok),
|
||||
};
|
||||
}
|
||||
|
||||
fn handle_variant_data_type(&mut self, tok: Tokenized) {
|
||||
let variant = self.current_variant.as_mut().unwrap();
|
||||
let variant_data = variant.data.as_mut().unwrap();
|
||||
self.current_state = match tok {
|
||||
Tokenized::Ident(ident) => {
|
||||
variant_data.type_ = ident;
|
||||
States::VariantData
|
||||
}
|
||||
_ => panic!("unexpected token: {:?}", tok),
|
||||
};
|
||||
}
|
||||
|
||||
fn handle_end(&mut self, tok: Tokenized) {
|
||||
panic!("unexpected tokens past ending brace: {:?}", tok);
|
||||
}
|
||||
}
|
||||
|
||||
// Continued from the above `EnumModel` definition. All methods are used for generating PollToken
|
||||
// implementation. The method for packing an enum into a u64 is as follows:
|
||||
// The method for packing an enum into a u64 is as follows:
|
||||
// 1) Reserve the lowest "ceil(log_2(x))" bits where x is the number of enum variants.
|
||||
// 2) Store the enum variant's index (0-based index based on order in the enum definition) in
|
||||
// reserved bits.
|
||||
|
|
@ -316,129 +25,138 @@ impl ParseState {
|
|||
// 1) Mask the raw token to just the reserved bits
|
||||
// 2) Match the reserved bits to the enum variant token.
|
||||
// 3) If the indicated enum variant had data, extract it from the unreserved bits.
|
||||
impl EnumModel {
|
||||
// Calculates the number of bits needed to store the variant index. Essentially the log base 2
|
||||
// of the number of variants, rounded up.
|
||||
fn variant_bits(&self) -> u32 {
|
||||
|
||||
// Calculates the number of bits needed to store the variant index. Essentially the log base 2
|
||||
// of the number of variants, rounded up.
|
||||
fn variant_bits(variants: &[Variant]) -> u32 {
|
||||
if variants.is_empty() {
|
||||
// The degenerate case of no variants.
|
||||
if self.variants.is_empty() {
|
||||
return 0;
|
||||
}
|
||||
self.variants.len().next_power_of_two().trailing_zeros()
|
||||
}
|
||||
|
||||
// Generates the function body for `as_raw_token`.
|
||||
fn generate_as_raw_token(&self) -> String {
|
||||
let variant_bits = self.variant_bits();
|
||||
let mut match_statement = "match *self {\n".to_owned();
|
||||
|
||||
// Each iteration corresponds to one variant's match arm.
|
||||
for (index, variant) in self.variants.iter().enumerate() {
|
||||
// The capture string is for everything between the variant identifier and the `=>` in
|
||||
// the match arm: the variant's data capture.
|
||||
let capture = match variant.data.as_ref() {
|
||||
Some(&EnumVariantData {
|
||||
name: Some(ref name),
|
||||
..
|
||||
}) => format!("{{ {}: data }}", name),
|
||||
Some(&EnumVariantData { .. }) => "(data)".to_owned(),
|
||||
None => "".to_owned(),
|
||||
};
|
||||
|
||||
// The modifier string ORs the variant index with extra bits from the variant data
|
||||
// field.
|
||||
let modifer = if variant.data.is_some() {
|
||||
format!(" | ((data as u64) << {})", variant_bits)
|
||||
} else {
|
||||
"".to_owned()
|
||||
};
|
||||
|
||||
// Assembly of the match arm.
|
||||
write!(
|
||||
match_statement,
|
||||
"{}::{}{} => {}{},\n",
|
||||
self.name, variant.name, capture, index, modifer
|
||||
).unwrap();
|
||||
}
|
||||
match_statement.push_str("}");
|
||||
match_statement
|
||||
}
|
||||
|
||||
// Generates the function body for `from_raw_token`.
|
||||
fn generate_from_raw_token(&self) -> String {
|
||||
let variant_bits = self.variant_bits();
|
||||
let variant_mask = (1 << variant_bits) - 1;
|
||||
|
||||
// The match expression only matches the bits for the variant index.
|
||||
let mut match_statement = format!("match data & 0x{:02x} {{\n", variant_mask);
|
||||
|
||||
// Each iteration corresponds to one variant's match arm.
|
||||
for (index, variant) in self.variants.iter().enumerate() {
|
||||
// The data string is for extracting the enum variant's data bits out of the raw token
|
||||
// data, which includes both variant index and data bits.
|
||||
let data = match variant.data.as_ref() {
|
||||
Some(&EnumVariantData {
|
||||
name: Some(ref name),
|
||||
ref type_,
|
||||
}) => format!("{{ {}: (data >> {}) as {} }}", name, variant_bits, type_),
|
||||
Some(&EnumVariantData {
|
||||
name: None,
|
||||
ref type_,
|
||||
}) => format!("((data >> {}) as {})", variant_bits, type_),
|
||||
None => "".to_owned(),
|
||||
};
|
||||
|
||||
// Assembly of the match arm.
|
||||
write!(
|
||||
match_statement,
|
||||
"{} => {}::{}{},\n",
|
||||
index, self.name, variant.name, data
|
||||
).unwrap();
|
||||
}
|
||||
match_statement.push_str("_ => unreachable!()\n}");
|
||||
match_statement
|
||||
0
|
||||
} else {
|
||||
variants.len().next_power_of_two().trailing_zeros()
|
||||
}
|
||||
}
|
||||
|
||||
// Because unit tests cannot create `TokenStream`s (apparently), we have an inner implementation
|
||||
// that deals in strings.
|
||||
fn poll_token_inner(src: &str) -> String {
|
||||
let src_tokens = src.split_whitespace();
|
||||
|
||||
// Parsing is done in two interleaved stages, tokenizing without context, followed by parsing
|
||||
// via state machine.
|
||||
let mut state = ParseState::new();
|
||||
for src_tok in src_tokens {
|
||||
let tok = Tokenized::from_str(src_tok);
|
||||
state.handle_token(tok);
|
||||
// Name of the field if it has one, otherwise 0 assuming this is the zeroth
|
||||
// field of a tuple variant.
|
||||
fn field_member(field: &Field) -> Member {
|
||||
match &field.ident {
|
||||
Some(name) => Member::Named(name.clone()),
|
||||
None => Member::Unnamed(Index::from(0)),
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
state.current_state,
|
||||
States::End,
|
||||
"unexpected end after parsing source enum"
|
||||
);
|
||||
// Generates the function body for `as_raw_token`.
|
||||
fn generate_as_raw_token(enum_name: &Ident, variants: &[Variant]) -> TokenStream {
|
||||
let variant_bits = variant_bits(variants);
|
||||
|
||||
// Each iteration corresponds to one variant's match arm.
|
||||
let cases = variants.iter().enumerate().map(|(index, variant)| {
|
||||
let variant_name = &variant.ident;
|
||||
let index = index as u64;
|
||||
|
||||
// The capture string is for everything between the variant identifier and the `=>` in
|
||||
// the match arm: the variant's data capture.
|
||||
let capture = variant.fields.iter().next().map(|field| {
|
||||
let member = field_member(&field);
|
||||
quote!({ #member: data })
|
||||
});
|
||||
|
||||
// The modifier string ORs the variant index with extra bits from the variant data
|
||||
// field.
|
||||
let modifier = match variant.fields {
|
||||
Fields::Named(_) | Fields::Unnamed(_) => Some(quote! {
|
||||
| ((data as u64) << #variant_bits)
|
||||
}),
|
||||
Fields::Unit => None,
|
||||
};
|
||||
|
||||
// Assembly of the match arm.
|
||||
quote! {
|
||||
#enum_name::#variant_name #capture => #index #modifier
|
||||
}
|
||||
});
|
||||
|
||||
quote! {
|
||||
match *self {
|
||||
#(
|
||||
#cases,
|
||||
)*
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Generates the function body for `from_raw_token`.
|
||||
fn generate_from_raw_token(enum_name: &Ident, variants: &[Variant]) -> TokenStream {
|
||||
let variant_bits = variant_bits(variants);
|
||||
let variant_mask = ((1 << variant_bits) - 1) as u64;
|
||||
|
||||
// Each iteration corresponds to one variant's match arm.
|
||||
let cases = variants.iter().enumerate().map(|(index, variant)| {
|
||||
let variant_name = &variant.ident;
|
||||
let index = index as u64;
|
||||
|
||||
// The data string is for extracting the enum variant's data bits out of the raw token
|
||||
// data, which includes both variant index and data bits.
|
||||
let data = variant.fields.iter().next().map(|field| {
|
||||
let member = field_member(&field);
|
||||
let ty = &field.ty;
|
||||
quote!({ #member: (data >> #variant_bits) as #ty })
|
||||
});
|
||||
|
||||
// Assembly of the match arm.
|
||||
quote! {
|
||||
#index => #enum_name::#variant_name #data
|
||||
}
|
||||
});
|
||||
|
||||
quote! {
|
||||
// The match expression only matches the bits for the variant index.
|
||||
match data & #variant_mask {
|
||||
#(
|
||||
#cases,
|
||||
)*
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The proc_macro::TokenStream type can only be constructed from within a
|
||||
// procedural macro, meaning that unit tests are not able to invoke `fn
|
||||
// poll_token` below as an ordinary Rust function. We factor out the logic into
|
||||
// a signature that deals with Syn and proc-macro2 types only which are not
|
||||
// restricted to a procedural macro invocation.
|
||||
fn poll_token_inner(input: DeriveInput) -> TokenStream {
|
||||
let variants: Vec<Variant> = match input.data {
|
||||
Data::Enum(data) => data.variants.into_iter().collect(),
|
||||
Data::Struct(_) | Data::Union(_) => panic!("input must be an enum"),
|
||||
};
|
||||
|
||||
for variant in &variants {
|
||||
assert!(variant.fields.iter().count() <= 1);
|
||||
}
|
||||
|
||||
// Given our basic model of a user given enum that is suitable as a token, we generate the
|
||||
// implementation. The implementation is NOT always well formed, such as when a variant's data
|
||||
// type is not bit shiftable or castable to u64, but we let Rust generate such errors as it
|
||||
// would be difficult to detect every kind of error. Importantly, every implementation that we
|
||||
// generate here and goes on to compile succesfully is sound.
|
||||
let model = state.model;
|
||||
format!(
|
||||
"impl PollToken for {} {{
|
||||
fn as_raw_token(&self) -> u64 {{
|
||||
{}
|
||||
}}
|
||||
|
||||
fn from_raw_token(data: u64) -> Self {{
|
||||
{}
|
||||
}}
|
||||
}}",
|
||||
model.name,
|
||||
model.generate_as_raw_token(),
|
||||
model.generate_from_raw_token()
|
||||
)
|
||||
let enum_name = input.ident;
|
||||
let as_raw_token = generate_as_raw_token(&enum_name, &variants);
|
||||
let from_raw_token = generate_from_raw_token(&enum_name, &variants);
|
||||
|
||||
quote! {
|
||||
impl PollToken for #enum_name {
|
||||
fn as_raw_token(&self) -> u64 {
|
||||
#as_raw_token
|
||||
}
|
||||
|
||||
fn from_raw_token(data: u64) -> Self {
|
||||
#from_raw_token
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Implements the PollToken trait for a given `enum`.
|
||||
|
|
@ -451,10 +169,7 @@ fn poll_token_inner(src: &str) -> String {
|
|||
/// zero. The number of bits truncated is equal to the number of bits used to store the variant
|
||||
/// index plus the number of bits above 64.
|
||||
#[proc_macro_derive(PollToken)]
|
||||
pub fn poll_token(input: TokenStream) -> TokenStream {
|
||||
// The token stream gets converted to a string in a rather regular way, which makes parsing
|
||||
// simpler. In particular, whitespace from the source enum is not carried over, instead replaced
|
||||
// with whatever the token stream's to_string function outputs. The rust parser has already
|
||||
// validated the syntax, so we can make lots of assumptions about the source being well formed.
|
||||
TokenStream::from_str(&poll_token_inner(&input.to_string())).unwrap()
|
||||
pub fn poll_token(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
|
||||
let input = parse_macro_input!(input as DeriveInput);
|
||||
poll_token_inner(input).into()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,279 +2,64 @@
|
|||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
mod tokenized {
|
||||
use Tokenized;
|
||||
use Tokenized::*;
|
||||
use quote::quote;
|
||||
use syn::{parse_quote, DeriveInput};
|
||||
|
||||
#[test]
|
||||
fn enum_() {
|
||||
assert_eq!(Tokenized::from_str("enum"), Enum);
|
||||
#[test]
|
||||
fn test_variant_bits() {
|
||||
let mut variants = vec![parse_quote!(A)];
|
||||
assert_eq!(::variant_bits(&variants), 0);
|
||||
|
||||
variants.push(parse_quote!(B));
|
||||
variants.push(parse_quote!(C));
|
||||
assert_eq!(::variant_bits(&variants), 2);
|
||||
|
||||
for _ in 0..1021 {
|
||||
variants.push(parse_quote!(Dynamic));
|
||||
}
|
||||
assert_eq!(::variant_bits(&variants), 10);
|
||||
|
||||
#[test]
|
||||
fn visibility() {
|
||||
assert_eq!(Tokenized::from_str("pub"), Visiblity);
|
||||
assert_eq!(Tokenized::from_str("pub(crate)"), Visiblity);
|
||||
assert_eq!(Tokenized::from_str("pub(my_module)"), Visiblity);
|
||||
assert_eq!(Tokenized::from_str("pub( crate )"), Visiblity);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ident() {
|
||||
assert_eq!(
|
||||
Tokenized::from_str("Important"),
|
||||
Ident("Important".to_owned())
|
||||
);
|
||||
assert_eq!(Tokenized::from_str("hello,"), Ident("hello".to_owned()));
|
||||
assert_eq!(Tokenized::from_str("world2"), Ident("world2".to_owned()));
|
||||
assert_eq!(Tokenized::from_str("A,"), Ident("A".to_owned()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn field_ident() {
|
||||
assert_eq!(
|
||||
Tokenized::from_str("index:"),
|
||||
FieldIdent("index".to_owned())
|
||||
);
|
||||
assert_eq!(Tokenized::from_str("a:"), FieldIdent("a".to_owned()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ident_and_type() {
|
||||
assert_eq!(
|
||||
Tokenized::from_str("a(u32)"),
|
||||
IdentAndType("a".to_owned(), "u32".to_owned())
|
||||
);
|
||||
assert_eq!(
|
||||
Tokenized::from_str("Socket(usize),"),
|
||||
IdentAndType("Socket".to_owned(), "usize".to_owned())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_brace() {
|
||||
assert_eq!(Tokenized::from_str("{"), OpenBrace);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn close_brace() {
|
||||
assert_eq!(Tokenized::from_str("}"), CloseBrace);
|
||||
assert_eq!(Tokenized::from_str("},"), CloseBrace);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic]
|
||||
fn empty() {
|
||||
Tokenized::from_str("");
|
||||
}
|
||||
}
|
||||
|
||||
mod parse_state {
|
||||
use Tokenized::*;
|
||||
use {EnumModel, EnumVariant, EnumVariantData, ParseState, States, Tokenized};
|
||||
|
||||
fn parse_tokens(tokens: &[Tokenized]) -> EnumModel {
|
||||
let mut state = ParseState::new();
|
||||
for token in tokens {
|
||||
state.handle_token(token.clone());
|
||||
}
|
||||
assert_eq!(
|
||||
state.current_state,
|
||||
States::End,
|
||||
"unexpected end after parsing source enum"
|
||||
);
|
||||
state.model
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_struct() {
|
||||
let model = parse_tokens(&[
|
||||
Visiblity,
|
||||
Enum,
|
||||
Ident("Blarg".to_owned()),
|
||||
OpenBrace,
|
||||
CloseBrace,
|
||||
]);
|
||||
let expected = EnumModel {
|
||||
name: "Blarg".to_string(),
|
||||
variants: Vec::new(),
|
||||
};
|
||||
assert_eq!(model, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic]
|
||||
fn invalid_token() {
|
||||
parse_tokens(&[
|
||||
Visiblity,
|
||||
Enum,
|
||||
Ident("Blarg".to_owned()),
|
||||
OpenBrace,
|
||||
CloseBrace,
|
||||
CloseBrace,
|
||||
]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_unit_variants() {
|
||||
let model = parse_tokens(&[
|
||||
Enum,
|
||||
Ident("Foo".to_owned()),
|
||||
OpenBrace,
|
||||
Ident("A".to_owned()),
|
||||
Ident("B".to_owned()),
|
||||
Ident("C".to_owned()),
|
||||
CloseBrace,
|
||||
]);
|
||||
let expected = EnumModel {
|
||||
name: "Foo".to_string(),
|
||||
variants: vec![
|
||||
EnumVariant {
|
||||
name: "A".to_owned(),
|
||||
data: None,
|
||||
},
|
||||
EnumVariant {
|
||||
name: "B".to_owned(),
|
||||
data: None,
|
||||
},
|
||||
EnumVariant {
|
||||
name: "C".to_owned(),
|
||||
data: None,
|
||||
},
|
||||
],
|
||||
};
|
||||
assert_eq!(model, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unnamed_data() {
|
||||
let model = parse_tokens(&[
|
||||
Enum,
|
||||
Ident("Foo".to_owned()),
|
||||
OpenBrace,
|
||||
IdentAndType("A".to_owned(), "u32".to_owned()),
|
||||
Ident("B".to_owned()),
|
||||
IdentAndType("C".to_owned(), "usize".to_owned()),
|
||||
CloseBrace,
|
||||
]);
|
||||
let expected = EnumModel {
|
||||
name: "Foo".to_string(),
|
||||
variants: vec![
|
||||
EnumVariant {
|
||||
name: "A".to_owned(),
|
||||
data: Some(EnumVariantData {
|
||||
name: None,
|
||||
type_: "u32".to_owned(),
|
||||
}),
|
||||
},
|
||||
EnumVariant {
|
||||
name: "B".to_owned(),
|
||||
data: None,
|
||||
},
|
||||
EnumVariant {
|
||||
name: "C".to_owned(),
|
||||
data: Some(EnumVariantData {
|
||||
name: None,
|
||||
type_: "usize".to_owned(),
|
||||
}),
|
||||
},
|
||||
],
|
||||
};
|
||||
assert_eq!(model, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn named_data() {
|
||||
let model = parse_tokens(&[
|
||||
Enum,
|
||||
Ident("Foo".to_owned()),
|
||||
OpenBrace,
|
||||
Ident("A".to_owned()),
|
||||
OpenBrace,
|
||||
FieldIdent("index".to_owned()),
|
||||
Ident("u16".to_owned()),
|
||||
CloseBrace,
|
||||
CloseBrace,
|
||||
]);
|
||||
let expected = EnumModel {
|
||||
name: "Foo".to_string(),
|
||||
variants: vec![EnumVariant {
|
||||
name: "A".to_owned(),
|
||||
data: Some(EnumVariantData {
|
||||
name: Some("index".to_owned()),
|
||||
type_: "u16".to_owned(),
|
||||
}),
|
||||
}],
|
||||
};
|
||||
assert_eq!(model, expected);
|
||||
}
|
||||
}
|
||||
|
||||
mod enum_model {
|
||||
use {EnumModel, EnumVariant};
|
||||
|
||||
#[test]
|
||||
fn variant_bits() {
|
||||
let mut model = EnumModel {
|
||||
name: "Baz".to_string(),
|
||||
variants: vec![EnumVariant {
|
||||
name: "A".to_owned(),
|
||||
data: None,
|
||||
}],
|
||||
};
|
||||
assert_eq!(model.variant_bits(), 0);
|
||||
|
||||
model.variants.append(&mut vec![
|
||||
EnumVariant {
|
||||
name: "B".to_owned(),
|
||||
data: None,
|
||||
},
|
||||
EnumVariant {
|
||||
name: "C".to_owned(),
|
||||
data: None,
|
||||
},
|
||||
]);
|
||||
assert_eq!(model.variant_bits(), 2);
|
||||
for _ in 0..1021 {
|
||||
model.variants.push(EnumVariant {
|
||||
name: "Dynamic".to_owned(),
|
||||
data: None,
|
||||
});
|
||||
}
|
||||
assert_eq!(model.variant_bits(), 10);
|
||||
model.variants.push(EnumVariant {
|
||||
name: "OneMore".to_owned(),
|
||||
data: None,
|
||||
});
|
||||
assert_eq!(model.variant_bits(), 11);
|
||||
}
|
||||
variants.push(parse_quote!(OneMore));
|
||||
assert_eq!(::variant_bits(&variants), 11);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn poll_token_e2e() {
|
||||
let input = "enum Token { A, B, C, D(usize), E { foobaz: u32 }, }";
|
||||
let output = ::poll_token_inner(input);
|
||||
let expected = "impl PollToken for Token {
|
||||
fn as_raw_token(&self) -> u64 {
|
||||
match *self {
|
||||
Token::A => 0,
|
||||
Token::B => 1,
|
||||
Token::C => 2,
|
||||
Token::D(data) => 3 | ((data as u64) << 3),
|
||||
Token::E{ foobaz: data } => 4 | ((data as u64) << 3),
|
||||
}
|
||||
}
|
||||
let input: DeriveInput = parse_quote! {
|
||||
enum Token {
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
D(usize),
|
||||
E { foobaz: u32 },
|
||||
}
|
||||
};
|
||||
|
||||
fn from_raw_token(data: u64) -> Self {
|
||||
match data & 0x07 {
|
||||
0 => Token::A,
|
||||
1 => Token::B,
|
||||
2 => Token::C,
|
||||
3 => Token::D((data >> 3) as usize),
|
||||
4 => Token::E{ foobaz: (data >> 3) as u32 },
|
||||
_ => unreachable!()
|
||||
}
|
||||
}
|
||||
}";
|
||||
assert_eq!(output.as_str(), expected);
|
||||
let actual = ::poll_token_inner(input);
|
||||
let expected = quote! {
|
||||
impl PollToken for Token {
|
||||
fn as_raw_token(&self) -> u64 {
|
||||
match *self {
|
||||
Token::A => 0u64,
|
||||
Token::B => 1u64,
|
||||
Token::C => 2u64,
|
||||
Token::D { 0: data } => 3u64 | ((data as u64) << 3u32),
|
||||
Token::E { foobaz: data } => 4u64 | ((data as u64) << 3u32),
|
||||
}
|
||||
}
|
||||
|
||||
fn from_raw_token(data: u64) -> Self {
|
||||
match data & 7u64 {
|
||||
0u64 => Token::A,
|
||||
1u64 => Token::B,
|
||||
2u64 => Token::C,
|
||||
3u64 => Token::D { 0: (data >> 3u32) as usize },
|
||||
4u64 => Token::E { foobaz: (data >> 3u32) as u32 },
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
assert_eq!(actual.to_string(), expected.to_string());
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue