new impl for #[args] attribute

pyo3cls/Cargo.toml

@@ -9,6 +9,7 @@ proc-macro = true
 [dependencies]
 quote="0.3"
 log="0.3"
+nom = "^3.0"
 env_logger = "0.4"
 
 [dependencies.syn]

pyo3cls/src/args.rs

@@ -0,0 +1,223 @@
+use nom::*;
+
+#[derive(Debug, PartialEq)]
+pub enum Argument {
+    VarArgsSeparator,
+    VarArgs(String),
+    KeywordArgs(String),
+    Arg(String, Option<String>),
+    Kwarg(String, String),
+}
+
+pub fn parse_arguments(value: &[u8]) -> Vec<Argument> {
+    let mut arguments = Vec::new();
+    let mut has_kw = false;
+    let mut has_varargs = false;
+    let mut has_kwargs = false;
+    let args_str = String::from_utf8_lossy(value);
+
+    let mut curr = value;
+    while curr.len() > 0 {
+        match get_arg(curr) {
+            IResult::Done(i, t) => {
+                match argument(t) {
+                    IResult::Done(rest, arg) => {
+                        if rest.len() != 0 {
+                            println!("can not parse arguments {:?} : {:?}",
+                                     args_str, String::from_utf8_lossy(t),
+                            );
+                        } else {
+                            match arg {
+                                Argument::VarArgsSeparator => {
+                                    if has_kwargs {
+                                        println!("syntax error, keyword arguments is defined: {:?}",
+                                                 args_str);
+                                        return Vec::new()
+                                    }
+                                    if has_varargs {
+                                        println!("arguments already define * (var args): {:?}",
+                                                 args_str);
+                                        return Vec::new()
+                                    } else {
+                                        arguments.push(Argument::VarArgsSeparator);
+                                        has_varargs = true;
+                                        has_kw = true;
+                                    }
+                                }
+                                Argument::VarArgs(s) => {
+                                    if has_kwargs {
+                                        println!("syntax error, keyword arguments is defined: {:?}",
+                                                 args_str);
+                                        return Vec::new()
+                                    }
+                                    if has_varargs {
+                                        println!("*(var args) is defined: {:?}", args_str);
+                                        return Vec::new()
+                                    } else {
+                                        arguments.push(Argument::VarArgs(s));
+                                        has_varargs = true;
+                                        has_kw = true;
+                                    }
+                                }
+                                Argument::KeywordArgs(s) => {
+                                    if has_kwargs {
+                                        println!("arguments already define ** (kw args): {:?}",
+                                                 args_str);
+                                        return Vec::new()
+                                    } else {
+                                        arguments.push(Argument::KeywordArgs(s));
+                                        has_kwargs = true;
+                                    }
+                                }
+                                Argument::Arg(s, opt) => {
+                                    if has_kwargs {
+                                        println!("syntax error, keyword arguments is defined: {:?}",
+                                                 args_str);
+                                        return Vec::new()
+                                    }
+                                    if let Some(opt) = opt {
+                                        has_kw = true;
+                                        if has_varargs {
+                                            arguments.push(Argument::Kwarg(s, opt))
+                                        } else {
+                                            arguments.push(Argument::Arg(s, Some(opt)))
+                                        }
+                                    } else {
+                                        if has_kw {
+                                            println!("syntax error, argument is not allowed after keyword argument: {:?}",
+                                                     args_str);
+                                            return Vec::new()
+                                        }
+                                        arguments.push(Argument::Arg(s, None));
+                                    }
+                                }
+                                Argument::Kwarg(_, _) => unreachable!()
+                            }
+                        }
+                    },
+                    IResult::Error(_) | IResult::Incomplete(_) => {
+                        println!("can not parse arguments {:?} : {:?}",
+                                 String::from_utf8_lossy(value), String::from_utf8_lossy(t),
+                        );
+                    }
+                };
+                if i.len() > 0 {
+                    curr = &i[1..i.len()];
+                } else {
+                    break
+                }
+            }
+            IResult::Error(_) | IResult::Incomplete(_) => {
+                println!("can not parse arguments {:?}", String::from_utf8_lossy(value));
+            }
+        }
+    }
+
+    arguments
+}
+
+named!(get_arg, ws!(take_while!(is_not_arg_end)));
+
+fn is_not_arg_end(c: u8) -> bool {
+    c as char != ','
+}
+
+named!(argument<&[u8], Argument>, alt_complete!(
+    parse_arg | parse_optional_arg | parse_kwargs | parse_var_arg | parse_var_arg_sep));
+
+named!(parse_var_arg_sep<&[u8], Argument>,
+       do_parse!(
+           char!('*') >> eof!() >> ( Argument::VarArgsSeparator )
+       )
+);
+named!(parse_var_arg<&[u8], Argument>,
+       do_parse!(
+           tag!("*") >>
+               take_while!(is_space) >> verify!(peek!(take!(1)), alphabetic) >>
+               name: take_while!(is_alphanumeric) >> take_while!(is_space) >> eof!() >> (
+                   Argument::VarArgs(String::from_utf8_lossy(name).to_string())
+               )
+       )
+);
+named!(parse_kwargs<&[u8], Argument>,
+       do_parse!(
+           tag!("**") >>
+               take_while!(is_space) >> verify!(peek!(take!(1)), alphabetic) >>
+               name: take_while!(is_alphanumeric) >> take_while!(is_space) >> eof!() >> (
+                   Argument::KeywordArgs(String::from_utf8_lossy(name).to_string())
+               )
+       )
+);
+
+named!(parse_arg<&[u8], Argument>,
+       do_parse!(
+           take_while!(is_space) >> verify!(peek!(take!(1)), alphabetic) >>
+               name: take_while!(is_alphanumeric) >> take_while!(is_space) >> eof!() >> (
+                   Argument::Arg(String::from_utf8_lossy(name).to_string(), None)
+               )
+       )
+);
+
+named!(parse_optional_arg<&[u8], Argument>,
+       do_parse!(
+           take_while!(is_space) >> verify!(peek!(take!(1)), alphabetic) >>
+               name: take_while!(is_alphanumeric) >>
+               take_while!(is_space) >> tag!("=") >>
+               value: take_while!(any) >> (
+                   Argument::Arg(
+                       String::from_utf8_lossy(name).to_string(),
+                       Some(String::from_utf8_lossy(value).to_string())
+                   )
+               )
+       )
+);
+
+fn any(_: u8) -> bool { true }
+
+fn alphabetic(b: &[u8]) -> bool {
+    is_alphabetic(b[0])
+}
+
+
+#[cfg(test)]
+mod test {
+    use args::{Argument, parse_arguments};
+
+    #[test]
+    fn test_errs() {
+        assert!(parse_arguments("123t2113est".as_ref()).is_empty());
+        assert!(parse_arguments("test=1, test2".as_ref()).is_empty());
+        assert!(parse_arguments("test=1, *, *args".as_ref()).is_empty());
+        assert!(parse_arguments("test=1, **kwargs, *args".as_ref()).is_empty());
+        assert!(parse_arguments("test=1, **kwargs, args".as_ref()).is_empty());
+    }
+
+    #[test]
+    fn test_simple_args() {
+        let args = parse_arguments("test1, test2, test3=None".as_ref());
+        assert!(args == vec![Argument::Arg("test1".to_owned(), None),
+                             Argument::Arg("test2".to_owned(), None),
+                             Argument::Arg("test3".to_owned(), Some("None".to_owned()))]);
+    }
+
+    #[test]
+    fn test_varargs() {
+        let args = parse_arguments("test1, test2=None, *, test3=None".as_ref());
+        assert!(args == vec![Argument::Arg("test1".to_owned(), None),
+                             Argument::Arg("test2".to_owned(), Some("None".to_owned())),
+                             Argument::VarArgsSeparator,
+                             Argument::Kwarg("test3".to_owned(), "None".to_owned())]);
+    }
+
+    #[test]
+    fn test_all() {
+        let args = parse_arguments(
+            "test1, test2=None, *args, test3=None, **kwargs".as_ref());
+        assert!(args == vec![Argument::Arg("test1".to_owned(), None),
+                             Argument::Arg("test2".to_owned(), Some("None".to_owned())),
+                             Argument::VarArgs("args".to_owned()),
+                             Argument::Kwarg("test3".to_owned(), "None".to_owned()),
+                             Argument::KeywordArgs("kwargs".to_owned())]);
+    }
+
+}

pyo3cls/src/lib.rs

@@ -7,6 +7,7 @@ extern crate proc_macro;
 extern crate syn;
 #[macro_use] extern crate quote;
 #[macro_use] extern crate log;
+#[macro_use] extern crate nom;
 
 use std::str::FromStr;
 use proc_macro::TokenStream;
@@ -18,6 +19,7 @@ mod py_impl;
 mod py_proto;
 mod py_method;
 mod py_ptr;
+mod args;
 mod defs;
 mod func;
 mod method;

pyo3cls/src/method.rs

@@ -1,7 +1,9 @@
 // Copyright (c) 2017-present PyO3 Project and Contributors
 
 use syn;
-use quote::{Tokens, ToTokens};
+use quote::{Tokens, Ident};
+
+use args::{Argument, parse_arguments};
 use utils::for_err_msg;
 
 
@@ -24,16 +26,9 @@ pub enum FnType {
     FnStatic,
 }
 
-#[derive(Clone, Debug)]
-pub enum FnAttr {
-    Args(syn::Ident),
-    Kwargs(syn::Ident),
-    Default(syn::Ident, Tokens),
-}
-
 pub struct FnSpec<'a> {
     pub tp: FnType,
-    pub attrs: Vec<FnAttr>,
+    pub attrs: Vec<Argument>,
     pub args: Vec<FnArg<'a>>,
     pub output: syn::Ty,
 }
@@ -108,8 +103,8 @@ impl<'a> FnSpec<'a> {
     pub fn is_args(&self, name: &syn::Ident) -> bool {
         for s in self.attrs.iter() {
             match *s {
-                FnAttr::Args(ref ident) =>
+                Argument::VarArgs(ref ident) =>
-                    return name == ident,
+                    return name.as_ref() == ident.as_str(),
                 _ => (),
             }
         }
@@ -119,7 +114,8 @@ impl<'a> FnSpec<'a> {
     pub fn accept_args(&self) -> bool {
         for s in self.attrs.iter() {
             match *s {
-                FnAttr::Args(_) => return true,
+                Argument::VarArgs(_) => return true,
+                Argument::VarArgsSeparator => return true,
                 _ => (),
             }
         }
@@ -129,8 +125,8 @@ impl<'a> FnSpec<'a> {
     pub fn is_kwargs(&self, name: &syn::Ident) -> bool {
         for s in self.attrs.iter() {
             match *s {
-                FnAttr::Kwargs(ref ident) =>
+                Argument::KeywordArgs(ref ident) =>
-                    return name == ident,
+                    return name.as_ref() == ident.as_str(),
                 _ => (),
             }
         }
@@ -140,7 +136,7 @@ impl<'a> FnSpec<'a> {
     pub fn accept_kwargs(&self) -> bool {
         for s in self.attrs.iter() {
             match *s {
-                FnAttr::Kwargs(_) => return true,
+                Argument::KeywordArgs(_) => return true,
                 _ => (),
             }
         }
@@ -150,9 +146,18 @@ impl<'a> FnSpec<'a> {
     pub fn default_value(&self, name: &syn::Ident) -> Option<Tokens> {
         for s in self.attrs.iter() {
             match *s {
-                FnAttr::Default(ref ident, ref val) => {
+                Argument::Arg(ref ident, ref opt) => {
-                    if ident == name {
+                    if ident.as_str() == name.as_ref() {
-                        return Some(val.clone())
+                        if let &Some(ref val) = opt {
+                            let i = Ident::from(val.as_str());
+                            return Some(quote!(#i))
+                        }
+                    }
+                },
+                Argument::Kwarg(ref ident, ref opt) => {
+                    if ident.as_str() == name.as_ref() {
+                        let i = Ident::from(opt.as_str());
+                        return Some(quote!(#i))
                     }
                 },
                 _ => (),
@@ -160,6 +165,20 @@ impl<'a> FnSpec<'a> {
         }
         None
     }
+
+    pub fn is_kw_only(&self, name: &syn::Ident) -> bool {
+        for s in self.attrs.iter() {
+            match *s {
+                Argument::Kwarg(ref ident, _) => {
+                    if ident.as_str() == name.as_ref() {
+                        return true
+                    }
+                },
+                _ => (),
+            }
+        }
+        false
+    }
 }
 
 fn check_arg_ty_and_optional<'a>(name: &'a syn::Ident, ty: &'a syn::Ty) -> Option<&'a syn::Ty> {
@@ -206,7 +225,7 @@ fn check_arg_ty_and_optional<'a>(name: &'a syn::Ident, ty: &'a syn::Ty) -> Optio
     }
 }
 
-fn parse_attributes(attrs: &mut Vec<syn::Attribute>) -> (FnType, Vec<FnAttr>) {
+fn parse_attributes(attrs: &mut Vec<syn::Attribute>) -> (FnType, Vec<Argument>) {
     let mut new_attrs = Vec::new();
     let mut spec = Vec::new();
     let mut res: Option<FnType> = None;
@@ -295,14 +314,6 @@ fn parse_attributes(attrs: &mut Vec<syn::Attribute>) -> (FnType, Vec<FnAttr>) {
                     "args" => {
                         spec.extend(parse_args(meta))
                     }
-                    "defaults" => {
-                        // parse: #[defaults(param2=12, param3=12)]
-                        for item in meta.iter() {
-                            if let Some(el) = parse_args_default(item) {
-                                spec.push(el)
-                            }
-                        }
-                    }
                     _ => {
                         new_attrs.push(attr.clone())
                     }
@@ -323,7 +334,7 @@ fn parse_attributes(attrs: &mut Vec<syn::Attribute>) -> (FnType, Vec<FnAttr>) {
 }
 
 /// parse: #[args(args="args", kw="kwargs")]
-fn parse_args(items: &Vec<syn::NestedMetaItem>) -> Vec<FnAttr> {
+fn parse_args(items: &Vec<syn::NestedMetaItem>) -> Vec<Argument> {
     let mut spec = Vec::new();
 
     for item in items.iter() {
@@ -332,38 +343,23 @@ fn parse_args(items: &Vec<syn::NestedMetaItem>) -> Vec<FnAttr> {
                 match *name {
                     syn::Lit::Str(ref name, _) => match ident.as_ref() {
                         "args" =>
-                            spec.push(FnAttr::Args(syn::Ident::from(name.clone()))),
+                            spec.push(Argument::VarArgs(name.clone())),
                         "kw" =>
-                            spec.push(FnAttr::Kwargs(syn::Ident::from(name.clone()))),
+                            spec.push(Argument::KeywordArgs(name.clone())),
                         _ => (),
                     },
                     _ => (),
                 }
             },
+            &syn::NestedMetaItem::Literal(syn::Lit::Str(ref args, _)) => {
+                println!("ARGS: {:?}", args);
+                for item in parse_arguments(args.as_ref()) {
+                    spec.push(item);
+                }
+            },
             _ => (),
         }
     }
 
     spec
 }
-
-fn parse_args_default(item: &syn::NestedMetaItem) -> Option<FnAttr> {
-    match *item {
-        syn::NestedMetaItem::MetaItem(syn::MetaItem::NameValue(ref name, ref lit)) => {
-            let mut t = Tokens::new();
-            match lit {
-                &syn::Lit::Str(ref val, _) => {
-                    syn::Ident::from(val.as_str()).to_tokens(&mut t);
-                },
-                _ => {
-                    lit.to_tokens(&mut t);
-                }
-            }
-            Some(FnAttr::Default(name.clone(), t))
-        }
-        _ => {
-            println!("expected name value {:?}", item);
-            None
-        }
-    }
-}

pyo3cls/src/py_method.rs

@@ -300,18 +300,24 @@ fn impl_arg_params(spec: &FnSpec, body: Tokens) -> Tokens {
     for arg in spec.args.iter() {
         if ! (spec.is_args(&arg.name) || spec.is_kwargs(&arg.name)) {
             let name = arg.name.as_ref();
-            let opt = if let Some(_) = arg.optional {
+            let kwonly = if spec.is_kw_only(&arg.name) {
-                syn::Ident::from("true")
-            } else {
-                if let Some(_) = spec.default_value(&arg.name) {
                 syn::Ident::from("true")
             } else {
                 syn::Ident::from("false")
-                }
             };
+
+            let opt = if let Some(_) = arg.optional {
+                syn::Ident::from("true")
+            } else if let Some(_) = spec.default_value(&arg.name) {
+                syn::Ident::from("true")
+            } else {
+                syn::Ident::from("false")
+            };
+
             params.push(
                 quote! {
-                    _pyo3::argparse::ParamDescription{name: #name, is_optional: #opt,}
+                    _pyo3::argparse::ParamDescription{
+                        name: #name, is_optional: #opt, kw_only: #kwonly}
                 }
             );
         }

src/argparse.rs

@@ -30,7 +30,9 @@ pub struct ParamDescription<'a> {
     /// The name of the parameter.
     pub name: &'a str,
     /// Whether the parameter is optional.
-    pub is_optional: bool
+    pub is_optional: bool,
+    /// Whether the parameter is optional.
+    pub kw_only: bool
 }
 
 /// Parse argument list
@@ -370,14 +372,16 @@ macro_rules! py_argparse_param_description {
     { $pname:ident : $ptype:ty = [ {} {} $rtype:tt ] } => (
         $crate::argparse::ParamDescription {
             name: stringify!($pname),
-            is_optional: false
+            is_optional: false,
+            kw_only: false,
         }
     );
     // optional parameters
     { $pname:ident : $ptype:ty = [ {} {$default:expr} {$($rtype:tt)*} ] } => (
         $crate::argparse::ParamDescription {
             name: stringify!($pname),
-            is_optional: true
+            is_optional: true,
+            kw_only: false,
         }
     );
 }