Merge pull request #15 from Servostar/5-create-tokenizer

5 create tokenizer

src/lex/lexer.l

@@ -6,11 +6,79 @@
     int yylex();
 %}
 
+
 /* disable the following functions */
 /* to avoid failing code check */
 %option nounput
 %option noinput
 
+
 %%
+"\n"    yyLineNumber++;
+
+":" return(':');
+"=" return('=');
+"+"  return('+');
+"-"  return('-');
+"*"  return('*');
+"/"  return('/');
+"," return(',');
+";"  return(';');
+"." return('.');
+"#" return('#');
+
+"("  return('(');
+")"  return(')');
+"["  return('[');
+"]"  return(']');
+"{"  return('{');
+"}"  return('}');
+">"  return('>');
+"<"  return('<');
+
+"int" return(KeyInt);
+"float" return(KeyFloat);
+"as" return (KeyAs);
+"short"  return(KeyShort);
+"long" return(KeyLong);
+"half"  return(KeyHalf);
+"double" return(KeyDouble); 
+"signed" return(KeySigned);
+"unsigned" return(Keyunsigned);
+"ref" return(KeyRef);
+"type" return(KeyType);
+"local" return(KeyLocal);
+"global" return(KeyGlobal);
+"static" return(KeyStatic);
+"if" return(KeyIf);
+"else" return(KeyElse);
+"while" return(KeyWhile);
+"in" return(KeyIn);
+"out" return(KeyOut);
+"fun" return(KeyFun);
+"==" return(OpEquals);
+"&&" return(OpAnd);
+"||" return(OpOr);
+"!!" return(OpNot);
+"^^" return(OpXor);
+"&" return(OpBitand);
+"|" return(OpBitor);
+"!" return(OpBitnot);
+"^" return(OpBitxor);
+"import" return(KeyImport);
+"silent" return(KeySilent);
+"box" return(KeyBox);
+"typeof" return(FunTypeof);
+"sizeof" return(FunSizeof);
+"filename" return(FunFilename);
+"funname" return(FunFunname);
+"lineno" return(FunLineno);
+"extsupport" return(FunExtsupport);
+
+[a-zA-Z_]+ { yylval.string = strdup(yytext); return(Ident); };
+[0-9]+    { yylval.string = strdup(yytext); return(ValInt); };
+\"([^\"\n])*\" {yylval.string = strdup(yytext); return(ValStr);};
+\"\"\"([^\"\n]|\\\n)*\"\"\" {yylval.string = strdup(yytext); return(ValMultistr);};
+[0-9]*\.[0-9]+ {yylval.string = strdup(yytext); return(ValFloat);};
 .;
 %%

src/yacc/parser.y

@@ -7,6 +7,54 @@
     extern int yylex();
 %}
 
+%union {
+    char *string;
+}
+
+%token KeyInt
+%token KeyFloat
+%token KeyAs
+%token <string> ValInt
+%token <string> Ident
+%token <string> ValFloat 
+%token <string> ValStr
+%token <string> ValMultistr
+%token KeyShort
+%token KeyLong
+%token KeyHalf
+%token KeyDouble
+%token KeySigned
+%token Keyunsigned
+%token KeyRef
+%token KeyType
+%token KeyLocal
+%token KeyGlobal
+%token KeyStatic
+%token KeyIf
+%token KeyElse
+%token KeyWhile
+%token KeyIn
+%token KeyOut
+%token KeyFun
+%token OpEquals
+%token OpAnd
+%token OpOr
+%token OpNot
+%token OpXor
+%token OpBitand
+%token OpBitor
+%token OpBitnot
+%token OpBitxor
+%token KeyImport
+%token KeySilent
+%token KeyBox
+%token FunTypeof
+%token FunSizeof
+%token FunFilename
+%token FunFunname
+%token FunLineno
+%token FunExtsupport
+
 %%
 program: ;
 %%