From b17ad14e2a7d15bd116f1b419e1312e65bab34b4 Mon Sep 17 00:00:00 2001
From: Michael Uleysky <uleysky@poi.dvo.ru>
Date: Wed, 9 Sep 2015 14:43:29 +1000
Subject: [PATCH] @include directive in lexical parser. Correct error handling
 in lexical parser.

---
 src/init.cpp             |  23 +++++---
 src/init.h               |   2 +-
 src/parser/grammatical.y |   9 ++--
 src/parser/lexical.l     | 112 +++++++++++++++++++++++----------------
 src/parser/parser.h      |  29 ++++++++--
 5 files changed, 116 insertions(+), 59 deletions(-)

diff --git a/src/init.cpp b/src/init.cpp
index dea807e..5346d01 100644
--- a/src/init.cpp
+++ b/src/init.cpp
@@ -6,9 +6,14 @@
 #include "arifmetic.h"
 #include "parser/parser.h"
 #include "parser/grammatical.h"
+// definitions for bison-bridge
+#define YYSTYPE CONFSTYPE
+#define YYLTYPE CONFLTYPE
 #include "parser/lexical.h"
+#undef YYSTYPE
+#undef YYLTYPE
 
-int ParseConfigFile(char* config)
+int ParseConfigFile(const char* config)
 {
  yyscan_t scanner;
  struct lexical_extra extra;
@@ -21,18 +26,24 @@ int ParseConfigFile(char* config)
   COUT(ERROR)<<"Can't open file "<<config<<std::endl;
   return 1;
  }
- extra.filename=config;
- extra.inclevel=0;
+
  extra.maxinclevel=10;
- extra.curline=1;
- extra.curpos=extra.curoffset=0;
+ extra.retcode=0;
+ extra.state.inclevel=0;
+ extra.state.curline=1;
+ extra.state.curpos=extra.state.curoffset=0;
+ char* cwd=get_current_dir_name();
+ extra.state.curdir=cwd;
+ free(cwd);
+ extra.ParsePath(config);
+
  conflex_init_extra(&extra,&scanner);
  confset_in(conffd,scanner);
 // {YYSTYPE qqq; while(conflex(&qqq,scanner)>0);}
  ret=confparse(scanner);
  conflex_destroy(scanner);
  fclose(conffd);
- return ret;
+ return ret+extra.retcode;
 }
 
 
diff --git a/src/init.h b/src/init.h
index d2578ae..98c73d8 100644
--- a/src/init.h
+++ b/src/init.h
@@ -6,7 +6,7 @@ typedef void* yyscan_t;
 #endif
 #include "deptree.h"
 
-int ParseConfigFile(char* config);
+int ParseConfigFile(const char* config);
 int RegisterArifmeticFunctions();
 int BuildDepTree(DepTree* deptree, UsedType& used);
 int CheckFunctions();
diff --git a/src/parser/grammatical.y b/src/parser/grammatical.y
index d98dd86..a48f188 100644
--- a/src/parser/grammatical.y
+++ b/src/parser/grammatical.y
@@ -1,10 +1,13 @@
-%name-prefix "conf"
 %language "c"
 %output "grammatical.cpp"
 %defines "grammatical.h"
 %param {yyscan_t scanner}
+%define api.prefix {conf}
 %define api.pure full
+%define lr.type ielr
 %define parse.lac full
+%define parse.error verbose
+//%define parse.trace
 %locations
 //%no-lines
 
@@ -23,7 +26,7 @@ typedef void* yyscan_t;
 #include "grammatical.h"
 inline void conferror(YYLTYPE *locp, yyscan_t sc, const char * str)
 {
- COUT(ERROR)<<str<<std::endl;
+ COUT(ERROR)<<std::endl<<" Grammatical parser: "<<str<<std::endl;
 }
 #include "lexical.h"
 %}
@@ -109,7 +112,7 @@ pair:
         ;
 
 object:
-        STRING                  {COUT(DEBUG)<<" STRING\n"; $$=new ObjectString($1);}
+        STRING                  {COUT(DEBUG)<<" STRING\n"; $$=new ObjectString($1); delete $1;}
         | BOOL                  {COUT(DEBUG)<<" BOOL\n"; $$=new ObjectBool($1);}
         | OBRACE list CBRACE    {COUT(DEBUG)<<" OBRACE list CBRACE\n"; $$=$2;}
         | expression            {COUT(DEBUG)<<" expression\n"; $$=$1;}
diff --git a/src/parser/lexical.l b/src/parser/lexical.l
index 485bea4..b601a67 100644
--- a/src/parser/lexical.l
+++ b/src/parser/lexical.l
@@ -2,7 +2,6 @@
 %option warn
 %option yylineno
 %option noyywrap
-%option yylineno
 %option header-file="lexical.h"
 %option outfile="lexical.cpp"
 %option prefix="conf"
@@ -10,66 +9,87 @@
 %option bison-bridge
 %option bison-locations
 %option nounput
+%option stack
 %x PSTRING
 %x PARSE
+%x INCLUDE
 
 %{
 #if __cplusplus > 199711L
 #define register      // Deprecated in C++11.
 #endif  // #if __cplusplus > 199711L
 #include <stdlib.h>
-#include <string>
 #include "../debug.h"
 #include "../object.h"
 #include "parser.h"
 #include "grammatical.h"
-// Get rid of warning on unneeded function
+// Get rid of warning on unused function
 #define YY_NO_INPUT
-static std::string str;
+// definitions for bison-bridge
+#define YYSTYPE CONFSTYPE
+#define YYLTYPE CONFLTYPE
+std::string str;
+#define yyterminate(ret) { while(YY_CURRENT_BUFFER) yypop_buffer_state(yyscanner); str.erase(); yyextra->retcode=-(ret); return (ret); }
+#define yyerrormessage(message,ret) {COUT(ERROR)<<std::endl<<" Lexical error in file "<<yyextra->state.curdir<<yyextra->state.filename<<" at line "<<yyextra->state.curline<<": "<<(message)<<std::endl; yyterminate(ret);}
+#define yynextline {yyextra->state.curline++; yyextra->state.curpos=0; yyextra->state.curoffset++;}
+#define yynextsym {yyextra->state.curpos+=yyleng; yyextra->state.curoffset+=yyleng;}
 %}
 
 %%
-<PARSE,INITIAL,PSTRING>\n                       yyextra->curline++; yyextra->curpos=0; yyextra->curoffset++; REJECT;
-<PARSE,INITIAL,PSTRING>.                        yyextra->curpos++; yyextra->curoffset++; REJECT;
-include\(\".+\"\);                              {/*
-                                                        if(yyextra->inclevel>=yyextra->maxinclevel) { COUT(ERROR)<<"Max include level reached in file "<<yyextra->filename<<" at line "<<yylineno<<std::endl; return 1; }
-                                                        yyscan_t scanner;
-                                                        struct lexical_extra extra;
-                                                        std::string fname(yytext+9,yyleng-12);
-                                                        FILE* fd;
-                                                        fd=fopen(fname.c_str(),"r");
-                                                        if(fd==0) { COUT(ERROR)<<"Can't open file "<<fname<<std::endl; return -1; }
-                                                        COUT(DEBUG)<<"Include "<<fname<<std::endl;
-                                                        extra.filename=fname.c_str();
-                                                        extra.inclevel=yyextra->inclevel+1;
-                                                        extra.maxinclevel=yyextra->maxinclevel;
-                                                        yylex_init_extra(&extra,&scanner);
-                                                        yyset_in(fd,scanner);
-                                                        yylex(scanner);
-                                                        yylex_destroy(scanner);
-                                                        fclose(fd);
+<PARSE,INITIAL>@include[ \t]*\"                 yy_push_state(INCLUDE,yyscanner); str.erase(); yynextsym;
+<INCLUDE>\"                                     {
+                                                 if(yyextra->state.inclevel>=yyextra->maxinclevel) yyerrormessage("maximal include level reached",-1);
+                                                 FILE* fd;
+
+                                                 // yyextra->curdir is directory with currently scanned file
+                                                 // !!!NONPORTABLE!!!
+                                                 if('/'==str[0]) fd=fopen(str.c_str(),"r");     // absolute path
+                                                 else
+                                                 {
+                                                  // first, try to search file in working directory
+                                                  fd=fopen(str.c_str(),"r");
+                                                  // if fail, try to search in directory with currently scanned file
+                                                  if(fd==0) fd=fopen((yyextra->state.curdir+str).c_str(),"r");
+                                                 }
+
+                                                 if(fd==0) yyerrormessage("can't open file "+str,-1);
+                                                 yynextsym;
+                                                 yyextra->fds.push(fd);
+                                                 yyextra->states.push(yyextra->state);
+                                                 COUT(DEBUG)<<std::endl<<"Include "<<str<<std::endl;
+
+                                                 yyextra->state.inclevel++;
+                                                 yyextra->ParsePath(str);
+                                                 yyextra->state.curline=1;
+                                                 yyextra->state.curpos=yyextra->state.curoffset=0;
 
-                                                */}
-[a-zA-Z][a-zA-Z0-9_]*                           COUT(MOREDEBUG)<<"NAME("<<yytext<<")"; BEGIN(PARSE); yylval_param->str=new std::string(yytext); return NAME;
-<PARSE>[+\-*/^]                                 COUT(MOREDEBUG)<<" OPERATION("<<yytext<<")"; return yytext[0];
-<PARSE>[0-9]+   				COUT(MOREDEBUG)<<" INTEGER("<<yytext<<")"; yylval_param->i=atoll(yytext); return INTEGER;
-<PARSE>[0-9]+(\.[0-9]*)?([eE][+-][0-9]+)?	COUT(MOREDEBUG)<<" REAL("<<yytext<<")"; yylval_param->r=atof(yytext); return REAL;
-<PARSE>[TF]                                     COUT(MOREDEBUG)<<" BOOL("<<yytext<<")"; yylval_param->b=(yytext[0]=='T')?true:false; return BOOL;
-<PARSE>\(					COUT(MOREDEBUG)<<" OBRACE()"; return OBRACE;
-<PARSE>\)					COUT(MOREDEBUG)<<" CBRACE()"; return CBRACE;
-<PARSE>\;					COUT(MOREDEBUG)<<" ENDL()"<<std::endl; BEGIN(0); return ENDL;
-<PARSE>=					COUT(MOREDEBUG)<<" ASSIGN()"; return ASSIGN;
-<PARSE>,                                        COUT(MOREDEBUG)<<" DELIM()"; return DELIM;
-<PARSE>[a-zA-Z][a-zA-Z0-9_]*		        COUT(MOREDEBUG)<<" IDENTIFIER("<<yytext<<")"; yylval_param->str=new std::string(yytext); return IDENTIFIER;
-<PARSE>\.                                       COUT(MOREDEBUG)<<" DOT()"; return yytext[0];
-<PARSE,INITIAL>[ \n\t]
-<PARSE,INITIAL>\#.*
-<PARSE>\"				        BEGIN(PSTRING); str.erase();
-<PARSE,INITIAL>.                                COUT(ERROR)<<"Unknown symbol "<<yytext<<" in file "<<yyextra->filename<<" at line "<<yylineno<<std::endl; yyterminate(); return -1;
-<PSTRING>\\\\					str+='\\';
-<PSTRING>\\\"					str+='\"';
-<PSTRING>\"					BEGIN(PARSE); COUT(MOREDEBUG)<<" STRING("<<str<<")"; yylval_param->str=&str; return STRING;
-<PSTRING>.					str+=yytext[0];
-<PSTRING><<EOF>>				COUT(ERROR)<<"Unclosed quote!"<<std::endl; str.erase(); yyterminate(); return -1;
-<<EOF>>						str.erase(); yyterminate(); return 0;
+                                                 yypush_buffer_state(yy_create_buffer(fd,YY_BUF_SIZE,yyscanner),yyscanner);
+                                                 // Get rid of warning on unused function
+                                                 if(__builtin_expect(yy_top_state(yyscanner)==PSTRING,0)) yyerrormessage("misterious error",-1);
+                                                 yy_pop_state(yyscanner);
+                                                }
+[a-zA-Z][a-zA-Z0-9_]*                           COUT(MOREDEBUG)<<"NAME("<<yytext<<")"; BEGIN(PARSE); yylval_param->str=new std::string(yytext); yynextsym; return NAME;
+<PARSE>[+\-*/^]                                 COUT(MOREDEBUG)<<" OPERATION("<<yytext<<")"; return yytext[0]; yynextsym;
+<PARSE>[0-9]+   				COUT(MOREDEBUG)<<" INTEGER("<<yytext<<")"; yylval_param->i=atoll(yytext); yynextsym; return INTEGER;
+<PARSE>[0-9]+(\.[0-9]*)?([eE][+-][0-9]+)?	COUT(MOREDEBUG)<<" REAL("<<yytext<<")"; yylval_param->r=atof(yytext); yynextsym; return REAL;
+<PARSE>[TF]                                     COUT(MOREDEBUG)<<" BOOL("<<yytext<<")"; yylval_param->b=(yytext[0]=='T')?true:false; yynextsym; return BOOL;
+<PARSE>\(					COUT(MOREDEBUG)<<" OBRACE()"; yynextsym; return OBRACE;
+<PARSE>\)					COUT(MOREDEBUG)<<" CBRACE()"; yynextsym; return CBRACE;
+<PARSE>\;					COUT(MOREDEBUG)<<" ENDL()"<<std::endl; BEGIN(0); yynextsym; return ENDL;
+<PARSE>=					COUT(MOREDEBUG)<<" ASSIGN()"; yynextsym; return ASSIGN;
+<PARSE>,                                        COUT(MOREDEBUG)<<" DELIM()"; yynextsym; return DELIM;
+<PARSE>[a-zA-Z][a-zA-Z0-9_]*		        COUT(MOREDEBUG)<<" IDENTIFIER("<<yytext<<")"; yylval_param->str=new std::string(yytext); yynextsym; return IDENTIFIER;
+<PARSE>\.                                       COUT(MOREDEBUG)<<" DOT()"; yynextsym; return yytext[0];
+<PARSE,INITIAL>[ \t]+                           yynextsym;
+<PARSE,INITIAL>\n                               yynextline;
+<PARSE,INITIAL>\#.*                             yynextsym;
+<PARSE>\"				        BEGIN(PSTRING); str.erase(); yynextsym;
+<PARSE,INITIAL>.                                yyerrormessage(std::string("unknown symbol ")+yytext+" at position "+std::to_string(yyextra->state.curpos),-1);
+<PSTRING,INCLUDE>\\\\				str+='\\'; yynextsym;
+<PSTRING,INCLUDE>\\\"	        		str+='\"'; yynextsym;
+<PSTRING>\n                                     str+=yytext[0]; yynextline;
+<PSTRING>\"					BEGIN(PARSE); COUT(MOREDEBUG)<<" STRING("<<str<<")"; yylval_param->str=new std::string(str); yynextsym;  return STRING;
+<PSTRING,INCLUDE>.				str+=yytext[0]; yynextsym;
+<PSTRING,INCLUDE><<EOF>>			yyerrormessage("unclosed quote",-1);
+<<EOF>>						yypop_buffer_state(yyscanner); if(YY_CURRENT_BUFFER) {yyextra->state=yyextra->states.top(); yyextra->states.pop(); fclose(yyextra->fds.top()); yyextra->fds.pop();} else yyterminate(0);
 %%
diff --git a/src/parser/parser.h b/src/parser/parser.h
index 088032d..dd745c5 100644
--- a/src/parser/parser.h
+++ b/src/parser/parser.h
@@ -1,11 +1,34 @@
 #ifndef PARSER_PARSER_H
 #define PARSER_PARSER_H
+#include <string>
+#include <stack>
+#include <stdio.h>
 
-struct lexical_extra
+struct lexical_state
 {
- const char* filename;
- unsigned int inclevel,maxinclevel;
+ std::string filename,curdir;
+ unsigned int inclevel;
  unsigned int curline,curpos,curoffset;
 };
 
+struct lexical_extra
+{
+ struct lexical_state state;
+ unsigned int maxinclevel;
+ int retcode;
+ std::stack<FILE*> fds;
+ std::stack<struct lexical_state> states;
+ ~lexical_extra()
+ {
+  while(fds.size()!=0) {fclose(fds.top()); fds.pop();} while(states.size()!=0) states.pop();
+ }
+ void ParsePath(const std::string& path)
+ {
+  // !!!NONPORTABLE!!!
+  if('/'==path[0]) state.curdir=path.substr(0,path.rfind('/')+1);                             // absolute path
+  else if(std::string::npos!=path.rfind('/')) state.curdir+=path.substr(0,path.rfind('/')+1); // relative path
+  state.filename=path.substr((path.rfind('/')!=std::string::npos)?(path.rfind('/')+1):0,std::string::npos);
+ }
+};
+
 #endif