【发布时间】：2017-06-05 04:33:47
【问题描述】：
假设我有一个由逗号分隔的正整数和字母字符串组成的简单语法。我想使用 Flex 和 Bison 解析这个语法，并且我想在 Flex 中使用多个输入缓冲区，无论出于何种原因（可能数据通过网络或串行线路或其他方式到达）。我看到的问题是，当一个字符串或一个整数（它们都是可变长度标记）在一个缓冲区的结尾和下一个缓冲区的开头之间拆分时，词法分析器会在应该只有一个时报告两个标记。
在下面的示例中，块是 10, asdf 和 g,。如果这一切都在一个缓冲区中，它将产生标记INT(10)COMMASTR(asdfg)COMMA。但是在与“asdf”不同的缓冲区中，词法分析器实际上会产生INT(10)COMMASTR(asdf)STR(g)COMMA。到达缓冲区末尾的逻辑似乎是（1）检查输入是否与令牌匹配，（2）重新填充缓冲区。我觉得应该反过来：（2）重新填充缓冲区，（1）检查输入是否与令牌匹配。
我想确保我在更改缓冲区的方式上没有做傻事。
标准输出/标准错误：
read_more_input: Setting up buffer containing: 10,
--accepting rule at line 48 ("10")
Starting parse
Entering state 0
Reading a token: Next token is token INT_TERM ()
Shifting token INT_TERM ()
Entering state 1
Return for a new token:
--accepting rule at line 50 (",")
Reading a token: Next token is token COMMA ()
Shifting token COMMA ()
Entering state 4
Reducing stack by rule 2 (line 67):
$1 = token INT_TERM ()
$2 = token COMMA ()
-> $$ = nterm int_non_term ()
Stack now 0
Entering state 3
Return for a new token:
--(end of buffer or a NUL)
--EOF (start condition 0)
read_more_input: Setting up buffer containing: asdf
--(end of buffer or a NUL)
--accepting rule at line 49 ("asdf")
Reading a token: Next token is token STR_TERM ()
Shifting token STR_TERM ()
Entering state 6
Return for a new token:
--(end of buffer or a NUL)
--EOF (start condition 0)
read_more_input: Setting up buffer containing: g,
--accepting rule at line 49 ("g")
Reading a token: Next token is token STR_TERM ()
syntax errorError: popping token STR_TERM ()
Stack now 0 3
Error: popping nterm int_non_term ()
Stack now 0
Cleanup: discarding lookahead token STR_TERM ()
Stack now 0
Lex 文件：
%{
#include <stdbool.h>
#include "yacc.h"
bool read_more_input(yyscan_t scanner);
%}
%option reentrant bison-bridge
%%
[0-9]+ { yylval->int_value = atoi(yytext); return INT_TERM; }
[a-zA-Z]+ { yylval->str_value = strdup(yytext); return STR_TERM; }
, { return COMMA; }
<<EOF>> {
if (!read_more_input(yyscanner)) {
yyterminate();
}
}
yacc 文件：
%{
// This appears to be a bug. This typedef breaks a dependency cycle between the headers.
// See https://stackoverflow.com/questions/44103798/cyclic-dependency-in-reentrant-flex-bison-headers-with-union-yystype
typedef void * yyscan_t;
#include <stdbool.h>
#include "yacc.h"
#include "lex.h"
%}
%define api.pure full
%lex-param {yyscan_t scanner}
%parse-param {yyscan_t scanner}
%define api.push-pull push
%union {
int int_value;
char * str_value;
}
%token <int_value> INT_TERM
%type <int_value> int_non_term
%token <str_value> STR_TERM
%type <str_value> str_non_term
%token COMMA
%%
complete : int_non_term str_non_term { printf(" === %d === %s === \n", $1, $2); }
int_non_term : INT_TERM COMMA { $$ = $1; }
str_non_term : STR_TERM COMMA { $$ = $1; }
%%
char * packets[]= {"10,", "asdf", "g,"};
int current_packet = 0;
bool read_more_input(yyscan_t scanner) {
if (current_packet >= 3) {
fprintf(stderr, "read_more_input: No more input\n");
return false;
}
fprintf(stderr, "read_more_input: Setting up buffer containing: %s\n", packets[current_packet]);
size_t buffer_size = strlen(packets[current_packet]) + 2;
char * buffer = (char *) calloc(buffer_size, sizeof(char));
memcpy(buffer, packets[current_packet], buffer_size - 2);
yy_scan_buffer(buffer, buffer_size, scanner);
current_packet++;
return true;
}
int main(int argc, char** argv) {
yyscan_t scanner;
yylex_init(&scanner) ;
read_more_input(scanner);
yyset_debug(1, scanner);
yydebug = 1;
int status;
yypstate *ps = yypstate_new ();
YYSTYPE pushed_value;
do {
status = yypush_parse(ps, yylex(&pushed_value, scanner), &pushed_value, scanner);
} while(status == YYPUSH_MORE);
yypstate_delete (ps);
yylex_destroy (scanner) ;
return 0;
}
【问题讨论】：
标签： c bison flex-lexer