加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
文件
该仓库未声明开源许可证文件(LICENSE),使用请关注具体项目描述及其代码上游依赖。
克隆/下载
Scanner.cpp 3.52 KB
一键复制 编辑 原始数据 按行查看 历史
873314461 提交于 2016-05-08 02:00 . 修改BUG
#include "Scanner.h"
#include <cstdio>
#include <algorithm>
#include <cctype>
Scanner::Scanner(std::string srcFileName, string grammar)
:fileName(srcFileName), line(1), column(1), currentChar(0), state(0), grammarFile(grammar)
{
vector<tuple<string, string, string> > temp = getGrammar(grammar);
dfa = DFA(temp);
dfa.makeNFA();
dfa.NfaToDfa();
input.open(fileName);
if (input.fail())
{
//错误处理
cout << "不能打开文件" << fileName << endl;
}
}
string Scanner::getToken()
{
preprocess();
bLine = line;
bColumn = column;
bool stop = false;
nearestAccept = 0;
state = 0;
buffer.clear();
while (!stop)
{
char c = peekChar();
if (c == EOF)
{
state = -2;
return "EOF";
}
c = transform(c);
pair<int, map<string, int> > res = dfa.find(state);
char s[2];
s[0] = c;
s[1] = '\0';
map<string, int>::iterator i = res.second.find(s);
if (i == res.second.end())
{
stop = true;
string temp = buffer.substr(nearestAccept);
buffer.erase(nearestAccept);
nextRead.insert(0, temp);
if (nearestAccept == 0)
{
cout << "错误:";
state = -1;
getNextChar();
buffer.push_back(currentChar);
}
}
else
{
state = i->second;
getNextChar();
buffer.push_back(currentChar);
pair<int, map<string, int> > res = dfa.find(state);
i = res.second.find("&");
if (i != res.second.end())
{
nearestAccept = buffer.size();
nearestState = state;
}
}
}
return buffer;
}
string Scanner::getFileName()
{
return fileName;
}
string Scanner::getTokenType()
{
if (nearestState == -1)
return "UNKNOW";
else if (nearestState == -2)
return "END_OF_FILE";
else
{
string temp = buffer;
std::transform(temp.begin(), temp.end(), temp.begin(), ::tolower);
set<string>::iterator res = keywords.find(temp);
if (res != keywords.end())
return "KEYWORD";
res = limiters.find(buffer);
if (res != limiters.end())
return "LIMITER";
res = operat.find(buffer);
if (res != operat.end())
return buffer;
return dfa.getType(nearestState);
}
}
bool Scanner::isEnd()
{
if (state == -2)
return true;
return false;
}
long Scanner::getLine()
{
return bLine;
}
long Scanner::getColumn()
{
return bColumn;
}
void Scanner::getNextChar()
{
if (nextRead.empty())
{
currentChar = input.get();
if (currentChar == '\n')
{
line++;
column = 1;
}
else
{
column++;
}
}
else
{
currentChar = nextRead[0];
nextRead.erase(0, 1);
}
}
char Scanner::peekChar()
{
char ch;
if (nextRead.empty())
{
ch = input.peek();
}
else
{
ch = nextRead[0];
}
return ch;
}
void Scanner::preprocess()
{
char c = peekChar();
while (std::isspace(c))
{
getNextChar();
c = peekChar();
}
}
vector<tuple<string, string, string>> Scanner::getGrammar(string grammarFile)
{
ifstream cin;
cin.open(grammarFile);
if (cin.fail())
{
cout << "不能打开文件" << fileName << endl;
}
vector<tuple<string, string, string> > res;
string temp;
int n;
cin >> n;
for (int i = 0; i < n; i++)
{
cin >> temp;
keywords.insert(temp);
}
cin >> n;
for (int i = 0; i < n; i++)
{
cin >> temp;
limiters.insert(temp);
}
cin >> n;
for (int i = 0; i < n; i++)
{
cin >> temp;
operat.insert(temp);
}
string begin;
cin >> begin;
res.push_back(std::make_tuple(begin, begin, begin));
string str1;
string str2;
while (cin >> temp)
{
cin >> str1;
if (str1 != "&")
{
cin >> str2;
}
else
{
str2 = "###";
}
res.push_back(std::make_tuple(temp, str1, str2));
}
return res;
}
char Scanner::transform(char ch)
{
if (std::isspace(ch))
ch = '!';
return ch;
}
Loading...
马建仓 AI 助手
尝试更多
代码解读
代码找茬
代码优化