---

scanner_spec.cc

Go to the documentation of this file.

/*
 *  File:       scanner_spec.cc
 *              $Id: scanner_spec.cc,v 1.9 2002/06/13 11:42:21 alec Exp $
 *
 *  Author:     Alec Panoviciu (alecu@email.com)
 * 
 *  Comments:
 *
 *  Revision history:
 *
 *  $Log: scanner_spec.cc,v $
 *  Revision 1.9  2002/06/13 11:42:21  alec
 *  added #line stuff
 *
 *  Revision 1.8  2002/05/22 01:43:00  alec
 *  *** empty log message ***
 *
 *  Revision 1.7  2002/05/08 10:36:53  alec
 *  added keyword tokens support
 *
 *  Revision 1.6  2002/05/07 10:02:18  alec
 *  fixed some bugs & mem leaks; added MORE tokens support
 *
 *  Revision 1.5  2002/05/04 17:39:22  alec
 *  the scanner works (slightly tested)
 *
 *  Revision 1.4  2002/04/29 17:55:41  alec
 *  regexps almost done
 *
 *  Revision 1.3  2002/04/29 09:34:10  alec
 *  scanner ptree building compiles
 *
 */

/*
  Copyright (C) 2002 Alexandru Panoviciu (alecu@email.com)

  This program is free software; you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation; either version 2 of the License, or
  (at your option) any later version.

  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License
  along with this program; if not, write to the Free Software
  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

 */

#include <algorithm>

#include "debug.h"
#include "scanner_spec.hh"
#include "dfa_re_node_builder.hh"
#include "itoken_spec.hh"

ScannerSpec::ScannerSpec (ITokenSpec &tokens_, PropRegistry &registry) :
  nodeBuilder(*new DfaReNodeBuilder(registry)),
  tokens(tokens_),
  starRe(NULL)
{
  // EOF has always id 0; we add it as a SPECIAl token (no regexp).
  tokens.addToken("eof", Position());
}


ScannerSpec::~ScannerSpec ()
{
  delete &nodeBuilder;
}


IReNodeBuilder& ScannerSpec::getReNodeBuilder ()
{
  return nodeBuilder;
}


void ScannerSpec::addRegToken (const vector<string> &states,
                               const string &name, ReNode *regexp,
                               const string &tokenAction,
                               const Position &pos)
  throw (ParseException)
{
  if (tokens.isToken(name))
    throw ParseException(pos, "Duplicate token name.");

  regexp = nodeBuilder.createCatNode(regexp,
                                     nodeBuilder.createEotNode(tokens.count(),
                                                               pos),
                                     pos);
  tokens.addToken(name, ITokenSpec::regular, regexp, tokenAction, pos);
  updateStates(states, dynamic_cast<DfaSourceRe*>(regexp));
}


void ScannerSpec::addSkipToken (const vector<string> &states,
                                const string &name, ReNode *regexp,
                                const string &tokenAction,
                                const Position &pos)
  throw (ParseException)
{
  if (tokens.isToken(name))
    throw ParseException(pos, "Duplicate token name.");

  regexp = nodeBuilder.createCatNode(regexp,
                               nodeBuilder.createEotNode(tokens.count(), pos),
                                     pos);
  tokens.addToken(name, ITokenSpec::skip, regexp, tokenAction, pos);

  updateStates(states, dynamic_cast<DfaSourceRe*>(regexp));
}


void ScannerSpec::addMoreToken (const vector<string> &states,
                                const string &name, ReNode *regexp,
                                const string &tokenAction,
                                const Position &pos)
  throw (ParseException)
{
  if (tokens.isToken(name))
    throw ParseException(pos, "Duplicate token name.");

  regexp = nodeBuilder.createCatNode(regexp,
                               nodeBuilder.createEotNode(tokens.count(), pos),
                                     pos);
  tokens.addToken(name, ITokenSpec::more, regexp, tokenAction, pos);

  updateStates(states, dynamic_cast<DfaSourceRe*>(regexp));
}


void ScannerSpec::addKeywordToken (const vector<string> &states,
                                   const string &name, ReNode *regexp,
                                   const string &tokenAction,
                                   const Position &pos)
  throw (ParseException)
{
  if (tokens.isToken(name))
    throw ParseException(pos, "Duplicate token name.");

  regexp = nodeBuilder.createCatNode(regexp,
                               nodeBuilder.createEotNode(tokens.count(), pos),
                                     pos);
  tokens.addToken(name, ITokenSpec::keyword, regexp, tokenAction, pos);

  updateStates(states, dynamic_cast<DfaSourceRe*>(regexp));
}

  
void ScannerSpec::addSpecialToken (const string &name, const Position &pos)
  throw (ParseException)
{
  if (tokens.isToken(name))
    throw ParseException(pos, "Duplicate token name.");

  tokens.addToken(name, pos);
}


void ScannerSpec::setPreambleCode (const string &block, const Position &pos)
{
  preambleCode.code = block;
  preambleCode.pos = pos;
}


void ScannerSpec::addCodeBlock (const string &block, const Position &pos)
{
  userCode.push_back(CodeChunk(pos, block));
}


void ScannerSpec::setInheritance (const string &inheritance_,
                                  const Position &pos)
{
  inheritance.pos = pos;
  inheritance.code = inheritance_;
}


void ScannerSpec::setClassName (const string &name_)
{
  className = name_;
}


bool operator == (const LexicalStateSpec &st, const string &s)
{
  return st.name == s;
}


void ScannerSpec::updateStates (const vector<string> &tokStates,
                                DfaSourceRe *regexp)
{
  /*
    this goes like this:
    - first, if no state was given, make it be START
    - then if the state name is "*", add the new regexp to all the states
                                (including *).
    - then, for each state in the list, add the new regexp; if the state is
                                new, add also the regexp corresponding to
                                "*").
  */
  vector<string> vs = tokStates;
  
  if (vs.size() == 0) vs.push_back("START");
  if (vs[0] == "*") {
    if (starRe != NULL)
      starRe = dynamic_cast<DfaSourceRe*>(nodeBuilder.createOrNode(starRe, regexp,
                                                                   Position()));
    else starRe = regexp;
    
    for (int i = 0; i < states.size(); i++)
      states[i].regexp =
        dynamic_cast<DfaSourceRe*>(nodeBuilder.createOrNode(states[i].regexp,
                                                            regexp->clone(),
                                                            Position()));
  } else {
    DfaSourceRe *tmp = regexp; // holds a new copy of regexp at each iteration
                             // the first one is the original itself
    for (int i = 0; i < vs.size(); i++)
    {
      vector<LexicalStateSpec>::iterator j;
      if (tmp == NULL) tmp = dynamic_cast<DfaSourceRe*>(regexp->clone());
      
      if ((j = find(states.begin(), states.end(), vs[i])) != states.end())
      { //is this state already added?
        (*j).regexp =
          dynamic_cast<DfaSourceRe*>(nodeBuilder.createOrNode((*j).regexp,
                                                              tmp,
                                                              Position()));
        tmp = NULL; // ask for a new copy of regexp
      } else 
      { //no, we have to create it and also add the regexp for * if any
        if (starRe != NULL)
          tmp =
            dynamic_cast<DfaSourceRe*>(nodeBuilder.createOrNode(starRe->clone(),
                                                                tmp,
                                                                Position()));
        states.push_back(LexicalStateSpec(vs[i], tmp));
        tmp = NULL;  // ask for a new copy of regexp
      }
    }
  }
}


#ifdef DEBUG

void ScannerSpec::dump (ostream &os) const
{
  os << "ScannerSpec duump: " << endl
     << "className: " << className << endl
     << "inheritance: " << inheritance.code << endl
     << "preambleCode:" << endl
     << preambleCode.code << endl
     << "----------------------------------------------------------" << endl
     << "userCode:" << endl;
  for (int i = 0; i < userCode.size(); i++)
    os << userCode[i].code;
  os << endl
     << "----------------------------------------------------------" << endl
     << "Tokens: " << endl;
  for (int i = 0; i < tokens.count(); i++)
    tokens[i].dump(os);
  os << "----------------------------------------------------------" << endl;
  os << "States:" << endl;
  for (int i = 0; i < states.size(); i++) {
    states[i].dump(os);
    os << "----------------------------------------------------------" << endl;
  }
}

void LexicalStateSpec::dump (ostream &os) const
{
  os << "State: " << name << endl
     << "Regexp: " << endl;
  regexp->dump(os);
  os << endl;
}


#endif

---