Tokenizer Struct Reference

#include <tokenizer.hpp>

Classes
struct	TOKEN_REF

Public Types
typedef int	myint

Public Member Functions
	Tokenizer (std::vector< TOKEN_DEF > &tokendefs)
TOKEN_REF	match (std::smatch &matches, const std::string &str) const
TOKEN_REF	match (std::smatch &matches, const std::string &str, size_t startpos) const

Static Public Member Functions
static unsigned int	count_captures (const std::string &exp)
static std::regex	CompileMultiRegex (const std::vector< TOKEN_DEF > &tokendefs)

Public Attributes
const std::vector< TOKEN_DEF > &	m_tokendefs
const std::regex	m_re

Detailed Description

Definition at line 61 of file tokenizer.hpp.

Member Typedef Documentation

myint

typedef int Tokenizer::myint

Definition at line 66 of file tokenizer.hpp.

Constructor & Destructor Documentation

Tokenizer()

Tokenizer::Tokenizer ( std::vector< TOKEN_DEF > &  tokendefs )

inline

Definition at line 156 of file tokenizer.hpp.

        : m_tokendefs(tokendefs),
          m_re(CompileMultiRegex(tokendefs))
    {
    }

Member Function Documentation

CompileMultiRegex()

static std::regex Tokenizer::CompileMultiRegex ( const std::vector< TOKEN_DEF > &  tokendefs )

inlinestatic

Definition at line 115 of file tokenizer.hpp.

    {
        std::string combinedString;
 
        if (tokendefs.size() == 0)
        {
            return std::regex();
        }
 
        // Validate all token definitions
        for (auto def : tokendefs)
        {
            size_t found = -1;
 
            // Count capture groups
            unsigned int count = count_captures(def.RegExString);
            if (count != 1)
            {
                throw "invalid count!\n";
            }
        }
 
        // Combine all expressions into one (one capture group for each)
        combinedString = "(?:" + tokendefs[0].RegExString + ")";
        for (size_t i = 1; i < tokendefs.size(); i++)
        {
            combinedString += "|(?:" + tokendefs[i].RegExString + ")";
        }
 
        return std::regex(combinedString, std::regex_constants::icase);
    }

count_captures()

static unsigned int Tokenizer::count_captures ( const std::string &  exp )

inlinestatic

Definition at line 70 of file tokenizer.hpp.

    {
        bool in_char_group = false;
        unsigned int count = 0;
 
        for (size_t i = 0; i < exp.size(); i++)
        {
            char c = exp[i];
 
            // Skip escaped characters
            if (c == '\\')
            {
                i++;
                continue;
            }
 
            if (in_char_group)
            {
                if (c == ']')
                {
                    in_char_group = false;
                }
                continue;
            }
 
            if (c == '[')
            {
                in_char_group = true;
                continue;
            }
 
            if (c == '(')
            {
                if (exp[i + 1] != '?')
                {
                    count++;
                }
            }
        }
 
        return count;
    }

Referenced by CompileMultiRegex().

match() [1/2]

TOKEN_REF Tokenizer::match ( std::smatch &  matches, const std::string &  str  ) const

inline

Definition at line 162 of file tokenizer.hpp.

    {
        return match(matches, str, 0);
    }

Referenced by TokenList::TokenList().

match() [2/2]

TOKEN_REF Tokenizer::match ( std::smatch &  matches, const std::string &  str, size_t  startpos  ) const

inline

Definition at line 167 of file tokenizer.hpp.

    {
        const std::string::const_iterator first = str.cbegin() + startpos;
        const std::string::const_iterator last = str.cend();
 
        // If we reached the end, there is nothing more to do
        if (first == last)
        {
            return TOKEN_REF{ static_cast<unsigned int>(startpos), 0, -1 };
        }
 
        time_t start_time = time(NULL);
 
        // Try to find a match
        if (!std::regex_search(first, last, matches, m_re))
        {
            throw "Failed to match\n";
        }
 
        search_time += time(NULL) - start_time;
 
        // Validate that it's at the start of the string
        if (matches.prefix().matched)
        {
            throw "Failed to match at current position!\n";
        }
        
        // We have a match, check which one it is
        for (size_t i = 1; i < matches.size(); i++)
        {
            if (matches[i].matched)
            {
                unsigned int len = static_cast<unsigned int>(matches.length(i));
                int type = m_tokendefs[i - 1].Type;
                return TOKEN_REF{ static_cast<unsigned int>(startpos), len, type};
            }
        }
 
        // We should never get here
        throw "Something went wrong!\n";
    }

Member Data Documentation

m_re

const std::regex Tokenizer::m_re

Definition at line 64 of file tokenizer.hpp.

Referenced by match().

m_tokendefs

const std::vector<TOKEN_DEF>& Tokenizer::m_tokendefs

Definition at line 63 of file tokenizer.hpp.

Referenced by match().

---

The documentation for this struct was generated from the following file:

• sdk/tools/asmpp/tokenizer.hpp