docs: function tokenize(source, options){
    var result = [];
    var tagStack = [];
    var lastTag = { children: result };
    var parseTag = false;
    var token;
    var state = TEXT;
    var pos = 0;
    var textStateEndPos = 0;
    var textEndPos;
    var bufferPos;
    var startPos;
    var m;
  
    /** @cut */ var attrMap;  // map to detect attribute duplicates
    /** @cut */ result.source_ = source;
    /** @cut */ result.warns = [];
  
    // skip starting whitespaces
    if (!options || options.trim !== false)
    {
      pos = textStateEndPos = source.match(/^\s*/)[0].length;
      source = source.trimRight();  // TODO: replace for basis.string.trimRight
    }
  
    while (pos < source.length || state != TEXT)
    {
      state.lastIndex = pos;
      startPos = pos;
  
      m = state.exec(source);
  
      if (!m || m.index !== pos)
      {
        // treat broken comment reference as comment content
        if (state == REFERENCE && token && token.type == TYPE_COMMENT)
        {
          state = COMMENT;
          continue;
        }
  
        if (parseTag)
          lastTag = tagStack.pop();
  
        if (token)
          lastTag.children.pop();
  
        if (token = lastTag.children.pop())
        {
          if (token.type == TYPE_TEXT && !token.refs)
            textStateEndPos -= token.value.length;
          else
            lastTag.children.push(token);
        }
  
        parseTag = false;
        state = TEXT;
        continue;
      }
  
      pos = state.lastIndex;
  
      //stat[state] = (stat[state] || 0) + 1;
      switch (state)
      {
        case TEXT:
  
          textEndPos = startPos + m[1].length;
  
          if (textStateEndPos != textEndPos)
          {
            var sourceText = textStateEndPos == startPos
              ? m[1]
              : source.substring(textStateEndPos, textEndPos);
  
            sourceText = sourceText.replace(/\s*(\r\n?|\n\r?)\s*/g, '');
  
            if (sourceText)
              lastTag.children.push({
                type: TYPE_TEXT,
                value: sourceText,
                range: {
                  start_: textStateEndPos,
                  end_: textEndPos
                }
              });
          }
  
          textStateEndPos = textEndPos;
  
          if (m[3])
          {
            lastTag.children.push({
              type: TYPE_TEXT,
              refs: ['l10n:' + m[3]],
              value: '{l10n:' + m[3] + '}',
              range: {
                start_: textEndPos,
                end_: pos
              }
            });
          }
          else if (m[2] == '{')
          {
            bufferPos = pos - 1;
            lastTag.children.push(token = {
              type: TYPE_TEXT,
              range: {
                start_: textEndPos,
                end_: textEndPos
              }
            });
            state = REFERENCE;
          }
          else if (m[4])
          {
            if (m[4] == '/')
            {
              token = null;
              state = CLOSE_TAG;
            }
            else //if (m[3] == '!--')
            {
              lastTag.children.push(token = {
                type: TYPE_COMMENT,
                range: {
                  start_: textEndPos,
                  end_: textEndPos
                }
              });
  
              if (m[5])
              {
                bufferPos = pos - m[5].length;
                state = REFERENCE;
              }
              else
              {
                bufferPos = pos;
                state = COMMENT;
              }
            }
          }
          else if (m[2]) // m[2] == '<' open tag
          {
            parseTag = true;
            tagStack.push(lastTag);
  
            lastTag.children.push(token = {
              type: TYPE_ELEMENT,
              attrs: [],
              children: [],
              range: {
                start_: textEndPos,
                end_: textEndPos
              }
            });
            lastTag = token;
  
            state = TAG_NAME;
  
            // create attribute map
            /** @cut */ attrMap = {};
          }
  
          break;
  
        case CLOSE_TAG:
          if (m[1] !== (lastTag.prefix ? lastTag.prefix + ':' : '') + lastTag.name)
          {
            lastTag.children.push({
              type: TYPE_TEXT,
              value: '</' + m[0],
              range: {
                start_: startPos - 2,
                end_: startPos + m[0].length
              }
            });
            /** @cut */ result.warns.push(['Wrong close tag: ' + source.substr(startPos - 2, m[0].length + 2), lastTag.children[lastTag.children.length - 1]]);
          }
          else
          {
            lastTag.range.end_ = startPos + m[0].length;
            lastTag = tagStack.pop();
          }
  
          state = TEXT;
          break;
  
        case TAG_NAME:
        case ATTRIBUTE_NAME_OR_END:
          if (m[2] == ':')
          {
            if (token.prefix)      // prefix was before, break tag parse
              state = BREAK_TAG_PARSE;
            else
              token.prefix = m[1];
  
            break;
          }
  
          if (m[1])
          {
            // store name (it may be null when check for attribute and end)
            token.name = m[1];
            token.range.end_ = startPos + m[1].length;
  
            // store attribute
            if (token.type == TYPE_ATTRIBUTE)
            {
              // detect attribute duplicates
              /** @cut */ var fullName = (token.prefix ? token.prefix + ':' : '') + token.name;
              /** @cut */ if (Object.prototype.hasOwnProperty.call(attrMap, fullName))
              /** @cut */   result.warns.push(['Duplicate attribute: ' + fullName, token]);
              /** @cut */ attrMap[fullName] = true;
  
              lastTag.attrs.push(token);
            }
          }
  
          if (m[2] == '{')
          {
            if (token.type == TYPE_ELEMENT)
              state = REFERENCE;
            else
              state = BREAK_TAG_PARSE;
  
            break;
          }
  
          if (m[3]) // end tag declaration
          {
            parseTag = false;
            lastTag.range.end_ = pos;
  
            if (m[3] == '/>' ||
                (!lastTag.prefix && SINGLETON_TAG.test(lastTag.name)))
            {
              /** @cut */ if (m[3] != '/>')
              /** @cut */   result.warns.push(['Tag <' + lastTag.name + '> doesn\'t closed explicit (use `/>` as tag ending)', lastTag]);
  
              lastTag = tagStack.pop();
            }
            else
            {
              // otherwise m[3] == '>'
              if (lastTag.prefix == 'b' && lastTag.name in TAG_IGNORE_CONTENT)
              {
                state = TAG_IGNORE_CONTENT[lastTag.name];
                break;
              }
            }
  
            state = TEXT;
            break;
          }
  
          if (m[2] == '=') // ATTRIBUTE_NAME_OR_END only
          {
            state = ATTRIBUTE_VALUE;
            break;
          }
  
          // m[2] == '\s+' next attr, state doesn't change
          token = {
            type: TYPE_ATTRIBUTE,
            range: {
              start_: pos,
              end_: pos
            }
          };
          state = ATTRIBUTE_NAME_OR_END;
          break;
  
        case COMMENT:
          token.value = source.substring(bufferPos, pos - 3);
          token.range.end_ = pos;
          state = TEXT;
          break;
  
        case REFERENCE:
          // add reference to token list name
          if (token.refs)
            token.refs.push(m[1]);
          else
            token.refs = [m[1]];
  
          // go next
          if (m[2] != '|') // m[2] == '}\s*'
          {
            if (token.type == TYPE_TEXT)
            {
              pos -= m[2].length - 1;
              token.value = source.substring(bufferPos, pos);
              token.range.end_ = pos;
              state = TEXT;
            }
            else if (token.type == TYPE_COMMENT)
            {
              state = COMMENT;
            }
            else if (token.type == TYPE_ATTRIBUTE && source[pos] == '=')
            {
              pos++;
              state = ATTRIBUTE_VALUE;
            }
            else // ATTRIBUTE || ELEMENT
            {
              token = {
                type: TYPE_ATTRIBUTE,
                range: {
                  start_: pos,
                  end_: pos
                }
              };
              state = ATTRIBUTE_NAME_OR_END;
            }
          }
  
          // continue to collect references
          break;
  
        case ATTRIBUTE_VALUE:
          token.value = m[1].replace(QUOTE_UNESCAPE, '"');
          token.range.end_ = pos;
          token.valueRange = {
            start_: startPos + 1,
            end_: startPos + 1 + m[1].length
          };
  
          token = {
            type: TYPE_ATTRIBUTE,
            range: {
              start_: pos,
              end_: pos
            }
          };
          state = ATTRIBUTE_NAME_OR_END;
  
          break;
  
        case TAG_IGNORE_CONTENT.text:
        case TAG_IGNORE_CONTENT.style:
          lastTag.children.push({
            type: TYPE_TEXT,
            value: m[1],
            range: {
              start_: startPos,
              end_: startPos + m[1].length
            }
          });
  
          lastTag = tagStack.pop();
  
          state = TEXT;
          break;
  
        default:
          throw SYNTAX_ERROR; // Must never to be here; bug in parser otherwise
      }
  
      if (state == TEXT)
        textStateEndPos = pos;
    }
  
    if (textStateEndPos != pos)
      lastTag.children.push({
        type: TYPE_TEXT,
        value: source.substring(textStateEndPos, pos),
        range: {
          start_: textStateEndPos,
          end_: pos
        }
      });
  
    // process attributes binding, join text tokens, normalize ranges and add location info
    postProcessing(result, options || {}, source);
  
    /** @cut */ if (lastTag.name)
    /** @cut */   result.warns.push(['No close tag for <' + (lastTag.prefix ? lastTag.prefix + ':' : '') + lastTag.name + '>', lastTag]);
    /** @cut */
    /** @cut */ if (!result.warns.length)
    /** @cut */   result.warns = false;
  
    result.templateTokens = true;
  
    return result;
  }