docs: function tokenBase(stream, state) {
      // Handle scope changes
      var leaving_expr = state.leaving_expr;
      if(stream.sol()) {
        leaving_expr = false;
      }
      state.leaving_expr = false;
      if(leaving_expr) {
        if(stream.match(/^'+/)) {
          return 'operator';
        }
  
      }
  
      if(stream.match(/^\.{2,3}/)) {
        return 'operator';
      }
  
      if (stream.eatSpace()) {
        return null;
      }
  
      var ch = stream.peek();
      // Handle Comments
      if (ch === '#') {
          stream.skipToEnd();
          return 'comment';
      }
      if(ch==='[') {
        state.scopes.push("[");
      }
  
      if(ch==='{') {
        state.scopes.push("{");
      }
  
      var scope=cur_scope(state);
  
      if(scope==='[' && ch===']') {
        state.scopes.pop();
        state.leaving_expr=true;
      }
  
      if(scope==='{' && ch==='}') {
        state.scopes.pop();
        state.leaving_expr=true;
      }
  
      if(ch===')') {
        state.leaving_expr = true;
      }
  
      var match;
      if(!in_array(state) && (match=stream.match(openers, false))) {
        state.scopes.push(match);
      }
  
      if(!in_array(state) && stream.match(closers, false)) {
        state.scopes.pop();
      }
  
      if(in_array(state)) {
        if(stream.match(/^end/)) {
          return 'number';
        }
  
      }
  
      if(stream.match(/^=>/)) {
        return 'operator';
      }
  
  
      // Handle Number Literals
      if (stream.match(/^[0-9\.]/, false)) {
        var imMatcher = RegExp(/^im\b/);
        var floatLiteral = false;
        // Floats
        if (stream.match(/^\d*\.(?!\.)\d+([ef][\+\-]?\d+)?/i)) { floatLiteral = true; }
        if (stream.match(/^\d+\.(?!\.)\d*/)) { floatLiteral = true; }
        if (stream.match(/^\.\d+/)) { floatLiteral = true; }
        if (floatLiteral) {
            // Float literals may be "imaginary"
            stream.match(imMatcher);
            state.leaving_expr = true;
            return 'number';
        }
        // Integers
        var intLiteral = false;
        // Hex
        if (stream.match(/^0x[0-9a-f]+/i)) { intLiteral = true; }
        // Binary
        if (stream.match(/^0b[01]+/i)) { intLiteral = true; }
        // Octal
        if (stream.match(/^0o[0-7]+/i)) { intLiteral = true; }
        // Decimal
        if (stream.match(/^[1-9]\d*(e[\+\-]?\d+)?/)) {
            intLiteral = true;
        }
        // Zero by itself with no other piece of number.
        if (stream.match(/^0(?![\dx])/i)) { intLiteral = true; }
        if (intLiteral) {
            // Integer literals may be "long"
            stream.match(imMatcher);
            state.leaving_expr = true;
            return 'number';
        }
      }
  
      if(stream.match(/^(::)|(<:)/)) {
        return 'operator';
      }
  
      // Handle symbols
      if(!leaving_expr && stream.match(symbol)) {
        return 'string';
      }
  
      // Handle operators and Delimiters
      if (stream.match(operators)) {
        return 'operator';
      }
  
  
      // Handle Strings
      if (stream.match(stringPrefixes)) {
        state.tokenize = tokenStringFactory(stream.current());
        return state.tokenize(stream, state);
      }
  
      if (stream.match(macro)) {
        return 'meta';
      }
  
  
      if (stream.match(delimiters)) {
        return null;
      }
  
      if (stream.match(keywords)) {
        return 'keyword';
      }
  
      if (stream.match(builtins)) {
        return 'builtin';
      }
  
  
      if (stream.match(identifiers)) {
        state.leaving_expr=true;
        return 'variable';
      }
      // Handle non-detected items
      stream.next();
      return ERRORCLASS;
    }