You are an expert in writing code in many different languages. Your goal is to perform code completion for the following code, keeping in mind the rest of the code and the file meta data.
    Metadata details:
    filename: mod.rs
    path of file: ./repos/libsql/vendored/sqlite3-parser/src/lexer/sql
    
    the code of the file until where you have to start completion: //! Adaptation/port of [`SQLite` tokenizer](http://www.sqlite.org/src/artifact?ci=trunk&filename=src/tokenize.c)
use fallible_iterator::FallibleIterator;
use memchr::memchr;

pub use crate::dialect::TokenType;
use crate::dialect::TokenType::*;
use crate::dialect::{
    is_identifier_continue, is_identifier_start, keyword_token, sentinel, MAX_KEYWORD_LEN,
};
use crate::parser::ast::Cmd;
use crate::parser::parse::{yyParser, YYCODETYPE};
use crate::parser::Context;

mod error;
#[cfg(test)]
mod test;

use crate::lexer::scan::ScanError;
use crate::lexer::scan::Splitter;
use crate::lexer::Scanner;
pub use crate::parser::ParserError;
pub use error::Error;

// TODO Extract scanning stuff and move this into the parser crate
// to make possible to use the tokenizer without depending on the parser...

pub struct Parser<'input> {
    input: &'input [u8],
    scanner: Scanner<Tokenizer>,
    parser: yyParser<'input>,
}

impl<'input> Parser<'input> {
    pub fn new(input: &'input [u8]) -> Parser<'input> {
        let lexer = Tokenizer::new();
        let scanner = Scanner::new(lexer);
        let ctx = Context::new(input);
        let parser = yyParser::new(ctx);
        Parser {
            input,
            scanner,
            parser,
        }
    }

    pub fn reset(&mut self, input: &'input [u8]) {
        self.input = input;
        self.scanner.reset();
    }

    pub fn line(&self) -> u64 {
        self.scanner.line()
    }
    pub fn column(&self) -> usize {
        self.scanner.column()
    }
}

/*
 ** Return the id of the next token in input.
 */
fn get_token(scanner: &mut Scanner<Tokenizer>, input: &[u8]) -> Result<TokenType, Error> {
    let mut t = {
        let (_, token_type) = match scanner.scan(input)? {
            (_, None, _) => {
             
    
    fn next(&mut self) -> Result<Option<Cmd>, Error> {
        //print!("line: {}, column: {}: ", self.scanner.line(), self.scanner.column());
        self.parser.ctx.reset();
        let mut last_token_parsed = TK_EOF;
        let mut eof = false;
        loop {
            let (start, (value, mut token_type), end) = match self.scanner.scan(self.input)? {
                (_, None, _) => {
                    eof = true;
                    break;
                }
                (start, Some(tuple), end) => (start, tuple, end),
            };
            let token = if token_type >= TK_WINDOW {
                debug_assert!(
                    token_type == TK_OVER || token_type == TK_FILTER || token_type == TK_WINDOW
                );
                self.scanner.mark();
                if token_type == TK_WINDOW {
                    token_type = analyze_window_keyword(&mut self.scanner, self.input)?;
                } else if token_type == TK_OVER {
                    token_type =
                        analyze_over_keyword(&mut self.scanner, self.input, last_token_parsed)?;
                } else if token_type == TK_FILTER {
                    token_type =
                        analyze_filter_keyword(&mut self.scanner, self.input, last_token_parsed)?;
                }
                self.scanner.reset_to_mark();
                token_type.to_token(start, value, end)
            } else {
                token_type.to_token(start, value, end)
            };
            //println!("({:?}, {:?})", token_type, token);
            try_with_position!(self.scanner, self.parser.sqlite3Parser(token_type, token));
            last_token_parsed = token_type;
            if self.parser.ctx.done() {
                //println!();
                break;
            }
        }
        if last_token_parsed == TK_EOF {
            return Ok(None); // empty input
        }
        /* Upon reaching the end of input, call the parser two more times
        with tokens TK_SEMI and 0, in that order. */
        if eof && self.parser.ctx.is_ok() {
            if last_token_parsed != TK_SEMI {
                try_with_position!(
                    self.scanner,
                    self.parser
                        .sqlite3Parser(TK_SEMI, sentinel(self.input.len()))
                );
            }
            try_with_position!(
                self.scanner,
                self.parser
                    .sqlite3Parser(TK_EOF, sentinel(self.input.len()))
            );
        }
        self.parser.sqlite3ParserFinalize();
        if let Some(e) = self.parser.ctx.error() {
            let err = Error::ParserError(e, Some((self.scanner.line(), self.scanner.column())));
            return Err(err);
        }
        let cmd = self.parser.ctx.cmd();
        Ok(cmd)
    }