# -*- coding: utf8 -*-

SRC = r'''

hello world

"hello\t\nworld"

'''

import ply.lex as lex

tokens = ['sym','str']

states = (('str','exclusive'),)

t_str_ignore = ''

def t_str(t):
    r'"'
    t.lexer.push_state('str')
    t.lexer.accumulator = ''
def t_str_str(t):
    r'"'
    t.lexer.pop_state()
    t.value = t.lexer.accumulator ; return t
def t_str_ln(t):
    r'\\n'
    t.lexer.accumulator += '\n'
def t_str_tab(t):
    r'\\t'
    t.lexer.accumulator += '\t'
def t_str_any(t):
    r'.'
    t.lexer.accumulator += t.value


t_ignore = ' \t\r\n'

def t_sym(t):
    r'[^ \t\r\n]+'
    return t

def t_ANY_error(t):
    raise SyntaxError(t)    

lexer = lex.lex()

lexer.input(SRC)

while True:
    token = lexer.token()
    if not token: break
    print token

import re

class Token:
    def __init__(self,V=''):
        # for PLY token must have type & value fields
        self.type  = self.__class__.__name__.lower() 
        self.value = V
    def __repr__(self):
        return '<%s:%s>' % (self.type,self.value)
    def __add__(self,str):
        self.value += str ; return self

def DFA(s):
    # start/stop indexes in `s` source code string
    p = 0 ; pe = len(s)
    # DFA `S`tate
    S = 'start'
    token = None
    def space(): return re.match(r'[ \t\r\n]',s[p])
    def asciz(): return re.match(r'[a-zA-Z0-9_]',s[p])
    while True:
        if S == 'start':
            if p >= pe: S = 'end' ; break                               # ?EOF
            elif space(): p += 1                                        # ~[ \t\r\n]
            elif asciz(): S = 'symbol' ; token = Token(s[p]) ; p += 1   # ~[a-zA-Z0-9_]
            elif s[p] == '"': S = 'string' ; token = Token('') ; p += 1 # ?"
            else: raise SyntaxError((S,p,s[p]))                         # ?else
        elif S == 'symbol':
            if p >= pe: S = 'end' ; print token ; break                 # ?EOF
            elif asciz(): token + s[p] ; p += 1                         # ~[a-zA-Z0-9_]
            elif space(): S = 'start' ; print token                     # ~[ \t\r\n]
            else: raise SyntaxError((S,p,s[p]))                         # ?else
        elif S == 'string':
            if p >= pe: raise SyntaxError((S,p,'EOF'))                  # ?EOF
            elif s[p] == 'ё': raise SyntaxError((S,p,s[p]))             # ё
            elif s[p] != '"': token + s[p] ; p += 1                     # !"
            elif s[p] == '"': S = 'start' ; p += 1; print token         #  "
            else: raise SyntaxError((S,p,s[p]))                         # ?else
        else:
            raise SyntaxError((S,p,s[p]))                               # S undef

DFA(SRC)
