"""Testcases for new cssutils.tokenize.Tokenizer

TODO: old tests as new ones are **not complete**!
"""

import sys

import pytest

import cssutils.tokenize2 as tokenize2
from cssutils.tokenize2 import Tokenizer


class TestTokenizer:
    testsall = {
        # IDENT
        'äöü߀': [('IDENT', 'äöü߀', 1, 1)],
        ' a ': [('S', ' ', 1, 1), ('IDENT', 'a', 1, 2), ('S', ' ', 1, 3)],
        '_a': [('IDENT', '_a', 1, 1)],
        '-a': [('IDENT', '-a', 1, 1)],
        '--a': [('IDENT', '--a', 1, 1)],
        'aA-_\200\377': [('IDENT', 'aA-_\200\377', 1, 1)],
        'a1': [('IDENT', 'a1', 1, 1)],
        # escapes must end with S or max 6 digits:
        '\\44 b': [('IDENT', 'Db', 1, 1)],
        '\\44  b': [('IDENT', 'D', 1, 1), ('S', ' ', 1, 5), ('IDENT', 'b', 1, 6)],
        '\\44\nb': [('IDENT', 'Db', 1, 1)],
        '\\44\rb': [('IDENT', 'Db', 1, 1)],
        '\\44\fb': [('IDENT', 'Db', 1, 1)],
        '\\44\n*': [('IDENT', 'D', 1, 1), ('CHAR', '*', 2, 1)],
        '\\44  a': [('IDENT', 'D', 1, 1), ('S', ' ', 1, 5), ('IDENT', 'a', 1, 6)],
        # TODO:
        # Note that this means that a "real" space after the escape sequence
        # must itself either be escaped or doubled:
        '\\44\\ x': [('IDENT', 'D\\ x', 1, 1)],
        '\\44  ': [('IDENT', 'D', 1, 1), ('S', ' ', 1, 5)],
        r'\44': [('IDENT', 'D', 1, 1)],
        r'\\': [('IDENT', r'\\', 1, 1)],
        r'\{': [('IDENT', r'\{', 1, 1)],
        r'\"': [('IDENT', r'\"', 1, 1)],
        r'\(': [('IDENT', r'\(', 1, 1)],
        r'\1 \22 \333 \4444 \55555 \666666 \777777 7 \7777777': [
            (
                ('IDENT', '\x01"\u0333\u4444\U00055555\\666666 \\777777 7', 1, 1)
                if sys.maxunicode > 0x10000
                else ('IDENT', '\x01"\u0333\u4444\\55555 \\666666 \\777777 7', 1, 1)
            ),
            ('S', ' ', 1, 43),
            ('IDENT', '\\7777777', 1, 44),
        ],
        # Not a function, important for media queries
        'and(': [('IDENT', 'and', 1, 1), ('CHAR', '(', 1, 4)],
        '\\1 b': [('IDENT', '\x01b', 1, 1)],
        # repeat: '\\44 b': [('IDENT', 'Db', 1, 1)],
        '\\123 b': [('IDENT', '\u0123b', 1, 1)],
        '\\1234 b': [('IDENT', '\u1234b', 1, 1)],
        '\\12345 b': [
            (
                ('IDENT', '\U00012345b', 1, 1)
                if sys.maxunicode > 0x10000
                else ('IDENT', '\\12345 b', 1, 1)
            )
        ],
        '\\123456 b': [('IDENT', '\\123456 b', 1, 1)],
        '\\1234567 b': [
            ('IDENT', '\\1234567', 1, 1),
            ('S', ' ', 1, 9),
            ('IDENT', 'b', 1, 10),
        ],
        '\\{\\}\\(\\)\\[\\]\\#\\@\\.\\,': [
            ('IDENT', '\\{\\}\\(\\)\\[\\]\\#\\@\\.\\,', 1, 1)
        ],
        # STRING
        ' "" ': [('S', ' ', 1, 1), ('STRING', '""', 1, 2), ('S', ' ', 1, 4)],
        ' "\'" ': [('S', ' ', 1, 1), ('STRING', '"\'"', 1, 2), ('S', ' ', 1, 5)],
        " '' ": [('S', ' ', 1, 1), ('STRING', "''", 1, 2), ('S', ' ', 1, 4)],
        "'\\\n'": [('STRING', "''", 1, 1)],
        "'\\\n\\\n\\\n'": [('STRING', "''", 1, 1)],
        "'\\\f'": [('STRING', "''", 1, 1)],
        "'\\\r'": [('STRING', "''", 1, 1)],
        "'1\\\n2'": [('STRING', "'12'", 1, 1)],
        "'1\\\r\n2'": [('STRING', "'12'", 1, 1)],
        # r'"\0020|\0020"': [('STRING', '"\\0020|\\0020"', 1, 1)],
        r'"\61|\0061"': [('STRING', '"a|a"', 1, 1)],
        # HASH
        ' #a ': [('S', ' ', 1, 1), ('HASH', '#a', 1, 2), ('S', ' ', 1, 4)],
        '#ccc': [('HASH', '#ccc', 1, 1)],
        '#111': [('HASH', '#111', 1, 1)],
        '#a1a1a1': [('HASH', '#a1a1a1', 1, 1)],
        '#1a1a1a': [('HASH', '#1a1a1a', 1, 1)],
        # NUMBER, for plus see CSS3
        ' 0 ': [('S', ' ', 1, 1), ('NUMBER', '0', 1, 2), ('S', ' ', 1, 3)],
        ' 0.1 ': [('S', ' ', 1, 1), ('NUMBER', '0.1', 1, 2), ('S', ' ', 1, 5)],
        ' .0 ': [('S', ' ', 1, 1), ('NUMBER', '.0', 1, 2), ('S', ' ', 1, 4)],
        ' -0 ': [
            ('S', ' ', 1, 1),
            # ('CHAR', '-', 1, 2),
            # ('NUMBER', '0', 1, 3),
            ('NUMBER', '-0', 1, 2),
            ('S', ' ', 1, 4),
        ],
        # PERCENTAGE
        ' 0% ': [('S', ' ', 1, 1), ('PERCENTAGE', '0%', 1, 2), ('S', ' ', 1, 4)],
        ' .5% ': [('S', ' ', 1, 1), ('PERCENTAGE', '.5%', 1, 2), ('S', ' ', 1, 5)],
        # URI
        ' url() ': [('S', ' ', 1, 1), ('URI', 'url()', 1, 2), ('S', ' ', 1, 7)],
        ' url(a) ': [('S', ' ', 1, 1), ('URI', 'url(a)', 1, 2), ('S', ' ', 1, 8)],
        ' url("a") ': [('S', ' ', 1, 1), ('URI', 'url("a")', 1, 2), ('S', ' ', 1, 10)],
        ' url( a ) ': [('S', ' ', 1, 1), ('URI', 'url( a )', 1, 2), ('S', ' ', 1, 10)],
        # UNICODE-RANGE
        # CDO
        ' <!-- ': [('S', ' ', 1, 1), ('CDO', '<!--', 1, 2), ('S', ' ', 1, 6)],
        '"<!--""-->"': [('STRING', '"<!--"', 1, 1), ('STRING', '"-->"', 1, 7)],
        # CDC
        ' --> ': [('S', ' ', 1, 1), ('CDC', '-->', 1, 2), ('S', ' ', 1, 5)],
        # S
        ' ': [('S', ' ', 1, 1)],
        '  ': [('S', '  ', 1, 1)],
        '\r': [('S', '\r', 1, 1)],
        '\n': [('S', '\n', 1, 1)],
        '\r\n': [('S', '\r\n', 1, 1)],
        '\f': [('S', '\f', 1, 1)],
        '\t': [('S', '\t', 1, 1)],
        '\r\n\r\n\f\t ': [('S', '\r\n\r\n\f\t ', 1, 1)],
        # COMMENT, for incomplete see later
        '/*x*/ ': [('COMMENT', '/*x*/', 1, 1), ('S', ' ', 1, 6)],
        # FUNCTION
        ' x( ': [('S', ' ', 1, 1), ('FUNCTION', 'x(', 1, 2), ('S', ' ', 1, 4)],
        # INCLUDES
        ' ~= ': [('S', ' ', 1, 1), ('INCLUDES', '~=', 1, 2), ('S', ' ', 1, 4)],
        '~==': [('INCLUDES', '~=', 1, 1), ('CHAR', '=', 1, 3)],
        # DASHMATCH
        ' |= ': [('S', ' ', 1, 1), ('DASHMATCH', '|=', 1, 2), ('S', ' ', 1, 4)],
        '|==': [('DASHMATCH', '|=', 1, 1), ('CHAR', '=', 1, 3)],
        # CHAR
        ' @ ': [('S', ' ', 1, 1), ('CHAR', '@', 1, 2), ('S', ' ', 1, 3)],
        # --- overwritten for CSS 2.1 ---
        # LBRACE
        ' { ': [('S', ' ', 1, 1), ('CHAR', '{', 1, 2), ('S', ' ', 1, 3)],
        # PLUS
        ' + ': [('S', ' ', 1, 1), ('CHAR', '+', 1, 2), ('S', ' ', 1, 3)],
        # GREATER
        ' > ': [('S', ' ', 1, 1), ('CHAR', '>', 1, 2), ('S', ' ', 1, 3)],
        # COMMA
        ' , ': [('S', ' ', 1, 1), ('CHAR', ',', 1, 2), ('S', ' ', 1, 3)],
        # class
        ' . ': [('S', ' ', 1, 1), ('CHAR', '.', 1, 2), ('S', ' ', 1, 3)],
    }

    tests3 = {
        # UNICODE-RANGE
        ' u+0 ': [('S', ' ', 1, 1), ('UNICODE-RANGE', 'u+0', 1, 2), ('S', ' ', 1, 5)],
        ' u+01 ': [('S', ' ', 1, 1), ('UNICODE-RANGE', 'u+01', 1, 2), ('S', ' ', 1, 6)],
        ' u+012 ': [
            ('S', ' ', 1, 1),
            ('UNICODE-RANGE', 'u+012', 1, 2),
            ('S', ' ', 1, 7),
        ],
        ' u+0123 ': [
            ('S', ' ', 1, 1),
            ('UNICODE-RANGE', 'u+0123', 1, 2),
            ('S', ' ', 1, 8),
        ],
        ' u+01234 ': [
            ('S', ' ', 1, 1),
            ('UNICODE-RANGE', 'u+01234', 1, 2),
            ('S', ' ', 1, 9),
        ],
        ' u+012345 ': [
            ('S', ' ', 1, 1),
            ('UNICODE-RANGE', 'u+012345', 1, 2),
            ('S', ' ', 1, 10),
        ],
        ' u+0123456 ': [
            ('S', ' ', 1, 1),
            ('UNICODE-RANGE', 'u+012345', 1, 2),
            ('NUMBER', '6', 1, 10),
            ('S', ' ', 1, 11),
        ],
        ' U+123456 ': [
            ('S', ' ', 1, 1),
            ('UNICODE-RANGE', 'U+123456', 1, 2),
            ('S', ' ', 1, 10),
        ],
        ' \\55+abcdef ': [
            ('S', ' ', 1, 1),
            ('UNICODE-RANGE', 'U+abcdef', 1, 2),
            ('S', ' ', 1, 12),
        ],
        ' \\75+abcdef ': [
            ('S', ' ', 1, 1),
            ('UNICODE-RANGE', 'u+abcdef', 1, 2),
            ('S', ' ', 1, 12),
        ],
        ' u+0-1 ': [
            ('S', ' ', 1, 1),
            ('UNICODE-RANGE', 'u+0-1', 1, 2),
            ('S', ' ', 1, 7),
        ],
        ' u+0-1, u+123456-abcdef ': [
            ('S', ' ', 1, 1),
            ('UNICODE-RANGE', 'u+0-1', 1, 2),
            ('CHAR', ',', 1, 7),
            ('S', ' ', 1, 8),
            ('UNICODE-RANGE', 'u+123456-abcdef', 1, 9),
            ('S', ' ', 1, 24),
        ],
        # specials
        'c\\olor': [('IDENT', 'c\\olor', 1, 1)],
        # '-1': [('CHAR', '-', 1, 1), ('NUMBER', '1', 1, 2)],
        # '-1px': [('CHAR', '-', 1, 1), ('DIMENSION', '1px', 1, 2)],
        '-1': [('NUMBER', '-1', 1, 1)],
        '-1px': [('DIMENSION', '-1px', 1, 1)],
        # ATKEYWORD
        ' @x ': [('S', ' ', 1, 1), ('ATKEYWORD', '@x', 1, 2), ('S', ' ', 1, 4)],
        '@X': [('ATKEYWORD', '@X', 1, 1)],
        '@\\x': [('ATKEYWORD', '@\\x', 1, 1)],
        # -
        '@1x': [('CHAR', '@', 1, 1), ('DIMENSION', '1x', 1, 2)],
        # DIMENSION
        ' 0px ': [('S', ' ', 1, 1), ('DIMENSION', '0px', 1, 2), ('S', ' ', 1, 5)],
        ' 1s ': [('S', ' ', 1, 1), ('DIMENSION', '1s', 1, 2), ('S', ' ', 1, 4)],
        '0.2EM': [('DIMENSION', '0.2EM', 1, 1)],
        '1p\\x': [('DIMENSION', '1p\\x', 1, 1)],
        '1PX': [('DIMENSION', '1PX', 1, 1)],
        # NUMBER
        ' - 0 ': [
            ('S', ' ', 1, 1),
            ('CHAR', '-', 1, 2),
            ('S', ' ', 1, 3),
            ('NUMBER', '0', 1, 4),
            ('S', ' ', 1, 5),
        ],
        ' + 0 ': [
            ('S', ' ', 1, 1),
            ('CHAR', '+', 1, 2),
            ('S', ' ', 1, 3),
            ('NUMBER', '0', 1, 4),
            ('S', ' ', 1, 5),
        ],
        # PREFIXMATCH
        ' ^= ': [('S', ' ', 1, 1), ('PREFIXMATCH', '^=', 1, 2), ('S', ' ', 1, 4)],
        '^==': [('PREFIXMATCH', '^=', 1, 1), ('CHAR', '=', 1, 3)],
        # SUFFIXMATCH
        ' $= ': [('S', ' ', 1, 1), ('SUFFIXMATCH', '$=', 1, 2), ('S', ' ', 1, 4)],
        '$==': [('SUFFIXMATCH', '$=', 1, 1), ('CHAR', '=', 1, 3)],
        # SUBSTRINGMATCH
        ' *= ': [('S', ' ', 1, 1), ('SUBSTRINGMATCH', '*=', 1, 2), ('S', ' ', 1, 4)],
        '*==': [('SUBSTRINGMATCH', '*=', 1, 1), ('CHAR', '=', 1, 3)],
        # BOM only at start
        '\xfe\xff ': [('BOM', '\xfe\xff', 1, 1), ('S', ' ', 1, 1)],
        ' \xfe\xff ': [('S', ' ', 1, 1), ('IDENT', '\xfe\xff', 1, 2), ('S', ' ', 1, 4)],
        '\xef\xbb\xbf ': [('BOM', '\xef\xbb\xbf', 1, 1), ('S', ' ', 1, 1)],
        ' \xef\xbb\xbf ': [
            ('S', ' ', 1, 1),
            ('IDENT', '\xef\xbb\xbf', 1, 2),
            ('S', ' ', 1, 5),
        ],
    }

    tests2 = {
        # escapes work not for a-f!
        # IMPORT_SYM
        ' @import ': [
            ('S', ' ', 1, 1),
            ('IMPORT_SYM', '@import', 1, 2),
            ('S', ' ', 1, 9),
        ],
        '@IMPORT': [('IMPORT_SYM', '@IMPORT', 1, 1)],
        '@\\49\r\nMPORT': [('IMPORT_SYM', '@\\49\r\nMPORT', 1, 1)],
        r'@\i\m\p\o\r\t': [('IMPORT_SYM', r'@\i\m\p\o\r\t', 1, 1)],
        r'@\I\M\P\O\R\T': [('IMPORT_SYM', r'@\I\M\P\O\R\T', 1, 1)],
        r'@\49 \04d\0050\0004f\000052\54': [
            ('IMPORT_SYM', r'@\49 \04d\0050\0004f\000052\54', 1, 1)
        ],
        r'@\69 \06d\0070\0006f\000072\74': [
            ('IMPORT_SYM', r'@\69 \06d\0070\0006f\000072\74', 1, 1)
        ],
        # PAGE_SYM
        ' @page ': [('S', ' ', 1, 1), ('PAGE_SYM', '@page', 1, 2), ('S', ' ', 1, 7)],
        '@PAGE': [('PAGE_SYM', '@PAGE', 1, 1)],
        r'@\pa\ge': [('PAGE_SYM', r'@\pa\ge', 1, 1)],
        r'@\PA\GE': [('PAGE_SYM', r'@\PA\GE', 1, 1)],
        r'@\50\41\47\45': [('PAGE_SYM', r'@\50\41\47\45', 1, 1)],
        r'@\70\61\67\65': [('PAGE_SYM', r'@\70\61\67\65', 1, 1)],
        # MEDIA_SYM
        ' @media ': [('S', ' ', 1, 1), ('MEDIA_SYM', '@media', 1, 2), ('S', ' ', 1, 8)],
        '@MEDIA': [('MEDIA_SYM', '@MEDIA', 1, 1)],
        r'@\med\ia': [('MEDIA_SYM', r'@\med\ia', 1, 1)],
        r'@\MED\IA': [('MEDIA_SYM', r'@\MED\IA', 1, 1)],
        '@\\4d\n\\45\r\\44\t\\49\r\nA': [
            ('MEDIA_SYM', '@\\4d\n\\45\r\\44\t\\49\r\nA', 1, 1)
        ],
        '@\\4d\n\\45\r\\44\t\\49\r\\41\f': [
            ('MEDIA_SYM', '@\\4d\n\\45\r\\44\t\\49\r\\41\f', 1, 1)
        ],
        '@\\6d\n\\65\r\\64\t\\69\r\\61\f': [
            ('MEDIA_SYM', '@\\6d\n\\65\r\\64\t\\69\r\\61\f', 1, 1)
        ],
        # FONT_FACE_SYM
        ' @font-face ': [
            ('S', ' ', 1, 1),
            ('FONT_FACE_SYM', '@font-face', 1, 2),
            ('S', ' ', 1, 12),
        ],
        '@FONT-FACE': [('FONT_FACE_SYM', '@FONT-FACE', 1, 1)],
        r'@f\o\n\t\-face': [('FONT_FACE_SYM', r'@f\o\n\t\-face', 1, 1)],
        r'@F\O\N\T\-FACE': [('FONT_FACE_SYM', r'@F\O\N\T\-FACE', 1, 1)],
        # TODO: "-" as hex!
        r'@\46\4f\4e\54\-\46\41\43\45': [
            ('FONT_FACE_SYM', r'@\46\4f\4e\54\-\46\41\43\45', 1, 1)
        ],
        r'@\66\6f\6e\74\-\66\61\63\65': [
            ('FONT_FACE_SYM', r'@\66\6f\6e\74\-\66\61\63\65', 1, 1)
        ],
        # CHARSET_SYM only if "@charset "!
        '@charset  ': [('CHARSET_SYM', '@charset ', 1, 1), ('S', ' ', 1, 10)],
        ' @charset  ': [
            ('S', ' ', 1, 1),
            ('CHARSET_SYM', '@charset ', 1, 2),  # not at start
            ('S', ' ', 1, 11),
        ],
        '@charset': [('ATKEYWORD', '@charset', 1, 1)],  # no ending S
        '@CHARSET ': [('ATKEYWORD', '@CHARSET', 1, 1), ('S', ' ', 1, 9)],  # uppercase
        '@cha\\rset ': [
            ('ATKEYWORD', '@cha\\rset', 1, 1),  # not literal
            ('S', ' ', 1, 10),
        ],
        # NAMESPACE_SYM
        ' @namespace ': [
            ('S', ' ', 1, 1),
            ('NAMESPACE_SYM', '@namespace', 1, 2),
            ('S', ' ', 1, 12),
        ],
        r'@NAMESPACE': [('NAMESPACE_SYM', r'@NAMESPACE', 1, 1)],
        r'@\na\me\s\pace': [('NAMESPACE_SYM', r'@\na\me\s\pace', 1, 1)],
        r'@\NA\ME\S\PACE': [('NAMESPACE_SYM', r'@\NA\ME\S\PACE', 1, 1)],
        r'@\4e\41\4d\45\53\50\41\43\45': [
            ('NAMESPACE_SYM', r'@\4e\41\4d\45\53\50\41\43\45', 1, 1)
        ],
        r'@\6e\61\6d\65\73\70\61\63\65': [
            ('NAMESPACE_SYM', r'@\6e\61\6d\65\73\70\61\63\65', 1, 1)
        ],
        # ATKEYWORD
        ' @unknown ': [
            ('S', ' ', 1, 1),
            ('ATKEYWORD', '@unknown', 1, 2),
            ('S', ' ', 1, 10),
        ],
        # STRING
        # strings with linebreak in it
        ' "\\na"\na': [
            ('S', ' ', 1, 1),
            ('STRING', '"\\na"', 1, 2),
            ('S', '\n', 1, 7),
            ('IDENT', 'a', 2, 1),
        ],
        " '\\na'\na": [
            ('S', ' ', 1, 1),
            ('STRING', "'\\na'", 1, 2),
            ('S', '\n', 1, 7),
            ('IDENT', 'a', 2, 1),
        ],
        ' "\\r\\n\\t\\n\\ra"a': [
            ('S', ' ', 1, 1),
            ('STRING', '"\\r\\n\\t\\n\\ra"', 1, 2),
            ('IDENT', 'a', 1, 15),
        ],
        # IMPORTANT_SYM is not IDENT!!!
        ' !important ': [
            ('S', ' ', 1, 1),
            ('CHAR', '!', 1, 2),
            ('IDENT', 'important', 1, 3),
            ('S', ' ', 1, 12),
        ],
        '! /*1*/ important ': [
            ('CHAR', '!', 1, 1),
            ('S', ' ', 1, 2),
            ('COMMENT', '/*1*/', 1, 3),
            ('S', ' ', 1, 8),
            ('IDENT', 'important', 1, 9),
            ('S', ' ', 1, 18),
        ],
        '! important': [
            ('CHAR', '!', 1, 1),
            ('S', ' ', 1, 2),
            ('IDENT', 'important', 1, 3),
        ],
        '!\n\timportant': [
            ('CHAR', '!', 1, 1),
            ('S', '\n\t', 1, 2),
            ('IDENT', 'important', 2, 2),
        ],
        '!IMPORTANT': [('CHAR', '!', 1, 1), ('IDENT', 'IMPORTANT', 1, 2)],
        r'!\i\m\p\o\r\ta\n\t': [
            ('CHAR', '!', 1, 1),
            ('IDENT', r'\i\m\p\o\r\ta\n\t', 1, 2),
        ],
        r'!\I\M\P\O\R\Ta\N\T': [
            ('CHAR', '!', 1, 1),
            ('IDENT', r'\I\M\P\O\R\Ta\N\T', 1, 2),
        ],
        r'!\49\4d\50\4f\52\54\41\4e\54': [
            ('CHAR', '!', 1, 1),
            ('IDENT', r'IMPORTANT', 1, 2),
        ],
        r'!\69\6d\70\6f\72\74\61\6e\74': [
            ('CHAR', '!', 1, 1),
            ('IDENT', r'important', 1, 2),
        ],
    }

    # overwriting tests in testsall
    tests2only = {
        # LBRACE
        ' { ': [('S', ' ', 1, 1), ('LBRACE', '{', 1, 2), ('S', ' ', 1, 3)],
        # PLUS
        ' + ': [('S', ' ', 1, 1), ('PLUS', '+', 1, 2), ('S', ' ', 1, 3)],
        # GREATER
        ' > ': [('S', ' ', 1, 1), ('GREATER', '>', 1, 2), ('S', ' ', 1, 3)],
        # COMMA
        ' , ': [('S', ' ', 1, 1), ('COMMA', ',', 1, 2), ('S', ' ', 1, 3)],
        # class
        ' . ': [('S', ' ', 1, 1), ('CLASS', '.', 1, 2), ('S', ' ', 1, 3)],
    }

    testsfullsheet = {
        # escape ends with explicit space but \r\n as single space
        '\\65\r\nb': [('IDENT', 'eb', 1, 1)],
        # STRING
        r'"\""': [('STRING', r'"\""', 1, 1)],
        r'"\" "': [('STRING', r'"\" "', 1, 1)],
        """'\\''""": [('STRING', """'\\''""", 1, 1)],
        ' "\na': [
            ('S', ' ', 1, 1),
            ('INVALID', '"', 1, 2),
            ('S', '\n', 1, 3),
            ('IDENT', 'a', 2, 1),
        ],
        # strings with linebreak in it
        ' "\\na\na': [
            ('S', ' ', 1, 1),
            ('INVALID', '"\\na', 1, 2),
            ('S', '\n', 1, 6),
            ('IDENT', 'a', 2, 1),
        ],
        ' "\\r\\n\\t\\n\\ra\na': [
            ('S', ' ', 1, 1),
            ('INVALID', '"\\r\\n\\t\\n\\ra', 1, 2),
            ('S', '\n', 1, 14),
            ('IDENT', 'a', 2, 1),
        ],
        # URI
        'ur\\l(a)': [('URI', 'ur\\l(a)', 1, 1)],
        'url(a)': [('URI', 'url(a)', 1, 1)],
        '\\55r\\4c(a)': [('URI', 'UrL(a)', 1, 1)],
        '\\75r\\6c(a)': [('URI', 'url(a)', 1, 1)],
        ' url())': [('S', ' ', 1, 1), ('URI', 'url()', 1, 2), ('CHAR', ')', 1, 7)],
        'url("x"))': [('URI', 'url("x")', 1, 1), ('CHAR', ')', 1, 9)],
        "url('x'))": [('URI', "url('x')", 1, 1), ('CHAR', ')', 1, 9)],
    }

    # tests if fullsheet=False is set on tokenizer
    testsfullsheetfalse = {
        # COMMENT incomplete
        '/*': [('CHAR', '/', 1, 1), ('CHAR', '*', 1, 2)],
        # INVALID incomplete
        ' " ': [('S', ' ', 1, 1), ('INVALID', '" ', 1, 2)],
        " 'abc\"with quote\" in it": [
            ('S', ' ', 1, 1),
            ('INVALID', "'abc\"with quote\" in it", 1, 2),
        ],
        # URI incomplete
        'url(a': [('FUNCTION', 'url(', 1, 1), ('IDENT', 'a', 1, 5)],
        'url("a': [('FUNCTION', 'url(', 1, 1), ('INVALID', '"a', 1, 5)],
        "url('a": [('FUNCTION', 'url(', 1, 1), ('INVALID', "'a", 1, 5)],
        "UR\\l('a": [('FUNCTION', 'UR\\l(', 1, 1), ('INVALID', "'a", 1, 6)],
    }

    # tests if fullsheet=True is set on tokenizer
    testsfullsheettrue = {
        # COMMENT incomplete
        '/*': [('COMMENT', '/**/', 1, 1)],
        #        # INVALID incomplete => STRING
        ' " ': [('S', ' ', 1, 1), ('STRING', '" "', 1, 2)],
        " 'abc\"with quote\" in it": [
            ('S', ' ', 1, 1),
            ('STRING', "'abc\"with quote\" in it'", 1, 2),
        ],
        # URI incomplete FUNC => URI
        'url(a': [('URI', 'url(a)', 1, 1)],
        'url( a': [('URI', 'url( a)', 1, 1)],
        'url("a': [('URI', 'url("a")', 1, 1)],
        'url( "a ': [('URI', 'url( "a ")', 1, 1)],
        "url('a": [('URI', "url('a')", 1, 1)],
        'url("a"': [('URI', 'url("a")', 1, 1)],
        "url('a'": [('URI', "url('a')", 1, 1)],
    }

    def setup_method(self):
        # log = cssutils.errorhandler.ErrorHandler()
        self.tokenizer = Tokenizer()

    def test_tokenize(self):
        "cssutils Tokenizer().tokenize()"
        import cssutils.cssproductions

        tokenizer = Tokenizer(
            cssutils.cssproductions.MACROS, cssutils.cssproductions.PRODUCTIONS
        )
        tests = {}
        tests.update(self.testsall)
        tests.update(self.tests2)
        tests.update(self.tests3)
        tests.update(self.testsfullsheet)
        tests.update(self.testsfullsheetfalse)
        for css in tests:
            # check token format
            tokens = tokenizer.tokenize(css)
            for i, actual in enumerate(tokens):
                expected = tests[css][i]
                assert expected == actual

            # check if all same number of tokens
            tokens = list(tokenizer.tokenize(css))
            assert len(tokens) == len(tests[css])

    def test_tokenizefullsheet(self):
        "cssutils Tokenizer().tokenize(fullsheet=True)"
        import cssutils.cssproductions

        tokenizer = Tokenizer(
            cssutils.cssproductions.MACROS, cssutils.cssproductions.PRODUCTIONS
        )
        tests = {}
        tests.update(self.testsall)
        tests.update(self.tests2)
        tests.update(self.tests3)
        tests.update(self.testsfullsheet)
        tests.update(self.testsfullsheettrue)
        for css in tests:
            # check token format
            tokens = tokenizer.tokenize(css, fullsheet=True)
            for i, actual in enumerate(tokens):
                try:
                    expected = tests[css][i]
                except IndexError:
                    # EOF is added
                    assert actual[0] == 'EOF'
                else:
                    assert expected == actual

            # check if all same number of tokens
            tokens = list(tokenizer.tokenize(css, fullsheet=True))
            # EOF is added so -1
            assert len(tokens) - 1 == len(tests[css])


class TestTokenizerUtils:
    """Tests for the util functions of tokenize"""

    has_at_cases = [
        ('foo', 0, 'foo', True),
        ('foo', 0, 'f', True),
        ('foo', 1, 'o', True),
        ('foo', 1, 'oo', True),
        ('foo', 4, 'foo', False),
        ('foo', 0, 'bar', False),
        ('foo', 0, 'foobar', False),
    ]

    @pytest.mark.parametrize("string, pos, text, expected", has_at_cases)
    def test_has_at(self, string, pos, text, expected):
        assert tokenize2.has_at(string, pos, text) == expected

    suffix_eq_cases = [
        ('foobar', 0, 'foobar', True),
        ('foobar', 3, 'bar', True),
        ('foobar', 3, 'foo', False),
        ('foobar', 10, 'bar', False),
    ]

    @pytest.mark.parametrize("string, pos, suffix, expected", suffix_eq_cases)
    def test_suffix_eq(self, string, pos, suffix, expected):
        assert tokenize2.suffix_eq(string, pos, suffix) == expected