366 lines
14 KiB
Python
366 lines
14 KiB
Python
|
"""Testcases for cssutils.codec"""
|
||
|
|
|
||
|
|
import codecs
|
||
|
|
import io
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
from cssutils import codec
|
||
|
|
|
||
|
|
iostream = io.BytesIO
|
||
|
|
|
||
|
|
|
||
|
|
class Queue:
|
||
|
|
"""
|
||
|
|
queue: write bytes at one end, read bytes from the other end
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self):
|
||
|
|
self._buffer = b""
|
||
|
|
|
||
|
|
def write(self, chars):
|
||
|
|
# TODO ???
|
||
|
|
if isinstance(chars, str):
|
||
|
|
chars = chars.encode()
|
||
|
|
elif isinstance(chars, int):
|
||
|
|
chars = bytes([chars])
|
||
|
|
|
||
|
|
self._buffer += chars
|
||
|
|
|
||
|
|
def read(self, size=-1):
|
||
|
|
if size < 0:
|
||
|
|
s = self._buffer
|
||
|
|
self._buffer = b""
|
||
|
|
return s
|
||
|
|
else:
|
||
|
|
s = self._buffer[:size]
|
||
|
|
self._buffer = self._buffer[size:]
|
||
|
|
return s
|
||
|
|
|
||
|
|
|
||
|
|
class TestCodec:
|
||
|
|
def test_detectencoding_str(self):
|
||
|
|
"codec.detectencoding_str()"
|
||
|
|
assert codec.detectencoding_str(b'') == (None, False)
|
||
|
|
assert codec.detectencoding_str(b'\xef') == (None, False)
|
||
|
|
assert codec.detectencoding_str('\xef\x33'.encode()) == ("utf-8", False)
|
||
|
|
assert codec.detectencoding_str('\xc3\xaf3'.encode()) == ("utf-8", False)
|
||
|
|
assert codec.detectencoding_str(b'\xef\xbb') == (None, False)
|
||
|
|
assert codec.detectencoding_str('\xef\xbb\x33'.encode()) == (
|
||
|
|
"utf-8",
|
||
|
|
False,
|
||
|
|
)
|
||
|
|
assert codec.detectencoding_str('\xef\xbb\xbf'.encode("utf-8-sig")) == (
|
||
|
|
"utf-8-sig",
|
||
|
|
True,
|
||
|
|
)
|
||
|
|
assert codec.detectencoding_str(b'\xff') == (None, False)
|
||
|
|
assert codec.detectencoding_str('\xff\x33'.encode()) == ("utf-8", False)
|
||
|
|
assert codec.detectencoding_str(b'\xff\xfe') == (None, False)
|
||
|
|
assert codec.detectencoding_str('\xff\xfe\x33'.encode("utf-16")) == (
|
||
|
|
"utf-16",
|
||
|
|
True,
|
||
|
|
)
|
||
|
|
assert codec.detectencoding_str(b'\xff\xfe\x00') == (
|
||
|
|
None,
|
||
|
|
False,
|
||
|
|
)
|
||
|
|
assert codec.detectencoding_str('\xff\xfe\x00\x33'.encode("utf-16")) == (
|
||
|
|
"utf-16",
|
||
|
|
True,
|
||
|
|
)
|
||
|
|
assert codec.detectencoding_str('\xff\xfe\x00\x00'.encode("utf-32")) == (
|
||
|
|
"utf-32",
|
||
|
|
True,
|
||
|
|
)
|
||
|
|
assert codec.detectencoding_str('\x00'.encode()) == (None, False)
|
||
|
|
assert codec.detectencoding_str('\x00\x33'.encode()) == ("utf-8", False)
|
||
|
|
assert codec.detectencoding_str('\x00\x00'.encode()) == (None, False)
|
||
|
|
assert codec.detectencoding_str('\x00\x00\x33'.encode()) == ("utf-8", False)
|
||
|
|
assert codec.detectencoding_str(b'\x00\x00\xfe') == (
|
||
|
|
None,
|
||
|
|
False,
|
||
|
|
)
|
||
|
|
assert codec.detectencoding_str('\x00\x00\x00\x33'.encode()) == ("utf-8", False)
|
||
|
|
assert codec.detectencoding_str('\x00\x00\x00@'.encode()) == (
|
||
|
|
"utf-32-be",
|
||
|
|
False,
|
||
|
|
)
|
||
|
|
assert codec.detectencoding_str('\x00\x00\xfe\xff'.encode('utf-32')) == (
|
||
|
|
"utf-32",
|
||
|
|
True,
|
||
|
|
)
|
||
|
|
assert codec.detectencoding_str(b'@') == (None, False)
|
||
|
|
assert codec.detectencoding_str('@\x33'.encode()) == ("utf-8", False)
|
||
|
|
assert codec.detectencoding_str('@\x00'.encode()) == (None, False)
|
||
|
|
assert codec.detectencoding_str('@\x00\x33'.encode()) == ("utf-8", False)
|
||
|
|
assert codec.detectencoding_str('@\x00\x00'.encode()) == (None, False)
|
||
|
|
assert codec.detectencoding_str('@\x00\x00\x33'.encode()) == ("utf-8", False)
|
||
|
|
assert codec.detectencoding_str('@\x00\x00\x00'.encode()) == (
|
||
|
|
"utf-32-le",
|
||
|
|
False,
|
||
|
|
)
|
||
|
|
assert codec.detectencoding_str(b'@c') == (None, False)
|
||
|
|
assert codec.detectencoding_str(b'@ch') == (None, False)
|
||
|
|
assert codec.detectencoding_str(b'@cha') == (None, False)
|
||
|
|
assert codec.detectencoding_str(b'@char') == (None, False)
|
||
|
|
assert codec.detectencoding_str(b'@chars') == (None, False)
|
||
|
|
assert codec.detectencoding_str(b'@charse') == (None, False)
|
||
|
|
assert codec.detectencoding_str(b'@charset') == (None, False)
|
||
|
|
assert codec.detectencoding_str(b'@charset ') == (None, False)
|
||
|
|
assert codec.detectencoding_str(b'@charset "') == (None, False)
|
||
|
|
assert codec.detectencoding_str(b'@charset "x') == (None, False)
|
||
|
|
assert codec.detectencoding_str(b'@charset ""') == ("", True)
|
||
|
|
assert codec.detectencoding_str(b'@charset "x"') == ("x", True)
|
||
|
|
assert codec.detectencoding_str(b"@", False) == (None, False)
|
||
|
|
assert codec.detectencoding_str(b"@", True) == ("utf-8", False)
|
||
|
|
assert codec.detectencoding_str(b"@c", False) == (None, False)
|
||
|
|
assert codec.detectencoding_str(b"@c", True) == ("utf-8", False)
|
||
|
|
|
||
|
|
def test_detectencoding_unicode(self):
|
||
|
|
"codec.detectencoding_unicode()"
|
||
|
|
# Unicode version (only parses the header)
|
||
|
|
assert codec.detectencoding_unicode('@charset "x') == (None, False)
|
||
|
|
assert codec.detectencoding_unicode('a {}') == ("utf-8", False)
|
||
|
|
assert codec.detectencoding_unicode('@charset "x', True) == (None, False)
|
||
|
|
assert codec.detectencoding_unicode('@charset "x"') == ("x", True)
|
||
|
|
|
||
|
|
def test_fixencoding(self):
|
||
|
|
"codec._fixencoding()"
|
||
|
|
s = '@charset "'
|
||
|
|
assert codec._fixencoding(s, "utf-8") is None
|
||
|
|
|
||
|
|
s = '@charset "x'
|
||
|
|
assert codec._fixencoding(s, "utf-8") is None
|
||
|
|
|
||
|
|
s = '@charset "x'
|
||
|
|
assert codec._fixencoding(s, "utf-8", True) == s
|
||
|
|
|
||
|
|
s = '@charset x'
|
||
|
|
assert codec._fixencoding(s, "utf-8") == s
|
||
|
|
|
||
|
|
s = '@charset "x"'
|
||
|
|
assert codec._fixencoding(s, "utf-8") == s.replace('"x"', '"utf-8"')
|
||
|
|
|
||
|
|
def test_decoder(self): # noqa: C901
|
||
|
|
"codecs.decoder"
|
||
|
|
|
||
|
|
def checkauto(encoding, input='@charset "x";g\xfcrk\u20ac{}'):
|
||
|
|
outputencoding = encoding
|
||
|
|
if outputencoding == "utf-8-sig":
|
||
|
|
outputencoding = "utf-8"
|
||
|
|
# Check stateless decoder with encoding autodetection
|
||
|
|
d = codecs.getdecoder("css")
|
||
|
|
assert d(input.encode(encoding))[0] == input.replace(
|
||
|
|
'"x"', '"%s"' % outputencoding
|
||
|
|
)
|
||
|
|
|
||
|
|
# Check stateless decoder with specified encoding
|
||
|
|
assert d(input.encode(encoding), encoding=encoding)[0] == input.replace(
|
||
|
|
'"x"', '"%s"' % outputencoding
|
||
|
|
)
|
||
|
|
|
||
|
|
if hasattr(codec, "getincrementaldecoder"):
|
||
|
|
# Check incremental decoder with encoding autodetection
|
||
|
|
id = codecs.getincrementaldecoder("css")()
|
||
|
|
assert "".join(id.iterdecode(input.encode(encoding))) == input.replace(
|
||
|
|
'"x"', '"%s"' % outputencoding
|
||
|
|
)
|
||
|
|
|
||
|
|
# Check incremental decoder with specified encoding
|
||
|
|
id = codecs.getincrementaldecoder("css")(encoding=encoding)
|
||
|
|
assert "".join(id.iterdecode(input.encode(encoding))) == input.replace(
|
||
|
|
'"x"', '"%s"' % outputencoding
|
||
|
|
)
|
||
|
|
|
||
|
|
# Check stream reader with encoding autodetection
|
||
|
|
q = Queue()
|
||
|
|
sr = codecs.getreader("css")(q)
|
||
|
|
result = []
|
||
|
|
# TODO: py3 only???
|
||
|
|
for c in input.encode(encoding):
|
||
|
|
q.write(c)
|
||
|
|
result.append(sr.read())
|
||
|
|
assert "".join(result) == input.replace('"x"', '"%s"' % outputencoding)
|
||
|
|
|
||
|
|
# Check stream reader with specified encoding
|
||
|
|
q = Queue()
|
||
|
|
sr = codecs.getreader("css")(q, encoding=encoding)
|
||
|
|
result = []
|
||
|
|
for c in input.encode(encoding):
|
||
|
|
q.write(c)
|
||
|
|
result.append(sr.read())
|
||
|
|
assert "".join(result) == input.replace('"x"', '"%s"' % outputencoding)
|
||
|
|
|
||
|
|
# Autodetectable encodings
|
||
|
|
checkauto("utf-8-sig")
|
||
|
|
checkauto("utf-16")
|
||
|
|
checkauto("utf-16-le")
|
||
|
|
checkauto("utf-16-be")
|
||
|
|
checkauto("utf-32")
|
||
|
|
checkauto("utf-32-le")
|
||
|
|
checkauto("utf-32-be")
|
||
|
|
|
||
|
|
def checkdecl(encoding, input='@charset "%s";g\xfcrk{}'):
|
||
|
|
# Check stateless decoder with encoding autodetection
|
||
|
|
d = codecs.getdecoder("css")
|
||
|
|
input = input % encoding
|
||
|
|
outputencoding = encoding
|
||
|
|
if outputencoding == "utf-8-sig":
|
||
|
|
outputencoding = "utf-8"
|
||
|
|
assert d(input.encode(encoding))[0] == input
|
||
|
|
|
||
|
|
# Check stateless decoder with specified encoding
|
||
|
|
assert d(input.encode(encoding), encoding=encoding)[0] == input
|
||
|
|
|
||
|
|
if hasattr(codec, "getincrementaldecoder"):
|
||
|
|
# Check incremental decoder with encoding autodetection
|
||
|
|
id = codecs.getincrementaldecoder("css")()
|
||
|
|
assert "".join(id.iterdecode(input.encode(encoding))) == input
|
||
|
|
|
||
|
|
# Check incremental decoder with specified encoding
|
||
|
|
id = codecs.getincrementaldecoder("css")(encoding)
|
||
|
|
assert "".join(id.iterdecode(input.encode(encoding))) == input
|
||
|
|
|
||
|
|
# Check stream reader with encoding autodetection
|
||
|
|
q = Queue()
|
||
|
|
sr = codecs.getreader("css")(q)
|
||
|
|
result = []
|
||
|
|
for c in input.encode(encoding):
|
||
|
|
q.write(c)
|
||
|
|
result.append(sr.read())
|
||
|
|
assert "".join(result) == input
|
||
|
|
|
||
|
|
# Check stream reader with specified encoding
|
||
|
|
q = Queue()
|
||
|
|
sr = codecs.getreader("css")(q, encoding=encoding)
|
||
|
|
result = []
|
||
|
|
for c in input.encode(encoding):
|
||
|
|
q.write(c)
|
||
|
|
result.append(sr.read())
|
||
|
|
assert "".join(result) == input
|
||
|
|
|
||
|
|
# Use correct declaration
|
||
|
|
checkdecl("utf-8")
|
||
|
|
checkdecl("iso-8859-1", '@charset "%s";g\xfcrk')
|
||
|
|
checkdecl("iso-8859-15")
|
||
|
|
checkdecl("cp1252")
|
||
|
|
|
||
|
|
# No recursion
|
||
|
|
with pytest.raises(ValueError):
|
||
|
|
b'@charset "css";div{}'.decode("css")
|
||
|
|
|
||
|
|
def test_encoder(self):
|
||
|
|
"codec.encoder"
|
||
|
|
|
||
|
|
def check(encoding, input='@charset "x";g\xfcrk\u20ac{}'):
|
||
|
|
outputencoding = encoding
|
||
|
|
if outputencoding == "utf-8-sig":
|
||
|
|
outputencoding = "utf-8"
|
||
|
|
|
||
|
|
# Check stateless encoder with encoding autodetection
|
||
|
|
e = codecs.getencoder("css")
|
||
|
|
inputdecl = input.replace('"x"', '"%s"' % encoding)
|
||
|
|
outputdecl = input.replace('"x"', '"%s"' % outputencoding)
|
||
|
|
assert e(inputdecl)[0].decode(encoding) == outputdecl
|
||
|
|
|
||
|
|
# Check stateless encoder with specified encoding
|
||
|
|
assert e(input, encoding=encoding)[0].decode(encoding) == outputdecl
|
||
|
|
|
||
|
|
if hasattr(codec, "getincrementalencoder"):
|
||
|
|
# Check incremental encoder with encoding autodetection
|
||
|
|
ie = codecs.getincrementalencoder("css")()
|
||
|
|
assert "".join(ie.iterencode(inputdecl)).decode(encoding) == outputdecl
|
||
|
|
|
||
|
|
# Check incremental encoder with specified encoding
|
||
|
|
ie = codecs.getincrementalencoder("css")(encoding=encoding)
|
||
|
|
assert "".join(ie.iterencode(input)).decode(encoding) == outputdecl
|
||
|
|
|
||
|
|
# Check stream writer with encoding autodetection
|
||
|
|
q = Queue()
|
||
|
|
sw = codecs.getwriter("css")(q)
|
||
|
|
for c in inputdecl: # .encode(outputencoding): # TODO: .encode()???
|
||
|
|
sw.write(c)
|
||
|
|
assert q.read().decode(encoding) == input.replace(
|
||
|
|
'"x"', '"%s"' % outputencoding
|
||
|
|
)
|
||
|
|
|
||
|
|
# Check stream writer with specified encoding
|
||
|
|
q = Queue()
|
||
|
|
sw = codecs.getwriter("css")(q, encoding=encoding)
|
||
|
|
for c in input:
|
||
|
|
sw.write(c)
|
||
|
|
assert q.read().decode(encoding) == input.replace(
|
||
|
|
'"x"', '"%s"' % outputencoding
|
||
|
|
)
|
||
|
|
|
||
|
|
# Autodetectable encodings
|
||
|
|
check("utf-8-sig")
|
||
|
|
check("utf-16")
|
||
|
|
check("utf-16-le")
|
||
|
|
check("utf-16-be")
|
||
|
|
check("utf-32")
|
||
|
|
check("utf-32-le")
|
||
|
|
check("utf-32-be")
|
||
|
|
check("utf-8")
|
||
|
|
check("iso-8859-1", '@charset "x";g\xfcrk{}')
|
||
|
|
check("iso-8859-15")
|
||
|
|
check("cp1252")
|
||
|
|
|
||
|
|
# No recursion
|
||
|
|
with pytest.raises(ValueError):
|
||
|
|
'@charset "css";div{}'.encode("css")
|
||
|
|
|
||
|
|
def test_decode_force(self):
|
||
|
|
"codec.decode (force)"
|
||
|
|
info = codecs.lookup("css")
|
||
|
|
|
||
|
|
def decodeall(input, **kwargs):
|
||
|
|
# Py 2.5: info.decode('@charset "utf-8"; x')
|
||
|
|
return info[1](input, **kwargs)[0]
|
||
|
|
|
||
|
|
def incdecode(input, **kwargs):
|
||
|
|
decoder = info.incrementaldecoder(**kwargs)
|
||
|
|
return decoder.decode(input)
|
||
|
|
|
||
|
|
def streamdecode(input, **kwargs):
|
||
|
|
stream = iostream(input) # py3 .decode('utf-8') but still error?!
|
||
|
|
reader = info.streamreader(stream, **kwargs)
|
||
|
|
return reader.read()
|
||
|
|
|
||
|
|
for d in (decodeall, incdecode, streamdecode):
|
||
|
|
# input = '@charset "utf-8"; \xc3\xbf'
|
||
|
|
# output = u'@charset "utf-8"; \xff'
|
||
|
|
# self.assertEqual(d(input), output)
|
||
|
|
#
|
||
|
|
# input = '@charset "utf-8"; \xc3\xbf'
|
||
|
|
# output = u'@charset "iso-8859-1"; \xc3\xbf'
|
||
|
|
# self.assertEqual(d(input, encoding="iso-8859-1", force=True), output)
|
||
|
|
#
|
||
|
|
# input = '\xc3\xbf'
|
||
|
|
# output = u'\xc3\xbf'
|
||
|
|
# self.assertEqual(d(input, encoding="iso-8859-1", force=True), output)
|
||
|
|
#
|
||
|
|
# input = '@charset "utf-8"; \xc3\xbf'
|
||
|
|
# output = u'@charset "utf-8"; \xff'
|
||
|
|
# self.assertEqual(d(input, encoding="iso-8859-1", force=False), output)
|
||
|
|
|
||
|
|
input = '@charset "utf-8"; \xff'.encode()
|
||
|
|
output = '@charset "utf-8"; \xff'
|
||
|
|
assert d(input) == output
|
||
|
|
|
||
|
|
# input = b'@charset "utf-8"; \xc3\xbf'
|
||
|
|
input = '@charset "utf-8"; \xff'.encode()
|
||
|
|
output = '@charset "iso-8859-1"; \xc3\xbf'
|
||
|
|
assert d(input, encoding="iso-8859-1", force=True) == output
|
||
|
|
|
||
|
|
# input = b'\xc3\xbf'
|
||
|
|
input = '\xff'.encode()
|
||
|
|
output = '\xc3\xbf'
|
||
|
|
assert d(input, encoding="iso-8859-1", force=True) == output
|
||
|
|
|
||
|
|
# input = b'@charset "utf-8"; \xc3\xbf'
|
||
|
|
input = '@charset "utf-8"; \xff'.encode()
|
||
|
|
output = '@charset "utf-8"; \xff'
|
||
|
|
assert d(input, encoding="iso-8859-1", force=False) == output
|