diff --git a/README.md b/README.md index afe217a..b280e5d 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,8 @@ scene = pywavefront.Wavefront('something.obj') A more complex example * `strict` (Default: `False`) will raise an exception if unsupported features are found in the obj or mtl file -* `encoding` (Default: `utf-8`) of the obj and mtl file(s) +* `encoding` (Default: `utf-8`) of the obj and mtl file(s). + UTF-8 files with or without a byte-order mark are accepted automatically. * `create_materials` (Default: `False`) will create materials if mtl file is missing or obj file references non-existing materials * `collect_faces` (Default: `False`) will collect triangle face data for every mesh. In case faces with more than three vertices are specified they will be triangulated. See the documentation of `ObjParser#consume_faces()` in [`obj.py`](https://github.com/pywavefront/PyWavefront/blob/master/pywavefront/obj.py). * `parse` (Default: `True`) decides if parsing should start immediately. diff --git a/pywavefront/parser.py b/pywavefront/parser.py index 0614558..e0bbf54 100644 --- a/pywavefront/parser.py +++ b/pywavefront/parser.py @@ -77,9 +77,14 @@ def create_line_generator(self): Should only yield non-empty lines """ + encoding = self.encoding + if encoding and codecs.lookup(encoding).name == 'utf-8': + # Consume a UTF-8 signature at the start, preserving U+FEFF in content. + encoding = 'utf-8-sig' + if self.file_name.suffix == ".gz": # FIXME: Converting to str for now for py34 compatibility - gz = gzip.open(str(self.file_name), mode='rt', encoding=self.encoding) + gz = gzip.open(str(self.file_name), mode='rt', encoding=encoding) for line in gz.readlines(): yield line @@ -87,7 +92,7 @@ def create_line_generator(self): gz.close() else: # FIXME: Converting to str for now for py34 compatibility - file = open(str(self.file_name), mode='r', encoding=self.encoding) + file = open(str(self.file_name), mode='r', encoding=encoding) for line in file: yield line diff --git a/tests/test_encoding.py b/tests/test_encoding.py new file mode 100644 index 0000000..e810aba --- /dev/null +++ b/tests/test_encoding.py @@ -0,0 +1,56 @@ +import codecs +import gzip + +import pytest + +import pywavefront + + +def write_source(path, text, compressed, bom=False, encoding='utf-8'): + data = text.encode(encoding) + if bom: + data = codecs.BOM_UTF8 + data + opener = gzip.open if compressed else open + with opener(str(path), 'wb') as stream: + stream.write(data) + + +@pytest.mark.parametrize( + 'obj_bom,mtl_bom', [(False, False), (True, False), (False, True), (True, True)]) +@pytest.mark.parametrize('compressed', [False, True]) +@pytest.mark.parametrize('encoding', [None, 'UTF8', 'utf-8-sig']) +@pytest.mark.parametrize('strict', [False, True]) +def test_utf8_bom(tmp_path, obj_bom, mtl_bom, compressed, encoding, strict): + suffix = '.gz' if compressed else '' + obj_path = tmp_path / ('triangle.obj' + suffix) + mtl_path = tmp_path / ('triangle.mtl' + suffix) + # U+FEFF inside a material name is data, not a file signature. + name = 'caf\u00e9\ufeffpaint' + obj = ('v 0 0 0\nv 1 0 0\nv 0 1 0\nmtllib {0}\n' + 'usemtl {1}\nf 1 2 3\n').format(mtl_path.name, name) + mtl = 'newmtl {0}\nKd 0.25 0.5 0.75\n'.format(name) + write_source(obj_path, obj, compressed, obj_bom) + write_source(mtl_path, mtl, compressed, mtl_bom) + kwargs = {'strict': strict, 'collect_faces': True} + if encoding is not None: + kwargs['encoding'] = encoding + scene = pywavefront.Wavefront(str(obj_path), **kwargs) + assert scene.vertices == [(0., 0., 0.), (1., 0., 0.), (0., 1., 0.)] + assert scene.mesh_list[0].faces == [[0, 1, 2]] + material = scene.materials[name] + assert scene.mesh_list[0].materials == [material] + assert material.diffuse[:3] == [0.25, 0.5, 0.75] + assert material.vertices == [0., 0., 0., 1., 0., 0., 0., 1., 0.] + + +@pytest.mark.parametrize('encoding', ['utf-16', 'iso-8859-1']) +@pytest.mark.parametrize('compressed', [False, True]) +def test_other_encodings(tmp_path, encoding, compressed): + suffix = '.gz' if compressed else '' + path = tmp_path / ('triangle.obj' + suffix) + obj = 'v 0 0 0\nv 1 0 0\nv 0 1 0\no caf\u00e9\nf 1 2 3\n' + write_source(path, obj, compressed, encoding=encoding) + scene = pywavefront.Wavefront(str(path), strict=True, encoding=encoding, + collect_faces=True) + assert scene.meshes['caf\u00e9'].faces == [[0, 1, 2]] + assert len(scene.vertices) == 3