Merge pull request #106 from defanator/allow-parse-non-unicode-configs

Allow parsing non-Unicode nginx configurations

Author: defanator

crossplane/lexer.py

@@ -145,7 +145,7 @@ def _balance_braces(tokens, filename=None):
 
 def lex(filename):
     """Generates tokens from an nginx config file"""
-    with io.open(filename, mode='r', encoding='utf-8') as f:
+    with io.open(filename, mode='r', encoding='utf-8', errors='replace') as f:
         it = _lex_file_object(f)
         it = _balance_braces(it, filename)
         for token, line, quoted in it:

tests/configs/non-unicode/nginx.conf

@@ -0,0 +1,8 @@
+http {
+    server {
+        location /city {
+            # Mölln
+            return 200 "Mölln\n";
+        }
+    }
+}

tests/test_parse.py

@@ -975,3 +975,60 @@ def test_comments_between_args():
             }
         ]
     }
+
+def test_non_unicode():
+    dirname = os.path.join(here, 'configs', 'non-unicode')
+    config = os.path.join(dirname, 'nginx.conf')
+
+    payload = crossplane.parse(config, comments=True)
+
+    assert payload == {
+        "errors": [],
+        "status": "ok",
+        "config": [
+            {
+                "status": "ok",
+                "errors": [],
+                "file": os.path.join(dirname, 'nginx.conf'),
+                "parsed": [
+                    {
+                        "directive": "http",
+                        "line": 1,
+                        "args": [],
+                        "block": [
+                            {
+                                "directive": "server",
+                                "line": 2,
+                                "args": [],
+                                "block": [
+                                    {
+                                        "directive": "location",
+                                        "line": 3,
+                                        "args": [
+                                            "/city"
+                                        ],
+                                        "block": [
+                                            {
+                                                "directive": "#",
+                                                "line": 4,
+                                                "args": [],
+                                                "comment": u" M\ufffdlln"
+                                            },
+                                            {
+                                                "directive": "return",
+                                                "line": 5,
+                                                "args": [
+                                                    "200",
+                                                    u"M\ufffdlln\\n"
+                                                ]
+                                            }
+                                        ]
+                                    }
+                                ]
+                            }
+                        ]
+                    }
+                ]
+            }
+        ]
+    }