Puremagic

Makefile

@@ -1,4 +1,5 @@
 PYTHON = python3
+ARGS =
 
 # In not in a virtualenv, add --user options for install commands.
 INSTALL_OPTS = `$(PYTHON) -c "import sys; print('' if hasattr(sys, 'real_prefix') else '--user')"`
@@ -21,9 +22,13 @@ SYSDEPS = \
 TEST_PREFIX = PYTHONWARNINGS=all FULLTEXT_TESTING=1
 
 test:  ## Run tests.
-	${MAKE} install-git-hooks
+	${MAKE} install
 	$(TEST_PREFIX) $(PYTHON) fulltext/test/__init__.py
 
+test-by-name:  ## Run a specific test by name.
+	${MAKE} install
+	$(TEST_PREFIX) $(PYTHON) -m unittest -v $(ARGS)
+
 ci:  ## Run CI tests.
 	${MAKE} sysdeps
 	${MAKE} pydeps

appveyor.yml

@@ -2,13 +2,9 @@ build: false
 
 environment:
   matrix:
-    - PYTHON: "C:\\Python35-x64"
-      PYTHON_VERSION: "3.5.x"
-      PYTHON_ARCH: "64"
-      ARCH: x86_64
-      VS_VER: "2015"
-      INSTANCENAME: "SQL2012SP1"
-
+    - PYTHON: "C:\\Python36"
+      PYTHON_VERSION: "3.6.x"
+      PYTHON_ARCH: "32"
 init:
   - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%"
 

fulltext/__init__.py

@@ -4,18 +4,21 @@
 import re
 import logging
 import os
-import mimetypes
 import sys
 
 from os.path import splitext
 
 from six import string_types
 from six import PY3
-from fulltext.util import warn
-from fulltext.util import magic
-from fulltext.util import is_file_path
+import fulltext.mimewrap
+from fulltext.magicwrap import magic
+from fulltext.mimewrap import ext_to_mimetype
+from fulltext.mimewrap import mimetype_to_backend
 from fulltext.util import fobj_to_tempfile
+from fulltext.util import is_file_path
 from fulltext.util import is_windows
+from fulltext.util import warn
+
 
 __all__ = ["get", "register_backend"]
 
@@ -32,77 +35,8 @@
 LOGGER.addHandler(logging.NullHandler())
 STRIP_WHITE = re.compile(r'[ \t\v\f\r\n]+')
 SENTINAL = object()
-MIMETYPE_TO_BACKENDS = {}
-EXTS_TO_MIMETYPES = {}
 MAGIC_BUFFER_SIZE = 1024
-
-mimetypes.init()
-_MIMETYPES_TO_EXT = dict([(v, k) for k, v in mimetypes.types_map.items()])
-
-# A list of extensions which will be treated as pure text.
-# This takes precedence over register_backend().
-# https://www.openoffice.org/dev_docs/source/file_extensions.html
-_TEXT_EXTS = set((
-    ".asm",  # Non-UNIX assembler source file
-    ".asp",  # Active Server Page
-    ".awk",  # An awk script file
-    ".bat",  # MS-DOS batch file
-    ".c",    # C language file
-    ".class",  # Compiled java source code file
-    ".cmd",  # Compiler command file
-    ".cpp",  # C++ language file
-    ".cxx",  # C++ language file
-    ".def",  # Win32 library definition file
-    ".dpc",  # Source dependency file containing list of dependencies
-    ".dpj",  # Java source dependency file containing list of dependencies
-    ".h",    # C header file
-    ".hpp",  # Generated C++ header or header plus plus file
-    ".hrc",  # An ".src",  # include header file
-    ".hxx",  # C++ header file
-    ".in",
-    ".inc",  # Include file
-    ".ini",  # Initialization file
-    ".inl",  # Inline header file
-    ".jar",  # Java classes archive file
-    ".java",  # Java language file
-    ".js",   # JavaScript code file
-    ".jsp",  # Java Server Page file
-    ".kdelnk",  # KDE1 configuration file
-    ".l",    # Lex source code file
-    ".ll",   # Lex source code file
-    ".lnx",  # Linux-specific makefile
-    ".log",  # Log file
-    ".lst",  # ASCII database file used in solenv
-    ".MacOS",
-    ".md",   # Markdown language.
-    ".mk",   # A dmake makefile
-    ".mod",  # BASIC module file
-    ".par",  # Script particles file
-    ".pl",   # Perl script
-    ".plc",  # Former build script file, now obsolete
-    ".pld",  # Former build script file, now obsolete
-    ".pm",   # Perl module file
-    ".pmk",  # Project makefiles
-    ".pre",  # Preprocessor output from scpcomp
-    ".py",   # Python
-    ".pyx",  # Cython
-    ".r",    # Resource file for Macintosh
-    ".rc",   # A dmake recursive makefile or a Win32 resource script file
-    ".rdb",  # Interface and type description database (type library)
-    ".res",  # Resource file
-    ".rst",  # Restructured text
-    ".s",    # Assembler source file (UNIX)
-    ".sbl",  # BASIC file
-    ".scp",  # Script source file
-    ".sh",   # Shell script
-    ".src",  # Source resource string file
-    ".txt",  # Language text file
-    ".y",    # Yacc source code file
-    ".yaml",  # Yaml
-    ".yml",  # Yaml
-    ".yxx",  # Bison source code file
-))
-
+register_backend = fulltext.mimewrap.register_backend
 
 # XXX: dirty hack for pyinstaller so that it includes these modules.
 # TODO: find a way to do this in pyinstaller.spec instead.
@@ -134,193 +68,6 @@
     from fulltext.backends import __zip  # NOQA
 
 
-# =====================================================================
-# --- backends
-# =====================================================================
-
-
-def register_backend(mimetype, module, extensions=None):
-    """Register a backend.
-    `mimetype`: a mimetype string (e.g. 'text/plain')
-    `module`: an import string (e.g. path.to.my.module)
-    `extensions`: a list of extensions (e.g. ['txt', 'text'])
-    """
-    if mimetype in MIMETYPE_TO_BACKENDS:
-        warn("overwriting %r mimetype which was already set" % mimetype)
-    MIMETYPE_TO_BACKENDS[mimetype] = module
-    if extensions is None:
-        try:
-            ext = _MIMETYPES_TO_EXT[mimetype]
-        except KeyError:
-            raise KeyError(
-                "mimetypes module has no extension associated "
-                "with %r mimetype; use 'extensions' arg yourself" % mimetype)
-        assert ext, ext
-        EXTS_TO_MIMETYPES[ext] = mimetype
-    else:
-        if not isinstance(extensions, (list, tuple, set, frozenset)):
-            raise TypeError("invalid extensions type (got %r)" % extensions)
-        for ext in set(extensions):
-            ext = ext if ext.startswith('.') else '.' + ext
-            assert ext, ext
-            EXTS_TO_MIMETYPES[ext] = mimetype
-
-
-register_backend(
-    'application/zip',
-    'fulltext.backends.__zip',
-    extensions=[".zip"])
-
-register_backend(
-    'application/x-rar-compressed',
-    'fulltext.backends.__rar',
-    extensions=['.rar'])
-
-for mt in ("text/xml", "application/xml", "application/x-xml"):
-    register_backend(
-        mt,
-        'fulltext.backends.__xml',
-        extensions=[".xml", ".xsd"])
-
-register_backend(
-    'application/vnd.ms-excel',
-    'fulltext.backends.__xlsx',
-    extensions=['.xls', '.xlsx'])
-
-register_backend(
-    'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
-    'fulltext.backends.__xlsx',
-    extensions=['.xlsx'])
-
-register_backend(
-    'text/plain',
-    'fulltext.backends.__text',
-    extensions=['.txt', '.text'])
-
-register_backend(
-    'application/rtf',
-    'fulltext.backends.__rtf',
-    extensions=['.rtf'])
-
-register_backend(
-    'application/vnd.openxmlformats-officedocument.presentationml.presentation',  # NOQA
-    'fulltext.backends.__pptx',
-    extensions=['.pptx'])
-
-register_backend(
-    'application/pdf',
-    'fulltext.backends.__pdf',
-    extensions=['.pdf'])
-
-register_backend(
-    'application/vnd.oasis.opendocument.text',
-    'fulltext.backends.__odt',
-    extensions=['.odt'])
-
-register_backend(
-    'application/vnd.oasis.opendocument.spreadsheet',
-    'fulltext.backends.__odt',
-    extensions=['.ods'])
-
-# images
-register_backend(
-    'image/jpeg',
-    'fulltext.backends.__ocr',
-    extensions=['.jpg', '.jpeg'])
-
-register_backend(
-    'image/bmp',
-    'fulltext.backends.__ocr',
-    extensions=['.bmp'])
-
-register_backend(
-    'image/png',
-    'fulltext.backends.__ocr',
-    extensions=['.png'])
-
-register_backend(
-    'image/gif',
-    'fulltext.backends.__ocr',
-    extensions=['.gif'])
-
-register_backend(
-    'application/x-hwp',
-    'fulltext.backends.__hwp',
-    extensions=['.hwp'])
-
-for mt in ('text/html', 'application/html', 'text/xhtml'):
-    register_backend(
-        mt,
-        'fulltext.backends.__html',
-        extensions=['.htm', '.html', '.xhtml'])
-
-register_backend(
-    'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
-    'fulltext.backends.__docx',
-    extensions=['.docx'])
-
-register_backend(
-    'application/msword',
-    'fulltext.backends.__doc',
-    extensions=['.doc'])
-
-for mt in ('text/csv', 'text/tsv', 'text/psv'):
-    register_backend(
-        mt,
-        'fulltext.backends.__csv',
-        extensions=['.csv', '.tsv', '.psv', '.tab'])
-
-for mt in ("application/epub", "application/epub+zip"):
-    register_backend(
-        mt,
-        'fulltext.backends.__epub',
-        extensions=[".epub"])
-
-register_backend(
-    'application/postscript',
-    'fulltext.backends.__ps',
-    extensions=[".ps", ".eps", ".ai"])
-
-register_backend(
-    'message/rfc822',
-    'fulltext.backends.__eml',
-    extensions=['.eml'])
-
-register_backend(
-    'application/mbox',
-    'fulltext.backends.__mbox',
-    extensions=['.mbox'])
-
-register_backend(
-    'application/vnd.ms-outlook',
-    'fulltext.backends.__msg',
-    extensions=['.msg'])
-
-register_backend(
-    'application/gzip',
-    'fulltext.backends.__gz',
-    extensions=['.gz'])
-
-register_backend(
-    'application/json',
-    'fulltext.backends.__json',
-    extensions=['.json'])
-
-# default backend.
-register_backend(
-    'application/octet-stream',
-    'fulltext.backends.__bin',
-    extensions=['.a', '.bin'])
-
-# Extensions which will be treated as pure text.
-# We just come up with a custom mime name.
-for ext in _TEXT_EXTS:
-    register_backend(
-        '[custom-fulltext-mime]/%s' % ext,
-        'fulltext.backends.__text',
-        extensions=[ext])
-
-
 # =====================================================================
 # --- utils
 # =====================================================================
@@ -428,7 +175,7 @@ def import_mod(mod_name):
 def backend_from_mime(mime):
     """Determine backend module object from a mime string."""
     try:
-        mod_name = MIMETYPE_TO_BACKENDS[mime]
+        mod_name = mimetype_to_backend(mime)
 
     except KeyError:
         msg = "No handler for %r, defaulting to %r" % (mime, DEFAULT_MIME)
@@ -437,7 +184,7 @@ def backend_from_mime(mime):
         else:
             LOGGER.debug(msg)
 
-        mod_name = MIMETYPE_TO_BACKENDS[DEFAULT_MIME]
+        mod_name = mimetype_to_backend(DEFAULT_MIME)
     mod = import_mod(mod_name)
     return mod
 
@@ -446,10 +193,8 @@ def backend_from_fname(name):
     """Determine backend module object from a file name."""
     ext = splitext(name)[1]
 
-    try:
-        mime = EXTS_TO_MIMETYPES[ext]
-
-    except KeyError:
+    mime = ext_to_mimetype(ext)
+    if mime is None:
         try:
             f = open(name, 'rb')
 
@@ -466,14 +211,14 @@ def backend_from_fname(name):
             else:
                 LOGGER.debug(msg)
 
-            mod_name = MIMETYPE_TO_BACKENDS[DEFAULT_MIME]
+            mod_name = mimetype_to_backend(DEFAULT_MIME)
 
         else:
             with f:
                 return backend_from_fobj(f)
 
     else:
-        mod_name = MIMETYPE_TO_BACKENDS[mime]
+        mod_name = mimetype_to_backend(mime)
 
     mod = import_mod(mod_name)
     return mod
@@ -550,9 +295,8 @@ def _get(path_or_file, default, mime, name, backend, encoding,
                     backend_mod = backend_from_fobj(path_or_file)
     else:
         if isinstance(backend, string_types):
-            try:
-                mime = EXTS_TO_MIMETYPES['.' + backend]
-            except KeyError:
+            mime = ext_to_mimetype('.' + backend)
+            if mime is None:
                 raise ValueError("invalid backend %r" % backend)
             backend_mod = backend_from_mime(mime)
         else: