WIP : Code refactoring (#77)

* 🔨 simplify the plugin interface. 🔥 no open function, 🔥 no more useless set_type, each plugin knows its type.

* This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst

* 🚜 csvr to csv_sheet

* 🚜 better naming for the readers

* 🤝 update gitignore

* 📚 update pyinstaller notes

* 🔥 remmove python 2 related compactibility codes

* 💄 update coding style

* 🔥 remove test files generated during tests

Co-authored-by: chfw <chfw@users.noreply.github.com>

Author: chfw

.gitignore

@@ -7,3 +7,4 @@ targets:
   - .travis.yml: custom_travis.yml.jj2
   - README.rst: io_readme.rst.jj2
   - "docs/source/index.rst": "docs/source/index.rst"
+  - .gitignore: gitignore.jj2

.moban.yml

@@ -4,8 +4,8 @@
 
 In alphabetical order:
 
-* `Craig Anderson <https://api.github.com/users/craiga>`_
-* `John Vandenberg <https://api.github.com/users/jayvdb>`_
-* `Stephen J. Fuhry <https://api.github.com/users/fuhrysteve>`_
-* `Stephen Rauch <https://api.github.com/users/stephenrauch>`_
-* `Víctor Antonio Hernández Monroy <https://api.github.com/users/antherkiv>`_
+* `Craig Anderson <https://github.com/craiga>`_
+* `John Vandenberg <https://github.com/jayvdb>`_
+* `Stephen J. Fuhry <https://github.com/fuhrysteve>`_
+* `Stephen Rauch <https://github.com/stephenrauch>`_
+* `Víctor Antonio Hernández Monroy <https://github.com/antherkiv>`_

CONTRIBUTORS.rst

@@ -1,11 +1,8 @@
 include README.rst
 include LICENSE
 include CHANGELOG.rst
+include CONTRIBUTORS.rst
 recursive-include tests *
-<<<<<<< HEAD
-=======
 recursive-include docs *
->>>>>>> master
-include docs/source/*
 include Makefile
 include test.sh

MANIFEST.in

@@ -74,6 +74,8 @@ Continue from previous example::
    :hide:
 
    >>> import os
+   >>> if os.path.exists("your_file.csv"):
+   ...     os.unlink("your_file.csv")
    >>> os.unlink("your_file__Sheet 1__0.csv")
    >>> os.unlink("your_file__Sheet 2__1.csv")
 

docs/source/extendedcsv.rst

@@ -60,3 +60,9 @@ Obvious, you could do both at the same time:
 
 The pagination support is available across all pyexcel-io plugins.
 
+.. testcode::
+   :hide:
+
+   >>> import os
+   >>> if os.path.exists("your_file.csv"):
+   ...     os.unlink("your_file.csv")

docs/source/pagination.rst

@@ -172,4 +172,5 @@ When you read it back you will have to specify encoding too.
 
    >>> import os
    >>> os.unlink("your_file.csv")
+   >>> os.unlink("test-utf16-encoding.csv")
    >>> os.unlink(test_file)

docs/source/plaincsv.rst

@@ -9,14 +9,16 @@ Built-in plugins for pyexcel-io
 
 In order to package every built-in plugins of pyexcel-io, you need to specify::
 
-    --hidden-import pyexcel_io.readers.csvr
+    --hidden-import pyexcel_io.readers.csv_in_file
+    --hidden-import pyexcel_io.readers.csv_in_memory
+    --hidden-import pyexcel_io.readers.csv_content
     --hidden-import pyexcel_io.readers.csvz
     --hidden-import pyexcel_io.readers.tsv
     --hidden-import pyexcel_io.readers.tsvz
-    --hidden-import pyexcel_io.writers.csv_file_writer
-    --hidden-import pyexcel_io.writers.csv_memory_writer
-    --hidden-import pyexcel_io.writers.tsv_file_writer
-    --hidden-import pyexcel_io.writers.tsv_memory_writer
+    --hidden-import pyexcel_io.writers.csv_in_file
+    --hidden-import pyexcel_io.writers.csv_in_memory
+    --hidden-import pyexcel_io.writers.tsv_in_file
+    --hidden-import pyexcel_io.writers.tsv_in_memory
     --hidden-import pyexcel_io.writers.csvz_writer
     --hidden-import pyexcel_io.writers.tsvz_writer
     --hidden-import pyexcel_io.database.importers.django

docs/source/pyinstaller.rst

@@ -46,3 +46,10 @@ And you may want use row_renderer to customize it to string:
    >>> data = get_data("your_file.csv", row_renderer=my_renderer)
    >>> data['your_file.csv']
    [['1', '21', '31'], ['2', '22', '32'], ['3', '23', '33']]
+
+.. testcode::
+   :hide:
+
+   >>> import os
+   >>> if os.path.exists("your_file.csv"):
+   ...     os.unlink("your_file.csv")

docs/source/renderer.rst

@@ -16,17 +16,7 @@
 import sys
 import types
 import logging
-
-PY2 = sys.version_info[0] == 2
-PY3_ABOVE = sys.version_info[0] >= 3
-PY26 = PY2 and sys.version_info[1] < 7
-PY27 = PY2 and sys.version_info[1] == 7
-PY27_ABOVE = PY27 or PY3_ABOVE
-
-if PY26:
-    from ordereddict import OrderedDict
-else:
-    from collections import OrderedDict
+from collections import OrderedDict
 
 try:
     from logging import NullHandler
@@ -37,24 +27,11 @@ def emit(self, record):
             pass
 
 
-if PY2:
-    from cStringIO import StringIO
-    from cStringIO import StringIO as BytesIO
-
-    text_type = unicode
-    irange = xrange
-
-    class Iterator(object):
-        def next(self):
-            return type(self).__next__(self)
-
-
-else:
-    from io import BytesIO, StringIO
+from io import BytesIO, StringIO
 
-    text_type = str
-    Iterator = object
-    irange = range
+text_type = str
+Iterator = object
+irange = range
 
 
 def isstream(instance):

pyexcel_io/_compact.py

@@ -176,8 +176,7 @@ def get_a_plugin(
     ):
         __file_type = file_type.lower()
         plugin = self.load_me_now(f"{location}-{__file_type}", library=library)
-        handler = plugin()
-        return handler
+        return plugin
 
     def raise_exception(self, file_type):
         file_type = file_type.split("-")[1]
@@ -224,14 +223,14 @@ def _do_additional_registration_for_new_plugins(plugin_info):
         )
 
 
-class FakeReaders:
+class AllReaders:
     def get_all_formats(self):
         return OLD_READERS.get_all_formats().union(
             NEW_READERS.get_all_formats()
         )
 
 
-class FakeWriters:
+class AllWriters:
     def get_all_formats(self):
         return OLD_WRITERS.get_all_formats().union(
             NEW_WRITERS.get_all_formats()
@@ -242,8 +241,8 @@ def get_all_formats(self):
 OLD_WRITERS = IOManager(WRITER_PLUGIN, ioutils.AVAILABLE_WRITERS)
 NEW_WRITERS = NewIOManager(NEW_WRITER_PLUGIN, ioutils.AVAILABLE_WRITERS)
 NEW_READERS = NewIOManager(NEW_READER_PLUGIN, ioutils.AVAILABLE_READERS)
-READERS = FakeReaders()
-WRITERS = FakeWriters()
+READERS = AllReaders()
+WRITERS = AllWriters()
 
 
 def load_plugins(plugin_name_patterns, path, black_list, white_list):

pyexcel_io/plugins.py

@@ -33,19 +33,21 @@ def __init__(self, file_type, library=None):
         self.keywords = None
 
     def open(self, file_name, **keywords):
-        self.reader = NEW_READERS.get_a_plugin(
+        reader_class = NEW_READERS.get_a_plugin(
             self.file_type, location="file", library=self.library
         )
         self.keywords, native_sheet_keywords = clean_keywords(keywords)
-        return self.reader.open(file_name, **native_sheet_keywords)
+        self.reader = reader_class(file_name, **native_sheet_keywords)
+        return self.reader
 
     def open_content(self, file_content, **keywords):
         self.keywords, native_sheet_keywords = clean_keywords(keywords)
         try:
-            self.reader = NEW_READERS.get_a_plugin(
+            reader_class = NEW_READERS.get_a_plugin(
                 self.file_type, location="content", library=self.library
             )
-            return self.reader.open(file_content, **native_sheet_keywords)
+            self.reader = reader_class(file_content, **native_sheet_keywords)
+            return self.reader
         except (
             exceptions.NoSupportingPluginFound,
             exceptions.SupportingPluginAvailableButNotInstalled,
@@ -57,10 +59,11 @@ def open_content(self, file_content, **keywords):
 
     def open_stream(self, file_stream, **keywords):
         self.keywords, native_sheet_keywords = clean_keywords(keywords)
-        self.reader = NEW_READERS.get_a_plugin(
+        reader_class = NEW_READERS.get_a_plugin(
             self.file_type, location="memory", library=self.library
         )
-        return self.reader.open(file_stream, **native_sheet_keywords)
+        self.reader = reader_class(file_stream, **native_sheet_keywords)
+        return self.reader
 
     def read_sheet_by_name(self, sheet_name):
         """

pyexcel_io/reader.py

@@ -10,17 +10,17 @@
 from pyexcel_io.plugins import NewIOPluginInfoChain
 
 NewIOPluginInfoChain(__name__).add_a_reader(
-    relative_plugin_class_path="csv_file_reader.FileReader",
+    relative_plugin_class_path="csv_in_file.FileReader",
     location="file",
     file_types=["csv"],
     stream_type="text",
 ).add_a_reader(
-    relative_plugin_class_path="csv_content_reader.ContentReader",
+    relative_plugin_class_path="csv_content.ContentReader",
     location="content",
     file_types=["csv"],
     stream_type="text",
 ).add_a_reader(
-    relative_plugin_class_path="csv_memory_reader.MemoryReader",
+    relative_plugin_class_path="csv_in_memory.MemoryReader",
     location="memory",
     file_types=["csv"],
     stream_type="text",

pyexcel_io/readers/__init__.py

@@ -0,0 +1,30 @@
+import mmap
+
+import pyexcel_io.constants as constants
+from pyexcel_io.book import _convert_content_to_stream
+from pyexcel_io.readers.csv_sheet import CSVMemoryMapIterator
+from pyexcel_io.readers.csv_in_memory import MemoryReader
+
+
+class ContentReader(MemoryReader):
+    file_type = constants.FILE_FORMAT_CSV
+
+    def __init__(self, file_content, **keywords):
+        file_stream = ContentReader.convert_content_to_stream(
+            file_content, self.file_type, **keywords
+        )
+        super().__init__(file_stream, **keywords)
+
+    @staticmethod
+    def convert_content_to_stream(file_content, file_type, **keywords):
+        encoding = keywords.get("encoding", "utf-8")
+        if isinstance(file_content, mmap.mmap):
+            # load from mmap
+            file_stream = CSVMemoryMapIterator(file_content, encoding)
+        else:
+            if isinstance(file_content, bytes):
+                file_content = file_content.decode(encoding)
+
+            file_stream = _convert_content_to_stream(file_content, file_type)
+
+        return file_stream

pyexcel_io/readers/csv_content.py

@@ -4,23 +4,18 @@
 
 from pyexcel_io import constants
 from pyexcel_io.sheet import NamedContent
-from pyexcel_io.readers.csvr import CSVFileReader
+from pyexcel_io.readers.csv_sheet import CSVFileReader
 
 DEFAULT_NEWLINE = "\r\n"
 
 
 class FileReader(object):
-    def __init__(self):
-        self.handles = []
-
-    def set_type(self, _):
-        pass
-
-    def open(self, file_name, **keywords):
+    def __init__(self, file_name, **keywords):
         """Load content from a file
         :params str filename: an accessible file path
         :returns: a book
         """
+        self.handles = []
         self.keywords = keywords
         self.__line_terminator = keywords.get(
             constants.KEYWORD_LINE_TERMINATOR, DEFAULT_NEWLINE

pyexcel_io/readers/csv_content_reader.py

@@ -3,24 +3,20 @@
 import pyexcel_io._compact as compact
 from pyexcel_io import constants
 from pyexcel_io.sheet import NamedContent
-from pyexcel_io.readers.csvr import CSVinMemoryReader
+from pyexcel_io.readers.csv_sheet import CSVinMemoryReader
 
 DEFAULT_SHEET_SEPARATOR_FORMATTER = f"---{constants.DEFAULT_NAME}---%s"
 
 
 class MemoryReader(object):
-    def __init__(self):
-        self.handles = []
-        self.file_type = constants.FILE_FORMAT_CSV
-
-    def set_type(self, _):
-        pass
+    file_type = constants.FILE_FORMAT_CSV
 
-    def open(self, file_stream, multiple_sheets=False, **keywords):
+    def __init__(self, file_stream, multiple_sheets=False, **keywords):
         """Load content from memory
         :params stream file_content: the actual file content in memory
         :returns: a book
         """
+        self.handles = []
         self.keywords = keywords
         self.__load_from_memory_flag = True
         self.__line_terminator = keywords.get(

pyexcel_io/readers/csv_file_reader.py renamed to pyexcel_io/readers/csv_in_file.py

@@ -12,18 +12,12 @@
 import chardet
 from pyexcel_io.sheet import NamedContent
 from pyexcel_io._compact import StringIO
-from pyexcel_io.readers.csvr import CSVinMemoryReader
+from pyexcel_io.readers.csv_sheet import CSVinMemoryReader
 
 
 class FileReader(object):
-    def __init__(self):
+    def __init__(self, file_alike_object, **keywords):
         self.content_array = []
-        self.keywords = None
-
-    def set_type(self, _):
-        pass
-
-    def open(self, file_alike_object, **keywords):
         try:
             self.zipfile = zipfile.ZipFile(file_alike_object, "r")
             sheets = [