New style writer (#76)

* 🔨 code up the new writer interface

* 🐴 prototype

* 🚜 new style csv writer for file, content and stream

* This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst

* 💚 add test dependency

* This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst

* 💚 update moban update yaml

* 💄 update coding style with latest black, isort

* ✨ tsv writers in new style

* 🐴 new style csvz writer

* 🐴 new style tsvz writer

* 🔥 remove old style csvz and tsvz writers

* 🔥 remove old style csv and tsv writer

* 📚 update changelog

* 🚜 code refactoring and update 📚

* This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst

* 🚜 rename sheet writer files

* 🔥 remove .moban.hashes

* 🔥 remove useless set_type function call

* 💄 update coding style

Co-authored-by: chfw <chfw@users.noreply.github.com>

Author: chfw

.github/workflows/moban-update.yml

@@ -5,23 +5,25 @@ jobs:
     runs-on: ubuntu-latest
     name: synchronize templates via moban
     steps:
-    - uses: actions/checkout@v2
-      with:
-        ref: ${{ github.head_ref }}
-    - name: Set up Python
-      uses: actions/setup-python@v1
-      with:
-        python-version: '3.7'
-    - name: check changes
-      run: |
-        pip install moban gitfs2 pypifs
-        moban
-        git status
-        git diff --exit-code
-    - name: Auto-commit
-      if: failure()
-      uses: docker://cdssnc/auto-commit-github-action
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        args: This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.head_ref }}
+      - name: Set up Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.7'
+      - name: check changes
+        run: |
+          pip install moban gitfs2 pypifs moban-jinja2-github moban-ansible
+          moban
+          git status
+          git diff --exit-code
+      - name: Auto-commit
+        if: failure()
+        uses: docker://cdssnc/auto-commit-github-action
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          args: >-
+            This is an auto-commit, updating project meta data,
+            such as changelog.rst, contributors.rst

CHANGELOG.rst

@@ -13,6 +13,11 @@ Change log
 
 #. python 3.6 lower versions are no longer supported
 
+**updated**
+
+#. pyexcel-io plugin interface has been rewritten. PyInstaller user will be
+   impacted.
+
 0.5.20 - 17.7.2019
 --------------------------------------------------------------------------------
 

changelog.yml

@@ -8,6 +8,9 @@ releases:
   - action: removed
     details:
     - 'python 3.6 lower versions are no longer supported'
+  - action: updated
+    details:
+    - 'pyexcel-io plugin interface has been rewritten. PyInstaller user will be impacted.'
   version: 0.6.0
   date: tbd
 - changes:

docs/source/pyinstaller.rst

@@ -1,6 +1,30 @@
 Packaging with PyInstaller
 ================================================================================
 
+With pyexcel-io v0.6.0, the way to package it has been changed because
+plugin interface update.
+
+Built-in plugins for pyexcel-io
+---------------------------------
+
+In order to package every built-in plugins of pyexcel-io, you need to specify::
+
+    --hidden-import pyexcel_io.readers.csvr
+    --hidden-import pyexcel_io.readers.csvz
+    --hidden-import pyexcel_io.readers.tsv
+    --hidden-import pyexcel_io.readers.tsvz
+    --hidden-import pyexcel_io.writers.csv_file_writer
+    --hidden-import pyexcel_io.writers.csv_memory_writer
+    --hidden-import pyexcel_io.writers.tsv_file_writer
+    --hidden-import pyexcel_io.writers.tsv_memory_writer
+    --hidden-import pyexcel_io.writers.csvz_writer
+    --hidden-import pyexcel_io.writers.tsvz_writer
+    --hidden-import pyexcel_io.database.importers.django
+    --hidden-import pyexcel_io.database.importers.sqlalchemy
+    --hidden-import pyexcel_io.database.exporters.django
+    --hidden-import pyexcel_io.database.exporters.sqlalchemy
+
+
 With pyexcel-io v0.4.0, the way to package it has been changed because it
 uses lml for all plugins.
 
@@ -14,9 +38,9 @@ In order to package every built-in plugins of pyexcel-io, you need to specify::
     --hidden-import pyexcel_io.readers.tsv
     --hidden-import pyexcel_io.readers.tsvz
     --hidden-import pyexcel_io.writers.csvw
-    --hidden-import pyexcel_io.readers.csvz
-    --hidden-import pyexcel_io.readers.tsv
-    --hidden-import pyexcel_io.readers.tsvz
+    --hidden-import pyexcel_io.writers.csvz
+    --hidden-import pyexcel_io.writers.tsv
+    --hidden-import pyexcel_io.writers.tsvz
     --hidden-import pyexcel_io.database.importers.django
     --hidden-import pyexcel_io.database.importers.sqlalchemy
     --hidden-import pyexcel_io.database.exporters.django

pyexcel-io.yml

@@ -15,6 +15,7 @@ test_dependencies:
   - pyexcel
   - pyexcel-xls
   - SQLAlchemy
+  - pyexcel-xlsxw
 extra_dependencies:
   - xls:
     - pyexcel-xls>=0.5.0

pyexcel_io/io.py

@@ -14,7 +14,7 @@
 from pyexcel_io import constants
 from pyexcel_io.reader import Reader
 from pyexcel_io.writer import Writer
-from pyexcel_io.plugins import OLD_READERS
+from pyexcel_io.plugins import OLD_READERS, OLD_WRITERS
 from pyexcel_io._compact import isstream
 from pyexcel_io.exceptions import (
     NoSupportingPluginFound,
@@ -244,7 +244,11 @@ def get_writer(
 
         file_type_given = False
 
-    writer = Writer(file_type, library)
+    try:
+        writer = OLD_WRITERS.get_a_plugin(file_type, library)
+    except (NoSupportingPluginFound, SupportingPluginAvailableButNotInstalled):
+        writer = Writer(file_type, library)
+
     if file_name:
         if file_type_given:
             writer.open_content(file_name, **keywords)

pyexcel_io/plugins.py

@@ -20,6 +20,7 @@
 READER_PLUGIN = "pyexcel-io reader"
 NEW_READER_PLUGIN = "pyexcel-io new reader"
 WRITER_PLUGIN = "pyexcel-io writer"
+NEW_WRITER_PLUGIN = "pyexcel-io new writer"
 
 
 class IOPluginInfo(PluginInfo):
@@ -83,6 +84,26 @@ def add_a_reader(
         )
         return self.add_a_plugin_instance(a_plugin_info)
 
+    def add_a_writer(
+        self,
+        relative_plugin_class_path=None,
+        locations=(),
+        file_types=(),
+        stream_type=None,
+    ):
+        """ add pyexcle-io writer plugin info """
+        a_plugin_info = IOPluginInfo(
+            NEW_WRITER_PLUGIN,
+            self._get_abs_path(relative_plugin_class_path),
+            file_types=[
+                f"{location}-{file_type}"
+                for file_type in file_types
+                for location in locations
+            ],
+            stream_type=stream_type,
+        )
+        return self.add_a_plugin_instance(a_plugin_info)
+
 
 class IOManager(PluginManager):
     """Manage pyexcel-io plugins"""
@@ -143,20 +164,19 @@ def get_all_formats(self):
 class NewIOManager(IOManager):
     def load_me_later(self, plugin_info):
         PluginManager.load_me_later(self, plugin_info)
-        _do_additional_registration(plugin_info)
+        _do_additional_registration_for_new_plugins(plugin_info)
 
     def register_a_plugin(self, cls, plugin_info):
         """ for dynamically loaded plugin """
         PluginManager.register_a_plugin(self, cls, plugin_info)
-        _do_additional_registration(plugin_info)
+        _do_additional_registration_for_new_plugins(plugin_info)
 
     def get_a_plugin(
         self, file_type=None, location=None, library=None, **keywords
     ):
         __file_type = file_type.lower()
         plugin = self.load_me_now(f"{location}-{__file_type}", library=library)
         handler = plugin()
-        handler.set_type(__file_type)
         return handler
 
     def raise_exception(self, file_type):
@@ -194,17 +214,36 @@ def _do_additional_registration(plugin_info):
         manager.register_a_file_type(file_type, plugin_info.stream_type, None)
 
 
+def _do_additional_registration_for_new_plugins(plugin_info):
+    for file_type in plugin_info.tags():
+        manager.register_stream_type(
+            file_type.split("-")[1], plugin_info.stream_type
+        )
+        manager.register_a_file_type(
+            file_type.split("-")[1], plugin_info.stream_type, None
+        )
+
+
 class FakeReaders:
     def get_all_formats(self):
         return OLD_READERS.get_all_formats().union(
             NEW_READERS.get_all_formats()
         )
 
 
+class FakeWriters:
+    def get_all_formats(self):
+        return OLD_WRITERS.get_all_formats().union(
+            NEW_WRITERS.get_all_formats()
+        )
+
+
 OLD_READERS = IOManager(READER_PLUGIN, ioutils.AVAILABLE_READERS)
-WRITERS = IOManager(WRITER_PLUGIN, ioutils.AVAILABLE_WRITERS)
+OLD_WRITERS = IOManager(WRITER_PLUGIN, ioutils.AVAILABLE_WRITERS)
+NEW_WRITERS = NewIOManager(NEW_WRITER_PLUGIN, ioutils.AVAILABLE_WRITERS)
 NEW_READERS = NewIOManager(NEW_READER_PLUGIN, ioutils.AVAILABLE_READERS)
 READERS = FakeReaders()
+WRITERS = FakeWriters()
 
 
 def load_plugins(plugin_name_patterns, path, black_list, white_list):

pyexcel_io/readers/__init__.py

@@ -7,7 +7,7 @@
     :copyright: (c) 2014-2020 by Onni Software Ltd.
     :license: New BSD License, see LICENSE for more details
 """
-from pyexcel_io.plugins import IOPluginInfoChain, NewIOPluginInfoChain
+from pyexcel_io.plugins import NewIOPluginInfoChain
 
 NewIOPluginInfoChain(__name__).add_a_reader(
     relative_plugin_class_path="csv_file_reader.FileReader",

pyexcel_io/writer.py

@@ -1,24 +1,43 @@
-from pyexcel_io.plugins import WRITERS
+from pyexcel_io.plugins import NEW_WRITERS
+from pyexcel_io._compact import isstream
+
+from .constants import MESSAGE_ERROR_03
 
 
 class Writer(object):
-    def __init__(self, file_type, library):
-        self.writer = WRITERS.get_a_plugin(file_type, library)
+    def __init__(self, file_type, library=None):
+        self.file_type = file_type
+        self.library = library
+        self.keyboards = None
 
     def open(self, file_name, **keywords):
+        self.writer = NEW_WRITERS.get_a_plugin(
+            self.file_type, library=self.library, location="file"
+        )
         self.writer.open(file_name, **keywords)
 
-    def open_stream(self, file_stream, **keywords):
-        self.writer.open_stream(file_stream, **keywords)
-
     def open_content(self, file_stream, **keywords):
-        self.writer.open_content(file_stream, **keywords)
+        if not isstream(file_stream):
+            raise IOError(MESSAGE_ERROR_03)
+        self.writer = NEW_WRITERS.get_a_plugin(
+            self.file_type, library=self.library, location="content"
+        )
+        self.writer.open(file_stream, **keywords)
 
-    def write(self, incoming_dict):
-        self.writer.write(incoming_dict)
+    def open_stream(self, file_stream, **keywords):
+        self.writer = NEW_WRITERS.get_a_plugin(
+            self.file_type, library=self.library, location="memory"
+        )
+        self.writer.open(file_stream, **keywords)
 
-    def create_sheet(self, sheet_name):
-        return self.writer.create_sheet(sheet_name)
+    def write(self, incoming_dict):
+        for sheet_name in incoming_dict:
+            sheet_writer = self.writer.create_sheet(sheet_name)
+            if sheet_writer:
+                sheet_writer.write_array(incoming_dict[sheet_name])
+                sheet_writer.close()
+            else:
+                raise Exception("Cannot create a sheet writer!")
 
     def close(self):
         self.writer.close()

pyexcel_io/writers/__init__.py

@@ -7,22 +7,36 @@
     :copyright: (c) 2014-2020 by Onni Software Ltd.
     :license: New BSD License, see LICENSE for more details
 """
-from pyexcel_io.plugins import IOPluginInfoChain
+from pyexcel_io.plugins import NewIOPluginInfoChain
 
-IOPluginInfoChain(__name__).add_a_writer(
-    relative_plugin_class_path="csvw.CSVBookWriter",
+NewIOPluginInfoChain(__name__).add_a_writer(
+    relative_plugin_class_path="csv_in_file.CsvFileWriter",
+    locations=["file", "content"],
     file_types=["csv"],
     stream_type="text",
 ).add_a_writer(
-    relative_plugin_class_path="tsv.TSVBookWriter",
+    relative_plugin_class_path="csv_in_memory.CsvMemoryWriter",
+    locations=["memory"],
+    file_types=["csv"],
+    stream_type="text",
+).add_a_writer(
+    relative_plugin_class_path="tsv_in_file.TsvFileWriter",
+    locations=["file", "content"],
+    file_types=["tsv"],
+    stream_type="text",
+).add_a_writer(
+    relative_plugin_class_path="tsv_in_memory.TsvMemoryWriter",
+    locations=["memory"],
     file_types=["tsv"],
     stream_type="text",
 ).add_a_writer(
-    relative_plugin_class_path="csvz.CSVZipBookWriter",
+    relative_plugin_class_path="csvz_writer.CsvZipWriter",
+    locations=["memory", "file", "content"],
     file_types=["csvz"],
     stream_type="binary",
 ).add_a_writer(
-    relative_plugin_class_path="tsvz.TSVZipBookWriter",
+    relative_plugin_class_path="tsvz_writer.TsvZipWriter",
+    locations=["memory", "file", "content"],
     file_types=["tsvz"],
     stream_type="binary",
 )

pyexcel_io/writers/csv_in_file.py

@@ -0,0 +1,25 @@
+from pyexcel_io.writers.csv_sheet import CSVFileWriter
+
+
+class CsvFileWriter:
+    def __init__(self):
+        self.__index = 0
+        self.writer = None
+
+    def open(self, file_alike_object, **keywords):
+        self._file_alike_object = file_alike_object
+        self._keywords = keywords
+
+    def create_sheet(self, name):
+        self.writer = CSVFileWriter(
+            self._file_alike_object,
+            name,
+            sheet_index=self.__index,
+            **self._keywords
+        )
+        self.__index = self.__index + 1
+        return self.writer
+
+    def close(self):
+        if self.writer:
+            self.writer.close()

pyexcel_io/writers/csv_in_memory.py

@@ -0,0 +1,24 @@
+from pyexcel_io.writers.csv_sheet import CSVMemoryWriter
+
+
+class CsvMemoryWriter:
+    def __init__(self):
+        self.__index = 0
+
+    def open(self, file_alike_object, **keywords):
+        self._file_alike_object = file_alike_object
+        self._keywords = keywords
+
+    def create_sheet(self, name):
+        writer_class = CSVMemoryWriter
+        writer = writer_class(
+            self._file_alike_object,
+            name,
+            sheet_index=self.__index,
+            **self._keywords
+        )
+        self.__index = self.__index + 1
+        return writer
+
+    def close(self):
+        pass