[Servers] Add Local server

It allows to store locally comics in CBZ, CBR (and maybe soon EPUB) 
formats.

Still experimental

Issue #224, #13

README.md

@@ -84,6 +84,7 @@ Dependencies:
 * `python-natsort`
 * `python-pillow`
 * `python-pure-protobuf`
+* `python-rarfile`
 * `python-unidecode`
 
 This is the best practice to test __Komikku__ without installing using meson and ninja.

flatpak/info.febvre.Komikku.json

@@ -32,6 +32,7 @@
         "python3-beautifulsoup4.json",
         "python3-brotli.json",
         "python3-cloudscraper.json",
+        "python3-rarfile.json",
         {
             "name" : "komikku",
             "buildsystem" : "meson",

flatpak/python3-rarfile.json

@@ -0,0 +1,14 @@
+{
+    "name": "python3-rarfile",
+    "buildsystem": "simple",
+    "build-commands": [
+        "pip3 install --verbose --exists-action=i --no-index --find-links=\"file://${PWD}\" --prefix=${FLATPAK_DEST} \"rarfile\" --no-build-isolation"
+    ],
+    "sources": [
+        {
+            "type": "file",
+            "url": "https://files.pythonhosted.org/packages/95/f4/c92fab227c7457e3b76a4096ccb655ded9deac869849cb03afbe55dfdc1e/rarfile-4.0-py3-none-any.whl",
+            "sha256": "1094869119012f95c31a6f22cc3a9edbdca61861b805241116adbe2d737b68f8"
+        }
+    ]
+}

komikku/card.py

@@ -62,9 +62,9 @@ class Card:
         self.sort_order_action.connect('activate', self.chapters_list.on_sort_order_changed)
         self.window.application.add_action(self.sort_order_action)
 
-        open_in_browser_action = Gio.SimpleAction.new('card.open-in-browser', None)
+        self.open_in_browser_action = Gio.SimpleAction.new('card.open-in-browser', None)
-        open_in_browser_action.connect('activate', self.on_open_in_browser_menu_clicked)
+        self.open_in_browser_action.connect('activate', self.on_open_in_browser_menu_clicked)
-        self.window.application.add_action(open_in_browser_action)
+        self.window.application.add_action(self.open_in_browser_action)
 
         self.chapters_list.add_actions()
 
@@ -202,6 +202,8 @@ class Card:
         self.window.menu_button.set_icon_name('view-more-symbolic')
         self.window.menu_button.show()
 
+        self.open_in_browser_action.set_enabled(self.manga.server_id != 'local')
+
         self.window.show_page('card', transition=transition)
 
     def refresh(self, chapters):
@@ -1031,14 +1033,22 @@ class InfoBox:
         authors = html_escape(', '.join(manga.authors)) if manga.authors else _('Unknown author')
         self.authors_label.set_markup(authors)
 
-        self.status_server_label.set_markup(
+        if manga.server_id != 'local':
-            '{0} · <a href="{1}">{2}</a> ({3})'.format(
+            self.status_server_label.set_markup(
-                _(manga.STATUSES[manga.status]) if manga.status else '-',
+                '{0} · <a href="{1}">{2}</a> ({3})'.format(
-                manga.server.get_manga_url(manga.slug, manga.url),
+                    _(manga.STATUSES[manga.status]) if manga.status else _('Unknown status'),
-                html_escape(manga.server.name),
+                    manga.server.get_manga_url(manga.slug, manga.url),
-                manga.server.lang.upper()
+                    html_escape(manga.server.name),
+                    manga.server.lang.upper()
+                )
+            )
+        else:
+            self.status_server_label.set_markup(
+                '{0} · {1}'.format(
+                    _('Unknown status'),
+                    html_escape(_('Local'))
+                )
             )
-        )
 
         if manga.genres:
             self.genres_label.set_markup(html_escape(', '.join(manga.genres)))

komikku/explorer.py

@@ -3,9 +3,11 @@
 # Author: Valéry Febvre <vfebvre@easter-eggs.com>
 
 from gettext import gettext as _
+import os
 import threading
 import time
 
+from gi.repository import Gio
 from gi.repository import GLib
 from gi.repository import GObject
 from gi.repository import Gtk
@@ -16,6 +18,7 @@ from komikku.models import Manga
 from komikku.models import Settings
 from komikku.servers import LANGUAGES
 from komikku.servers.utils import get_allowed_servers_list
+from komikku.utils import get_data_dir
 from komikku.utils import html_escape
 from komikku.utils import log_error_traceback
 from komikku.utils import create_paintable_from_data
@@ -144,23 +147,33 @@ class Explorer(Gtk.Stack):
         # Server logo
         logo = Gtk.Image()
         logo.set_size_request(28, 28)
-        if data['logo_path']:
+        if data['id'] != 'local':
-            logo.set_from_file(data['logo_path'])
+            if data['logo_path']:
+                logo.set_from_file(data['logo_path'])
+        else:
+            logo.set_from_icon_name('folder-symbolic')
         box.append(logo)
 
         # Server title & language
         vbox = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=0)
 
+        if data['id'] != 'local':
+            title = data['name']
+            if data['is_nsfw']:
+                title += ' (NSFW)'
+            subtitle = LANGUAGES[data['lang']]
+        else:
+            title = _('Local')
+            subtitle = _('Comics stored locally as archives in CBZ/CBR formats')
+
         label = Gtk.Label(xalign=0, hexpand=True)
         label.set_ellipsize(Pango.EllipsizeMode.END)
-        title = data['name']
-        if data['is_nsfw']:
-            title += ' (NSFW)'
         label.set_text(title)
         vbox.append(label)
 
         label = Gtk.Label(xalign=0)
-        label.set_text(LANGUAGES[data['lang']])
+        label.set_wrap(True)
+        label.set_text(subtitle)
         label.add_css_class('subtitle')
         vbox.append(label)
 
@@ -171,8 +184,38 @@ class Explorer(Gtk.Stack):
             label = Gtk.Image.new_from_icon_name('dialog-password-symbolic')
             box.append(label)
 
+        if data['id'] == 'local':
+            # Info button
+            button = Gtk.MenuButton(valign=Gtk.Align.CENTER)
+            button.set_icon_name('help-about-symbolic')
+            popover = Gtk.Popover()
+            label = Gtk.Label()
+            label.set_markup("""A specific folder structure is required
+for local comics to be properly processed.
+
+Each comic must have its own folder which
+must contain the chapters as archive files
+in CBZ or CBR formats.
+
+The folder's name will be used as name
+for the comic.
+
+The 'unrar' utility is required for
+CBR format archives.
+""")
+            popover.set_child(label)
+            button.set_popover(popover)
+            box.append(button)
+
+            # Button to open local folder
+            button = Gtk.Button(valign=Gtk.Align.CENTER)
+            button.set_icon_name('folder-symbolic')
+            button.set_tooltip_text(_('Open local folder'))
+            button.connect('clicked', self.open_local_folder)
+            box.append(button)
+
         # Button to pin/unpin
-        button = Gtk.ToggleButton()
+        button = Gtk.ToggleButton(valign=Gtk.Align.CENTER)
         button.set_icon_name('view-pin-symbolic')
         button.set_active(data['id'] in Settings.get_default().pinned_servers)
         button.connect('toggled', self.toggle_server_pinned_state, row)
@@ -427,6 +470,10 @@ class Explorer(Gtk.Stack):
             self.on_server_clicked(self.servers_page_listbox, child_row)
             break
 
+    def open_local_folder(self, _button):
+        path = os.path.join(get_data_dir(), 'local')
+        Gio.app_info_launch_default_for_uri(f'file://{path}')
+
     def populate_card(self, manga_data):
         def run(server, manga_slug):
             try:
@@ -469,14 +516,22 @@ class Explorer(Gtk.Stack):
             authors = html_escape(', '.join(self.manga_data['authors'])) if self.manga_data['authors'] else _('Unknown author')
             self.card_page_authors_label.set_markup(authors)
 
-            self.card_page_status_server_label.set_markup(
+            if self.manga_data['server_id'] != 'local':
-                '{0} · <a href="{1}">{2}</a> ({3})'.format(
+                self.card_page_status_server_label.set_markup(
-                    _(Manga.STATUSES[self.manga_data['status']]) if self.manga_data['status'] else '-',
+                    '{0} · <a href="{1}">{2}</a> ({3})'.format(
-                    self.server.get_manga_url(self.manga_data['slug'], self.manga_data.get('url')),
+                        _(Manga.STATUSES[self.manga_data['status']]) if self.manga_data['status'] else _('Unknown status'),
-                    html_escape(self.server.name),
+                        self.server.get_manga_url(self.manga_data['slug'], self.manga_data.get('url')),
-                    self.server.lang.upper()
+                        html_escape(self.server.name),
+                        self.server.lang.upper()
+                    )
+                )
+            else:
+                self.card_page_status_server_label.set_markup(
+                    '{0} · {1}'.format(
+                        _('Unknown status'),
+                        html_escape(_('Local'))
+                    )
                 )
-            )
 
             if self.manga_data['genres']:
                 self.card_page_genres_label.set_markup(html_escape(', '.join(self.manga_data['genres'])))
@@ -587,7 +642,7 @@ class Explorer(Gtk.Stack):
                 row.add_css_class('explorer-section-listboxrow')
                 label = Gtk.Label(xalign=0)
                 label.add_css_class('subtitle')
-                label.set_text(LANGUAGES[server_data['lang']].upper())
+                label.set_text(LANGUAGES[server_data['lang']].upper() if server_data['lang'] else _('Other'))
                 row.set_child(label)
                 self.servers_page_listbox.append(row)
 
@@ -650,7 +705,10 @@ class Explorer(Gtk.Stack):
                 row = Gtk.ListBoxRow()
                 row.add_css_class('explorer-section-listboxrow')
                 row.manga_data = None
-                label = Gtk.Label(label=_('MOST POPULARS'), xalign=0)
+                if server.id != 'local':
+                    label = Gtk.Label(label=_('Most populars').upper(), xalign=0)
+                else:
+                    label = Gtk.Label(label=_('Collection').upper(), xalign=0)
                 label.add_css_class('subtitle')
                 row.set_child(label)
 

komikku/models/database.py

@@ -589,7 +589,8 @@ class Manga:
 
         db_conn.close()
 
-        if os.path.exists(self.path):
+        # Delete folder except when server is 'local'
+        if os.path.exists(self.path) and self.server_id != 'local':
             shutil.rmtree(self.path)
 
     def get_next_chapter(self, chapter, direction=1):
@@ -721,7 +722,7 @@ class Manga:
                     chapter_data.update(dict(
                         manga_id=self.id,
                         rank=rank,
-                        downloaded=0,
+                        downloaded=chapter_data.get('downloaded', 0),
                         recent=1,
                         read=0,
                     ))
@@ -786,7 +787,7 @@ class Chapter:
         data.update(dict(
             manga_id=manga_id,
             rank=rank,
-            downloaded=0,
+            downloaded=data.get('downloaded', 0),
             recent=0,
             read=0,
         ))
@@ -885,6 +886,14 @@ class Chapter:
 
         return page_path
 
+    def get_page_data(self, index):
+        """
+        Return page image data: buffer, mime type, name
+
+        Useful for locally stored manga. Image data (bytes) are retrieved directly from archive.
+        """
+        return self.manga.server.get_manga_chapter_page_image(self.manga.slug, self.manga.name, self.slug, self.pages[index])
+
     def get_page_path(self, index):
         if not self.pages:
             return None

komikku/reader/pager/page.py

@@ -10,6 +10,7 @@ from gi.repository import GObject
 from gi.repository import Gtk
 
 from komikku.activity_indicator import ActivityIndicator
+from komikku.utils import create_picture_from_data
 from komikku.utils import create_picture_from_file
 from komikku.utils import create_picture_from_resource
 from komikku.utils import log_error_traceback
@@ -29,9 +30,11 @@ class Page(Gtk.Overlay):
         self.window = self.reader.window
 
         self.chapter = self.init_chapter = chapter
+        self.data = None
         self.index = self.init_index = index
         self.init_height = None
         self.path = None
+        self.picture = None
 
         self._status = None    # rendering, rendered, offlimit, cleaned
         self.error = None      # connection error, server error or corrupt file error
@@ -50,8 +53,6 @@ class Page(Gtk.Overlay):
 
         self.set_child(self.scrolledwindow)
 
-        self.picture = None
-
         # Activity indicator
         self.activity_indicator = ActivityIndicator()
         self.add_overlay(self.activity_indicator)
@@ -181,19 +182,25 @@ class Page(Gtk.Overlay):
 
             self.loadable = True
 
-            page_path = self.chapter.get_page_path(self.index)
+            if self.chapter.manga.server_id != 'local':
-            if page_path is None:
+                page_path = self.chapter.get_page_path(self.index)
-                try:
+                if page_path is None:
-                    page_path = self.chapter.get_page(self.index)
+                    try:
-                    if page_path:
+                        page_path = self.chapter.get_page(self.index)
-                        self.path = page_path
+                        if page_path:
-                    else:
+                            self.path = page_path
-                        error_code, error_message = 'server', None
+                        else:
-                        on_error('server')
+                            error_code, error_message = 'server', None
-                except Exception as e:
+                            on_error('server')
-                    error_code, error_message = 'connection', log_error_traceback(e)
+                    except Exception as e:
+                        error_code, error_message = 'connection', log_error_traceback(e)
+                else:
+                    self.path = page_path
             else:
-                self.path = page_path
+                self.data = self.chapter.get_page_data(self.index)
+                # FIXME
+                if self.data is None:
+                    self.error = 'corrupt_file'
 
             GLib.idle_add(complete, error_code, error_message)
 
@@ -205,9 +212,12 @@ class Page(Gtk.Overlay):
 
         self.activity_indicator.start()
 
-        thread = threading.Thread(target=run)
+        if self.chapter.manga.server_id != 'local':
-        thread.daemon = True
+            thread = threading.Thread(target=run)
-        thread.start()
+            thread.daemon = True
+            thread.start()
+        else:
+            run()
 
     def rescale(self):
         if self.status == 'rendered':
@@ -219,10 +229,14 @@ class Page(Gtk.Overlay):
 
     def set_image(self, size=None):
         if self.picture is None:
-            if self.path is None:
+            if self.path is None and self.data is None:
                 picture = create_picture_from_resource('/info/febvre/Komikku/images/missing_file.png')
             else:
-                picture = create_picture_from_file(self.path, subdivided=self.reader.reading_mode == 'webtoon')
+                if self.path:
+                    picture = create_picture_from_file(self.path, subdivided=self.reader.reading_mode == 'webtoon')
+                else:
+                    picture = create_picture_from_data(self.data['buffer'], subdivided=self.reader.reading_mode == 'webtoon')
+
                 if picture is None:
                     GLib.unlink(self.path)
 

komikku/servers/local/__init__.py

@@ -0,0 +1,214 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2019-2022 Valéry Febvre
+# SPDX-License-Identifier: GPL-3.0-only or GPL-3.0-or-later
+# Author: Valéry Febvre <vfebvre@easter-eggs.com>
+
+import os
+import rarfile
+import zipfile
+
+from komikku.servers import Server
+from komikku.servers.utils import convert_image
+from komikku.servers.utils import get_buffer_mime_type
+from komikku.utils import get_data_dir
+
+IMG_EXTENSIONS = ['bmp', 'gif', 'jpg', 'jpeg', 'png', 'webp']
+
+
+def is_archive(path):
+    if zipfile.is_zipfile(path):
+        return True
+    elif rarfile.is_rarfile(path):
+        return True
+
+    return False
+
+
+class Archive:
+    def __init__(self, path):
+        if zipfile.is_zipfile(path):
+            self.obj = CBZ(path)
+
+        elif rarfile.is_rarfile(path):
+            self.obj = CBR(path)
+
+    def get_namelist(self):
+        names = []
+        for name in self.obj.get_namelist():
+            _root, ext = os.path.splitext(name)
+            if ext[1:].lower() in IMG_EXTENSIONS:
+                names.append(name)
+
+        return names
+
+    def get_name_buffer(self, name):
+        return self.obj.get_name_buffer(name)
+
+
+class CBR:
+    def __init__(self, path):
+        self.path = path
+
+    def get_namelist(self):
+        with rarfile.RarFile(self.path) as archive:
+            names = archive.namelist()
+
+        return names
+
+    def get_name_buffer(self, name):
+        with rarfile.RarFile(self.path) as archive:
+            try:
+                buffer = archive.read(name)
+            except Exception:
+                # `unrar` command line tool is missing, bad/invalid archive, not RAR archive...
+                buffer = None
+
+        return buffer
+
+
+class CBZ:
+    def __init__(self, path):
+        self.path = path
+
+    def get_namelist(self):
+        with zipfile.ZipFile(self.path) as archive:
+            names = archive.namelist()
+
+        return names
+
+    def get_name_buffer(self, name):
+        with zipfile.ZipFile(self.path) as archive:
+            buffer = archive.read(name)
+
+        return buffer
+
+
+class Local(Server):
+    id = 'local'
+    name = 'Local'
+    lang = ''
+
+    def get_manga_cover_image(self, data):
+        if data is None:
+            return None
+
+        archive = Archive(data['path'])
+        buffer = archive.get_name_buffer(data['name'])
+
+        if buffer is None:
+            return None
+
+        mime_type = get_buffer_mime_type(buffer)
+
+        if not mime_type.startswith('image'):
+            return None
+
+        if mime_type == 'image/webp':
+            buffer = convert_image(buffer, ret_type='bytes')
+
+        return buffer
+
+    def get_manga_data(self, initial_data):
+        data = initial_data.copy()
+        data.update(dict(
+            authors=[],
+            scanlators=[],
+            genres=[],
+            status=None,
+            chapters=[],
+            synopsis=None,
+            cover=None,
+            server_id=self.id,
+            url=None,
+        ))
+
+        dir_path = os.path.join(get_data_dir(), self.id, data['slug'])
+        if not os.path.exists(dir_path):
+            return None
+
+        # Chapters
+        for _path, _dirs, files in os.walk(dir_path):
+            for file in sorted(files):
+                path = os.path.join(dir_path, file)
+                if not is_archive(os.path.join(dir_path, file)):
+                    continue
+
+                data['chapters'].append(dict(
+                    slug=file,
+                    title=os.path.splitext(file)[0],
+                    date=None,
+                    downloaded=1,
+                ))
+
+        # Cover is by default 1st page of 1st chapter
+        if len(data['chapters']) > 0:
+            path = os.path.join(dir_path, data['chapters'][0]['slug'])
+            archive = Archive(path)
+            data['cover'] = dict(
+                path=path,
+                name=archive.get_namelist()[0],
+            )
+
+        return data
+
+    def get_manga_chapter_data(self, manga_slug, manga_name, chapter_slug, chapter_url):
+        path = os.path.join(get_data_dir(), self.id, manga_slug, chapter_slug)
+        if not os.path.exists(path):
+            return None
+
+        archive = Archive(path)
+
+        data = dict(
+            pages=[],
+        )
+        for name in archive.get_namelist():
+            data['pages'].append(dict(
+                slug=name,
+                image=None,
+            ))
+
+        return data
+
+    def get_manga_chapter_page_image(self, manga_slug, manga_name, chapter_slug, page):
+        path = os.path.join(get_data_dir(), self.id, manga_slug, chapter_slug)
+        if not os.path.exists(path):
+            return None
+
+        archive = Archive(path)
+        content = archive.get_name_buffer(page['slug'])
+
+        mime_type = get_buffer_mime_type(content)
+        if not mime_type.startswith('image'):
+            return None
+
+        return dict(
+            buffer=content,
+            mime_type=mime_type,
+            name=page['slug'],
+        )
+
+    def get_manga_url(self, slug, url):
+        return None
+
+    def get_most_populars(self):
+        return self.search('')
+
+    def search(self, term):
+        dir_path = os.path.join(get_data_dir(), self.id)
+
+        result = []
+        for path, _dirs, _files in os.walk(dir_path):
+            if path == dir_path:
+                continue
+
+            name = os.path.basename(path)
+            if term and term.lower() not in name.lower():
+                continue
+
+            result.append(dict(
+                name=name,
+                slug=name,
+            ))
+
+        return result

komikku/utils.py

@@ -31,9 +31,22 @@ from gi.repository.GdkPixbuf import InterpType
 from gi.repository.GdkPixbuf import Pixbuf
 from gi.repository.GdkPixbuf import PixbufAnimation
 
+from komikku.servers.utils import get_buffer_mime_type
+
 logger = logging.getLogger('komikku')
 
 
+def create_picture_from_data(data, static_animation=False, subdivided=False):
+    mime_type = get_buffer_mime_type(data)
+
+    if mime_type == 'image/gif' and not static_animation:
+        return PictureAnimation.new_from_data(data)
+    elif subdivided:
+        return PictureSubdivided.new_from_data(data)
+    else:
+        return Picture.new_from_data(data)
+
+
 def create_picture_from_file(path, static_animation=False, subdivided=False):
     format, _width, _height = Pixbuf.get_file_info(path)
     if format is None:
@@ -88,7 +101,7 @@ def expand_cover(buffer):
     """Convert a cover that is in landscape format (rare) to portrait format"""
 
     def get_dominant_color(img):
-        # Resize imgae to reduce number of colors
+        # Resize image to reduce number of colors
         colors = img.resize((150, 150), resample=0).getcolors(150 * 150)
         sorted_colors = sorted(colors, key=lambda t: t[0])
 
@@ -151,31 +164,33 @@ def get_data_dir():
     data_dir_path = GLib.get_user_data_dir()
     app_profile = Gio.Application.get_default().profile
 
-    # Check if inside flatpak sandbox
+    if not is_flatpak():
-    if is_flatpak():
+        base_path = data_dir_path
-        return data_dir_path
+        data_dir_path = os.path.join(base_path, 'komikku')
+        if app_profile == 'development':
+            data_dir_path += '-devel'
+        elif app_profile == 'beta':
+            data_dir_path += '-beta'
 
-    base_path = data_dir_path
+        if not os.path.exists(data_dir_path):
-    data_dir_path = os.path.join(base_path, 'komikku')
+            os.mkdir(data_dir_path)
-    if app_profile == 'development':
-        data_dir_path += '-devel'
-    elif app_profile == 'beta':
-        data_dir_path += '-beta'
 
-    if not os.path.exists(data_dir_path):
+            # Until version 0.11.0, data files (chapters, database) were stored in a wrong place
-        os.mkdir(data_dir_path)
+            from komikku.servers.utils import get_servers_list
 
-        # Until version 0.11.0, data files (chapters, database) were stored in a wrong place
+            must_be_moved = ['komikku.db', 'komikku_backup.db', ]
-        from komikku.servers.utils import get_servers_list
+            for server in get_servers_list(include_disabled=True):
+                must_be_moved.append(server['id'])
 
-        must_be_moved = ['komikku.db', 'komikku_backup.db', ]
+            for name in must_be_moved:
-        for server in get_servers_list(include_disabled=True):
+                data_path = os.path.join(base_path, name)
-            must_be_moved.append(server['id'])
+                if os.path.exists(data_path):
+                    os.rename(data_path, os.path.join(data_dir_path, name))
 
-        for name in must_be_moved:
+    # Create folder for 'local' server
-            data_path = os.path.join(base_path, name)
+    data_local_dir_path = os.path.join(data_dir_path, 'local')
-            if os.path.exists(data_path):
+    if not os.path.exists(data_local_dir_path):
-                os.rename(data_path, os.path.join(data_dir_path, name))
+        os.mkdir(data_local_dir_path)
 
     return data_dir_path
 
@@ -560,6 +575,11 @@ class PictureSubdivided(Gtk.Box):
         self.orig_width = pixbuf.get_width()
         self.orig_height = pixbuf.get_height()
 
+    @classmethod
+    def new_from_data(cls, data):
+        stream = Gio.MemoryInputStream.new_from_data(data, None)
+        return cls(None, Pixbuf.new_from_stream(stream))
+
     @classmethod
     def new_from_file(cls, path):
         return cls(path, Pixbuf.new_from_file(path))