diff options
author | Unit 193 <unit193@unit193.net> | 2020-09-01 18:58:32 -0400 |
---|---|---|
committer | Unit 193 <unit193@unit193.net> | 2020-09-01 18:58:32 -0400 |
commit | 4ec4285f016962ca2632e08159a8ab3923db0d4d (patch) | |
tree | 2ce4c30c0a9bfb7713a1a18dc886ea1708f4760d | |
parent | 5c70c52739bfef9ec6e2cac3e6799edc6fa14f2e (diff) | |
parent | 261c8c2bc74969e2242a153297895684742b6995 (diff) | |
download | gallery-dl-4ec4285f016962ca2632e08159a8ab3923db0d4d.tar.bz2 gallery-dl-4ec4285f016962ca2632e08159a8ab3923db0d4d.tar.xz gallery-dl-4ec4285f016962ca2632e08159a8ab3923db0d4d.tar.zst |
Update upstream source from tag 'upstream/1.14.5'
Update to upstream version '1.14.5'
with Debian dir 41719034f3c9c9a9dbba0a65928783d0e4a02afd
-rw-r--r-- | CHANGELOG.md | 15 | ||||
-rw-r--r-- | PKG-INFO | 10 | ||||
-rw-r--r-- | README.rst | 8 | ||||
-rw-r--r-- | data/man/gallery-dl.1 | 2 | ||||
-rw-r--r-- | data/man/gallery-dl.conf.5 | 9 | ||||
-rw-r--r-- | docs/gallery-dl.conf | 5 | ||||
-rw-r--r-- | gallery_dl.egg-info/PKG-INFO | 10 | ||||
-rw-r--r-- | gallery_dl/extractor/500px.py | 171 | ||||
-rw-r--r-- | gallery_dl/extractor/aryion.py | 36 | ||||
-rw-r--r-- | gallery_dl/extractor/exhentai.py | 8 | ||||
-rw-r--r-- | gallery_dl/extractor/foolslide.py | 17 | ||||
-rw-r--r-- | gallery_dl/extractor/furaffinity.py | 59 | ||||
-rw-r--r-- | gallery_dl/extractor/gelbooru.py | 2 | ||||
-rw-r--r-- | gallery_dl/extractor/hentaihand.py | 137 | ||||
-rw-r--r-- | gallery_dl/extractor/hitomi.py | 14 | ||||
-rw-r--r-- | gallery_dl/extractor/imgur.py | 54 | ||||
-rw-r--r-- | gallery_dl/extractor/mangapark.py | 17 | ||||
-rw-r--r-- | gallery_dl/extractor/reddit.py | 33 | ||||
-rw-r--r-- | gallery_dl/extractor/redgifs.py | 3 | ||||
-rw-r--r-- | gallery_dl/extractor/subscribestar.py | 8 | ||||
-rw-r--r-- | gallery_dl/extractor/xvideos.py | 2 | ||||
-rw-r--r-- | gallery_dl/version.py | 2 | ||||
-rw-r--r-- | test/test_results.py | 6 |
23 files changed, 415 insertions, 213 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index fa9f17c..b38c9c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## 1.14.5 - 2020-08-30 +### Additions +- [aryion] add username/password support ([#960](https://github.com/mikf/gallery-dl/issues/960)) +- [exhentai] add ability to specify a custom image limit ([#940](https://github.com/mikf/gallery-dl/issues/940)) +- [furaffinity] add `search` extractor ([#915](https://github.com/mikf/gallery-dl/issues/915)) +- [imgur] add `search` and `tag` extractors ([#934](https://github.com/mikf/gallery-dl/issues/934)) +### Fixes +- [500px] fix extraction and update URL patterns ([#956](https://github.com/mikf/gallery-dl/issues/956)) +- [aryion] update folder mime type list ([#945](https://github.com/mikf/gallery-dl/issues/945)) +- [gelbooru] fix extraction without API +- [hentaihand] update to new site layout +- [hitomi] fix redirect processing +- [reddit] handle deleted galleries ([#953](https://github.com/mikf/gallery-dl/issues/953)) +- [reddit] improve gallery extraction ([#955](https://github.com/mikf/gallery-dl/issues/955)) + ## 1.14.4 - 2020-08-15 ### Additions - [blogger] add `search` extractor ([#925](https://github.com/mikf/gallery-dl/issues/925)) @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: gallery_dl -Version: 1.14.4 +Version: 1.14.5 Summary: Command-line program to download image-galleries and -collections from several image hosting sites Home-page: https://github.com/mikf/gallery-dl Author: Mike Fährmann @@ -94,8 +94,8 @@ Description: ========== put it into your `PATH <https://en.wikipedia.org/wiki/PATH_(variable)>`__, and run it inside a command prompt (like ``cmd.exe``). - - `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.14.4/gallery-dl.exe>`__ - - `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.14.4/gallery-dl.bin>`__ + - `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.14.5/gallery-dl.exe>`__ + - `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.14.5/gallery-dl.bin>`__ These executables include a Python 3.8 interpreter and all required Python packages. @@ -216,7 +216,7 @@ Description: ========== a username & password pair. This is necessary for ``pixiv``, ``nijie``, and ``seiga`` and optional for - ``danbooru``, ``e621``, ``exhentai``, ``idolcomplex``, ``inkbunny``, + ``aryion``, ``danbooru``, ``e621``, ``exhentai``, ``idolcomplex``, ``inkbunny``, ``instagram``, ``luscious``, ``sankaku``, ``subscribestar``, ``tsumino``, and ``twitter``. @@ -311,7 +311,7 @@ Description: ========== .. _gallery-dl-example.conf: https://github.com/mikf/gallery-dl/blob/master/docs/gallery-dl-example.conf .. _configuration.rst: https://github.com/mikf/gallery-dl/blob/master/docs/configuration.rst .. _Supported Sites: https://github.com/mikf/gallery-dl/blob/master/docs/supportedsites.rst - .. _stable: https://github.com/mikf/gallery-dl/archive/v1.14.4.tar.gz + .. _stable: https://github.com/mikf/gallery-dl/archive/v1.14.5.tar.gz .. _dev: https://github.com/mikf/gallery-dl/archive/master.tar.gz .. _Python: https://www.python.org/downloads/ @@ -83,8 +83,8 @@ Download a standalone executable file, put it into your `PATH <https://en.wikipedia.org/wiki/PATH_(variable)>`__, and run it inside a command prompt (like ``cmd.exe``). -- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.14.4/gallery-dl.exe>`__ -- `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.14.4/gallery-dl.bin>`__ +- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.14.5/gallery-dl.exe>`__ +- `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.14.5/gallery-dl.bin>`__ These executables include a Python 3.8 interpreter and all required Python packages. @@ -205,7 +205,7 @@ Some extractors require you to provide valid login credentials in the form of a username & password pair. This is necessary for ``pixiv``, ``nijie``, and ``seiga`` and optional for -``danbooru``, ``e621``, ``exhentai``, ``idolcomplex``, ``inkbunny``, +``aryion``, ``danbooru``, ``e621``, ``exhentai``, ``idolcomplex``, ``inkbunny``, ``instagram``, ``luscious``, ``sankaku``, ``subscribestar``, ``tsumino``, and ``twitter``. @@ -300,7 +300,7 @@ access to *gallery-dl*. Authorize it and you will be shown one or more .. _gallery-dl-example.conf: https://github.com/mikf/gallery-dl/blob/master/docs/gallery-dl-example.conf .. _configuration.rst: https://github.com/mikf/gallery-dl/blob/master/docs/configuration.rst .. _Supported Sites: https://github.com/mikf/gallery-dl/blob/master/docs/supportedsites.rst -.. _stable: https://github.com/mikf/gallery-dl/archive/v1.14.4.tar.gz +.. _stable: https://github.com/mikf/gallery-dl/archive/v1.14.5.tar.gz .. _dev: https://github.com/mikf/gallery-dl/archive/master.tar.gz .. _Python: https://www.python.org/downloads/ diff --git a/data/man/gallery-dl.1 b/data/man/gallery-dl.1 index e554159..2437195 100644 --- a/data/man/gallery-dl.1 +++ b/data/man/gallery-dl.1 @@ -1,4 +1,4 @@ -.TH "GALLERY-DL" "1" "2020-08-15" "1.14.4" "gallery-dl Manual" +.TH "GALLERY-DL" "1" "2020-08-30" "1.14.5" "gallery-dl Manual" .\" disable hyphenation .nh diff --git a/data/man/gallery-dl.conf.5 b/data/man/gallery-dl.conf.5 index 67e51d4..a5b1f4d 100644 --- a/data/man/gallery-dl.conf.5 +++ b/data/man/gallery-dl.conf.5 @@ -1,4 +1,4 @@ -.TH "GALLERY-DL.CONF" "5" "2020-08-15" "1.14.4" "gallery-dl Manual" +.TH "GALLERY-DL.CONF" "5" "2020-08-30" "1.14.5" "gallery-dl Manual" .\" disable hyphenation .nh .\" disable justification (adjust text to left margin only) @@ -276,6 +276,8 @@ Specifying a username and password is required for and optional for .br +* \f[I]aryion\f[] +.br * \f[I]danbooru\f[] .br * \f[I]e621\f[] @@ -858,7 +860,7 @@ depending on the input URL .SS extractor.exhentai.limits .IP "Type:" 6 -\f[I]bool\f[] +\f[I]bool\f[] or \f[I]integer\f[] .IP "Default:" 9 \f[I]true\f[] @@ -867,6 +869,9 @@ depending on the input URL Check image download limits and stop extraction when they are exceeded. +If this value is an \f[I]integer\f[], it gets used as the limit maximum +instead of the value listed on \f[I]https://e-hentai.org/home.php\f[] + .SS extractor.exhentai.original .IP "Type:" 6 \f[I]bool\f[] diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index 2db802d..56147e9 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -18,6 +18,11 @@ { "external": false }, + "aryion": + { + "username": null, + "password": null + }, "blogger": { "videos": true diff --git a/gallery_dl.egg-info/PKG-INFO b/gallery_dl.egg-info/PKG-INFO index 8f6f112..a2fafb1 100644 --- a/gallery_dl.egg-info/PKG-INFO +++ b/gallery_dl.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: gallery-dl -Version: 1.14.4 +Version: 1.14.5 Summary: Command-line program to download image-galleries and -collections from several image hosting sites Home-page: https://github.com/mikf/gallery-dl Author: Mike Fährmann @@ -94,8 +94,8 @@ Description: ========== put it into your `PATH <https://en.wikipedia.org/wiki/PATH_(variable)>`__, and run it inside a command prompt (like ``cmd.exe``). - - `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.14.4/gallery-dl.exe>`__ - - `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.14.4/gallery-dl.bin>`__ + - `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.14.5/gallery-dl.exe>`__ + - `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.14.5/gallery-dl.bin>`__ These executables include a Python 3.8 interpreter and all required Python packages. @@ -216,7 +216,7 @@ Description: ========== a username & password pair. This is necessary for ``pixiv``, ``nijie``, and ``seiga`` and optional for - ``danbooru``, ``e621``, ``exhentai``, ``idolcomplex``, ``inkbunny``, + ``aryion``, ``danbooru``, ``e621``, ``exhentai``, ``idolcomplex``, ``inkbunny``, ``instagram``, ``luscious``, ``sankaku``, ``subscribestar``, ``tsumino``, and ``twitter``. @@ -311,7 +311,7 @@ Description: ========== .. _gallery-dl-example.conf: https://github.com/mikf/gallery-dl/blob/master/docs/gallery-dl-example.conf .. _configuration.rst: https://github.com/mikf/gallery-dl/blob/master/docs/configuration.rst .. _Supported Sites: https://github.com/mikf/gallery-dl/blob/master/docs/supportedsites.rst - .. _stable: https://github.com/mikf/gallery-dl/archive/v1.14.4.tar.gz + .. _stable: https://github.com/mikf/gallery-dl/archive/v1.14.5.tar.gz .. _dev: https://github.com/mikf/gallery-dl/archive/master.tar.gz .. _Python: https://www.python.org/downloads/ diff --git a/gallery_dl/extractor/500px.py b/gallery_dl/extractor/500px.py index 7ecdef7..96cb021 100644 --- a/gallery_dl/extractor/500px.py +++ b/gallery_dl/extractor/500px.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019 Mike Fährmann +# Copyright 2019-2020 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -9,7 +9,7 @@ """Extractors for https://500px.com/""" from .common import Extractor, Message -from .. import text +import json BASE_PATTERN = r"(?:https?://)?(?:web\.)?500px\.com" @@ -48,7 +48,7 @@ class _500pxExtractor(Extractor): def photos(self): """Returns an iterable containing all relevant photo IDs""" - def _extend(self, photos): + def _extend(self, edges): """Extend photos with additional metadata and higher resolution URLs""" url = "https://api.500px.com/v1/photos" params = { @@ -62,40 +62,42 @@ class _500pxExtractor(Extractor): "liked_by" : "1", "following_sample" : "100", "image_size" : "4096", - "ids" : ",".join(str(p["id"]) for p in photos), + "ids" : ",".join( + str(edge["node"]["legacyId"]) for edge in edges), } - data = self._api_call(url, params)["photos"] - for photo in photos: - pid = str(photo["id"]) - photo.update(data[pid]) - return photos + data = self._request_api(url, params)["photos"] + return [ + data[str(edge["node"]["legacyId"])] + for edge in edges + ] - def _api_call(self, url, params, csrf_token=None): + def _request_api(self, url, params, csrf_token=None): headers = {"Origin": self.root, "X-CSRF-Token": csrf_token} return self.request(url, headers=headers, params=params).json() - def _pagination(self, url, params, csrf): - params["page"] = 1 - while True: - data = self._api_call(url, params, csrf) - yield from self._extend(data["photos"]) - - if params["page"] >= data["total_pages"]: - return - params["page"] += 1 + def _request_graphql(self, opname, variables, query_hash): + url = "https://api.500px.com/graphql" + params = { + "operationName": opname, + "variables" : json.dumps(variables), + "extensions" : '{"persistedQuery":{"version":1' + ',"sha256Hash":"' + query_hash + '"}}', + } + return self.request(url, params=params).json()["data"] class _500pxUserExtractor(_500pxExtractor): """Extractor for photos from a user's photostream on 500px.com""" subcategory = "user" - pattern = BASE_PATTERN + r"/(?!photo/)([^/?&#]+)/?(?:$|\?|#)" + pattern = BASE_PATTERN + r"/(?!photo/)(?:p/)?([^/?&#]+)/?(?:$|\?|#)" test = ( - ("https://500px.com/light_expression_photography", { + ("https://500px.com/p/light_expression_photography", { "pattern": r"https?://drscdn.500px.org/photo/\d+/m%3D4096/v2", "range": "1-99", "count": 99, }), + ("https://500px.com/light_expression_photography"), ("https://web.500px.com/light_expression_photography"), ) @@ -104,72 +106,97 @@ class _500pxUserExtractor(_500pxExtractor): self.user = match.group(1) def photos(self): - # get csrf token and user id from webpage - url = "{}/{}".format(self.root, self.user) - page = self.request(url).text - csrf_token, pos = text.extract(page, 'csrf-token" content="', '"') - user_id , pos = text.extract(page, '/user/', '"', pos) + variables = {"username": self.user, "pageSize": 20} + photos = self._request_graphql( + "OtherPhotosQuery", variables, + "54524abbdc809f8d4e10d37839e8ab2d" + "3035413688cad9c7fbece13b66637e9d", + )["user"]["photos"] - # get user photos - url = "https://api.500px.com/v1/photos" - params = { - "feature" : "user", - "stream" : "photos", - "rpp" : "50", - "user_id" : user_id, - } - return self._pagination(url, params, csrf_token) + while True: + yield from self._extend(photos["edges"]) + + if not photos["pageInfo"]["hasNextPage"]: + return + + variables["cursor"] = photos["pageInfo"]["endCursor"] + photos = self._request_graphql( + "OtherPhotosPaginationContainerQuery", variables, + "6d31e01104456ce642a2c6fc2f936812" + "b0f2a65c442d03e1521d769c20efe507", + )["userByUsername"]["photos"] class _500pxGalleryExtractor(_500pxExtractor): """Extractor for photo galleries on 500px.com""" subcategory = "gallery" directory_fmt = ("{category}", "{user[username]}", "{gallery[name]}") - pattern = BASE_PATTERN + r"/(?!photo/)([^/?&#]+)/galleries/([^/?&#]+)" - test = ("https://500px.com/fashvamp/galleries/lera", { - "url": "002dc81dee5b4a655f0e31ad8349e8903b296df6", - "count": 3, - "keyword": { - "gallery": dict, - "user": dict, - }, - }) + pattern = (BASE_PATTERN + r"/(?!photo/)(?:p/)?" + r"([^/?&#]+)/galleries/([^/?&#]+)") + test = ( + ("https://500px.com/p/fashvamp/galleries/lera", { + "url": "002dc81dee5b4a655f0e31ad8349e8903b296df6", + "count": 3, + "keyword": { + "gallery": dict, + "user": dict, + }, + }), + ("https://500px.com/fashvamp/galleries/lera"), + ) def __init__(self, match): _500pxExtractor.__init__(self, match) self.user_name, self.gallery_name = match.groups() - self.user_id = self.gallery_id = self.csrf_token = None + self.user_id = self._photos = None def metadata(self): - # get csrf token and user id from webpage - url = "{}/{}/galleries/{}".format( - self.root, self.user_name, self.gallery_name) - page = self.request(url).text - self.csrf_token, pos = text.extract(page, 'csrf-token" content="', '"') - self.user_id , pos = text.extract(page, 'App.CuratorId =', '\n', pos) - self.user_id = self.user_id.strip(" '\";") - - # get gallery metadata; transform gallery name into id - url = "https://api.500px.com/v1/users/{}/galleries/{}".format( - self.user_id, self.gallery_name) - params = { - # "include_user": "true", - "include_cover": "1", - "cover_size": "2048", + user = self._request_graphql( + "ProfileRendererQuery", {"username": self.user_name}, + "db1dba2cb7b7e94916d1005db16fea1a39d6211437b691c4de2f1a606c21c5fb", + )["profile"] + self.user_id = str(user["legacyId"]) + + variables = { + "galleryOwnerLegacyId": self.user_id, + "ownerLegacyId" : self.user_id, + "slug" : self.gallery_name, + "token" : None, + "pageSize" : 20, + } + gallery = self._request_graphql( + "GalleriesDetailQueryRendererQuery", variables, + "1afc7dede86ff73456b4defbc5aeb593e330b990943d114cbef7da5be0d7ce2f", + )["gallery"] + + self._photos = gallery["photos"] + del gallery["photos"] + return { + "gallery": gallery, + "user" : user, } - data = self._api_call(url, params, self.csrf_token) - self.gallery_id = data["gallery"]["id"] - return data def photos(self): - url = "https://api.500px.com/v1/users/{}/galleries/{}/items".format( - self.user_id, self.gallery_id) - params = { - "sort" : "position", - "sort_direction" : "asc", - "rpp" : "50", + photos = self._photos + variables = { + "ownerLegacyId": self.user_id, + "slug" : self.gallery_name, + "token" : None, + "pageSize" : 20, } - return self._pagination(url, params, self.csrf_token) + + while True: + yield from self._extend(photos["edges"]) + + if not photos["pageInfo"]["hasNextPage"]: + return + + variables["cursor"] = photos["pageInfo"]["endCursor"] + photos = self._request_graphql( + "GalleriesDetailPaginationContainerQuery", variables, + "3fcbc9ea1589f31c86fc43a0a02c2163" + "cab070f9d376651f270de9f30f031539", + )["galleryByOwnerIdAndSlugOrToken"]["photos"] class _500pxImageExtractor(_500pxExtractor): @@ -226,5 +253,5 @@ class _500pxImageExtractor(_500pxExtractor): self.photo_id = match.group(1) def photos(self): - photos = ({"id": self.photo_id},) - return self._extend(photos) + edges = ({"node": {"legacyId": self.photo_id}},) + return self._extend(edges) diff --git a/gallery_dl/extractor/aryion.py b/gallery_dl/extractor/aryion.py index 04bb146..2e4c4d4 100644 --- a/gallery_dl/extractor/aryion.py +++ b/gallery_dl/extractor/aryion.py @@ -9,7 +9,8 @@ """Extractors for https://aryion.com/""" from .common import Extractor, Message -from .. import text, util +from .. import text, util, exception +from ..cache import cache BASE_PATTERN = r"(?:https?://)?(?:www\.)?aryion\.com/g4" @@ -21,6 +22,8 @@ class AryionExtractor(Extractor): directory_fmt = ("{category}", "{user!l}", "{path:J - }") filename_fmt = "{id} {title}.{extension}" archive_fmt = "{id}" + cookiedomain = ".aryion.com" + cookienames = ("phpbb3_rl7a3_sid",) root = "https://aryion.com" def __init__(self, match): @@ -28,7 +31,30 @@ class AryionExtractor(Extractor): self.user = match.group(1) self.recursive = True + def login(self): + username, password = self._get_auth_info() + if username: + self._update_cookies(self._login_impl(username, password)) + + @cache(maxage=14*24*3600, keyarg=1) + def _login_impl(self, username, password): + self.log.info("Logging in as %s", username) + + url = self.root + "/forum/ucp.php?mode=login" + data = { + "username": username, + "password": password, + "login": "Login", + } + + response = self.request(url, method="POST", data=data) + if b"You have been successfully logged in." not in response.content: + raise exception.AuthenticationError() + return {c: response.cookies[c] for c in self.cookienames} + def items(self): + self.login() + for post_id in self.posts(): post = self._parse_post(post_id) if post: @@ -68,6 +94,7 @@ class AryionExtractor(Extractor): # folder if headers["content-type"] in ( "application/x-folder", + "application/x-comic-folder", "application/x-comic-folder-nomerge", ): return False @@ -184,11 +211,16 @@ class AryionPostExtractor(AryionExtractor): "_mtime" : "Sat, 16 Feb 2019 19:30:34 GMT", }, }), - # folder (#694) + # x-folder (#694) ("https://aryion.com/g4/view/588928", { "pattern": pattern, "count": ">= 8", }), + # x-comic-folder (#945) + ("https://aryion.com/g4/view/537379", { + "pattern": pattern, + "count": 2, + }), ) def posts(self): diff --git a/gallery_dl/extractor/exhentai.py b/gallery_dl/extractor/exhentai.py index 4cb10b4..80c7187 100644 --- a/gallery_dl/extractor/exhentai.py +++ b/gallery_dl/extractor/exhentai.py @@ -47,6 +47,12 @@ class ExhentaiExtractor(Extractor): self.wait_min = self.config("wait-min", 3) self.wait_max = self.config("wait-max", 6) + if type(self.limits) is int: + self._limit_max = self.limits + self.limits = True + else: + self._limit_max = 0 + self._remaining = 0 if self.wait_max < self.wait_min: self.wait_max = self.wait_min @@ -331,6 +337,8 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor): page = self.request(url, cookies=cookies).text current, pos = text.extract(page, "<strong>", "</strong>") maximum, pos = text.extract(page, "<strong>", "</strong>", pos) + if self._limit_max: + maximum = self._limit_max self.log.debug("Image Limits: %s/%s", current, maximum) self._remaining = text.parse_int(maximum) - text.parse_int(current) diff --git a/gallery_dl/extractor/foolslide.py b/gallery_dl/extractor/foolslide.py index 715e294..e624a65 100644 --- a/gallery_dl/extractor/foolslide.py +++ b/gallery_dl/extractor/foolslide.py @@ -208,21 +208,16 @@ EXTRACTORS = { "pattern": r"(?:(?:www\.)?sensescans\.com/reader" r"|reader\.sensescans\.com)", "test-chapter": ( - (("https://sensescans.com/reader/read/" - "magi__labyrinth_of_magic/en/37/369/"), { - "url": "8bbc59a995640bbb944c0b1be06a490909b58be1", - "keyword": "07acd84fb18a9f1fd6dff5befe711bcca0ff9988", - }), - (("https://reader.sensescans.com/read/" - "magi__labyrinth_of_magic/en/37/369/"), { - "url": "8bbc59a995640bbb944c0b1be06a490909b58be1", - "keyword": "07acd84fb18a9f1fd6dff5befe711bcca0ff9988", + ("https://sensescans.com/reader/read/ao_no_orchestra/en/0/26/", { + "url": "bbd428dc578f5055e9f86ad635b510386cd317cd", + "keyword": "083ef6f8831c84127fe4096fa340a249be9d1424", }), + ("https://reader.sensescans.com/read/ao_no_orchestra/en/0/26/"), ), "test-manga": ("https://sensescans.com/reader/series/yotsubato/", { - "url": "ee4dca7c421bf15ac039200f8c0bcb0858153640", - "keyword": "f94961bd731bd878bbd4d48555bc3ace1d937364", + "url": "305e6eb6160e3bb90c3de39ff5fb7c971e052087", + "keyword": "562fb5a7362a4cb43d59d5c8a6ea8080fc65cf99", }), }, "worldthree": { diff --git a/gallery_dl/extractor/furaffinity.py b/gallery_dl/extractor/furaffinity.py index 61226b6..6dfd75d 100644 --- a/gallery_dl/extractor/furaffinity.py +++ b/gallery_dl/extractor/furaffinity.py @@ -30,15 +30,21 @@ class FuraffinityExtractor(Extractor): self.offset = 0 def items(self): + metadata = self.metadata() for post_id in util.advance(self.posts(), self.offset): post = self._parse_post(post_id) if post: + if metadata: + post.update(metadata) yield Message.Directory, post yield Message.Url, post["url"], post def posts(self): return self._pagination() + def metadata(self): + return None + def skip(self, num): self.offset += num return num @@ -133,6 +139,40 @@ class FuraffinityExtractor(Extractor): yield from text.extract_iter(page, 'id="sid-', '"') path = text.extract(page, 'right" href="', '"')[0] + def _pagination_search(self, query): + url = self.root + "/search/" + data = { + "page" : 0, + "next_page" : "Next", + "order-by" : "relevancy", + "order-direction": "desc", + "range" : "all", + "rating-general" : "on", + "rating-mature" : "on", + "rating-adult" : "on", + "type-art" : "on", + "type-music" : "on", + "type-flash" : "on", + "type-story" : "on", + "type-photo" : "on", + "type-poetry" : "on", + "mode" : "extended", + } + data.update(query) + if "page" in query: + data["page"] = text.parse_int(query["page"]) + + while True: + page = self.request(url, method="POST", data=data).text + post_id = None + + for post_id in text.extract_iter(page, 'id="sid-', '"'): + yield post_id + + if not post_id: + return + data["page"] += 1 + class FuraffinityGalleryExtractor(FuraffinityExtractor): """Extractor for a furaffinity user's gallery""" @@ -171,6 +211,25 @@ class FuraffinityFavoriteExtractor(FuraffinityExtractor): return self._pagination_favorites() +class FuraffinitySearchExtractor(FuraffinityExtractor): + """Extractor for furaffinity search results""" + subcategory = "search" + directory_fmt = ("{category}", "Search", "{search}") + pattern = BASE_PATTERN + r"/search/?\?([^#]+)" + test = ("https://www.furaffinity.net/search/?q=cute", { + "pattern": r"https://d.facdn.net/art/[^/]+/\d+/\d+.\w+\.\w+", + "range": "45-50", + "count": 6, + }) + + def metadata(self): + self.query = text.parse_query(self.user) + return {"search": self.query.get("q")} + + def posts(self): + return self._pagination_search(self.query) + + class FuraffinityPostExtractor(FuraffinityExtractor): """Extractor for individual posts on furaffinity""" subcategory = "post" diff --git a/gallery_dl/extractor/gelbooru.py b/gallery_dl/extractor/gelbooru.py index 612c742..edadd31 100644 --- a/gallery_dl/extractor/gelbooru.py +++ b/gallery_dl/extractor/gelbooru.py @@ -55,7 +55,7 @@ class GelbooruExtractor(booru.XmlParserMixin, while True: page = self.request(url, params=params).text - ids = list(text.extract_iter(page, '<a id="p', '"')) + ids = list(text.extract_iter(page, '<span id="s', '"')) yield from ids if len(ids) < self.per_page: return diff --git a/gallery_dl/extractor/hentaihand.py b/gallery_dl/extractor/hentaihand.py index 302999b..7635bf1 100644 --- a/gallery_dl/extractor/hentaihand.py +++ b/gallery_dl/extractor/hentaihand.py @@ -10,74 +10,61 @@ from .common import GalleryExtractor, Extractor, Message from .. import text, util -import collections +import json class HentaihandGalleryExtractor(GalleryExtractor): """Extractor for image galleries on hentaihand.com""" category = "hentaihand" root = "https://hentaihand.com" - pattern = (r"(?i)(?:https?://)?(?:www\.)?hentaihand\.com" - r"/(?:comi|view)c/(\d+)") + pattern = r"(?:https?://)?(?:www\.)?hentaihand\.com/\w+/comic/([\w-]+)" test = ( - ("https://hentaihand.com/comic/272772/kouda-tomohiro-chiyomi-bl", { - "pattern": r"https://i.hentaihand.com/.*/images/full/\d+.jpg$", + (("https://hentaihand.com/en/comic/kouda-tomohiro-chiyomi-" + "blizzard-comic-aun-2016-12-english-nanda-sore-scans"), { + "pattern": r"https://cdn.hentaihand.com/.*/images/304546/\d+.jpg$", "count": 19, "keyword": { - "artists" : ["kouda tomohiro"], - "categories": ["manga"], - "date" : "Feb. 6, 2020, 3:19 p.m.", - "gallery_id": 272772, + "artists" : ["Kouda Tomohiro"], + "date" : "dt:2020-02-06 00:00:00", + "gallery_id": 304546, "lang" : "en", "language" : "English", - "relationships": ["family", "step family"], + "relationships": ["Family", "Step family"], "tags" : list, "title" : r"re:\[Kouda Tomohiro\] Chiyomi Blizzard", - "title_jp" : r"re:\[幸田朋弘\] ちよみブリザード", + "title_alt" : r"re:\[幸田朋弘\] ちよみブリザード", + "type" : "Manga", }, }), - ("https://hentaihand.com/viewc/272772/kouda-tomohiro-chiyomi-bl"), ) def __init__(self, match): - self.gallery_id = match.group(1) - url = "{}/comic/{}".format(self.root, self.gallery_id) + self.slug = match.group(1) + url = "{}/api/comics/{}".format(self.root, self.slug) GalleryExtractor.__init__(self, match, url) def metadata(self, page): - extr = text.extract_from(page) - - title_en = text.unescape(extr("<h1>", "<")) - title_jp = text.unescape(extr("<h2>", "<")) - tags = extr('<section id="tags"', "</section>") - + info = json.loads(page) data = { - "gallery_id" : text.parse_int(self.gallery_id), - "title" : title_en or title_jp, - "title_en" : title_en, - "title_jp" : title_jp, - - # impossible to parse with strptime() - "date" : extr('datetime="', '"'), + "gallery_id" : text.parse_int(info["id"]), + "title" : info["title"], + "title_alt" : info["alternative_title"], + "slug" : self.slug, + "type" : info["category"]["name"], + "language" : info["language"]["name"], + "lang" : util.language_to_code(info["language"]["name"]), + "tags" : [t["slug"] for t in info["tags"]], + "date" : text.parse_datetime( + info["uploaded_at"], "%Y-%m-%d"), } - - tdict = collections.defaultdict(list) - for path in text.extract_iter(tags, 'href="/', '"'): - kind, _, name = path.partition("/") - tdict[kind].append(name.replace("+", " ")) - data.update(tdict) - - if "languages" in data: - data["language"] = data["languages"][-1].capitalize() - data["lang"] = util.language_to_code(data["language"]) - del data["languages"] + for key in ("artists", "authors", "groups", "characters", + "relationships", "parodies"): + data[key] = [v["name"] for v in info[key]] return data def images(self, _): - url = "{}/viewc/{}/1".format(self.root, self.gallery_id) - page = self.request(url).text - images = text.extract(page, "var images", ";")[0] - return [(img, None) for img in text.extract_iter(images, "'", "'")] + info = self.request(self.gallery_url + "/images").json() + return [(img["source_url"], img) for img in info["images"]] class HentaihandTagExtractor(Extractor): @@ -86,49 +73,49 @@ class HentaihandTagExtractor(Extractor): subcategory = "tag" root = "https://hentaihand.com" pattern = (r"(?i)(?:https?://)?(?:www\.)?hentaihand\.com" - r"(/(?:parody|characters|tags|artists|groups|languages" - r"|categories|relationships)/[^#]+)") + r"/\w+/(parody|character|tag|artist|group|language" + r"|category|relationship)/([^/?&#]+)") test = ( - ("https://hentaihand.com/artists/tony+taka", { + ("https://hentaihand.com/en/artist/himuro", { "pattern": HentaihandGalleryExtractor.pattern, - "count": ">= 50", + "count": ">= 18", }), - ("https://hentaihand.com/artists/tony+taka/popular?page=2"), - ("https://hentaihand.com/tags/full+color"), - ("https://hentaihand.com/languages/japanese"), - ("https://hentaihand.com/categories/manga"), + ("https://hentaihand.com/en/tag/full-color"), + ("https://hentaihand.com/fr/language/japanese"), + ("https://hentaihand.com/zh/category/manga"), ) def __init__(self, match): Extractor.__init__(self, match) - self.path, _, query = match.group(1).partition("?") - self.query = text.parse_query(query) - self.query["page"] = text.parse_int(self.query.get("page"), 1) + self.type, self.key = match.groups() def items(self): - yield Message.Version, 1 - url = self.root + self.path - params = self.query.copy() - data = {"_extractor": HentaihandGalleryExtractor} - + if self.type[-1] == "y": + tpl = self.type[:-1] + "ies" + else: + tpl = self.type + "s" + + url = "{}/api/{}/{}".format(self.root, tpl, self.key) + tid = self.request(url, notfound=self.type).json()["id"] + + url = self.root + "/api/comics" + params = { + "per_page": "18", + tpl : tid, + "page" : 1, + "q" : "", + "sort" : "uploaded_at", + "order" : "desc", + "duration": "day", + } while True: - page = self.request(url, params=params).text + info = self.request(url, params=params).json() - for path in text.extract_iter(page, '<a href="/comic/', '"'): - yield Message.Queue, self.root + "/comic/" + path, data + for gallery in info["data"]: + gurl = "{}/en/comic/{}".format(self.root, gallery["slug"]) + gallery["_extractor"] = HentaihandGalleryExtractor + yield Message.Queue, gurl, gallery - pos = page.find(">(current)<") - if pos < 0 or page.find('class="page-link" href="', pos) < 0: - break + if params["page"] >= info["last_page"]: + return params["page"] += 1 - - -class HentaihandSearchExtractor(HentaihandTagExtractor): - """Extractor for search results on hentaihand.com""" - subcategory = "search" - pattern = r"(?i)(?:https?://)?(?:www\.)?hentaihand\.com(/search/?[^#]+)" - test = ("https://hentaihand.com/search?q=color", { - "pattern": HentaihandGalleryExtractor.pattern, - "range": "1-50", - "count": 50, - }) diff --git a/gallery_dl/extractor/hitomi.py b/gallery_dl/extractor/hitomi.py index da53113..209a4f2 100644 --- a/gallery_dl/extractor/hitomi.py +++ b/gallery_dl/extractor/hitomi.py @@ -111,9 +111,9 @@ class HitomiGalleryExtractor(GalleryExtractor): response = self.request(url, fatal=False) if b"<title>Redirect</title>" not in response.content: break - url = text.extract(response.text, "href='", "'")[0] - if not url.startswith("http"): - url = text.urljoin(self.root, url) + url = text.extract( + response.text, 'http-equiv="refresh" content="', '"', + )[0].partition("=")[2] if response.status_code >= 400: return {} @@ -158,9 +158,9 @@ class HitomiTagExtractor(Extractor): subcategory = "tag" pattern = (r"(?:https?://)?hitomi\.la/" r"(tag|artist|group|series|type|character)/" - r"([^/?&#]+)-\d+\.html") + r"([^/?&#]+)\.html") test = ( - ("https://hitomi.la/tag/screenshots-japanese-1.html", { + ("https://hitomi.la/tag/screenshots-japanese.html", { "pattern": HitomiGalleryExtractor.pattern, "count": ">= 35", }), @@ -175,6 +175,10 @@ class HitomiTagExtractor(Extractor): Extractor.__init__(self, match) self.type, self.tag = match.groups() + tag, _, num = self.tag.rpartition("-") + if num.isdecimal(): + self.tag = tag + def items(self): url = "https://ltn.hitomi.la/{}/{}.nozomi".format(self.type, self.tag) data = {"_extractor": HitomiGalleryExtractor} diff --git a/gallery_dl/extractor/imgur.py b/gallery_dl/extractor/imgur.py index 25328ab..190a4ff 100644 --- a/gallery_dl/extractor/imgur.py +++ b/gallery_dl/extractor/imgur.py @@ -59,7 +59,7 @@ class ImgurImageExtractor(ImgurExtractor): subcategory = "image" filename_fmt = "{category}_{id}{title:?_//}.{extension}" archive_fmt = "{id}" - pattern = BASE_PATTERN + r"/(?!gallery)(\w{7}|\w{5})[sbtmlh]?\.?" + pattern = BASE_PATTERN + r"/(?!gallery|search)(\w{7}|\w{5})[sbtmlh]?\.?" test = ( ("https://imgur.com/21yMxCS", { "url": "6f2dcfb86815bdd72808c313e5f715610bc7b9b2", @@ -304,8 +304,40 @@ class ImgurSubredditExtractor(ImgurExtractor): return self._items_queue(self.api.gallery_subreddit(self.key)) +class ImgurTagExtractor(ImgurExtractor): + """Extractor for imgur tag searches""" + subcategory = "tag" + pattern = BASE_PATTERN + r"/t/([^/?&#]+)$" + test = ("https://imgur.com/t/animals", { + "range": "1-100", + "count": 100, + "pattern": r"https?://(i.imgur.com|imgur.com/a)/[\w.]+", + }) + + def items(self): + return self._items_queue(self.api.gallery_tag(self.key)) + + +class ImgurSearchExtractor(ImgurExtractor): + """Extractor for imgur search results""" + subcategory = "search" + pattern = BASE_PATTERN + r"/search(?:/[^?&#]+)?/?\?q=([^&#]+)" + test = ("https://imgur.com/search?q=cute+cat", { + "range": "1-100", + "count": 100, + "pattern": r"https?://(i.imgur.com|imgur.com/a)/[\w.]+", + }) + + def items(self): + key = text.unquote(self.key.replace("+", " ")) + return self._items_queue(self.api.gallery_search(key)) + + class ImgurAPI(): + """Interface for the Imgur API + Ref: https://apidocs.imgur.com/ + """ def __init__(self, extractor): self.extractor = extractor self.headers = { @@ -317,6 +349,11 @@ class ImgurAPI(): endpoint = "account/{}/gallery_favorites".format(account) return self._pagination(endpoint) + def gallery_search(self, query): + endpoint = "gallery/search" + params = {"q": query} + return self._pagination(endpoint, params) + def account_submissions(self, account): endpoint = "account/{}/submissions".format(account) return self._pagination(endpoint) @@ -325,16 +362,21 @@ class ImgurAPI(): endpoint = "gallery/r/{}".format(subreddit) return self._pagination(endpoint) + def gallery_tag(self, tag): + endpoint = "gallery/t/{}".format(tag) + return self._pagination(endpoint, key="items") + def album(self, album_hash): return self._call("album/" + album_hash) def image(self, image_hash): return self._call("image/" + image_hash) - def _call(self, endpoint): + def _call(self, endpoint, params=None): try: return self.extractor.request( - "https://api.imgur.com/3/" + endpoint, headers=self.headers, + "https://api.imgur.com/3/" + endpoint, + params=params, headers=self.headers, ).json()["data"] except exception.HttpError as exc: if exc.status != 403 or b"capacity" not in exc.response.content: @@ -342,11 +384,13 @@ class ImgurAPI(): self.extractor.sleep(seconds=600) return self._call(endpoint) - def _pagination(self, endpoint): + def _pagination(self, endpoint, params=None, key=None): num = 0 while True: - data = self._call("{}/{}".format(endpoint, num)) + data = self._call("{}/{}".format(endpoint, num), params) + if key: + data = data[key] if not data: return yield from data diff --git a/gallery_dl/extractor/mangapark.py b/gallery_dl/extractor/mangapark.py index 3d64acd..59a046c 100644 --- a/gallery_dl/extractor/mangapark.py +++ b/gallery_dl/extractor/mangapark.py @@ -53,18 +53,19 @@ class MangaparkChapterExtractor(MangaparkBase, ChapterExtractor): pattern = (r"(?:https?://)?(?:www\.)?mangapark\.(me|net|com)" r"/manga/([^?&#]+/i\d+)") test = ( - ("https://mangapark.net/manga/gosu/i811615/c55/1", { + ("https://mangapark.net/manga/gosu/i811653/c055/1", { "count": 50, - "keyword": "2bb16a50dbac9577ead62b41db9a01a0419c0ae2", + "keyword": "8344bdda8cd8414e7729a4e148379f147e3437da", }), (("https://mangapark.net/manga" - "/ad-astra-per-aspera-hata-kenjirou/i662054/c001.2/1"), { + "/ad-astra-per-aspera-hata-kenjirou/i662051/c001.2/1"), { "count": 40, - "keyword": "8e9cce4ed0e25d12a45e02f840d6f32ef838e257", + "keyword": "2bb3a8f426383ea13f17ff5582f3070d096d30ac", }), - ("https://mangapark.net/manga/gekkan-shoujo-nozaki-kun/i655476/c70", { + (("https://mangapark.net/manga" + "/gekkan-shoujo-nozaki-kun/i2067426/v7/c70/1"), { "count": 15, - "keyword": "19f730617074d65f91c0781f429de324890925bf", + "keyword": "edc14993c4752cee3a76e09b2f024d40d854bfd1", }), ("https://mangapark.me/manga/gosu/i811615/c55/1"), ("https://mangapark.com/manga/gosu/i811615/c55/1"), @@ -119,8 +120,8 @@ class MangaparkMangaExtractor(MangaparkBase, MangaExtractor): r"(/manga/[^/?&#]+)/?$") test = ( ("https://mangapark.net/manga/aria", { - "url": "9b0b31e4992260876f56d7bfc8ff0ae71295c4f4", - "keyword": "6e44744a28d01b889b1e8291847abd84b591590d", + "url": "9b62883c25c8de471f8ab43651e1448536c4ce3f", + "keyword": "eb4a9b273c69acf31efa731eba713e1cfa14bab6", }), ("https://mangapark.me/manga/aria"), ("https://mangapark.com/manga/aria"), diff --git a/gallery_dl/extractor/reddit.py b/gallery_dl/extractor/reddit.py index cb70fe5..9c6892a 100644 --- a/gallery_dl/extractor/reddit.py +++ b/gallery_dl/extractor/reddit.py @@ -57,12 +57,8 @@ class RedditExtractor(Extractor): yield Message.Url, url, submission elif "gallery_data" in submission: - meta = submission["media_metadata"] - items = submission["gallery_data"]["items"] - for submission["num"], item in enumerate(items, 1): - url = meta[item["media_id"]]["s"]["u"] - url = url.partition("?")[0] - url = url.replace("/preview.", "/i.", 1) + for submission["num"], url in enumerate( + self._extract_gallery(submission), 1): text.nameext_from_url(url, submission) yield Message.Url, url, submission @@ -118,6 +114,22 @@ class RedditExtractor(Extractor): def submissions(self): """Return an iterable containing all (submission, comments) tuples""" + def _extract_gallery(self, submission): + if submission["gallery_data"] is None: + self.log.warning("gallery '%s' was deleted", submission["id"]) + return + + meta = submission["media_metadata"] + for item in submission["gallery_data"]["items"]: + src = meta[item["media_id"]]["s"] + url = src.get("u") or src.get("gif") or src.get("mp4") + if url: + yield url.partition("?")[0].replace("/preview.", "/i.", 1) + else: + self.log.error("Unable to get download URL for item '%s'", + item["media_id"]) + self.log.debug(src) + class RedditSubredditExtractor(RedditExtractor): """Extractor for URLs from subreddits on reddit.com""" @@ -188,6 +200,15 @@ class RedditSubmissionExtractor(RedditExtractor): "content": "1e7dde4ee7d5f4c4b45749abfd15b2dbfa27df3f", "count": 3, }), + # deleted gallery (#953) + ("https://www.reddit.com/gallery/icfgzv", { + "count": 0, + }), + # animated gallery items (#955) + ("https://www.reddit.com/r/araragi/comments/ib32hm", { + "pattern": r"https://i\.redd\.it/\w+\.gif", + "count": 2, + }), ("https://old.reddit.com/r/lavaporn/comments/2a00np/"), ("https://np.reddit.com/r/lavaporn/comments/2a00np/"), ("https://m.reddit.com/r/lavaporn/comments/2a00np/"), diff --git a/gallery_dl/extractor/redgifs.py b/gallery_dl/extractor/redgifs.py index 4477825..0f02e8b 100644 --- a/gallery_dl/extractor/redgifs.py +++ b/gallery_dl/extractor/redgifs.py @@ -58,7 +58,8 @@ class RedgifsImageExtractor(RedgifsExtractor): r"|gifdeliverynetwork.com)/([A-Za-z]+)") test = ( ("https://redgifs.com/watch/foolishforkedabyssiniancat", { - "pattern": r"https://\w+.redgifs.com/FoolishForkedAbyss.+.mp4", + "pattern": r"https://\w+\.(redgifs|gfycat)\.com" + r"/FoolishForkedAbyssiniancat\.mp4", "content": "f6e03f1df9a2ff2a74092f53ee7580d2fb943533", }), ("https://www.gifdeliverynetwork.com/foolishforkedabyssiniancat"), diff --git a/gallery_dl/extractor/subscribestar.py b/gallery_dl/extractor/subscribestar.py index 076d0c0..38b39d4 100644 --- a/gallery_dl/extractor/subscribestar.py +++ b/gallery_dl/extractor/subscribestar.py @@ -147,13 +147,13 @@ class SubscribestarUserExtractor(SubscribestarExtractor): "author_nick": "SubscribeStar", "content": str, "date" : "type:datetime", - "height" : int, "id" : int, - "pinned" : bool, "post_id": int, - "type" : "re:image|video", + "type" : "re:image|video|attachment", "url" : str, - "width" : int, + "?pinned": bool, + "?height": int, + "?width" : int, }, }), ("https://www.subscribestar.com/subscribestar", { diff --git a/gallery_dl/extractor/xvideos.py b/gallery_dl/extractor/xvideos.py index 80a3614..2548ead 100644 --- a/gallery_dl/extractor/xvideos.py +++ b/gallery_dl/extractor/xvideos.py @@ -31,7 +31,7 @@ class XvideosGalleryExtractor(XvideosBase, GalleryExtractor): r"/([^/?&#]+)/photos/(\d+)") test = ( ("https://www.xvideos.com/profiles/pervertedcouple/photos/751031", { - "url": "4f0d992e5dc39def2c3ac8e099d17bf09e76e3c7", + "url": "cb4657a37eea5ab6b1d333491cee7eeb529b0645", "keyword": { "gallery": { "id" : 751031, diff --git a/gallery_dl/version.py b/gallery_dl/version.py index b2b59e0..9af9a43 100644 --- a/gallery_dl/version.py +++ b/gallery_dl/version.py @@ -6,4 +6,4 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -__version__ = "1.14.4" +__version__ = "1.14.5" diff --git a/test/test_results.py b/test/test_results.py index 1f2f699..fbbb79c 100644 --- a/test/test_results.py +++ b/test/test_results.py @@ -26,15 +26,13 @@ TRAVIS_SKIP = { "archivedmoe", "archiveofsins", "thebarchive", "fireden", "4plebs", "sankaku", "idolcomplex", "mangahere", "readcomiconline", "mangadex", "sankakucomplex", "warosu", "fuskator", "patreon", "komikcast", - "instagram", + "instagram", "ngomik", } # temporary issues, etc. BROKEN = { - "hentaihand", + "dokireader", "imagevenue", - "mangapark", - "ngomik", "photobucket", "worldthree", } |