aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLibravatarUnit 193 <unit193@unit193.net>2020-09-01 18:58:25 -0400
committerLibravatarUnit 193 <unit193@unit193.net>2020-09-01 18:58:25 -0400
commit261c8c2bc74969e2242a153297895684742b6995 (patch)
tree86b6196089f2820874d3cf92e544dfa29cd334c5
parent7cf59dc17c3607e096292462ed15d391be4e3dfd (diff)
downloadgallery-dl-261c8c2bc74969e2242a153297895684742b6995.tar.bz2
gallery-dl-261c8c2bc74969e2242a153297895684742b6995.tar.xz
gallery-dl-261c8c2bc74969e2242a153297895684742b6995.tar.zst
New upstream version 1.14.5.upstream/1.14.5
-rw-r--r--CHANGELOG.md15
-rw-r--r--PKG-INFO10
-rw-r--r--README.rst8
-rw-r--r--data/man/gallery-dl.12
-rw-r--r--data/man/gallery-dl.conf.59
-rw-r--r--docs/gallery-dl.conf5
-rw-r--r--gallery_dl.egg-info/PKG-INFO10
-rw-r--r--gallery_dl/extractor/500px.py171
-rw-r--r--gallery_dl/extractor/aryion.py36
-rw-r--r--gallery_dl/extractor/exhentai.py8
-rw-r--r--gallery_dl/extractor/foolslide.py17
-rw-r--r--gallery_dl/extractor/furaffinity.py59
-rw-r--r--gallery_dl/extractor/gelbooru.py2
-rw-r--r--gallery_dl/extractor/hentaihand.py137
-rw-r--r--gallery_dl/extractor/hitomi.py14
-rw-r--r--gallery_dl/extractor/imgur.py54
-rw-r--r--gallery_dl/extractor/mangapark.py17
-rw-r--r--gallery_dl/extractor/reddit.py33
-rw-r--r--gallery_dl/extractor/redgifs.py3
-rw-r--r--gallery_dl/extractor/subscribestar.py8
-rw-r--r--gallery_dl/extractor/xvideos.py2
-rw-r--r--gallery_dl/version.py2
-rw-r--r--test/test_results.py6
23 files changed, 415 insertions, 213 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fa9f17c..b38c9c8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,20 @@
# Changelog
+## 1.14.5 - 2020-08-30
+### Additions
+- [aryion] add username/password support ([#960](https://github.com/mikf/gallery-dl/issues/960))
+- [exhentai] add ability to specify a custom image limit ([#940](https://github.com/mikf/gallery-dl/issues/940))
+- [furaffinity] add `search` extractor ([#915](https://github.com/mikf/gallery-dl/issues/915))
+- [imgur] add `search` and `tag` extractors ([#934](https://github.com/mikf/gallery-dl/issues/934))
+### Fixes
+- [500px] fix extraction and update URL patterns ([#956](https://github.com/mikf/gallery-dl/issues/956))
+- [aryion] update folder mime type list ([#945](https://github.com/mikf/gallery-dl/issues/945))
+- [gelbooru] fix extraction without API
+- [hentaihand] update to new site layout
+- [hitomi] fix redirect processing
+- [reddit] handle deleted galleries ([#953](https://github.com/mikf/gallery-dl/issues/953))
+- [reddit] improve gallery extraction ([#955](https://github.com/mikf/gallery-dl/issues/955))
+
## 1.14.4 - 2020-08-15
### Additions
- [blogger] add `search` extractor ([#925](https://github.com/mikf/gallery-dl/issues/925))
diff --git a/PKG-INFO b/PKG-INFO
index afc4636..644b647 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: gallery_dl
-Version: 1.14.4
+Version: 1.14.5
Summary: Command-line program to download image-galleries and -collections from several image hosting sites
Home-page: https://github.com/mikf/gallery-dl
Author: Mike Fährmann
@@ -94,8 +94,8 @@ Description: ==========
put it into your `PATH <https://en.wikipedia.org/wiki/PATH_(variable)>`__,
and run it inside a command prompt (like ``cmd.exe``).
- - `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.14.4/gallery-dl.exe>`__
- - `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.14.4/gallery-dl.bin>`__
+ - `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.14.5/gallery-dl.exe>`__
+ - `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.14.5/gallery-dl.bin>`__
These executables include a Python 3.8 interpreter
and all required Python packages.
@@ -216,7 +216,7 @@ Description: ==========
a username & password pair. This is necessary for
``pixiv``, ``nijie``, and ``seiga``
and optional for
- ``danbooru``, ``e621``, ``exhentai``, ``idolcomplex``, ``inkbunny``,
+ ``aryion``, ``danbooru``, ``e621``, ``exhentai``, ``idolcomplex``, ``inkbunny``,
``instagram``, ``luscious``, ``sankaku``, ``subscribestar``, ``tsumino``,
and ``twitter``.
@@ -311,7 +311,7 @@ Description: ==========
.. _gallery-dl-example.conf: https://github.com/mikf/gallery-dl/blob/master/docs/gallery-dl-example.conf
.. _configuration.rst: https://github.com/mikf/gallery-dl/blob/master/docs/configuration.rst
.. _Supported Sites: https://github.com/mikf/gallery-dl/blob/master/docs/supportedsites.rst
- .. _stable: https://github.com/mikf/gallery-dl/archive/v1.14.4.tar.gz
+ .. _stable: https://github.com/mikf/gallery-dl/archive/v1.14.5.tar.gz
.. _dev: https://github.com/mikf/gallery-dl/archive/master.tar.gz
.. _Python: https://www.python.org/downloads/
diff --git a/README.rst b/README.rst
index 2148c42..6f5c4bb 100644
--- a/README.rst
+++ b/README.rst
@@ -83,8 +83,8 @@ Download a standalone executable file,
put it into your `PATH <https://en.wikipedia.org/wiki/PATH_(variable)>`__,
and run it inside a command prompt (like ``cmd.exe``).
-- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.14.4/gallery-dl.exe>`__
-- `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.14.4/gallery-dl.bin>`__
+- `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.14.5/gallery-dl.exe>`__
+- `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.14.5/gallery-dl.bin>`__
These executables include a Python 3.8 interpreter
and all required Python packages.
@@ -205,7 +205,7 @@ Some extractors require you to provide valid login credentials in the form of
a username & password pair. This is necessary for
``pixiv``, ``nijie``, and ``seiga``
and optional for
-``danbooru``, ``e621``, ``exhentai``, ``idolcomplex``, ``inkbunny``,
+``aryion``, ``danbooru``, ``e621``, ``exhentai``, ``idolcomplex``, ``inkbunny``,
``instagram``, ``luscious``, ``sankaku``, ``subscribestar``, ``tsumino``,
and ``twitter``.
@@ -300,7 +300,7 @@ access to *gallery-dl*. Authorize it and you will be shown one or more
.. _gallery-dl-example.conf: https://github.com/mikf/gallery-dl/blob/master/docs/gallery-dl-example.conf
.. _configuration.rst: https://github.com/mikf/gallery-dl/blob/master/docs/configuration.rst
.. _Supported Sites: https://github.com/mikf/gallery-dl/blob/master/docs/supportedsites.rst
-.. _stable: https://github.com/mikf/gallery-dl/archive/v1.14.4.tar.gz
+.. _stable: https://github.com/mikf/gallery-dl/archive/v1.14.5.tar.gz
.. _dev: https://github.com/mikf/gallery-dl/archive/master.tar.gz
.. _Python: https://www.python.org/downloads/
diff --git a/data/man/gallery-dl.1 b/data/man/gallery-dl.1
index e554159..2437195 100644
--- a/data/man/gallery-dl.1
+++ b/data/man/gallery-dl.1
@@ -1,4 +1,4 @@
-.TH "GALLERY-DL" "1" "2020-08-15" "1.14.4" "gallery-dl Manual"
+.TH "GALLERY-DL" "1" "2020-08-30" "1.14.5" "gallery-dl Manual"
.\" disable hyphenation
.nh
diff --git a/data/man/gallery-dl.conf.5 b/data/man/gallery-dl.conf.5
index 67e51d4..a5b1f4d 100644
--- a/data/man/gallery-dl.conf.5
+++ b/data/man/gallery-dl.conf.5
@@ -1,4 +1,4 @@
-.TH "GALLERY-DL.CONF" "5" "2020-08-15" "1.14.4" "gallery-dl Manual"
+.TH "GALLERY-DL.CONF" "5" "2020-08-30" "1.14.5" "gallery-dl Manual"
.\" disable hyphenation
.nh
.\" disable justification (adjust text to left margin only)
@@ -276,6 +276,8 @@ Specifying a username and password is required for
and optional for
.br
+* \f[I]aryion\f[]
+.br
* \f[I]danbooru\f[]
.br
* \f[I]e621\f[]
@@ -858,7 +860,7 @@ depending on the input URL
.SS extractor.exhentai.limits
.IP "Type:" 6
-\f[I]bool\f[]
+\f[I]bool\f[] or \f[I]integer\f[]
.IP "Default:" 9
\f[I]true\f[]
@@ -867,6 +869,9 @@ depending on the input URL
Check image download limits
and stop extraction when they are exceeded.
+If this value is an \f[I]integer\f[], it gets used as the limit maximum
+instead of the value listed on \f[I]https://e-hentai.org/home.php\f[]
+
.SS extractor.exhentai.original
.IP "Type:" 6
\f[I]bool\f[]
diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf
index 2db802d..56147e9 100644
--- a/docs/gallery-dl.conf
+++ b/docs/gallery-dl.conf
@@ -18,6 +18,11 @@
{
"external": false
},
+ "aryion":
+ {
+ "username": null,
+ "password": null
+ },
"blogger":
{
"videos": true
diff --git a/gallery_dl.egg-info/PKG-INFO b/gallery_dl.egg-info/PKG-INFO
index 8f6f112..a2fafb1 100644
--- a/gallery_dl.egg-info/PKG-INFO
+++ b/gallery_dl.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 2.1
Name: gallery-dl
-Version: 1.14.4
+Version: 1.14.5
Summary: Command-line program to download image-galleries and -collections from several image hosting sites
Home-page: https://github.com/mikf/gallery-dl
Author: Mike Fährmann
@@ -94,8 +94,8 @@ Description: ==========
put it into your `PATH <https://en.wikipedia.org/wiki/PATH_(variable)>`__,
and run it inside a command prompt (like ``cmd.exe``).
- - `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.14.4/gallery-dl.exe>`__
- - `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.14.4/gallery-dl.bin>`__
+ - `Windows <https://github.com/mikf/gallery-dl/releases/download/v1.14.5/gallery-dl.exe>`__
+ - `Linux <https://github.com/mikf/gallery-dl/releases/download/v1.14.5/gallery-dl.bin>`__
These executables include a Python 3.8 interpreter
and all required Python packages.
@@ -216,7 +216,7 @@ Description: ==========
a username & password pair. This is necessary for
``pixiv``, ``nijie``, and ``seiga``
and optional for
- ``danbooru``, ``e621``, ``exhentai``, ``idolcomplex``, ``inkbunny``,
+ ``aryion``, ``danbooru``, ``e621``, ``exhentai``, ``idolcomplex``, ``inkbunny``,
``instagram``, ``luscious``, ``sankaku``, ``subscribestar``, ``tsumino``,
and ``twitter``.
@@ -311,7 +311,7 @@ Description: ==========
.. _gallery-dl-example.conf: https://github.com/mikf/gallery-dl/blob/master/docs/gallery-dl-example.conf
.. _configuration.rst: https://github.com/mikf/gallery-dl/blob/master/docs/configuration.rst
.. _Supported Sites: https://github.com/mikf/gallery-dl/blob/master/docs/supportedsites.rst
- .. _stable: https://github.com/mikf/gallery-dl/archive/v1.14.4.tar.gz
+ .. _stable: https://github.com/mikf/gallery-dl/archive/v1.14.5.tar.gz
.. _dev: https://github.com/mikf/gallery-dl/archive/master.tar.gz
.. _Python: https://www.python.org/downloads/
diff --git a/gallery_dl/extractor/500px.py b/gallery_dl/extractor/500px.py
index 7ecdef7..96cb021 100644
--- a/gallery_dl/extractor/500px.py
+++ b/gallery_dl/extractor/500px.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
-# Copyright 2019 Mike Fährmann
+# Copyright 2019-2020 Mike Fährmann
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -9,7 +9,7 @@
"""Extractors for https://500px.com/"""
from .common import Extractor, Message
-from .. import text
+import json
BASE_PATTERN = r"(?:https?://)?(?:web\.)?500px\.com"
@@ -48,7 +48,7 @@ class _500pxExtractor(Extractor):
def photos(self):
"""Returns an iterable containing all relevant photo IDs"""
- def _extend(self, photos):
+ def _extend(self, edges):
"""Extend photos with additional metadata and higher resolution URLs"""
url = "https://api.500px.com/v1/photos"
params = {
@@ -62,40 +62,42 @@ class _500pxExtractor(Extractor):
"liked_by" : "1",
"following_sample" : "100",
"image_size" : "4096",
- "ids" : ",".join(str(p["id"]) for p in photos),
+ "ids" : ",".join(
+ str(edge["node"]["legacyId"]) for edge in edges),
}
- data = self._api_call(url, params)["photos"]
- for photo in photos:
- pid = str(photo["id"])
- photo.update(data[pid])
- return photos
+ data = self._request_api(url, params)["photos"]
+ return [
+ data[str(edge["node"]["legacyId"])]
+ for edge in edges
+ ]
- def _api_call(self, url, params, csrf_token=None):
+ def _request_api(self, url, params, csrf_token=None):
headers = {"Origin": self.root, "X-CSRF-Token": csrf_token}
return self.request(url, headers=headers, params=params).json()
- def _pagination(self, url, params, csrf):
- params["page"] = 1
- while True:
- data = self._api_call(url, params, csrf)
- yield from self._extend(data["photos"])
-
- if params["page"] >= data["total_pages"]:
- return
- params["page"] += 1
+ def _request_graphql(self, opname, variables, query_hash):
+ url = "https://api.500px.com/graphql"
+ params = {
+ "operationName": opname,
+ "variables" : json.dumps(variables),
+ "extensions" : '{"persistedQuery":{"version":1'
+ ',"sha256Hash":"' + query_hash + '"}}',
+ }
+ return self.request(url, params=params).json()["data"]
class _500pxUserExtractor(_500pxExtractor):
"""Extractor for photos from a user's photostream on 500px.com"""
subcategory = "user"
- pattern = BASE_PATTERN + r"/(?!photo/)([^/?&#]+)/?(?:$|\?|#)"
+ pattern = BASE_PATTERN + r"/(?!photo/)(?:p/)?([^/?&#]+)/?(?:$|\?|#)"
test = (
- ("https://500px.com/light_expression_photography", {
+ ("https://500px.com/p/light_expression_photography", {
"pattern": r"https?://drscdn.500px.org/photo/\d+/m%3D4096/v2",
"range": "1-99",
"count": 99,
}),
+ ("https://500px.com/light_expression_photography"),
("https://web.500px.com/light_expression_photography"),
)
@@ -104,72 +106,97 @@ class _500pxUserExtractor(_500pxExtractor):
self.user = match.group(1)
def photos(self):
- # get csrf token and user id from webpage
- url = "{}/{}".format(self.root, self.user)
- page = self.request(url).text
- csrf_token, pos = text.extract(page, 'csrf-token" content="', '"')
- user_id , pos = text.extract(page, '/user/', '"', pos)
+ variables = {"username": self.user, "pageSize": 20}
+ photos = self._request_graphql(
+ "OtherPhotosQuery", variables,
+ "54524abbdc809f8d4e10d37839e8ab2d"
+ "3035413688cad9c7fbece13b66637e9d",
+ )["user"]["photos"]
- # get user photos
- url = "https://api.500px.com/v1/photos"
- params = {
- "feature" : "user",
- "stream" : "photos",
- "rpp" : "50",
- "user_id" : user_id,
- }
- return self._pagination(url, params, csrf_token)
+ while True:
+ yield from self._extend(photos["edges"])
+
+ if not photos["pageInfo"]["hasNextPage"]:
+ return
+
+ variables["cursor"] = photos["pageInfo"]["endCursor"]
+ photos = self._request_graphql(
+ "OtherPhotosPaginationContainerQuery", variables,
+ "6d31e01104456ce642a2c6fc2f936812"
+ "b0f2a65c442d03e1521d769c20efe507",
+ )["userByUsername"]["photos"]
class _500pxGalleryExtractor(_500pxExtractor):
"""Extractor for photo galleries on 500px.com"""
subcategory = "gallery"
directory_fmt = ("{category}", "{user[username]}", "{gallery[name]}")
- pattern = BASE_PATTERN + r"/(?!photo/)([^/?&#]+)/galleries/([^/?&#]+)"
- test = ("https://500px.com/fashvamp/galleries/lera", {
- "url": "002dc81dee5b4a655f0e31ad8349e8903b296df6",
- "count": 3,
- "keyword": {
- "gallery": dict,
- "user": dict,
- },
- })
+ pattern = (BASE_PATTERN + r"/(?!photo/)(?:p/)?"
+ r"([^/?&#]+)/galleries/([^/?&#]+)")
+ test = (
+ ("https://500px.com/p/fashvamp/galleries/lera", {
+ "url": "002dc81dee5b4a655f0e31ad8349e8903b296df6",
+ "count": 3,
+ "keyword": {
+ "gallery": dict,
+ "user": dict,
+ },
+ }),
+ ("https://500px.com/fashvamp/galleries/lera"),
+ )
def __init__(self, match):
_500pxExtractor.__init__(self, match)
self.user_name, self.gallery_name = match.groups()
- self.user_id = self.gallery_id = self.csrf_token = None
+ self.user_id = self._photos = None
def metadata(self):
- # get csrf token and user id from webpage
- url = "{}/{}/galleries/{}".format(
- self.root, self.user_name, self.gallery_name)
- page = self.request(url).text
- self.csrf_token, pos = text.extract(page, 'csrf-token" content="', '"')
- self.user_id , pos = text.extract(page, 'App.CuratorId =', '\n', pos)
- self.user_id = self.user_id.strip(" '\";")
-
- # get gallery metadata; transform gallery name into id
- url = "https://api.500px.com/v1/users/{}/galleries/{}".format(
- self.user_id, self.gallery_name)
- params = {
- # "include_user": "true",
- "include_cover": "1",
- "cover_size": "2048",
+ user = self._request_graphql(
+ "ProfileRendererQuery", {"username": self.user_name},
+ "db1dba2cb7b7e94916d1005db16fea1a39d6211437b691c4de2f1a606c21c5fb",
+ )["profile"]
+ self.user_id = str(user["legacyId"])
+
+ variables = {
+ "galleryOwnerLegacyId": self.user_id,
+ "ownerLegacyId" : self.user_id,
+ "slug" : self.gallery_name,
+ "token" : None,
+ "pageSize" : 20,
+ }
+ gallery = self._request_graphql(
+ "GalleriesDetailQueryRendererQuery", variables,
+ "1afc7dede86ff73456b4defbc5aeb593e330b990943d114cbef7da5be0d7ce2f",
+ )["gallery"]
+
+ self._photos = gallery["photos"]
+ del gallery["photos"]
+ return {
+ "gallery": gallery,
+ "user" : user,
}
- data = self._api_call(url, params, self.csrf_token)
- self.gallery_id = data["gallery"]["id"]
- return data
def photos(self):
- url = "https://api.500px.com/v1/users/{}/galleries/{}/items".format(
- self.user_id, self.gallery_id)
- params = {
- "sort" : "position",
- "sort_direction" : "asc",
- "rpp" : "50",
+ photos = self._photos
+ variables = {
+ "ownerLegacyId": self.user_id,
+ "slug" : self.gallery_name,
+ "token" : None,
+ "pageSize" : 20,
}
- return self._pagination(url, params, self.csrf_token)
+
+ while True:
+ yield from self._extend(photos["edges"])
+
+ if not photos["pageInfo"]["hasNextPage"]:
+ return
+
+ variables["cursor"] = photos["pageInfo"]["endCursor"]
+ photos = self._request_graphql(
+ "GalleriesDetailPaginationContainerQuery", variables,
+ "3fcbc9ea1589f31c86fc43a0a02c2163"
+ "cab070f9d376651f270de9f30f031539",
+ )["galleryByOwnerIdAndSlugOrToken"]["photos"]
class _500pxImageExtractor(_500pxExtractor):
@@ -226,5 +253,5 @@ class _500pxImageExtractor(_500pxExtractor):
self.photo_id = match.group(1)
def photos(self):
- photos = ({"id": self.photo_id},)
- return self._extend(photos)
+ edges = ({"node": {"legacyId": self.photo_id}},)
+ return self._extend(edges)
diff --git a/gallery_dl/extractor/aryion.py b/gallery_dl/extractor/aryion.py
index 04bb146..2e4c4d4 100644
--- a/gallery_dl/extractor/aryion.py
+++ b/gallery_dl/extractor/aryion.py
@@ -9,7 +9,8 @@
"""Extractors for https://aryion.com/"""
from .common import Extractor, Message
-from .. import text, util
+from .. import text, util, exception
+from ..cache import cache
BASE_PATTERN = r"(?:https?://)?(?:www\.)?aryion\.com/g4"
@@ -21,6 +22,8 @@ class AryionExtractor(Extractor):
directory_fmt = ("{category}", "{user!l}", "{path:J - }")
filename_fmt = "{id} {title}.{extension}"
archive_fmt = "{id}"
+ cookiedomain = ".aryion.com"
+ cookienames = ("phpbb3_rl7a3_sid",)
root = "https://aryion.com"
def __init__(self, match):
@@ -28,7 +31,30 @@ class AryionExtractor(Extractor):
self.user = match.group(1)
self.recursive = True
+ def login(self):
+ username, password = self._get_auth_info()
+ if username:
+ self._update_cookies(self._login_impl(username, password))
+
+ @cache(maxage=14*24*3600, keyarg=1)
+ def _login_impl(self, username, password):
+ self.log.info("Logging in as %s", username)
+
+ url = self.root + "/forum/ucp.php?mode=login"
+ data = {
+ "username": username,
+ "password": password,
+ "login": "Login",
+ }
+
+ response = self.request(url, method="POST", data=data)
+ if b"You have been successfully logged in." not in response.content:
+ raise exception.AuthenticationError()
+ return {c: response.cookies[c] for c in self.cookienames}
+
def items(self):
+ self.login()
+
for post_id in self.posts():
post = self._parse_post(post_id)
if post:
@@ -68,6 +94,7 @@ class AryionExtractor(Extractor):
# folder
if headers["content-type"] in (
"application/x-folder",
+ "application/x-comic-folder",
"application/x-comic-folder-nomerge",
):
return False
@@ -184,11 +211,16 @@ class AryionPostExtractor(AryionExtractor):
"_mtime" : "Sat, 16 Feb 2019 19:30:34 GMT",
},
}),
- # folder (#694)
+ # x-folder (#694)
("https://aryion.com/g4/view/588928", {
"pattern": pattern,
"count": ">= 8",
}),
+ # x-comic-folder (#945)
+ ("https://aryion.com/g4/view/537379", {
+ "pattern": pattern,
+ "count": 2,
+ }),
)
def posts(self):
diff --git a/gallery_dl/extractor/exhentai.py b/gallery_dl/extractor/exhentai.py
index 4cb10b4..80c7187 100644
--- a/gallery_dl/extractor/exhentai.py
+++ b/gallery_dl/extractor/exhentai.py
@@ -47,6 +47,12 @@ class ExhentaiExtractor(Extractor):
self.wait_min = self.config("wait-min", 3)
self.wait_max = self.config("wait-max", 6)
+ if type(self.limits) is int:
+ self._limit_max = self.limits
+ self.limits = True
+ else:
+ self._limit_max = 0
+
self._remaining = 0
if self.wait_max < self.wait_min:
self.wait_max = self.wait_min
@@ -331,6 +337,8 @@ class ExhentaiGalleryExtractor(ExhentaiExtractor):
page = self.request(url, cookies=cookies).text
current, pos = text.extract(page, "<strong>", "</strong>")
maximum, pos = text.extract(page, "<strong>", "</strong>", pos)
+ if self._limit_max:
+ maximum = self._limit_max
self.log.debug("Image Limits: %s/%s", current, maximum)
self._remaining = text.parse_int(maximum) - text.parse_int(current)
diff --git a/gallery_dl/extractor/foolslide.py b/gallery_dl/extractor/foolslide.py
index 715e294..e624a65 100644
--- a/gallery_dl/extractor/foolslide.py
+++ b/gallery_dl/extractor/foolslide.py
@@ -208,21 +208,16 @@ EXTRACTORS = {
"pattern": r"(?:(?:www\.)?sensescans\.com/reader"
r"|reader\.sensescans\.com)",
"test-chapter": (
- (("https://sensescans.com/reader/read/"
- "magi__labyrinth_of_magic/en/37/369/"), {
- "url": "8bbc59a995640bbb944c0b1be06a490909b58be1",
- "keyword": "07acd84fb18a9f1fd6dff5befe711bcca0ff9988",
- }),
- (("https://reader.sensescans.com/read/"
- "magi__labyrinth_of_magic/en/37/369/"), {
- "url": "8bbc59a995640bbb944c0b1be06a490909b58be1",
- "keyword": "07acd84fb18a9f1fd6dff5befe711bcca0ff9988",
+ ("https://sensescans.com/reader/read/ao_no_orchestra/en/0/26/", {
+ "url": "bbd428dc578f5055e9f86ad635b510386cd317cd",
+ "keyword": "083ef6f8831c84127fe4096fa340a249be9d1424",
}),
+ ("https://reader.sensescans.com/read/ao_no_orchestra/en/0/26/"),
),
"test-manga":
("https://sensescans.com/reader/series/yotsubato/", {
- "url": "ee4dca7c421bf15ac039200f8c0bcb0858153640",
- "keyword": "f94961bd731bd878bbd4d48555bc3ace1d937364",
+ "url": "305e6eb6160e3bb90c3de39ff5fb7c971e052087",
+ "keyword": "562fb5a7362a4cb43d59d5c8a6ea8080fc65cf99",
}),
},
"worldthree": {
diff --git a/gallery_dl/extractor/furaffinity.py b/gallery_dl/extractor/furaffinity.py
index 61226b6..6dfd75d 100644
--- a/gallery_dl/extractor/furaffinity.py
+++ b/gallery_dl/extractor/furaffinity.py
@@ -30,15 +30,21 @@ class FuraffinityExtractor(Extractor):
self.offset = 0
def items(self):
+ metadata = self.metadata()
for post_id in util.advance(self.posts(), self.offset):
post = self._parse_post(post_id)
if post:
+ if metadata:
+ post.update(metadata)
yield Message.Directory, post
yield Message.Url, post["url"], post
def posts(self):
return self._pagination()
+ def metadata(self):
+ return None
+
def skip(self, num):
self.offset += num
return num
@@ -133,6 +139,40 @@ class FuraffinityExtractor(Extractor):
yield from text.extract_iter(page, 'id="sid-', '"')
path = text.extract(page, 'right" href="', '"')[0]
+ def _pagination_search(self, query):
+ url = self.root + "/search/"
+ data = {
+ "page" : 0,
+ "next_page" : "Next",
+ "order-by" : "relevancy",
+ "order-direction": "desc",
+ "range" : "all",
+ "rating-general" : "on",
+ "rating-mature" : "on",
+ "rating-adult" : "on",
+ "type-art" : "on",
+ "type-music" : "on",
+ "type-flash" : "on",
+ "type-story" : "on",
+ "type-photo" : "on",
+ "type-poetry" : "on",
+ "mode" : "extended",
+ }
+ data.update(query)
+ if "page" in query:
+ data["page"] = text.parse_int(query["page"])
+
+ while True:
+ page = self.request(url, method="POST", data=data).text
+ post_id = None
+
+ for post_id in text.extract_iter(page, 'id="sid-', '"'):
+ yield post_id
+
+ if not post_id:
+ return
+ data["page"] += 1
+
class FuraffinityGalleryExtractor(FuraffinityExtractor):
"""Extractor for a furaffinity user's gallery"""
@@ -171,6 +211,25 @@ class FuraffinityFavoriteExtractor(FuraffinityExtractor):
return self._pagination_favorites()
+class FuraffinitySearchExtractor(FuraffinityExtractor):
+ """Extractor for furaffinity search results"""
+ subcategory = "search"
+ directory_fmt = ("{category}", "Search", "{search}")
+ pattern = BASE_PATTERN + r"/search/?\?([^#]+)"
+ test = ("https://www.furaffinity.net/search/?q=cute", {
+ "pattern": r"https://d.facdn.net/art/[^/]+/\d+/\d+.\w+\.\w+",
+ "range": "45-50",
+ "count": 6,
+ })
+
+ def metadata(self):
+ self.query = text.parse_query(self.user)
+ return {"search": self.query.get("q")}
+
+ def posts(self):
+ return self._pagination_search(self.query)
+
+
class FuraffinityPostExtractor(FuraffinityExtractor):
"""Extractor for individual posts on furaffinity"""
subcategory = "post"
diff --git a/gallery_dl/extractor/gelbooru.py b/gallery_dl/extractor/gelbooru.py
index 612c742..edadd31 100644
--- a/gallery_dl/extractor/gelbooru.py
+++ b/gallery_dl/extractor/gelbooru.py
@@ -55,7 +55,7 @@ class GelbooruExtractor(booru.XmlParserMixin,
while True:
page = self.request(url, params=params).text
- ids = list(text.extract_iter(page, '<a id="p', '"'))
+ ids = list(text.extract_iter(page, '<span id="s', '"'))
yield from ids
if len(ids) < self.per_page:
return
diff --git a/gallery_dl/extractor/hentaihand.py b/gallery_dl/extractor/hentaihand.py
index 302999b..7635bf1 100644
--- a/gallery_dl/extractor/hentaihand.py
+++ b/gallery_dl/extractor/hentaihand.py
@@ -10,74 +10,61 @@
from .common import GalleryExtractor, Extractor, Message
from .. import text, util
-import collections
+import json
class HentaihandGalleryExtractor(GalleryExtractor):
"""Extractor for image galleries on hentaihand.com"""
category = "hentaihand"
root = "https://hentaihand.com"
- pattern = (r"(?i)(?:https?://)?(?:www\.)?hentaihand\.com"
- r"/(?:comi|view)c/(\d+)")
+ pattern = r"(?:https?://)?(?:www\.)?hentaihand\.com/\w+/comic/([\w-]+)"
test = (
- ("https://hentaihand.com/comic/272772/kouda-tomohiro-chiyomi-bl", {
- "pattern": r"https://i.hentaihand.com/.*/images/full/\d+.jpg$",
+ (("https://hentaihand.com/en/comic/kouda-tomohiro-chiyomi-"
+ "blizzard-comic-aun-2016-12-english-nanda-sore-scans"), {
+ "pattern": r"https://cdn.hentaihand.com/.*/images/304546/\d+.jpg$",
"count": 19,
"keyword": {
- "artists" : ["kouda tomohiro"],
- "categories": ["manga"],
- "date" : "Feb. 6, 2020, 3:19 p.m.",
- "gallery_id": 272772,
+ "artists" : ["Kouda Tomohiro"],
+ "date" : "dt:2020-02-06 00:00:00",
+ "gallery_id": 304546,
"lang" : "en",
"language" : "English",
- "relationships": ["family", "step family"],
+ "relationships": ["Family", "Step family"],
"tags" : list,
"title" : r"re:\[Kouda Tomohiro\] Chiyomi Blizzard",
- "title_jp" : r"re:\[幸田朋弘\] ちよみブリザード",
+ "title_alt" : r"re:\[幸田朋弘\] ちよみブリザード",
+ "type" : "Manga",
},
}),
- ("https://hentaihand.com/viewc/272772/kouda-tomohiro-chiyomi-bl"),
)
def __init__(self, match):
- self.gallery_id = match.group(1)
- url = "{}/comic/{}".format(self.root, self.gallery_id)
+ self.slug = match.group(1)
+ url = "{}/api/comics/{}".format(self.root, self.slug)
GalleryExtractor.__init__(self, match, url)
def metadata(self, page):
- extr = text.extract_from(page)
-
- title_en = text.unescape(extr("<h1>", "<"))
- title_jp = text.unescape(extr("<h2>", "<"))
- tags = extr('<section id="tags"', "</section>")
-
+ info = json.loads(page)
data = {
- "gallery_id" : text.parse_int(self.gallery_id),
- "title" : title_en or title_jp,
- "title_en" : title_en,
- "title_jp" : title_jp,
-
- # impossible to parse with strptime()
- "date" : extr('datetime="', '"'),
+ "gallery_id" : text.parse_int(info["id"]),
+ "title" : info["title"],
+ "title_alt" : info["alternative_title"],
+ "slug" : self.slug,
+ "type" : info["category"]["name"],
+ "language" : info["language"]["name"],
+ "lang" : util.language_to_code(info["language"]["name"]),
+ "tags" : [t["slug"] for t in info["tags"]],
+ "date" : text.parse_datetime(
+ info["uploaded_at"], "%Y-%m-%d"),
}
-
- tdict = collections.defaultdict(list)
- for path in text.extract_iter(tags, 'href="/', '"'):
- kind, _, name = path.partition("/")
- tdict[kind].append(name.replace("+", " "))
- data.update(tdict)
-
- if "languages" in data:
- data["language"] = data["languages"][-1].capitalize()
- data["lang"] = util.language_to_code(data["language"])
- del data["languages"]
+ for key in ("artists", "authors", "groups", "characters",
+ "relationships", "parodies"):
+ data[key] = [v["name"] for v in info[key]]
return data
def images(self, _):
- url = "{}/viewc/{}/1".format(self.root, self.gallery_id)
- page = self.request(url).text
- images = text.extract(page, "var images", ";")[0]
- return [(img, None) for img in text.extract_iter(images, "'", "'")]
+ info = self.request(self.gallery_url + "/images").json()
+ return [(img["source_url"], img) for img in info["images"]]
class HentaihandTagExtractor(Extractor):
@@ -86,49 +73,49 @@ class HentaihandTagExtractor(Extractor):
subcategory = "tag"
root = "https://hentaihand.com"
pattern = (r"(?i)(?:https?://)?(?:www\.)?hentaihand\.com"
- r"(/(?:parody|characters|tags|artists|groups|languages"
- r"|categories|relationships)/[^#]+)")
+ r"/\w+/(parody|character|tag|artist|group|language"
+ r"|category|relationship)/([^/?&#]+)")
test = (
- ("https://hentaihand.com/artists/tony+taka", {
+ ("https://hentaihand.com/en/artist/himuro", {
"pattern": HentaihandGalleryExtractor.pattern,
- "count": ">= 50",
+ "count": ">= 18",
}),
- ("https://hentaihand.com/artists/tony+taka/popular?page=2"),
- ("https://hentaihand.com/tags/full+color"),
- ("https://hentaihand.com/languages/japanese"),
- ("https://hentaihand.com/categories/manga"),
+ ("https://hentaihand.com/en/tag/full-color"),
+ ("https://hentaihand.com/fr/language/japanese"),
+ ("https://hentaihand.com/zh/category/manga"),
)
def __init__(self, match):
Extractor.__init__(self, match)
- self.path, _, query = match.group(1).partition("?")
- self.query = text.parse_query(query)
- self.query["page"] = text.parse_int(self.query.get("page"), 1)
+ self.type, self.key = match.groups()
def items(self):
- yield Message.Version, 1
- url = self.root + self.path
- params = self.query.copy()
- data = {"_extractor": HentaihandGalleryExtractor}
-
+ if self.type[-1] == "y":
+ tpl = self.type[:-1] + "ies"
+ else:
+ tpl = self.type + "s"
+
+ url = "{}/api/{}/{}".format(self.root, tpl, self.key)
+ tid = self.request(url, notfound=self.type).json()["id"]
+
+ url = self.root + "/api/comics"
+ params = {
+ "per_page": "18",
+ tpl : tid,
+ "page" : 1,
+ "q" : "",
+ "sort" : "uploaded_at",
+ "order" : "desc",
+ "duration": "day",
+ }
while True:
- page = self.request(url, params=params).text
+ info = self.request(url, params=params).json()
- for path in text.extract_iter(page, '<a href="/comic/', '"'):
- yield Message.Queue, self.root + "/comic/" + path, data
+ for gallery in info["data"]:
+ gurl = "{}/en/comic/{}".format(self.root, gallery["slug"])
+ gallery["_extractor"] = HentaihandGalleryExtractor
+ yield Message.Queue, gurl, gallery
- pos = page.find(">(current)<")
- if pos < 0 or page.find('class="page-link" href="', pos) < 0:
- break
+ if params["page"] >= info["last_page"]:
+ return
params["page"] += 1
-
-
-class HentaihandSearchExtractor(HentaihandTagExtractor):
- """Extractor for search results on hentaihand.com"""
- subcategory = "search"
- pattern = r"(?i)(?:https?://)?(?:www\.)?hentaihand\.com(/search/?[^#]+)"
- test = ("https://hentaihand.com/search?q=color", {
- "pattern": HentaihandGalleryExtractor.pattern,
- "range": "1-50",
- "count": 50,
- })
diff --git a/gallery_dl/extractor/hitomi.py b/gallery_dl/extractor/hitomi.py
index da53113..209a4f2 100644
--- a/gallery_dl/extractor/hitomi.py
+++ b/gallery_dl/extractor/hitomi.py
@@ -111,9 +111,9 @@ class HitomiGalleryExtractor(GalleryExtractor):
response = self.request(url, fatal=False)
if b"<title>Redirect</title>" not in response.content:
break
- url = text.extract(response.text, "href='", "'")[0]
- if not url.startswith("http"):
- url = text.urljoin(self.root, url)
+ url = text.extract(
+ response.text, 'http-equiv="refresh" content="', '"',
+ )[0].partition("=")[2]
if response.status_code >= 400:
return {}
@@ -158,9 +158,9 @@ class HitomiTagExtractor(Extractor):
subcategory = "tag"
pattern = (r"(?:https?://)?hitomi\.la/"
r"(tag|artist|group|series|type|character)/"
- r"([^/?&#]+)-\d+\.html")
+ r"([^/?&#]+)\.html")
test = (
- ("https://hitomi.la/tag/screenshots-japanese-1.html", {
+ ("https://hitomi.la/tag/screenshots-japanese.html", {
"pattern": HitomiGalleryExtractor.pattern,
"count": ">= 35",
}),
@@ -175,6 +175,10 @@ class HitomiTagExtractor(Extractor):
Extractor.__init__(self, match)
self.type, self.tag = match.groups()
+ tag, _, num = self.tag.rpartition("-")
+ if num.isdecimal():
+ self.tag = tag
+
def items(self):
url = "https://ltn.hitomi.la/{}/{}.nozomi".format(self.type, self.tag)
data = {"_extractor": HitomiGalleryExtractor}
diff --git a/gallery_dl/extractor/imgur.py b/gallery_dl/extractor/imgur.py
index 25328ab..190a4ff 100644
--- a/gallery_dl/extractor/imgur.py
+++ b/gallery_dl/extractor/imgur.py
@@ -59,7 +59,7 @@ class ImgurImageExtractor(ImgurExtractor):
subcategory = "image"
filename_fmt = "{category}_{id}{title:?_//}.{extension}"
archive_fmt = "{id}"
- pattern = BASE_PATTERN + r"/(?!gallery)(\w{7}|\w{5})[sbtmlh]?\.?"
+ pattern = BASE_PATTERN + r"/(?!gallery|search)(\w{7}|\w{5})[sbtmlh]?\.?"
test = (
("https://imgur.com/21yMxCS", {
"url": "6f2dcfb86815bdd72808c313e5f715610bc7b9b2",
@@ -304,8 +304,40 @@ class ImgurSubredditExtractor(ImgurExtractor):
return self._items_queue(self.api.gallery_subreddit(self.key))
+class ImgurTagExtractor(ImgurExtractor):
+ """Extractor for imgur tag searches"""
+ subcategory = "tag"
+ pattern = BASE_PATTERN + r"/t/([^/?&#]+)$"
+ test = ("https://imgur.com/t/animals", {
+ "range": "1-100",
+ "count": 100,
+ "pattern": r"https?://(i.imgur.com|imgur.com/a)/[\w.]+",
+ })
+
+ def items(self):
+ return self._items_queue(self.api.gallery_tag(self.key))
+
+
+class ImgurSearchExtractor(ImgurExtractor):
+ """Extractor for imgur search results"""
+ subcategory = "search"
+ pattern = BASE_PATTERN + r"/search(?:/[^?&#]+)?/?\?q=([^&#]+)"
+ test = ("https://imgur.com/search?q=cute+cat", {
+ "range": "1-100",
+ "count": 100,
+ "pattern": r"https?://(i.imgur.com|imgur.com/a)/[\w.]+",
+ })
+
+ def items(self):
+ key = text.unquote(self.key.replace("+", " "))
+ return self._items_queue(self.api.gallery_search(key))
+
+
class ImgurAPI():
+ """Interface for the Imgur API
+ Ref: https://apidocs.imgur.com/
+ """
def __init__(self, extractor):
self.extractor = extractor
self.headers = {
@@ -317,6 +349,11 @@ class ImgurAPI():
endpoint = "account/{}/gallery_favorites".format(account)
return self._pagination(endpoint)
+ def gallery_search(self, query):
+ endpoint = "gallery/search"
+ params = {"q": query}
+ return self._pagination(endpoint, params)
+
def account_submissions(self, account):
endpoint = "account/{}/submissions".format(account)
return self._pagination(endpoint)
@@ -325,16 +362,21 @@ class ImgurAPI():
endpoint = "gallery/r/{}".format(subreddit)
return self._pagination(endpoint)
+ def gallery_tag(self, tag):
+ endpoint = "gallery/t/{}".format(tag)
+ return self._pagination(endpoint, key="items")
+
def album(self, album_hash):
return self._call("album/" + album_hash)
def image(self, image_hash):
return self._call("image/" + image_hash)
- def _call(self, endpoint):
+ def _call(self, endpoint, params=None):
try:
return self.extractor.request(
- "https://api.imgur.com/3/" + endpoint, headers=self.headers,
+ "https://api.imgur.com/3/" + endpoint,
+ params=params, headers=self.headers,
).json()["data"]
except exception.HttpError as exc:
if exc.status != 403 or b"capacity" not in exc.response.content:
@@ -342,11 +384,13 @@ class ImgurAPI():
self.extractor.sleep(seconds=600)
return self._call(endpoint)
- def _pagination(self, endpoint):
+ def _pagination(self, endpoint, params=None, key=None):
num = 0
while True:
- data = self._call("{}/{}".format(endpoint, num))
+ data = self._call("{}/{}".format(endpoint, num), params)
+ if key:
+ data = data[key]
if not data:
return
yield from data
diff --git a/gallery_dl/extractor/mangapark.py b/gallery_dl/extractor/mangapark.py
index 3d64acd..59a046c 100644
--- a/gallery_dl/extractor/mangapark.py
+++ b/gallery_dl/extractor/mangapark.py
@@ -53,18 +53,19 @@ class MangaparkChapterExtractor(MangaparkBase, ChapterExtractor):
pattern = (r"(?:https?://)?(?:www\.)?mangapark\.(me|net|com)"
r"/manga/([^?&#]+/i\d+)")
test = (
- ("https://mangapark.net/manga/gosu/i811615/c55/1", {
+ ("https://mangapark.net/manga/gosu/i811653/c055/1", {
"count": 50,
- "keyword": "2bb16a50dbac9577ead62b41db9a01a0419c0ae2",
+ "keyword": "8344bdda8cd8414e7729a4e148379f147e3437da",
}),
(("https://mangapark.net/manga"
- "/ad-astra-per-aspera-hata-kenjirou/i662054/c001.2/1"), {
+ "/ad-astra-per-aspera-hata-kenjirou/i662051/c001.2/1"), {
"count": 40,
- "keyword": "8e9cce4ed0e25d12a45e02f840d6f32ef838e257",
+ "keyword": "2bb3a8f426383ea13f17ff5582f3070d096d30ac",
}),
- ("https://mangapark.net/manga/gekkan-shoujo-nozaki-kun/i655476/c70", {
+ (("https://mangapark.net/manga"
+ "/gekkan-shoujo-nozaki-kun/i2067426/v7/c70/1"), {
"count": 15,
- "keyword": "19f730617074d65f91c0781f429de324890925bf",
+ "keyword": "edc14993c4752cee3a76e09b2f024d40d854bfd1",
}),
("https://mangapark.me/manga/gosu/i811615/c55/1"),
("https://mangapark.com/manga/gosu/i811615/c55/1"),
@@ -119,8 +120,8 @@ class MangaparkMangaExtractor(MangaparkBase, MangaExtractor):
r"(/manga/[^/?&#]+)/?$")
test = (
("https://mangapark.net/manga/aria", {
- "url": "9b0b31e4992260876f56d7bfc8ff0ae71295c4f4",
- "keyword": "6e44744a28d01b889b1e8291847abd84b591590d",
+ "url": "9b62883c25c8de471f8ab43651e1448536c4ce3f",
+ "keyword": "eb4a9b273c69acf31efa731eba713e1cfa14bab6",
}),
("https://mangapark.me/manga/aria"),
("https://mangapark.com/manga/aria"),
diff --git a/gallery_dl/extractor/reddit.py b/gallery_dl/extractor/reddit.py
index cb70fe5..9c6892a 100644
--- a/gallery_dl/extractor/reddit.py
+++ b/gallery_dl/extractor/reddit.py
@@ -57,12 +57,8 @@ class RedditExtractor(Extractor):
yield Message.Url, url, submission
elif "gallery_data" in submission:
- meta = submission["media_metadata"]
- items = submission["gallery_data"]["items"]
- for submission["num"], item in enumerate(items, 1):
- url = meta[item["media_id"]]["s"]["u"]
- url = url.partition("?")[0]
- url = url.replace("/preview.", "/i.", 1)
+ for submission["num"], url in enumerate(
+ self._extract_gallery(submission), 1):
text.nameext_from_url(url, submission)
yield Message.Url, url, submission
@@ -118,6 +114,22 @@ class RedditExtractor(Extractor):
def submissions(self):
"""Return an iterable containing all (submission, comments) tuples"""
+ def _extract_gallery(self, submission):
+ if submission["gallery_data"] is None:
+ self.log.warning("gallery '%s' was deleted", submission["id"])
+ return
+
+ meta = submission["media_metadata"]
+ for item in submission["gallery_data"]["items"]:
+ src = meta[item["media_id"]]["s"]
+ url = src.get("u") or src.get("gif") or src.get("mp4")
+ if url:
+ yield url.partition("?")[0].replace("/preview.", "/i.", 1)
+ else:
+ self.log.error("Unable to get download URL for item '%s'",
+ item["media_id"])
+ self.log.debug(src)
+
class RedditSubredditExtractor(RedditExtractor):
"""Extractor for URLs from subreddits on reddit.com"""
@@ -188,6 +200,15 @@ class RedditSubmissionExtractor(RedditExtractor):
"content": "1e7dde4ee7d5f4c4b45749abfd15b2dbfa27df3f",
"count": 3,
}),
+ # deleted gallery (#953)
+ ("https://www.reddit.com/gallery/icfgzv", {
+ "count": 0,
+ }),
+ # animated gallery items (#955)
+ ("https://www.reddit.com/r/araragi/comments/ib32hm", {
+ "pattern": r"https://i\.redd\.it/\w+\.gif",
+ "count": 2,
+ }),
("https://old.reddit.com/r/lavaporn/comments/2a00np/"),
("https://np.reddit.com/r/lavaporn/comments/2a00np/"),
("https://m.reddit.com/r/lavaporn/comments/2a00np/"),
diff --git a/gallery_dl/extractor/redgifs.py b/gallery_dl/extractor/redgifs.py
index 4477825..0f02e8b 100644
--- a/gallery_dl/extractor/redgifs.py
+++ b/gallery_dl/extractor/redgifs.py
@@ -58,7 +58,8 @@ class RedgifsImageExtractor(RedgifsExtractor):
r"|gifdeliverynetwork.com)/([A-Za-z]+)")
test = (
("https://redgifs.com/watch/foolishforkedabyssiniancat", {
- "pattern": r"https://\w+.redgifs.com/FoolishForkedAbyss.+.mp4",
+ "pattern": r"https://\w+\.(redgifs|gfycat)\.com"
+ r"/FoolishForkedAbyssiniancat\.mp4",
"content": "f6e03f1df9a2ff2a74092f53ee7580d2fb943533",
}),
("https://www.gifdeliverynetwork.com/foolishforkedabyssiniancat"),
diff --git a/gallery_dl/extractor/subscribestar.py b/gallery_dl/extractor/subscribestar.py
index 076d0c0..38b39d4 100644
--- a/gallery_dl/extractor/subscribestar.py
+++ b/gallery_dl/extractor/subscribestar.py
@@ -147,13 +147,13 @@ class SubscribestarUserExtractor(SubscribestarExtractor):
"author_nick": "SubscribeStar",
"content": str,
"date" : "type:datetime",
- "height" : int,
"id" : int,
- "pinned" : bool,
"post_id": int,
- "type" : "re:image|video",
+ "type" : "re:image|video|attachment",
"url" : str,
- "width" : int,
+ "?pinned": bool,
+ "?height": int,
+ "?width" : int,
},
}),
("https://www.subscribestar.com/subscribestar", {
diff --git a/gallery_dl/extractor/xvideos.py b/gallery_dl/extractor/xvideos.py
index 80a3614..2548ead 100644
--- a/gallery_dl/extractor/xvideos.py
+++ b/gallery_dl/extractor/xvideos.py
@@ -31,7 +31,7 @@ class XvideosGalleryExtractor(XvideosBase, GalleryExtractor):
r"/([^/?&#]+)/photos/(\d+)")
test = (
("https://www.xvideos.com/profiles/pervertedcouple/photos/751031", {
- "url": "4f0d992e5dc39def2c3ac8e099d17bf09e76e3c7",
+ "url": "cb4657a37eea5ab6b1d333491cee7eeb529b0645",
"keyword": {
"gallery": {
"id" : 751031,
diff --git a/gallery_dl/version.py b/gallery_dl/version.py
index b2b59e0..9af9a43 100644
--- a/gallery_dl/version.py
+++ b/gallery_dl/version.py
@@ -6,4 +6,4 @@
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
-__version__ = "1.14.4"
+__version__ = "1.14.5"
diff --git a/test/test_results.py b/test/test_results.py
index 1f2f699..fbbb79c 100644
--- a/test/test_results.py
+++ b/test/test_results.py
@@ -26,15 +26,13 @@ TRAVIS_SKIP = {
"archivedmoe", "archiveofsins", "thebarchive", "fireden", "4plebs",
"sankaku", "idolcomplex", "mangahere", "readcomiconline", "mangadex",
"sankakucomplex", "warosu", "fuskator", "patreon", "komikcast",
- "instagram",
+ "instagram", "ngomik",
}
# temporary issues, etc.
BROKEN = {
- "hentaihand",
+ "dokireader",
"imagevenue",
- "mangapark",
- "ngomik",
"photobucket",
"worldthree",
}