[vk] improve metadata extraction and URL pattern (fixes #1691)

- always fetch all user metadata
- use 'user[name]' for directory names if available
This commit is contained in:
Mike Fährmann 2021-07-15 00:43:42 +02:00
parent b9783403d9
commit 36a2aff363
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88

View File

@ -17,43 +17,60 @@ class VkPhotosExtractor(Extractor):
"""Extractor for photos from a vk user""" """Extractor for photos from a vk user"""
category = "vk" category = "vk"
subcategory = "photos" subcategory = "photos"
directory_fmt = ("{category}", "{user[id]}") directory_fmt = ("{category}", "{user[name]|user[id]}")
filename_fmt = "{id}.{extension}" filename_fmt = "{id}.{extension}"
archive_fmt = "{id}" archive_fmt = "{id}"
root = "https://vk.com" root = "https://vk.com"
request_interval = 1.0 request_interval = 1.0
pattern = r"(?:https://)?(?:www\.|m\.)?vk\.com/(?:albums|photos|id)(\d+)" pattern = (r"(?:https://)?(?:www\.|m\.)?vk\.com/(?:"
r"(?:albums|photos|id)(-?\d+)|([^/?#]+))")
test = ( test = (
("https://vk.com/id398982326", { ("https://vk.com/id398982326", {
"pattern": r"https://sun\d+-\d+\.userapi\.com/c\d+/v\d+" "pattern": r"https://sun\d+-\d+\.userapi\.com/c\d+/v\d+"
r"/[0-9a-f]+/[\w-]+\.jpg", r"/[0-9a-f]+/[\w-]+\.jpg",
"count": ">= 35", "count": ">= 35",
"keywords": {
"id": r"re:\d+",
"user": {
"id": "398982326",
"info": "Мы за Движуху! m1ni SounD #4 [EROmusic]",
"name": "",
"nick": "Dobrov Kurva",
},
},
}),
("https://vk.com/cosplayinrussia", {
"range": "75-100",
"keywords": {
"id": r"re:\d+",
"user": {
"id" : "-165740836",
"info": "Предложка открыта, кидайте ваши косплейчики. При "
"правильном оформлении они будут опубликованы",
"name": "cosplayinrussia",
"nick": "Косплей | Cosplay 18+",
},
},
}), }),
("https://m.vk.com/albums398982326"), ("https://m.vk.com/albums398982326"),
("https://www.vk.com/id398982326?profile=1"), ("https://www.vk.com/id398982326?profile=1"),
("https://vk.com/albums-165740836"),
) )
def __init__(self, match): def __init__(self, match):
Extractor.__init__(self, match) Extractor.__init__(self, match)
self.user_id = match.group(1) self.user_id, self.user_name = match.groups()
def items(self): def items(self):
user_id = self.user_id if self.user_id:
user_id = self.user_id
if self.config("metadata"): prefix = "public" if user_id[0] == "-" else "id"
url = "{}/id{}".format(self.root, user_id) url = "{}/{}{}".format(self.root, prefix, user_id.lstrip("-"))
extr = text.extract_from(self.request(url).text) data = self._extract_profile(url)
data = {"user": {
"id" : user_id,
"nick": text.unescape(extr(
"<title>", " | VK<")),
"name": text.unescape(extr(
'<h1 class="page_name">', "<")).replace(" ", " "),
"info": text.unescape(text.remove_html(extr(
'<span class="current_text">', '</span')))
}}
else: else:
data = {"user": {"id": user_id}} url = "{}/{}".format(self.root, self.user_name)
data = self._extract_profile(url)
user_id = data["user"]["id"]
photos_url = "{}/photos{}".format(self.root, user_id) photos_url = "{}/photos{}".format(self.root, user_id)
headers = { headers = {
@ -86,3 +103,15 @@ class VkPhotosExtractor(Extractor):
if cnt <= 40 or offset == params["offset"]: if cnt <= 40 or offset == params["offset"]:
return return
params["offset"] = offset params["offset"] = offset
def _extract_profile(self, url):
extr = text.extract_from(self.request(url).text)
return {"user": {
"name": text.unescape(extr(
'rel="canonical" href="https://vk.com/', '"')),
"nick": text.unescape(extr(
'<h1 class="page_name">', "<")).replace(" ", " "),
"info": text.unescape(text.remove_html(extr(
'<span class="current_text">', '</span'))),
"id" : extr('<a href="/albums', '"'),
}}