[vk] improve metadata extraction and URL pattern (fixes #1691)
- always fetch all user metadata - use 'user[name]' for directory names if available
This commit is contained in:
parent
b9783403d9
commit
36a2aff363
@ -17,43 +17,60 @@ class VkPhotosExtractor(Extractor):
|
|||||||
"""Extractor for photos from a vk user"""
|
"""Extractor for photos from a vk user"""
|
||||||
category = "vk"
|
category = "vk"
|
||||||
subcategory = "photos"
|
subcategory = "photos"
|
||||||
directory_fmt = ("{category}", "{user[id]}")
|
directory_fmt = ("{category}", "{user[name]|user[id]}")
|
||||||
filename_fmt = "{id}.{extension}"
|
filename_fmt = "{id}.{extension}"
|
||||||
archive_fmt = "{id}"
|
archive_fmt = "{id}"
|
||||||
root = "https://vk.com"
|
root = "https://vk.com"
|
||||||
request_interval = 1.0
|
request_interval = 1.0
|
||||||
pattern = r"(?:https://)?(?:www\.|m\.)?vk\.com/(?:albums|photos|id)(\d+)"
|
pattern = (r"(?:https://)?(?:www\.|m\.)?vk\.com/(?:"
|
||||||
|
r"(?:albums|photos|id)(-?\d+)|([^/?#]+))")
|
||||||
test = (
|
test = (
|
||||||
("https://vk.com/id398982326", {
|
("https://vk.com/id398982326", {
|
||||||
"pattern": r"https://sun\d+-\d+\.userapi\.com/c\d+/v\d+"
|
"pattern": r"https://sun\d+-\d+\.userapi\.com/c\d+/v\d+"
|
||||||
r"/[0-9a-f]+/[\w-]+\.jpg",
|
r"/[0-9a-f]+/[\w-]+\.jpg",
|
||||||
"count": ">= 35",
|
"count": ">= 35",
|
||||||
|
"keywords": {
|
||||||
|
"id": r"re:\d+",
|
||||||
|
"user": {
|
||||||
|
"id": "398982326",
|
||||||
|
"info": "Мы за Движуху! – m1ni SounD #4 [EROmusic]",
|
||||||
|
"name": "",
|
||||||
|
"nick": "Dobrov Kurva",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
("https://vk.com/cosplayinrussia", {
|
||||||
|
"range": "75-100",
|
||||||
|
"keywords": {
|
||||||
|
"id": r"re:\d+",
|
||||||
|
"user": {
|
||||||
|
"id" : "-165740836",
|
||||||
|
"info": "Предложка открыта, кидайте ваши косплейчики. При "
|
||||||
|
"правильном оформлении они будут опубликованы",
|
||||||
|
"name": "cosplayinrussia",
|
||||||
|
"nick": "Косплей | Cosplay 18+",
|
||||||
|
},
|
||||||
|
},
|
||||||
}),
|
}),
|
||||||
("https://m.vk.com/albums398982326"),
|
("https://m.vk.com/albums398982326"),
|
||||||
("https://www.vk.com/id398982326?profile=1"),
|
("https://www.vk.com/id398982326?profile=1"),
|
||||||
|
("https://vk.com/albums-165740836"),
|
||||||
)
|
)
|
||||||
|
|
||||||
def __init__(self, match):
|
def __init__(self, match):
|
||||||
Extractor.__init__(self, match)
|
Extractor.__init__(self, match)
|
||||||
self.user_id = match.group(1)
|
self.user_id, self.user_name = match.groups()
|
||||||
|
|
||||||
def items(self):
|
def items(self):
|
||||||
user_id = self.user_id
|
if self.user_id:
|
||||||
|
user_id = self.user_id
|
||||||
if self.config("metadata"):
|
prefix = "public" if user_id[0] == "-" else "id"
|
||||||
url = "{}/id{}".format(self.root, user_id)
|
url = "{}/{}{}".format(self.root, prefix, user_id.lstrip("-"))
|
||||||
extr = text.extract_from(self.request(url).text)
|
data = self._extract_profile(url)
|
||||||
data = {"user": {
|
|
||||||
"id" : user_id,
|
|
||||||
"nick": text.unescape(extr(
|
|
||||||
"<title>", " | VK<")),
|
|
||||||
"name": text.unescape(extr(
|
|
||||||
'<h1 class="page_name">', "<")).replace(" ", " "),
|
|
||||||
"info": text.unescape(text.remove_html(extr(
|
|
||||||
'<span class="current_text">', '</span')))
|
|
||||||
}}
|
|
||||||
else:
|
else:
|
||||||
data = {"user": {"id": user_id}}
|
url = "{}/{}".format(self.root, self.user_name)
|
||||||
|
data = self._extract_profile(url)
|
||||||
|
user_id = data["user"]["id"]
|
||||||
|
|
||||||
photos_url = "{}/photos{}".format(self.root, user_id)
|
photos_url = "{}/photos{}".format(self.root, user_id)
|
||||||
headers = {
|
headers = {
|
||||||
@ -86,3 +103,15 @@ class VkPhotosExtractor(Extractor):
|
|||||||
if cnt <= 40 or offset == params["offset"]:
|
if cnt <= 40 or offset == params["offset"]:
|
||||||
return
|
return
|
||||||
params["offset"] = offset
|
params["offset"] = offset
|
||||||
|
|
||||||
|
def _extract_profile(self, url):
|
||||||
|
extr = text.extract_from(self.request(url).text)
|
||||||
|
return {"user": {
|
||||||
|
"name": text.unescape(extr(
|
||||||
|
'rel="canonical" href="https://vk.com/', '"')),
|
||||||
|
"nick": text.unescape(extr(
|
||||||
|
'<h1 class="page_name">', "<")).replace(" ", " "),
|
||||||
|
"info": text.unescape(text.remove_html(extr(
|
||||||
|
'<span class="current_text">', '</span'))),
|
||||||
|
"id" : extr('<a href="/albums', '"'),
|
||||||
|
}}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user