mikf
/
gallery-dl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[500px] update GraphQL queries 

500px changed its method from query hashes to sending the entire query
string for every request.
master
Mike Fährmann 2021-06-14 16:13:08 +02:00
parent e9993451c3
commit b56e245094
No known key found for this signature in database
GPG Key ID: 5680CA389D365A88
2 changed files with 397 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

407
gallery_dl/extractor/500px.py
View File

@ -11,7 +11,6 @@
from .common import Extractor, Message
import json
BASE_PATTERN = r"(?:https?://)?(?:web\.)?500px\.com"
@ -78,15 +77,14 @@ class _500pxExtractor(Extractor):
headers = {"Origin": self.root, "X-CSRF-Token": csrf_token}
return self.request(url, headers=headers, params=params).json()
def _request_graphql(self, opname, variables, query_hash):
def _request_graphql(self, opname, variables):
url = "https://api.500px.com/graphql"
params = {
data = {
"operationName": opname,
"variables" : json.dumps(variables),
"extensions" : '{"persistedQuery":{"version":1'
',"sha256Hash":"' + query_hash + '"}}',
"query" : QUERIES[opname],
}
return self.request(url, params=params).json()["data"]
return self.request(url, method="POST", json=data).json()["data"]
class _500pxUserExtractor(_500pxExtractor):
@ -111,8 +109,6 @@ class _500pxUserExtractor(_500pxExtractor):
variables = {"username": self.user, "pageSize": 20}
photos = self._request_graphql(
"OtherPhotosQuery", variables,
"018a5e5117bd72bdf28066aad02c4f2d"
"8acdf7f6127215d231da60e24080eb1b",
)["user"]["photos"]
while True:
@ -124,8 +120,6 @@ class _500pxUserExtractor(_500pxExtractor):
variables["cursor"] = photos["pageInfo"]["endCursor"]
photos = self._request_graphql(
"OtherPhotosPaginationContainerQuery", variables,
"b4af70d42c71a5e43f0be36ce60dc81e"
"9742ebc117cde197350f2b86b5977d98",
)["userByUsername"]["photos"]
@ -159,7 +153,6 @@ class _500pxGalleryExtractor(_500pxExtractor):
def metadata(self):
user = self._request_graphql(
"ProfileRendererQuery", {"username": self.user_name},
"fcecc7028c308115b0defebc63acec3fe3c12df86a602c3e1785ba5cfb8fff47",
)["profile"]
self.user_id = str(user["legacyId"])
@ -172,7 +165,6 @@ class _500pxGalleryExtractor(_500pxExtractor):
}
gallery = self._request_graphql(
"GalleriesDetailQueryRendererQuery", variables,
"eda3c77ca4efe4b3347ec9c08befe3bd2c58099ebfb1f680d829fcd26d34f12d",
)["gallery"]
self._photos = gallery["photos"]
@ -261,3 +253,394 @@ class _500pxImageExtractor(_500pxExtractor):
def photos(self):
edges = ({"node": {"legacyId": self.photo_id}},)
return self._extend(edges)
QUERIES = {
"OtherPhotosQuery": """\
query OtherPhotosQuery($username: String!, $pageSize: Int) {
user: userByUsername(username: $username) {
...OtherPhotosPaginationContainer_user_RlXb8
id
}
}
fragment OtherPhotosPaginationContainer_user_RlXb8 on User {
photos(first: $pageSize, privacy: PROFILE, sort: ID_DESC) {
edges {
node {
id
legacyId
canonicalPath
width
height
name
isLikedByMe
notSafeForWork
photographer: uploader {
id
legacyId
username
displayName
canonicalPath
followedByUsers {
isFollowedByMe
}
}
images(sizes: [33, 35]) {
size
url
jpegUrl
webpUrl
id
}
__typename
}
cursor
}
totalCount
pageInfo {
endCursor
hasNextPage
}
}
}
""",
"OtherPhotosPaginationContainerQuery": """\
query OtherPhotosPaginationContainerQuery($username: String!, $pageSize: Int, $cursor: String) {
userByUsername(username: $username) {
...OtherPhotosPaginationContainer_user_3e6UuE
id
}
}
fragment OtherPhotosPaginationContainer_user_3e6UuE on User {
photos(first: $pageSize, after: $cursor, privacy: PROFILE, sort: ID_DESC) {
edges {
node {
id
legacyId
canonicalPath
width
height
name
isLikedByMe
notSafeForWork
photographer: uploader {
id
legacyId
username
displayName
canonicalPath
followedByUsers {
isFollowedByMe
}
}
images(sizes: [33, 35]) {
size
url
jpegUrl
webpUrl
id
}
__typename
}
cursor
}
totalCount
pageInfo {
endCursor
hasNextPage
}
}
}
""",
"ProfileRendererQuery": """\
query ProfileRendererQuery($username: String!) {
profile: userByUsername(username: $username) {
id
legacyId
userType: type
username
firstName
displayName
registeredAt
canonicalPath
avatar {
...ProfileAvatar_avatar
id
}
userProfile {
firstname
lastname
state
country
city
about
id
}
socialMedia {
website
twitter
instagram
facebook
id
}
coverPhotoUrl
followedByUsers {
totalCount
isFollowedByMe
}
followingUsers {
totalCount
}
membership {
expiryDate
membershipTier: tier
photoUploadQuota
refreshPhotoUploadQuotaAt
paymentStatus
id
}
profileTabs {
tabs {
name
visible
}
}
...EditCover_cover
photoStats {
likeCount
viewCount
}
photos(privacy: PROFILE) {
totalCount
}
licensingPhotos(status: ACCEPTED) {
totalCount
}
portfolio {
id
status
userDisabled
}
}
}
fragment EditCover_cover on User {
coverPhotoUrl
}
fragment ProfileAvatar_avatar on UserAvatar {
images(sizes: [MEDIUM, LARGE]) {
size
url
id
}
}
""",
"GalleriesDetailQueryRendererQuery": """\
query GalleriesDetailQueryRendererQuery($galleryOwnerLegacyId: ID!, $ownerLegacyId: String, $slug: String, $token: String, $pageSize: Int, $gallerySize: Int) {
galleries(galleryOwnerLegacyId: $galleryOwnerLegacyId, first: $gallerySize) {
edges {
node {
legacyId
description
name
privacy
canonicalPath
notSafeForWork
buttonName
externalUrl
cover {
images(sizes: [35, 33]) {
size
webpUrl
jpegUrl
id
}
id
}
photos {
totalCount
}
id
}
}
}
gallery: galleryByOwnerIdAndSlugOrToken(ownerLegacyId: $ownerLegacyId, slug: $slug, token: $token) {
...GalleriesDetailPaginationContainer_gallery_RlXb8
id
}
}
fragment GalleriesDetailPaginationContainer_gallery_RlXb8 on Gallery {
id
legacyId
name
privacy
notSafeForWork
ownPhotosOnly
canonicalPath
publicSlug
lastPublishedAt
photosAddedSinceLastPublished
reportStatus
creator {
legacyId
id
}
cover {
images(sizes: [33, 32, 36, 2048]) {
url
size
webpUrl
id
}
id
}
description
externalUrl
buttonName
photos(first: $pageSize) {
totalCount
edges {
cursor
node {
id
legacyId
canonicalPath
name
description
category
uploadedAt
location
width
height
isLikedByMe
photographer: uploader {
id
legacyId
username
displayName
canonicalPath
avatar {
images(sizes: SMALL) {
url
id
}
id
}
followedByUsers {
totalCount
isFollowedByMe
}
}
images(sizes: [33, 32]) {
size
url
webpUrl
id
}
__typename
}
}
pageInfo {
endCursor
hasNextPage
}
}
}
""",
"GalleriesDetailPaginationContainerQuery": """\
query GalleriesDetailPaginationContainerQuery($ownerLegacyId: String, $slug: String, $token: String, $pageSize: Int, $cursor: String) {
galleryByOwnerIdAndSlugOrToken(ownerLegacyId: $ownerLegacyId, slug: $slug, token: $token) {
...GalleriesDetailPaginationContainer_gallery_3e6UuE
id
}
}
fragment GalleriesDetailPaginationContainer_gallery_3e6UuE on Gallery {
id
legacyId
name
privacy
notSafeForWork
ownPhotosOnly
canonicalPath
publicSlug
lastPublishedAt
photosAddedSinceLastPublished
reportStatus
creator {
legacyId
id
}
cover {
images(sizes: [33, 32, 36, 2048]) {
url
size
webpUrl
id
}
id
}
description
externalUrl
buttonName
photos(first: $pageSize, after: $cursor) {
totalCount
edges {
cursor
node {
id
legacyId
canonicalPath
name
description
category
uploadedAt
location
width
height
isLikedByMe
photographer: uploader {
id
legacyId
username
displayName
canonicalPath
avatar {
images(sizes: SMALL) {
url
id
}
id
}
followedByUsers {
totalCount
isFollowedByMe
}
}
images(sizes: [33, 32]) {
size
url
webpUrl
id
}
__typename
}
}
pageInfo {
endCursor
hasNextPage
}
}
}
""",
}

2
setup.cfg
View File

@ -1,3 +1,5 @@
[flake8]
exclude = gallery_dl/__init__.py,gallery_dl/__main__.py,setup.py,build,scripts,archive
ignore = E203,E226,W504
per-file-ignores =
gallery_dl/extractor/500px.py: E501