Browse Source

Merge branch 'rust_backend'

master
TGRCDev 1 month ago
parent
commit
fa28b91252
Signed by: tgrcdev <tgrc@tgrc.dev> GPG Key ID: 3A975199085322C3
16 changed files with 1452 additions and 580 deletions
  1. +6
    -1
      .gitignore
  2. +353
    -126
      __init__.py
  3. +1
    -6
      config_EXAMPLE.json
  4. +0
    -251
      igdb_utils.py
  5. +15
    -0
      lib/wcwp_rust/Cargo.toml
  6. +80
    -0
      lib/wcwp_rust/src/errors.rs
  7. +306
    -0
      lib/wcwp_rust/src/igdb.rs
  8. +5
    -0
      lib/wcwp_rust/src/lib.rs
  9. +311
    -0
      lib/wcwp_rust/src/python.rs
  10. +303
    -0
      lib/wcwp_rust/src/steam.rs
  11. +19
    -0
      lib/wcwp_rust/src/wcwp.rs
  12. +31
    -15
      static/scripts/app.js
  13. +5
    -0
      static/styles/app.css
  14. +12
    -0
      static/styles/default_dark.css
  15. +0
    -179
      steam_utils.py
  16. +5
    -2
      templates/base_page.html

+ 6
- 1
.gitignore View File

@@ -3,4 +3,9 @@ config.json
venv*
.vscode
igdb-cache.sqlite
bearer-token.json
bearer-token.json
lib/wcwp_rust/target
lib/wcwp_rust/Cargo.lock
lib/whatcanweplay.pyd
lib/bin
.wcwp-commit-hash

+ 353
- 126
__init__.py View File

@@ -14,18 +14,51 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.

try:
import shutil, platform, os
path_prefix = os.path.dirname(os.path.abspath(__file__))
source = ""
dest = ""
if platform.system() == "Windows":
source = "lib/wcwp_rust/target/release/whatcanweplay.dll"
dest = "lib/bin/whatcanweplay.pyd"
else:
source = "lib/wcwp_rust/target/release/libwhatcanweplay.so"
dest = "lib/bin/whatcanweplay.so"
source = os.path.join(path_prefix, source)
dest = os.path.join(path_prefix, dest)

if not os.path.exists(dest):
os.makedirs(os.path.dirname(dest), exist_ok=True)
shutil.copy2(source, dest)
else:
source_time = os.path.getmtime(source)
dest_time = os.path.getmtime(dest)
try:
if dest_time < source_time:
print("Updating WhatCanWePlay rust library with newer library file...")
shutil.copy2(source, dest)
except Exception:
print("Failed to update WhatCanWePlay rust library. It will be re-attempted when next launched.")
from .lib.bin import whatcanweplay as wcwp
except Exception as e:
print("Failed to load WhatCanWePlay rust library. Please go to \"lib/wcwp_rust\" and run \"cargo build --release\"")
print(e)
raise e
print("WhatCanWePlay rust lib loaded (" + str(wcwp.__file__) + ")")

from flask import Flask, request, jsonify, Response, render_template, redirect, session, url_for, make_response
import requests
from urllib import parse
from werkzeug.exceptions import BadRequest
import json
from .steam_utils import get_steam_user_info, get_steam_user_friend_list, get_owned_steam_games
from .igdb_utils import get_steam_game_info
from requests import HTTPError
import secrets
from datetime import timezone, datetime, timedelta
from itsdangerous import URLSafeSerializer
from os import path
import traceback

# Load config
def create_app():
@@ -33,25 +66,30 @@ def create_app():
config = json.load(open(path.join(root_path, "config.json"), "r"))
steam_key = config["steam-key"]
igdb_key = config["igdb-client-id"]
igdb_secret = config["igdb-secret"]
debug = config.get("debug", config.get("DEBUG", False))
enable_api_tests = config.get("enable-api-tests", debug)
cookie_max_age_dict = config.get("cookie-max-age", {})
info_max_age_dict = config.get("info-max-age", {})
cache_max_age_dict = config.get("igdb-cache-max-age", config.get("igdb-cache-info-age", {}))
source_url = config.get("source-url", "")
contact_email = config["contact-email"]
privacy_email = config.get("privacy-email", contact_email)
connect_timeout = config.get("connect-timeout", 0.0)
commit_hash_filename = config.get("commit-hash-file", ".wcwp-commit-hash")
donate_url = config.get("donate-url", "")
if connect_timeout <= 0.0:
connect_timeout = None
read_timeout = config.get("read-timeout", 0.0)
if read_timeout <= 0.0:
read_timeout = None
cache_file = config.get("igdb-cache-file")
if not os.path.isabs(cache_file):
cache_file = os.path.join(root_path, cache_file)

# Create uWSGI callable
app = Flask(__name__)
app.debug = debug
app.secret_key = config.get("secret-key", secrets.token_hex()) # If not set, cookies will be invalidated every time the app is reloaded
app.secret_key = config["secret-key"]

# Hide requests to /steam_login to prevent linking Steam ID to IP in logs
from werkzeug import serving
@@ -68,20 +106,61 @@ def create_app():
cookie_max_age = timedelta(**cookie_max_age_dict).total_seconds()
if cookie_max_age == 0:
cookie_max_age = None
info_max_age = timedelta(**info_max_age_dict).total_seconds()
# Setup cache info max age
cache_max_age = 0.0
if cache_file:
cache_max_age = timedelta(**cache_max_age_dict).total_seconds()

print("cookies set to expire after %f seconds" % cookie_max_age)
print("cache set to expire after %f seconds" % cache_max_age)

def fetch_and_store_commit_hash():
f = open(commit_hash_filename, "w")
import subprocess
args = ['--git-dir=' + os.path.join(os.path.abspath(os.path.dirname(__file__)), ".git"), 'rev-parse', '--short', 'HEAD']
try:
try:
commit_hash = subprocess.check_output(['git'] + args).decode("utf-8").strip()
f.write(commit_hash)
f.close()
return commit_hash
except:
commit_hash = subprocess.check_output(['/usr/bin/git'] + args).decode("utf-8").strip()
f.write(commit_hash)
f.close()
return commit_hash
except:
return ""

@app.before_first_request
def before_first_request():
fetch_and_store_commit_hash()

print("cookies set to expire after {} seconds".format(cookie_max_age))
print("steam info set to refresh after {} seconds".format(info_max_age))
def get_commit_hash():
try:
f = open(commit_hash_filename, "r")
hash = f.read()
return hash
except Exception:
traceback.print_exc()
return ""

def basic_info_dict():
email_rev = contact_email.split("@")
return {
basic_info = {
"contact_email_user_reversed": email_rev[0][::-1],
"contact_email_domain_reversed": email_rev[1][::-1],
"source_url": source_url,
"donate_url": donate_url
}

commit = get_commit_hash()
if commit:
basic_info["commit"] = commit
return basic_info

# Tries to fetch the Steam info cookie, returns an errcode and a dict
#
# Errcodes:
@@ -113,18 +192,20 @@ def create_app():

def refresh_steam_cookie(steamid: int, response):
if steamid <= 0:
response.set_cookie("steam_info", "", secure=True)
return {}

info = get_steam_user_info(steam_key, [steamid], connect_timeout, read_timeout)

if info["errcode"] != 0 or not info["users"].get(steamid, {}).get("exists", False):
response.set_cookie("steam_info", "", secure=True)
response.set_cookie("steam_info", "", secure=True, httponly=True)
return {}
info = info["users"][steamid]
if info_max_age:
info["expires"] = (datetime.now(timezone.utc) + timedelta(seconds=info_max_age)).timestamp()
info = {}
try:
info = wcwp.steam.get_steam_users_info(steam_key, [steamid])[0]
except IndexError:
response.set_cookie("steam_info", "", secure=True, httponly=True)
return {}
except Exception:
traceback.print_exc()
response.set_cookie("steam_info", "", secure=True, httponly=True)
return {}

ser = URLSafeSerializer(app.secret_key)
response.set_cookie(
"steam_info",
@@ -143,7 +224,7 @@ def create_app():
if errcode == 3:
steam_info = refresh_steam_cookie(steam_info.get("steam_id", -1), response)
elif errcode != 0:
response.set_cookie("steam_info", "", secure=True)
response.set_cookie("steam_info", "", secure=True, httponly=True)
steam_info = {}
response.data = render_template("home.html", steam_info=steam_info, **basic_info_dict())
@@ -157,7 +238,7 @@ def create_app():
def steam_login():
if request.method == "POST":
steam_openid_url = 'https://steamcommunity.com/openid/login'
return_url = url_for("steam_login", _external=True)
return_url = request.base_url
params = {
'openid.ns': "http://specs.openid.net/auth/2.0",
'openid.identity': "http://specs.openid.net/auth/2.0/identifier_select",
@@ -183,7 +264,7 @@ def create_app():
def steam_logout():
response = redirect(url_for("index"))
response.headers["X-Robots-Tag"] = "none"
response.set_cookie("steam_info", "", secure=True)
response.set_cookie("steam_info", "", secure=True, httponly=True)
return response

def validate_steam_identity(params):
@@ -214,46 +295,196 @@ def create_app():
)
errcode, steam_info = fetch_steam_cookie(request)
if "steam_id" not in steam_info.keys():
return ("Not signed in to Steam", 403)
return (
"Not signed in to Steam. Please refresh the page.",
403
)
friends_info = get_steam_user_friend_list(
steam_key,
steam_info["steam_id"],
connect_timeout,
read_timeout
)
try:
friends_info = wcwp.steam.get_friend_list(
steam_key,
steam_info["steam_id"]
)
for user in friends_info:
if "steam_id" in user.keys():
user["steam_id"] = str(user["steam_id"])
user["exists"] = True

return jsonify(friends_info)
except wcwp.steam.BadWebkeyException:
traceback.print_exc()
return (
"Site has bad Steam API key. Please contact us about this error at " + contact_email,
500
)
except wcwp.steam.ServerErrorException:
traceback.print_exc()
return (
"Steam had an internal server error. Please try again later.",
500
)
except wcwp.steam.BadResponseException:
traceback.print_exc()
return (
"Steam returned an unparseable response. Please try again later.",
500
)
except wcwp.steam.FriendListPrivateException:
traceback.print_exc()
return (
"WhatCanWePlay cannot retrieve your friend list. Please change your friend list visibility to public and refresh the page.",
500
)
except Exception:
traceback.print_exc()
if debug:
return (
traceback.format_exc(),
500
)
else:
traceback.print_exc()
return (
"An unknown error has occurred. Please try again later.",
500
)

errcode = friends_info.pop("errcode")

if errcode == 1:
return ("Site has bad Steam API key. Please contact us about this error at " + contact_email, 500)
elif errcode == 2:
return ("Steam took too long to respond. Please try again later.", 500)
elif errcode == 3:
return ("Steam took too long to transmit info. Please try again later.", 500)
elif errcode == 4:
return ("Your Friend List is not publicly accessible, and cannot be retrieved by WhatCanWePlay. Please set your Friend list visibility to Public and refresh the page.", 500)
elif errcode == -1:
return ("An unknown error occurred. Please try again later.", 500)
def refresh_igdb_token():
try:
token_path = path.join(root_path, "bearer-token.json")
token = wcwp.igdb.fetch_twitch_token(igdb_key, igdb_secret)
token["expiry"] = datetime.now(timezone.utc).timestamp() + token.get("expires_in", 0)
token.pop("expires_in")
json.dump(token, open(token_path, "w"))
return token.get("access_token", "")
except Exception:
traceback.print_exc()
return ""

def get_igdb_token():
try:
token_path = path.join(root_path, "bearer-token.json")
if path.exists(token_path):
token_file = open(token_path)
token = json.load(token_file)
token_file.close()
if datetime.now(timezone.utc).timestamp() >= token.get("expiry", 0):
return refresh_igdb_token()
else:
return token.get("access_token", "")
else:
return refresh_igdb_token()
except Exception:
traceback.print_exc()
return ""

cache_init_query = """
CREATE TABLE IF NOT EXISTS game (
steam_id INTEGER PRIMARY KEY,
igdb_id INTEGER,
name STRING,
supported_players INTEGER DEFAULT(0),
cover_id STRING,
has_multiplayer BOOLEAN,
expiry REAL DEFAULT(0.0)
);
"""

CACHE_VERSION = 1

def initialize_cache():
import sqlite3
cache = sqlite3.connect(cache_file)
cache.execute(cache_init_query)
#cache.execute("PRAGMA user_version = ?;", [CACHE_VERSION]) # Doesn't work?
cache.execute("PRAGMA user_version = %d" % CACHE_VERSION)
return cache
def cache_is_correct_version(cache):
return cache.execute("PRAGMA user_version;").fetchone()[0] == CACHE_VERSION

def update_cached_games(game_info):
if not cache_file:
return
friends_info = get_steam_user_info(steam_key, friends_info["friends"], connect_timeout, read_timeout)

errcode = friends_info.pop("errcode")
if errcode == 1:
return ("Site has bad Steam API key. Please contact us about this error at " + contact_email, 500)
elif errcode == 2:
return ("Steam took too long to respond. Please try again later.", 500)
elif errcode == 3:
return ("Steam took too long to transmit info. Please try again later.", 500)
elif errcode == -1:
return ("An unknown error occurred. Please try again later.", 500)
try:
import sqlite3
cache = None

if os.path.exists(cache_file):
cache = sqlite3.connect(cache_file)
# Check if cache is correct version
if not cache_is_correct_version(cache):
# Cache is the wrong version, rebuild
print("Cache file is the wrong version! Rebuilding... ")
cache.close()
os.remove(cache_file)
cache = initialize_cache()
else:
cache = initialize_cache()
insert_info = [
[
game.get("steam_id"),
game.get("igdb_id"),
game.get("name"),
game.get("supported_players"),
game.get("cover_id"),
game.get("has_multiplayer"),
datetime.now(timezone.utc).timestamp() + cache_max_age
] for game in game_info
]
cache.executemany(
"INSERT OR REPLACE INTO game VALUES (?,?,?,?,?,?,?);",
insert_info
)

cache.commit()
cache.close()
except Exception:
print("FAILED TO UPDATE CACHE DB")
traceback.print_exc()
return
# returns [info of cached games], (set of uncached ids)
def get_cached_games(steam_ids):
if not cache_file:
return [], set(steam_ids)

game_info = []
uncached = set(steam_ids)
for user in friends_info["users"].values():
if "steam_id" in user.keys():
user["steam_id"] = str(user["steam_id"])
user["exists"] = True
try:
import sqlite3
cache = sqlite3.connect(cache_file)
cache.row_factory = sqlite3.Row

return jsonify(friends_info["users"])
if not cache_is_correct_version(cache):
return [], set(steam_ids)
query_str = "SELECT * FROM game WHERE steam_id IN (%s)" % ("?" + (",?" * (len(steam_ids) - 1))) # Construct a query with arbitrary parameter length
cursor = cache.execute(
query_str,
steam_ids
)
for row in cursor.fetchall():
game = dict(row)
if datetime.now(timezone.utc).timestamp() < game.pop("expiry"):
# Info hasn't expired
game_info.append(game)
uncached.remove(game["steam_id"])

# Expired info gets updated during update_cached_games()
except Exception:
print("EXCEPTION THROWN WHILE QUERYING GAME CACHE!")
traceback.print_exc()
return [], set(steam_ids)
return game_info, uncached

# Errcodes
# -1: An error occurred with a message. Additional fields: "message"
@@ -316,90 +547,86 @@ def create_app():
json.dumps({"message": "Games intersection is capped at 10 users.", "errcode": -1}),
200
)
free_games = bool(body.get("include_free_games", False))

all_own = None
try:
token = get_igdb_token()
game_ids = wcwp.steam.intersect_owned_game_ids(steam_key, list(steamids))

for steamid in steamids:
user_owned_games = get_owned_steam_games(steam_key, steamid, free_games, connect_timeout, read_timeout)
errcode = user_owned_games["errcode"]
fetched_game_count = 0
cached_game_count = 0
game_info = []

if errcode == 1:
return (
json.dumps({"message": "Site has bad Steam API key. Please contact us about this error at {}.".format(contact_email), "errcode": -1}),
200
)
elif errcode == 2:
return (
json.dumps({"message": "Steam took too long to respond. Please try again later.", "errcode": -1}),
200
)
elif errcode == 3:
return (
json.dumps({"message": "Steam took too long to transmit info. Please try again later.", "errcode": -1}),
200
)
elif errcode == 4:
return (
json.dumps({"user": str(steamid), "errcode": 1}),
200
)
elif errcode == -1:
return (
json.dumps({"message": "An unknown error occurred. Please try again later.", "errcode": -1}),
200
)
games = user_owned_games.get("games")
if len(games) == 0:
return (
json.dumps({"user": str(steamid), "errcode": 2}),
200
)
if not all_own:
all_own = set(games)
else:
all_own = all_own & set(games)
if len(all_own) == 0:
break
# Step two: Fetch the game info
game_info = get_steam_game_info(igdb_key, all_own, connect_timeout, read_timeout)
if game_ids:
game_info, uncached_ids = get_cached_games(game_ids)

errcode = game_info["errcode"]
cached_game_count = len(game_info)

if errcode == 1:
if uncached_ids:
fetched_info, not_found = wcwp.igdb.get_steam_game_info(igdb_key, token, list(uncached_ids))

cache_info_update = fetched_info
if not_found:
for uncached_id in [id for id in not_found]:
cache_info_update.append({"steam_id": uncached_id}) # Cache empty data to prevent further IGDB fetch attempts
update_cached_games(cache_info_update) # TODO: Spin up separate process for caching?

game_info += fetched_info
fetched_game_count = len(fetched_info)
print("Intersection resulted in %d games (%d from cache, %d from IGDB)" % (len(game_info), cached_game_count, fetched_game_count))

return jsonify({
"message": "Intersected successfully",
"games": game_info,
"errcode": 0
})
except wcwp.steam.BadWebkeyException:
traceback.print_exc()
return (
json.dumps({"message": "Site has bad IGDB API key. Please contact us about this error at {}.".format(contact_email), "errcode": -1}),
200
json.dumps({"message": "Site has bad Steam API key. Please contact us about this error at " + contact_email, "errcode": -1}),
500
)
elif errcode == 2:
except wcwp.steam.ServerErrorException:
traceback.print_exc()
return (
json.dumps({"message": "IGDB took too long to respond. Please try again later.", "errcode": -1}),
200
json.dumps({"message": "Steam had an internal server error. Please try again later.", "errcode": -1}),
500
)
elif errcode == 3:
except wcwp.steam.BadResponseException:
traceback.print_exc()
return (
json.dumps({"message": "IGDB took too long to transmit info. Please try again later.", "errcode": -1}),
200
json.dumps({"message": "Steam returned an unparseable response. Please try again later.", "errcode": -1}),
500
)
elif errcode == 4:
except wcwp.steam.GamesListPrivateException as e:
if debug:
print(e)
else:
print("Intersection interrupted due to private games list")
return (
json.dumps({"message": "WhatCanWePlay failed to acquire an IGDB token. Please contact us about this error at {}.".format(contact_email), "errcode": -1})
json.dumps({"errcode": 1, "user": str(e.args[1])}),
500
)
elif errcode == -1:
except wcwp.steam.GamesListEmptyException as e:
if debug:
print(e)
else:
print("Intersection interrupted due to private games list")
return (
json.dumps({"message": "An unknown error occurred. Please try again later.", "errcode": -1}),
200
json.dumps({"errcode": 2, "user": str(e.args[1])}),
500
)

return jsonify({
"message": "Intersected successfully",
"games": list(game_info.get("games", {}).values()),
"errcode": 0
})
except Exception:
traceback.print_exc()
if debug:
return (
json.dumps({"message": traceback.format_exc(), "errcode": -1}),
500
)
else:
return (
json.dumps({"message": "An unknown error has occurred. Please try again later.", "errcode": -1}),
500
)

return app

+ 1
- 6
config_EXAMPLE.json View File

@@ -4,22 +4,17 @@
"steam-key": "YOUR STEAM KEY HERE",
"igdb-client-id": "YOUR IGDB KEY HERE",
"igdb-secret": "YOUR IGDB SECRET HERE",
"www-host": "yoururl.here",
"cookie-max-age": {
"days": 0,
"weeks": 0
},
"info-max-age": {
"hours": 0,
"minutes": 0
},
"source-url": "https://git.tgrc.dev/tgrcdev/whatcanweplay",
"contact-email": "contact@whatcanweplay.net",
"privacy-email": "privacy@whatcanweplay.net",
"connect-timeout": 8,
"read-timeout": 30,
"igdb-cache-file": "igdb-cache.sqlite",
"igdb-cache-info-age": {
"igdb-cache-max-age": {
"weeks": 4
}
}

+ 0
- 251
igdb_utils.py View File

@@ -1,251 +0,0 @@
# This file is a part of WhatCanWePlay
# Copyright (C) 2020 TGRCDev

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.

import requests
import json
import sqlite3
from requests.exceptions import ConnectTimeout, ReadTimeout
from typing import Dict, Collection, Any, Mapping, Optional
from datetime import timedelta, datetime, timezone
from os import path

api_base = "https://api.igdb.com/v4/"

root_path = path.dirname(__file__)
config = json.load(open(path.join(root_path, "config.json"), "r"))
debug = config.get("debug", config.get("DEBUG", False))
db_filename = config.get("igdb-cache-filename", "igdb-cache.sqlite")
if not path.isabs(db_filename):
db_filename = path.join(root_path, db_filename)

create_db_query = """
CREATE TABLE IF NOT EXISTS game_info (
steam_id INTEGER PRIMARY KEY,
igdb_id INTEGER,
name TEXT,
cover_id TEXT,
has_multiplayer BOOL,
supported_players TEXT,
time_cached REAL
);
"""

info_age_dict = config.get("igdb-cache-info-age", {})
info_age = timedelta(**info_age_dict).total_seconds()

def fetch_and_store_token() -> str:
r = requests.post(
"https://id.twitch.tv/oauth2/token",
json={
"client_id": config["igdb-client-id"],
"client_secret": config["igdb-secret"],
"grant_type": "client_credentials"
}
)

if debug:
r.raise_for_status()
try:
token = r.json()
token_dict = {
"token": token["access_token"],
"expire-time": datetime.now(timezone.utc).timestamp() + token["expires_in"],
"token_type": token["token_type"]
}
token_file = open(path.join(root_path, "bearer-token.json"), "w")
json.dump(token_dict, token_file)
token_file.close()
return token["access_token"]
except Exception as e:
print("Exception thrown while parsing token return")
print(e)
return ""

def get_or_refresh_token() -> str:
try:
token_path = path.join(root_path, "bearer-token.json")
if path.exists(token_path):
token_file = open(token_path)
token = json.load(token_file)
token_file.close()
if datetime.now(timezone.utc).timestamp() >= token.get("expire-time", 0):
return fetch_and_store_token()
else:
return token.get("token", "")
else:
return fetch_and_store_token()
except json.JSONDecodeError as e:
print("Failed to decode token JSON.")
print(e)
return ""
except Exception as e:
print("Failed to fetch bearer token.")
print(e)
return ""

def get_cached_games(appids: Collection[int]) -> Dict[int, Dict[str, Any]]:
query = """
SELECT steam_id, igdb_id, name, cover_id, has_multiplayer, supported_players
FROM game_info
WHERE time_cached > ? AND steam_id in ({})
""".format(",".join(
["?" for _ in range(len(appids))]
))
db_handle = sqlite3.connect(db_filename)
db_handle.execute(create_db_query)

cursor = db_handle.cursor()
cursor.execute(query, [datetime.now(timezone.utc).timestamp() - info_age, *list(appids)])
results = cursor.fetchall()

return {
game[0]: {
"steam_id": game[0],
"name": game[2],
"cover_id": game[3],
"has_multiplayer": game[4],
"supported_players": game[5]
} if game[1] else {}
for game in results
}

def update_cached_games(game_info: Mapping[int, Mapping[str, Any]]):
query = """
INSERT OR REPLACE INTO game_info
(steam_id, igdb_id, name, cover_id, has_multiplayer, supported_players, time_cached)
VALUES (?,?,?,?,?,?,?)
"""
db_handle = sqlite3.connect(db_filename)
db_handle.execute(create_db_query)
cursor = db_handle.cursor()
now = datetime.now(timezone.utc).timestamp()
cursor.execute("BEGIN TRANSACTION")
cursor.executemany(query, [[
game.get("steam_id"),
game.get("igdb_id"),
game.get("name"),
game.get("cover_id"),
game.get("has_multiplayer"),
game.get("supported_players"),
now] for game in game_info.values()]
)
cursor.execute("END TRANSACTION")

def get_steam_game_info(webkey: str, appids: Collection[int], connect_timeout: Optional[float] = None, read_timeout: Optional[float] = None) -> Dict[int, Dict[str, Any]]:
if len(appids) == 0:
return {"errcode":0, "games":{}}
appid_set = set(appids)
cache_error = False

try:
cached_games = get_cached_games(appid_set)
except Exception as e:
print("An exception occurred while retrieving cached games: " + str(e))
cached_games = {}
cache_error = True

if len(cached_games) == len(appid_set):
return {"errcode": 0, "games": cached_games}

uncached_ids = appid_set - set(cached_games.keys())
uncached_ids_list = list(uncached_ids)

token = get_or_refresh_token()
if not token:
return {"errcode": 4}

games_dict = {}
retrieved_games = 0
while retrieved_games < len(uncached_ids_list):
game_slice = uncached_ids_list[retrieved_games:500]

retrieved_games += len(game_slice)

try:
r = requests.post(
api_base + "external_games",
data = "fields uid,game.name,game.game_modes,game.multiplayer_modes.onlinemax,game.multiplayer_modes.onlinecoopmax,game.cover.image_id; where uid = ({}) & category = 1; limit {};".format(",".join(map(str, game_slice)), len(game_slice)),
headers = {
"Client-ID": webkey,
"Authorization": "Bearer " + token,
"Accept": "application/json"
},
timeout = (connect_timeout, read_timeout)
)
except ConnectTimeout:
return {"errcode": 2}
except ReadTimeout:
return {"errcode": 3}
if r.status_code == 403:
return {"errcode": 1}
elif r.status_code != 200:
if debug:
r.raise_for_status()
return {"errcode": -1}
for game in r.json():
steam_id = game.get("uid", None)
if not steam_id:
continue
steam_id = int(steam_id)
game = game.get("game", None)
if not game:
continue
uncached_ids.discard(steam_id)

game_modes = game.get("game_modes", [])

is_multiplayer = (2 in game_modes or 5 in game_modes)

maxplayers = -1
if is_multiplayer:
for mode in game.get("multiplayer_modes", []):
maxplayers = max(max(mode.get("onlinemax", 1), mode.get("onlinecoopmax", 1)), maxplayers)
else:
maxplayers = 1

games_dict[steam_id] = {
"steam_id": steam_id,
"igdb_id": game["id"],
"name": game["name"],
"cover_id": game.get("cover", {}).get("image_id", ""),
"has_multiplayer": is_multiplayer,
"supported_players": str(maxplayers) if maxplayers > 0 else "?"
}
# Any games that couldn't be retrieved probably dont exist. Store them so they don't trigger an IGDB fetch.
for nonexist_id in uncached_ids:
games_dict[nonexist_id] = {"steam_id": nonexist_id}
if len(games_dict) == 0:
return {"errcode":0, "games":cached_games}
for id in games_dict.keys():
cached_games[id] = games_dict[id]
if not cache_error:
try:
update_cached_games(games_dict)
except Exception as e:
print("An exception occurred while updating cached games: " + str(e))

return {
"errcode": 0,
"games": cached_games
}

+ 15
- 0
lib/wcwp_rust/Cargo.toml View File

@@ -0,0 +1,15 @@
[package]
name = "whatcanweplay"
version = "0.1.0"
authors = ["TGRCDev <tgrc@tgrc.dev>"]
edition = "2018"

[lib]
name = "whatcanweplay"
crate-type = ["cdylib"]

[dependencies]
reqwest = { version = "0.10", features = ["json", "blocking"] }
serde_json = "1.0"
serde = { version = "1.0", features = ["derive", "std"] }
pyo3 = { version = "0.12", features = ["extension-module"] }

+ 80
- 0
lib/wcwp_rust/src/errors.rs View File

@@ -0,0 +1,80 @@
#[derive(Debug)]
pub enum IGDBError {
UnknownError(String), // Unhandled error
ServerError, // IGDB had an internal error
BadResponse, // IGDB returned an unparseable response
BadClient, // The supplied client ID is wrong
BadSecret, // The supplied client secret is wrong
BadToken, // The supplied bearer token is wrong
BadAuth, // Some part of the supplied authentication is wrong
}

use std::fmt;

impl fmt::Display for IGDBError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
IGDBError::ServerError => return write!(f, "IGDB had an internal server error"),
IGDBError::BadResponse => return write!(f, "IGDB returned an unparseable response"),
IGDBError::BadClient => return write!(f, "IGDB rejected the provided client ID"),
IGDBError::BadSecret => return write!(f, "IGDB rejected the provided client secret"),
IGDBError::BadToken => return write!(f, "IGDB rejected the provided bearer token"),
IGDBError::BadAuth => return write!(f, "IGDB rejected some part of the provided authentication"),
IGDBError::UnknownError(err_string) => return write!(f, "{}", &err_string),
_ => return write!(f, "IGDB API had an unknown error")
}
}
}

#[derive(Debug)]
pub enum SteamError {
UnknownError(String), // Unhandled error
ServerError, // Steam had an internal error
BadResponse, // Steam returned an unparseable response
BadWebkey, // The supplied webkey is wrong
GamesListPrivate(u64), // The games list of the given steam id is unretrievable, probably due to game list privacy settings
GamesListEmpty(u64), // Intersection failed, the user with the given steam id had no games
FriendListPrivate, // The steam id passed to get_friend_list() has their friend list set to private
}

impl fmt::Display for SteamError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
SteamError::ServerError => return write!(f, "Steam had an internal server error"),
SteamError::BadResponse => return write!(f, "Steam returned an unparseable response"),
SteamError::BadWebkey => return write!(f, "Steam rejected the provided webkey"),
SteamError::UnknownError(err_string) => return write!(f, "{}", &err_string),
SteamError::GamesListPrivate(steamid) => return write!(f, "The user with Steam ID {} has their games list set to private", steamid),
SteamError::GamesListEmpty(steamid) => return write!(f, "The user with the Steam ID {} has no games to intersect", steamid),
SteamError::FriendListPrivate => return write!(f, "The given user has their friend list set to private"),
_ => return write!(f, "Steam had an unknown error")
}
}
}

#[derive(Debug)]
pub enum WCWPError {
SteamError(SteamError), // Steam API Error
IGDBError(IGDBError), // IGDB API Error
}

impl fmt::Display for WCWPError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
WCWPError::SteamError(e) => return e.fmt(f),
WCWPError::IGDBError(e) => return e.fmt(f),
}
}
}

impl From<SteamError> for WCWPError {
fn from(e: SteamError) -> Self {
return WCWPError::SteamError(e);
}
}

impl From<IGDBError> for WCWPError {
fn from(e: IGDBError) -> Self {
return WCWPError::IGDBError(e);
}
}

+ 306
- 0
lib/wcwp_rust/src/igdb.rs View File

@@ -0,0 +1,306 @@
use reqwest;
use serde::{Deserialize, Serialize};
use serde_json;
use std::collections::HashSet;
use std::fmt::{Debug, Write};
use crate::errors::IGDBError;
use reqwest::StatusCode;
use serde_json::Value;
use std::convert::{TryFrom, TryInto};
use std::cmp::{max, min};

const API_URL : &str = "https://api.igdb.com/v4/";
const TOKEN_URL : &str = "https://id.twitch.tv/oauth2/token";

#[derive(Serialize, Deserialize, Debug)]
/// Struct for retrieving a bearer token from the Twitch Developer API
pub struct Token {
pub access_token: String,
pub expires_in: u64
}

#[derive(Debug, Serialize)]
/// Struct for games described by IGDB
pub struct GameInfo {
pub steam_id: u64,
pub igdb_id: u64,
pub name: String,
pub supported_players: u8,
/// Cover image url
///
/// Usage: `https://images.igdb.com/igdb/image/upload/t_cover_small/<cover_id>.jpg`
pub cover_id: String,

pub has_multiplayer: bool,
}

/// Takes a serde_json Value and tries to get an i64
/// either by parsing an i64 directly, or by parsing
/// a stringified i64.
fn parse_u8_from_value(value: &Value) -> Option<u8> {
let num = parse_u64_from_value(value);
match num {
Some(num) =>
return num.try_into().ok(),
None =>
return None
}
}

/// Takes a serde_json Value and tries to get an i64
/// either by parsing an u64 directly, or by parsing
/// a stringified u64.
fn parse_u64_from_value(value: &Value) -> Option<u64> {
match value {
Value::Number(num) => {
return num.as_u64();
},
Value::String(string) => {
return string.parse().ok();
},
_ => return None
}
}

impl TryFrom<&Value> for GameInfo {
type Error = IGDBError;

/// Used to extract game information from IGDB query results
fn try_from(value: &Value) -> Result<Self, IGDBError> {
let steam_id : u64;
match parse_u64_from_value(&value["uid"]) {
Some(id) => steam_id = id,
None => return Err(IGDBError::BadResponse)
}

let game = &value["game"];
if !game.is_object()
{
return Err(IGDBError::BadResponse);
}

let igdb_id : u64;
match parse_u64_from_value(&game["id"]) {
Some(id) => igdb_id = id,
None => return Err(IGDBError::BadResponse)
}

let name : &str;
match game["name"].as_str() {
Some(val) => name = val,
None => return Err(IGDBError::BadResponse),
}

let cover_id : &str;
match game["cover"]["image_id"].as_str() {
Some(val) => cover_id = val,
None => cover_id = "",
}

// Figure out multiplayer and whatnot

let game_modes = &game["game_modes"];
let has_multiplayer: bool;
let supported_players : u8;

if game_modes.is_array() {
let gamemode_arr = game_modes.as_array().unwrap();
has_multiplayer = gamemode_arr.iter().any(|x| x == 5 || x == 2); // TODO: This isn't really needed with game["multiplayer_modes"], replace it
if has_multiplayer
{
let multiplayer_modes = &game["multiplayer_modes"];
if let Some(multimodes) = multiplayer_modes.as_array() {
let mut mostplayers = 0;
for mode in multimodes.iter() {
let onlinemax : u8 = parse_u8_from_value(&mode["onlinemax"]).unwrap_or(0);
let coopmax : u8 = parse_u8_from_value(&mode["onlinecoopmax"]).unwrap_or(0);
mostplayers = max(onlinemax, coopmax);
}
supported_players = max(min(255u8, mostplayers), 0u8).try_into().unwrap();
}
else
{
supported_players = 0; // Displayed as ? for number of supported players
}
}
else
{
supported_players = 1;
}
}
else
{
has_multiplayer = false;
supported_players = 1;
}

return Ok(GameInfo {
steam_id,
igdb_id,
name: name.to_string(),
cover_id: cover_id.to_string(),
supported_players,
has_multiplayer,
});
}
}

/// Fetches a Twitch app bearer token for use with the IGDB API.
///
/// # Errors
///
/// `IGDBError::BadClient` is returned if the client ID is invalid.
///
/// `IGDBError::BadSecret` is returned if the client secret is invalid.
///
/// `IGDBError::ServerError` is returned if Twitch returned an error code.
///
/// `IGDBError::UnknownError` is returned for any unexpected status codes, or if Twitch was unreachable.
pub fn get_twitch_token(client_id: &str, secret: &str) -> Result<Token, IGDBError> {
let client = reqwest::blocking::Client::new();

let res = client.post(TOKEN_URL)
.query(&[
("client_id", client_id),
("client_secret", secret),
("grant_type", "client_credentials")
]).send();
if let Err(e) = res {
return Err(IGDBError::UnknownError(e.to_string()));
}

let res = res.unwrap();
if !res.status().is_success() {
match res.status() {
StatusCode::BAD_REQUEST =>
return Err(IGDBError::BadClient),
StatusCode::FORBIDDEN =>
return Err(IGDBError::BadSecret),
StatusCode::INTERNAL_SERVER_ERROR | StatusCode::BAD_GATEWAY =>
return Err(IGDBError::ServerError),
_ =>
return Err(IGDBError::UnknownError(res.text().unwrap_or("Unknown error".to_string())))
}
}
let response = res.text().expect("Reqwest errored while getting response text");
let t: Token = serde_json::from_str(&response).expect(&format!("Serde failed to load JSON response from IGDB (response: {})", response));
return Ok(t);
}

/// Fetch the info for the provided set of steam app IDs
///
/// Returns a tuple of two `Vec`s, a `Vec` containing the info of found games, and a `Vec` of app IDs not found.
/// If `appids` is empty, returns two empty `Vec`s.
///
/// # Errors
///
/// `IGDBError::BadAuth` is returned if either the `client_id` or `bearer_token` are invalid.
///
/// `IGDBError::ServerError` is returned if IGDB was unable to process the request.
///
///`IGDBError::UnknownError` is returned for any unexpected status codes, or if IGDB was unreachable.
pub fn get_steam_game_info(client_id: &str, bearer_token: &str, appids: &[u64]) -> Result<(Vec<GameInfo>, HashSet<u64>), IGDBError> {
if appids.is_empty()
{
return Ok((Vec::new(), HashSet::new()));
}

let client = reqwest::blocking::Client::new();
let mut games_info : Vec<GameInfo> = Vec::new();
let mut not_found : HashSet<u64> = HashSet::with_capacity(appids.len());

for current_slice in appids.chunks(500)
{
let mut id_str = current_slice[0].to_string();

for n in current_slice[1..].iter()
{
write!(&mut id_str, ",{}", n).unwrap();
}

let response = client.post(&format!("{}{}", API_URL, "external_games"))
.header("Client-ID", client_id)
.header("Authorization", format!("Bearer {}", bearer_token))
.header("Accept", "application/json")
.body(format!(
"fields uid,game.name,game.game_modes,game.multiplayer_modes.onlinemax,
game.multiplayer_modes.onlinecoopmax,game.cover.image_id;
where uid = ({}) & category = 1; limit {};",
id_str, current_slice.len()
)).send();

if let Err(e) = response {
return Err(IGDBError::UnknownError(e.to_string()));
}

let response = response.unwrap();
if !response.status().is_success() {
match response.status() {
StatusCode::UNAUTHORIZED => {
return Err(IGDBError::BadClient);
},
StatusCode::FORBIDDEN => {
return Err(IGDBError::BadAuth);
},
StatusCode::INTERNAL_SERVER_ERROR | StatusCode::BAD_GATEWAY =>
return Err(IGDBError::ServerError),
_ =>
return Err(IGDBError::UnknownError(
response.text().unwrap_or("Unknown error".to_string()))
),
}
}

for id in current_slice.iter()
{ not_found.insert(*id); }

if let Ok(games_json) = response.json::<Value>() {
if let Some(games) = games_json.as_array() {
for game in games.iter() {
if let Ok(game_info) = GameInfo::try_from(game) {
not_found.remove(&game_info.steam_id);
games_info.push(game_info);
}
}
}
}
}

return Ok((games_info, not_found));
}

#[test]
fn unwrap_token() {
let test_token = "
{
\"access_token\": \"thisisnotarealtokenbutpretenditis\",
\"refresh_token\": \"\",
\"expires_in\": 3600,
\"scope\": [],
\"token_type\": \"bearer\"
}";

let t: Token = serde_json::from_str(test_token).unwrap();

println!("{:#?}", t);
}

#[test]
fn unwrap_game() {
let games = "[{\"uid\": \"1\", \"game\":{\"id\": 5, \"name\":\"Test\"}}]";

let games_json: Value = serde_json::from_str(games).unwrap();

for game in games_json.as_array().unwrap()
{
let g: GameInfo = game.try_into().unwrap();
println!("{:#?}", g);
}
}

+ 5
- 0
lib/wcwp_rust/src/lib.rs View File

@@ -0,0 +1,5 @@
pub mod igdb;
pub mod wcwp;
pub mod steam;
pub mod errors;
mod python;

+ 311
- 0
lib/wcwp_rust/src/python.rs View File

@@ -0,0 +1,311 @@
use pyo3::prelude::*;
use pyo3::wrap_pyfunction;
use pyo3::types::{PyTuple, PyList};

pub mod conversions {
use pyo3::prelude::*;
use pyo3::types::PyDict;
use pyo3::conversion::ToPyObject;

use crate::igdb;
impl IntoPy<PyObject> for igdb::Token {
fn into_py(self, py: Python) -> PyObject {
let obj = PyDict::new(py);
obj.set_item("access_token", self.access_token).unwrap();
obj.set_item("expires_in", self.expires_in).unwrap();
return obj.into();
}
}

impl IntoPy<PyObject> for igdb::GameInfo {
fn into_py(self, py: Python) -> PyObject {
let obj = PyDict::new(py);
obj.set_item("steam_id", self.steam_id).unwrap();
obj.set_item("igdb_id", self.igdb_id).unwrap();
obj.set_item("name", self.name).unwrap();
obj.set_item("supported_players", self.supported_players).unwrap();
obj.set_item("cover_id", self.cover_id).unwrap();
obj.set_item("has_multiplayer", self.has_multiplayer).unwrap();

return obj.into();
}
}

impl ToPyObject for igdb::GameInfo {
fn to_object(&self, py: Python) -> PyObject {
let obj = PyDict::new(py);
obj.set_item("steam_id", self.steam_id).unwrap();
obj.set_item("igdb_id", self.igdb_id).unwrap();
obj.set_item("name", self.name.clone()).unwrap();
obj.set_item("supported_players", self.supported_players).unwrap();
obj.set_item("cover_id", self.cover_id.clone()).unwrap();
obj.set_item("has_multiplayer", self.has_multiplayer).unwrap();

return obj.into();
}
}

use crate::steam;
impl IntoPy<PyObject> for steam::SteamUser {
fn into_py(self, py: Python) -> PyObject {
let obj = PyDict::new(py);
obj.set_item("steam_id", self.steam_id).unwrap();
obj.set_item("screen_name", self.screen_name).unwrap();
obj.set_item("avatar_thumb", self.avatar_thumb).unwrap();
obj.set_item("avatar", self.avatar).unwrap();
obj.set_item("visibility", self.visibility).unwrap();
obj.set_item("online", self.online).unwrap();

return obj.into();
}
}

impl ToPyObject for steam::SteamUser {
fn to_object(&self, py: Python) -> PyObject {
let obj = PyDict::new(py);
obj.set_item("steam_id", self.steam_id).unwrap();
obj.set_item("screen_name", self.screen_name.clone()).unwrap();
obj.set_item("avatar_thumb", self.avatar_thumb.clone()).unwrap();
obj.set_item("avatar", self.avatar.clone()).unwrap();
obj.set_item("visibility", self.visibility).unwrap();
obj.set_item("online", self.online).unwrap();

return obj.into();
}
}
}

use crate::igdb;

pub mod igdb_exceptions {
use pyo3::create_exception;
use pyo3::exceptions::{PyException};
use pyo3::PyErr;

use crate::errors::IGDBError;

create_exception!(igdb, IGDBException, PyException);

create_exception!(igdb, UnknownErrorException, IGDBException);
create_exception!(igdb, ServerErrorException, IGDBException);
create_exception!(igdb, BadClientException, IGDBException);
create_exception!(igdb, BadResponseException, IGDBException);
create_exception!(igdb, BadSecretException, IGDBException);
create_exception!(igdb, BadTokenException, IGDBException);
create_exception!(igdb, BadAuthException, IGDBException);

impl From<IGDBError> for PyErr {
fn from(e: IGDBError) -> PyErr {
match e {
IGDBError::ServerError => ServerErrorException::new_err(e.to_string()),
IGDBError::BadResponse => BadResponseException::new_err(e.to_string()),
IGDBError::BadClient => BadClientException::new_err(e.to_string()),
IGDBError::BadSecret => BadSecretException::new_err(e.to_string()),
IGDBError::BadToken => BadTokenException::new_err(e.to_string()),
IGDBError::BadAuth => BadAuthException::new_err(e.to_string()),
_ => UnknownErrorException::new_err(e.to_string()),
}
}
}
}

#[pyfunction]
pub fn fetch_twitch_token(_py: Python, client_id: &str, secret: &str) -> PyResult<igdb::Token> {
let token = igdb::get_twitch_token(client_id, secret);
match token {
Ok(token) => return Ok(token),
Err(e) => return Err(e.into())
}
}

#[pyfunction]
pub fn get_steam_game_info(_py: Python, client_id: &str, bearer_token: &str, appids: Vec<u64>) -> PyResult<PyObject> {
let result = igdb::get_steam_game_info(client_id, bearer_token, &appids);
match result {
Err(e) => {
return Err(e.into());
},
Ok((games, not_found)) => {
let games = PyList::new(_py, games);
let not_found = PyList::new(_py, &not_found);
let tuple : Vec<PyObject> = vec!(games.into(), not_found.into());
return Ok(PyTuple::new(_py, tuple).into());
}
}
}

macro_rules! expose_exception {
($py:expr, $m:expr, $exc:ty) => {
$m.add(stringify!($exc), $py.get_type::<$exc>());
}
}

fn igdb_mod(py: &Python, m: &PyModule) -> PyResult<()> {
m.add_function(wrap_pyfunction!(fetch_twitch_token, m)?)?;
m.add_function(wrap_pyfunction!(get_steam_game_info, m)?)?;

use igdb_exceptions::*;

// Exceptions
expose_exception!(py, m, IGDBException)?;
expose_exception!(py, m, UnknownErrorException)?;
expose_exception!(py, m, ServerErrorException)?;
expose_exception!(py, m, BadClientException)?;
expose_exception!(py, m, BadResponseException)?;
expose_exception!(py, m, BadSecretException)?;
expose_exception!(py, m, BadTokenException)?;
expose_exception!(py, m, BadAuthException)?;

return Ok(());
}

use crate::steam;

pub mod steam_exceptions {
use pyo3::create_exception;
use pyo3::exceptions::{PyException};
use pyo3::PyErr;

use crate::errors::SteamError;

create_exception!(steam, SteamException, PyException);

create_exception!(steam, UnknownErrorException, SteamException);
create_exception!(steam, ServerErrorException, SteamException);
create_exception!(steam, BadResponseException, SteamException);
create_exception!(steam, BadWebkeyException, SteamException);
create_exception!(steam, GamesListPrivateException, SteamException);
create_exception!(steam, GamesListEmptyException, SteamException);
create_exception!(steam, FriendListPrivateException, SteamException);

impl From<SteamError> for PyErr {
fn from(e: SteamError) -> PyErr {
match e {
SteamError::ServerError => ServerErrorException::new_err(e.to_string()),
SteamError::BadResponse => BadResponseException::new_err(e.to_string()),
SteamError::BadWebkey => BadWebkeyException::new_err(e.to_string()),
SteamError::FriendListPrivate => FriendListPrivateException::new_err(e.to_string()),
SteamError::GamesListPrivate(steamid) => GamesListPrivateException::new_err((e.to_string(), steamid)),
SteamError::GamesListEmpty(steamid) => GamesListEmptyException::new_err((e.to_string(), steamid)),
_ => UnknownErrorException::new_err(e.to_string()),
}
}
}
}

#[pyfunction]
pub fn get_steam_users_info(_py: Python, webkey: &str, steamids: Vec<u64>) -> PyResult<PyObject> {
let result = steam::get_steam_users_info(webkey, &steamids);

match result {
Err(e) => {
return Err(e.into());
},
Ok(users) => {
return Ok(PyList::new(_py, users).into());
},
}
}

#[pyfunction]
pub fn get_owned_steam_games(_py: Python, webkey: &str, steamid: u64) -> PyResult<PyObject> {
let result = steam::get_owned_steam_games(webkey, steamid);

match result {
Err(e) => {
return Err(e.into());
},
Ok(game_ids) => {
return Ok(PyList::new(_py, game_ids).into());
}
}
}

#[pyfunction]
pub fn get_friend_list(_py: Python, webkey: &str, steamid: u64) -> PyResult<PyObject> {
let result = steam::get_friend_list(webkey, steamid);

match result {
Err(e) => {
return Err(e.into());
},
Ok(friends) => {
return Ok(PyList::new(_py, friends).into());
}
}
}

#[pyfunction]
pub fn intersect_owned_game_ids(_py: Python, webkey: &str, steamids: Vec<u64>) -> PyResult<PyObject> {
let result = steam::intersect_owned_game_ids(webkey, &steamids);

match result {
Err(e) => {
return Err(e.into());
},
Ok(appids) => {
return Ok(PyList::new(_py, appids).into());
}
}
}

fn steam_mod(py: &Python, m: &PyModule) -> PyResult<()> {
m.add_function(wrap_pyfunction!(get_steam_users_info, m)?)?;
m.add_function(wrap_pyfunction!(get_owned_steam_games, m)?)?;
m.add_function(wrap_pyfunction!(get_friend_list, m)?)?;
m.add_function(wrap_pyfunction!(intersect_owned_game_ids, m)?)?;

use steam_exceptions::*;

// Exceptions
expose_exception!(py, m, SteamException)?;
expose_exception!(py, m, UnknownErrorException)?;
expose_exception!(py, m, ServerErrorException)?;
expose_exception!(py, m, BadResponseException)?;
expose_exception!(py, m, BadWebkeyException)?;
expose_exception!(py, m, FriendListPrivateException)?;
expose_exception!(py, m, GamesListEmptyException)?;
expose_exception!(py, m, GamesListPrivateException)?;

return Ok(());
}

use crate::wcwp;

pub mod wcwp_exceptions {
use pyo3::PyErr;

use crate::errors::WCWPError;

impl From<WCWPError> for PyErr {
fn from(e: WCWPError) -> PyErr {
match e {
WCWPError::IGDBError(e) => return e.into(),
WCWPError::SteamError(e) => return e.into(),
}
}
}
}

#[pyfunction]
pub fn intersect_owned_games(_py: Python, webkey: &str, igdb_id: &str, igdb_token: &str, steamids: Vec<u64>) -> PyResult<PyObject> {
let result = wcwp::intersect_owned_games(webkey, igdb_id, igdb_token, &steamids)?;

return Ok(PyList::new(_py, result).into());
}

#[pymodule]
fn whatcanweplay(py: Python, m: &PyModule) -> PyResult<()> {
let submod = PyModule::new(py, "igdb")?;
igdb_mod(&py, submod)?;
m.add_submodule(submod)?;

let submod = PyModule::new(py, "steam")?;
steam_mod(&py, submod)?;
m.add_submodule(submod)?;

m.add_function(wrap_pyfunction!(intersect_owned_games, m)?)?;

return Ok(());
}

+ 303
- 0
lib/wcwp_rust/src/steam.rs View File

@@ -0,0 +1,303 @@
use serde::{Serialize, Deserialize};

const API_URL: &str = "https://api.steampowered.com/";

#[derive(Debug, Serialize, Deserialize)]
pub struct SteamUser {
#[serde(rename(deserialize = "steamid"))]
#[serde(deserialize_with = "u64_string_parse")]
pub steam_id: u64,

#[serde(rename(deserialize = "personaname"))]
pub screen_name: String,

#[serde(rename(deserialize = "avatar"))]
pub avatar_thumb: String,

#[serde(rename(deserialize = "avatarmedium"))]
pub avatar: String,

#[serde(rename(deserialize = "communityvisibilitystate"))]
pub visibility: i8,

#[serde(rename(deserialize = "personastate"))]
#[serde(deserialize_with = "bool_from_int")]
pub online: bool
}

use crate::errors::SteamError;

use reqwest;
use reqwest::{StatusCode, Url};

use std::fmt::Write;
use std::collections::HashSet;

use serde::de::{self, Deserializer};

fn bool_from_int<'de, D>(deserializer: D) -> Result<bool, D::Error>
where
D: Deserializer<'de>,
{
match u8::deserialize(deserializer)? {
0 => Ok(false),
_ => Ok(true),
}
}

fn u64_string_parse<'de, D>(deserializer: D) -> Result<u64, D::Error>
where
D: Deserializer<'de>,
{
let val : serde_json::Value = Deserialize::deserialize(deserializer)?;
match val {
serde_json::Value::Number(num) => {
if let Some(num) = num.as_u64() {
return Ok(num);
}
},
serde_json::Value::String(string) => {
if let Ok(num) = string.parse() {
return Ok(num);
}
}
_ => {}
}

return Err(de::Error::custom(&"expected u64 or stringified u64"));
}

pub fn get_steam_users_info(webkey: &str, steamids: &[u64]) -> Result<Vec<SteamUser>, SteamError> {
if steamids.is_empty() {
return Ok(Vec::new());
}
let client = reqwest::blocking::Client::new();
let mut id_str = steamids[0].to_string();

for n in steamids[1..].iter()
{
write!(&mut id_str, ",{}", n).unwrap();
}

let base_url = Url::parse(API_URL).unwrap();

let response = client.get(base_url.join("ISteamUser/GetPlayerSummaries/v2/").unwrap())
.query(&[
("key", webkey),
("format", "json"),
("steamids", &id_str)
])
.send();
if let Err(e) = response {
return Err(SteamError::UnknownError(e.to_string()));
}

let response = response.unwrap();

if !response.status().is_success() {
match response.status() {
StatusCode::BAD_GATEWAY | StatusCode::INTERNAL_SERVER_ERROR =>
return Err(SteamError::ServerError),
StatusCode::FORBIDDEN =>
return Err(SteamError::BadWebkey),
_ =>
return Err(SteamError::UnknownError(response.text().unwrap_or("Unknown error".to_string())))
}
}

let response_json = response.json();
if let Err(_) = response_json {
return Err(SteamError::BadResponse); // TODO: Turn BadResponse into BadResponse(String, String) to hold response and error text?
}
let mut response: serde_json::Value = response_json.unwrap();
let players = &mut response["response"]["players"];
let mut users = Vec::new();

if players.is_array() {
for player_json in players.as_array_mut().unwrap().iter_mut() {
let user_info: Option<SteamUser> = serde_json::from_value(player_json.take()).unwrap();
if let Some(user) = user_info {
users.push(user);
}
}
}

return Ok(users);
}

///
/// # Errors
///
/// `SteamError::BadWebkey` is returned if the provided webkey is invalid
///
/// `SteamError::ServerError` is returned if the given steamid does not exist, or if the server had an error processing the request.
/// (We can't differentiate between the two, they're both returned as 500 status code)
pub fn get_owned_steam_games(webkey: &str, steamid: u64) -> Result<HashSet<u64>, SteamError> {
let base_url = Url::parse(API_URL).unwrap();
let client = reqwest::blocking::Client::new();

let response = client.get(base_url.join("IPlayerService/GetOwnedGames/v0001/").unwrap())
.query(&[
("key", webkey),
("steamid", &steamid.to_string()),
("include_appinfo", "false"),
("include_played_free_games", "true"),
("format", "json")
]).send();
if let Err(e) = response {
return Err(SteamError::UnknownError(e.to_string()));
}

let response = response.unwrap();

if !response.status().is_success() {
match response.status() {
StatusCode::UNAUTHORIZED =>
return Err(SteamError::BadWebkey),
StatusCode::INTERNAL_SERVER_ERROR =>
return Err(SteamError::ServerError),
_ =>
return Err(SteamError::UnknownError(response.text().unwrap_or("Unknown error".to_string()))),
}
}

let response_json = response.json();
if let Err(_) = response_json {
return Err(SteamError::BadResponse);
}

let response_json: serde_json::Value = response_json.unwrap();

let mut app_ids = HashSet::new();

let games_arr = &response_json["response"]["games"];
if let Some(games) = games_arr.as_array() {
for game in games {
let id = &game["appid"];
if let Some(id) = id.as_u64() {
app_ids.insert(id);
}
}
}

return Ok(app_ids);
}

pub fn get_friend_list(webkey: &str, steamid: u64) -> Result<Vec<SteamUser>, SteamError>
{
let base_url = Url::parse(API_URL).unwrap();
let client = reqwest::blocking::Client::new();
let response = client.get(base_url.join("ISteamUser/GetFriendList/v0001/").unwrap())
.query(&[
("key", webkey),
("steamid", &steamid.to_string()),
("relationship", "friend"),
("format", "json")
]).send();
if let Err(e) = response {
return Err(SteamError::UnknownError(e.to_string()));
}

let response = response.unwrap();

if !response.status().is_success() {
match response.status() {
StatusCode::UNAUTHORIZED =>
return Err(SteamError::BadWebkey),
StatusCode::INTERNAL_SERVER_ERROR =>
return Err(SteamError::ServerError),
_ =>
return Err(SteamError::UnknownError(response.text().unwrap_or("Unknown error".to_string()))),
}
}

let response_json = response.json();
if let Err(_) = response_json {
return Err(SteamError::BadResponse);
}

let response_json: serde_json::Value = response_json.unwrap();

let friendslist = &response_json["friendslist"];
if friendslist.is_null()
{
return Err(SteamError::FriendListPrivate);
}
let friendslist = &friendslist["friends"];
let mut user_ids = Vec::new();

if let Some(friendslist) = friendslist.as_array()
{
for friend in friendslist
{
let id_val = &friend["steamid"];
match id_val {
serde_json::Value::Number(num) => {
if let Some(num) = num.as_u64()
{
user_ids.push(num);
}
},
serde_json::Value::String(numstr) => {
if let Ok(num) = numstr.parse() {
user_ids.push(num);
}
},
_ => {},
}
}
}
else
{
return Err(SteamError::FriendListPrivate);
}

if user_ids.is_empty() {
return Ok(Vec::new())
}

let friends_info = get_steam_users_info(webkey, &user_ids)?;

return Ok(friends_info);
}

pub fn intersect_owned_game_ids(webkey: &str, steamids: &[u64])-> Result<HashSet<u64>, SteamError>
{
if steamids.is_empty()
{
return Ok(HashSet::new());
}

let mut games_set = get_owned_steam_games(webkey, steamids[0])?;

if games_set.is_empty()
{
return Err(SteamError::GamesListEmpty(steamids[0]));
}

for &id in steamids[1..].iter() {
let next_set = get_owned_steam_games(webkey, id)?;

if next_set.is_empty()
{
return Err(SteamError::GamesListEmpty(id));
}

games_set = &games_set & &next_set; // Intersect the two sets

if games_set.is_empty()
{ // No common owned games
return Ok(HashSet::new());
}
}

return Ok(games_set);
}

+ 19
- 0
lib/wcwp_rust/src/wcwp.rs View File

@@ -0,0 +1,19 @@
use crate::{igdb, steam};
use crate::errors::WCWPError;

use std::iter::FromIterator;

pub fn intersect_owned_games(webkey: &str, igdb_id: &str, igdb_token: &str, steamids: &[u64]) -> Result<Vec<igdb::GameInfo>, WCWPError>
{
if steamids.is_empty()
{
return Ok(Vec::new());
}

let games_set = steam::intersect_owned_game_ids(webkey, steamids)?;

let games_list = Vec::from_iter(games_set.into_iter());
let (games_info, _) = igdb::get_steam_game_info(igdb_id, igdb_token, &games_list)?;

return Ok(games_info);
}

+ 31
- 15
static/scripts/app.js View File

@@ -36,6 +36,8 @@ var current_slide_timeout;

var fetching = false;

var main_user_id = 0;

window.addEventListener("load", function() {
submit = document.getElementById("submit-button");
submit.addEventListener("click", submitButtonClicked);
@@ -64,7 +66,7 @@ window.addEventListener("load", function() {
child.src = default_avatar_url;
break;
case "user-name":
child.innerHTML = "";
child.innerText = "";
break;
case "user-checkbox":
delete child.dataset.steamId;
@@ -72,6 +74,8 @@ window.addEventListener("load", function() {
}
}

var main_user_info = {}

for(var i = 0; i < main_user.children.length; i++)
{
var child = main_user.children[i];
@@ -89,8 +93,19 @@ window.addEventListener("load", function() {
else
{
userCheckboxClicked(child);
main_user_id = child.dataset.steamId;
main_user_info["steam_id"] = main_user_id;
}
}
else if(child.className == "user-name")
{
main_user_info["screen_name"] = child.children[0].innerText;
}
}

if(main_user_id != 0)
{
user_info[main_user_id] = main_user_info
}

// Fetch friends list
@@ -121,9 +136,9 @@ function submitButtonClicked()
error_div.style.display = "none"
fetching = true;
submit.disabled = true;
submit.innerHTML = "Fetching..."
submit.innerText = "Fetching..."
back.disabled = true;
back.innerHTML = "Fetching..."
back.innerText = "Fetching..."
users_cover.style.display = "block";

if(app.className.includes("on-users") || app.className.includes("slide-to-users"))
@@ -163,9 +178,9 @@ function submitButtonClicked()
.catch(apiError)
.finally(function() {
submit.disabled = false;
submit.innerHTML = "Find Games";
submit.innerText = "Find Games";
back.disabled = false;
back.innerHTML = "Back";
back.innerText = "Back";
fetching = false;
users_cover.style.display = "none"
})
@@ -174,16 +189,16 @@ function submitButtonClicked()
function intersectResponse(data) {
if(data["errcode"] == 1)
{ // User has non-visible games list
displayError("WhatCanWePlay cannot access the games list of " + user_info[data["user"]]["screen_name"] + ". This either means that their Game details visibility is not Public, or they are being rate-limited by Steam for having too many requests. You can try one of the following fixes:\
<br><br>- Ask " + user_info[data["user"]]["screen_name"] + " to set their Game details to Public\
<br>- Remove " + user_info[data["user"]]["screen_name"] + " from your selected users\
displayError("WhatCanWePlay cannot access the games list of <span class='err-user-name'>" + user_info[data["user"]]["screen_name"] + "</span>. This either means that their Game details visibility is not Public, or they are being rate-limited by Steam for having too many requests. You can try one of the following fixes:\
<br><br>- Ask <span class='err-user-name'>" + user_info[data["user"]]["screen_name"] + "</span> to set their Game details to Public\
<br>- Remove <span class='err-user-name'>" + user_info[data["user"]]["screen_name"] + "</span> from your selected users\
<br>- Try again later\
");
return;
}
else if(data["errcode"] == 2)
{ // User has empty games list
displayError(user_info[data["user"]]["screen_name"] + " has an empty games list, and cannot possibly share any common games with the selected users. Please deselect " + user_infp[data["user"]]["screen_name"] + " and try again.")
displayError("<span class='err-user-name'>" + user_info[data["user"]]["screen_name"] + "</span> has an empty games list, and cannot possibly share any common games with the selected users. Please deselect <span class='err-user-name'>" + user_info[data["user"]]["screen_name"] + "</span> and try again.")
return;
}
else if(data["errcode"] != 0)
@@ -284,15 +299,16 @@ function intersectResponse(data) {
}
break;
case "game-title":
child.innerHTML = game["name"]
child.innerText = game["name"]
break;
case "user-count":
Array.from(child.children).forEach(function(child) {
if(child.className == "user-number")
{
child.innerHTML = game["supported_players"]
if(game["supported_players"] == "?")
child.innerText = game["supported_players"]
if(game["supported_players"] == "0")
{
child.innerText = "?"
child.classList.add("short")
child.title = "WhatCanWePlay was unable to retrieve the player count for this game from the IGDB"
}
@@ -419,7 +435,7 @@ function friendDataFetched(data)
}
break;
case "user-name":
child.innerHTML = user["screen_name"];
child.innerText = user["screen_name"];
break;
case "user-checkbox":
if(user["visibility"] != 3)
@@ -472,12 +488,12 @@ function userCheckboxClicked(box)
if(len >= 2)
{
submit.disabled = false;
submit.innerHTML = "Find Games"
submit.innerText = "Find Games"
}
else
{
submit.disabled = true;
submit.innerHTML = "Select " + (len == 0 ? "Two Users" : "One User")
submit.innerText = "Select " + (len == 0 ? "Two Users" : "One User")
}
}
else