Browse Source

Merge branch 'rust_backend'

master
TGRCDev 1 month ago
parent
commit
fa28b91252
Signed by: tgrcdev <tgrc@tgrc.dev> GPG Key ID: 3A975199085322C3
16 changed files with 1452 additions and 580 deletions
  1. +6
    -1
      .gitignore
  2. +353
    -126
      __init__.py
  3. +1
    -6
      config_EXAMPLE.json
  4. +0
    -251
      igdb_utils.py
  5. +15
    -0
      lib/wcwp_rust/Cargo.toml
  6. +80
    -0
      lib/wcwp_rust/src/errors.rs
  7. +306
    -0
      lib/wcwp_rust/src/igdb.rs
  8. +5
    -0
      lib/wcwp_rust/src/lib.rs
  9. +311
    -0
      lib/wcwp_rust/src/python.rs
  10. +303
    -0
      lib/wcwp_rust/src/steam.rs
  11. +19
    -0
      lib/wcwp_rust/src/wcwp.rs
  12. +31
    -15
      static/scripts/app.js
  13. +5
    -0
      static/styles/app.css
  14. +12
    -0
      static/styles/default_dark.css
  15. +0
    -179
      steam_utils.py
  16. +5
    -2
      templates/base_page.html

+ 6
- 1
.gitignore View File

@@ -3,4 +3,9 @@ config.json
venv* venv*
.vscode .vscode
igdb-cache.sqlite igdb-cache.sqlite
bearer-token.json
bearer-token.json
lib/wcwp_rust/target
lib/wcwp_rust/Cargo.lock
lib/whatcanweplay.pyd
lib/bin
.wcwp-commit-hash

+ 353
- 126
__init__.py View File

@@ -14,18 +14,51 @@
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>. # along with this program. If not, see <https://www.gnu.org/licenses/>.


try:
import shutil, platform, os
path_prefix = os.path.dirname(os.path.abspath(__file__))
source = ""
dest = ""
if platform.system() == "Windows":
source = "lib/wcwp_rust/target/release/whatcanweplay.dll"
dest = "lib/bin/whatcanweplay.pyd"
else:
source = "lib/wcwp_rust/target/release/libwhatcanweplay.so"
dest = "lib/bin/whatcanweplay.so"
source = os.path.join(path_prefix, source)
dest = os.path.join(path_prefix, dest)

if not os.path.exists(dest):
os.makedirs(os.path.dirname(dest), exist_ok=True)
shutil.copy2(source, dest)
else:
source_time = os.path.getmtime(source)
dest_time = os.path.getmtime(dest)
try:
if dest_time < source_time:
print("Updating WhatCanWePlay rust library with newer library file...")
shutil.copy2(source, dest)
except Exception:
print("Failed to update WhatCanWePlay rust library. It will be re-attempted when next launched.")
from .lib.bin import whatcanweplay as wcwp
except Exception as e:
print("Failed to load WhatCanWePlay rust library. Please go to \"lib/wcwp_rust\" and run \"cargo build --release\"")
print(e)
raise e
print("WhatCanWePlay rust lib loaded (" + str(wcwp.__file__) + ")")

from flask import Flask, request, jsonify, Response, render_template, redirect, session, url_for, make_response from flask import Flask, request, jsonify, Response, render_template, redirect, session, url_for, make_response
import requests import requests
from urllib import parse from urllib import parse
from werkzeug.exceptions import BadRequest from werkzeug.exceptions import BadRequest
import json import json
from .steam_utils import get_steam_user_info, get_steam_user_friend_list, get_owned_steam_games
from .igdb_utils import get_steam_game_info
from requests import HTTPError from requests import HTTPError
import secrets import secrets
from datetime import timezone, datetime, timedelta from datetime import timezone, datetime, timedelta
from itsdangerous import URLSafeSerializer from itsdangerous import URLSafeSerializer
from os import path from os import path
import traceback


# Load config # Load config
def create_app(): def create_app():
@@ -33,25 +66,30 @@ def create_app():
config = json.load(open(path.join(root_path, "config.json"), "r")) config = json.load(open(path.join(root_path, "config.json"), "r"))
steam_key = config["steam-key"] steam_key = config["steam-key"]
igdb_key = config["igdb-client-id"] igdb_key = config["igdb-client-id"]
igdb_secret = config["igdb-secret"]
debug = config.get("debug", config.get("DEBUG", False)) debug = config.get("debug", config.get("DEBUG", False))
enable_api_tests = config.get("enable-api-tests", debug) enable_api_tests = config.get("enable-api-tests", debug)
cookie_max_age_dict = config.get("cookie-max-age", {}) cookie_max_age_dict = config.get("cookie-max-age", {})
info_max_age_dict = config.get("info-max-age", {})
cache_max_age_dict = config.get("igdb-cache-max-age", config.get("igdb-cache-info-age", {}))
source_url = config.get("source-url", "") source_url = config.get("source-url", "")
contact_email = config["contact-email"] contact_email = config["contact-email"]
privacy_email = config.get("privacy-email", contact_email) privacy_email = config.get("privacy-email", contact_email)
connect_timeout = config.get("connect-timeout", 0.0) connect_timeout = config.get("connect-timeout", 0.0)
commit_hash_filename = config.get("commit-hash-file", ".wcwp-commit-hash")
donate_url = config.get("donate-url", "") donate_url = config.get("donate-url", "")
if connect_timeout <= 0.0: if connect_timeout <= 0.0:
connect_timeout = None connect_timeout = None
read_timeout = config.get("read-timeout", 0.0) read_timeout = config.get("read-timeout", 0.0)
if read_timeout <= 0.0: if read_timeout <= 0.0:
read_timeout = None read_timeout = None
cache_file = config.get("igdb-cache-file")
if not os.path.isabs(cache_file):
cache_file = os.path.join(root_path, cache_file)


# Create uWSGI callable # Create uWSGI callable
app = Flask(__name__) app = Flask(__name__)
app.debug = debug app.debug = debug
app.secret_key = config.get("secret-key", secrets.token_hex()) # If not set, cookies will be invalidated every time the app is reloaded
app.secret_key = config["secret-key"]


# Hide requests to /steam_login to prevent linking Steam ID to IP in logs # Hide requests to /steam_login to prevent linking Steam ID to IP in logs
from werkzeug import serving from werkzeug import serving
@@ -68,20 +106,61 @@ def create_app():
cookie_max_age = timedelta(**cookie_max_age_dict).total_seconds() cookie_max_age = timedelta(**cookie_max_age_dict).total_seconds()
if cookie_max_age == 0: if cookie_max_age == 0:
cookie_max_age = None cookie_max_age = None
info_max_age = timedelta(**info_max_age_dict).total_seconds()
# Setup cache info max age
cache_max_age = 0.0
if cache_file:
cache_max_age = timedelta(**cache_max_age_dict).total_seconds()

print("cookies set to expire after %f seconds" % cookie_max_age)
print("cache set to expire after %f seconds" % cache_max_age)

def fetch_and_store_commit_hash():
f = open(commit_hash_filename, "w")
import subprocess
args = ['--git-dir=' + os.path.join(os.path.abspath(os.path.dirname(__file__)), ".git"), 'rev-parse', '--short', 'HEAD']
try:
try:
commit_hash = subprocess.check_output(['git'] + args).decode("utf-8").strip()
f.write(commit_hash)
f.close()
return commit_hash
except:
commit_hash = subprocess.check_output(['/usr/bin/git'] + args).decode("utf-8").strip()
f.write(commit_hash)
f.close()
return commit_hash
except:
return ""

@app.before_first_request
def before_first_request():
fetch_and_store_commit_hash()


print("cookies set to expire after {} seconds".format(cookie_max_age))
print("steam info set to refresh after {} seconds".format(info_max_age))
def get_commit_hash():
try:
f = open(commit_hash_filename, "r")
hash = f.read()
return hash
except Exception:
traceback.print_exc()
return ""


def basic_info_dict(): def basic_info_dict():
email_rev = contact_email.split("@") email_rev = contact_email.split("@")
return {
basic_info = {
"contact_email_user_reversed": email_rev[0][::-1], "contact_email_user_reversed": email_rev[0][::-1],
"contact_email_domain_reversed": email_rev[1][::-1], "contact_email_domain_reversed": email_rev[1][::-1],
"source_url": source_url, "source_url": source_url,
"donate_url": donate_url "donate_url": donate_url
} }


commit = get_commit_hash()
if commit:
basic_info["commit"] = commit
return basic_info

# Tries to fetch the Steam info cookie, returns an errcode and a dict # Tries to fetch the Steam info cookie, returns an errcode and a dict
# #
# Errcodes: # Errcodes:
@@ -113,18 +192,20 @@ def create_app():


def refresh_steam_cookie(steamid: int, response): def refresh_steam_cookie(steamid: int, response):
if steamid <= 0: if steamid <= 0:
response.set_cookie("steam_info", "", secure=True)
return {}

info = get_steam_user_info(steam_key, [steamid], connect_timeout, read_timeout)

if info["errcode"] != 0 or not info["users"].get(steamid, {}).get("exists", False):
response.set_cookie("steam_info", "", secure=True)
response.set_cookie("steam_info", "", secure=True, httponly=True)
return {} return {}
info = info["users"][steamid]
if info_max_age:
info["expires"] = (datetime.now(timezone.utc) + timedelta(seconds=info_max_age)).timestamp()
info = {}
try:
info = wcwp.steam.get_steam_users_info(steam_key, [steamid])[0]
except IndexError:
response.set_cookie("steam_info", "", secure=True, httponly=True)
return {}
except Exception:
traceback.print_exc()
response.set_cookie("steam_info", "", secure=True, httponly=True)
return {}

ser = URLSafeSerializer(app.secret_key) ser = URLSafeSerializer(app.secret_key)
response.set_cookie( response.set_cookie(
"steam_info", "steam_info",
@@ -143,7 +224,7 @@ def create_app():
if errcode == 3: if errcode == 3:
steam_info = refresh_steam_cookie(steam_info.get("steam_id", -1), response) steam_info = refresh_steam_cookie(steam_info.get("steam_id", -1), response)
elif errcode != 0: elif errcode != 0:
response.set_cookie("steam_info", "", secure=True)
response.set_cookie("steam_info", "", secure=True, httponly=True)
steam_info = {} steam_info = {}
response.data = render_template("home.html", steam_info=steam_info, **basic_info_dict()) response.data = render_template("home.html", steam_info=steam_info, **basic_info_dict())
@@ -157,7 +238,7 @@ def create_app():
def steam_login(): def steam_login():
if request.method == "POST": if request.method == "POST":
steam_openid_url = 'https://steamcommunity.com/openid/login' steam_openid_url = 'https://steamcommunity.com/openid/login'
return_url = url_for("steam_login", _external=True)
return_url = request.base_url
params = { params = {
'openid.ns': "http://specs.openid.net/auth/2.0", 'openid.ns': "http://specs.openid.net/auth/2.0",
'openid.identity': "http://specs.openid.net/auth/2.0/identifier_select", 'openid.identity': "http://specs.openid.net/auth/2.0/identifier_select",
@@ -183,7 +264,7 @@ def create_app():
def steam_logout(): def steam_logout():
response = redirect(url_for("index")) response = redirect(url_for("index"))
response.headers["X-Robots-Tag"] = "none" response.headers["X-Robots-Tag"] = "none"
response.set_cookie("steam_info", "", secure=True)
response.set_cookie("steam_info", "", secure=True, httponly=True)
return response return response


def validate_steam_identity(params): def validate_steam_identity(params):
@@ -214,46 +295,196 @@ def create_app():
) )
errcode, steam_info = fetch_steam_cookie(request) errcode, steam_info = fetch_steam_cookie(request)
if "steam_id" not in steam_info.keys(): if "steam_id" not in steam_info.keys():
return ("Not signed in to Steam", 403)
return (
"Not signed in to Steam. Please refresh the page.",
403
)
friends_info = get_steam_user_friend_list(
steam_key,
steam_info["steam_id"],
connect_timeout,
read_timeout
)
try:
friends_info = wcwp.steam.get_friend_list(
steam_key,
steam_info["steam_id"]
)
for user in friends_info:
if "steam_id" in user.keys():
user["steam_id"] = str(user["steam_id"])
user["exists"] = True

return jsonify(friends_info)
except wcwp.steam.BadWebkeyException:
traceback.print_exc()
return (
"Site has bad Steam API key. Please contact us about this error at " + contact_email,
500
)
except wcwp.steam.ServerErrorException:
traceback.print_exc()
return (
"Steam had an internal server error. Please try again later.",
500
)
except wcwp.steam.BadResponseException:
traceback.print_exc()
return (
"Steam returned an unparseable response. Please try again later.",
500
)
except wcwp.steam.FriendListPrivateException:
traceback.print_exc()
return (
"WhatCanWePlay cannot retrieve your friend list. Please change your friend list visibility to public and refresh the page.",
500
)
except Exception:
traceback.print_exc()
if debug:
return (
traceback.format_exc(),
500
)
else:
traceback.print_exc()
return (
"An unknown error has occurred. Please try again later.",
500
)


errcode = friends_info.pop("errcode")

if errcode == 1:
return ("Site has bad Steam API key. Please contact us about this error at " + contact_email, 500)
elif errcode == 2:
return ("Steam took too long to respond. Please try again later.", 500)
elif errcode == 3:
return ("Steam took too long to transmit info. Please try again later.", 500)
elif errcode == 4:
return ("Your Friend List is not publicly accessible, and cannot be retrieved by WhatCanWePlay. Please set your Friend list visibility to Public and refresh the page.", 500)
elif errcode == -1:
return ("An unknown error occurred. Please try again later.", 500)
def refresh_igdb_token():
try:
token_path = path.join(root_path, "bearer-token.json")
token = wcwp.igdb.fetch_twitch_token(igdb_key, igdb_secret)
token["expiry"] = datetime.now(timezone.utc).timestamp() + token.get("expires_in", 0)
token.pop("expires_in")
json.dump(token, open(token_path, "w"))
return token.get("access_token", "")
except Exception:
traceback.print_exc()
return ""

def get_igdb_token():
try:
token_path = path.join(root_path, "bearer-token.json")
if path.exists(token_path):
token_file = open(token_path)
token = json.load(token_file)
token_file.close()
if datetime.now(timezone.utc).timestamp() >= token.get("expiry", 0):
return refresh_igdb_token()
else:
return token.get("access_token", "")
else:
return refresh_igdb_token()
except Exception:
traceback.print_exc()
return ""

cache_init_query = """
CREATE TABLE IF NOT EXISTS game (
steam_id INTEGER PRIMARY KEY,
igdb_id INTEGER,
name STRING,
supported_players INTEGER DEFAULT(0),
cover_id STRING,
has_multiplayer BOOLEAN,
expiry REAL DEFAULT(0.0)
);
"""

CACHE_VERSION = 1

def initialize_cache():
import sqlite3
cache = sqlite3.connect(cache_file)
cache.execute(cache_init_query)
#cache.execute("PRAGMA user_version = ?;", [CACHE_VERSION]) # Doesn't work?
cache.execute("PRAGMA user_version = %d" % CACHE_VERSION)
return cache
def cache_is_correct_version(cache):
return cache.execute("PRAGMA user_version;").fetchone()[0] == CACHE_VERSION

def update_cached_games(game_info):
if not cache_file:
return
friends_info = get_steam_user_info(steam_key, friends_info["friends"], connect_timeout, read_timeout)

errcode = friends_info.pop("errcode")
if errcode == 1:
return ("Site has bad Steam API key. Please contact us about this error at " + contact_email, 500)
elif errcode == 2:
return ("Steam took too long to respond. Please try again later.", 500)
elif errcode == 3:
return ("Steam took too long to transmit info. Please try again later.", 500)
elif errcode == -1:
return ("An unknown error occurred. Please try again later.", 500)
try:
import sqlite3
cache = None

if os.path.exists(cache_file):
cache = sqlite3.connect(cache_file)
# Check if cache is correct version
if not cache_is_correct_version(cache):
# Cache is the wrong version, rebuild
print("Cache file is the wrong version! Rebuilding... ")
cache.close()
os.remove(cache_file)
cache = initialize_cache()
else:
cache = initialize_cache()
insert_info = [
[
game.get("steam_id"),
game.get("igdb_id"),
game.get("name"),
game.get("supported_players"),
game.get("cover_id"),
game.get("has_multiplayer"),
datetime.now(timezone.utc).timestamp() + cache_max_age
] for game in game_info
]
cache.executemany(
"INSERT OR REPLACE INTO game VALUES (?,?,?,?,?,?,?);",
insert_info
)

cache.commit()
cache.close()
except Exception:
print("FAILED TO UPDATE CACHE DB")
traceback.print_exc()
return
# returns [info of cached games], (set of uncached ids)
def get_cached_games(steam_ids):
if not cache_file:
return [], set(steam_ids)

game_info = []
uncached = set(steam_ids)
for user in friends_info["users"].values():
if "steam_id" in user.keys():
user["steam_id"] = str(user["steam_id"])
user["exists"] = True
try:
import sqlite3
cache = sqlite3.connect(cache_file)
cache.row_factory = sqlite3.Row


return jsonify(friends_info["users"])
if not cache_is_correct_version(cache):
return [], set(steam_ids)
query_str = "SELECT * FROM game WHERE steam_id IN (%s)" % ("?" + (",?" * (len(steam_ids) - 1))) # Construct a query with arbitrary parameter length
cursor = cache.execute(
query_str,
steam_ids
)
for row in cursor.fetchall():
game = dict(row)
if datetime.now(timezone.utc).timestamp() < game.pop("expiry"):
# Info hasn't expired
game_info.append(game)
uncached.remove(game["steam_id"])

# Expired info gets updated during update_cached_games()
except Exception:
print("EXCEPTION THROWN WHILE QUERYING GAME CACHE!")
traceback.print_exc()
return [], set(steam_ids)
return game_info, uncached


# Errcodes # Errcodes
# -1: An error occurred with a message. Additional fields: "message" # -1: An error occurred with a message. Additional fields: "message"
@@ -316,90 +547,86 @@ def create_app():
json.dumps({"message": "Games intersection is capped at 10 users.", "errcode": -1}), json.dumps({"message": "Games intersection is capped at 10 users.", "errcode": -1}),
200 200
) )
free_games = bool(body.get("include_free_games", False))


all_own = None
try:
token = get_igdb_token()
game_ids = wcwp.steam.intersect_owned_game_ids(steam_key, list(steamids))


for steamid in steamids:
user_owned_games = get_owned_steam_games(steam_key, steamid, free_games, connect_timeout, read_timeout)
errcode = user_owned_games["errcode"]
fetched_game_count = 0
cached_game_count = 0
game_info = []


if errcode == 1:
return (
json.dumps({"message": "Site has bad Steam API key. Please contact us about this error at {}.".format(contact_email), "errcode": -1}),
200
)
elif errcode == 2:
return (
json.dumps({"message": "Steam took too long to respond. Please try again later.", "errcode": -1}),
200
)
elif errcode == 3:
return (
json.dumps({"message": "Steam took too long to transmit info. Please try again later.", "errcode": -1}),
200
)
elif errcode == 4:
return (
json.dumps({"user": str(steamid), "errcode": 1}),
200
)
elif errcode == -1:
return (
json.dumps({"message": "An unknown error occurred. Please try again later.", "errcode": -1}),
200
)
games = user_owned_games.get("games")
if len(games) == 0:
return (
json.dumps({"user": str(steamid), "errcode": 2}),
200
)
if not all_own:
all_own = set(games)
else:
all_own = all_own & set(games)
if len(all_own) == 0:
break
# Step two: Fetch the game info
game_info = get_steam_game_info(igdb_key, all_own, connect_timeout, read_timeout)
if game_ids:
game_info, uncached_ids = get_cached_games(game_ids)


errcode = game_info["errcode"]
cached_game_count = len(game_info)


if errcode == 1:
if uncached_ids:
fetched_info, not_found = wcwp.igdb.get_steam_game_info(igdb_key, token, list(uncached_ids))

cache_info_update = fetched_info
if not_found:
for uncached_id in [id for id in not_found]:
cache_info_update.append({"steam_id": uncached_id}) # Cache empty data to prevent further IGDB fetch attempts
update_cached_games(cache_info_update) # TODO: Spin up separate process for caching?

game_info += fetched_info
fetched_game_count = len(fetched_info)
print("Intersection resulted in %d games (%d from cache, %d from IGDB)" % (len(game_info), cached_game_count, fetched_game_count))

return jsonify({
"message": "Intersected successfully",
"games": game_info,
"errcode": 0
})
except wcwp.steam.BadWebkeyException:
traceback.print_exc()
return ( return (
json.dumps({"message": "Site has bad IGDB API key. Please contact us about this error at {}.".format(contact_email), "errcode": -1}),
200
json.dumps({"message": "Site has bad Steam API key. Please contact us about this error at " + contact_email, "errcode": -1}),
500
) )
elif errcode == 2:
except wcwp.steam.ServerErrorException:
traceback.print_exc()
return ( return (
json.dumps({"message": "IGDB took too long to respond. Please try again later.", "errcode": -1}),
200
json.dumps({"message": "Steam had an internal server error. Please try again later.", "errcode": -1}),
500
) )
elif errcode == 3:
except wcwp.steam.BadResponseException:
traceback.print_exc()
return ( return (
json.dumps({"message": "IGDB took too long to transmit info. Please try again later.", "errcode": -1}),
200
json.dumps({"message": "Steam returned an unparseable response. Please try again later.", "errcode": -1}),
500
) )
elif errcode == 4:
except wcwp.steam.GamesListPrivateException as e:
if debug:
print(e)
else:
print("Intersection interrupted due to private games list")
return ( return (
json.dumps({"message": "WhatCanWePlay failed to acquire an IGDB token. Please contact us about this error at {}.".format(contact_email), "errcode": -1})
json.dumps({"errcode": 1, "user": str(e.args[1])}),
500
) )
elif errcode == -1:
except wcwp.steam.GamesListEmptyException as e:
if debug:
print(e)
else:
print("Intersection interrupted due to private games list")
return ( return (
json.dumps({"message": "An unknown error occurred. Please try again later.", "errcode": -1}),
200
json.dumps({"errcode": 2, "user": str(e.args[1])}),
500
) )

return jsonify({
"message": "Intersected successfully",
"games": list(game_info.get("games", {}).values()),
"errcode": 0
})
except Exception:
traceback.print_exc()
if debug:
return (
json.dumps({"message": traceback.format_exc(), "errcode": -1}),
500
)
else:
return (
json.dumps({"message": "An unknown error has occurred. Please try again later.", "errcode": -1}),
500
)


return app return app

+ 1
- 6
config_EXAMPLE.json View File

@@ -4,22 +4,17 @@
"steam-key": "YOUR STEAM KEY HERE", "steam-key": "YOUR STEAM KEY HERE",
"igdb-client-id": "YOUR IGDB KEY HERE", "igdb-client-id": "YOUR IGDB KEY HERE",
"igdb-secret": "YOUR IGDB SECRET HERE", "igdb-secret": "YOUR IGDB SECRET HERE",
"www-host": "yoururl.here",
"cookie-max-age": { "cookie-max-age": {
"days": 0, "days": 0,
"weeks": 0 "weeks": 0
}, },
"info-max-age": {
"hours": 0,
"minutes": 0
},
"source-url": "https://git.tgrc.dev/tgrcdev/whatcanweplay", "source-url": "https://git.tgrc.dev/tgrcdev/whatcanweplay",
"contact-email": "contact@whatcanweplay.net", "contact-email": "contact@whatcanweplay.net",
"privacy-email": "privacy@whatcanweplay.net", "privacy-email": "privacy@whatcanweplay.net",
"connect-timeout": 8, "connect-timeout": 8,
"read-timeout": 30, "read-timeout": 30,
"igdb-cache-file": "igdb-cache.sqlite", "igdb-cache-file": "igdb-cache.sqlite",
"igdb-cache-info-age": {
"igdb-cache-max-age": {
"weeks": 4 "weeks": 4
} }
} }

+ 0
- 251
igdb_utils.py View File

@@ -1,251 +0,0 @@
# This file is a part of WhatCanWePlay
# Copyright (C) 2020 TGRCDev

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.

import requests
import json
import sqlite3
from requests.exceptions import ConnectTimeout, ReadTimeout
from typing import Dict, Collection, Any, Mapping, Optional
from datetime import timedelta, datetime, timezone
from os import path

api_base = "https://api.igdb.com/v4/"

root_path = path.dirname(__file__)
config = json.load(open(path.join(root_path, "config.json"), "r"))
debug = config.get("debug", config.get("DEBUG", False))
db_filename = config.get("igdb-cache-filename", "igdb-cache.sqlite")
if not path.isabs(db_filename):
db_filename = path.join(root_path, db_filename)

create_db_query = """
CREATE TABLE IF NOT EXISTS game_info (
steam_id INTEGER PRIMARY KEY,
igdb_id INTEGER,
name TEXT,
cover_id TEXT,
has_multiplayer BOOL,
supported_players TEXT,
time_cached REAL
);
"""

info_age_dict = config.get("igdb-cache-info-age", {})
info_age = timedelta(**info_age_dict).total_seconds()

def fetch_and_store_token() -> str:
r = requests.post(
"https://id.twitch.tv/oauth2/token",
json={
"client_id": config["igdb-client-id"],
"client_secret": config["igdb-secret"],
"grant_type": "client_credentials"
}
)

if debug:
r.raise_for_status()
try:
token = r.json()
token_dict = {
"token": token["access_token"],
"expire-time": datetime.now(timezone.utc).timestamp() + token["expires_in"],
"token_type": token["token_type"]
}
token_file = open(path.join(root_path, "bearer-token.json"), "w")
json.dump(token_dict, token_file)
token_file.close()
return token["access_token"]
except Exception as e:
print("Exception thrown while parsing token return")
print(e)
return ""

def get_or_refresh_token() -> str:
try:
token_path = path.join(root_path, "bearer-token.json")
if path.exists(token_path):
token_file = open(token_path)
token = json.load(token_file)
token_file.close()
if datetime.now(timezone.utc).timestamp() >= token.get("expire-time", 0):
return fetch_and_store_token()
else:
return token.get("token", "")
else:
return fetch_and_store_token()
except json.JSONDecodeError as e:
print("Failed to decode token JSON.")
print(e)
return ""
except Exception as e:
print("Failed to fetch bearer token.")
print(e)
return ""

def get_cached_games(appids: Collection[int]) -> Dict[int, Dict[str, Any]]:
query = """
SELECT steam_id, igdb_id, name, cover_id, has_multiplayer, supported_players
FROM game_info
WHERE time_cached > ? AND steam_id in ({})
""".format(",".join(
["?" for _ in range(len(appids))]
))
db_handle = sqlite3.connect(db_filename)
db_handle.execute(create_db_query)

cursor = db_handle.cursor()
cursor.execute(query, [datetime.now(timezone.utc).timestamp() - info_age, *list(appids)])
results = cursor.fetchall()

return {
game[0]: {
"steam_id": game[0],
"name": game[2],
"cover_id": game[3],
"has_multiplayer": game[4],
"supported_players": game[5]
} if game[1] else {}
for game in results
}

def update_cached_games(game_info: Mapping[int, Mapping[str, Any]]):
query = """
INSERT OR REPLACE INTO game_info
(steam_id, igdb_id, name, cover_id, has_multiplayer, supported_players, time_cached)
VALUES (?,?,?,?,?,?,?)
"""
db_handle = sqlite3.connect(db_filename)
db_handle.execute(create_db_query)
cursor = db_handle.cursor()
now = datetime.now(timezone.utc).timestamp()
cursor.execute("BEGIN TRANSACTION")
cursor.executemany(query, [[
game.get("steam_id"),
game.get("igdb_id"),
game.get("name"),
game.get("cover_id"),
game.get("has_multiplayer"),
game.get("supported_players"),
now] for game in game_info.values()]
)
cursor.execute("END TRANSACTION")

def get_steam_game_info(webkey: str, appids: Collection[int], connect_timeout: Optional[float] = None, read_timeout: Optional[float] = None) -> Dict[int, Dict[str, Any]]:
if len(appids) == 0:
return {"errcode":0, "games":{}}
appid_set = set(appids)
cache_error = False

try:
cached_games = get_cached_games(appid_set)
except Exception as e:
print("An exception occurred while retrieving cached games: " + str(e))
cached_games = {}
cache_error = True

if len(cached_games) == len(appid_set):
return {"errcode": 0, "games": cached_games}

uncached_ids = appid_set - set(cached_games.keys())
uncached_ids_list = list(uncached_ids)

token = get_or_refresh_token()
if not token:
return {"errcode": 4}

games_dict = {}
retrieved_games = 0
while retrieved_games < len(uncached_ids_list):
game_slice = uncached_ids_list[retrieved_games:500]

retrieved_games += len(game_slice)

try:
r = requests.post(
api_base + "external_games",
data = "fields uid,game.name,game.game_modes,game.multiplayer_modes.onlinemax,game.multiplayer_modes.onlinecoopmax,game.cover.image_id; where uid = ({}) & category = 1; limit {};".format(",".join(map(str, game_slice)), len(game_slice)),
headers = {
"Client-ID": webkey,
"Authorization": "Bearer " + token,
"Accept": "application/json"
},
timeout = (connect_timeout, read_timeout)
)
except ConnectTimeout:
return {"errcode": 2}
except ReadTimeout:
return {"errcode": 3}
if r.status_code == 403:
return {"errcode": 1}
elif r.status_code != 200:
if debug:
r.raise_for_status()
return {"errcode": -1}
for game in r.json():
steam_id = game.get("uid", None)
if not steam_id:
continue
steam_id = int(steam_id)
game = game.get("game", None)
if not game:
continue
uncached_ids.discard(steam_id)

game_modes = game.get("game_modes", [])

is_multiplayer = (2 in game_modes or 5 in game_modes)

maxplayers = -1
if is_multiplayer:
for mode in game.get("multiplayer_modes", []):
maxplayers = max(max(mode.get("onlinemax", 1), mode.get("onlinecoopmax", 1)), maxplayers)
else:
maxplayers = 1

games_dict[steam_id] = {
"steam_id": steam_id,
"igdb_id": game["id"],
"name": game["name"],
"cover_id": game.get("cover", {}).get("image_id", ""),
"has_multiplayer": is_multiplayer,
"supported_players": str(maxplayers) if maxplayers > 0 else "?"
}
# Any games that couldn't be retrieved probably dont exist. Store them so they don't trigger an IGDB fetch.
for nonexist_id in uncached_ids:
games_dict[nonexist_id] = {"steam_id": nonexist_id}
if len(games_dict) == 0:
return {"errcode":0, "games":cached_games}
for id in games_dict.keys():
cached_games[id] = games_dict[id]
if not cache_error:
try:
update_cached_games(games_dict)
except Exception as e:
print("An exception occurred while updating cached games: " + str(e))

return {
"errcode": 0,
"games": cached_games
}

+ 15
- 0
lib/wcwp_rust/Cargo.toml View File

@@ -0,0 +1,15 @@
[package]
name = "whatcanweplay"
version = "0.1.0"
authors = ["TGRCDev <tgrc@tgrc.dev>"]
edition = "2018"

[lib]
name = "whatcanweplay"
crate-type = ["cdylib"]

[dependencies]
reqwest = { version = "0.10", features = ["json", "blocking"] }
serde_json = "1.0"
serde = { version = "1.0", features = ["derive", "std"] }
pyo3 = { version = "0.12", features = ["extension-module"] }

+ 80
- 0
lib/wcwp_rust/src/errors.rs View File

@@ -0,0 +1,80 @@
#[derive(Debug)]
pub enum IGDBError {
UnknownError(String), // Unhandled error
ServerError, // IGDB had an internal error
BadResponse, // IGDB returned an unparseable response
BadClient, // The supplied client ID is wrong
BadSecret, // The supplied client secret is wrong
BadToken, // The supplied bearer token is wrong
BadAuth, // Some part of the supplied authentication is wrong
}

use std::fmt;

impl fmt::Display for IGDBError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
IGDBError::ServerError => return write!(f, "IGDB had an internal server error"),
IGDBError::BadResponse => return write!(f, "IGDB returned an unparseable response"),
IGDBError::BadClient => return write!(f, "IGDB rejected the provided client ID"),
IGDBError::BadSecret => return write!(f, "IGDB rejected the provided client secret"),
IGDBError::BadToken => return write!(f, "IGDB rejected the provided bearer token"),
IGDBError::BadAuth => return write!(f, "IGDB rejected some part of the provided authentication"),
IGDBError::UnknownError(err_string) => return write!(f, "{}", &err_string),
_ => return write!(f, "IGDB API had an unknown error")
}
}
}

#[derive(Debug)]
pub enum SteamError {
UnknownError(String), // Unhandled error
ServerError, // Steam had an internal error
BadResponse, // Steam returned an unparseable response
BadWebkey, // The supplied webkey is wrong
GamesListPrivate(u64), // The games list of the given steam id is unretrievable, probably due to game list privacy settings
GamesListEmpty(u64), // Intersection failed, the user with the given steam id had no games
FriendListPrivate, // The steam id passed to get_friend_list() has their friend list set to private
}

impl fmt::Display for SteamError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
SteamError::ServerError => return write!(f, "Steam had an internal server error"),
SteamError::BadResponse => return write!(f, "Steam returned an unparseable response"),
SteamError::BadWebkey => return write!(f, "Steam rejected the provided webkey"),
SteamError::UnknownError(err_string) => return write!(f, "{}", &err_string),
SteamError::GamesListPrivate(steamid) => return write!(f, "The user with Steam ID {} has their games list set to private", steamid),
SteamError::GamesListEmpty(steamid) => return write!(f, "The user with the Steam ID {} has no games to intersect", steamid),
SteamError::FriendListPrivate => return write!(f, "The given user has their friend list set to private"),
_ => return write!(f, "Steam had an unknown error")
}
}
}

#[derive(Debug)]
pub enum WCWPError {
SteamError(SteamError), // Steam API Error
IGDBError(IGDBError), // IGDB API Error
}

impl fmt::Display for WCWPError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
WCWPError::SteamError(e) => return e.fmt(f),
WCWPError::IGDBError(e) => return e.fmt(f),
}
}
}

impl From<SteamError> for WCWPError {
fn from(e: SteamError) -> Self {
return WCWPError::SteamError(e);
}
}

impl From<IGDBError> for WCWPError {
fn from(e: IGDBError) -> Self {
return WCWPError::IGDBError(e);
}
}

+ 306
- 0
lib/wcwp_rust/src/igdb.rs View File

@@ -0,0 +1,306 @@
use reqwest;
use serde::{Deserialize, Serialize};
use serde_json;
use std::collections::HashSet;
use std::fmt::{Debug, Write};
use crate::errors::IGDBError;
use reqwest::StatusCode;
use serde_json::Value;
use std::convert::{TryFrom, TryInto};
use std::cmp::{max, min};

const API_URL : &str = "https://api.igdb.com/v4/";
const TOKEN_URL : &str = "https://id.twitch.tv/oauth2/token";

#[derive(Serialize, Deserialize, Debug)]
/// Struct for retrieving a bearer token from the Twitch Developer API
pub struct Token {
pub access_token: String,
pub expires_in: u64
}

#[derive(Debug, Serialize)]
/// Struct for games described by IGDB
pub struct GameInfo {
pub steam_id: u64,
pub igdb_id: u64,
pub name: String,
pub supported_players: u8,
/// Cover image url
///
/// Usage: `https://images.igdb.com/igdb/image/upload/t_cover_small/<cover_id>.jpg`
pub cover_id: String,

pub has_multiplayer: bool,
}

/// Takes a serde_json Value and tries to get an i64
/// either by parsing an i64 directly, or by parsing
/// a stringified i64.
fn parse_u8_from_value(value: &Value) -> Option<u8> {
let num = parse_u64_from_value(value);
match num {
Some(num) =>
return num.try_into().ok(),
None =>
return None
}
}

/// Takes a serde_json Value and tries to get an i64
/// either by parsing an u64 directly, or by parsing
/// a stringified u64.
fn parse_u64_from_value(value: &Value) -> Option<u64> {
match value {
Value::Number(num) => {
return num.as_u64();
},
Value::String(string) => {
return string.parse().ok();
},
_ => return None
}
}

impl TryFrom<&Value> for GameInfo {
type Error = IGDBError;

/// Used to extract game information from IGDB query results
fn try_from(value: &Value) -> Result<Self, IGDBError> {
let steam_id : u64;
match parse_u64_from_value(&value["uid"]) {
Some(id) => steam_id = id,
None => return Err(IGDBError::BadResponse)
}

let game = &value["game"];
if !game.is_object()
{
return Err(IGDBError::BadResponse);
}

let igdb_id : u64;
match parse_u64_from_value(&game["id"]) {
Some(id) => igdb_id = id,
None => return Err(IGDBError::BadResponse)
}

let name : &str;
match game["name"].as_str() {
Some(val) => name = val,
None => return Err(IGDBError::BadResponse),
}

let cover_id : &str;
match game["cover"]["image_id"].as_str() {
Some(val) => cover_id = val,
None => cover_id = "",
}

// Figure out multiplayer and whatnot

let game_modes = &game["game_modes"];
let has_multiplayer: bool;
let supported_players : u8;

if game_modes.is_array() {
let gamemode_arr = game_modes.as_array().unwrap();
has_multiplayer = gamemode_arr.iter().any(|x| x == 5 || x == 2); // TODO: This isn't really needed with game["multiplayer_modes"], replace it
if has_multiplayer
{
let multiplayer_modes = &game["multiplayer_modes"];
if let Some(multimodes) = multiplayer_modes.as_array() {
let mut mostplayers = 0;
for mode in multimodes.iter() {
let onlinemax : u8 = parse_u8_from_value(&mode["onlinemax"]).unwrap_or(0);
let coopmax : u8 = parse_u8_from_value(&mode["onlinecoopmax"]).unwrap_or(0);
mostplayers = max(onlinemax, coopmax);
}
supported_players = max(min(255u8, mostplayers), 0u8).try_into().unwrap();
}
else
{
supported_players = 0; // Displayed as ? for number of supported players
}
}
else
{
supported_players = 1;
}
}
else
{
has_multiplayer = false;
supported_players = 1;
}

return Ok(GameInfo {
steam_id,
igdb_id,
name: name.to_string(),
cover_id: cover_id.to_string(),
supported_players,
has_multiplayer,
});
}
}

/// Fetches a Twitch app bearer token for use with the IGDB API.
///
/// # Errors
///
/// `IGDBError::BadClient` is returned if the client ID is invalid.
///
/// `IGDBError::BadSecret` is returned if the client secret is invalid.
///
/// `IGDBError::ServerError` is returned if Twitch returned an error code.
///
/// `IGDBError::UnknownError` is returned for any unexpected status codes, or if Twitch was unreachable.
pub fn get_twitch_token(client_id: &str, secret: &str) -> Result<Token, IGDBError> {
let client = reqwest::blocking::Client::new();

let res = client.post(TOKEN_URL)
.query(&[
("client_id", client_id),
("client_secret", secret),
("grant_type", "client_credentials")
]).send();
if let Err(e) = res {
return Err(IGDBError::UnknownError(e.to_string()));
}

let res = res.unwrap();
if !res.status().is_success() {
match res.status() {
StatusCode::BAD_REQUEST =>
return Err(IGDBError::BadClient),
StatusCode::FORBIDDEN =>
return Err(IGDBError::BadSecret),
StatusCode::INTERNAL_SERVER_ERROR | StatusCode::BAD_GATEWAY =>
return Err(IGDBError::ServerError),
_ =>
return Err(IGDBError::UnknownError(res.text().unwrap_or("Unknown error".to_string())))
}
}
let response = res.text().expect("Reqwest errored while getting response text");
let t: Token = serde_json::from_str(&response).expect(&format!("Serde failed to load JSON response from IGDB (response: {})", response));
return Ok(t);
}

/// Fetch the info for the provided set of steam app IDs
///
/// Returns a tuple of two `Vec`s, a `Vec` containing the info of found games, and a `Vec` of app IDs not found.
/// If `appids` is empty, returns two empty `Vec`s.
///
/// # Errors
///
/// `IGDBError::BadAuth` is returned if either the `client_id` or `bearer_token` are invalid.
///
/// `IGDBError::ServerError` is returned if IGDB was unable to process the request.
///
///`IGDBError::UnknownError` is returned for any unexpected status codes, or if IGDB was unreachable.
pub fn get_steam_game_info(client_id: &str, bearer_token: &str, appids: &[u64]) -> Result<(Vec<GameInfo>, HashSet<u64>), IGDBError> {
if appids.is_empty()
{