Merge branch 'rust_backend'

.gitignore

@@ -3,4 +3,9 @@ config.json
 venv*
 .vscode
 igdb-cache.sqlite
-bearer-token.json
+bearer-token.json
+lib/wcwp_rust/target
+lib/wcwp_rust/Cargo.lock
+lib/whatcanweplay.pyd
+lib/bin
+.wcwp-commit-hash

__init__.py

@@ -14,18 +14,51 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+try:
+    import shutil, platform, os
+    path_prefix = os.path.dirname(os.path.abspath(__file__))
+    source = ""
+    dest = ""
+    if platform.system() == "Windows":
+        source = "lib/wcwp_rust/target/release/whatcanweplay.dll"
+        dest = "lib/bin/whatcanweplay.pyd"
+    else:
+        source = "lib/wcwp_rust/target/release/libwhatcanweplay.so"
+        dest = "lib/bin/whatcanweplay.so"
+    source = os.path.join(path_prefix, source)
+    dest = os.path.join(path_prefix, dest)
+
+    if not os.path.exists(dest):
+        os.makedirs(os.path.dirname(dest), exist_ok=True)
+        shutil.copy2(source, dest)
+    else:
+        source_time = os.path.getmtime(source)
+        dest_time = os.path.getmtime(dest)
+        try:
+            if dest_time < source_time:
+                print("Updating WhatCanWePlay rust library with newer library file...")
+                shutil.copy2(source, dest)
+        except Exception:
+            print("Failed to update WhatCanWePlay rust library. It will be re-attempted when next launched.")
+    
+    from .lib.bin import whatcanweplay as wcwp
+except Exception as e:
+    print("Failed to load WhatCanWePlay rust library. Please go to \"lib/wcwp_rust\" and run \"cargo build --release\"")
+    print(e)
+    raise e
+print("WhatCanWePlay rust lib loaded (" + str(wcwp.__file__) + ")")
+
 from flask import Flask, request, jsonify, Response, render_template, redirect, session, url_for, make_response
 import requests
 from urllib import parse
 from werkzeug.exceptions import BadRequest
 import json
-from .steam_utils import get_steam_user_info, get_steam_user_friend_list, get_owned_steam_games
-from .igdb_utils import get_steam_game_info
 from requests import HTTPError
 import secrets
 from datetime import timezone, datetime, timedelta
 from itsdangerous import URLSafeSerializer
 from os import path
+import traceback
 
 # Load config
 def create_app():
@@ -33,25 +66,30 @@ def create_app():
     config = json.load(open(path.join(root_path, "config.json"), "r"))
     steam_key = config["steam-key"]
     igdb_key = config["igdb-client-id"]
+    igdb_secret = config["igdb-secret"]
     debug = config.get("debug", config.get("DEBUG", False))
     enable_api_tests = config.get("enable-api-tests", debug)
     cookie_max_age_dict = config.get("cookie-max-age", {})
-    info_max_age_dict = config.get("info-max-age", {})
+    cache_max_age_dict = config.get("igdb-cache-max-age", config.get("igdb-cache-info-age", {}))
     source_url = config.get("source-url", "")
     contact_email = config["contact-email"]
     privacy_email = config.get("privacy-email", contact_email)
     connect_timeout = config.get("connect-timeout", 0.0)
+    commit_hash_filename = config.get("commit-hash-file", ".wcwp-commit-hash")
     donate_url = config.get("donate-url", "")
     if connect_timeout <= 0.0:
         connect_timeout = None
     read_timeout = config.get("read-timeout", 0.0)
     if read_timeout <= 0.0:
         read_timeout = None
+    cache_file = config.get("igdb-cache-file")
+    if not os.path.isabs(cache_file):
+        cache_file = os.path.join(root_path, cache_file)
 
     # Create uWSGI callable
     app = Flask(__name__)
     app.debug = debug
-    app.secret_key = config.get("secret-key", secrets.token_hex()) # If not set, cookies will be invalidated every time the app is reloaded
+    app.secret_key = config["secret-key"]
 
     # Hide requests to /steam_login to prevent linking Steam ID to IP in logs
     from werkzeug import serving
@@ -68,20 +106,61 @@ def create_app():
     cookie_max_age = timedelta(**cookie_max_age_dict).total_seconds()
     if cookie_max_age == 0:
         cookie_max_age = None
-    info_max_age = timedelta(**info_max_age_dict).total_seconds()
+    
+    # Setup cache info max age
+    cache_max_age = 0.0
+    if cache_file:
+        cache_max_age = timedelta(**cache_max_age_dict).total_seconds()
+
+    print("cookies set to expire after %f seconds" % cookie_max_age)
+    print("cache set to expire after %f seconds" % cache_max_age)
+
+    def fetch_and_store_commit_hash():
+        f = open(commit_hash_filename, "w")
+        import subprocess
+        args = ['--git-dir=' + os.path.join(os.path.abspath(os.path.dirname(__file__)), ".git"), 'rev-parse', '--short', 'HEAD']
+        try:
+            try:
+                commit_hash = subprocess.check_output(['git'] + args).decode("utf-8").strip()
+                f.write(commit_hash)
+                f.close()
+                return commit_hash
+            except:
+                commit_hash = subprocess.check_output(['/usr/bin/git'] + args).decode("utf-8").strip()
+                f.write(commit_hash)
+                f.close()
+                return commit_hash
+        except:
+            return ""
+
+    @app.before_first_request
+    def before_first_request():
+        fetch_and_store_commit_hash()
 
-    print("cookies set to expire after {} seconds".format(cookie_max_age))
-    print("steam info set to refresh after {} seconds".format(info_max_age))
+    def get_commit_hash():
+        try:
+            f = open(commit_hash_filename, "r")
+            hash = f.read()
+            return hash
+        except Exception:
+            traceback.print_exc()
+            return ""
 
     def basic_info_dict():
         email_rev = contact_email.split("@")
-        return {
+        basic_info = {
             "contact_email_user_reversed": email_rev[0][::-1],
             "contact_email_domain_reversed": email_rev[1][::-1],
             "source_url": source_url,
             "donate_url": donate_url
         }
 
+        commit = get_commit_hash()
+        if commit:
+            basic_info["commit"] = commit
+        
+        return basic_info
+
     # Tries to fetch the Steam info cookie, returns an errcode and a dict
     #
     # Errcodes:
@@ -113,18 +192,20 @@ def create_app():
 
     def refresh_steam_cookie(steamid: int, response):
         if steamid <= 0:
-            response.set_cookie("steam_info", "", secure=True)
-            return {}
-
-        info = get_steam_user_info(steam_key, [steamid], connect_timeout, read_timeout)
-
-        if info["errcode"] != 0 or not info["users"].get(steamid, {}).get("exists", False):
-            response.set_cookie("steam_info", "", secure=True)
+            response.set_cookie("steam_info", "", secure=True, httponly=True)
             return {}
         
-        info = info["users"][steamid]
-        if info_max_age:
-            info["expires"] = (datetime.now(timezone.utc) + timedelta(seconds=info_max_age)).timestamp()
+        info = {}
+        try:
+            info = wcwp.steam.get_steam_users_info(steam_key, [steamid])[0]
+        except IndexError:
+            response.set_cookie("steam_info", "", secure=True, httponly=True)
+            return {}
+        except Exception:
+            traceback.print_exc()
+            response.set_cookie("steam_info", "", secure=True, httponly=True)
+            return {}
+
         ser = URLSafeSerializer(app.secret_key)
         response.set_cookie(
             "steam_info",
@@ -143,7 +224,7 @@ def create_app():
         if errcode == 3:
             steam_info = refresh_steam_cookie(steam_info.get("steam_id", -1), response)
         elif errcode != 0:
-            response.set_cookie("steam_info", "", secure=True)
+            response.set_cookie("steam_info", "", secure=True, httponly=True)
             steam_info = {}
         
         response.data = render_template("home.html", steam_info=steam_info, **basic_info_dict())
@@ -157,7 +238,7 @@ def create_app():
     def steam_login():
         if request.method == "POST":
             steam_openid_url = 'https://steamcommunity.com/openid/login'
-            return_url = url_for("steam_login", _external=True)
+            return_url = request.base_url
             params = {
                 'openid.ns': "http://specs.openid.net/auth/2.0",
                 'openid.identity': "http://specs.openid.net/auth/2.0/identifier_select",
@@ -183,7 +264,7 @@ def create_app():
     def steam_logout():
         response = redirect(url_for("index"))
         response.headers["X-Robots-Tag"] = "none"
-        response.set_cookie("steam_info", "", secure=True)
+        response.set_cookie("steam_info", "", secure=True, httponly=True)
         return response
 
     def validate_steam_identity(params):
@@ -214,46 +295,196 @@ def create_app():
             )
         errcode, steam_info = fetch_steam_cookie(request)
         if "steam_id" not in steam_info.keys():
-            return ("Not signed in to Steam", 403)
+            return (
+                "Not signed in to Steam. Please refresh the page.",
+                403
+            )
         
-        friends_info = get_steam_user_friend_list(
-            steam_key,
-            steam_info["steam_id"],
-            connect_timeout,
-            read_timeout
-        )
+        try:
+            friends_info = wcwp.steam.get_friend_list(
+                steam_key,
+                steam_info["steam_id"]
+            )
+            
+            for user in friends_info:
+                if "steam_id" in user.keys():
+                    user["steam_id"] = str(user["steam_id"])
+                    user["exists"] = True
+
+            return jsonify(friends_info)
+        except wcwp.steam.BadWebkeyException:
+            traceback.print_exc()
+            return (
+                "Site has bad Steam API key. Please contact us about this error at " + contact_email,
+                500
+            )
+        except wcwp.steam.ServerErrorException:
+            traceback.print_exc()
+            return (
+                "Steam had an internal server error. Please try again later.",
+                500
+            )
+        except wcwp.steam.BadResponseException:
+            traceback.print_exc()
+            return (
+                "Steam returned an unparseable response. Please try again later.",
+                500
+            )
+        except wcwp.steam.FriendListPrivateException:
+            traceback.print_exc()
+            return (
+                "WhatCanWePlay cannot retrieve your friend list. Please change your friend list visibility to public and refresh the page.",
+                500
+            )
+        except Exception:
+            traceback.print_exc()
+            if debug:
+                return (
+                    traceback.format_exc(),
+                    500
+                )
+            else:
+                traceback.print_exc()
+                return (
+                    "An unknown error has occurred. Please try again later.",
+                    500
+                )
 
-        errcode = friends_info.pop("errcode")
-
-        if errcode == 1:
-            return ("Site has bad Steam API key. Please contact us about this error at " + contact_email, 500)
-        elif errcode == 2:
-            return ("Steam took too long to respond. Please try again later.", 500)
-        elif errcode == 3:
-            return ("Steam took too long to transmit info. Please try again later.", 500)
-        elif errcode == 4:
-            return ("Your Friend List is not publicly accessible, and cannot be retrieved by WhatCanWePlay. Please set your Friend list visibility to Public and refresh the page.", 500)
-        elif errcode == -1:
-            return ("An unknown error occurred. Please try again later.", 500)
+    def refresh_igdb_token():
+        try:
+            token_path = path.join(root_path, "bearer-token.json")
+            token = wcwp.igdb.fetch_twitch_token(igdb_key, igdb_secret)
+            token["expiry"] = datetime.now(timezone.utc).timestamp() + token.get("expires_in", 0)
+            token.pop("expires_in")
+            json.dump(token, open(token_path, "w"))
+            return token.get("access_token", "")
+        except Exception:
+            traceback.print_exc()
+            return ""
+
+    def get_igdb_token():
+        try:
+            token_path = path.join(root_path, "bearer-token.json")
+            if path.exists(token_path):
+                token_file = open(token_path)
+                token = json.load(token_file)
+                token_file.close()
+                if datetime.now(timezone.utc).timestamp() >= token.get("expiry", 0):
+                    return refresh_igdb_token()
+                else:
+                    return token.get("access_token", "")
+            else:
+                return refresh_igdb_token()
+        except Exception:
+            traceback.print_exc()
+            return ""
+
+    cache_init_query = """
+    CREATE TABLE IF NOT EXISTS game (
+        steam_id INTEGER PRIMARY KEY,
+        igdb_id INTEGER,
+        name STRING,
+        supported_players INTEGER DEFAULT(0),
+        cover_id STRING,
+        has_multiplayer BOOLEAN,
+        expiry REAL DEFAULT(0.0)
+    );
+    """
+
+    CACHE_VERSION = 1
+
+    def initialize_cache():
+        import sqlite3
+        cache = sqlite3.connect(cache_file)
+        cache.execute(cache_init_query)
+        #cache.execute("PRAGMA user_version = ?;", [CACHE_VERSION]) # Doesn't work?
+        cache.execute("PRAGMA user_version = %d" % CACHE_VERSION)
+        return cache
+    
+    def cache_is_correct_version(cache):
+        return cache.execute("PRAGMA user_version;").fetchone()[0] == CACHE_VERSION
+
+    def update_cached_games(game_info):
+        if not cache_file:
+            return
         
-        friends_info = get_steam_user_info(steam_key, friends_info["friends"], connect_timeout, read_timeout)
-
-        errcode = friends_info.pop("errcode")
-        if errcode == 1:
-            return ("Site has bad Steam API key. Please contact us about this error at " + contact_email, 500)
-        elif errcode == 2:
-            return ("Steam took too long to respond. Please try again later.", 500)
-        elif errcode == 3:
-            return ("Steam took too long to transmit info. Please try again later.", 500)
-        elif errcode == -1:
-            return ("An unknown error occurred. Please try again later.", 500)
+        try:
+            import sqlite3
+            cache = None
+
+            if os.path.exists(cache_file):
+                cache = sqlite3.connect(cache_file)
+                # Check if cache is correct version
+                if not cache_is_correct_version(cache):
+                    # Cache is the wrong version, rebuild
+                    print("Cache file is the wrong version! Rebuilding... ")
+                    cache.close()
+                    os.remove(cache_file)
+                    cache = initialize_cache()
+            else:
+                cache = initialize_cache()
+            
+            insert_info = [
+                [
+                    game.get("steam_id"),
+                    game.get("igdb_id"),
+                    game.get("name"),
+                    game.get("supported_players"),
+                    game.get("cover_id"),
+                    game.get("has_multiplayer"),
+                    datetime.now(timezone.utc).timestamp() + cache_max_age
+                ] for game in game_info
+            ]
+            
+            cache.executemany(
+                "INSERT OR REPLACE INTO game VALUES (?,?,?,?,?,?,?);",
+                insert_info
+            )
+
+            cache.commit()
+            cache.close()
+        except Exception:
+            print("FAILED TO UPDATE CACHE DB")
+            traceback.print_exc()
+            return
+    
+    # returns [info of cached games], (set of uncached ids)
+    def get_cached_games(steam_ids):
+        if not cache_file:
+            return [], set(steam_ids)
+
+        game_info = []
+        uncached = set(steam_ids)
         
-        for user in friends_info["users"].values():
-            if "steam_id" in user.keys():
-                user["steam_id"] = str(user["steam_id"])
-                user["exists"] = True
+        try:
+            import sqlite3
+            cache = sqlite3.connect(cache_file)
+            cache.row_factory = sqlite3.Row
 
-        return jsonify(friends_info["users"])
+            if not cache_is_correct_version(cache):
+                return [], set(steam_ids)
+            
+            query_str = "SELECT * FROM game WHERE steam_id IN (%s)" % ("?" + (",?" * (len(steam_ids) - 1))) # Construct a query with arbitrary parameter length
+            
+            cursor = cache.execute(
+                query_str,
+                steam_ids
+            )
+            for row in cursor.fetchall():
+                game = dict(row)
+                if datetime.now(timezone.utc).timestamp() < game.pop("expiry"):
+                    # Info hasn't expired
+                    game_info.append(game)
+                    uncached.remove(game["steam_id"])
+
+                # Expired info gets updated during update_cached_games()
+        except Exception:
+            print("EXCEPTION THROWN WHILE QUERYING GAME CACHE!")
+            traceback.print_exc()
+            return [], set(steam_ids)
+        
+        return game_info, uncached
+        
 
     # Errcodes
     # -1: An error occurred with a message. Additional fields: "message"
@@ -316,90 +547,86 @@ def create_app():
                 json.dumps({"message": "Games intersection is capped at 10 users.", "errcode": -1}),
                 200
             )
-        
-        free_games = bool(body.get("include_free_games", False))
 
-        all_own = None
+        try:
+            token = get_igdb_token()
+            game_ids = wcwp.steam.intersect_owned_game_ids(steam_key, list(steamids))
 
-        for steamid in steamids:
-            user_owned_games = get_owned_steam_games(steam_key, steamid, free_games, connect_timeout, read_timeout)
-            errcode = user_owned_games["errcode"]
+            fetched_game_count = 0
+            cached_game_count = 0
+            game_info = []
 
-            if errcode == 1:
-                return (
-                    json.dumps({"message": "Site has bad Steam API key. Please contact us about this error at {}.".format(contact_email), "errcode": -1}),
-                    200
-                )
-            elif errcode == 2:
-                return (
-                    json.dumps({"message": "Steam took too long to respond. Please try again later.", "errcode": -1}),
-                    200
-                )
-            elif errcode == 3:
-                return (
-                    json.dumps({"message": "Steam took too long to transmit info. Please try again later.", "errcode": -1}),
-                    200
-                )
-            elif errcode == 4:
-                return (
-                    json.dumps({"user": str(steamid), "errcode": 1}),
-                    200
-                )
-            elif errcode == -1:
-                return (
-                    json.dumps({"message": "An unknown error occurred. Please try again later.", "errcode": -1}),
-                    200
-                )
-            
-            games = user_owned_games.get("games")
-            if len(games) == 0:
-                return (
-                    json.dumps({"user": str(steamid), "errcode": 2}),
-                    200
-                )
-            
-            if not all_own:
-                all_own = set(games)
-            else:
-                all_own = all_own & set(games)
-            
-            if len(all_own) == 0:
-                break
-        
-        # Step two: Fetch the game info
-        game_info = get_steam_game_info(igdb_key, all_own, connect_timeout, read_timeout)
+            if game_ids:
+                game_info, uncached_ids = get_cached_games(game_ids)
 
-        errcode = game_info["errcode"]
+                cached_game_count = len(game_info)
 
-        if errcode == 1:
+                if uncached_ids:
+                    fetched_info, not_found = wcwp.igdb.get_steam_game_info(igdb_key, token, list(uncached_ids))
+
+                    cache_info_update = fetched_info
+                    if not_found:
+                        for uncached_id in [id for id in not_found]:
+                            cache_info_update.append({"steam_id": uncached_id}) # Cache empty data to prevent further IGDB fetch attempts
+                    update_cached_games(cache_info_update) # TODO: Spin up separate process for caching?
+
+                    game_info += fetched_info
+                    fetched_game_count = len(fetched_info)
+            
+            print("Intersection resulted in %d games (%d from cache, %d from IGDB)" % (len(game_info), cached_game_count, fetched_game_count))
+
+            return jsonify({
+                "message": "Intersected successfully",
+                "games": game_info,
+                "errcode": 0
+            })
+        except wcwp.steam.BadWebkeyException:
+            traceback.print_exc()
             return (
-                json.dumps({"message": "Site has bad IGDB API key. Please contact us about this error at {}.".format(contact_email), "errcode": -1}),
-                200
+                json.dumps({"message": "Site has bad Steam API key. Please contact us about this error at " + contact_email, "errcode": -1}),
+                500
             )
-        elif errcode == 2:
+        except wcwp.steam.ServerErrorException:
+            traceback.print_exc()
             return (
-                json.dumps({"message": "IGDB took too long to respond. Please try again later.", "errcode": -1}),
-                200
+                json.dumps({"message": "Steam had an internal server error. Please try again later.", "errcode": -1}),
+                500
             )
-        elif errcode == 3:
+        except wcwp.steam.BadResponseException:
+            traceback.print_exc()
             return (
-                json.dumps({"message": "IGDB took too long to transmit info. Please try again later.", "errcode": -1}),
-                200
+                json.dumps({"message": "Steam returned an unparseable response. Please try again later.", "errcode": -1}),
+                500
             )
-        elif errcode == 4:
+        except wcwp.steam.GamesListPrivateException as e:
+            if debug:
+                print(e)
+            else:
+                print("Intersection interrupted due to private games list")
             return (
-                json.dumps({"message": "WhatCanWePlay failed to acquire an IGDB token. Please contact us about this error at {}.".format(contact_email), "errcode": -1})
+                json.dumps({"errcode": 1, "user": str(e.args[1])}),
+                500
             )
-        elif errcode == -1:
+        except wcwp.steam.GamesListEmptyException as e:
+            if debug:
+                print(e)
+            else:
+                print("Intersection interrupted due to private games list")
             return (
-                json.dumps({"message": "An unknown error occurred. Please try again later.", "errcode": -1}),
-                200
+                json.dumps({"errcode": 2, "user": str(e.args[1])}),
+                500
             )
-
-        return jsonify({
-            "message": "Intersected successfully",
-            "games": list(game_info.get("games", {}).values()),
-            "errcode": 0
-        })
+        except Exception:
+            traceback.print_exc()
+            if debug:
+                return (
+                    json.dumps({"message": traceback.format_exc(), "errcode": -1}),
+                    500
+                )
+            else:
+                return (
+                    json.dumps({"message": "An unknown error has occurred. Please try again later.", "errcode": -1}),
+                    500
+                )
 
     return app

config_EXAMPLE.json

@@ -4,22 +4,17 @@
     "steam-key": "YOUR STEAM KEY HERE",
     "igdb-client-id": "YOUR IGDB KEY HERE",
     "igdb-secret": "YOUR IGDB SECRET HERE",
-    "www-host": "yoururl.here",
     "cookie-max-age": {
         "days": 0,
         "weeks": 0
     },
-    "info-max-age": {
-        "hours": 0,
-        "minutes": 0
-    },
     "source-url": "https://git.tgrc.dev/tgrcdev/whatcanweplay",
     "contact-email": "contact@whatcanweplay.net",
     "privacy-email": "privacy@whatcanweplay.net",
     "connect-timeout": 8,
     "read-timeout": 30,
     "igdb-cache-file": "igdb-cache.sqlite",
-    "igdb-cache-info-age": {
+    "igdb-cache-max-age": {
         "weeks": 4
     }
 }

igdb_utils.py

@@ -1,251 +0,0 @@
-# This file is a part of WhatCanWePlay
-# Copyright (C) 2020 TGRCDev
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published
-# by the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-import requests
-import json
-import sqlite3
-from requests.exceptions import ConnectTimeout, ReadTimeout
-from typing import Dict, Collection, Any, Mapping, Optional
-from datetime import timedelta, datetime, timezone
-from os import path
-
-api_base = "https://api.igdb.com/v4/"
-
-root_path = path.dirname(__file__)
-config = json.load(open(path.join(root_path, "config.json"), "r"))
-debug = config.get("debug", config.get("DEBUG", False))
-db_filename = config.get("igdb-cache-filename", "igdb-cache.sqlite")
-if not path.isabs(db_filename):
-    db_filename = path.join(root_path, db_filename)
-
-create_db_query = """
-    CREATE TABLE IF NOT EXISTS game_info (
-        steam_id INTEGER PRIMARY KEY,
-        igdb_id INTEGER,
-        name TEXT,
-        cover_id TEXT,
-        has_multiplayer BOOL,
-        supported_players TEXT,
-        time_cached REAL
-    );
-"""
-
-info_age_dict = config.get("igdb-cache-info-age", {})
-info_age = timedelta(**info_age_dict).total_seconds()
-
-def fetch_and_store_token() -> str:
-    r = requests.post(
-        "https://id.twitch.tv/oauth2/token",
-        json={
-            "client_id": config["igdb-client-id"],
-            "client_secret": config["igdb-secret"],
-            "grant_type": "client_credentials"
-        }
-    )
-
-    if debug:
-        r.raise_for_status()
-    
-    try:
-        token = r.json()
-        token_dict = {
-            "token": token["access_token"],
-            "expire-time": datetime.now(timezone.utc).timestamp() + token["expires_in"],
-            "token_type": token["token_type"]
-        }
-        token_file = open(path.join(root_path, "bearer-token.json"), "w")
-        json.dump(token_dict, token_file)
-        token_file.close()
-        return token["access_token"]
-    except Exception as e:
-        print("Exception thrown while parsing token return")
-        print(e)
-        return ""
-
-def get_or_refresh_token() -> str:
-    try:
-        token_path = path.join(root_path, "bearer-token.json")
-        if path.exists(token_path):
-            token_file = open(token_path)
-            token = json.load(token_file)
-            token_file.close()
-            if datetime.now(timezone.utc).timestamp() >= token.get("expire-time", 0):
-                return fetch_and_store_token()
-            else:
-                return token.get("token", "")
-        else:
-            return fetch_and_store_token()
-    except json.JSONDecodeError as e:
-        print("Failed to decode token JSON.")
-        print(e)
-        return ""
-    except Exception as e:
-        print("Failed to fetch bearer token.")
-        print(e)
-        return ""
-
-def get_cached_games(appids: Collection[int]) -> Dict[int, Dict[str, Any]]:
-    query = """
-    SELECT steam_id, igdb_id, name, cover_id, has_multiplayer, supported_players
-    FROM game_info
-    WHERE time_cached > ? AND steam_id in ({})
-    """.format(",".join(
-        ["?" for _ in range(len(appids))]
-    ))
-    db_handle = sqlite3.connect(db_filename)
-    db_handle.execute(create_db_query)
-
-    cursor = db_handle.cursor()
-    cursor.execute(query, [datetime.now(timezone.utc).timestamp() - info_age, *list(appids)])
-    results = cursor.fetchall()
-
-    return {
-        game[0]: {
-            "steam_id": game[0],
-            "name": game[2],
-            "cover_id": game[3],
-            "has_multiplayer": game[4],
-            "supported_players": game[5]
-        } if game[1] else {}
-        for game in results
-    }
-
-def update_cached_games(game_info: Mapping[int, Mapping[str, Any]]):
-    query = """
-    INSERT OR REPLACE INTO game_info
-    (steam_id, igdb_id, name, cover_id, has_multiplayer, supported_players, time_cached)
-    VALUES (?,?,?,?,?,?,?)
-    """
-    db_handle = sqlite3.connect(db_filename)
-    db_handle.execute(create_db_query)
-    cursor = db_handle.cursor()
-    now = datetime.now(timezone.utc).timestamp()
-    cursor.execute("BEGIN TRANSACTION")
-    cursor.executemany(query, [[
-        game.get("steam_id"),
-        game.get("igdb_id"),
-        game.get("name"),
-        game.get("cover_id"),
-        game.get("has_multiplayer"),
-        game.get("supported_players"),
-        now] for game in game_info.values()]
-    )
-    cursor.execute("END TRANSACTION")
-
-def get_steam_game_info(webkey: str, appids: Collection[int], connect_timeout: Optional[float] = None, read_timeout: Optional[float] = None) -> Dict[int, Dict[str, Any]]:
-    if len(appids) == 0:
-        return {"errcode":0, "games":{}}
-    
-    appid_set = set(appids)
-    
-    cache_error = False
-
-    try:
-        cached_games = get_cached_games(appid_set)
-    except Exception as e:
-        print("An exception occurred while retrieving cached games: " + str(e))
-        cached_games = {}
-        cache_error = True
-
-    if len(cached_games) == len(appid_set):
-        return {"errcode": 0, "games": cached_games}
-
-    uncached_ids = appid_set - set(cached_games.keys())
-    uncached_ids_list = list(uncached_ids)
-
-    token = get_or_refresh_token()
-    if not token:
-        return {"errcode": 4}
-
-    games_dict = {}
-    retrieved_games = 0
-    while retrieved_games < len(uncached_ids_list):
-        game_slice = uncached_ids_list[retrieved_games:500]
-
-        retrieved_games += len(game_slice)
-
-        try:
-            r = requests.post(
-                api_base + "external_games",
-                data = "fields uid,game.name,game.game_modes,game.multiplayer_modes.onlinemax,game.multiplayer_modes.onlinecoopmax,game.cover.image_id; where uid = ({}) & category = 1; limit {};".format(",".join(map(str, game_slice)), len(game_slice)),
-                headers = {
-                    "Client-ID": webkey,
-                    "Authorization": "Bearer " + token,
-                    "Accept": "application/json"
-                },
-                timeout = (connect_timeout, read_timeout)
-            )
-        except ConnectTimeout:
-            return {"errcode": 2}
-        except ReadTimeout:
-            return {"errcode": 3}
-        if r.status_code == 403:
-            return {"errcode": 1}
-        elif r.status_code != 200:
-            if debug:
-                r.raise_for_status()
-            return {"errcode": -1}
-        
-        for game in r.json():
-            steam_id = game.get("uid", None)
-            if not steam_id:
-                continue
-            steam_id = int(steam_id)
-            game = game.get("game", None)
-            if not game:
-                continue
-            uncached_ids.discard(steam_id)
-
-            game_modes = game.get("game_modes", [])
-
-            is_multiplayer = (2 in game_modes or 5 in game_modes)
-
-            maxplayers = -1
-            if is_multiplayer:
-                for mode in game.get("multiplayer_modes", []):
-                    maxplayers = max(max(mode.get("onlinemax", 1), mode.get("onlinecoopmax", 1)), maxplayers)
-            else:
-                maxplayers = 1
-
-            games_dict[steam_id] = {
-                "steam_id": steam_id,
-                "igdb_id": game["id"],
-                "name": game["name"],
-                "cover_id": game.get("cover", {}).get("image_id", ""),
-                "has_multiplayer": is_multiplayer,
-                "supported_players": str(maxplayers) if maxplayers > 0 else "?"
-            }
-    
-    # Any games that couldn't be retrieved probably dont exist. Store them so they don't trigger an IGDB fetch.
-    for nonexist_id in uncached_ids:
-        games_dict[nonexist_id] = {"steam_id": nonexist_id}
-    
-    if len(games_dict) == 0:
-        return {"errcode":0, "games":cached_games}
-    
-    for id in games_dict.keys():
-        cached_games[id] = games_dict[id]
-    
-    if not cache_error:
-        try:
-            update_cached_games(games_dict)
-        except Exception as e:
-            print("An exception occurred while updating cached games: " + str(e))
-
-    return {
-        "errcode": 0,
-        "games": cached_games
-    }

lib/wcwp_rust/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "whatcanweplay"
+version = "0.1.0"
+authors = ["TGRCDev <tgrc@tgrc.dev>"]
+edition = "2018"
+
+[lib]
+name = "whatcanweplay"
+crate-type = ["cdylib"]
+
+[dependencies]
+reqwest = { version = "0.10", features = ["json", "blocking"] }
+serde_json = "1.0"
+serde = { version = "1.0", features = ["derive", "std"] }
+pyo3 = { version = "0.12", features = ["extension-module"] }

lib/wcwp_rust/src/errors.rs

@@ -0,0 +1,80 @@
+#[derive(Debug)]
+pub enum IGDBError {
+    UnknownError(String), // Unhandled error
+    ServerError, // IGDB had an internal error
+    BadResponse, // IGDB returned an unparseable response
+    BadClient, // The supplied client ID is wrong
+    BadSecret, // The supplied client secret is wrong
+    BadToken, // The supplied bearer token is wrong
+    BadAuth, // Some part of the supplied authentication is wrong
+}
+
+use std::fmt;
+
+impl fmt::Display for IGDBError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            IGDBError::ServerError => return write!(f, "IGDB had an internal server error"),
+            IGDBError::BadResponse => return write!(f, "IGDB returned an unparseable response"),
+            IGDBError::BadClient => return write!(f, "IGDB rejected the provided client ID"),
+            IGDBError::BadSecret => return write!(f, "IGDB rejected the provided client secret"),
+            IGDBError::BadToken => return write!(f, "IGDB rejected the provided bearer token"),
+            IGDBError::BadAuth => return write!(f, "IGDB rejected some part of the provided authentication"),
+            IGDBError::UnknownError(err_string) => return write!(f, "{}", &err_string),
+            _ => return write!(f, "IGDB API had an unknown error")
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum SteamError {
+    UnknownError(String), // Unhandled error
+    ServerError, // Steam had an internal error
+    BadResponse, // Steam returned an unparseable response
+    BadWebkey, // The supplied webkey is wrong
+    GamesListPrivate(u64), // The games list of the given steam id is unretrievable, probably due to game list privacy settings
+    GamesListEmpty(u64), // Intersection failed, the user with the given steam id had no games
+    FriendListPrivate, // The steam id passed to get_friend_list() has their friend list set to private
+}
+
+impl fmt::Display for SteamError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            SteamError::ServerError => return write!(f, "Steam had an internal server error"),
+            SteamError::BadResponse => return write!(f, "Steam returned an unparseable response"),
+            SteamError::BadWebkey => return write!(f, "Steam rejected the provided webkey"),
+            SteamError::UnknownError(err_string) => return write!(f, "{}", &err_string),
+            SteamError::GamesListPrivate(steamid) => return write!(f, "The user with Steam ID {} has their games list set to private", steamid),
+            SteamError::GamesListEmpty(steamid) => return write!(f, "The user with the Steam ID {} has no games to intersect", steamid),
+            SteamError::FriendListPrivate => return write!(f, "The given user has their friend list set to private"),
+            _ => return write!(f, "Steam had an unknown error")
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum WCWPError {
+    SteamError(SteamError), // Steam API Error
+    IGDBError(IGDBError), // IGDB API Error
+}
+
+impl fmt::Display for WCWPError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            WCWPError::SteamError(e) => return e.fmt(f),
+            WCWPError::IGDBError(e) => return e.fmt(f),
+        }
+    }
+}
+
+impl From<SteamError> for WCWPError {
+    fn from(e: SteamError) -> Self {
+        return WCWPError::SteamError(e);
+    }
+}
+
+impl From<IGDBError> for WCWPError {
+    fn from(e: IGDBError) -> Self {
+        return WCWPError::IGDBError(e);
+    }
+}

lib/wcwp_rust/src/igdb.rs

@@ -0,0 +1,306 @@
+use reqwest;
+use serde::{Deserialize, Serialize};
+use serde_json;
+use std::collections::HashSet;
+use std::fmt::{Debug, Write};
+use crate::errors::IGDBError;
+use reqwest::StatusCode;
+use serde_json::Value;
+use std::convert::{TryFrom, TryInto};
+use std::cmp::{max, min};
+
+const API_URL : &str = "https://api.igdb.com/v4/";
+const TOKEN_URL : &str = "https://id.twitch.tv/oauth2/token";
+
+#[derive(Serialize, Deserialize, Debug)]
+/// Struct for retrieving a bearer token from the Twitch Developer API
+pub struct Token {
+    pub access_token: String,
+    pub expires_in: u64
+}
+
+#[derive(Debug, Serialize)]
+/// Struct for games described by IGDB
+pub struct GameInfo {
+    pub steam_id: u64,
+    pub igdb_id: u64,
+    pub name: String,
+    pub supported_players: u8,
+    
+    /// Cover image url
+    ///
+    /// Usage: `https://images.igdb.com/igdb/image/upload/t_cover_small/<cover_id>.jpg`
+    pub cover_id: String, 
+
+    pub has_multiplayer: bool,
+}
+
+/// Takes a serde_json Value and tries to get an i64
+/// either by parsing an i64 directly, or by parsing
+/// a stringified i64.
+fn parse_u8_from_value(value: &Value) -> Option<u8> {
+    let num = parse_u64_from_value(value);
+    match num {
+        Some(num) =>
+            return num.try_into().ok(),
+        None =>
+            return None
+    }
+}
+
+/// Takes a serde_json Value and tries to get an i64
+/// either by parsing an u64 directly, or by parsing
+/// a stringified u64.
+fn parse_u64_from_value(value: &Value) -> Option<u64> {
+    match value {
+        Value::Number(num) => {
+            return num.as_u64();
+        },
+        Value::String(string) => {
+            return string.parse().ok();
+        },
+        _ => return None
+    }
+}
+
+impl TryFrom<&Value> for GameInfo {
+    type Error = IGDBError;
+
+    /// Used to extract game information from IGDB query results
+    fn try_from(value: &Value) -> Result<Self, IGDBError> {
+        let steam_id : u64;
+        match parse_u64_from_value(&value["uid"]) {
+            Some(id) => steam_id = id,
+            None => return Err(IGDBError::BadResponse)
+        }
+
+        let game = &value["game"];
+        if !game.is_object()
+        {
+            return Err(IGDBError::BadResponse);
+        }
+
+        let igdb_id : u64;
+        match parse_u64_from_value(&game["id"]) {
+            Some(id) => igdb_id = id,
+            None => return Err(IGDBError::BadResponse)
+        }
+
+        let name : &str;
+        match game["name"].as_str() {
+            Some(val) => name = val,
+            None => return Err(IGDBError::BadResponse),
+        }
+
+        let cover_id : &str;
+        match game["cover"]["image_id"].as_str() {
+            Some(val) => cover_id = val,
+            None => cover_id = "",
+        }
+
+        // Figure out multiplayer and whatnot
+
+        let game_modes = &game["game_modes"];
+        let has_multiplayer: bool;
+        let supported_players : u8;
+
+        if game_modes.is_array() {
+            let gamemode_arr = game_modes.as_array().unwrap();
+            
+            has_multiplayer = gamemode_arr.iter().any(|x| x == 5 || x == 2); // TODO: This isn't really needed with game["multiplayer_modes"], replace it
+            if has_multiplayer
+            {
+                let multiplayer_modes = &game["multiplayer_modes"];
+                if let Some(multimodes) = multiplayer_modes.as_array() {
+                    let mut mostplayers = 0;
+                    for mode in multimodes.iter() {
+                        let onlinemax : u8 = parse_u8_from_value(&mode["onlinemax"]).unwrap_or(0);
+                        let coopmax : u8 = parse_u8_from_value(&mode["onlinecoopmax"]).unwrap_or(0);
+                        mostplayers = max(onlinemax, coopmax);
+                    }
+                    supported_players = max(min(255u8, mostplayers), 0u8).try_into().unwrap();
+                }
+                else
+                {
+                    supported_players = 0; // Displayed as ? for number of supported players
+                }
+            }
+            else
+            {
+                supported_players = 1;
+            }
+        }
+        else
+        {
+            has_multiplayer = false;
+            supported_players = 1;
+        }
+
+        return Ok(GameInfo {
+            steam_id,
+            igdb_id,
+            name: name.to_string(),
+            cover_id: cover_id.to_string(),
+            supported_players,
+            has_multiplayer,
+        });
+    }
+}
+
+/// Fetches a Twitch app bearer token for use with the IGDB API.
+///
+/// # Errors
+///
+/// `IGDBError::BadClient` is returned if the client ID is invalid.
+///
+/// `IGDBError::BadSecret` is returned if the client secret is invalid.
+///
+/// `IGDBError::ServerError` is returned if Twitch returned an error code.
+///
+/// `IGDBError::UnknownError` is returned for any unexpected status codes, or if Twitch was unreachable.
+pub fn get_twitch_token(client_id: &str, secret: &str) -> Result<Token, IGDBError> {
+    let client = reqwest::blocking::Client::new();
+
+    let res = client.post(TOKEN_URL)
+        .query(&[
+            ("client_id", client_id),
+            ("client_secret", secret),
+            ("grant_type", "client_credentials")
+        ]).send();
+    
+    if let Err(e) = res {
+        return Err(IGDBError::UnknownError(e.to_string()));
+    }
+
+    let res = res.unwrap();
+    
+    if !res.status().is_success() {
+        match res.status() {
+            StatusCode::BAD_REQUEST => 
+                return Err(IGDBError::BadClient),
+            StatusCode::FORBIDDEN => 
+                return Err(IGDBError::BadSecret),
+            StatusCode::INTERNAL_SERVER_ERROR | StatusCode::BAD_GATEWAY => 
+                return Err(IGDBError::ServerError),
+            _ =>
+                return Err(IGDBError::UnknownError(res.text().unwrap_or("Unknown error".to_string())))
+        }
+    }
+    
+    let response = res.text().expect("Reqwest errored while getting response text");
+    let t: Token = serde_json::from_str(&response).expect(&format!("Serde failed to load JSON response from IGDB (response: {})", response));
+    
+    return Ok(t);
+}
+
+/// Fetch the info for the provided set of steam app IDs
+///
+/// Returns a tuple of two `Vec`s, a `Vec` containing the info of found games, and a `Vec` of app IDs not found.
+/// If `appids` is empty, returns two empty `Vec`s.
+///
+/// # Errors
+///
+/// `IGDBError::BadAuth` is returned if either the `client_id` or `bearer_token` are invalid.
+/// 
+/// `IGDBError::ServerError` is returned if IGDB was unable to process the request.
+/// 
+///`IGDBError::UnknownError` is returned for any unexpected status codes, or if IGDB was unreachable.
+pub fn get_steam_game_info(client_id: &str, bearer_token: &str, appids: &[u64]) -> Result<(Vec<GameInfo>, HashSet<u64>), IGDBError> {
+    if appids.is_empty()
+    {