greatly enhanced the functionality of the generate_emu_config script + build script + updated .gitignore

.gitignore

@@ -14,3 +14,15 @@ base.lib
 steamclient.exp
 steamclient.lib
 out/*
+
+scripts/.py/
+scripts/.venv/
+scripts/.env/
+scripts/.vscode/
+scripts/backup/
+scripts/bin/
+scripts/build_tmp/
+scripts/login_temp/
+scripts/**/__pycache__/
+scripts/generate_emu_config.spec
+scripts/my_login.txt

scripts/controller_config_generator/parse_controller_vdf.py

@@ -160,6 +160,7 @@ def generate_controller_config(controller_vdf, config_dir):
 
     #print(all_bindings)
 
+    if all_bindings:
         if not os.path.exists(config_dir):
             os.makedirs(config_dir)
 

scripts/external_components/ach_watcher_gen.py

@@ -0,0 +1,173 @@
+import copy
+import os
+import time
+import json
+
+def __ClosestDictKey(targetKey : str, srcDict : dict[str, object] | set[str]) -> str | None:
+    for k in srcDict:
+        if k.lower() == f"{targetKey}".lower():
+            return k
+    
+    return None
+
+def __generate_ach_watcher_schema(lang: str, app_id: int, achs: list[dict]) -> list[dict]:
+    out_achs_list = []
+    for idx in range(len(achs)):
+        ach = copy.deepcopy(achs[idx])
+        out_ach_data = {}
+        
+        # adjust the displayName
+        displayName = ""
+        ach_displayName = ach.get("displayName", None)
+        if ach_displayName:
+            if type(ach_displayName) == dict: # this is a dictionary
+                displayName : str = ach_displayName.get(lang, "")
+                if not displayName and ach_displayName: # has some keys but language not found
+                    #print(f'[?] Missing language "{lang}" in "displayName" of achievement {ach["name"]}')
+                    nearestLang = __ClosestDictKey(lang, ach_displayName)
+                    if nearestLang:
+                        #print(f'[?] Best matching language "{nearestLang}"')
+                        displayName = ach_displayName[nearestLang]
+                    else:
+                        print(f'[?] Missing language "{lang}", using displayName from the first language for achievement {ach["name"]}')
+                        displayName : str = list(ach_displayName.values())[0]
+            else: # single string (or anything else)
+                displayName = ach_displayName
+            
+            del ach["displayName"]
+        else:
+            print(f'[?] Missing "displayName" in achievement {ach["name"]}')
+        
+        out_ach_data["displayName"] = displayName
+        
+        desc = ""
+        ach_desc = ach.get("description", None)
+        if ach_desc:
+            if type(ach_desc) == dict: # this is a dictionary
+                desc : str = ach_desc.get(lang, "")
+                if not desc and ach_desc: # has some keys but language not found
+                    #print(f'[?] Missing language "{lang}" in "description" of achievement {ach["name"]}')
+                    nearestLang = __ClosestDictKey(lang, ach_desc)
+                    if nearestLang:
+                        #print(f'[?] Best matching language "{nearestLang}"')
+                        desc = ach_desc[nearestLang]
+                    else:
+                        print(f'[?] Missing language "{lang}", using description from the first language for achievement {ach["name"]}')
+                        desc : str = list(ach_desc.values())[0]
+            else: # single string (or anything else)
+                desc = ach_desc
+            
+            del ach["description"]
+        else:
+            print(f'[?] Missing "description" in achievement {ach["name"]}')
+        
+        # adjust the description
+        out_ach_data["description"] = desc
+
+        # copy the rest of the data
+        out_ach_data.update(ach)
+
+        # add links to icon, icongray, and icon_gray
+        base_icon_url = r'https://cdn.cloudflare.steamstatic.com/steamcommunity/public/images/apps'
+        icon_hash = out_ach_data.get("icon", None)
+        if icon_hash:
+            out_ach_data["icon"] = f'{base_icon_url}/{app_id}/{icon_hash}'
+        else:
+            out_ach_data["icon"] = ""
+
+        icongray_hash = out_ach_data.get("icongray", None)
+        if icongray_hash:
+            out_ach_data["icongray"] = f'{base_icon_url}/{app_id}/{icongray_hash}'
+        else:
+            out_ach_data["icongray"] = ""
+
+        icon_gray_hash = out_ach_data.get("icon_gray", None)
+        if icon_gray_hash:
+            del out_ach_data["icon_gray"] # use the old key
+            out_ach_data["icongray"] = f'{base_icon_url}/{app_id}/{icon_gray_hash}'
+        
+        if "hidden" in out_ach_data:
+            try:
+                out_ach_data["hidden"] = int(out_ach_data["hidden"])
+            except Exception as e:
+                pass
+        else:
+            out_ach_data["hidden"] = 0
+
+        out_achs_list.append(out_ach_data)
+    
+    return out_achs_list
+
+def generate_all_ach_watcher_schemas(
+    base_out_dir : str,
+    appid: int,
+    app_name : str,
+    app_exe : str,
+    achs: list[dict],
+    small_icon_hash : str) -> None:
+    
+    ach_watcher_out_dir = os.path.join(base_out_dir, "Achievement Watcher", "steam_cache", "schema")
+    print(f"generating schemas for Achievement Watcher in: {ach_watcher_out_dir}")
+
+    if app_exe:
+        print(f"detected app exe: '{app_exe}'")
+    else:
+        print(f"[X] couldn't detect app exe")
+    
+    # if not achs:
+    #     print("[X] No achievements were found for Achievement Watcher")
+    #     return
+    
+    small_icon_url = ''
+    if small_icon_hash:
+        small_icon_url = f"https://cdn.cloudflare.steamstatic.com/steamcommunity/public/images/apps/{appid}/{small_icon_hash}.jpg"
+    images_base_url = r'https://cdn.cloudflare.steamstatic.com/steam/apps'
+    ach_watcher_base_schema = {
+        "appid": appid,
+        "name": app_name,
+        "binary": app_exe,
+        "achievement": {
+            "total": len(achs),
+        },
+        "img": {
+            "header":     f"{images_base_url}/{appid}/header.jpg",
+            "background": f"{images_base_url}/{appid}/page_bg_generated_v6b.jpg",
+            "portrait":   f"{images_base_url}/{appid}/library_600x900.jpg",
+            "hero":       f"{images_base_url}/{appid}/library_hero.jpg",
+            "icon":       small_icon_url,
+        },
+        "apiVersion": 1,
+    }
+    
+    langs : set[str] = set()
+    for ach in achs:
+        displayNameLangs = ach.get("displayName", None)
+        if displayNameLangs and type(displayNameLangs) == dict:
+            langs.update(list(displayNameLangs.keys()))
+            
+        descriptionLangs = ach.get("description", None)
+        if descriptionLangs and type(descriptionLangs) == dict:
+            langs.update(list(descriptionLangs.keys()))
+
+    if "token" in langs:
+        langs.remove("token")
+    
+    tokenKey = __ClosestDictKey("token", langs)
+    if tokenKey:
+        langs.remove(tokenKey)
+    
+    if not langs:
+        print("[X] Couldn't detect supported languages, assuming English is the only supported language for Achievement Watcher")
+        langs = ["english"]
+    
+    for lang in langs:
+        out_schema_folder = os.path.join(ach_watcher_out_dir, lang)
+        if not os.path.exists(out_schema_folder):
+            os.makedirs(out_schema_folder)
+            time.sleep(0.050)
+
+        out_schema = copy.copy(ach_watcher_base_schema)
+        out_schema["achievement"]["list"] = __generate_ach_watcher_schema(lang, appid, achs)
+        out_schema_file = os.path.join(out_schema_folder, f'{appid}.db')
+        with open(out_schema_file, "wt", encoding='utf-8') as f:
+            json.dump(out_schema, f, ensure_ascii=False, indent=2)

scripts/external_components/app_details.py

@@ -0,0 +1,230 @@
+import os
+import json
+import queue
+import threading
+import time
+import requests
+import urllib.parse
+from external_components import (
+    safe_name
+)
+
+def __downloader_thread(q : queue.Queue[tuple[str, str]]):
+    while True:
+        url, path = q.get()
+        if not url:
+            q.task_done()
+            return
+
+        # try 3 times
+        for download_trial in range(3):
+            try:
+                r = requests.get(url)
+                if r.status_code == requests.codes.ok: # if download was successfull
+                    with open(path, "wb") as f:
+                        f.write(r.content)
+
+                    break
+            except Exception:
+                pass
+
+            time.sleep(0.1)
+        
+        q.task_done()
+
+def __remove_url_query(url : str) -> str:
+    url_parts = urllib.parse.urlsplit(url)
+    url_parts_list = list(url_parts)
+    url_parts_list[3] = '' # remove query
+    return str(urllib.parse.urlunsplit(url_parts_list))
+
+def __download_screenshots(
+        base_out_dir : str,
+        appid : int,
+        app_details : dict,
+        download_screenshots : bool,
+        download_thumbnails : bool):
+    if not download_screenshots and not download_thumbnails:
+        return
+
+    screenshots : list[dict[str, object]] = app_details.get(f'{appid}', {}).get('data', {}).get('screenshots', [])
+    if not screenshots:
+        print(f'[?] no screenshots or thumbnails are available')
+        return
+    
+    screenshots_out_dir = os.path.join(base_out_dir, "screenshots")
+    if download_screenshots:
+        print(f"downloading screenshots in: {screenshots_out_dir}")
+        if not os.path.exists(screenshots_out_dir):
+            os.makedirs(screenshots_out_dir)
+            time.sleep(0.025)
+
+    thumbnails_out_dir = os.path.join(screenshots_out_dir, "thumbnails")
+    if download_thumbnails:
+        print(f"downloading screenshots thumbnails in: {thumbnails_out_dir}")
+        if not os.path.exists(thumbnails_out_dir):
+            os.makedirs(thumbnails_out_dir)
+            time.sleep(0.025)
+
+    q : queue.Queue[tuple[str, str]] = queue.Queue()
+
+    max_threads = 20
+    for i in range(max_threads):
+        threading.Thread(target=__downloader_thread, args=(q,), daemon=True).start()
+
+    for scrn in screenshots:
+        if download_screenshots:
+            full_image_url = scrn.get('path_full', None)
+            if full_image_url:
+                full_image_url_sanitized = __remove_url_query(full_image_url)
+                image_hash_name = f'{full_image_url_sanitized.rsplit("/", 1)[-1]}'.rstrip()
+                if image_hash_name:
+                    q.put((full_image_url_sanitized, os.path.join(screenshots_out_dir, image_hash_name)))
+                else:
+                    print(f'[X] cannot download screenshot from url: "{full_image_url}", failed to get image name')
+    
+        if download_thumbnails:
+            thumbnail_url = scrn.get('path_thumbnail', None)
+            if thumbnail_url:
+                thumbnail_url_sanitized = __remove_url_query(thumbnail_url)
+                image_hash_name = f'{thumbnail_url_sanitized.rsplit("/", 1)[-1]}'.rstrip()
+                if image_hash_name:
+                    q.put((thumbnail_url_sanitized, os.path.join(thumbnails_out_dir, image_hash_name)))
+                else:
+                    print(f'[X] cannot download screenshot thumbnail from url: "{thumbnail_url}", failed to get image name')
+        
+    q.join()
+
+    for i in range(max_threads):
+        q.put((None, None))
+    
+    q.join()
+        
+    print(f"finished downloading app screenshots")
+
+PREFERED_VIDS = [
+    'trailer', 'gameplay', 'announcement'
+]
+
+def __download_videos(base_out_dir : str, appid : int, app_details : dict):
+    videos : list[dict[str, object]] = app_details.get(f'{appid}', {}).get('data', {}).get('movies', [])
+    if not videos:
+        print(f'[?] no videos were found')
+        return
+    
+    videos_out_dir = os.path.join(base_out_dir, "videos")
+    print(f"downloading app videos in: {videos_out_dir}")
+
+    first_vid : tuple[str, str] = None
+    prefered_vid : tuple[str, str] = None
+    for vid in videos:
+        vid_name = f"{vid.get('name', '')}"
+        webm_url = vid.get('webm', {}).get("480", None)
+        mp4_url = vid.get('mp4', {}).get("480", None)
+
+        ext : str = None
+        prefered_url : str = None
+        if mp4_url:
+            prefered_url = mp4_url
+            ext = 'mp4'
+        elif webm_url:
+            prefered_url = webm_url
+            ext = 'webm'
+        else: # no url is found
+            print(f'[X] no url is found for video "{vid_name}"')
+            continue
+    
+        vid_url_sanitized = __remove_url_query(prefered_url)
+        vid_name_in_url = f'{vid_url_sanitized.rsplit("/", 1)[-1]}'.rstrip()
+        vid_name = safe_name.create_safe_name(vid_name)
+        if vid_name:
+            vid_name = f'{vid_name}.{ext}'
+        else:
+            vid_name = vid_name_in_url
+
+        if vid_name:
+            if not first_vid:
+                first_vid = (vid_url_sanitized, vid_name)
+
+            if any(vid_name.lower().find(candidate) > -1 for candidate in PREFERED_VIDS):
+                prefered_vid = (vid_url_sanitized, vid_name)
+
+            if prefered_vid:
+                break
+        else:
+            print(f'[X] cannot download video from url: "{prefered_url}", failed to get vido name')
+    
+    if not first_vid and not prefered_vid:
+        print(f'[X] no video url could be found')
+        return
+    elif not prefered_vid:
+        prefered_vid = first_vid
+
+    if not os.path.exists(videos_out_dir):
+        os.makedirs(videos_out_dir)
+        time.sleep(0.05)
+
+    q : queue.Queue[tuple[str, str]] = queue.Queue()
+
+    max_threads = 1
+    for i in range(max_threads):
+        threading.Thread(target=__downloader_thread, args=(q,), daemon=True).start()
+
+    # TODO download all videos
+    print(f'donwloading video: "{prefered_vid[1]}"')
+    q.put((prefered_vid[0], os.path.join(videos_out_dir, prefered_vid[1])))
+    q.join()
+
+    for i in range(max_threads):
+        q.put((None, None))
+    
+    q.join()
+        
+    print(f"finished downloading app videos")
+
+
+def download_app_details(
+    base_out_dir : str,
+    info_out_dir : str,
+    appid : int,
+    download_screenshots : bool,
+    download_thumbnails : bool,
+    download_vids : bool):
+
+    details_out_file = os.path.join(info_out_dir, "app_details.json")
+    print(f"downloading app details in: {details_out_file}")
+
+    app_details : dict = None
+    last_exception : Exception | str = None
+    # try 3 times
+    for download_trial in range(3):
+        try:
+            r = requests.get(f'http://store.steampowered.com/api/appdetails?appids={appid}&format=json')
+            if r.status_code == requests.codes.ok: # if download was successfull
+                result : dict = r.json()
+                json_ok = result.get(f'{appid}', {}).get('success', False)
+                if json_ok:
+                    app_details = result
+                    break
+                else:
+                    last_exception = "JSON success was False"
+        except Exception as e:
+            last_exception = e
+
+        time.sleep(0.1)
+
+    if not app_details:
+        err = "[X] failed to download app details"
+        if last_exception:
+            err += f', last error: "{last_exception}"'
+        
+        print(err)
+        return
+
+    with open(details_out_file, "wt", encoding='utf-8') as f:
+        json.dump(app_details, f, ensure_ascii=False, indent=2)
+    
+    __download_screenshots(base_out_dir, appid, app_details, download_screenshots, download_thumbnails)
+        
+    if download_vids:
+        __download_videos(base_out_dir, appid, app_details)

scripts/external_components/app_images.py

@@ -0,0 +1,94 @@
+import os
+import threading
+import time
+import requests
+
+
+def download_app_images(
+    base_out_dir : str,
+    appid : int,
+    clienticon : str,
+    icon : str,
+    logo : str,
+    logo_small : str):
+
+    icons_out_dir = os.path.join(base_out_dir, "images")
+    print(f"downloading common app images in: {icons_out_dir}")
+
+    def downloader_thread(image_name : str, image_url : str):
+        # try 3 times
+        for download_trial in range(3):
+            try:
+                r = requests.get(image_url)
+                if r.status_code == requests.codes.ok: # if download was successfull
+                    with open(os.path.join(icons_out_dir, image_name), "wb") as f:
+                        f.write(r.content)
+
+                    break
+            except Exception as ex:
+                pass
+
+            time.sleep(0.1)
+
+    app_images_names = [
+        r'capsule_184x69.jpg',
+        r'capsule_231x87.jpg',
+        r'capsule_231x87_alt_assets_0.jpg',
+        r'capsule_467x181.jpg',
+        r'capsule_616x353.jpg',
+        r'capsule_616x353_alt_assets_0.jpg',
+        r'library_600x900.jpg',
+        r'library_600x900_2x.jpg',
+        r'library_hero.jpg',
+        r'broadcast_left_panel.jpg',
+        r'broadcast_right_panel.jpg',
+        r'page.bg.jpg',
+        r'page_bg_raw.jpg',
+        r'page_bg_generated.jpg',
+        r'page_bg_generated_v6b.jpg',
+        r'header.jpg',
+        r'header_alt_assets_0.jpg',
+        r'hero_capsule.jpg',
+        r'logo.png',
+    ]
+
+    if not os.path.exists(icons_out_dir):
+        os.makedirs(icons_out_dir)
+        time.sleep(0.050)
+
+    threads_list : list[threading.Thread] = []
+    for image_name in app_images_names:
+        image_url = f'https://cdn.cloudflare.steamstatic.com/steam/apps/{appid}/{image_name}'
+        t = threading.Thread(target=downloader_thread, args=(image_name, image_url), daemon=True)
+        threads_list.append(t)
+        t.start()
+    
+    community_images_url = f'https://cdn.cloudflare.steamstatic.com/steamcommunity/public/images/apps/{appid}'
+    if clienticon:
+        image_url = f'{community_images_url}/{clienticon}.ico'
+        t = threading.Thread(target=downloader_thread, args=('clienticon.ico', image_url), daemon=True)
+        threads_list.append(t)
+        t.start()
+
+    if icon:
+        image_url = f'{community_images_url}/{icon}.jpg'
+        t = threading.Thread(target=downloader_thread, args=('icon.jpg', image_url), daemon=True)
+        threads_list.append(t)
+        t.start()
+
+    if logo:
+        image_url = f'{community_images_url}/{logo}.jpg'
+        t = threading.Thread(target=downloader_thread, args=('logo.jpg', image_url), daemon=True)
+        threads_list.append(t)
+        t.start()
+
+    if logo_small:
+        image_url = f'{community_images_url}/{logo_small}.jpg'
+        t = threading.Thread(target=downloader_thread, args=('logo_small.jpg', image_url), daemon=True)
+        threads_list.append(t)
+        t.start()
+
+    for t in threads_list:
+        t.join()
+
+    print(f"finished downloading common app images")

scripts/external_components/cdx_gen.py

@@ -0,0 +1,157 @@
+import os
+
+__cdx_ini = '''
+###        мллллл   м
+###      Алллл плл лВ ппплллллллм     пппппллВллм     мВлллп
+###     Блллп   Бллп        ппллллА           пллл  Блллп
+###    Вллл      п         ллВ  плллБ           АллВллл
+###   Вллл        млллллм  ллл   пллл мллллллм  Бллллл
+###   лллА     Аллллп  плВ ллл    лллВллВ  Алл  лллВллл
+###   Бллл     ллллА    лл ллл   Алллллллллллп лллБ Бллл
+###    Алллм мллпВллм млл Влл   лллБлллА     млллА   Алллм
+###      плллллп   плллВп  ллп АлллА  плллллллВлп       пВллм
+###                        мллллллБ
+###          пппллВмммммлВлллВпп    
+###
+###
+### Game data is stored at %SystemDrive%\\Users\\Public\\Documents\\Steam\\CODEX\\{cdx_id}
+###
+
+[Settings]
+###
+### Game identifier (http://store.steampowered.com/app/{cdx_id})
+###
+AppId={cdx_id}
+###
+### Steam Account ID, set it to 0 to get a random Account ID
+###
+#AccountId=0
+### 
+### Name of the current player
+###
+UserName=Player2
+###
+### Language that will be used in the game
+###
+Language=english
+###
+### Enable lobby mode
+###
+LobbyEnabled=1
+###
+### Lobby port to listen on
+###
+#LobbyPort=31183
+###
+### Enable/Disable Steam overlay
+###
+Overlays=1
+###
+### Set Steam connection to offline mode
+###
+Offline=0
+###
+
+[Interfaces]
+###
+### Steam Client API interface versions
+###
+SteamAppList=STEAMAPPLIST_INTERFACE_VERSION001
+SteamApps=STEAMAPPS_INTERFACE_VERSION008
+SteamClient=SteamClient017
+SteamController=SteamController008
+SteamFriends=SteamFriends017
+SteamGameServer=SteamGameServer013
+SteamGameServerStats=SteamGameServerStats001
+SteamHTMLSurface=STEAMHTMLSURFACE_INTERFACE_VERSION_005
+SteamHTTP=STEAMHTTP_INTERFACE_VERSION003
+SteamInput=SteamInput002
+SteamInventory=STEAMINVENTORY_INTERFACE_V003
+SteamMatchGameSearch=SteamMatchGameSearch001
+SteamMatchMaking=SteamMatchMaking009
+SteamMatchMakingServers=SteamMatchMakingServers002
+SteamMusic=STEAMMUSIC_INTERFACE_VERSION001
+SteamMusicRemote=STEAMMUSICREMOTE_INTERFACE_VERSION001
+SteamNetworking=SteamNetworking006
+SteamNetworkingSockets=SteamNetworkingSockets008
+SteamNetworkingUtils=SteamNetworkingUtils003
+SteamParentalSettings=STEAMPARENTALSETTINGS_INTERFACE_VERSION001
+SteamParties=SteamParties002
+SteamRemotePlay=STEAMREMOTEPLAY_INTERFACE_VERSION001
+SteamRemoteStorage=STEAMREMOTESTORAGE_INTERFACE_VERSION014
+SteamScreenshots=STEAMSCREENSHOTS_INTERFACE_VERSION003
+SteamTV=STEAMTV_INTERFACE_V001
+SteamUGC=STEAMUGC_INTERFACE_VERSION015
+SteamUser=SteamUser021
+SteamUserStats=STEAMUSERSTATS_INTERFACE_VERSION012
+SteamUtils=SteamUtils010
+SteamVideo=STEAMVIDEO_INTERFACE_V002
+###
+
+[DLC]
+###
+### Automatically unlock all DLCs
+###
+DLCUnlockall=0
+###
+### Identifiers for DLCs
+###
+#ID=Name
+{cdx_dlc_list}
+###
+
+[AchievementIcons]
+###
+### Bitmap Icons for Achievements
+###
+#halloween_8 Achieved=steam_settings\\img\\halloween_8.jpg
+#halloween_8 Unachieved=steam_settings\\img\\unachieved\\halloween_8.jpg
+{cdx_ach_list}
+###
+
+[Crack]
+00ec7837693245e3=b7d5bc716512b5d6
+
+'''
+
+
+def generate_cdx_ini(
+    base_out_dir : str,
+    appid: int,
+    dlc: list[tuple[int, str]],
+    achs: list[dict]) -> None:
+
+    cdx_ini_path = os.path.join(base_out_dir, "steam_emu.ini")
+    print(f"generating steam_emu.ini for CODEX emulator in: {cdx_ini_path}")
+
+    dlc_list = [f"{d[0]}={d[1]}" for d in dlc]
+    achs_list = []
+    for ach in achs:
+        icon = ach.get("icon", None)
+        if icon:
+            icon = f"steam_settings\\img\\{icon}"
+        else:
+            icon = 'steam_settings\\img\\steam_default_icon_unlocked.jpg'
+
+        icon_gray = ach.get("icon_gray", None)
+        if icon_gray:
+            icon_gray = f"steam_settings\\img\\{icon_gray}"
+        else:
+            icon_gray = 'steam_settings\\img\\steam_default_icon_locked.jpg'
+
+        icongray = ach.get("icongray", None)
+        if icongray:
+            icon_gray = f"steam_settings\\img\\{icongray}"
+
+        achs_list.append(f'{ach["name"]} Achieved={icon}') # unlocked
+        achs_list.append(f'{ach["name"]} Unachieved={icon_gray}') # locked
+
+    formatted_ini = __cdx_ini.format(
+        cdx_id = appid,
+        cdx_dlc_list = "\n".join(dlc_list),
+        cdx_ach_list = "\n".join(achs_list)
+    )
+    
+    with open(cdx_ini_path, "wt", encoding='utf-8') as f:
+        f.writelines(formatted_ini)
+

scripts/external_components/safe_name.py

@@ -0,0 +1,22 @@
+
+import re
+
+
+ALLOWED_CHARS = set([
+    '`', '~', '!', '@',
+    '#', '$', '%', '&',
+    '(', ')', '-', '_',
+    '=', '+', '[', '{',
+    ']', '}', ';', '\'',
+    ',', '.', ' ', '\t',
+    '®', '™',
+])
+
+def create_safe_name(app_name : str):
+    safe_name = ''.join(c for c in f'{app_name}' if c.isalnum() or c in ALLOWED_CHARS)\
+        .rstrip()\
+        .rstrip('.')\
+        .replace('\t', ' ')
+    safe_name = re.sub('\s\s+', ' ', safe_name)
+    return safe_name
+

scripts/generate_emu_config.py

@@ -1,11 +1,9 @@
-
+import pathlib
-USERNAME = ""
+import time
-PASSWORD = ""
-
-#steam ids with public profiles that own a lot of games
-TOP_OWNER_IDS = [76561198028121353, 76561198001237877, 76561198355625888, 76561198001678750, 76561198237402290, 76561197979911851, 76561198152618007, 76561197969050296, 76561198213148949, 76561198037867621, 76561198108581917]
-
 from stats_schema_achievement_gen import achievements_gen
+from external_components import (
+    ach_watcher_gen, cdx_gen, app_images, app_details, safe_name
+)
 from controller_config_generator import parse_controller_vdf
 from steam.client import SteamClient
 from steam.client.cdn import CDNClient
@@ -20,59 +18,7 @@ import urllib.request
 import urllib.error
 import threading
 import queue
-
+import shutil
-prompt_for_unavailable = True
-
-if len(sys.argv) < 2:
-    print("\nUsage: {} appid appid appid etc..\n\nExample: {} 480\n".format(sys.argv[0], sys.argv[0]))
-    exit(1)
-
-appids = []
-for id in sys.argv[1:]:
-    appids +=  [int(id)]
-
-client = SteamClient()
-if not os.path.exists("login_temp"):
-    os.makedirs("login_temp")
-client.set_credential_location("login_temp")
-
-if (len(USERNAME) == 0 or len(PASSWORD) == 0):
-    client.cli_login()
-else:
-    result = client.login(USERNAME, password=PASSWORD)
-    auth_code, two_factor_code = None, None
-    while result in (EResult.AccountLogonDenied, EResult.InvalidLoginAuthCode,
-                        EResult.AccountLoginDeniedNeedTwoFactor, EResult.TwoFactorCodeMismatch,
-                        EResult.TryAnotherCM, EResult.ServiceUnavailable,
-                        EResult.InvalidPassword,
-                        ):
-
-        if result == EResult.InvalidPassword:
-            print("invalid password, the password you set is wrong.")
-            exit(1)
-
-        elif result in (EResult.AccountLogonDenied, EResult.InvalidLoginAuthCode):
-            prompt = ("Enter email code: " if result == EResult.AccountLogonDenied else
-                        "Incorrect code. Enter email code: ")
-            auth_code, two_factor_code = input(prompt), None
-
-        elif result in (EResult.AccountLoginDeniedNeedTwoFactor, EResult.TwoFactorCodeMismatch):
-            prompt = ("Enter 2FA code: " if result == EResult.AccountLoginDeniedNeedTwoFactor else
-                        "Incorrect code. Enter 2FA code: ")
-            auth_code, two_factor_code = None, input(prompt)
-
-        elif result in (EResult.TryAnotherCM, EResult.ServiceUnavailable):
-            if prompt_for_unavailable and result == EResult.ServiceUnavailable:
-                while True:
-                    answer = input("Steam is down. Keep retrying? [y/n]: ").lower()
-                    if answer in 'yn': break
-
-                prompt_for_unavailable = False
-                if answer == 'n': break
-
-            client.reconnect(maxdelay=15)
-
-        result = client.login(USERNAME, PASSWORD, None, auth_code, two_factor_code)
 
 
 def get_stats_schema(client, game_id, owner_id):
@@ -85,16 +31,18 @@ def get_stats_schema(client, game_id, owner_id):
     client.send(message)
     return client.wait_msg(EMsg.ClientGetUserStatsResponse, timeout=5)
 
-def download_achievement_images(game_id, image_names, output_folder):
+def download_achievement_images(game_id : int, image_names : set[str], output_folder : str):
-    q = queue.Queue()
+    print(f"downloading achievements images inside '{output_folder }', images count = {len(image_names)}")
+    q : queue.Queue[str] = queue.Queue()
 
     def downloader_thread():
         while True:
             name = q.get()
-            succeeded = False
             if name is None:
                 q.task_done()
                 return
+            
+            succeeded = False
             for u in ["https://cdn.akamai.steamstatic.com/steamcommunity/public/images/apps/", "https://cdn.cloudflare.steamstatic.com/steamcommunity/public/images/apps/"]:
                 url = "{}{}/{}".format(u, game_id, name)
                 try:
@@ -110,9 +58,10 @@ def download_achievement_images(game_id, image_names, output_folder):
                     print("URLError downloading", url, e.code)
             if not succeeded:
                 print("error could not download", name)
+            
             q.task_done()
 
-    num_threads = 20
+    num_threads = 50
     for i in range(num_threads):
         threading.Thread(target=downloader_thread, daemon=True).start()
 
@@ -123,34 +72,319 @@ def download_achievement_images(game_id, image_names, output_folder):
     for i in range(num_threads):
         q.put(None)
     q.join()
+    print("finished downloading achievements images")
 
+#steam ids with public profiles that own a lot of games
+# https://steamladder.com/ladder/games/
+# in browser console:
+#const links = $x("/html/body/div[3]/table/tbody/tr/td[2]/a[@href]/@href");
+#console.clear();
+#for (let index = 0; index < links.length; index++) {
+#    const usr_link = links[index].textContent.split('/').filter(s => s);
+#    const usr_id = usr_link[usr_link.length - 1]
+#    console.log(usr_id)
+#}
+TOP_OWNER_IDS = set([
+    76561198213148949,
+    76561198108581917,
+    76561198028121353,
+    76561197979911851,
+    76561198355625888,
+    76561198237402290,
+    76561197969050296,
+    76561198152618007,
+    76561198001237877,
+    76561198037867621,
+    76561198001678750,
+    76561198217186687,
+    76561198094227663,
+    76561197993544755,
+    76561197963550511,
+    76561198095049646,
+    76561197973009892,
+    76561197969810632,
+    76561198388522904,
+    76561198864213876,
+    # 76561198017975643,
+    # 76561198044596404,
+    # 76561197976597747,
+    # 76561197962473290,
+    # 76561197976968076,
+    # 76561198235911884,
+    # 76561198313790296,
+    # 76561198407953371,
+    # 76561198063574735,
+    # 76561198122859224,
+    # 76561198154462478,
+    # 76561197996432822,
+    # 76561197979667190,
+    # 76561198139084236,
+    # 76561198842864763,
+    # 76561198096081579,
+    # 76561198019712127,
+    # 76561198033715344,
+    # 76561198121398682,
+    # 76561198027233260,
+    # 76561198104323854,
+    # 76561197995070100,
+    # 76561198001221571,
+    # 76561198005337430,
+    # 76561198085065107,
+    # 76561198027214426,
+    # 76561198062901118,
+    # 76561198008181611,
+    # 76561198124872187,
+    # 76561198048373585,
+    # 76561197974742349,
+    # 76561198040421250,
+    # 76561198017902347,
+    # 76561198010615256,
+    # 76561197970825215,
+    # 76561198077213101,
+    # 76561197971011821,
+    # 76561197992133229,
+    # 76561197963534359,
+    # 76561198077248235,
+    # 76561198152760885,
+    # 76561198256917957,
+    # 76561198326510209,
+    # 76561198019009765,
+    # 76561198047438206,
+    # 76561198128158703,
+    # 76561198037809069,
+    # 76561198121336040,
+    # 76561198102767019,
+    # 76561198063728345,
+    # 76561198082995144,
+    # 76561197981111953,
+    # 76561197995008105,
+    # 76561198109083829,
+    # 76561197968410781,
+    # 76561198808371265,
+    # 76561198025858988,
+    # 76561198252374474,
+    # 76561198382166453,
+    # 76561198396723427,
+    # 76561197992548975,
+    # 76561198134044398,
+    # 76561198029503957,
+    # 76561197990233857,
+    # 76561197971026489,
+    # 76561197965978376,
+    # 76561197976796589,
+    # 76561197994616562,
+    # 76561197984235967,
+    # 76561197992967892,
+    # 76561198097945516,
+    # 76561198251835488,
+    # 76561198281128349,
+    # 76561198044387084,
+    # 76561198015685843,
+    # 76561197993312863,
+    # 76561198020125851,
+    # 76561198006391846,
+    # 76561198158932704,
+    # 76561198039492467,
+    # 76561198035552258,
+    # 76561198031837797,
+    # 76561197982718230,
+    # 76561198025653291,
+    # 76561197972951657,
+    # 76561198269242105,
+    # 76561198004332929,
+    # 76561197972378106,
+    # 76561197962630138,
+    # 76561198192399786,
+    # 76561198119667710,
+    # 76561198120120943,
+    # 76561198015992850,
+    # 76561198096632451,
+    # 76561198008797636,
+    # 76561198118726910,
+    # 76561198018254158,
+    # 76561198061393233,
+    # 76561198086250077,
+    # 76561198025391492,
+    # 76561198050474710,
+    # 76561197997477460,
+    # 76561198105279930,
+    # 76561198026221141,
+    # 76561198443388781,
+    # 76561197981228012,
+    # 76561197986240493,
+    # 76561198003041763,
+    # 76561198056971296,
+    # 76561198072936438,
+    # 76561198264362271,
+    # 76561198101049562,
+    # 76561198831075066,
+    # 76561197991699268,
+    # 76561198042965266,
+    # 76561198019555404,
+    # 76561198111433283,
+    # 76561197984010356,
+    # 76561198427572372,
+    # 76561198071709714,
+    # 76561198034213886,
+    # 76561198846208086,
+    # 76561197991613008,
+    # 76561197978640923,
+    # 76561198009596142,
+    # 76561199173688191,
+    # 76561198294806446,
+    # 76561197992105918,
+    # 76561198155124847,
+    # 76561198032614383,
+    # 76561198051740093,
+    # 76561198051725954,
+    # 76561198048151962,
+    # 76561198172367910,
+    # 76561198043532513,
+    # 76561198029532782,
+    # 76561198106145311,
+    # 76561198020746864,
+    # 76561198122276418,
+    # 76561198844130640,
+    # 76561198890581618,
+    # 76561198021180815,
+    # 76561198046642155,
+    # 76561197985091630,
+    # 76561198119915053,
+    # 76561198318547224,
+    # 76561198426000196,
+    # 76561197988052802,
+    # 76561198008549198,
+    # 76561198054210948,
+    # 76561198028011423,
+    # 76561198026306582,
+    # 76561198079227501,
+    # 76561198070220549,
+    # 76561198034503074,
+    # 76561198172925593,
+    # 76561198286209051,
+    # 76561197998058239,
+    # 76561198057648189,
+    # 76561197982273259,
+    # 76561198093579202,
+    # 76561198035612474,
+    # 76561197970307937,
+    # 76561197996825541,
+    # 76561197981027062,
+    # 76561198019841907,
+    # 76561197970727958,
+    # 76561197967716198,
+    # 76561197970545939,
+    # 76561198315929726,
+    # 76561198093753361,
+    # 76561198413266831,
+    # 76561198045540632,
+    # 76561198015514779,
+    # 76561198004532679,
+    # 76561198080773680,
+    # 76561198079896896,
+    # 76561198005299723,
+    # 76561198337784749,
+    # 76561198150126284,
+    # 76561197988445370,
+    # 76561198258304011,
+    # 76561198321551799,
+    # 76561197973701057,
+    # 76561197973230221,
+    # 76561198002535276,
+    # 76561198100306249,
+    # 76561198116086535,
+    # 76561197970970678,
+    # 76561198085238363,
+    # 76561198007200913,
+    # 76561198025111129,
+    # 76561198068747739,
+    # 76561197970539274,
+    # 76561198148627568,
+    # 76561197970360549,
+    # 76561198098314980,
+    # 76561197972529138,
+    # 76561198007403855,
+    # 76561197977403803,
+    # 76561198124865933,
+    # 76561197981323238,
+    # 76561197960330700,
+    # 76561198217979953,
+    # 76561197960366517,
+    # 76561198044067612,
+    # 76561197967197052,
+    # 76561198027066612,
+    # 76561198072833066,
+    # 76561198033967307,
+    # 76561198104561325,
+    # 76561198272374716,
+    # 76561197970127197,
+    # 76561197970257188,
+    # 76561198026921217,
+    # 76561198027904347,
+    # 76561198062469228,
+    # 76561198026278913,
+    # 76561197970548935,
+    # 76561197966617426,
+    # 76561198356842617,
+    # 76561198034276722,
+    # 76561198355953202,
+    # 76561197986603983,
+    # 76561197967923946,
+    # 76561197961542845,
+    # 76561198121938079,
+    # 76561197992357639,
+    # 76561198002536379,
+    # 76561198017054389,
+    # 76561198031129658,
+    # 76561198020728639,
+])
 
-def generate_achievement_stats(client, game_id, output_directory, backup_directory):
+def generate_achievement_stats(client, game_id : int, output_directory, backup_directory) -> list[dict]:
-    achievement_images_dir = os.path.join(output_directory, "achievement_images")
+    steam_id_list = TOP_OWNER_IDS.copy()
-    images_to_download = []
+    steam_id_list.add(client.steam_id)
-    steam_id_list = TOP_OWNER_IDS + [client.steam_id]
+    stats_schema_found = None
-    for x in steam_id_list:
+    print(f"finding achievements stats...")
-        out = get_stats_schema(client, game_id, x)
+    for id in steam_id_list:
-        if out is not None:
+        #print(f"finding achievements stats using account ID {id}...")
-            if len(out.body.schema) > 0:
+        out = get_stats_schema(client, game_id, id)
-                with open(os.path.join(backup_directory, 'UserGameStatsSchema_{}.bin'.format(appid)), 'wb') as f:
+        if out is not None and len(out.body.schema) > 0:
-                    f.write(out.body.schema)
+            stats_schema_found = out
-                achievements, stats = achievements_gen.generate_stats_achievements(out.body.schema, output_directory)
+            #print(f"found achievement stats using account ID {id}")
-                for ach in achievements:
-                    if "icon" in ach:
-                        images_to_download.append(ach["icon"])
-                    if "icon_gray" in ach:
-                        images_to_download.append(ach["icon_gray"])
             break
-            else:
-                pass
-                # print("no schema", out)
 
-    if (len(images_to_download) > 0):
+    if stats_schema_found is None: # nothing found
+        print(f"[X] app id {game_id} has not achievements")
+        return []
+
+    achievement_images_dir = os.path.join(output_directory, "img")
+    images_to_download : set[str] = set()
+    
+    with open(os.path.join(backup_directory, f'UserGameStatsSchema_{game_id}.bin'), 'wb') as f:
+        f.write(stats_schema_found.body.schema)
+    (
+        achievements, stats,
+        copy_default_unlocked_img, copy_default_locked_img
+    ) = achievements_gen.generate_stats_achievements(stats_schema_found.body.schema, output_directory)
+    
+    for ach in achievements:
+        icon = f"{ach.get('icon', '')}".strip()
+        if icon:
+            images_to_download.add(icon)
+        icon_gray = f"{ach.get('icon_gray', '')}".strip()
+        if icon_gray:
+            images_to_download.add(icon_gray)
+
+    if images_to_download:
         if not os.path.exists(achievement_images_dir):
             os.makedirs(achievement_images_dir)
+        if copy_default_unlocked_img:
+            shutil.copy("steam_default_icon_unlocked.jpg", achievement_images_dir)
+        if copy_default_locked_img:
+            shutil.copy("steam_default_icon_locked.jpg", achievement_images_dir)
         download_achievement_images(game_id, images_to_download, achievement_images_dir)
 
+    return achievements
+
 def get_ugc_info(client, published_file_id):
     return client.send_um_and_wait('PublishedFile.GetDetails#1', {
             'publishedfileids': [published_file_id],
@@ -201,11 +435,11 @@ def get_inventory_info(client, game_id):
         })
 
 def generate_inventory(client, game_id):
-    inventory = get_inventory_info(client, appid)
+    inventory = get_inventory_info(client, game_id)
     if inventory.header.eresult != EResult.OK:
         return None
 
-    url = "https://api.steampowered.com/IGameInventory/GetItemDefArchive/v0001?appid={}&digest={}".format(game_id, inventory.body.digest)
+    url = f"https://api.steampowered.com/IGameInventory/GetItemDefArchive/v0001?appid={game_id}&digest={inventory.body.digest}"
     try:
         with urllib.request.urlopen(url) as response:
             return response.read()
@@ -217,52 +451,257 @@ def generate_inventory(client, game_id):
 
 def get_dlc(raw_infos):
     try:
-        try:
-            dlc_list = set(map(lambda a: int(a), raw_infos["extended"]["listofdlc"].split(",")))
-        except:
         dlc_list = set()
         depot_app_list = set()
+        all_depots = set()
+        try:
+            dlc_list = set(map(lambda a: int(f"{a}".strip()), raw_infos["extended"]["listofdlc"].split(",")))
+        except Exception:
+            dlc_list = set()
+        
         if "depots" in raw_infos:
-            depots = raw_infos["depots"]
+            depots : dict[str, object] = raw_infos["depots"]
             for dep in depots:
                 depot_info = depots[dep]
                 if "dlcappid" in depot_info:
                     dlc_list.add(int(depot_info["dlcappid"]))
                 if "depotfromapp" in depot_info:
                     depot_app_list.add(int(depot_info["depotfromapp"]))
-        return (dlc_list, depot_app_list)
+                if dep.isnumeric():
-    except:
+                    all_depots.add(int(dep))
+        
+        return (dlc_list, depot_app_list, all_depots)
+    except Exception:
         print("could not get dlc infos, are there any dlcs ?")
-        return (set(), set())
+        return (set(), set(), set())
 
-for appid in appids:
+EXTRA_FEATURES: list[tuple[str, str]] = [
-    backup_dir = os.path.join("backup","{}".format(appid))
+    ("disable_account_avatar.txt",                   "disable avatar functionality."),
-    out_dir = os.path.join("{}".format( "{}_output".format(appid)), "steam_settings")
+    ("disable_networking.txt",                       "disable all networking functionality."),
+    ("disable_overlay.txt",                          "disable the overlay."),
+    ("disable_overlay_achievement_notification.txt", "disable the achievement notifications."),
+    ("disable_overlay_friend_notification.txt",      "disable the friend invite and message notifications."),
+    ("disable_source_query.txt",                     "Do not send server details for the server browser. Only works for game servers."),
+]
 
+def disable_all_extra_features(emu_settings_dir : str) -> None:
+    for item in EXTRA_FEATURES:
+        with open(os.path.join(emu_settings_dir, item[0]), 'wt', encoding='utf-8') as f:
+            f.write(item[1])
+
+
+def help():
+    exe_name = os.path.basename(sys.argv[0])
+    print(f"\nUsage: {exe_name} [-shots] [-thumbs] [-vid] [-imgs] [-name] [-cdx] [-aw] [-clean] appid appid appid ... ")
+    print(f" Example: {exe_name} 421050 420 480")
+    print(f" Example: {exe_name} -shots -thumbs -vid -imgs -name -cdx -aw -clean 421050")
+    print("\nSwitches:")
+    print(" -shots:  download screenshots for each app if they're available")
+    print(" -thumbs: download screenshots thumbnails for each app if they're available")
+    print(" -vid:    download the first video available for each app: trailer, gameplay, announcement, etc...")
+    print(" -imgs:   download common images for each app: Steam generated background, icon, logo, etc...")
+    print(" -name:   save the output of each app in a folder with the same name as the app, unsafe characters are discarded")
+    print(" -cdx:    generate .ini file for CODEX Steam emu for each app")
+    print(" -aw:     generate schemas of all possible languages for Achievement Watcher")
+    print(" -clean:  delete any folder/file with the same name as the output before generating any data")
+    print("\nAll switches are optional except app id, at least 1 app id must be provided\n")
+
+
+def main():
+    USERNAME = ""
+    PASSWORD = ""
+
+    DOWNLOAD_SCREESHOTS = False
+    DOWNLOAD_THUMBNAILS = False
+    DOWNLOAD_VIDEOS = False
+    DOWNLOAD_COMMON_IMAGES = False
+    SAVE_APP_NAME = False
+    GENERATE_CODEX_INI = False
+    GENERATE_ACHIEVEMENT_WATCHER_SCHEMAS = False
+    CLEANUP_BEFORE_GENERATING = False
+
+    prompt_for_unavailable = True
+
+    if len(sys.argv) < 2:
+        help()
+        sys.exit(1)
+
+    appids : set[int] = set()
+    for appid in sys.argv[1:]:
+        if f'{appid}'.isnumeric():
+            appids.add(int(appid))
+        elif f'{appid}'.lower() == '-shots':
+            DOWNLOAD_SCREESHOTS = True
+        elif f'{appid}'.lower() == '-thumbs':
+            DOWNLOAD_THUMBNAILS = True
+        elif f'{appid}'.lower() == '-vid':
+            DOWNLOAD_VIDEOS = True
+        elif f'{appid}'.lower() == '-imgs':
+            DOWNLOAD_COMMON_IMAGES = True
+        elif f'{appid}'.lower() == '-name':
+            SAVE_APP_NAME = True
+        elif f'{appid}'.lower() == '-cdx':
+            GENERATE_CODEX_INI = True
+        elif f'{appid}'.lower() == '-aw':
+            GENERATE_ACHIEVEMENT_WATCHER_SCHEMAS = True
+        elif f'{appid}'.lower() == '-clean':
+            CLEANUP_BEFORE_GENERATING = True
+        else:
+            print(f'[X] invalid switch: {appid}')
+            help()
+            sys.exit(1)
+    
+    if not appids:
+        print(f'[X] no app id was provided')
+        help()
+        sys.exit(1)
+
+    client = SteamClient()
+    if not os.path.exists("login_temp"):
+        os.makedirs("login_temp")
+    client.set_credential_location("login_temp")
+
+    if os.path.isfile("my_login.txt"):
+        filedata = ['']
+        with open("my_login.txt", "r") as f:
+            filedata = f.readlines()
+        filedata = list(map(lambda s: s.strip().replace("\r", "").replace("\n", ""), filedata))
+        filedata = [l for l in filedata if l]
+        if len(filedata) == 2:
+            USERNAME = filedata[0]
+            PASSWORD = filedata[1]
+
+    if (len(USERNAME) == 0 or len(PASSWORD) == 0):
+        client.cli_login()
+    else:
+        result = client.login(USERNAME, password=PASSWORD)
+        auth_code, two_factor_code = None, None
+        while result in (EResult.AccountLogonDenied, EResult.InvalidLoginAuthCode,
+                            EResult.AccountLoginDeniedNeedTwoFactor, EResult.TwoFactorCodeMismatch,
+                            EResult.TryAnotherCM, EResult.ServiceUnavailable,
+                            EResult.InvalidPassword,
+                            ):
+
+            if result == EResult.InvalidPassword:
+                print("invalid password, the password you set is wrong.")
+                exit(1)
+
+            elif result in (EResult.AccountLogonDenied, EResult.InvalidLoginAuthCode):
+                prompt = ("Enter email code: " if result == EResult.AccountLogonDenied else
+                            "Incorrect code. Enter email code: ")
+                auth_code, two_factor_code = input(prompt), None
+
+            elif result in (EResult.AccountLoginDeniedNeedTwoFactor, EResult.TwoFactorCodeMismatch):
+                prompt = ("Enter 2FA code: " if result == EResult.AccountLoginDeniedNeedTwoFactor else
+                            "Incorrect code. Enter 2FA code: ")
+                auth_code, two_factor_code = None, input(prompt)
+
+            elif result in (EResult.TryAnotherCM, EResult.ServiceUnavailable):
+                if prompt_for_unavailable and result == EResult.ServiceUnavailable:
+                    while True:
+                        answer = input("Steam is down. Keep retrying? [y/n]: ").lower()
+                        if answer in 'yn': break
+
+                    prompt_for_unavailable = False
+                    if answer == 'n': break
+
+                client.reconnect(maxdelay=15)
+
+            result = client.login(USERNAME, PASSWORD, None, auth_code, two_factor_code)
+
+    for appid in appids:
+        print(f"********* generating info for app id {appid} *********")
+        raw = client.get_product_info(apps=[appid])
+        game_info : dict = raw["apps"][appid]
+
+        game_info_common : dict = game_info.get("common", {})
+        app_name = game_info_common.get("name", "")
+        app_name_on_disk = f"{appid}"
+        if app_name:
+            print(f"App name on store: '{app_name}'")
+            if SAVE_APP_NAME:
+                sanitized_name = safe_name.create_safe_name(app_name)
+                if sanitized_name:
+                    app_name_on_disk = f'{sanitized_name}-{appid}'
+        else:
+            app_name = f"Unknown_Steam_app_{appid}" # we need this for later use in the Achievement Watcher
+            print(f"[X] Couldn't find app name on store")
+
+        root_backup_dir = "backup"
+        backup_dir = os.path.join(root_backup_dir, f"{appid}")
         if not os.path.exists(backup_dir):
             os.makedirs(backup_dir)
 
-    if not os.path.exists(out_dir):
+        root_out_dir = "output"
-        os.makedirs(out_dir)
+        base_out_dir = os.path.join(root_out_dir, app_name_on_disk)
+        emu_settings_dir = os.path.join(base_out_dir, "steam_settings")
+        info_out_dir = os.path.join(base_out_dir, "info")
 
-    print("outputting config to", out_dir)
+        if CLEANUP_BEFORE_GENERATING:
+            print("cleaning output folder before generating any data")
+            base_dir_path = pathlib.Path(base_out_dir)
+            if base_dir_path.is_file():
+                base_dir_path.unlink()
+                time.sleep(0.05)
+            elif base_dir_path.is_dir():
+                shutil.rmtree(base_dir_path)
+                time.sleep(0.05)
             
-    raw = client.get_product_info(apps=[appid])
+            while base_dir_path.exists():
-    game_info = raw["apps"][appid]
+                time.sleep(0.05)
 
-    if "common" in game_info:
+        if not os.path.exists(emu_settings_dir):
-        game_info_common = game_info["common"]
+            os.makedirs(emu_settings_dir)
+
+        if not os.path.exists(info_out_dir):
+            os.makedirs(info_out_dir)
+
+        print(f"output dir: '{base_out_dir}'")
+
+        with open(os.path.join(info_out_dir, "product_info.json"), "wt", encoding='utf-8') as f:
+            json.dump(game_info, f, ensure_ascii=False, indent=2)
+        
+        app_details.download_app_details(
+            base_out_dir, info_out_dir,
+            appid,
+            DOWNLOAD_SCREESHOTS,
+            DOWNLOAD_THUMBNAILS,
+            DOWNLOAD_VIDEOS)
+
+        clienticon : str = None
+        icon : str = None
+        logo : str = None
+        logo_small : str = None
+        achievements : list[dict] = []
+        languages : list[str] = []
+        app_exe = ''
+        if game_info_common:
+            if "clienticon" in game_info_common:
+                clienticon = f"{game_info_common['clienticon']}"
+            
+            if "icon" in game_info_common:
+                icon = f"{game_info_common['icon']}"
+            
+            if "logo" in game_info_common:
+                logo = f"{game_info_common['logo']}"
+            
+            if "logo_small" in game_info_common:
+                logo_small = f"{game_info_common['logo_small']}"
+            
+            #print(f"generating achievement stats")
             #if "community_visible_stats" in game_info_common: #NOTE: checking this seems to skip stats on a few games so it's commented out
-        generate_achievement_stats(client, appid, out_dir, backup_dir)
+            achievements = generate_achievement_stats(client, appid, emu_settings_dir, backup_dir)
+
             if "supported_languages" in game_info_common:
-            with open(os.path.join(out_dir, "supported_languages.txt"), 'w') as f:
+                langs : dict[str, dict] = game_info_common["supported_languages"]
-                languages = game_info_common["supported_languages"]
+                languages = [lang for lang in langs if langs[lang].get("supported", "").lower() == "true"]
-                for l in languages:
-                    if "supported" in languages[l] and languages[l]["supported"] == "true":
-                        f.write("{}\n".format(l))
 
+        if languages:
+            with open(os.path.join(emu_settings_dir, "supported_languages.txt"), 'wt', encoding='utf-8') as f:
+                for lang in languages:
+                    f.write(f'{lang}\n')
         
-    with open(os.path.join(out_dir, "steam_appid.txt"), 'w') as f:
+        with open(os.path.join(emu_settings_dir, "steam_appid.txt"), 'w') as f:
             f.write(str(appid))
 
         if "depots" in game_info:
@@ -270,25 +709,35 @@ for appid in appids:
                 if "public" in game_info["depots"]["branches"]:
                     if "buildid" in game_info["depots"]["branches"]["public"]:
                         buildid = game_info["depots"]["branches"]["public"]["buildid"]
-                    with open(os.path.join(out_dir, "build_id.txt"), 'w') as f:
+                        with open(os.path.join(emu_settings_dir, "build_id.txt"), 'wt', encoding='utf-8') as f:
                             f.write(str(buildid))
 
-    dlc_config_list = []
+        dlc_config_list : list[tuple[int, str]] = []
-    dlc_list, depot_app_list = get_dlc(game_info)
+        dlc_list, depot_app_list, all_depots = get_dlc(game_info)
-    dlc_infos_backup = ""
+        dlc_raw = {}
-    if (len(dlc_list) > 0):
+        if dlc_list:
             dlc_raw = client.get_product_info(apps=dlc_list)["apps"]
             for dlc in dlc_raw:
+                dlc_name = ''
                 try:
-                dlc_config_list.append((dlc, dlc_raw[dlc]["common"]["name"]))
+                    dlc_name = f'{dlc_raw[dlc]["common"]["name"]}'
-            except:
+                except Exception:
-                dlc_config_list.append((dlc, None))
+                    pass
-        dlc_infos_backup = json.dumps(dlc_raw, indent=4)
                 
-    with open(os.path.join(out_dir, "DLC.txt"), 'w', encoding="utf-8") as f:
+                if not dlc_name:
+                    dlc_name = f"Unknown Steam app {dlc}"
+                
+                dlc_config_list.append((dlc, dlc_name))
+
+        if dlc_config_list:
+            with open(os.path.join(emu_settings_dir, "DLC.txt"), 'wt', encoding="utf-8") as f:
                 for x in dlc_config_list:
-            if (x[1] is not None):
+                    f.write(f"{x[0]}={x[1]}\n")
-                f.write("{}={}\n".format(x[0], x[1]))
+        
+        if all_depots:
+            with open(os.path.join(emu_settings_dir, "depots.txt"), 'wt', encoding="utf-8") as f:
+                for game_depot in all_depots:
+                    f.write(f"{game_depot}\n")
         
         config_generated = False
         if "config" in game_info:
@@ -306,7 +755,7 @@ for appid in appids:
                     out_vdf = download_published_file(client, int(id), os.path.join(backup_dir, controller_type + str(id)))
                     if out_vdf is not None and not config_generated:
                         if (controller_type in ["controller_xbox360", "controller_xboxone"] and (("default" in enabled_branches) or ("public" in enabled_branches))):
-                        parse_controller_vdf.generate_controller_config(out_vdf.decode('utf-8'), os.path.join(out_dir, "controller"))
+                            parse_controller_vdf.generate_controller_config(out_vdf.decode('utf-8'), os.path.join(emu_settings_dir, "controller"))
                             config_generated = True
             if "steamcontrollertouchconfigdetails" in game_info["config"]:
                 controller_details = game_info["config"]["steamcontrollertouchconfigdetails"]
@@ -320,6 +769,57 @@ for appid in appids:
                         enabled_branches = details["enabled_branches"]
                     print(id, controller_type)
                     out_vdf = download_published_file(client, int(id), os.path.join(backup_dir, controller_type + str(id)))
+            if "launch" in game_info["config"]:
+                launch_configs = game_info["config"]["launch"]
+                with open(os.path.join(info_out_dir, "launch_config.json"), "wt", encoding='utf-8') as f:
+                    json.dump(launch_configs, f, ensure_ascii=False, indent=2)
+                
+                first_app_exe : str = None
+                prefered_app_exe : str = None
+                unwanted_app_exes = ["launch", "start", "play", "try", "demo", "_vr",]
+                for cfg in launch_configs.values():
+                    if "executable" in cfg:
+                        app_exe = f'{cfg["executable"]}'
+                    
+                    if app_exe.lower().endswith(".exe"):
+                        app_exe = app_exe.replace("\\", "/").split('/')[-1]
+                        if first_app_exe is None:
+                            first_app_exe = app_exe
+                        if all(app_exe.lower().find(unwanted_exe) < 0 for unwanted_exe in unwanted_app_exes):
+                            prefered_app_exe = app_exe
+                            break
+                
+                if prefered_app_exe:
+                    app_exe = prefered_app_exe
+                elif first_app_exe:
+                    app_exe = first_app_exe
+
+        if GENERATE_ACHIEVEMENT_WATCHER_SCHEMAS:
+            ach_watcher_gen.generate_all_ach_watcher_schemas(
+                base_out_dir,
+                appid,
+                app_name,
+                app_exe,
+                achievements,
+                icon)
+        
+        if GENERATE_CODEX_INI:
+            cdx_gen.generate_cdx_ini(
+                base_out_dir,
+                appid,
+                dlc_config_list,
+                achievements)
+
+        if DOWNLOAD_COMMON_IMAGES:
+            app_images.download_app_images(
+                base_out_dir,
+                appid,
+                clienticon,
+                icon,
+                logo,
+                logo_small)
+
+        disable_all_extra_features(emu_settings_dir)
 
         inventory_data = generate_inventory(client, appid)
         if inventory_data is not None:
@@ -342,15 +842,25 @@ for appid in appids:
                 out_inventory[index] = x
                 default_items[index] = 1
 
-        out_json_inventory = json.dumps(out_inventory, indent=2)
+            with open(os.path.join(emu_settings_dir, "items.json"), "wt", encoding='utf-8') as f:
-        with open(os.path.join(out_dir, "items.json"), "w") as f:
+                json.dump(out_inventory, f, ensure_ascii=False, indent=2)
-            f.write(out_json_inventory)
+
-        out_json_inventory = json.dumps(default_items, indent=2)
+            with open(os.path.join(emu_settings_dir, "default_items.json"), "wt", encoding='utf-8') as f:
-        with open(os.path.join(out_dir, "default_items.json"), "w") as f:
+                json.dump(default_items, f, ensure_ascii=False, indent=2)
-            f.write(out_json_inventory)
+
+        with open(os.path.join(backup_dir, "product_info.json"), "wt", encoding='utf-8') as f:
+            json.dump(game_info, f, ensure_ascii=False, indent=2)
+        
+        with open(os.path.join(backup_dir, "dlc_product_info.json"), "wt", encoding='utf-8') as f:
+            json.dump(dlc_raw, f, ensure_ascii=False, indent=2)
+        
+        print(f"######### done for app id {appid} #########\n\n")
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as e:
+        print("Unexpected error:")
+        print(e)
+        sys.exit(1)
 
-    game_info_backup = json.dumps(game_info, indent=4)
-    with open(os.path.join(backup_dir, "product_info.json"), "w") as f:
-        f.write(game_info_backup)
-    with open(os.path.join(backup_dir, "dlc_product_info.json"), "w") as f:
-        f.write(dlc_infos_backup)

scripts/rebuild.bat

@@ -0,0 +1,37 @@
+@echo off
+
+setlocal
+pushd "%~dp0"
+
+set "venv=.env"
+set "out_dir=bin"
+set "build_temp_dir=build_tmp"
+set "tool_name=generate_emu_config"
+set "icon_file=icon\Froyoshark-Enkel-Steam.ico"
+set "main_file=generate_emu_config.py"
+
+if exist "%out_dir%" (
+    rmdir /s /q "%out_dir%"
+)
+
+if exist "%build_temp_dir%" (
+    rmdir /s /q "%build_temp_dir%"
+)
+
+del /f /q "*.spec"
+
+call "%venv%\Scripts\activate.bat"
+
+pyinstaller "%main_file%" --distpath "%out_dir%" -y --clean --onedir --name "%tool_name%" --noupx --console -i "%icon_file%" --workpath "%build_temp_dir%" --collect-submodules "steam"
+
+copy /y "steam_default_icon_locked.jpg" "%out_dir%\%tool_name%\"
+copy /y "steam_default_icon_unlocked.jpg" "%out_dir%\%tool_name%\"
+
+echo:
+echo =============
+echo Built inside : "%out_dir%\"
+
+:script_end
+popd
+endlocal
+

scripts/recreate_venv.bat

@@ -0,0 +1,15 @@
+@echo off
+
+cd /d "%~dp0"
+
+set "venv=.env"
+set "reqs_file=requirements.txt"
+
+if exist "%venv%" (
+    rmdir /s /q "%venv%"
+)
+
+python -m venv "%venv%"
+timeout /t 1 /nobreak
+call "%venv%\Scripts\activate.bat"
+pip install -r "%reqs_file%"

scripts/requirements.txt

@@ -0,0 +1,2 @@
+steam[client]
+pyinstaller

scripts/stats_schema_achievement_gen/achievements_gen.py

@@ -2,6 +2,7 @@ import vdf
 import sys
 import os
 import json
+import copy
 
 
 STAT_TYPE_INT = '1'
@@ -9,11 +10,13 @@ STAT_TYPE_FLOAT = '2'
 STAT_TYPE_AVGRATE = '3'
 STAT_TYPE_BITS = '4'
 
-def generate_stats_achievements(schema, config_directory):
+def generate_stats_achievements(
+        schema, config_directory
+    ) -> tuple[list[dict], list[dict], bool, bool]:
     schema = vdf.binary_loads(schema)
     # print(schema)
-    achievements_out = []
+    achievements_out : list[dict] = []
-    stats_out = []
+    stats_out : list[dict] = []
 
     for appid in schema:
         sch = schema[appid]
@@ -25,15 +28,19 @@ def generate_stats_achievements(schema, config_directory):
                 for ach_num in achs:
                     out = {}
                     ach = achs[ach_num]
-                    out["hidden"] = '0'
+                    out['hidden'] = 0
                     for x in ach['display']:
                         value = ach['display'][x]
                         if x == 'name':
                             x = 'displayName'
-                        if x == 'desc':
+                        elif x == 'desc':
                             x = 'description'
-                        if x == 'Hidden':
+                        elif x == 'Hidden' or f'{x}'.lower() == 'hidden':
                             x = 'hidden'
+                            try:
+                                value = int(value)
+                            except Exception as e:
+                                pass
                         out[x] = value
                     out['name'] = ach['name']
                     if 'progress' in ach:
@@ -57,20 +64,39 @@ def generate_stats_achievements(schema, config_directory):
                 stats_out += [out]
             #print(stat_info[s])
 
+    copy_default_unlocked_img = False
+    copy_default_locked_img = False
+    output_ach = copy.deepcopy(achievements_out)
+    for out_ach in output_ach:
+        icon = out_ach.get("icon", None)
+        if icon:
+            out_ach["icon"] = f"img/{icon}"
+        else:
+            out_ach["icon"] = r'img/steam_default_icon_unlocked.jpg'
+            copy_default_unlocked_img = True
 
+        icon_gray = out_ach.get("icon_gray", None)
+        if icon_gray:
+            out_ach["icon_gray"] = f"img/{icon_gray}"
+        else:
+            out_ach["icon_gray"] = r'img/steam_default_icon_locked.jpg'
+            copy_default_locked_img = True
 
-    output_ach = json.dumps(achievements_out, indent=4)
+        icongray = out_ach.get("icongray", None)
-    output_stats = ""
+        if icongray:
+            out_ach["icongray"] = f"{icongray}"
+
+    output_stats : list[str] = []
     for s in stats_out:
         default_num = 0
-        if (s['type'] == 'int'):
+        if f"{s['type']}".lower() == 'int':
             try:
                 default_num = int(s['default'])
             except ValueError:
                 default_num = int(float(s['default']))
         else:
             default_num = float(s['default'])
-        output_stats += "{}={}={}\n".format(s['name'], s['type'], default_num)
+        output_stats.append(f"{s['name']}={s['type']}={default_num}\n")
 
     # print(output_ach)
     # print(output_stats)
@@ -78,13 +104,16 @@ def generate_stats_achievements(schema, config_directory):
     if not os.path.exists(config_directory):
         os.makedirs(config_directory)
 
-    with open(os.path.join(config_directory, "achievements.json"), 'w') as f:
+    if output_ach:
-        f.write(output_ach)
+        with open(os.path.join(config_directory, "achievements.json"), 'wt', encoding='utf-8') as f:
+            json.dump(output_ach, f, indent=2)
 
-    with open(os.path.join(config_directory, "stats.txt"), 'w', encoding='utf-8') as f:
+    if output_stats:
-        f.write(output_stats)
+        with open(os.path.join(config_directory, "stats.txt"), 'wt', encoding='utf-8') as f:
+            f.writelines(output_stats)
 
-    return (achievements_out, stats_out)
+    return (achievements_out, stats_out,
+            copy_default_unlocked_img, copy_default_locked_img)
 
 if __name__ == '__main__':
     if len(sys.argv) < 2: