Created
April 20, 2022 23:53
-
-
Save xtex404/3c4c91b09c21b064a61471f55fb2fd6d to your computer and use it in GitHub Desktop.
Revisions
-
xtex404 created this gist
Apr 20, 2022 .There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,702 @@ #!venv/Scripts/python # # super janky quickly thrown together python script that'll create playlists from beatsaber favorites. it'll also # create playlists from songs i've passed before and got fullcombos on. # # it also creates text files of the bsr request keys for the songs, so I can use them as source lists for my streamer.bot # that fills my request queue if i need some suggestions. whatever. # # this code is terrible. it's not optimized. you shouldn't use it. one day i'll rewrite this to not suck, but i'm lazy # today and I just wanna play. i was so lazy i didn't feel like trying to figure out how to parse the protobuf song cache # that already exists, so I cheat and grab the JSON version. this is gross. i'm so sorry. # # seriously. don't run this unless you understand what it's doing -- and you change the filenames/etc, because it uses my # settings and not yours. # import base64 import datetime import json import os import sys import zipfile from pathlib import Path import pytz import requests from dateutil.parser import parse as parsedate # from http.client import HTTPConnection # py3 # # log = logging.getLogger('urllib3') # log.setLevel(logging.DEBUG) # # # logging from urllib3 to console # ch = logging.StreamHandler() # ch.setLevel(logging.DEBUG) # log.addHandler(ch) # # # print statements from `http.client.HTTPConnection` to console/stdout # HTTPConnection.debuglevel = 1 utc = pytz.utc mytz = pytz.timezone('America/New_York') MIN_AGE_SEC = 86400 MAX_AGE_SEC = 246400 BS_DIR = Path('D:/SteamLibrary/steamapps/common/Beat Saber') BS_FAVORITES_PLAYLIST_FILE = BS_DIR / 'Playlists/xedlock_favorites.bplist' BS_PASSED_PLAYLIST_FILE = BS_DIR / 'Playlists/xedlock_passed.bplist' BS_FULLCOMBO_PLAYLIST_FILE = BS_DIR / 'Playlists/xedlock_fullcombo.bplist' PLAYLIST_PNG_IMAGE = Path('C:/Users/xedlock/OneDrive/AppData/streaming/assets/logos/xedlock/rainbow/xx256.png') BS_SONGS_CACHE_FILE = BS_DIR / 'UserData/SongCache/SongDetails_Full.json' BS_SONGS_CACHE_FILE_SMALL = BS_DIR / 'UserData/SongCache/SongDetails.json' BS_SONGS_KEY_FILE = BS_DIR / 'UserData/SongCache/BSR2Hash.json' BS_SONGS_TITLE_FILE = BS_DIR / 'UserData/SongCache/BSR2Titles.json' DOWNLOAD_PATH = Path('C:/Users/xedlock/AppData/LocalLow/Temp') BS_SONGS_DOWNLOAD_FILE = DOWNLOAD_PATH / '004SongDetails.zip' BS_SONGS_DOWNLOAD_ETAG = DOWNLOAD_PATH / '004SongDetails.zip.etag' BS_DATA_URL = 'https://github.com/andruzzzhka/BeatSaberScrappedData/raw/master/combinedScrappedData.zip' def refresh_data_cache(): force_refresh = False if BS_SONGS_CACHE_FILE.exists(): modified_time = utc.localize(datetime.datetime.fromtimestamp(BS_SONGS_CACHE_FILE.stat().st_mtime)) else: modified_time = utc.localize(datetime.datetime.fromtimestamp(165888000)) force_refresh = True modified_time_http = modified_time.strftime('%a, %d %b %Y %H:%M:%S GMT') time_now = utc.localize(datetime.datetime.now()) elapsed = round(time_now.timestamp() - modified_time.timestamp()) if not force_refresh and elapsed < MIN_AGE_SEC: print(f'"{BS_SONGS_CACHE_FILE}" was modified {elapsed} seconds ago (less than {MIN_AGE_SEC}).') return True if elapsed < MAX_AGE_SEC: if BS_SONGS_DOWNLOAD_ETAG.exists(): with open(BS_SONGS_DOWNLOAD_ETAG, 'r') as f: my_etag = f.readline().strip() else: my_etag = None else: my_etag = None force_refresh = True # make the request headers = dict() if modified_time_http is not None: headers['If-Modified-Since'] = modified_time_http if my_etag is not None: headers['If-None-Match'] = my_etag req = requests.head(url=BS_DATA_URL, headers=headers, allow_redirects=True) their_etag = None if req.status_code == 200: their_etag = req.headers.get('Etag', None) if their_etag: # there's surely a better way to do this? if their_etag.startswith('W/"'): their_etag = their_etag.replace('W/"', '') their_etag = their_etag.replace('"', '') elif req.status_code == 304 and not force_refresh: print(f"URL returned status code 304 - not modified.") return True if my_etag == their_etag and not force_refresh: print(f"ETag identical, file appears to have not been modified?") return True # okay. let's get it then? print(f"Retrieving song data from remote cache...") req = requests.get(url=BS_DATA_URL, headers=headers, allow_redirects=True) if req.status_code == 200: with open(BS_SONGS_DOWNLOAD_FILE, 'wb') as f: f.write(req.content) their_date = req.headers.get('Date', None) if their_date: their_modified_date = parsedate(their_date) print(f'Modified Date: {their_modified_date}') their_modified_date_ts = their_modified_date.timestamp() # os.utime(BS_SONGS_CACHE_FILE, (their_modified_date_ts,their_modified_date_ts)) else: their_modified_date_ts = None their_etag = req.headers.get('Etag', None) if their_etag: # there's surely a better way to do this? if their_etag.startswith('W/"'): their_etag = their_etag.replace('W/"', '') their_etag = their_etag.replace('"', '') with open(BS_SONGS_DOWNLOAD_ETAG, 'w') as f: f.write(their_etag.strip()) else: BS_SONGS_DOWNLOAD_ETAG.unlink() # now let's unzip the file print("Unzipping data file...") raw_data = None with zipfile.ZipFile(BS_SONGS_DOWNLOAD_FILE, mode='r') as z: file_list = z.namelist() if len(file_list) != 1: print("unexpected files in zipfile. aborting.") sys.exit(1) with z.open(file_list[0]) as zz: raw_data = zz.read() print("Parsing source cache file...") json_data = json.loads(raw_data) bsr_data = dict() bsr_key_lookup = dict() bsr_title_lookup = dict() out_dict = dict() for R in json_data: bhash = R['Hash'].upper() bkey = R['Key'].lower() bsr_key_lookup[bkey] = bhash del R['Hash'] bsr_data[bhash] = R bsr_title_lookup[bkey] = R['SongName'] with open(BS_SONGS_CACHE_FILE, 'w') as of: json.dump(bsr_data, of) with open(BS_SONGS_KEY_FILE, 'w') as of: json.dump(bsr_key_lookup, of, sort_keys=True) with open(BS_SONGS_TITLE_FILE, 'w') as of: json.dump(bsr_title_lookup, of, sort_keys=True) # gross, but i'm lazy bsr_data_small = dict() for k, d in bsr_data.items(): del d['Diffs'] del d['Chars'] bsr_data_small[k] = d with open(BS_SONGS_CACHE_FILE_SMALL, 'w') as of: json.dump(bsr_data_small, of, sort_keys=True) os.utime(BS_SONGS_CACHE_FILE, (their_modified_date_ts, their_modified_date_ts)) os.utime(BS_SONGS_KEY_FILE, (their_modified_date_ts, their_modified_date_ts)) os.utime(BS_SONGS_CACHE_FILE_SMALL, (their_modified_date_ts, their_modified_date_ts)) os.utime(BS_SONGS_TITLE_FILE, (their_modified_date_ts, their_modified_date_ts)) return True elif req.status_code == 304: print(f"Delayed response to unmodified file, but OK.") return True else: return False return True if refresh_data_cache() is False: sys.exit(0) BS_PLAYERDATA = Path('C:/Users/xedlock/AppData/LocalLow/Hyperbolic Magnetism/Beat Saber/PlayerData.dat') print(f"Reading Beat Saber favorites..") with open(BS_PLAYERDATA, 'r') as f: player_data = json.load(f) favorites = player_data['localPlayers'][0]['favoritesLevelIds'] played_songs = player_data['localPlayers'][0]['levelsStatsData'] # # favorites # print(f"Found {len(favorites)} favorites songs in PlayerData.dat") pdata = dict() pdata['playlistTitle'] = 'XFavorites' pdata['playlistAuthor'] = 'xedlock' pdata['playlistDescription'] = 'Playlist generated from Beat Saber favorites' pdata['customData'] = dict() pdata['customData']['AllowDuplicates'] = False song_list = list() with open(BS_SONGS_CACHE_FILE_SMALL, 'r') as f: song_data = json.load(f) # # can't figure out what time format this is supposed to use -- every file is different different. # # 2022-04-18T19:31:44.083174-04:00 # now_time_string = datetime.datetime.now().astimezone().isoformat() # # 2022-04-18T23:31:33Z # now_time_string = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") # # 2022-04-18T23:31:57.339634+00:00 now_time_string = utc.localize(datetime.datetime.utcnow()).isoformat() bsr_keys = list() print(f"Creating playlist ...") for s in favorites: this_entry: dict = dict() if s.startswith('custom_level_'): the_hash = s.replace('custom_level_', '').upper() the_song = song_data.get(the_hash, None) if the_song: custom_data: dict = dict() this_entry['songName'] = the_song['SongName'] if the_song.get('SongSubName', None): this_entry['songSubName'] = the_song['SongSubName'] if the_song.get('SongAuthorName', None): this_entry['songAuthorName'] = the_song.get('SongAuthorName', None) this_entry['levelAuthorName'] = the_song.get('LevelAuthorName', None) this_entry['key'] = the_song['Key'] bsr_keys.append(the_song['Key'].lower()) this_entry['hash'] = the_hash this_entry['level_id'] = s # if the_song.get('Bpm', None): # this_entry['bpm'] = the_song.get('Bpm') # if the_song.get('Downvotes', None): # this_entry['downvotes'] = the_song.get('Downvotes') # if the_song.get('Upvotes', None): # this_entry['upvotes'] = the_song.get('Upvotes') custom_data['name'] = the_song['SongName'] if the_song.get('Uploader', None): custom_data['uploader'] = the_song.get('Uploader') this_entry['customData'] = custom_data if the_song.get('Uploaded', None): this_entry['uploaded'] = the_song.get('Uploaded', None) this_entry['dateAdded'] = now_time_string song_list.append(this_entry) print(f" * {this_entry['songName']}") else: print(f"!! Cannot find song level: {s}") else: this_entry['songName'] = s this_entry['levelAuthorName'] = '' this_entry['level_id'] = s song_list.append(this_entry) sorted_list = sorted(song_list, key=lambda i: (i['songName'].lower())) pdata['songs'] = sorted_list print(f"Adding cover image to playlist ({PLAYLIST_PNG_IMAGE})") if PLAYLIST_PNG_IMAGE.is_file(): with open(PLAYLIST_PNG_IMAGE, 'rb') as img_file: base64_data = base64.b64encode(img_file.read()).decode('utf-8') if len(base64_data) > 64: pdata['image'] = f"data:image/png;base64,{base64_data}" print(f"Writing new playlist: {BS_FAVORITES_PLAYLIST_FILE}") with open(BS_FAVORITES_PLAYLIST_FILE, 'w') as of: json.dump(pdata, of, indent=4) key_list_file = Path(str(BS_FAVORITES_PLAYLIST_FILE).replace('.bplist', '.bsrkeys.txt')) print(f"Writing new playlist key_list: {key_list_file}") bsr_keys.sort() with open(key_list_file, 'w') as of: for k in bsr_keys: of.write(f"{k}\n") print(f"Wrote {len(sorted_list)} songs to {BS_FAVORITES_PLAYLIST_FILE}") print("done.") # ###################################################################################################################### # all played # print(f"Found {len(played_songs)} played songs in PlayerData.dat") pdata = dict() pdata['playlistTitle'] = 'XPassed' pdata['playlistAuthor'] = 'xedlock' pdata['playlistDescription'] = 'Playlist generated from Beat Saber PlayerData played' pdata['customData'] = dict() pdata['customData']['AllowDuplicates'] = False song_list = list() # with open(BS_SONGS_CACHE_FILE_SMALL, 'r') as f: # song_data = json.load(f) # # can't figure out what time format this is supposed to use -- every file is different different. # # 2022-04-18T19:31:44.083174-04:00 # now_time_string = datetime.datetime.now().astimezone().isoformat() # # 2022-04-18T23:31:33Z # now_time_string = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") # # 2022-04-18T23:31:57.339634+00:00 now_time_string = utc.localize(datetime.datetime.utcnow()).isoformat() playlist = dict() bsr_keys = list() print(f"Creating playlist ...") for song_entry in played_songs: this_entry: dict = dict() high_score = song_entry.get('highScore', None) valid_score = song_entry.get('validScore', None) level_id = song_entry.get('levelId', None) # skip this if it wasn't finished if high_score is None or valid_score is None or level_id is None: continue if not isinstance(high_score, int): continue if high_score < 50000: continue if not valid_score: continue characteristic = song_entry.get('beatmapCharacteristicName', 'Standard') difficulty = song_entry.get('difficulty', 0) if difficulty == 5: difficulty_name = 'Expert+' elif difficulty == 4: difficulty_name = 'Expert' elif difficulty == 3: difficulty_name = 'Hard' elif difficulty == 2: difficulty_name = 'Normal' elif difficulty == 1: difficulty_name = 'Easy' else: difficulty = 0 difficulty_name = None if level_id in playlist and difficulty > 0 and difficulty_name is not None: # this already exists, so we're just adding a difficulty playlist_entry = playlist[level_id] difficulty_found = False difficulties = playlist_entry.get('difficulties', list()) if difficulties and isinstance(difficulties, list): for diffs in playlist_entry['difficulties']: diffname = diffs.get('name', None) if not diffname: continue if diffname.lower() == difficulty_name.lower(): difficulty_found = True if not difficulty_found: new_difficulty = dict() new_difficulty['characteristic'] = characteristic new_difficulty['name'] = difficulty_name difficulties.append(new_difficulty) playlist[level_id]['difficulties'] = difficulties continue if level_id.startswith('custom_level_'): the_hash = level_id.replace('custom_level_', '').upper() the_song = song_data.get(the_hash, None) if the_song: custom_data: dict = dict() difficulties = list() this_entry['songName'] = the_song['SongName'] if the_song.get('SongSubName', None): this_entry['songSubName'] = the_song['SongSubName'] if the_song.get('SongAuthorName', None): this_entry['songAuthorName'] = the_song.get('SongAuthorName', None) this_entry['levelAuthorName'] = the_song.get('LevelAuthorName', None) if difficulty_name is not None: difficulty = dict() difficulty['characteristic'] = characteristic difficulty['name'] = difficulty_name difficulties.append(difficulty) this_entry['difficulties'] = difficulties this_entry['key'] = the_song['Key'] bsr_keys.append(the_song['Key'].lower()) this_entry['hash'] = the_hash this_entry['level_id'] = s # if the_song.get('Bpm', None): # this_entry['bpm'] = the_song.get('Bpm') # if the_song.get('Downvotes', None): # this_entry['downvotes'] = the_song.get('Downvotes') # if the_song.get('Upvotes', None): # this_entry['upvotes'] = the_song.get('Upvotes') custom_data['name'] = the_song['SongName'] if the_song.get('Uploader', None): custom_data['uploader'] = the_song.get('Uploader') if the_song.get('Uploaded', None): this_entry['uploaded'] = the_song.get('Uploaded', None) this_entry['customData'] = custom_data this_entry['dateAdded'] = now_time_string playlist[level_id] = this_entry print(f" * {this_entry['songName']}") else: print(f"!! Cannot find song level: {s}") else: this_entry['songName'] = s this_entry['levelAuthorName'] = '' this_entry['level_id'] = s custom_data = dict() difficulties = list() if difficulty_name is not None: difficulty = dict() difficulty['characteristic'] = characteristic difficulty['name'] = difficulty_name difficulties.append(difficulty) this_entry['difficulties'] = difficulties this_entry['customData'] = custom_data playlist[level_id] = this_entry # convert my playlist dict into a list song_list = list() for key, data in playlist.items(): # if 'difficulties' in data: # if len(data['difficulties']) == 1: # if data['difficulties'][0]['name'] == 'Easy': # continue song_list.append(data) sorted_list = sorted(song_list, key=lambda i: (i['songName'].lower())) pdata['songs'] = sorted_list print(f"Adding cover image to playlist ({PLAYLIST_PNG_IMAGE})") if PLAYLIST_PNG_IMAGE.is_file(): with open(PLAYLIST_PNG_IMAGE, 'rb') as img_file: base64_data = base64.b64encode(img_file.read()).decode('utf-8') if len(base64_data) > 64: pdata['image'] = f"data:image/png;base64,{base64_data}" print(f"Writing new playlist: {BS_PASSED_PLAYLIST_FILE}") with open(BS_PASSED_PLAYLIST_FILE, 'w') as of: json.dump(pdata, of, indent=4) key_list_file = Path(str(BS_PASSED_PLAYLIST_FILE).replace('.bplist', '.bsrkeys.txt')) print(f"Writing new playlist key_list: {key_list_file}") bsr_keys.sort() with open(key_list_file, 'w') as of: for k in bsr_keys: of.write(f"{k}\n") print(f"Wrote {len(sorted_list)} songs to {BS_PASSED_PLAYLIST_FILE}") print("done.") # ###################################################################################################################### # full combo (yes, this code is a duplicate, yes, it's gross. # print(f"Found {len(played_songs)} played songs in PlayerData.dat") pdata = dict() pdata['playlistTitle'] = 'XFullCombo' pdata['playlistAuthor'] = 'xedlock' pdata['playlistDescription'] = 'Playlist generated from Beat Saber PlayerData played' pdata['customData'] = dict() pdata['customData']['AllowDuplicates'] = False song_list = list() # with open(BS_SONGS_CACHE_FILE_SMALL, 'r') as f: # song_data = json.load(f) # # can't figure out what time format this is supposed to use -- every file is different different. # # 2022-04-18T19:31:44.083174-04:00 # now_time_string = datetime.datetime.now().astimezone().isoformat() # # 2022-04-18T23:31:33Z # now_time_string = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") # # 2022-04-18T23:31:57.339634+00:00 now_time_string = utc.localize(datetime.datetime.utcnow()).isoformat() playlist = dict() bsr_keys = list() print(f"Creating playlist ...") for song_entry in played_songs: this_entry: dict = dict() high_score = song_entry.get('highScore', None) valid_score = song_entry.get('validScore', None) level_id = song_entry.get('levelId', None) full_combo = song_entry.get('fullCombo', False) # skip this if it wasn't finished if high_score is None or valid_score is None or level_id is None: continue if not isinstance(high_score, int): continue if high_score < 50000: continue if not full_combo: continue if not valid_score: continue characteristic = song_entry.get('beatmapCharacteristicName', 'Standard') difficulty = song_entry.get('difficulty', 0) if difficulty == 5: difficulty_name = 'Expert+' elif difficulty == 4: difficulty_name = 'Expert' elif difficulty == 3: difficulty_name = 'Hard' elif difficulty == 2: difficulty_name = 'Normal' elif difficulty == 1: difficulty_name = 'Easy' else: difficulty = 0 difficulty_name = None if level_id in playlist and difficulty > 0 and difficulty_name is not None: # this already exists, so we're just adding a difficulty playlist_entry = playlist[level_id] difficulty_found = False difficulties = playlist_entry.get('difficulties', list()) if difficulties and isinstance(difficulties, list): for diffs in playlist_entry['difficulties']: diffname = diffs.get('name', None) if not diffname: continue if diffname.lower() == difficulty_name.lower(): difficulty_found = True if not difficulty_found: new_difficulty = dict() new_difficulty['characteristic'] = characteristic new_difficulty['name'] = difficulty_name difficulties.append(new_difficulty) playlist[level_id]['difficulties'] = difficulties continue if level_id.startswith('custom_level_'): the_hash = level_id.replace('custom_level_', '').upper() the_song = song_data.get(the_hash, None) if the_song: custom_data: dict = dict() difficulties = list() this_entry['songName'] = the_song['SongName'] if the_song.get('SongSubName', None): this_entry['songSubName'] = the_song['SongSubName'] if the_song.get('SongAuthorName', None): this_entry['songAuthorName'] = the_song.get('SongAuthorName', None) this_entry['levelAuthorName'] = the_song.get('LevelAuthorName', None) if difficulty_name is not None: difficulty = dict() difficulty['characteristic'] = characteristic difficulty['name'] = difficulty_name difficulties.append(difficulty) this_entry['difficulties'] = difficulties this_entry['key'] = the_song['Key'] bsr_keys.append(the_song['Key'].lower()) this_entry['hash'] = the_hash this_entry['level_id'] = s # if the_song.get('Bpm', None): # this_entry['bpm'] = the_song.get('Bpm') # if the_song.get('Downvotes', None): # this_entry['downvotes'] = the_song.get('Downvotes') # if the_song.get('Upvotes', None): # this_entry['upvotes'] = the_song.get('Upvotes') custom_data['name'] = the_song['SongName'] if the_song.get('Uploader', None): custom_data['uploader'] = the_song.get('Uploader') if the_song.get('Uploaded', None): this_entry['uploaded'] = the_song.get('Uploaded', None) this_entry['customData'] = custom_data this_entry['dateAdded'] = now_time_string playlist[level_id] = this_entry print(f" * {this_entry['songName']}") else: print(f"!! Cannot find song level: {s}") else: this_entry['songName'] = s this_entry['levelAuthorName'] = '' this_entry['level_id'] = s custom_data = dict() difficulties = list() if difficulty_name is not None: difficulty = dict() difficulty['characteristic'] = characteristic difficulty['name'] = difficulty_name difficulties.append(difficulty) this_entry['difficulties'] = difficulties this_entry['customData'] = custom_data playlist[level_id] = this_entry # convert my playlist dict into a list song_list = list() for key, data in playlist.items(): # if 'difficulties' in data: # if len(data['difficulties']) == 1: # if data['difficulties'][0]['name'] == 'Easy': # continue song_list.append(data) sorted_list = sorted(song_list, key=lambda i: (i['songName'].lower())) pdata['songs'] = sorted_list print(f"Adding cover image to playlist ({PLAYLIST_PNG_IMAGE})") if PLAYLIST_PNG_IMAGE.is_file(): with open(PLAYLIST_PNG_IMAGE, 'rb') as img_file: base64_data = base64.b64encode(img_file.read()).decode('utf-8') if len(base64_data) > 64: pdata['image'] = f"data:image/png;base64,{base64_data}" print(f"Writing new playlist: {BS_FULLCOMBO_PLAYLIST_FILE}") with open(BS_FULLCOMBO_PLAYLIST_FILE, 'w') as of: json.dump(pdata, of, indent=4) key_list_file = Path(str(BS_FULLCOMBO_PLAYLIST_FILE).replace('.bplist', '.bsrkeys.txt')) print(f"Writing new playlist key_list: {key_list_file}") bsr_keys.sort() with open(key_list_file, 'w') as of: for k in bsr_keys: of.write(f"{k}\n") print(f"Wrote {len(sorted_list)} songs to {BS_FULLCOMBO_PLAYLIST_FILE}") print("done.")