commit
b7d08b57f3
3 changed files with 394 additions and 0 deletions
-
21LICENSE.md
-
72README.md
-
301steam_stat_utils.py
@ -0,0 +1,21 @@ |
|||
MIT License |
|||
|
|||
Copyright (c) 2019 James Hofstra |
|||
|
|||
Permission is hereby granted, free of charge, to any person obtaining a copy |
|||
of this software and associated documentation files (the "Software"), to deal |
|||
in the Software without restriction, including without limitation the rights |
|||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
|||
copies of the Software, and to permit persons to whom the Software is |
|||
furnished to do so, subject to the following conditions: |
|||
|
|||
The above copyright notice and this permission notice shall be included in all |
|||
copies or substantial portions of the Software. |
|||
|
|||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
|||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
|||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
|||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
|||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
|||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
|||
SOFTWARE. |
|||
@ -0,0 +1,72 @@ |
|||
# Steam Stat Utils |
|||
|
|||
Command line utility for parsing detailed Steam stat information. |
|||
|
|||
## Background |
|||
|
|||
`steam_stat_utils.py` fetches data primarily from two sources: the Steam client's local filesystem cache and the Steam Web API. With this information you can view global stats as well as signed-in users' stats. |
|||
|
|||
You can also dump raw data for local/offline playtesting, such as with [Goldberg Emulator](https://mr_goldberg.gitlab.io/goldberg_emulator/ "Goldberg Emulator"). Steam's copy of data can change over time and as your games receive major updates, so this process should be repeated regularly. |
|||
|
|||
## Usage |
|||
|
|||
Usage details are in development and likely to change with future commits. |
|||
|
|||
`steam_stat_utils.py path/to/steam_dir appid [userid]` |
|||
|
|||
## Requirements |
|||
|
|||
- Python 3.5 or higher |
|||
- [vdf](https://github.com/ValvePython/vdf "vdf") |
|||
|
|||
## Tested Configurations |
|||
|
|||
- Python 3.5.3 on Debian GNU/Linux (WSL) |
|||
|
|||
## Planned Additions and Improvements |
|||
|
|||
1. Reading/writing [average-rate](https://partner.steamgames.com/doc/features/achievements#AVGRATE "average-rate") type stats (currently treated the same as floating-point type) |
|||
2. Adding and improving usage parameters |
|||
3. Writing default values to `stats.txt` file format used by Goldberg Emulator |
|||
|
|||
## Limitations |
|||
|
|||
- Writing raw data back to Steam's local cache is not supported. If you want to manipulate your own player stats, look up Steam Achievement Manager. |
|||
- The current version of Goldberg Emulator doesn't support achievements or global stats yet. To enable global stats support, I am using a custom build with the following changes (as of August 2019): |
|||
|
|||
1. In `dll/local_storage.h` add the following preprocessor directive next to the other defines near the top of the file: |
|||
|
|||
``` |
|||
#define GLOBALSTATS_STORAGE_FOLDER "globalstats" |
|||
``` |
|||
|
|||
2. In `dll/steam_user_stats.h` replace the two GetGlobalStat placeholder functions with these implementations: |
|||
|
|||
``` |
|||
// Gets the lifetime totals for an aggregated stat |
|||
bool GetGlobalStat( const char *pchStatName, int64 *pData ) |
|||
{ |
|||
PRINT_DEBUG("GetGlobalStat %s\n", pchStatName); |
|||
if (!pchStatName || !pData) return false; |
|||
std::lock_guard<std::recursive_mutex> lock(global_mutex); |
|||
|
|||
int read_data = local_storage->get_data(GLOBALSTATS_STORAGE_FOLDER, pchStatName, (char* )pData, sizeof(*pData)); |
|||
if (read_data == sizeof(int64)) |
|||
return true; |
|||
|
|||
return false; |
|||
} |
|||
|
|||
bool GetGlobalStat( const char *pchStatName, double *pData ) |
|||
{ |
|||
PRINT_DEBUG("GetGlobalStat %s\n", pchStatName); |
|||
if (!pchStatName || !pData) return false; |
|||
std::lock_guard<std::recursive_mutex> lock(global_mutex); |
|||
|
|||
int read_data = local_storage->get_data(GLOBALSTATS_STORAGE_FOLDER, pchStatName, (char* )pData, sizeof(*pData)); |
|||
if (read_data == sizeof(int64)) |
|||
return true; |
|||
|
|||
return false; |
|||
} |
|||
``` |
|||
@ -0,0 +1,301 @@ |
|||
#!/usr/bin/env python3 |
|||
""" |
|||
Command line utility for parsing detailed Steam stat information. |
|||
""" |
|||
import json |
|||
import pathlib |
|||
import struct |
|||
import time |
|||
import urllib.error |
|||
import urllib.parse |
|||
import urllib.request |
|||
import vdf |
|||
|
|||
# Output folder names |
|||
FOLDER_ACH = 'achievements' |
|||
FOLDER_GLOBAL = 'globalstats' |
|||
FOLDER_STAT = 'stats' |
|||
|
|||
# Stats data types |
|||
STAT_TYPE_INT = 1 |
|||
STAT_TYPE_FLOAT = 2 |
|||
STAT_TYPE_AVGRATE = 3 |
|||
STAT_TYPE_BITS = 4 |
|||
|
|||
# Helpers to pack and unpack binary data |
|||
STRUCT_INT32 = struct.Struct('<i') |
|||
STRUCT_INT64 = struct.Struct('<q') |
|||
STRUCT_FLOAT = struct.Struct('<f') |
|||
STRUCT_DOUBLE = struct.Struct('<d') |
|||
|
|||
|
|||
|
|||
# Functions to retrieve data from the Steam client's local filesystem cache |
|||
|
|||
def _stat_default(stat): |
|||
mappers = { |
|||
STAT_TYPE_INT: int, |
|||
STAT_TYPE_FLOAT: float, |
|||
STAT_TYPE_AVGRATE: float, |
|||
STAT_TYPE_BITS: int |
|||
} |
|||
return mappers[stat['type_int']](stat.get('Default', '0')) |
|||
|
|||
def get_local_stats(base_dir, appid, userid=None): |
|||
""" |
|||
Parses Steam's appcache for local stat values. |
|||
|
|||
In addition to fetching the stats for the given userid, global stats |
|||
will be set to their default value. |
|||
|
|||
Parameters |
|||
---------- |
|||
base_dir : str |
|||
Location of Steam install directory |
|||
appid: str |
|||
The game to fetch stats for |
|||
userid : str |
|||
The user to fetch stats for (if None, default stat values are used) |
|||
|
|||
Returns |
|||
------- |
|||
dict |
|||
The data contained in the response |
|||
""" |
|||
|
|||
local_stats = {FOLDER_ACH: {}, FOLDER_GLOBAL: {}, FOLDER_STAT: {}} |
|||
path_name = pathlib.Path(base_dir, 'appcache', 'stats') |
|||
|
|||
# Read in schema file |
|||
schema_file = 'UserGameStatsSchema_{}.bin'.format(appid) |
|||
with path_name.joinpath(schema_file).open('rb') as fh: |
|||
schema = vdf.binary_loads(fh.read()) |
|||
|
|||
# Read in cache file |
|||
if userid is not None: |
|||
cache_file = 'UserGameStats_{}_{}.bin'.format(userid, appid) |
|||
with path_name.joinpath(cache_file).open('rb') as fh: |
|||
cache = vdf.binary_loads(fh.read()) |
|||
else: |
|||
cache = {'cache': {'crc': 0, 'PendingChanges': 0}} |
|||
|
|||
for stat in schema[appid]['stats'].values(): |
|||
# Get default and cached stat values |
|||
default = _stat_default(stat) |
|||
if stat['id'] in cache['cache']: |
|||
data = cache['cache'][stat['id']]['data'] |
|||
# data is read as int32 - convert to float if applicable |
|||
if isinstance(default, float): |
|||
data = STRUCT_FLOAT.unpack(STRUCT_INT32.pack(data))[0] |
|||
else: |
|||
data = default |
|||
|
|||
if stat['type_int'] == STAT_TYPE_BITS: |
|||
# Store name:achieved of each bit in FOLDER_ACH |
|||
for bit in stat['bits'].values(): |
|||
local_stats[FOLDER_ACH][bit['name']] = (data >> bit['bit']) & 1 |
|||
else: |
|||
# Store name:data in FOLDER_STAT |
|||
# If aggregated, also store name:default in FOLDER_GLOBAL |
|||
local_stats[FOLDER_STAT][stat['name']] = data |
|||
if stat.get('aggregated', '0') == '1': |
|||
local_stats[FOLDER_GLOBAL][stat['name']] = default |
|||
|
|||
return local_stats |
|||
|
|||
# Functions to retrieve data from the Steam Web API |
|||
|
|||
def webapi_get(interface, method, version, params): |
|||
""" |
|||
Fetches data using Steam's public Web API. |
|||
|
|||
Methods that use HTTP POST, or that are only available to |
|||
publishers, cannot be called through this function. |
|||
|
|||
Parameters |
|||
---------- |
|||
interface : str |
|||
The interface associated with the method call |
|||
method: str |
|||
The name of the method to call |
|||
version : int |
|||
The version number associated with the method call |
|||
params : dict |
|||
The arguments (if any) to pass to the method |
|||
|
|||
Returns |
|||
------- |
|||
bytes |
|||
The data contained in the response |
|||
""" |
|||
base_url = 'https://api.steampowered.com/' |
|||
base_url += '{}/{}/v{}/'.format(interface, method, version) |
|||
if len(params) > 0: |
|||
base_url += '?{}'.format(urllib.parse.urlencode(params)) |
|||
with urllib.request.urlopen(base_url) as buf: |
|||
return buf.read() |
|||
|
|||
def get_global_stats(appid, names): |
|||
""" |
|||
Fetches the cumulative totals for global stats. |
|||
|
|||
Parameters |
|||
---------- |
|||
appid : str |
|||
The game to fetch stats for |
|||
names: list |
|||
The names of the stats to fetch totals for |
|||
|
|||
Returns |
|||
------- |
|||
dict |
|||
The data contained in the response |
|||
""" |
|||
interface = 'ISteamUserStats' |
|||
method = 'GetGlobalStatsForGame' |
|||
version = 1 |
|||
global_stats = {} |
|||
|
|||
# Build parameters |
|||
count = len(names) |
|||
params = {'appid': appid, 'count': count} |
|||
for i, name in enumerate(names): |
|||
params['name[{}]'.format(i)] = name |
|||
|
|||
try: |
|||
resp = webapi_get(interface, method, version, params) |
|||
except urllib.error.HTTPError: |
|||
pass |
|||
else: |
|||
resp = json.loads(resp.decode('utf-8')).get('response', {}) |
|||
if resp.get('result', 0) == 1: |
|||
for stat_name, stat_data in resp['globalstats'].items(): |
|||
global_stats[stat_name] = stat_data['total'] |
|||
|
|||
return global_stats |
|||
|
|||
def get_global_stats_history(appid, names, startdate, enddate): |
|||
""" |
|||
Fetches the daily totals for global stats in the time range given. |
|||
|
|||
The GetGlobalStatsForGame method is limited to retrieving 60 days |
|||
of daily totals per call. In the future, this implementation may |
|||
be changed to split time ranges of more than 60 days into multiple |
|||
API calls. |
|||
|
|||
Parameters |
|||
---------- |
|||
appid : str |
|||
The game to fetch stats for |
|||
names: list |
|||
The names of the stats to fetch totals for |
|||
startdate: int |
|||
Unix timestamp for the start of the time range |
|||
enddate: int |
|||
Unix timestamp for the end of the time range |
|||
|
|||
Returns |
|||
------- |
|||
dict |
|||
The data contained in the response |
|||
""" |
|||
interface = 'ISteamUserStats' |
|||
method = 'GetGlobalStatsForGame' |
|||
version = 1 |
|||
global_stats = {} |
|||
time.sleep(1) |
|||
|
|||
# Build parameters |
|||
count = len(names) |
|||
params = {'appid': appid, 'count': count, 'startdate': startdate, 'enddate': enddate} |
|||
for i, name in enumerate(names): |
|||
params['name[{}]'.format(i)] = name |
|||
|
|||
try: |
|||
resp = webapi_get(interface, method, version, params) |
|||
except urllib.error.HTTPError: |
|||
pass |
|||
else: |
|||
resp = json.loads(resp.decode('utf-8')).get('response', {}) |
|||
if resp.get('result', 0) == 1: |
|||
for stat_name, stat_data in resp['globalstats'].items(): |
|||
global_stats[stat_name] = stat_data.get('history') |
|||
|
|||
return global_stats |
|||
|
|||
# Functions to dump raw data |
|||
|
|||
def dump_raw_stats(stats, target_dir=None): |
|||
""" |
|||
Writes raw stats to files in target_dir. |
|||
|
|||
Parameters |
|||
---------- |
|||
stats : dict |
|||
The stats to dump |
|||
target_dir : str, optional |
|||
Directory to store stats in (defaults to working |
|||
directory if None). |
|||
""" |
|||
if target_dir is None: |
|||
target_dir = pathlib.Path.cwd() |
|||
|
|||
# Dump stats to target_dir |
|||
for folder in stats: |
|||
fp = pathlib.Path(target_dir, folder) |
|||
fp.mkdir(parents=True, exist_ok=True) |
|||
for statname in stats[folder].keys(): |
|||
data = stats[folder][statname] |
|||
# 4 bytes for GetStat, 8 bytes for GetGlobalStat |
|||
if folder == FOLDER_GLOBAL: |
|||
mapper = STRUCT_DOUBLE if isinstance(data, float) else STRUCT_INT64 |
|||
else: |
|||
mapper = STRUCT_FLOAT if isinstance(data, float) else STRUCT_INT32 |
|||
with fp.joinpath(statname).open('wb') as fh: |
|||
fh.write(mapper.pack(data)) |
|||
|
|||
# Program interface |
|||
|
|||
def main(): |
|||
import sys |
|||
if len(sys.argv) > 2: |
|||
# Test fetching local player stats |
|||
if len(sys.argv) > 3: |
|||
stats = get_local_stats(sys.argv[1], sys.argv[2], sys.argv[3]) |
|||
else: |
|||
stats = get_local_stats(sys.argv[1], sys.argv[2]) |
|||
# Test fetching global stats Web API use |
|||
global_stat_names = [n for n in stats[FOLDER_GLOBAL].keys()] |
|||
global_stats = get_global_stats(sys.argv[2], global_stat_names) |
|||
for stat_name, stat_data in global_stats.items(): |
|||
if stat_data is None: |
|||
continue |
|||
prev_data = stats[FOLDER_GLOBAL][stat_name] |
|||
if isinstance(prev_data, float): |
|||
stats[FOLDER_GLOBAL][stat_name] = float(stat_data) |
|||
else: |
|||
stats[FOLDER_GLOBAL][stat_name] = int(stat_data) |
|||
# Test dumping raw stats |
|||
dump_raw_stats(stats) |
|||
else: |
|||
print('No input specified.') |
|||
|
|||
def main_history(): |
|||
import sys |
|||
if len(sys.argv) > 4: |
|||
stats = get_local_stats(sys.argv[1], sys.argv[2]) |
|||
global_stat_names = [n for n in stats[FOLDER_GLOBAL].keys()] |
|||
# Make multiple calls to get daily totals |
|||
global_stat_hist = {} |
|||
def divide_names(l, n): |
|||
for i in range(0, len(l), n): |
|||
yield l[i:i+n] |
|||
for partial_names in divide_names(global_stat_names, 4): |
|||
global_stats = get_global_stats_history(sys.argv[2], partial_names, sys.argv[3], sys.argv[4]) |
|||
global_stat_hist.update(global_stats) |
|||
print(global_stat_hist) |
|||
else: |
|||
print('No input specified.') |
|||
|
|||
if __name__ == '__main__': |
|||
main() |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue