Browse Source

Initial commit

master
James Hofstra 10 months ago
commit
b7d08b57f3
3 changed files with 394 additions and 0 deletions
  1. +21
    -0
      LICENSE.md
  2. +72
    -0
      README.md
  3. +301
    -0
      steam_stat_utils.py

+ 21
- 0
LICENSE.md View File

@@ -0,0 +1,21 @@
MIT License

Copyright (c) 2019 James Hofstra

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

+ 72
- 0
README.md View File

@@ -0,0 +1,72 @@
# Steam Stat Utils

Command line utility for parsing detailed Steam stat information.

## Background

`steam_stat_utils.py` fetches data primarily from two sources: the Steam client's local filesystem cache and the Steam Web API. With this information you can view global stats as well as signed-in users' stats.

You can also dump raw data for local/offline playtesting, such as with [Goldberg Emulator](https://mr_goldberg.gitlab.io/goldberg_emulator/ "Goldberg Emulator"). Steam's copy of data can change over time and as your games receive major updates, so this process should be repeated regularly.

## Usage

Usage details are in development and likely to change with future commits.

`steam_stat_utils.py path/to/steam_dir appid [userid]`

## Requirements

- Python 3.5 or higher
- [vdf](https://github.com/ValvePython/vdf "vdf")

## Tested Configurations

- Python 3.5.3 on Debian GNU/Linux (WSL)

## Planned Additions and Improvements

1. Reading/writing [average-rate](https://partner.steamgames.com/doc/features/achievements#AVGRATE "average-rate") type stats (currently treated the same as floating-point type)
2. Adding and improving usage parameters
3. Writing default values to `stats.txt` file format used by Goldberg Emulator

## Limitations

- Writing raw data back to Steam's local cache is not supported. If you want to manipulate your own player stats, look up Steam Achievement Manager.
- The current version of Goldberg Emulator doesn't support achievements or global stats yet. To enable global stats support, I am using a custom build with the following changes (as of August 2019):

1. In `dll/local_storage.h` add the following preprocessor directive next to the other defines near the top of the file:

```
#define GLOBALSTATS_STORAGE_FOLDER "globalstats"
```

2. In `dll/steam_user_stats.h` replace the two GetGlobalStat placeholder functions with these implementations:

```
// Gets the lifetime totals for an aggregated stat
bool GetGlobalStat( const char *pchStatName, int64 *pData )
{
PRINT_DEBUG("GetGlobalStat %s\n", pchStatName);
if (!pchStatName || !pData) return false;
std::lock_guard<std::recursive_mutex> lock(global_mutex);

int read_data = local_storage->get_data(GLOBALSTATS_STORAGE_FOLDER, pchStatName, (char* )pData, sizeof(*pData));
if (read_data == sizeof(int64))
return true;

return false;
}

bool GetGlobalStat( const char *pchStatName, double *pData )
{
PRINT_DEBUG("GetGlobalStat %s\n", pchStatName);
if (!pchStatName || !pData) return false;
std::lock_guard<std::recursive_mutex> lock(global_mutex);

int read_data = local_storage->get_data(GLOBALSTATS_STORAGE_FOLDER, pchStatName, (char* )pData, sizeof(*pData));
if (read_data == sizeof(int64))
return true;

return false;
}
```

+ 301
- 0
steam_stat_utils.py View File

@@ -0,0 +1,301 @@
#!/usr/bin/env python3
"""
Command line utility for parsing detailed Steam stat information.
"""
import json
import pathlib
import struct
import time
import urllib.error
import urllib.parse
import urllib.request
import vdf

# Output folder names
FOLDER_ACH = 'achievements'
FOLDER_GLOBAL = 'globalstats'
FOLDER_STAT = 'stats'

# Stats data types
STAT_TYPE_INT = 1
STAT_TYPE_FLOAT = 2
STAT_TYPE_AVGRATE = 3
STAT_TYPE_BITS = 4

# Helpers to pack and unpack binary data
STRUCT_INT32 = struct.Struct('<i')
STRUCT_INT64 = struct.Struct('<q')
STRUCT_FLOAT = struct.Struct('<f')
STRUCT_DOUBLE = struct.Struct('<d')



# Functions to retrieve data from the Steam client's local filesystem cache

def _stat_default(stat):
mappers = {
STAT_TYPE_INT: int,
STAT_TYPE_FLOAT: float,
STAT_TYPE_AVGRATE: float,
STAT_TYPE_BITS: int
}
return mappers[stat['type_int']](stat.get('Default', '0'))

def get_local_stats(base_dir, appid, userid=None):
"""
Parses Steam's appcache for local stat values.
In addition to fetching the stats for the given userid, global stats
will be set to their default value.
Parameters
----------
base_dir : str
Location of Steam install directory
appid: str
The game to fetch stats for
userid : str
The user to fetch stats for (if None, default stat values are used)
Returns
-------
dict
The data contained in the response
"""
local_stats = {FOLDER_ACH: {}, FOLDER_GLOBAL: {}, FOLDER_STAT: {}}
path_name = pathlib.Path(base_dir, 'appcache', 'stats')
# Read in schema file
schema_file = 'UserGameStatsSchema_{}.bin'.format(appid)
with path_name.joinpath(schema_file).open('rb') as fh:
schema = vdf.binary_loads(fh.read())
# Read in cache file
if userid is not None:
cache_file = 'UserGameStats_{}_{}.bin'.format(userid, appid)
with path_name.joinpath(cache_file).open('rb') as fh:
cache = vdf.binary_loads(fh.read())
else:
cache = {'cache': {'crc': 0, 'PendingChanges': 0}}
for stat in schema[appid]['stats'].values():
# Get default and cached stat values
default = _stat_default(stat)
if stat['id'] in cache['cache']:
data = cache['cache'][stat['id']]['data']
# data is read as int32 - convert to float if applicable
if isinstance(default, float):
data = STRUCT_FLOAT.unpack(STRUCT_INT32.pack(data))[0]
else:
data = default
if stat['type_int'] == STAT_TYPE_BITS:
# Store name:achieved of each bit in FOLDER_ACH
for bit in stat['bits'].values():
local_stats[FOLDER_ACH][bit['name']] = (data >> bit['bit']) & 1
else:
# Store name:data in FOLDER_STAT
# If aggregated, also store name:default in FOLDER_GLOBAL
local_stats[FOLDER_STAT][stat['name']] = data
if stat.get('aggregated', '0') == '1':
local_stats[FOLDER_GLOBAL][stat['name']] = default
return local_stats

# Functions to retrieve data from the Steam Web API

def webapi_get(interface, method, version, params):
"""
Fetches data using Steam's public Web API.
Methods that use HTTP POST, or that are only available to
publishers, cannot be called through this function.
Parameters
----------
interface : str
The interface associated with the method call
method: str
The name of the method to call
version : int
The version number associated with the method call
params : dict
The arguments (if any) to pass to the method
Returns
-------
bytes
The data contained in the response
"""
base_url = 'https://api.steampowered.com/'
base_url += '{}/{}/v{}/'.format(interface, method, version)
if len(params) > 0:
base_url += '?{}'.format(urllib.parse.urlencode(params))
with urllib.request.urlopen(base_url) as buf:
return buf.read()

def get_global_stats(appid, names):
"""
Fetches the cumulative totals for global stats.
Parameters
----------
appid : str
The game to fetch stats for
names: list
The names of the stats to fetch totals for
Returns
-------
dict
The data contained in the response
"""
interface = 'ISteamUserStats'
method = 'GetGlobalStatsForGame'
version = 1
global_stats = {}
# Build parameters
count = len(names)
params = {'appid': appid, 'count': count}
for i, name in enumerate(names):
params['name[{}]'.format(i)] = name
try:
resp = webapi_get(interface, method, version, params)
except urllib.error.HTTPError:
pass
else:
resp = json.loads(resp.decode('utf-8')).get('response', {})
if resp.get('result', 0) == 1:
for stat_name, stat_data in resp['globalstats'].items():
global_stats[stat_name] = stat_data['total']
return global_stats

def get_global_stats_history(appid, names, startdate, enddate):
"""
Fetches the daily totals for global stats in the time range given.
The GetGlobalStatsForGame method is limited to retrieving 60 days
of daily totals per call. In the future, this implementation may
be changed to split time ranges of more than 60 days into multiple
API calls.
Parameters
----------
appid : str
The game to fetch stats for
names: list
The names of the stats to fetch totals for
startdate: int
Unix timestamp for the start of the time range
enddate: int
Unix timestamp for the end of the time range
Returns
-------
dict
The data contained in the response
"""
interface = 'ISteamUserStats'
method = 'GetGlobalStatsForGame'
version = 1
global_stats = {}
time.sleep(1)
# Build parameters
count = len(names)
params = {'appid': appid, 'count': count, 'startdate': startdate, 'enddate': enddate}
for i, name in enumerate(names):
params['name[{}]'.format(i)] = name
try:
resp = webapi_get(interface, method, version, params)
except urllib.error.HTTPError:
pass
else:
resp = json.loads(resp.decode('utf-8')).get('response', {})
if resp.get('result', 0) == 1:
for stat_name, stat_data in resp['globalstats'].items():
global_stats[stat_name] = stat_data.get('history')
return global_stats

# Functions to dump raw data

def dump_raw_stats(stats, target_dir=None):
"""
Writes raw stats to files in target_dir.
Parameters
----------
stats : dict
The stats to dump
target_dir : str, optional
Directory to store stats in (defaults to working
directory if None).
"""
if target_dir is None:
target_dir = pathlib.Path.cwd()
# Dump stats to target_dir
for folder in stats:
fp = pathlib.Path(target_dir, folder)
fp.mkdir(parents=True, exist_ok=True)
for statname in stats[folder].keys():
data = stats[folder][statname]
# 4 bytes for GetStat, 8 bytes for GetGlobalStat
if folder == FOLDER_GLOBAL:
mapper = STRUCT_DOUBLE if isinstance(data, float) else STRUCT_INT64
else:
mapper = STRUCT_FLOAT if isinstance(data, float) else STRUCT_INT32
with fp.joinpath(statname).open('wb') as fh:
fh.write(mapper.pack(data))

# Program interface

def main():
import sys
if len(sys.argv) > 2:
# Test fetching local player stats
if len(sys.argv) > 3:
stats = get_local_stats(sys.argv[1], sys.argv[2], sys.argv[3])
else:
stats = get_local_stats(sys.argv[1], sys.argv[2])
# Test fetching global stats Web API use
global_stat_names = [n for n in stats[FOLDER_GLOBAL].keys()]
global_stats = get_global_stats(sys.argv[2], global_stat_names)
for stat_name, stat_data in global_stats.items():
if stat_data is None:
continue
prev_data = stats[FOLDER_GLOBAL][stat_name]
if isinstance(prev_data, float):
stats[FOLDER_GLOBAL][stat_name] = float(stat_data)
else:
stats[FOLDER_GLOBAL][stat_name] = int(stat_data)
# Test dumping raw stats
dump_raw_stats(stats)
else:
print('No input specified.')

def main_history():
import sys
if len(sys.argv) > 4:
stats = get_local_stats(sys.argv[1], sys.argv[2])
global_stat_names = [n for n in stats[FOLDER_GLOBAL].keys()]
# Make multiple calls to get daily totals
global_stat_hist = {}
def divide_names(l, n):
for i in range(0, len(l), n):
yield l[i:i+n]
for partial_names in divide_names(global_stat_names, 4):
global_stats = get_global_stats_history(sys.argv[2], partial_names, sys.argv[3], sys.argv[4])
global_stat_hist.update(global_stats)
print(global_stat_hist)
else:
print('No input specified.')

if __name__ == '__main__':
main()

Loading…
Cancel
Save