Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add date as an argument in the replay.user mode in the hlt client #426

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions tools/hlt_client/hlt_client/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,9 @@ def _parse_arguments():
help='Number of replays to fetch')
replay_user_parser.add_argument('-d', '--destination', dest='destination', action='store', type=str, required=True,
help="In which folder to store all resulting replay files.")
replay_user_parser.add_argument('-t', '--date', action='store', type=str, dest='date', default=None,
help="Fetch replay files matching the specified date. To fetch a day's files user"
"the YYYYMMDD format.")
# .Modes.Replay.Modes.Date
replay_regex_parser = replay_subparser.add_parser(REPLAY_MODE_DATE, help='Retrieve replays based on regex')
replay_regex_parser.add_argument('-t', '--date', action='store', type=str, dest='date', required=True,
Expand Down
21 changes: 16 additions & 5 deletions tools/hlt_client/hlt_client/download_game.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import os
import zstd
import re
import datetime

import requests
import multiprocessing
Expand Down Expand Up @@ -125,32 +126,40 @@ def __init__(self, destination, date, all_bots=False):

class UserGameDownloader(GameDownloader):
_USER_BOT_URI = 'https://api.halite.io/v1/api/user/{}/match?limit={}&offset={}'
_DATE_BOT_URI = 'https://api.halite.io/v1/api/user/{}/match?limit={}&offset={}&filter=time_played,>=,{}'
_FETCH_THRESHOLD = 250
_BUCKETS = []

def __init__(self, destination, user_id, limit):
def __init__(self, destination, user_id, limit, date):
"""
Download games for a user
:param destination: Where to download
:param user_id: Which user's replays to fetch
:param limit: How many replays to fetch (max)
:param date: Which date to download
"""
self.destination = destination
self.objects = self._parse_user_metadata(self._fetch_metadata(user_id, limit))
self.objects = self._parse_user_metadata(self._fetch_metadata(user_id, limit, date))

def _fetch_metadata(self, user_id, limit):
def _fetch_metadata(self, user_id, limit, date):
"""
Retrieves paginated game metadata from the halite servers for a specified user up to limit items
:param user_id: The id of the user to fetch
:param limit: The maximum number of items to fetch
:param date: Which date to download
:return: The full metadata of items
"""
print('Fetching Metadata')
current = 0
result_set = []
while current <= limit:
current_limit = self._FETCH_THRESHOLD if ((limit - current) >= self._FETCH_THRESHOLD) else (limit - current)
result_set += requests.get(self._USER_BOT_URI.format(user_id, current_limit, current)).json()
if date is None:
result_set += requests.get(self._USER_BOT_URI.format(user_id, current_limit, current)).json()
else:
requested_date = datetime.datetime.strptime(date,'%Y%m%d').strftime('%Y-%m-%d')
result_set += [ replay for replay in requests.get(self._DATE_BOT_URI.format(user_id, current_limit, current, requested_date)).json() \
if datetime.datetime.strptime(replay["time_played"],'%a, %d %b %Y %H:%M:%S GMT').strftime('%Y-%m-%d') == requested_date ]
current += self._FETCH_THRESHOLD
print('Finished metadata fetch. Found {} game files.'.format(len(result_set)))
return result_set
Expand Down Expand Up @@ -197,5 +206,7 @@ def download(mode, destination, date, all_bots, default_user_id, user_id, limit)
elif mode == client.REPLAY_MODE_USER:
if not (default_user_id or user_id):
raise ValueError("Cannot run default mode without authenticating .Please run `client.py --auth` first.")
UserGameDownloader(destination, default_user_id if not user_id else user_id, limit).get_objects()
if date != None and not _valid_date(date):
raise ValueError("Date must match format YYYYMMDD")
UserGameDownloader(destination, default_user_id if not user_id else user_id, limit, date).get_objects()
print('Finished writing files to desired location')