diff --git a/mgz/cli.py b/mgz/cli.py index 14c685c..9812a0b 100644 --- a/mgz/cli.py +++ b/mgz/cli.py @@ -11,7 +11,6 @@ from construct.core import ConstructError from tabulate import tabulate -import tqdm import mgz import mgz.const @@ -23,8 +22,6 @@ LOGGER = logging.getLogger(__name__) -CMD_PLAY = 'play' -CMD_EXTRACT = 'extract' CMD_INFO = 'info' CMD_CHAT = 'chat' CMD_VALIDATE = 'validate' @@ -34,44 +31,6 @@ CMD_PAD = 'pad' -class TqdmStream: # pylint: disable=too-few-public-methods - """Log handler for TQDM.""" - - @classmethod - def write(cls, msg): - """Handle progress bars and logs.""" - tqdm.tqdm.write(msg, end='') - - -async def play_rec(playback, path): - """Play a recorded game.""" - if not playback: - raise RuntimeError('playback not supported') - from mgz.playback import Client, progress_bar - with open(path, 'rb') as handle: - summary = Summary(handle) - client = await Client.create( - playback, path, summary.get_start_time(), summary.get_duration() - ) - async for _, _, _ in progress_bar(client.sync(), client.duration): - pass - - -async def extract_rec(playback, path, select=None): - """Extract data from a recorded game.""" - with open(path, 'rb') as handle: - summary = Summary(handle, playback=playback) - data = await summary.async_extract(30000) - print('version: {}, runtime: {}'.format(data['version'], data['runtime'])) - for key, records in data.items(): - if select and key != select: - continue - print(key) - print('-------------') - for record in records: - print(record) - - def print_info(path): """Print basic info.""" with open(path, 'rb') as handle: @@ -215,13 +174,7 @@ def print_histogram(path): async def run(args): # pylint: disable=too-many-branches """Entry point.""" - if args.cmd == CMD_PLAY: - for rec in args.rec_path: - await play_rec(args.playback.split(',')[0], rec) - elif args.cmd == CMD_EXTRACT: - for rec in args.rec_path: - await extract_rec(args.playback.split(',')[0], rec, args.select) - elif args.cmd == CMD_INFO: + if args.cmd == CMD_INFO: for rec in args.rec_path: print_info(rec) elif args.cmd == CMD_CHAT: @@ -247,19 +200,12 @@ async def run(args): # pylint: disable=too-many-branches def get_args(): """Get arguments.""" parser = argparse.ArgumentParser() - parser.add_argument('-p', '--playback', - default=os.environ.get('AOC_PLAYBACK', '')) subparsers = parser.add_subparsers(dest='cmd') subparsers.required = True info = subparsers.add_parser(CMD_INFO) info.add_argument('rec_path', nargs='+') chat = subparsers.add_parser(CMD_CHAT) chat.add_argument('rec_path', nargs='+') - play = subparsers.add_parser(CMD_PLAY) - play.add_argument('rec_path', nargs='+') - extract = subparsers.add_parser(CMD_EXTRACT) - extract.add_argument('-s', '--select') - extract.add_argument('rec_path', nargs='+') validate = subparsers.add_parser(CMD_VALIDATE) validate.add_argument('rec_path', nargs='+') dump = subparsers.add_parser(CMD_DUMP) diff --git a/mgz/model/compat.py b/mgz/model/compat.py index 3f3215c..22020e6 100644 --- a/mgz/model/compat.py +++ b/mgz/model/compat.py @@ -55,7 +55,7 @@ def empty_achievements(): class ModelSummary: """Compatibility layer between Model and Summary classes.""" - def __init__(self, handle, playback=None): + def __init__(self, handle): self.match = parse_match(handle) self.size = self.match.file.size @@ -268,6 +268,3 @@ def get_map(self): ) for t in self.match.map.tiles ] ) - - def can_playback(self): - return False diff --git a/mgz/playback.py b/mgz/playback.py deleted file mode 100644 index 5aa6ae0..0000000 --- a/mgz/playback.py +++ /dev/null @@ -1,177 +0,0 @@ -"""API for recorded game playback.""" - -import asyncio -import logging -import os -import shutil -import struct -import time -from enum import Enum - -import aiohttp -import flatbuffers -import tqdm - -from AOC import ConfigMessage -from AOC import GameMessage - -from mgz import fast - - -LOGGER = logging.getLogger(__name__) -WS_URL = 'ws://{}' -MAX_ATTEMPTS = 5 -TOTAL_TIMEOUT = 60 * 4 -IO_TIMEOUT = 10 - - -class Source(Enum): - """Message source.""" - MGZ = 0 - MEMORY = 1 - - -def make_config(interval=1000, cycles=1): - """Make a configuration flatbuffer.""" - builder = flatbuffers.Builder(64) - ConfigMessage.ConfigMessageStart(builder) - ConfigMessage.ConfigMessageAddMessageIntervalMs(builder, interval) - ConfigMessage.ConfigMessageAddUpdateCycles(builder, cycles) - builder.Finish(ConfigMessage.ConfigMessageEnd(builder)) - return bytes(builder.Output()) - - -def read_message(buf): - """Read a message from flatbuffer as a dict.""" - return GameMessage.GameMessage.GetRootAsGameMessage(buf, 0) - - -async def progress_bar(generator, duration): - """Apply a progress bar - visualize progress in console.""" - config = { - 'total': duration, - 'unit': ' ticks', - 'miniters': int(duration/10000), - 'mininterval': 1 - } - with tqdm.tqdm(**config) as bar_: - last = 0 - async for tick, source, data in generator: - increment = tick - last - if increment > 0: - bar_.update(increment) - last = tick - yield tick, source, data - - -class Client(): - """Represents a client instance.""" - - def __init__(self, playback, rec_path): - """Initialize.""" - parts = playback.split(':') - self.socket = ':'.join(parts[0:-1]) - self.dropbox_path = os.path.abspath(os.path.expanduser(parts[-1])) - self.rec_path = os.path.abspath(os.path.expanduser(rec_path)) - self._handle = open(self.rec_path, 'rb') - header_len, _ = struct.unpack(' self.total_timeout: - LOGGER.warning("playback time exceeded timeout (%d%% completed)", int(mgz_time/self.duration * 100)) - break - try: - while not mgz_done and (mgz_time <= ws_time or ws_done): - op_type, payload = fast.operation(self._handle) - if op_type == fast.Operation.SYNC: - mgz_time += payload[0] - elif op_type == fast.Operation.CHAT and payload: - yield mgz_time, Source.MGZ, (op_type, payload.strip(b'\x00')) - elif op_type == fast.Operation.ACTION: - yield mgz_time, Source.MGZ, (op_type, payload) - except EOFError: - LOGGER.info("MGZ parsing stream finished") - mgz_done = True - while not ws_done and (ws_time <= mgz_time or mgz_done): - try: - data = await asyncio.wait_for(self.read_state().__anext__(), timeout=self.io_timeout) - except (asyncio.TimeoutError, asyncio.streams.IncompleteReadError): - LOGGER.warning("state reader timeout") - keep_reading = False - break - ws_time = data.WorldTime() - if data.GameFinished(): - LOGGER.info("state reader stream finished") - ws_done = True - yield ws_time, Source.MEMORY, data - if not keep_reading: - break - if ws_done and mgz_done: - LOGGER.info("synchronization finished in %.2f seconds", time.time() - start) - else: - raise RuntimeError("synchronization timeout encountered") - await self.session.close() - self._handle.close() diff --git a/mgz/summary/__init__.py b/mgz/summary/__init__.py index d2a9d63..97b3f86 100644 --- a/mgz/summary/__init__.py +++ b/mgz/summary/__init__.py @@ -9,7 +9,7 @@ class SummaryStub: - def __call__(self, data, playback=None, fallback=False): + def __call__(self, data, fallback=False): try: header = decompress(data) version, game, save, log = parse_version(header, data) @@ -20,11 +20,11 @@ def __call__(self, data, playback=None, fallback=False): if supported and not fallback: logger.info("using model summary") try: - return ModelSummary(data, playback) + return ModelSummary(data) except RuntimeError as e: logger.warning(f"could not fast parse; falling back: {e}") logger.info("using full summary") - return FullSummary(data, playback) + return FullSummary(data) Summary = SummaryStub() diff --git a/mgz/summary/extract.py b/mgz/summary/extract.py deleted file mode 100644 index a5a9ef6..0000000 --- a/mgz/summary/extract.py +++ /dev/null @@ -1,513 +0,0 @@ -""""Extract data via playback.""" - -import math -import logging -import time -from collections import defaultdict, deque -from datetime import timedelta - -from mgz import fast -from mgz.playback import Client, Source - - -LOGGER = logging.getLogger(__name__) -CLASS_UNIT = 70 -CLASS_BUILDING = 80 -TECH_STATE_AVAILABLE = 2 -TECH_STATE_RESEARCHING = 3 -TECH_STATE_DONE = 4 -DE_TECH_STATE_AVAILABLE = 1 -DE_TECH_STATE_RESEARCHING = 2 -DE_TECH_STATE_DONE = 3 -DE_TECH_STATE_CANT_RESEARCH = -1 -STATUS_WINNER = 1 - - -def has_diff(data, **kwargs): - """Simple dict subset diff.""" - for key, value in kwargs.items(): - if data.get(key) != value: - return True - return False - - -def flatten_research(research): - """Flatten research data.""" - flat = [] - for pid, techs in research.items(): - for tid, values in techs.items(): - flat.append(dict(values, player_number=pid, technology_id=tid)) - return flat - - -def build_fb_timeseries_record(tick, player): - """Build a timeseries record.""" - return { - 'timestamp': tick, - 'player_number': player.Id(), - 'population': player.Population(), - 'military': player.MilitaryPopulation(), - 'percent_explored': player.PercentMapExplored(), - 'headroom': int(player.Headroom()), - 'food': int(player.Food()), - 'wood': int(player.Wood()), - 'stone': int(player.Stone()), - 'gold': int(player.Gold()), - 'total_housed_time': int(player.CumulativeHousedTime()), - 'total_popcapped_time': int(player.CumulativePopCappedTime()), - 'relics_captured': int(player.VictoryPointsAndAttributes().VpRelicsCaptured()), - 'relic_gold': int(player.VictoryPointsAndAttributes().VpRelicGold()), - 'trade_profit': int(player.VictoryPointsAndAttributes().VpTradeProfit()), - 'tribute_received': int(player.VictoryPointsAndAttributes().VpTributeReceived()), - 'tribute_sent': int(player.VictoryPointsAndAttributes().VpTributeSent()), - 'total_food': int(player.VictoryPointsAndAttributes().VpTotalFood()), - 'total_wood': int(player.VictoryPointsAndAttributes().VpTotalWood()), - 'total_gold': int(player.VictoryPointsAndAttributes().VpTotalGold()), - 'total_stone': int(player.VictoryPointsAndAttributes().VpTotalStone()), - 'value_spent_objects': int(player.VictoryPointsAndAttributes().AttrValueSpentObjects()), - 'value_spent_research': int(player.VictoryPointsAndAttributes().AttrValueSpentResearch()), - 'value_lost_units': int(player.VictoryPointsAndAttributes().AttrValueLostUnits()), - 'value_lost_buildings': int(player.VictoryPointsAndAttributes().AttrValueLostBuildings()), - 'value_current_units': int(player.VictoryPointsAndAttributes().AttrValueCurrentUnits()), - 'value_current_buildings': int(player.VictoryPointsAndAttributes().AttrValueCurrentBuildings()), - 'value_objects_destroyed': int(player.VictoryPointsAndAttributes().AttrValueObjectsDestroyed()), - 'kills': int(player.VictoryPointsAndAttributes().AttrKills()), - 'deaths': int(player.VictoryPointsAndAttributes().AttrDeaths()), - 'razes': int(player.VictoryPointsAndAttributes().AttrRazes()) - } - - -def build_json_timeseries_record(tick, player): - """Build a timeseries record.""" - return { - 'timestamp': tick, - 'player_number': player['id'], - 'population': player['attributes']['population'], - 'military': player['attributes']['militaryPopulation'], - 'percent_explored': player['attributes']['percentMapExplored'], - 'headroom': int(player['attributes']['populationFreeRoom']), - 'food': int(player['attributes']['food']), - 'wood': int(player['attributes']['wood']), - 'stone': int(player['attributes']['stone']), - 'gold': int(player['attributes']['gold']), - 'total_housed_time': int(player['cumulativeHousedTime']), - 'total_popcapped_time': int(player['cumulativePopCappedTime']), - 'relics_captured': int(player['attributes']['relicsCaptured']), - 'relic_gold': int(player['attributes']['relicIncomeSum']), - 'trade_profit': int(player['attributes']['tradeIncomeSum']), - 'tribute_received': int(player['attributes']['tributeReceived']), - 'tribute_sent': int(player['attributes']['tributeSent']), - 'total_food': int(player['attributes']['foodTotalGathered']), - 'total_wood': int(player['attributes']['woodTotalGathered']), - 'total_gold': int(player['attributes']['goldTotalGathered']), - 'total_stone': int(player['attributes']['stoneTotalGathered']), - 'value_spent_objects': int(player['attributes']['objectCostSum']), - 'value_spent_research': int(player['attributes']['techCostSum']), - 'value_lost_units': int(player['attributes']['unitsLostValue']), - 'value_lost_buildings': int(player['attributes']['buildingsLostValue']), - 'value_current_units': int(player['attributes']['valueOfArmy']), - 'value_current_buildings': int(player['attributes']['valueOfBuildings']), - 'value_objects_destroyed': int(player['attributes']['killsValue'] + player['attributes']['razingsValue']), - 'value_units_killed': int(player['attributes']['killsValue']), - 'value_buildings_razed': int(player['attributes']['razingsValue']), - 'trained': int(player['attributes']['totalUnitsTrained']), - 'converted': int(player['attributes']['unitsConverted']), - 'kills': int(player['attributes']['unitsKilled']), - 'deaths': int(player['attributes']['unitsLost']), - 'razes': int(player['attributes']['razings']), - 'buildings_lost': int(player['attributes']['buildingsLost']), - 'hit_points_razed': int(player['attributes']['hitPointsRazed']), - 'hit_points_killed': int(player['attributes']['hitPointsKilled']), - 'villager_high': int(player['victoryPoints']['maxVillagers']), - 'military_high': int(player['victoryPoints']['maxMilitary']), - 'total_score': int(player['victoryPoints']['total']), - 'military_score': int(player['victoryPoints']['military']), - 'economy_score': int(player['victoryPoints']['economy']), - 'society_score': int(player['victoryPoints']['society']), - 'technology_score': int(player['victoryPoints']['technology']) - } - - -def update_research(player_number, state, index, tick, research, researching_state, done_state, available_state): - """Update research structure.""" - if state == researching_state and index not in research[player_number]: - research[player_number][index] = { - 'started': tick, - 'finished': None - } - elif state == done_state and index in research[player_number]: - research[player_number][index]['finished'] = tick - elif state == available_state and index in research[player_number]: - del research[player_number][index] - - -def update_market(tick, food, wood, stone, market): - """Update market coefficients structure.""" - last = None - change = True - if market: - last = market[-1] - change = has_diff(last, food=food, wood=wood, stone=stone) - if change: - market.append({ - 'timestamp': tick, - 'food': food, - 'wood': wood, - 'stone': stone - }) - - -def add_objects(owner_id, instance_id, class_id, object_id, created, pos_x, pos_y, objects): - """Add objects.""" - player_number = owner_id if owner_id > 0 else None - if class_id not in [CLASS_UNIT, CLASS_BUILDING]: - return - if instance_id not in objects: - objects[instance_id] = { - 'initial_player_number': player_number, - 'initial_object_id': object_id, - 'initial_class_id': class_id, - 'created': created, - 'destroyed': None, - 'destroyed_by_instance_id': None, - 'destroyed_building_percent': None, - 'deleted': False, - 'created_x': pos_x, - 'created_y': pos_y, - 'destroyed_x': None, - 'destroyed_y': None, - 'building_started': None, - 'building_completed': None, - 'total_idle_time': None - } - - -def update_objects(tick, owner_id, instance_id, class_id, object_id, killed, deleted, pos_x, pos_y, percent, killed_by, start, complete, idle, tech_id, objects, state, last): - """Update object/state structures.""" - player_number = owner_id if owner_id and owner_id > 0 else None - if instance_id not in objects: - return - if not class_id: - class_id = objects[instance_id]['initial_class_id'] - if not killed: - killed = objects[instance_id].get('killed') - data = {} - if class_id == CLASS_UNIT and pos_x is not None and pos_y is not None: - data.update({ - 'destroyed_x': pos_x, - 'destroyed_y': pos_y - }) - - if killed and killed > 0 and instance_id in objects: - data.update({ - 'destroyed': killed, - 'deleted': deleted - }) - if class_id == CLASS_BUILDING: - data['destroyed_building_percent'] = percent - if killed in objects: - data.update({ - 'destroyed_by_instance_id': killed_by - }) - objects[instance_id].update(data) - - if class_id == CLASS_BUILDING and instance_id in objects: - if start and start > 0 and objects[instance_id]['building_started'] is None: - objects[instance_id]['building_started'] = start - if complete and complete > 0 and objects[instance_id]['building_completed'] is None: - objects[instance_id]['building_completed'] = complete - - if instance_id in objects and idle: - objects[instance_id]['total_idle_time'] = int(idle) - - researching_technology_id = tech_id if tech_id and tech_id > 0 else None - change = ( - instance_id not in last or - has_diff( - last[instance_id], - player_number=player_number, - class_id=class_id, - object_id=object_id, - researching_technology_id=researching_technology_id - ) - ) - snapshot = { - 'timestamp': tick, - 'instance_id': instance_id, - 'player_number': player_number, - 'object_id': object_id, - 'class_id': class_id, - 'researching_technology_id': researching_technology_id - } - if change: - state.append(snapshot) - last[instance_id] = snapshot - - -def enrich_actions(actions, objects, states): - """Attach player to anonymous actions.""" - owners = defaultdict(list) - for state in states: - owners[state['instance_id']].append((state['timestamp'], state['player_number'])) - last_tick = None - last_seen = [] - for (tick, action_type, payload) in actions: - if 'player_id' not in payload and 'object_ids' in payload: - for object_id in payload['object_ids']: - if object_id not in owners: - continue - for (timestamp, player_number) in owners[object_id]: - if timestamp > tick: - payload['player_id'] = player_number - break - if object_id not in objects: - continue - if 'player_id' not in payload: - payload['player_id'] = objects[object_id]['initial_player_number'] - break - action = (tick, action_type, payload) - if tick != last_tick: - last_seen = [] - last_tick = tick - if action not in last_seen and 'player_id' in payload and payload['player_id'] is not None: - yield action - last_seen.append(action) - - -def add_action_count(timeseries, actions): - i = iter(actions) - tick, _, payload = next(i) - last_ts = 0 - for record in timeseries: - if record['timestamp'] > last_ts: - count = defaultdict(int) - while tick < record['timestamp']: - count[payload['player_id']] += 1 - try: - tick, _, payload = next(i) - except StopIteration: - break - record['action_count'] = count[record['player_number']] - last_ts = record['timestamp'] - -def _dist(a, b): - return math.sqrt((b[0] - a[0])**2 + (b[1] - a[1])**2) - -def add_map_control(timeseries, action_list, players, teams, duration): - #opps[pid] = {pid: xy} - coords = {} - lookup = {player['number']:player for player in players} - opps = defaultdict(list) - - for i, team_a in enumerate(teams): - for player_number in team_a: - coords[player_number] = lookup[player_number]['position'] - for j, team_b in enumerate(teams): - if i == j: - continue - opps[player_number] += team_b - #for opponent_number in team_b: - # opps[player_number] += team_b[opponent_number] = lookup[opponent_number]['position'] - - percents = defaultdict(list) - - last = {} - dk = {} - for player in players: - last[player['number']] = [] #deque(maxlen=30) - dk[player['number']] = deque(maxlen=30) - - - def next_action(act): - tick, _, payload = next(act) - if 'x' not in payload or 'y' not in payload or 'player_id' not in payload: - return tick, None, None - to_me = _dist((payload['x'], payload['y']), coords[payload['player_id']]) - options = [] - for opp in opps[payload['player_id']]: - to_opp = _dist((payload['x'], payload['y']), coords[opp]) - between = _dist(coords[payload['player_id']], coords[opp]) - options.append(round((((to_me - to_opp) + between) / (between * 2)) * 100)) - return tick, payload['player_id'], max(options) - - - it = iter(action_list) - tick, player_number, control = next_action(it) - for record in timeseries: - while tick < record['timestamp']: - if control: - if len(last[player_number]) == 30: - last[player_number] = [] - last[player_number].append(control) - mc = last[record['player_number']] - dk[player_number].append(sum(mc)/len(mc) if len(mc) > 0 else 0) - try: - tick, player_number, control = next_action(it) - except StopIteration: - break - mc = dk[record['player_number']] - record['map_control'] = sum(mc)/len(mc) if len(mc) > 0 else 0 - print(record['timestamp'], record['player_number'], record['map_control']) - -def transform_objects(objects): - """Transform objects.""" - obj_list = [] - for i, obj in objects.items(): - data = dict(obj, instance_id=i) - if data['destroyed'] is None: - data['destroyed_x'] = None - data['destroyed_y'] = None - obj_list.append(data) - return obj_list - - -def transform_seed_objects(objects): - """Map seed objects to state format.""" - return {obj['instance_id']: { - 'initial_player_number': obj['player_number'], - 'initial_object_id': obj['object_id'], - 'initial_class_id': obj['class_id'], - 'created': 0, - 'created_x': obj['x'], - 'created_y':obj['y'], - 'destroyed': None, - 'destroyed_by_instance_id': None, - 'destroyed_building_percent': None, - 'deleted': False, - 'destroyed_x': None, - 'destroyed_y': None, - 'building_started': None, - 'building_completed': None, - 'total_idle_time': None - } for obj in objects} - - -async def get_extracted_data(start_time, duration, playback, handle, interval, seed_objects, players, teams): - """Get extracted data.""" - timeseries = [] - research = defaultdict(dict) - market = [] - objects = transform_seed_objects(seed_objects) - state = [] - last = {} - actions = [] - version = None - start = time.time() - client = await Client.create(playback, handle.name, start_time, duration, interval) - async for tick, source, message in client.sync(): - if source == Source.MEMORY: - if not version: - version = message.StateReaderVersion().decode('ascii') - - update_market(tick, message.MarketCoefficients().Food(), message.MarketCoefficients().Wood(), message.MarketCoefficients().Stone(), market) - - # Add any new objects before updating to ensure fks are present for updates - for i in range(0, message.ObjectsLength()): - obj = message.Objects(i) - add_objects(obj.OwnerId(), obj.Id(), obj.MasterObjectClass(), obj.MasterObjectId(), obj.CreatedTime(), obj.Position().X(), obj.Position().Y(), objects) - - for i in range(0, message.ObjectsLength()): - obj = message.Objects(i) - update_objects( - tick, obj.OwnerId(), obj.Id(), obj.MasterObjectClass(), obj.MasterObjectId(), obj.KilledTime(), - obj.DeletedByOwner(), obj.Position().X(), obj.Position().Y(), obj.BuildingPercentComplete(), - obj.KilledByUnitId(), obj.BuildingStartTime(), obj.BuildingCompleteTime(), obj.CumulativeIdleTime(), - obj.CurrentlyResearchingTechId(), objects, state, last - ) - - for i in range(0, message.PlayersLength()): - player = message.Players(i) - timeseries.append(build_fb_timeseries_record(tick, player)) - for j in range(0, player.TechsLength()): - tech = player.Techs(j) - update_research( - player.Id(), tech.State(), tech.IdIndex(), tech.Time(), research, - TECH_STATE_RESEARCHING, TECH_STATE_DONE, TECH_STATE_AVAILABLE - ) - - elif source == Source.MGZ: - if message[0] == fast.Operation.ACTION: - actions.append((tick, *message[1])) - - handle.close() - - action_list = list(enrich_actions(actions, objects, state)) - add_action_count(timeseries, action_list) - add_map_control(timeseries, action_list, players, teams, duration) - - return { - 'version': version, - 'runtime': timedelta(seconds=int(time.time() - start)), - 'timeseries': timeseries, - 'research': flatten_research(research), - 'market': market, - 'objects': transform_objects(objects), - 'state': state, - 'actions': action_list, - 'winners': set() - } - - -def external_extracted_data(data, seed_objects, players, teams, actions): - """Merge externally-sourced extracted data.""" - research = defaultdict(dict) - objects = transform_seed_objects(seed_objects) - state = [] - market = [] - last = {} - timeseries = [] - winners = set() - version = data['version'] - interval = data['messageInterval'] - for message in data['messages']: - update_market(message['time'], message['world']['foodPrice'], message['world']['woodPrice'], message['world']['stonePrice'], market) - - # Add any new objects before updating to ensure fks are present for updates - for obj in message['objects']: - # Only add objects the first time we see them - if 'ownerId' not in obj or 'masterObjectClass' not in obj: - continue - add_objects( - obj['ownerId'], obj['id'], obj['masterObjectClass'], obj['masterObjectId'], obj['createdTime'], - obj['position']['x'], obj['position']['y'], objects - ) - - for obj in message['objects']: - update_objects( - message['time'], obj.get('ownerId'), obj['id'], obj.get('masterObjectClass'), obj.get('masterObjectId'), - obj.get('killedTime'), None, obj.get('position', {}).get('x'), obj.get('position', {}).get('y'), obj.get('buildingPercentComplete'), - None, obj.get('buildingStartTime'), obj.get('buildingCompleteTime'), obj.get('cumulativeIdleTime'), None, objects, state, last - ) - - for player in message['players'][1:]: - timeseries.append(build_json_timeseries_record(message['time'], player)) - if player['status'] == STATUS_WINNER: - winners.add(player['id']) - - for event in message['events']: - if event['data']['tag'] == 'techStateChange': - update_research( - event['data']['playerId'], event['data']['state'], event['data']['index'], event['worldTime'], - research, DE_TECH_STATE_RESEARCHING, DE_TECH_STATE_DONE, DE_TECH_STATE_AVAILABLE - ) - - for obj in objects.values(): - if not obj['destroyed']: - obj['destroyed_x'] = None - obj['destroyed_y'] = None - - action_list = list(enrich_actions(actions, objects, state)) - add_action_count(timeseries, action_list) - - return { - 'version': version, - 'interval': interval, - 'runtime': None, - 'timeseries': timeseries, - 'research': flatten_research(research), - 'market': market, - 'objects': [dict(obj, instance_id=i) for i, obj in objects.items()], - 'state': state, - 'actions': action_list, - 'winners': winners - } diff --git a/mgz/summary/full.py b/mgz/summary/full.py index 0fdc068..dbdc07d 100644 --- a/mgz/summary/full.py +++ b/mgz/summary/full.py @@ -43,12 +43,11 @@ class FullSummary: # pylint: disable=too-many-public-methods Access match summary data. """ - def __init__(self, handle, playback=None): + def __init__(self, handle): """Initialize.""" self.size = len(handle.read()) handle.seek(0) self._handle = handle - self._playback = playback self._cache = { 'dataset': None, 'teams': None, @@ -87,9 +86,6 @@ def __init__(self, handle, playback=None): except (construct.core.ConstructError, zlib.error, ValueError) as e: raise RuntimeError("invalid mgz file: {}".format(e)) - if isinstance(self._playback, io.TextIOWrapper): - self.extract() - def _process_body(self): # pylint: disable=too-many-locals, too-many-statements, too-many-branches """Process rec body.""" start_time = time.time() @@ -373,46 +369,3 @@ def get_mirror(self): def get_played(self): if self._header.de: return self._header.de.timestamp - - def can_playback(self): - """Indicate whether playback is possible.""" - return self._playback - - async def async_extract(self, interval=1000): - """Full extraction.""" - if not self.can_playback(): - raise RuntimeError('extraction not supported') - - from mgz.summary.extract import get_extracted_data - - temp = tempfile.NamedTemporaryFile() - self._handle.seek(0) - temp.write(self._handle.read()) - - return await get_extracted_data( - self.get_start_time(), - self.get_duration(), - self._playback, - temp, interval, - self.get_objects()['objects'], - self.get_players(), - self.get_teams() - ) - - def extract(self, interval=1000): - """Async wrapper around full extraction.""" - if not self._cache['extraction']: - if isinstance(self._playback, io.TextIOWrapper): - from mgz.summary.extract import external_extracted_data - - self._cache['extraction'] = external_extracted_data( - json.loads(self._playback.read()), - self.get_objects()['objects'], - self.get_players(), - self.get_teams(), - self._actions - ) - else: - loop = asyncio.get_event_loop() - self._cache['extraction'] = loop.run_until_complete(self.async_extract(interval)) - return self._cache['extraction'] diff --git a/setup.py b/setup.py index 70e89eb..105fdce 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name='mgz', - version='1.8.2', + version='1.8.3', description='Parse Age of Empires 2 recorded games.', url='https://github.com/happyleavesaoc/aoc-mgz/', license='MIT', @@ -11,13 +11,10 @@ author_email='happyleaves.tfr@gmail.com', packages=find_packages(), install_requires=[ - 'aiohttp>=3.6.2', - 'aocref>=2.0.7', + 'aocref>=2.0.8', 'construct==2.8.16', 'dataclasses==0.8; python_version < "3.7"', - 'flatbuffers>=1.10', - 'tabulate>=0.8.2', - 'tqdm>=4.28.1', + 'tabulate>=0.9.0', ], entry_points = { 'console_scripts': ['mgz=mgz.cli:main'],