From 9d0267d8a0dfec1c021bd40b20523d8bd43aac52 Mon Sep 17 00:00:00 2001 From: smiley Date: Fri, 31 Mar 2017 00:52:43 +0200 Subject: [PATCH] added colors, recipes & skins --- sql/gw2_colors.sql | 25 +++++ sql/gw2_recipes.sql | 25 +++++ sql/gw2_skins.sql | 26 +++++ src/Updaters/Items/Colors.php | 166 +++++++++++++++++++++++++++++++ src/Updaters/Items/Recipes.php | 173 +++++++++++++++++++++++++++++++++ src/Updaters/Items/Skins.php | 171 ++++++++++++++++++++++++++++++++ 6 files changed, 586 insertions(+) create mode 100644 sql/gw2_colors.sql create mode 100644 sql/gw2_recipes.sql create mode 100644 sql/gw2_skins.sql create mode 100644 src/Updaters/Items/Colors.php create mode 100644 src/Updaters/Items/Recipes.php create mode 100644 src/Updaters/Items/Skins.php diff --git a/sql/gw2_colors.sql b/sql/gw2_colors.sql new file mode 100644 index 0000000..dff615e --- /dev/null +++ b/sql/gw2_colors.sql @@ -0,0 +1,25 @@ +CREATE TABLE `gw2_colors` ( + `id` INT(10) UNSIGNED NOT NULL, + `item_id` INT(10) UNSIGNED NOT NULL, + `hue` ENUM ('Blue', 'Brown', 'Gray', 'Green', 'Orange', 'Purple', 'Red', 'Yellow') DEFAULT NULL, + `material` ENUM ('Leather', 'Metal', 'Vibrant') DEFAULT NULL, + `rarity` ENUM ('Common', 'Rare', 'Starter', 'Uncommon') DEFAULT NULL, + `name_de` TINYTEXT NOT NULL, + `name_en` TINYTEXT NOT NULL, + `name_es` TINYTEXT NOT NULL, + `name_fr` TINYTEXT NOT NULL, + `name_zh` TEXT NOT NULL, + `data_de` TEXT NOT NULL, + `data_en` TEXT NOT NULL, + `data_es` TEXT NOT NULL, + `data_fr` TEXT NOT NULL, + `data_zh` TEXT NOT NULL, + `updated` TINYINT(1) UNSIGNED NOT NULL DEFAULT '0', + `update_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `date_added` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) + ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4 + COLLATE = utf8mb4_bin; + diff --git a/sql/gw2_recipes.sql b/sql/gw2_recipes.sql new file mode 100644 index 0000000..f714238 --- /dev/null +++ b/sql/gw2_recipes.sql @@ -0,0 +1,25 @@ +CREATE TABLE `gw2_recipes` ( + `id` INT(10) UNSIGNED NOT NULL DEFAULT '0', + `output_id` INT(10) UNSIGNED NOT NULL DEFAULT '0', + `output_count` SMALLINT(3) UNSIGNED NOT NULL DEFAULT '0', + `disciplines` SMALLINT(3) UNSIGNED NOT NULL DEFAULT '0', + `rating` SMALLINT(3) UNSIGNED NOT NULL DEFAULT '0', + `type` TINYTEXT NOT NULL, + `from_item` TINYINT(1) UNSIGNED NOT NULL DEFAULT '0', + `ing_id_1` INT(10) UNSIGNED NOT NULL DEFAULT '0', + `ing_count_1` SMALLINT(3) UNSIGNED NOT NULL DEFAULT '0', + `ing_id_2` INT(10) UNSIGNED NOT NULL DEFAULT '0', + `ing_count_2` SMALLINT(3) UNSIGNED NOT NULL DEFAULT '0', + `ing_id_3` INT(10) UNSIGNED NOT NULL DEFAULT '0', + `ing_count_3` SMALLINT(3) UNSIGNED NOT NULL DEFAULT '0', + `ing_id_4` INT(10) UNSIGNED NOT NULL DEFAULT '0', + `ing_count_4` SMALLINT(3) UNSIGNED NOT NULL DEFAULT '0', + `data` TEXT NOT NULL, + `updated` TINYINT(1) UNSIGNED NOT NULL DEFAULT '0', + `update_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `date_added` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) + ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4 + COLLATE = utf8mb4_bin; diff --git a/sql/gw2_skins.sql b/sql/gw2_skins.sql new file mode 100644 index 0000000..d012082 --- /dev/null +++ b/sql/gw2_skins.sql @@ -0,0 +1,26 @@ +CREATE TABLE `gw2_skins` ( + `id` INT(10) UNSIGNED NOT NULL, + `signature` VARCHAR(40) NOT NULL, + `file_id` INT(10) NOT NULL DEFAULT '0', + `type` TINYTEXT NOT NULL, + `subtype` TINYTEXT NOT NULL, + `properties` TINYTEXT NOT NULL, + `name_de` TINYTEXT NOT NULL, + `name_en` TINYTEXT NOT NULL, + `name_es` TINYTEXT NOT NULL, + `name_fr` TINYTEXT NOT NULL, + `name_zh` TINYTEXT NOT NULL, + `data_de` TEXT NOT NULL, + `data_en` TEXT NOT NULL, + `data_es` TEXT NOT NULL, + `data_fr` TEXT NOT NULL, + `data_zh` TEXT NOT NULL, + `updated` TINYINT(1) UNSIGNED NOT NULL DEFAULT '0', + `update_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `date_added` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) + ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4 + COLLATE = utf8mb4_bin; + diff --git a/src/Updaters/Items/Colors.php b/src/Updaters/Items/Colors.php new file mode 100644 index 0000000..719dac9 --- /dev/null +++ b/src/Updaters/Items/Colors.php @@ -0,0 +1,166 @@ + + * @copyright 2017 Smiley + * @license MIT + */ + +namespace chillerlan\GW2DB\Updaters\Items; + +use chillerlan\GW2DB\Helpers; +use chillerlan\GW2DB\Updaters\{MultiRequestAbstract, UpdaterException}; +use chillerlan\TinyCurl\{ResponseInterface, URL}; + +/** + */ +class Colors extends MultiRequestAbstract{ + + /** + * @var array + */ + protected $colors; + + /** + * @throws \chillerlan\GW2DB\Updaters\UpdaterException + */ + public function init(){ + $this->refreshIDs('colors', getenv('TABLE_GW2_COLORS')); + + $sql = 'SELECT `id`, `data_de`, `data_en`, `data_es`, `data_fr`, `data_zh`, UNIX_TIMESTAMP(`update_time`) AS `update_time`, UNIX_TIMESTAMP(`date_added`) AS `date_added` FROM `'.getenv('TABLE_GW2_COLORS').'`'; + + $this->colors = $this->DBDriverInterface->raw($sql, 'id', true, true); + + if(!$this->colors || !is_array($this->colors)){ + throw new UpdaterException('failed to fetch color data from db'); + } + + $urls = []; + + foreach(array_chunk($this->colors, self::CHUNK_SIZE) as $chunk){ + foreach(self::API_LANGUAGES as $lang){ + $urls[] = new URL(self::API_BASE.'/colors', ['lang' => $lang, 'ids' => implode(',', array_column($chunk, 'id'))]); + } + } + + $this->fetchMulti($urls); + $this->updateStats(); + $this->logToCLI(__METHOD__.': end'); + } + + /** + * @param \chillerlan\TinyCurl\ResponseInterface $response + * + * @return mixed + */ + protected function processResponse(ResponseInterface $response){ + $info = $response->info; + + parse_str(parse_url($info->url, PHP_URL_QUERY), $params); + + $this->lang = $response->headers->{'content-language'} ?: $params['lang']; + + if(!$this->checkResponseLanguage($this->lang)){ + return false; + } + + $sql = 'UPDATE '.getenv('TABLE_GW2_COLORS').' SET `name_'.$this->lang.'`= ?, `data_'.$this->lang.'`= ? WHERE `id` = ?'; + + $query = $this->DBDriverInterface->multi_callback($sql, $response->json_array, [$this, 'callback']); + + if(!$query){ + $this->logToCLI('SQL insert failed, retrying URL. ('.$info->url.')'); + + return new URL($info->url); + } + + if(!empty($this->changes)){ + $sql = 'INSERT INTO `'.getenv('TABLE_GW2_DIFF').'` (`db_id`, `type`, `lang`, `date`, `data`) VALUES (?,?,?,?,?)'; + + if($this->DBDriverInterface->multi($sql, $this->changes)){ + $this->changes = []; + } + } + + $this->logToCLI('['.$this->lang.'] '.md5($info->url).' updated'); + } + + /** + * @param array $color + * + * @return array + */ + public function callback(array $color){ + $old = json_decode(@$this->colors[$color['id']]['data_'.$this->lang], true) ?? []; + $diff = Helpers\array_diff_assoc_recursive($old, $color, true); + + if(!empty($old) && !empty($diff)){ + $this->changes[] = [ + $color['id'], + 'color', + $this->lang, + $this->colors[$color['id']]['update_time'] ?? $this->colors[$color['id']]['date_added'] ?? time(), + json_encode($old), + ]; + + $this->logToCLI('['.$this->lang.'] color changed #'.$color['id'].' '.print_r($diff, true)); + } + + $this->logToCLI('['.$this->lang.'] updated color data #'.$color['id']); + + return [ + $color['name'], + json_encode($color), + $color['id'], + ]; + } + + /** + * @throws \chillerlan\GW2DB\Updaters\UpdaterException + */ + protected function updateStats(){ + $sql = 'SELECT `data_en` FROM `'.getenv('TABLE_GW2_COLORS').'`'; + + $this->colors = $this->DBDriverInterface->raw($sql, null, true, true); + + if(!$this->colors || !is_array($this->colors)){ + throw new UpdaterException('failed to fetch color data from db'); + } + + $sql = 'UPDATE '.getenv('TABLE_GW2_COLORS').' SET `hue`= ?, `material`= ?, `rarity`= ?, `updated`= ? WHERE `id` = ?'; + + $query = $this->DBDriverInterface->multi_callback($sql, $this->colors, [$this, 'statsCallback']); + + if(!$query){ + throw new UpdaterException('failed to update stats'); + } + } + + /** + * @param array $color + * + * @return array + */ + public function statsCallback(array $color):array{ + $data = json_decode($color['data_en']); + + list($hue, $material, $rarity) = !empty($data->categories) ? $data->categories : [null, null, null]; + + $this->logToCLI('updated color stats #'.$data->id); + + return [ + $hue, + $material, + $rarity, + 1, + $data->id, + ]; + } + +} + + diff --git a/src/Updaters/Items/Recipes.php b/src/Updaters/Items/Recipes.php new file mode 100644 index 0000000..c8697d6 --- /dev/null +++ b/src/Updaters/Items/Recipes.php @@ -0,0 +1,173 @@ + + * @copyright 2017 Smiley + * @license MIT + */ + +namespace chillerlan\GW2DB\Updaters\Items; + +use chillerlan\GW2DB\Helpers; +use chillerlan\GW2DB\Updaters\{MultiRequestAbstract, UpdaterException}; +use chillerlan\TinyCurl\{ResponseInterface, URL}; + +/** + * + */ +class Recipes extends MultiRequestAbstract{ + + const CRAFT_ARMORSMITH = 0x1; + const CRAFT_ARTIFICER = 0x2; + const CRAFT_CHEF = 0x4; + const CRAFT_HUNTSMAN = 0x8; + const CRAFT_JEWELER = 0x10; + const CRAFT_LEATHERWORKER = 0x20; + const CRAFT_TAILOR = 0x40; + const CRAFT_WEAPONSMITH = 0x80; + const CRAFT_SCRIBE = 0x100; + + /** + * @var array + */ + protected $recipes; + + /** + * @throws \chillerlan\GW2DB\Updaters\UpdaterException + */ + public function init(){ + $this->refreshIDs('recipes', getenv('TABLE_GW2_RECIPES')); + + $sql = 'SELECT `id`, `data`, UNIX_TIMESTAMP(`update_time`) AS `update_time`, UNIX_TIMESTAMP(`date_added`) AS `date_added` FROM `'.getenv('TABLE_GW2_RECIPES').'`'; + + $this->recipes = $this->DBDriverInterface->raw($sql, 'id', true, true); + + if(!$this->recipes || !is_array($this->recipes)){ + throw new UpdaterException('failed to fetch recipe IDs from db'); + } + + $urls = []; + + foreach(array_chunk($this->recipes, self::CHUNK_SIZE) as $chunk){ + $urls[] = new URL(self::API_BASE.'/recipes', ['ids' => implode(',', array_column($chunk, 'id'))]); + } + + $this->fetchMulti($urls); + $this->logToCLI(__METHOD__.': end'); + } + + /** + * @param \chillerlan\TinyCurl\ResponseInterface $response + * + * @return mixed + */ + protected function processResponse(ResponseInterface $response){ + $info = $response->info; + $json = $response->json_array; + + if(!is_array($json) || empty($json)){ + return false; + } + + $sql = 'UPDATE `'.getenv('TABLE_GW2_RECIPES').'` SET `output_id` = ?, `output_count` = ?, `disciplines`= ?, + `rating` = ?, `type` = ?, `from_item` = ?, `ing_id_1` = ?, `ing_count_1` = ?, + `ing_id_2` = ?, `ing_count_2` = ?, `ing_id_3` = ?, `ing_count_3` = ?, + `ing_id_4` = ?, `ing_count_4` = ?, `data` = ?, `updated` = ? + WHERE `id` = ?'; + + $query = $this->DBDriverInterface->multi_callback($sql, $json, [$this, 'callback']); + + if(!$query){ + $this->logToCLI('SQL insert failed, retrying URL. ('.$info->url.')'); + + return new URL($info->url); + } + + if(!empty($this->changes)){ + $sql = 'INSERT INTO `'.getenv('TABLE_GW2_DIFF').'` (`db_id`, `type`, `date`, `data`) VALUES (?,?,?,?)'; + + if($this->DBDriverInterface->multi($sql, $this->changes)){ + $this->changes = []; + } + } + + return true; + } + + /** + * @param array $recipe + * + * @return array + */ + public function callback(array $recipe):array{ + $recipe = $this->sortRecipe($recipe); + $old = $this->sortRecipe(json_decode(@$this->recipes[$recipe['id']]['data'], true) ?? []); + $diff = Helpers\array_diff_assoc_recursive($old, $recipe, true); + + if(!empty($old) && !empty($diff)){ + + $this->changes[] = [ + $recipe['id'], + 'recipe', + $this->recipes[$recipe['id']]['update_time'] ?? $this->recipes[$recipe['id']]['date_added'] ?? time(), + json_encode($old), + ]; + + $this->logToCLI('recipe changed #'.$recipe['id'].' '.print_r($diff, true)); + } + + $disciplines = array_map(function($value){ + return constant('self::CRAFT_'.strtoupper($value)); + }, $recipe['disciplines']); + + $this->logToCLI('updated recipe #'.$recipe['id']); + + return [ + $recipe['output_item_id'], + $recipe['output_item_count'], + Helpers\set_bitflag($disciplines), + $recipe['min_rating'], + $recipe['type'], + isset($recipe['flags']) && is_array($recipe['flags']) && in_array('LearnedFromItem', $recipe['flags']), + $recipe['ingredients'][0]['item_id'] ?? 0, + $recipe['ingredients'][0]['count'] ?? 0, + $recipe['ingredients'][1]['item_id'] ?? 0, + $recipe['ingredients'][1]['count'] ?? 0, + $recipe['ingredients'][2]['item_id'] ?? 0, + $recipe['ingredients'][2]['count'] ?? 0, + $recipe['ingredients'][3]['item_id'] ?? 0, + $recipe['ingredients'][3]['count'] ?? 0, + json_encode($recipe), + 1, + $recipe['id'], + ]; + } + + /** + * preserve order of the ingrediends + * + * @param array $recipe + * + * @return array + */ + protected function sortRecipe(array $recipe):array{ + $ingredients = $recipe['ingredients'] ?? null; + $guild_ingredients = $recipe['guild_ingredients'] ?? null; + + $recipe = Helpers\array_sort_recursive($recipe); + + if($ingredients){ + $recipe['ingredients'] = $ingredients; + } + + if($guild_ingredients){ + $recipe['guild_ingredients'] = $guild_ingredients; + } + + return $recipe; + } +} diff --git a/src/Updaters/Items/Skins.php b/src/Updaters/Items/Skins.php new file mode 100644 index 0000000..ecafed9 --- /dev/null +++ b/src/Updaters/Items/Skins.php @@ -0,0 +1,171 @@ + + * @copyright 2017 Smiley + * @license MIT + */ + +namespace chillerlan\GW2DB\Updaters\Items; + +use chillerlan\GW2DB\Helpers; +use chillerlan\GW2DB\Updaters\{MultiRequestAbstract, UpdaterException}; +use chillerlan\TinyCurl\{ResponseInterface, URL}; + +/** + * + */ +class Skins extends MultiRequestAbstract{ + + /** + * @var array + */ + protected $skins; + + /** + * @throws \chillerlan\GW2DB\Updaters\UpdaterException + */ + public function init(){ + $this->refreshIDs('skins', getenv('TABLE_GW2_SKINS')); + + $sql = 'SELECT `id`, `data_de`, `data_en`, `data_es`, `data_fr`, `data_zh`, UNIX_TIMESTAMP(`update_time`) AS `update_time`, UNIX_TIMESTAMP(`date_added`) AS `date_added` FROM `'.getenv('TABLE_GW2_SKINS').'`'; + + $this->skins = $this->DBDriverInterface->raw($sql, 'id', true, true); + + if(!$this->skins || !is_array($this->skins)){ + throw new UpdaterException('failed to fetch skin data from db'); + } + + $urls = []; + + foreach(array_chunk($this->skins, self::CHUNK_SIZE) as $chunk){ + foreach(self::API_LANGUAGES as $lang){ + $urls[] = new URL(self::API_BASE.'/skins', ['lang' => $lang, 'ids' => implode(',', array_column($chunk, 'id'))]); + } + } + + $this->fetchMulti($urls); + $this->updateStats(); + $this->logToCLI(__METHOD__.': end'); + } + + /** + * @param \chillerlan\TinyCurl\ResponseInterface $response + * + * @return mixed + */ + protected function processResponse(ResponseInterface $response){ + $info = $response->info; + + parse_str(parse_url($info->url, PHP_URL_QUERY), $params); + + $this->lang = $response->headers->{'content-language'} ?: $params['lang']; + + if(!$this->checkResponseLanguage($this->lang)){ + return false; + } + + $sql = 'UPDATE '.getenv('TABLE_GW2_SKINS').' SET `name_'.$this->lang.'`= ?, `data_'.$this->lang.'`= ? WHERE `id` = ?'; + + $query = $this->DBDriverInterface->multi_callback($sql, $response->json_array, [$this, 'callback']); + + if(!$query){ + $this->logToCLI('SQL insert failed, retrying URL. ('.$info->url.')'); + + return new URL($info->url); + } + + if(!empty($this->changes)){ + $sql = 'INSERT INTO `'.getenv('TABLE_GW2_DIFF').'` (`db_id`, `type`, `lang`, `date`, `data`) VALUES (?,?,?,?,?)'; + + if($this->DBDriverInterface->multi($sql, $this->changes)){ + $this->changes = []; + } + } + + $this->logToCLI('['.$this->lang.'] '.md5($info->url).' updated'); + + return true; + } + + /** + * @param array $skin + * + * @return array + */ + public function callback(array $skin):array{ + $skin = Helpers\array_sort_recursive($skin); + $old = Helpers\array_sort_recursive(json_decode(@$this->skins[$skin['id']]['data_'.$this->lang], true) ?? []); + $diff = Helpers\array_diff_assoc_recursive($old, $skin, true); + + if(!empty($old) && !empty($diff)){ + + $this->changes[] = [ + $skin['id'], + 'skin', + $this->lang, + $this->skins[$skin['id']]['update_time'] ?? $this->skins[$skin['id']]['date_added'] ?? time(), + json_encode($old), + ]; + + $this->logToCLI('['.$this->lang.'] skin changed #'.$skin['id'].' '.print_r($diff, true)); + } + + $this->logToCLI('['.$this->lang.'] updated skin data #'.$skin['id']); + + return [ + $skin['name'], + json_encode($skin), + $skin['id'], + ]; + } + + /** + * @throws \chillerlan\GW2DB\Updaters\UpdaterException + */ + protected function updateStats(){ + $sql = 'SELECT `data_en` FROM `'.getenv('TABLE_GW2_SKINS').'`'; + + $this->skins = $this->DBDriverInterface->raw($sql, null, true, true); + + if(!$this->skins || !is_array($this->skins)){ + throw new UpdaterException('failed to fetch skin data from db'); + } + + $sql = 'UPDATE '.getenv('TABLE_GW2_SKINS').' SET `signature`= ?, `file_id`= ?, `type`= ?, + `subtype`= ?, `properties` = ?, `updated`= ? WHERE `id` = ?'; + + $query = $this->DBDriverInterface->multi_callback($sql, $this->skins, [$this, 'statsCallback']); + + if(!$query){ + throw new UpdaterException('failed to update stats'); + } + } + + /** + * @param array $skin + * + * @return array + */ + public function statsCallback(array $skin):array{ + $data = json_decode($skin['data_en']); + + $file_id = explode('/', str_replace(['https://render.guildwars2.com/file/', '.png'], '', $data->icon ?? '')); + + $this->logToCLI('updated skin stats #'.$data->id); + + return [ + $file_id[0], + $file_id[1] ?? 0, + $data->type, + $data->details->type ?? '', + $data->details->weight_class ?? $data->details->damage_type ?? '', + 1, + $data->id, + ]; + } +}