diff --git a/config/default.ini b/config/default.ini index 4febe67..c557121 100644 --- a/config/default.ini +++ b/config/default.ini @@ -28,3 +28,6 @@ AVE_CHECK_FOR_UPDATES_DAYS=7 AVE_LOG_EVENT=true AVE_LOG_ERROR=true AVE_AVATAR_GENERATOR_VARIANTS="1.0 1.5 1.75 2.0 2.5" +AVE_BACKUP_COMPRESS_LEVEL=5 +AVE_BACKUP_COMPRESS_TYPE="7z" +AVE_BACKUP_MAX_ALLOWED_PACKET=536870912 diff --git a/includes/AVE.php b/includes/AVE.php index 052dddd..dea0245 100644 --- a/includes/AVE.php +++ b/includes/AVE.php @@ -29,7 +29,7 @@ class AVE extends CommandLine { public bool $open_log = false; private string $app_name = "AVE"; - private string $version = "1.4.0"; + private string $version = "1.4.1"; private ?string $command; private array $arguments; private string $logo; diff --git a/includes/services/DataBaseBackup.php b/includes/services/DataBaseBackup.php index 7e7c008..f1129da 100644 --- a/includes/services/DataBaseBackup.php +++ b/includes/services/DataBaseBackup.php @@ -9,12 +9,12 @@ class DataBaseBackup { - protected PDO $conn; + protected ?PDO $source; + protected ?PDO $destination; protected string $database; private int $query_limit; private int $insert_limit; - private bool $max_allowed_packet; private string $header; private string $footer; private array $types_no_quotes = [ @@ -29,16 +29,30 @@ class DataBaseBackup { 'year', ]; - public function __construct(string $path, int $query_limit = 50000, int $insert_limit = 100, bool $max_allowed_packet = false, string $date_format = "Y-m-d_His"){ + public function __construct(string $path, int $query_limit = 50000, int $insert_limit = 100, string $date_format = "Y-m-d_His"){ $date = date($date_format); $this->query_limit = $query_limit; $this->insert_limit = $insert_limit; - $this->max_allowed_packet = $max_allowed_packet; $this->path = $path.DIRECTORY_SEPARATOR.$date; $this->header = base64_decode("U0VUIFNRTF9NT0RFID0gIk5PX0FVVE9fVkFMVUVfT05fWkVSTyI7ClNUQVJUIFRSQU5TQUNUSU9OOwpTRVQgdGltZV96b25lID0gIiswMDowMCI7CgovKiE0MDEwMSBTRVQgQE9MRF9DSEFSQUNURVJfU0VUX0NMSUVOVD1AQENIQVJBQ1RFUl9TRVRfQ0xJRU5UICovOwovKiE0MDEwMSBTRVQgQE9MRF9DSEFSQUNURVJfU0VUX1JFU1VMVFM9QEBDSEFSQUNURVJfU0VUX1JFU1VMVFMgKi87Ci8qITQwMTAxIFNFVCBAT0xEX0NPTExBVElPTl9DT05ORUNUSU9OPUBAQ09MTEFUSU9OX0NPTk5FQ1RJT04gKi87Ci8qITQwMTAxIFNFVCBOQU1FUyB1dGY4bWI0ICovOw=="); $this->footer = base64_decode("Q09NTUlUOwoKLyohNDAxMDEgU0VUIENIQVJBQ1RFUl9TRVRfQ0xJRU5UPUBPTERfQ0hBUkFDVEVSX1NFVF9DTElFTlQgKi87Ci8qITQwMTAxIFNFVCBDSEFSQUNURVJfU0VUX1JFU1VMVFM9QE9MRF9DSEFSQUNURVJfU0VUX1JFU1VMVFMgKi87Ci8qITQwMTAxIFNFVCBDT0xMQVRJT05fQ09OTkVDVElPTj1AT0xEX0NPTExBVElPTl9DT05ORUNUSU9OICovOw=="); } + public function getOutput() : string { + return $this->path; + } + + public function set_max_allowed_packet(int $value) : bool { + try { + $this->destination->query("SET GLOBAL `max_allowed_packet` = $value;"); + return true; + } + catch(PDOException $e){ + echo " ".$e->getMessage()."\r\n"; + return false; + } + } + public function connect(string $host, string $user, string $password, string $dbname, int $port = 3306) : bool { $options = [ PDO::ATTR_EMULATE_PREPARES => true, @@ -46,7 +60,7 @@ public function connect(string $host, string $user, string $password, string $db PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, ]; try { - $this->conn = new PDO("mysql:dbname=$dbname;host=$host;port=$port", $user, $password, $options); + $this->source = new PDO("mysql:dbname=$dbname;host=$host;port=$port", $user, $password, $options); } catch(PDOException $e){ echo " Failed to connect:\r\n"; @@ -57,17 +71,43 @@ public function connect(string $host, string $user, string $password, string $db return true; } + public function connect_destination(string $host, string $user, string $password, string $dbname, int $port = 3306) : bool { + $options = [ + PDO::ATTR_EMULATE_PREPARES => true, + PDO::MYSQL_ATTR_INIT_COMMAND => 'SET SESSION SQL_BIG_SELECTS=1;', + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + ]; + try { + $this->destination = new PDO("mysql:dbname=$dbname;host=$host;port=$port", $user, $password, $options); + } + catch(PDOException $e){ + echo " Failed to connect:\r\n"; + echo " ".$e->getMessage()."\r\n"; + return false; + } + return true; + } + public function disconnect() : void { - $this->conn = null; + $this->source = null; + } + + public function disconnect_destination() : void { + $this->destination = null; } public function escape(mixed $string) : string { return preg_replace('~[\x00\x0A\x0D\x1A\x22\x27\x5C]~u', '\\\$0', strval($string)); } + public function isDestinationEmpty() : bool { + $tables = $this->destination->query('SHOW TABLES', PDO::FETCH_OBJ); + return $tables->rowCount() == 0; + } + public function getTables() : array { $data = []; - $tables = $this->conn->query('SHOW TABLES', PDO::FETCH_OBJ); + $tables = $this->source->query('SHOW TABLES', PDO::FETCH_OBJ); foreach($tables as $table){ array_push($data, $table->{'Tables_in_'.$this->database}); } @@ -76,7 +116,7 @@ public function getTables() : array { public function getColumns(string $table) : array { $data = []; - $columns = $this->conn->query("SELECT COLUMN_NAME, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE `TABLE_SCHEMA` = '$this->database' AND `TABLE_NAME` = '$table'", PDO::FETCH_OBJ); + $columns = $this->source->query("SELECT COLUMN_NAME, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE `TABLE_SCHEMA` = '$this->database' AND `TABLE_NAME` = '$table'", PDO::FETCH_OBJ); foreach($columns as $column){ $data[$column->COLUMN_NAME] = strtolower($column->DATA_TYPE); } @@ -84,7 +124,7 @@ public function getColumns(string $table) : array { } public function getCreation(string $table) : string { - $creation = $this->conn->query("SHOW CREATE TABLE `$table`", PDO::FETCH_OBJ); + $creation = $this->source->query("SHOW CREATE TABLE `$table`", PDO::FETCH_OBJ); $data = $creation->fetch(PDO::FETCH_OBJ); return $data->{'Create Table'}.';'; } @@ -110,7 +150,7 @@ public function getInsert(string $table, array $columns) : string { return "INSERT INTO `$table` ($columns_string)"; } - public function makeForTable(string $table, bool $backup_structure = true, bool $backup_data = true) : void { + public function backupTable(string $table, bool $backup_structure = true, bool $backup_data = true) : void { if(!file_exists($this->path)) mkdir($this->path, 0777, true); $offset = 0; $file_path = $this->path.DIRECTORY_SEPARATOR."$table.sql"; @@ -123,18 +163,17 @@ public function makeForTable(string $table, bool $backup_structure = true, bool if($backup_structure){ fwrite($file, $this->getDrop($table)."\n\n"); fwrite($file, $this->getCreation($table)."\n\n"); - if($this->max_allowed_packet) fwrite($file, "SET GLOBAL `max_allowed_packet` = 524288000;\r\n\r\n"); } if($backup_data){ - $results = $this->conn->query("SELECT count(*) AS cnt FROM `$table`"); + $results = $this->source->query("SELECT count(*) AS cnt FROM `$table`"); $row = $results->fetch(PDO::FETCH_OBJ); $count = $row->cnt; if($count > 0){ while($offset < $count){ $percent = sprintf("%.02f", ($offset / $count) * 100.0); echo " Table: $table Progress: $percent % \r"; - $rows = $this->conn->query("SELECT * FROM `$table` LIMIT $offset, $this->query_limit", PDO::FETCH_OBJ); + $rows = $this->source->query("SELECT * FROM `$table` LIMIT $offset, $this->query_limit", PDO::FETCH_OBJ); $seek = 0; foreach($rows as $row){ if($seek == 0){ @@ -194,10 +233,89 @@ public function makeForTable(string $table, bool $backup_structure = true, bool fclose($file); } - public function makeForAll(bool $backup_structure = true, bool $backup_data = true) : void { + public function cloneTable(string $table) : void { + $offset = 0; + $columns = $this->getColumns($table); + + $this->destination->query($this->getHeader()); + $this->destination->query($this->getDrop($table)); + $this->destination->query($this->getCreation($table)); + + $results = $this->source->query("SELECT count(*) AS cnt FROM `$table`"); + $row = $results->fetch(PDO::FETCH_OBJ); + $count = $row->cnt; + if($count > 0){ + while($offset < $count){ + $percent = sprintf("%.02f", ($offset / $count) * 100.0); + echo " Table: $table Progress: $percent % \r"; + $rows = $this->source->query("SELECT * FROM `$table` LIMIT $offset, $this->query_limit", PDO::FETCH_OBJ); + $seek = 0; + foreach($rows as $row){ + if($seek == 0){ + $query = $this->getInsert($table, array_keys($columns))." VALUES\n"; + } + $values = []; + foreach($columns as $column => $type){ + if(is_null($row->$column)){ + $values[] = "NULL"; + } else if($type == 'bit'){ + if(empty($row->$column)){ + $values[] = "b'0'"; + } else { + $values[] = "b'".decbin(intval($row->$column))."'"; + } + } else if($type == 'blob' || $type == 'binary' || $type == 'longblob'){ + if(empty($row->$column)){ + $values[] = "''"; + } else { + $values[] = "0x".bin2hex($row->$column); + } + } else { + if(in_array($type, $this->types_no_quotes)){ + $values[] = $row->$column; + } else { + $values[] = "'".$this->escape($row->$column)."'"; + } + } + } + $query .= '('.implode(',',$values).'),'."\n"; + unset($values); + $seek++; + if($seek >= $this->insert_limit){ + $seek = 0; + $this->destination->query(substr($query, 0, -2).";"); + unset($query); + } + $offset++; + $percent = sprintf("%.02f", ($offset / $count) * 100.0); + echo " Table: $table Progress: $percent % \r"; + } + $percent = sprintf("%.02f", ($offset / $count) * 100.0); + echo " Table: $table Progress: $percent % \r"; + unset($rows); + } + if(isset($query)){ + $this->destination->query(substr($query, 0, -2).";"); + unset($query); + } + } else { + echo " Table: $table Progress: 100.00 % \r"; + } + + $this->destination->query($this->getFooter()); + } + + public function backupAll(bool $backup_structure = true, bool $backup_data = true) : void { + $tables = $this->getTables(); + foreach($tables as $table){ + $this->backupTable($table); + } + } + + public function cloneAll() : void { $tables = $this->getTables(); foreach($tables as $table){ - $this->makeForTable($table); + $this->cloneTable($table); } } diff --git a/includes/tools/MySQLTools.php b/includes/tools/MySQLTools.php index 2d26708..3ebf059 100644 --- a/includes/tools/MySQLTools.php +++ b/includes/tools/MySQLTools.php @@ -33,6 +33,7 @@ public function help() : void { ' 2 - Open config folder', ' 3 - Show connections', ' 4 - Make backup', + ' 5 - Clone DB1 to DB2 (overwrite)', ]); } @@ -45,6 +46,7 @@ public function action(string $action) : bool { case '2': return $this->ToolOpenConfigFolder(); case '3': return $this->ToolShowConnections(); case '4': return $this->ToolMakeBackup(); + case '5': return $this->ToolMakeClone(); } return false; } @@ -145,10 +147,10 @@ public function ToolConfigureConnection() : bool { $backup['data'] = strtoupper($this->ave->get_input()); if(!in_array($backup['data'][0] ?? '?', ['Y', 'N'])) goto set_backup_data; - set_max_allowed_packet: - echo " Put SET max_allowed_packet before insert query (Y/N): "; - $backup['max_allowed_packet'] = strtoupper($this->ave->get_input()); - if(!in_array($backup['max_allowed_packet'][0] ?? '?', ['Y', 'N'])) goto set_max_allowed_packet; + set_backup_compress: + echo " Compress after backup (Y/N): "; + $backup['compress'] = strtoupper($this->ave->get_input()); + if(!in_array($backup['compress'][0] ?? '?', ['Y', 'N'])) goto set_backup_compress; $ini = $this->getConfig($label); $ini->update([ @@ -160,9 +162,9 @@ public function ToolConfigureConnection() : bool { 'FOLDER_DATE_FORMAT' => "Y-m-d_His", 'BACKUP_QUERY_LIMIT' => 50000, 'BACKUP_INSERT_LIMIT' => 100, - 'SET_MAX_ALLOWED_PACKET' => $backup['max_allowed_packet'] == 'Y', 'BACKUP_TYPE_STRUCTURE' => $backup['structure'][0] == 'Y', 'BACKUP_TYPE_DATA' => $backup['data'][0] == 'Y', + 'BACKUP_COMPRESS' => $backup['compress'][0] == 'Y', 'BACKUP_PATH' => $output, ], true); @@ -242,11 +244,16 @@ public function ToolMakeBackup() : bool { } $ini = $this->getConfig($label); + $path = $ini->get('BACKUP_PATH').DIRECTORY_SEPARATOR.$label; + + if(!$this->ave->is_valid_device($path)){ + echo " Output device \"$path\" is not available.\r\n"; + goto set_label; + } $this->ave->write_log("Initialize backup for \"$label\""); echo " Initialize backup service\r\n"; - $path = $ini->get('BACKUP_PATH').DIRECTORY_SEPARATOR.$label; - $backup = new DataBaseBackup($path, $ini->get('BACKUP_QUERY_LIMIT'), $ini->get('BACKUP_INSERT_LIMIT'), $ini->get('SET_MAX_ALLOWED_PACKET'), $ini->get('FOLDER_DATE_FORMAT')); + $backup = new DataBaseBackup($path, $ini->get('BACKUP_QUERY_LIMIT'), $ini->get('BACKUP_INSERT_LIMIT'), $ini->get('FOLDER_DATE_FORMAT')); echo " Connecting to: ".$ini->get('DB_HOST').':'.$ini->get('DB_PORT').'@'.$ini->get('DB_USER')."\r\n"; if(!$backup->connect($ini->get('DB_HOST'), $ini->get('DB_USER'), $ini->get('DB_PASSWORD'), $ini->get('DB_NAME'), $ini->get('DB_PORT'))) goto set_label; @@ -259,19 +266,136 @@ public function ToolMakeBackup() : bool { foreach($tables as $table){ $progress++; $this->ave->write_log("Create backup for table $table"); - $backup->makeForTable($table, $ini->get('BACKUP_TYPE_STRUCTURE'), $ini->get('BACKUP_TYPE_DATA')); + $backup->backupTable($table, $ini->get('BACKUP_TYPE_STRUCTURE'), $ini->get('BACKUP_TYPE_DATA')); echo "\n"; $this->ave->set_progress_ex('Tables', $progress, $total); } echo "\n"; $this->ave->write_log("Finish backup for \"$label\""); + $backup->disconnect(); + + $output = $backup->getOutput(); + $cl = $this->ave->config->get('AVE_BACKUP_COMPRESS_LEVEL'); + $at = $this->ave->config->get('AVE_BACKUP_COMPRESS_TYPE'); + if($ini->get('BACKUP_COMPRESS', false)){ + echo " Compressing backup\r\n"; + $this->ave->write_log("Compress backup"); + $sql = $output.DIRECTORY_SEPARATOR."*.sql"; + system("7z a -mx$cl -t$at \"$output.7z\" \"$sql\""); + echo "\r\n"; + if(file_exists("$output.7z")){ + echo " Compress backup into \"$output.sql\" success\r\n"; + $this->ave->write_log("Compress backup into \"$output.sql\" success"); + foreach($tables as $table){ + $this->ave->unlink($output.DIRECTORY_SEPARATOR."$table.sql"); + } + $this->ave->rmdir($output); + $this->ave->open_file($ini->get('BACKUP_PATH')); + } else { + echo " Compress backup into \"$output.sql\" fail\r\n"; + $this->ave->write_log("Compress backup into \"$output.sql\" fail"); + $this->ave->open_file($output); + } + } else { + $this->ave->open_file($output); + } $this->ave->open_logs(true); - $this->ave->open_file($path); $this->ave->pause(" Backup for \"$label\" done, press enter to back to menu"); return false; } + public function ToolMakeClone() : bool { + $this->ave->clear(); + $this->ave->set_subtool("MakeClone"); + + set_label_source: + echo " Source label: "; + $source = $this->ave->get_input(); + if($source == '#') return false; + if(!preg_match('/(?=[a-zA-Z0-9_\-]{3,20}$)/i', $source)) goto set_label_source; + + if(!file_exists($this->getConfigPath($source))){ + echo " Source label \"$source\" not exists.\r\n"; + goto set_label_source; + } + + $ini_source = $this->getConfig($source); + $path = $ini_source->get('BACKUP_PATH').DIRECTORY_SEPARATOR.$source; + + $this->ave->write_log("Initialize backup for \"$source\""); + echo " Initialize backup service\r\n"; + $backup = new DataBaseBackup($path, $ini_source->get('BACKUP_QUERY_LIMIT'), $ini_source->get('BACKUP_INSERT_LIMIT'), $ini_source->get('FOLDER_DATE_FORMAT')); + + echo " Connecting to: ".$ini_source->get('DB_HOST').':'.$ini_source->get('DB_PORT').'@'.$ini_source->get('DB_USER')."\r\n"; + if(!$backup->connect($ini_source->get('DB_HOST'), $ini_source->get('DB_USER'), $ini_source->get('DB_PASSWORD'), $ini_source->get('DB_NAME'), $ini_source->get('DB_PORT'))) goto set_label_source; + + set_label_destination: + echo " Destination label: "; + $destination = $this->ave->get_input(); + if($destination == '#') return false; + if(!preg_match('/(?=[a-zA-Z0-9_\-]{3,20}$)/i', $destination)) goto set_label_destination; + + if(!file_exists($this->getConfigPath($destination))){ + echo " Destination label \"$destination\" not exists.\r\n"; + goto set_label_destination; + } + + if($source == $destination){ + echo " Destination label must be different than source label.\r\n"; + goto set_label_destination; + } + + $ini_dest = $this->getConfig($destination); + + if($ini_source->get('DB_HOST') == $ini_dest->get('DB_HOST') && $ini_source->get('DB_USER') == $ini_dest->get('DB_USER') && $ini_source->get('DB_NAME') == $ini_dest->get('DB_NAME') && $ini_source->get('DB_PORT') == $ini_dest->get('DB_PORT')){ + echo " Destination database is same as source database.\r\n"; + goto set_label_destination; + } + + echo " Connecting to: ".$ini_dest->get('DB_HOST').':'.$ini_dest->get('DB_PORT').'@'.$ini_dest->get('DB_USER')."\r\n"; + if(!$backup->connect_destination($ini_dest->get('DB_HOST'), $ini_dest->get('DB_USER'), $ini_dest->get('DB_PASSWORD'), $ini_dest->get('DB_NAME'), $ini_dest->get('DB_PORT'))) goto set_label_destination; + + if(!$backup->isDestinationEmpty()){ + echo " Output database is not empty, continue (Y/N): "; + $confirmation = strtoupper($this->ave->get_input()); + if($confirmation != 'Y'){ + $this->ave->pause(" Clone \"$source\" to \"$destination\" aborted, press enter to back to menu"); + return false; + } + } + + $v = $this->ave->config->get('AVE_BACKUP_MAX_ALLOWED_PACKET'); + echo " Try call SET GLOBAL `max_allowed_packet` = $v; (Y/N): "; + $confirmation = strtoupper($this->ave->get_input()); + if($confirmation == 'Y'){ + if(!$backup->set_max_allowed_packet($v)){ + echo "SET GLOBAL `max_allowed_packet` = $v; fail, continue\r\n\r\n"; + } + } + + echo " Clone \"$source\" to \"$destination\"\r\n\r\n"; + $tables = $backup->getTables(); + $progress = 0; + $total = count($tables); + $this->ave->set_progress_ex('Tables', $progress, $total); + foreach($tables as $table){ + $progress++; + $this->ave->write_log("Clone table $table"); + $backup->cloneTable($table); + echo "\n"; + $this->ave->set_progress_ex('Tables', $progress, $total); + } + echo "\n"; + $this->ave->write_log("Finish clone \"$source\" to \"$destination\""); + $backup->disconnect(); + $backup->disconnect_destination(); + + $this->ave->open_logs(true); + $this->ave->pause(" Clone for \"$source\" to \"$destination\" done, press enter to back to menu"); + return false; + } + } ?> diff --git a/version b/version index e21e727..13175fd 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.4.0 \ No newline at end of file +1.4.1 \ No newline at end of file