|
| 1 | +#!/usr/bin/env bash |
| 2 | +# |
| 3 | +# Copyright 2022 WeBank |
| 4 | +# |
| 5 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | +# you may not use this file except in compliance with the License. |
| 7 | +# You may obtain a copy of the License at |
| 8 | +# |
| 9 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +# |
| 11 | +# Unless required by applicable law or agreed to in writing, software |
| 12 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | +# See the License for the specific language governing permissions and |
| 15 | +# limitations under the License. |
| 16 | +# |
| 17 | + |
| 18 | +# Use to upgrade from 0.1.0 to 0.2.0 |
| 19 | + |
| 20 | +if [ -f "~/.bashrc" ];then |
| 21 | + echo "Warning! user bashrc file does not exist." |
| 22 | +else |
| 23 | + source ~/.bashrc |
| 24 | +fi |
| 25 | + |
| 26 | +shellDir=`dirname $0` |
| 27 | +workDir=`cd ${shellDir}/..;pwd` |
| 28 | + |
| 29 | +interact_echo(){ |
| 30 | + while [ 1 ]; do |
| 31 | + read -p "$1 (Y/N)" yn |
| 32 | + if [[ "${yn}x" == "Yx" ]] || [[ "${yn}x" == "yx" ]]; then |
| 33 | + return 0 |
| 34 | + elif [[ "${yn}x" == "Nx" ]] || [[ "${yn}x" == "nx" ]]; then |
| 35 | + return 1 |
| 36 | + else |
| 37 | + echo "Unknown choose: [$yn], please choose again." |
| 38 | + fi |
| 39 | + done |
| 40 | +} |
| 41 | + |
| 42 | +interact_echo "Are you sure the current version of Streamis is 0.1.0 and need to upgrade to 0.2.0 ?" |
| 43 | +if [[ $? == 0 ]]; then |
| 44 | + source ${workDir}/conf/db.sh |
| 45 | + echo "<------ Will connect to [${MYSQL_HOST}:${MYSQL_PORT}] to upgrade the tables in database... ------>" |
| 46 | + mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 << EOF 1>/dev/null |
| 47 | + /*Modify the table column*/ |
| 48 | + ALTER TABLE \`linkis_stream_job\` MODIFY COLUMN \`project_name\` varchar(100) DEFAULT NULL; |
| 49 | + ALTER TABLE \`linkis_stream_job\` MODIFY COLUMN \`name\` varchar(200) DEFAULT NULL; |
| 50 | + ALTER TABLE \`linkis_stream_project\` MODIFY COLUMN \`name\` varchar(100) DEFAULT NULL; |
| 51 | + ALTER TABLE \`linkis_stream_task\` MODIFY COLUMN \`job_id\` varchar(200) DEFAULT NULL; |
| 52 | + ALTER TABLE \`linkis_stream_task\` MODIFY COLUMN \`linkis_job_id\` varchar(200) DEFAULT NULL; |
| 53 | +
|
| 54 | + ALTER TABLE \`linkis_stream_project\` ADD create_time datetime DEFAULT NULL; |
| 55 | + ALTER TABLE \`linkis_stream_project\` ADD last_update_by varchar(50) DEFAULT NULL; |
| 56 | + ALTER TABLE \`linkis_stream_project\` ADD last_update_time datetime DEFAULT NULL; |
| 57 | + ALTER TABLE \`linkis_stream_project\` ADD is_deleted tinyint unsigned DEFAULT 0; |
| 58 | +
|
| 59 | + /*Add indexes into the tables*/ |
| 60 | + ALTER TABLE \`linkis_stream_job\` ADD UNIQUE KEY(\`project_name\`, \`name\`); |
| 61 | + ALTER TABLE \`linkis_stream_job_version\` ADD UNIQUE KEY(\`job_id\`, \`version\`); |
| 62 | +
|
| 63 | + /*Add new tables*/ |
| 64 | + DROP TABLE IF EXISTS \`linkis_stream_project_privilege\`; |
| 65 | + CREATE TABLE \`linkis_stream_project_privilege\` ( |
| 66 | + \`id\` bigint(20) NOT NULL AUTO_INCREMENT, |
| 67 | + \`project_id\` bigint(20) NOT NULL, |
| 68 | + \`user_name\` varchar(100) NOT NULL, |
| 69 | + \`privilege\` tinyint(1) DEFAULT '0' NOT NULL COMMENT '1:发布权限 ，2:编辑权限 ，3:查看权限', |
| 70 | + PRIMARY KEY (\`id\`) USING BTREE |
| 71 | + ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='项目权限表'; |
| 72 | +
|
| 73 | + DROP TABLE IF EXISTS \`linkis_stream_job_config_def\`; |
| 74 | + CREATE TABLE \`linkis_stream_job_config_def\` ( |
| 75 | + \`id\` bigint(20) NOT NULL AUTO_INCREMENT, |
| 76 | + \`key\` varchar(100) COLLATE utf8_bin NOT NULL, |
| 77 | + \`name\` varchar(100) COLLATE utf8_bin DEFAULT NULL COMMENT 'Equals option', |
| 78 | + \`type\` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT 'NONE' COMMENT 'def type, NONE: 0, INPUT: 1, SELECT: 2', |
| 79 | + \`sort\` int(10) DEFAULT '0' COMMENT 'In order to sort the configurations that have the same level', |
| 80 | + \`description\` varchar(200) COLLATE utf8_bin DEFAULT NULL COMMENT 'Description of configuration', |
| 81 | + \`validate_type\` varchar(50) COLLATE utf8_bin DEFAULT NULL COMMENT 'Method the validate the configuration', |
| 82 | + \`validate_rule\` varchar(100) COLLATE utf8_bin DEFAULT NULL COMMENT 'Value of validation rule', |
| 83 | + \`style\` varchar(200) COLLATE utf8_bin DEFAULT '' COMMENT 'Display style', |
| 84 | + \`visiable\` tinyint(1) NOT NULL DEFAULT '1' COMMENT '0: hidden, 1: display', |
| 85 | + \`level\` tinyint(1) NOT NULL DEFAULT '1' COMMENT '0: root, 1: leaf', |
| 86 | + \`unit\` varchar(25) COLLATE utf8_bin DEFAULT NULL COMMENT 'Unit symbol', |
| 87 | + \`default_value\` varchar(200) COLLATE utf8_bin DEFAULT NULL COMMENT 'Default value', |
| 88 | + \`ref_values\` varchar(200) COLLATE utf8_bin DEFAULT '', |
| 89 | + \`parent_ref\` bigint(20) DEFAULT NULL COMMENT 'Parent key of configuration def', |
| 90 | + \`required\` tinyint(1) NOT NULL DEFAULT '0' COMMENT 'If the value of configuration is necessary', |
| 91 | + \`is_temp\` tinyint(1) DEFAULT '0' COMMENT 'Temp configuration', |
| 92 | + PRIMARY KEY (\`id\`), |
| 93 | + UNIQUE KEY \`config_def_key\` (\`key\`) |
| 94 | + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; |
| 95 | +
|
| 96 | + DROP TABLE IF EXISTS \`linkis_stream_job_config\`; |
| 97 | + CREATE TABLE \`linkis_stream_job_config\` ( |
| 98 | + \`job_id\` bigint(20) NOT NULL, |
| 99 | + \`job_name\` varchar(200) COLLATE utf8_bin NOT NULL COMMENT 'Just store the job name', |
| 100 | + \`key\` varchar(100) COLLATE utf8_bin NOT NULL, |
| 101 | + \`value\` varchar(500) COLLATE utf8_bin NOT NULL, |
| 102 | + \`ref_def_id\` bigint(20) DEFAULT NULL COMMENT 'Refer to id in config_def table', |
| 103 | + PRIMARY KEY (\`job_id\`,\`key\`), |
| 104 | + KEY \`config_def_id\` (\`ref_def_id\`) |
| 105 | + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; |
| 106 | +
|
| 107 | + /*Execute dml*/ |
| 108 | + source ${workDir}/db/streamis_dml.sql |
| 109 | +
|
| 110 | + /*Data migration*/ |
| 111 | + INSERT INTO \`linkis_stream_job_config\`(\`key\`, \`value\`, \`job_id\`, \`job_name\`, \`ref_def_id\`) SELECT ov.config_key, ov.config_value, ov.job_id, ov.job_name, d.id as refer_id from linkis_stream_configuration_config_value ov left join linkis_stream_job_config_def d on ov.config_key = d.key WHERE ov.config_value IS NOT NULL AND ov.job_name IS NOT NULL GROUP BY ov.job_id,ov.config_key; |
| 112 | + UPDATE linkis_stream_job_config SET \`key\` = "wds.linkis.flink.taskmanager.memory" WHERE \`key\` = "flink.taskmanager.memory"; |
| 113 | + UPDATE linkis_stream_job_config SET \`key\` = "wds.linkis.flink.taskmanager.cpus" WHERE \`key\` = "flink.taskmanager.cpu.cores"; |
| 114 | + UPDATE linkis_stream_job_config SET \`key\` = "wds.linkis.flink.taskmanager.cpus" WHERE \`key\` = "wds.linkis.flink.taskManager.cpus"; |
| 115 | + UPDATE linkis_stream_job_config SET \`key\` = "wds.linkis.flink.taskmanager.numberOfTaskSlots" WHERE \`key\` = "flink.taskmanager.numberOfTaskSlots"; |
| 116 | + UPDATE linkis_stream_job_config SET \`key\` = "wds.linkis.flink.app.parallelism" WHERE \`key\` = "wds.linkis.engineconn.flink.app.parallelism"; |
| 117 | + UPDATE linkis_stream_job_config SET \`key\` = "wds.linkis.flink.jobmanager.memory" WHERE \`key\` = "flink.jobmanager.memory"; |
| 118 | + UPDATE linkis_stream_job_config c SET \`ref_def_id\` = (SELECT d.id FROM linkis_stream_job_config_def d WHERE d.\`key\` = c.\`key\`) WHERE c.ref_def_id IS NULL; |
| 119 | + SELECT @flink_extra_param_id:=id FROM linkis_stream_job_config_def WHERE \`key\` = "wds.linkis.flink.custom"; |
| 120 | + UPDATE linkis_stream_job_config SET ref_def_id = @flink_extra_param_id WHERE ref_def_id IS NULL; |
| 121 | +
|
| 122 | + /*Drop tables*/ |
| 123 | + /*DROP TABLE \`linkis_stream_configuration_config_key\`*/ |
| 124 | + /*DROP TABLE \`linkis_stream_configuration_config_value\`*/ |
| 125 | +
|
| 126 | + /*update tables data*/ |
| 127 | + delimiter %% |
| 128 | +
|
| 129 | + create procedure update_project() |
| 130 | + BEGIN |
| 131 | + -- 声明变量 |
| 132 | + DECLARE projectname varchar(50); |
| 133 | + DECLARE done INT default 0; |
| 134 | +
|
| 135 | + -- 创建游标，并设置游标所指的数据 |
| 136 | + DECLARE cur CURSOR for |
| 137 | + SELECT distinct j.project_name from linkis_stream_job j; |
| 138 | + -- 游标执行完，即遍历结束。设置done的值为1 |
| 139 | + DECLARE CONTINUE HANDLER for not FOUND set done = 1; |
| 140 | + -- 开启游标 |
| 141 | + open cur; |
| 142 | + -- 执行循环 |
| 143 | + posLoop: |
| 144 | + LOOP |
| 145 | + -- 从游标中取出projectname |
| 146 | + FETCH cur INTO projectname ; |
| 147 | + -- 如果done的值为1，即遍历结束，结束循环 |
| 148 | + IF done = 1 THEN |
| 149 | + LEAVE posLoop; |
| 150 | + -- 注意，if语句需要添加END IF结束IF |
| 151 | + END IF; |
| 152 | + insert into linkis_stream_project(\`name\`,\`create_by\`,\`create_time\`) values (projectname,\'system\',now()); |
| 153 | + -- 关闭循环 |
| 154 | + END LOOP posLoop; |
| 155 | + -- 关闭游标 |
| 156 | + CLOSE cur; |
| 157 | + -- 关闭分隔标记 |
| 158 | + END %% |
| 159 | +
|
| 160 | + create procedure update_project_privilege() |
| 161 | + BEGIN |
| 162 | + -- 声明变量 |
| 163 | + DECLARE projectid bigint(20); |
| 164 | + DECLARE create_by varchar(50); |
| 165 | + DECLARE done INT default 0; |
| 166 | +
|
| 167 | + -- 创建游标，并设置游标所指的数据 |
| 168 | + DECLARE cur CURSOR for |
| 169 | + SELECT distinct p.id,j.create_by from linkis_stream_project p,linkis_stream_job j where p.name =j.project_name ; |
| 170 | + -- 游标执行完，即遍历结束。设置done的值为1 |
| 171 | + DECLARE CONTINUE HANDLER for not FOUND set done = 1; |
| 172 | + -- 开启游标 |
| 173 | + open cur; |
| 174 | + -- 执行循环 |
| 175 | + posLoop: |
| 176 | + LOOP |
| 177 | + -- 从游标中取出id |
| 178 | + FETCH cur INTO projectid ,create_by; |
| 179 | + -- 如果done的值为1，即遍历结束，结束循环 |
| 180 | + IF done = 1 THEN |
| 181 | + LEAVE posLoop; |
| 182 | + -- 注意，if语句需要添加END IF结束IF |
| 183 | + END IF; |
| 184 | +
|
| 185 | + insert into linkis_stream_project_privilege (project_id ,user_name ,privilege) values (projectid,create_by,2); |
| 186 | + -- 关闭循环 |
| 187 | + END LOOP posLoop; |
| 188 | + -- 关闭游标 |
| 189 | + CLOSE cur; |
| 190 | + -- 关闭分隔标记 |
| 191 | + END %% |
| 192 | + delimiter ; |
| 193 | +
|
| 194 | + call update_project; |
| 195 | + call update_project_privilege; |
| 196 | +
|
| 197 | + drop PROCEDURE update_project; |
| 198 | + drop PROCEDURE update_project_privilege; |
| 199 | +
|
| 200 | +EOF |
| 201 | + echo "<------ End to upgrade ------>" |
| 202 | +fi |
| 203 | + |
| 204 | + |
| 205 | + |
0 commit comments