V2.10.0 (#205)

Author: frf12

_cmd.py

@@ -30,7 +30,7 @@
 from optparse import OptionParser, BadOptionError, Option, IndentedHelpFormatter
 
 from core import ObdHome
-from _stdio import IO, FormtatText
+from _stdio import IO, FormatText
 from _lock import LockMode
 from _types import Capacity
 from tool import DirectoryUtil, FileUtil, NetUtil, COMMAND_ENV
@@ -226,17 +226,22 @@ def parse_command(self):
             self.parser.allow_undefine = self.dev_mode
         return super(ObdCommand, self).parse_command()
 
+    def _init_log(self):
+        trace_id = uuid()
+        log_dir = os.path.join(self.OBD_PATH, 'log')
+        DirectoryUtil.mkdir(log_dir)
+        log_path = os.path.join(log_dir, 'obd')
+        ROOT_IO.init_trace_logger(log_path, 'obd', trace_id)
+        ROOT_IO.exit_msg = '''Trace ID: {trace_id}
+If you want to view detailed obd logs, please run: obd display-trace {trace_id}'''.format(trace_id=trace_id)
+
     def do_command(self):
         self.parse_command()
         self.init_home()
-        trace_id = uuid()
         ret = False
         try:
-            log_dir = os.path.join(self.OBD_PATH, 'log')
-            DirectoryUtil.mkdir(log_dir)
-            log_path = os.path.join(log_dir, 'obd')
-            if self.enable_log:
-                ROOT_IO.init_trace_logger(log_path, 'obd', trace_id)
+            if self.has_trace and self.enable_log:
+                self._init_log()
             ROOT_IO.track_limit += 1
             ROOT_IO.verbose('cmd: %s' % self.cmds)
             ROOT_IO.verbose('opts: %s' % self.opts)
@@ -245,7 +250,7 @@ def do_command(self):
             obd.set_cmds(self.cmds)
             ret = self._do_command(obd)
             if not ret:
-                ROOT_IO.print(DOC_LINK_MSG)
+                ROOT_IO.exit_msg = DOC_LINK_MSG + "\n" + ROOT_IO.exit_msg
         except NotImplementedError:
             ROOT_IO.exception('command \'%s\' is not implemented' % self.prev_cmd)
         except LockError:
@@ -257,9 +262,6 @@ def do_command(self):
         except:
             e = sys.exc_info()[1]
             ROOT_IO.exception('Running Error: %s' % e)
-        if self.has_trace:
-            ROOT_IO.print('Trace ID: %s' % trace_id)
-            ROOT_IO.print('If you want to view detailed obd logs, please run: obd display-trace %s' % trace_id)
         return ret
 
     def _do_command(self, obd):
@@ -859,7 +861,7 @@ def _do_command(self, obd):
             res = obd.deploy_cluster(self.cmds[0])
             self.background_telemetry_task(obd)
             if res:
-                obd.stdio.print(FormtatText.success('Please execute ` obd cluster start %s ` to start' % self.cmds[0]))
+                obd.stdio.print(FormatText.success('Please execute ` obd cluster start %s ` to start' % self.cmds[0]))
             return res
         else:
             return self._show_help()
@@ -900,6 +902,7 @@ class ClusterComponentDeleteCommand(ClusterMirrorCommand):
 
     def __init__(self):
         super(ClusterComponentDeleteCommand, self).__init__('del', 'Add components for cluster')
+        self.parser.add_option('-f', '--force', action='store_true', help="Force delete components.", default=False)
         self.parser.add_option('--ignore-standby', '--igs', action='store_true', help="Force kill the observer while standby tenant in others cluster exists.")
 
     def init(self, cmd, args):
@@ -1129,6 +1132,7 @@ def __init__(self):
         self.parser.add_option('--tablegroup', type='string', help="Tenant tablegroup.")
         self.parser.add_option('--primary-zone', type='string', help="Tenant primary zone. [RANDOM].", default='RANDOM')
         self.parser.add_option('--locality', type='string', help="Tenant locality.")
+        self.parser.add_option('--time-zone', type='string', help="Tenant time zone. The default tenant time_zone is [+08:00].")
         self.parser.add_option('-s', '--variables', type='string', help="Set the variables for the system tenant. [ob_tcp_invited_nodes='%'].", default="ob_tcp_invited_nodes='%'")
         self.parser.add_option('-o', '--optimize', type='string', help="Specify scenario optimization when creating a tenant, the default is consistent with the cluster dimension.\n{express_oltp, complex_oltp, olap, htap, kv}\nSupported since version 4.3.")
 
@@ -1406,7 +1410,7 @@ def __init__(self):
         self.parser.add_option('--sysbench-script-dir', type='string', help='The directory of the sysbench lua script file. [/usr/sysbench/share/sysbench]', default='/usr/sysbench/share/sysbench')
         self.parser.add_option('--table-size', type='int', help='Number of data initialized per table. [20000]', default=20000)
         self.parser.add_option('--tables', type='int', help='Number of initialization tables. [30]', default=30)
-        self.parser.add_option('--threads', type='int', help='Number of threads to use. [16]', default=16)
+        self.parser.add_option('--threads', type='string', help='Number of threads to use. [16]', default='16')
         self.parser.add_option('--time', type='int', help='Limit for total execution time in seconds. [60]', default=60)
         self.parser.add_option('--interval', type='int', help='Periodically report intermediate statistics with a specified time interval in seconds. 0 disables intermediate reports. [10]', default=10)
         self.parser.add_option('--events', type='int', help='Limit for total number of events.')
@@ -1447,6 +1451,8 @@ def __init__(self):
         self.parser.add_option('-O', '--optimization', type='int', help='Optimization level {0/1/2}. [1] 0 - No optimization. 1 - Optimize some of the parameters which do not need to restart servers. 2 - Optimize all the parameters and maybe RESTART SERVERS for better performance.', default=1)
         self.parser.add_option('--test-only', action='store_true', help='Only testing SQLs are executed. No initialization is executed.')
         self.parser.add_option('-S', '--skip-cluster-status-check', action='store_true', help='Skip cluster status check', default=False)
+        self.parser.add_option('--direct-load', action='store_true', help="Enable load data by direct feature.")
+        self.parser.add_option('--parallel', type='int', help='The degree of parallelism for loading data. [max_cpu * unit_count]')
 
     def _do_command(self, obd):
         if self.cmds:
@@ -1605,9 +1611,10 @@ def __init__(self):
         super(CommandsCommand, self).__init__('command', 'Common tool commands')
         self.parser.add_option('-c', '--components', type='string', help='The components used by the command. The first component in the configuration will be used by default in interactive commands, and all available components will be used by default in non-interactive commands.')
         self.parser.add_option('-s', '--servers', type='string', help='The servers used by the command. The first server in the configuration will be used by default in interactive commands, and all available servers will be used by default in non-interactive commands.')
+        self.parser.undefine_warn = False
 
     def _do_command(self, obd):
-        if len(self.cmds) == 2:
+        if len(self.cmds) in [2, 3]:
             return obd.commands(self.cmds[0], self.cmds[1], self.opts)
         else:
             return self._show_help()

_deploy.py

@@ -28,6 +28,7 @@
 import hashlib
 from copy import deepcopy
 from enum import Enum
+from datetime import datetime
 
 from ruamel.yaml.comments import CommentedMap
 
@@ -628,9 +629,15 @@ def del_depend(self, name, component_name):
 
     def get_depend_servers(self, name):
         if name not in self._depends:
-            return None
+            return []
         cluster_config = self._depends[name]
         return deepcopy(cluster_config.original_servers)
+    
+    def get_be_depend_servers(self, name):
+        if name not in self._be_depends:
+            return []
+        cluster_config = self._be_depends[name]
+        return deepcopy(cluster_config.original_servers)
 
     def get_depend_added_servers(self, name):
         if name not in self._depends:
@@ -1033,11 +1040,12 @@ class  DeployInstallMode(object):
 
 class DeployInfo(object):
 
-    def __init__(self, name, status, components=None, config_status=DeployConfigStatus.UNCHNAGE):
+    def __init__(self, name, status, components=None, config_status=DeployConfigStatus.UNCHNAGE, create_date=None):
         self.status = status
         self.name = name
         self.components = components if components else {}
         self.config_status = config_status
+        self.create_date = create_date
 
     def __str__(self):
         info = ['%s (%s)' % (self.name, self.status.value)]
@@ -1218,19 +1226,23 @@ def scale_out(self, config_path):
                 source_data = self.yaml_loader.load(f)
                 for key in source_data:
                     if key in ['user', 'unuse_lib_repository', 'auto_create_tenant']:
-                        self.stdio.error(err.EC_COMPONENT_CHANGE_CONFIG.format(key))
+                        self.stdio.error(err.EC_COMPONENT_CHANGE_CONFIG.format(message=key))
                         ret = False
                     elif issubclass(type(source_data[key]), dict):
                         new_depends = source_data[key].get('depends', [])
+                        if key not in self.components:
+                            self.stdio.error(err.EC_COMPONENT_NOT_EXISTS.format(component=key))
+                            ret = False
+                            break
                         if new_depends and new_depends != self.components[key].depends:
                             self.stdio.error(err.EC_COMPONENT_CHANGE_CONFIG.format(message='depends:{}'.format(key)))
                             ret = False
                         # temp _depends
                         depends[key] = self.components[key].depends
-                    
+
                     if not self._merge_component(key, source_data[key]):
                         ret = False
-                    
+
                 for comp in depends:
                     conf = self.components[comp]
                     for name in depends[comp]:
@@ -1243,7 +1255,7 @@ def scale_out(self, config_path):
             ret = False
         return ret
 
-    def add_components(self, config_path):
+    def add_components(self, config_path, ignore_exist=False):
         ret = True
         depends = {}
         try:
@@ -1255,6 +1267,8 @@ def add_components(self, config_path):
                         ret = False
                     elif issubclass(type(source_data[key]), dict):
                         if key in self.components:
+                            if ignore_exist:
+                                continue
                             self.stdio.error(err.EC_COMPONENT_EXISTS.format(component=key))
                             ret = False
                             continue
@@ -1578,6 +1592,7 @@ def deploy_info(self):
                         getattr(DeployStatus, data['status'], DeployStatus.STATUS_CONFIGURED),
                         ConfigUtil.get_value_from_dict(data, 'components', OrderedDict()),
                         getattr(DeployConfigStatus, ConfigUtil.get_value_from_dict(data, 'config_status', '_'), DeployConfigStatus.UNCHNAGE),
+                        ConfigUtil.get_value_from_dict(data, 'create_date', None),
                     )
             except:
                 self._info = DeployInfo(self.name, DeployStatus.STATUS_CONFIGURED)
@@ -1654,6 +1669,7 @@ def dump_deploy_info(self):
                     'components': self.deploy_info.components,
                     'status': self.deploy_info.status.name,
                     'config_status': self.deploy_info.config_status.name,
+                    'create_date': self.deploy_info.create_date,
                 }
                 yaml.dump(data, f)
             return True
@@ -1664,6 +1680,8 @@ def dump_deploy_info(self):
     def _update_deploy_status(self, status):
         old = self.deploy_info.status
         self.deploy_info.status = status
+        if old == DeployStatus.STATUS_DEPLOYED and status == DeployStatus.STATUS_RUNNING:
+            self.deploy_info.create_date = datetime.now().strftime('%Y-%m-%d')
         if self.dump_deploy_info():
             return True
         self.deploy_info.status = old

_errno.py

@@ -139,6 +139,8 @@ class InitDirFailedErrorMessage(object):
 EC_COMPONENT_FAILED_TO_MERGE_CONFIG = OBDErrorCodeTemplate(1023, 'Failed to merge config: {message}')
 EC_COMPONENT_NO_REMAINING_COMPS = OBDErrorCodeTemplate(1024, 'The cluster will have no remaining components. If you are absolutely sure about DELETING ALL COMPONENTS, please use "obd cluster destroy <deploy>" command to completely destroy the cluster')
 EC_COMPONENT_PASSWD_ERROR = OBDErrorCodeTemplate(1025, '({ip}) {component} {key} invalid. (Rule: {rule})')
+EC_RUNNING_CLUSTER_NO_REDEPLOYED = OBDErrorCodeTemplate(1026, 'Could not modify {key} when the cluster is in the working status(`production_mode` is True, not support this operation).')
+EC_COMPONENT_DIR_NOT_EMPTY = OBDErrorCodeTemplate(1027, 'If you are sure the directory can be emptied, run `obd cluster deploy -f {deploy_name}` to perform forced deployment.')
 
 WC_ULIMIT_CHECK = OBDErrorCodeTemplate(1007, '({server}) The recommended number of {key} is {need} (Current value: {now})')
 WC_AIO_NOT_ENOUGH = OBDErrorCodeTemplate(1011, '({ip}) The recommended value of fs.aio-max-nr is 1048576 (Current value: {current})')
@@ -160,6 +162,8 @@ class InitDirFailedErrorMessage(object):
 EC_OBSERVER_FAILED_TO_REGISTER_WITH_DETAILS = OBDErrorCodeTemplate(2005, 'Failed to register cluster. {appname} may have been registered in {obconfig_url}.')
 EC_OBSERVER_MULTI_NET_DEVICE = OBDErrorCodeTemplate(2006, '{ip} has more than one network interface. Please set `devname` for ({server})')
 EC_OBSERVER_PING_FAILED = OBDErrorCodeTemplate(2007, '{ip1} {devname} fail to ping {ip2}. Please check configuration `devname`')
+EC_OBSERVER_PING_NOT_FOUND = OBDErrorCodeTemplate(2007, '/usr/bin/ping: No such file or directory. You can run `sudo yum install iputils` or `sudo apt-get install iputils-ping`.')
+EC_OBSERVER_PING_FAILED_SUID = OBDErrorCodeTemplate(2007, 'If the error message `operation not permitted` appears, please check the ping file permissions. You can try running `sudo chmod u+s /usr/bin/ping`')
 EC_OBSERVER_PING_FAILED_WITH_NO_DEVNAME = OBDErrorCodeTemplate(2007, '{ip1} fail to ping {ip2}. Please check your network')
 EC_OBSERVER_TIME_OUT_OF_SYNC = OBDErrorCodeTemplate(2008, 'Cluster clocks are out of sync')
 EC_OBSERVER_PRODUCTION_MODE_LIMIT = OBDErrorCodeTemplate(2009, '({server}): when production_mode is True, {key} can not be less then {limit}')
@@ -180,6 +184,7 @@ class InitDirFailedErrorMessage(object):
 # obagent
 EC_OBAGENT_RELOAD_FAILED = OBDErrorCodeTemplate(4000, 'Fail to reload {server}')
 EC_OBAGENT_SEND_CONFIG_FAILED = OBDErrorCodeTemplate(4001, 'Fail to send config file to {server}')
+WC_OBAGENT_SERVER_NAME_ERROR = OBDErrorCodeTemplate(4002, '{servers}: Failed to obtain the configuration of the OceanBase database component. \nPlease ensure that the server configurations are consistent between the OBAgent and OceanBase database components.')
 
 # obproxy
 EC_OBPROXY_NEED_CONFIG = OBDErrorCodeTemplate(4100, '{server} need config "rs_list" or "obproxy_config_server_url"')
@@ -220,8 +225,9 @@ class InitDirFailedErrorMessage(object):
 EC_OCP_SERVER_NOT_ENOUGH_MEMORY_CACHED = OBDErrorCodeTemplate(4364, '({ip}) not enough memory. (Free: {free}, Buff/Cache: {cached}, Need: {need})')
 EC_OCP_SERVER_NOT_ENOUGH_MEMORY = OBDErrorCodeTemplate(4364, '({ip}) not enough memory. (Free: {free}, Need: {need})')
 EC_OCP_SERVER_NOT_ENOUGH_DISK = OBDErrorCodeTemplate(4365, '({ip}) {disk} not enough disk space. (Avail: {avail}, Need: {need})')
-EC_OCP_SERVER_RESOURCE_NOT_ENOUGH = OBDErrorCodeTemplate(4366, 'There is not enough {resource}. (Avail: {avail}, need: {need})')
-
+EC_OCP_SERVER_RESOURCE_NOT_ENOUGH = OBDErrorCodeTemplate(4366, 'There is not enough {resource}. (Avail: {avail}, Need: {need})')
+EC_OCP_SERVER_EXIST_METADB_TENANT_MEMORY_NOT_ENOUGH = OBDErrorCodeTemplate(4367, 'The allocated memory for the provided meta database is currently insufficient for creating a tenant. Available: {avail}, Need: {need}.')
+EC_OCP_SERVER_NOT_EXIST_METADB_TENANT_MEMORY_NOT_ENOUGH = OBDErrorCodeTemplate(4368, 'The allocated memory for the provided meta database is currently insufficient for creating a tenant. Available: {avail}, Need: {need}(Available = memory_limit [{memory_limit}] - system_memory [{system_memory}] - sys tenant memory [{sys_tenant_memory}]. Need = ocp meta tenant memory [{ocp_meta_tenant_memory}] + ocp_monitor_tenant_memory [{ocp_monitor_tenant_memory}]).')
 
 
 WC_OCP_EXPRESS_FAILED_TO_GET_DISK_INFO = OBDErrorCodeTemplate(4303, '({ip}) failed to get disk information, skip disk space check')
@@ -300,7 +306,10 @@ class InitDirFailedErrorMessage(object):
 SUG_OCP_SERVER_REDUCE_MEM = OBDErrorSuggestionTemplate('Please reduce the `memory_size`', fix_eval=[FixEval(FixEval.DEL, 'memory_size')])
 SUG_OCP_SERVER_REDUCE_DISK = OBDErrorSuggestionTemplate('Please reduce the `logging_file_total_size_cap`', fix_eval=[FixEval(FixEval.DEL, 'logging_file_total_size_cap')])
 SUG_OCP_SERVER_EDIT_ADMIN_PASSWD_ERROR = OBDErrorSuggestionTemplate('Please edit the `admin_password`, must be 8 to 32 characters in length, containing at least 3 types from digits, lowercase letters, uppercase letters and the following special characters: ~!@#%^&*_-+=|(){{}}[]:;,.?/)', fix_eval=[FixEval(FixEval.DEL, 'admin_password')], auto_fix=True)
+SUG_OCP_SERVER_MACHINE_TIME = OBDErrorSuggestionTemplate('Please ensure that the machine time is synchronized with the ob time')
 SUG_SUDO_NOPASSWD = OBDErrorSuggestionTemplate('Please execute `bash -c \'echo "{user} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers`\' as root in {ip}.')
+SUG_OCP_SERVER_EXIST_METADB_TENANT_NOT_ENOUGH = OBDErrorSuggestionTemplate('Please reduce the ocp meta tenant memory or ocp monitor tenant memory')
+SUG_OCP_SERVER_NOT_EXIST_METADB_TENANT_NOT_ENOUGH = OBDErrorSuggestionTemplate('Please increase the meta db memory_limit and reduce the ocp meta tenant memory or ocp monitor tenant memory')
 SUG_OB_SYS_USERNAME = OBDErrorSuggestionTemplate('Please delete the "ob_sys_username" parameter.')
 SUG_OB_SYS_PASSWORD = OBDErrorSuggestionTemplate('''Please set the "ob_sys_password" for oblogproxy by configuring the "cdcro_password" parameter in the "oceanbase" or "oceanbase-ce" component.''')
 SUG_OBAGENT_EDIT_HTTP_BASIC_AUTH_PASSWORD = OBDErrorSuggestionTemplate('Please edit the `http_basic_auth_password`, cannot contain characters other than uppercase letters, lowercase characters, digits, special characters:~^*{{}}[]_-+', fix_eval=[FixEval(FixEval.DEL, 'http_basic_auth_password')], auto_fix=True)