repo-sync-2024-04-02T11:59:40+0800 (#266)

* repo-sync-2024-04-02T11:59:40+0800 * repo-sync-2024-04-03T10:08:35+0800
secretflow · Apr 3, 2024 · 5c3abc6 · 5c3abc6
1 parent 75a1a24
commit 5c3abc6
Show file tree

Hide file tree

Showing 152 changed files with 7,568 additions and 1,602 deletions.
diff --git a/Makefile b/Makefile
@@ -8,7 +8,7 @@ IMG := secretflow/kuscia:${TAG}
 # TEST_SUITE used by integration test
 TEST_SUITE ?= all
 
-ENVOY_IMAGE ?= secretflow/kuscia-envoy:0.3.0.dev231122
+ENVOY_IMAGE ?= secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-envoy:0.4.0.dev20240402
 DEPS_IMAGE ?= secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-deps:0.5.0b0
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)

diff --git a/WORKSPACE b/WORKSPACE
@@ -3,6 +3,7 @@ workspace(name = "kuscia")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
 
+
 maybe(
     http_archive,
     sha256 = "2c6a36c7b5a55accae063667ef3c55f2642e67476d96d355ff0acb13dbb47f09",

diff --git a/build/dockerfile/kuscia-anolis.Dockerfile b/build/dockerfile/kuscia-anolis.Dockerfile
@@ -1,5 +1,5 @@
 ARG DEPS_IMAGE="secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-deps:0.5.0b0"
-ARG KUSCIA_ENVOY_IMAGE="secretflow/kuscia-envoy:0.3.0.dev231122"
+ARG KUSCIA_ENVOY_IMAGE="secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-envoy:0.4.0.dev20240402"
 ARG PROM_NODE_EXPORTER="prom/node-exporter:v1.7.0"
 
 FROM ${DEPS_IMAGE} as deps

diff --git a/cmd/kuscia/confloader/kuscia_config.go b/cmd/kuscia/confloader/kuscia_config.go
@@ -70,11 +70,11 @@ type RunkConfig struct {
 	KubeconfigFile string   `yaml:"kubeconfigFile"`
 }
 
-func (runk RunkConfig) convert2K8sProviderCfg() (k8s config.K8sProviderCfg) {
-	k8s.Namespace = runk.Namespace
-	k8s.KubeconfigFile = runk.KubeconfigFile
-	k8s.DNS.Servers = runk.DNSServers
-	return
+func (runk RunkConfig) overwriteK8sProviderCfg(k8sCfg config.K8sProviderCfg) config.K8sProviderCfg {
+	k8sCfg.Namespace = runk.Namespace
+	k8sCfg.KubeconfigFile = runk.KubeconfigFile
+	k8sCfg.DNS.Servers = runk.DNSServers
+	return k8sCfg
 }
 
 type ImageConfig struct {
@@ -154,14 +154,18 @@ func (lite *LiteKusciaConfig) OverwriteKusciaConfig(kusciaConfig *KusciaConfig)
 	kusciaConfig.DataMesh = lite.DataMesh
 	kusciaConfig.Agent.AllowPrivileged = lite.Agent.AllowPrivileged
 	kusciaConfig.Agent.Provider.Runtime = lite.Runtime
-	kusciaConfig.Agent.Provider.K8s = lite.Runk.convert2K8sProviderCfg()
-	kusciaConfig.Agent.Provider.K8s.Backend = lite.Agent.Provider.K8s.Backend
-	kusciaConfig.Agent.Provider.K8s.LabelsToAdd = lite.Agent.Provider.K8s.LabelsToAdd
-	kusciaConfig.Agent.Provider.K8s.AnnotationsToAdd = lite.Agent.Provider.K8s.AnnotationsToAdd
+	kusciaConfig.Agent.Provider.K8s = lite.Runk.overwriteK8sProviderCfg(lite.Agent.Provider.K8s)
 	kusciaConfig.Agent.Capacity = lite.Capacity
-	if len(lite.Agent.Plugins) > 0 {
-		kusciaConfig.Agent.Plugins = lite.Agent.Plugins
+
+	for _, p := range lite.Agent.Plugins {
+		for j, pp := range kusciaConfig.Agent.Plugins {
+			if p.Name == pp.Name {
+				kusciaConfig.Agent.Plugins[j] = p
+				break
+			}
+		}
 	}
+
 	kusciaConfig.Master.Endpoint = lite.MasterEndpoint
 	kusciaConfig.DomainRoute.DomainCsrData = generateCsrData(lite.DomainID, lite.DomainKeyData, lite.LiteDeployToken)
 	kusciaConfig.Debug = lite.Debug
@@ -200,14 +204,18 @@ func (autonomy *AutomonyKusciaConfig) OverwriteKusciaConfig(kusciaConfig *Kuscia
 	kusciaConfig.DomainKeyData = autonomy.DomainKeyData
 	kusciaConfig.Agent.AllowPrivileged = autonomy.Agent.AllowPrivileged
 	kusciaConfig.Agent.Provider.Runtime = autonomy.Runtime
-	kusciaConfig.Agent.Provider.K8s = autonomy.Runk.convert2K8sProviderCfg()
-	kusciaConfig.Agent.Provider.K8s.Backend = autonomy.Agent.Provider.K8s.Backend
-	kusciaConfig.Agent.Provider.K8s.LabelsToAdd = autonomy.Agent.Provider.K8s.LabelsToAdd
-	kusciaConfig.Agent.Provider.K8s.AnnotationsToAdd = autonomy.Agent.Provider.K8s.AnnotationsToAdd
+	kusciaConfig.Agent.Provider.K8s = autonomy.Runk.overwriteK8sProviderCfg(autonomy.Agent.Provider.K8s)
 	kusciaConfig.Agent.Capacity = autonomy.Capacity
-	if len(autonomy.Agent.Plugins) > 0 {
-		kusciaConfig.Agent.Plugins = autonomy.Agent.Plugins
+
+	for _, p := range autonomy.Agent.Plugins {
+		for j, pp := range kusciaConfig.Agent.Plugins {
+			if p.Name == pp.Name {
+				kusciaConfig.Agent.Plugins[j] = p
+				break
+			}
+		}
 	}
+
 	kusciaConfig.ConfLoaders = autonomy.ConfLoaders
 	kusciaConfig.SecretBackends = autonomy.SecretBackends
 	if autonomy.KusciaAPI != nil {

diff --git a/cmd/kuscia/modules/domainroute.go b/cmd/kuscia/modules/domainroute.go
@@ -54,8 +54,14 @@ func NewDomainRoute(i *Dependencies) Module {
 		externalTLS = i.DomainRoute.ExternalTLS
 	}
 
-	if i.Protocol == common.NOTLS {
+	protocol := i.Protocol
+	switch protocol {
+	case common.NOTLS:
 		externalTLS = nil
+	case common.TLS, common.MTLS:
+		externalTLS = &kusciaconfig.TLSConfig{
+			EnableTLS: true,
+		}
 	}
 
 	if externalTLS != nil && externalTLS.EnableTLS {

diff --git a/cmd/kuscia/modules/scheduler.go b/cmd/kuscia/modules/scheduler.go
@@ -65,6 +65,8 @@ func NewScheduler(i *Dependencies) Module {
 	o.Authentication.TolerateInClusterLookupFailure = true
 	o.Authentication.RemoteKubeConfigFileOptional = true
 	o.Authorization.RemoteKubeConfigFileOptional = true
+	o.Authorization.RemoteKubeConfigFile = i.KubeconfigFile
+	o.Authentication.RemoteKubeConfigFile = i.KubeconfigFile
 
 	// Set the PairName but leave certificate directory blank to generate in-memory by default
 	o.SecureServing.ServerCert.CertDirectory = ""

diff --git a/cmd/kuscia/modules/ssexporter.go b/cmd/kuscia/modules/ssexporter.go
@@ -40,7 +40,7 @@ func NewSsExporter(i *Dependencies) Module {
 		domainID:           i.DomainID,
 		rootDir:            i.RootDir,
 		metricUpdatePeriod: i.MetricUpdatePeriod,
-		ssExportPort:       string(i.SsExportPort),
+		ssExportPort:       i.SsExportPort,
 	}
 }
 

diff --git a/crds/v1alpha1/kuscia.secretflow_clusterdomainroutes.yaml b/crds/v1alpha1/kuscia.secretflow_clusterdomainroutes.yaml
@@ -174,18 +174,28 @@ spec:
                 - tokenGenMethod
                 type: object
               transit:
-                description: Transit entity. If it is not empty, the requests between
-                  nodes need to be transferred through a third party.
+                description: Transit entity. If transitMethod is THIRD-DOMAIN, requests
+                  from source to destination need to be transferred through a third
+                  party, domain field must be set. If transitMethod is REVERSE-TUNNEL,
+                  requests from source to destination need to be transferred through
+                  local gateway chunked transfer encoding.
                 properties:
                   domain:
-                    description: DomainTransit means to forward the request through
-                      the domain.
+                    description: DomainTransit defines the information of the third
+                      domain.
                     properties:
                       domainID:
                         type: string
                     required:
                     - domainID
                     type: object
+                  transitMethod:
+                    description: TransitMethod means to forward the request through
+                      a specific entity, THIRD-DOMAIN by default.
+                    enum:
+                    - THIRD-DOMAIN
+                    - REVERSE-TUNNEL
+                    type: string
                 type: object
             required:
             - authenticationType

diff --git a/crds/v1alpha1/kuscia.secretflow_domainroutes.yaml b/crds/v1alpha1/kuscia.secretflow_domainroutes.yaml
@@ -171,18 +171,28 @@ spec:
                 - tokenGenMethod
                 type: object
               transit:
-                description: Transit entity. If it is not empty, the requests between
-                  nodes need to be transferred through a third party.
+                description: Transit entity. If transitMethod is THIRD-DOMAIN, requests
+                  from source to destination need to be transferred through a third
+                  party, domain field must be set. If transitMethod is REVERSE-TUNNEL,
+                  requests from source to destination need to be transferred through
+                  local gateway chunked transfer encoding.
                 properties:
                   domain:
-                    description: DomainTransit means to forward the request through
-                      the domain.
+                    description: DomainTransit defines the information of the third
+                      domain.
                     properties:
                       domainID:
                         type: string
                     required:
                     - domainID
                     type: object
+                  transitMethod:
+                    description: TransitMethod means to forward the request through
+                      a specific entity, THIRD-DOMAIN by default.
+                    enum:
+                    - THIRD-DOMAIN
+                    - REVERSE-TUNNEL
+                    type: string
                 type: object
             required:
             - authenticationType

diff --git a/docs/deployment/Docker_deployment_kuscia/deploy_master_lite_cn.md b/docs/deployment/Docker_deployment_kuscia/deploy_master_lite_cn.md
@@ -21,20 +21,27 @@ export KUSCIA_IMAGE=secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/k
 获取部署脚本，部署脚本会下载到当前目录：
 
 ```bash
-docker pull $KUSCIA_IMAGE && docker run --rm $KUSCIA_IMAGE cat /home/kuscia/scripts/deploy/deploy.sh > deploy.sh && chmod u+x deploy.sh
+docker pull $KUSCIA_IMAGE && docker run --rm $KUSCIA_IMAGE cat /home/kuscia/scripts/deploy/kuscia.sh > kuscia.sh && chmod u+x kuscia.sh
 ```
 
-启动 master，默认会在当前目录下创建 ${USER}-kuscia-master/data、${USER}-kuscia-master/logs、${USER}-kuscia-master/kuscia.yaml 用来存储 master 的数据、日志和配置文件：
+生成 master 节点的配置文件：
+```bash
+# -n 参数传递的是 master 节点 ID，DomainID 需全局唯一，生产环境建议使用公司名称-部门名称-节点名称，如: antgroup-secretflow-master
+docker run -it --rm ${KUSCIA_IMAGE} kuscia init --mode master --domain "antgroup-secretflow-master" > kuscia_master.yaml
+```
+
+启动 master，默认会在当前目录下创建 ${USER}-kuscia-master/{data、logs} 用来存储 master 的数据、日志：
 
 ```bash
-# -n 参数传递的是Master节点 ID，DomainID需全局唯一，生产环境建议使用公司名称-部门名称-节点名称，如: antgroup-secretflow-master
 # -p 参数传递的是 master 容器映射到主机的端口，保证和主机上现有的端口不冲突即可
 # -k 参数传递的是 master 容器 KusciaAPI 映射到主机的 HTTP 端口，保证和主机上现有的端口不冲突即可
-./deploy.sh master -n antgroup-secretflow-master -p 18080 -k 18082
+./kuscia.sh start -c kuscia_master.yaml -p 18080 -k 18081
 ```
+
 <span style="color:red;">注意：<br>
 1、如果 master 的入口网络存在网关时，为了确保节点与 master 之间通信正常，需要网关符合一些要求，详情请参考[这里](./networkrequirements.md) <br>
-2、master 节点默认使用 sqlite 作为存储，如果生产部署，需要配置链接到 mysql 数据库的连接串，具体配置可以参考[这里](./kuscia_config_cn.md#id3)</span>
+2、master 节点默认使用 sqlite 作为存储，如果生产部署，需要配置链接到 mysql 数据库的连接串，具体配置可以参考[这里](./kuscia_config_cn.md#id3)<br>
+3、需要对合作方暴露的 Kuscia 端口，可参考 [Kuscia 端口介绍](../kuscia_ports_cn.md) </span>
 
 建议使用 curl -kvvv https://ip:port; 检查一下是否访问能通，正常情况下返回的 HTTP 错误码是 401，内容是：unauthorized。
 示例如下：
@@ -122,18 +129,24 @@ export KUSCIA_IMAGE=secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/k
 获取部署脚本，部署脚本会下载到当前目录：
 
 ```bash
-docker pull $KUSCIA_IMAGE && docker run --rm $KUSCIA_IMAGE cat /home/kuscia/scripts/deploy/deploy.sh > deploy.sh && chmod u+x deploy.sh
+docker pull $KUSCIA_IMAGE && docker run --rm $KUSCIA_IMAGE cat /home/kuscia/scripts/deploy/kuscia.sh > kuscia.sh && chmod u+x kuscia.sh
+```
+
+生成 alice 节点的配置文件：
+```bash
+# --domain 参数传递的是节点 ID
+# --lite-deploy-token 参数传递的是节点部署的 Token
+# --master-endpoint 参数传递的是 master 容器对外暴露的 https://IP:PORT，假设 master 对外暴露的 IP 是 1.1.1.1，端口是18080
+docker run -it --rm ${KUSCIA_IMAGE} kuscia init --mode lite --domain "alice" --master-endpoint "https://1.1.1.1:18080" --lite-deploy-token "abcdefg" > lite_alice.yaml
 ```
 
 启动 alice，默认会在当前目录下创建 ${USER}-kuscia-lite-alice/data 目录用来存放 alice 的数据：
 ```bash
-# -n 参数传递的是节点 ID
-# -t 参数传递的是节点部署的 Token
-# -m 参数传递的是 master 容器对外暴露的 https://IP:PORT，假设 master 对外暴露的 IP 是1.1.1.1，端口是18080
 # -p 参数传递的是节点容器映射到主机的端口，保证和主机上现有的端口不冲突即可
-./deploy.sh lite -n alice -t abcdefg -m https://1.1.1.1:18080 -p 28080
+# -k 参数传递的是 lite 容器 KusciaAPI 映射到主机的 HTTP 端口，保证和主机上现有的端口不冲突即可
+./kuscia.sh start -c lite_alice.yaml -p 28080 -k 28081
 ```
-> 如果 master 与多个 lite 节点部署在同一个物理机上，可以用 -p -k -g 参数指定下端口号（例如：./deploy.sh lite -n alice -t abcdefg -m https://1.1.1.1:18080 -p 28080 -k 2008 -g 2009），防止出现端口冲突
+> 如果 master 与多个 lite 节点部署在同一个物理机上，可以用 -p -k -g -q 参数指定下端口号（例如：./kuscia.sh start -c lite_alice.yaml -p 28080 -k 28081 -g 28082 -q 28083），防止出现端口冲突。
 
 #### 部署 lite 节点 bob
 
@@ -167,18 +180,24 @@ export KUSCIA_IMAGE=secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/k
 获取部署脚本，部署脚本会下载到当前目录：
 
 ```bash
-docker pull $KUSCIA_IMAGE && docker run --rm $KUSCIA_IMAGE cat /home/kuscia/scripts/deploy/deploy.sh > deploy.sh && chmod u+x deploy.sh
+docker pull $KUSCIA_IMAGE && docker run --rm $KUSCIA_IMAGE cat /home/kuscia/scripts/deploy/kuscia.sh > kuscia.sh && chmod u+x kuscia.sh
+```
+
+生成 bob 节点的配置文件：
+```bash
+# --domain 参数传递的是节点 ID
+# --lite-deploy-token 参数传递的是节点部署的 Token
+# --master-endpoint 参数传递的是 master 容器对外暴露的 https://IP:PORT，假设 master 对外暴露的 IP 是 1.1.1.1，端口是18080
+docker run -it --rm ${KUSCIA_IMAGE} kuscia init --mode lite --domain "bob" --master-endpoint "https://1.1.1.1:18080" --lite-deploy-token "hijklmn" > lite_bob.yaml
 ```
 
 启动 bob，默认会在当前目录下创建 ${USER}-kuscia-lite-bob/data 目录用来存放 bob 的数据：
 ```bash
-# -n 参数传递的是节点 ID
-# -t 参数传递的是节点部署的 Token
-# -m 参数传递的是 master 容器对外暴露的 https://IP:PORT，假设 master 对外暴露的 IP 是1.1.1.1，端口是18080
 # -p 参数传递的是节点容器映射到主机的端口，保证和主机上现有的端口不冲突即可
-./deploy.sh lite -n bob -t hijklmn -m https://1.1.1.1:18080 -p 38080
+# -k 参数传递的是 lite 容器 KusciaAPI 映射到主机的 HTTP 端口，保证和主机上现有的端口不冲突即可
+./kuscia.sh start -c lite_bob.yaml -p 38080 -k 38081
 ```
-> 如果 master 与多个 lite 节点部署在同一个物理机上，可以用 -p -k -g 参数指定下端口号（例如：./deploy.sh lite -n alice -t abcdefg -m https://1.1.1.1:18080 -p 38080 -k 2010 -g 2011），防止出现端口冲突
+> 如果 master 与多个 lite 节点部署在同一个物理机上，可以用 -p -k -g -q 参数指定下端口号（例如：./kuscia.sh start -c lite_bob.yaml -p 38080 -k 38081 -g 38082 -q 38083），防止出现端口冲突。
 
 ### 配置授权
 

diff --git a/docs/deployment/Docker_deployment_kuscia/deploy_p2p_cn.md b/docs/deployment/Docker_deployment_kuscia/deploy_p2p_cn.md
@@ -24,20 +24,28 @@ export KUSCIA_IMAGE=secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/k
 获取部署脚本，部署脚本会下载到当前目录：
 
 ```
-docker pull $KUSCIA_IMAGE && docker run --rm $KUSCIA_IMAGE cat /home/kuscia/scripts/deploy/deploy.sh > deploy.sh && chmod u+x deploy.sh
+docker pull $KUSCIA_IMAGE && docker run --rm $KUSCIA_IMAGE cat /home/kuscia/scripts/deploy/kuscia.sh > kuscia.sh && chmod u+x kuscia.sh
 ```
 
-启动节点，默认会在当前目录下创建 ${USER}-kuscia-autonomy-alice/data 目录用来存放 alice 的数据。部署节点需要使用 `deploy.sh` 脚本并传入特定的参数：
+生成 alice 节点配置文件：
+```bash
+# --domain 参数传递的是节点 ID
+docker run -it --rm ${KUSCIA_IMAGE} kuscia init --mode autonomy --domain "alice" > autonomy_alice.yaml
+```
+
+启动节点，默认会在当前目录下创建 ${USER}-kuscia-autonomy-alice/data 目录用来存放 alice 的数据。部署节点需要使用 `kuscia.sh` 脚本并传入节点配置文件：
 
 ```bash
-# -n 参数传递的是节点 ID。
 # -p 参数传递的是节点容器映射到主机的 HTTPS 端口，保证和主机上现有的端口不冲突即可
 # -k 参数传递的是节点容器 KusciaAPI 映射到主机的 MTLS 端口，保证和主机上现有的端口不冲突即可
-./deploy.sh autonomy -n alice -p 11080 -k 8082
+./kuscia.sh start -c autonomy_alice.yaml -p 11080 -k 11081
 ```
+> 如果多个 lite 节点部署在同一个物理机上，可以用 -p -k -g -q 参数指定下端口号（例如：./kuscia.sh start -c autonomy_alice.yaml -p 11080 -k 11081 -g 11082 -q 11083），防止出现端口冲突。
+
 <span style="color:red;">注意：<br>
 1、如果节点之间的入口网络存在网关时，为了确保节点与 master 之间通信正常，需要网关符合一些要求，详情请参考[这里](./networkrequirements.md) <br>
-2、alice、bob 节点默认使用 sqlite 作为存储，如果生产部署，需要配置链接到 mysql 数据库的连接串，具体配置可以参考[这里](./kuscia_config_cn.md#id3)</span>
+2、alice、bob 节点默认使用 sqlite 作为存储，如果生产部署，需要配置链接到 mysql 数据库的连接串，具体配置可以参考[这里](./kuscia_config_cn.md#id3)<br>
+3、需要对合作方暴露的 Kuscia 端口，可参考 [Kuscia 端口介绍](../kuscia_ports_cn.md) </span>
 
 
 ### 部署 bob 节点
@@ -86,10 +94,10 @@ docker cp ${USER}-kuscia-autonomy-bob:/home/kuscia/var/certs/domain.crt bob.doma
 docker cp bob.domain.crt ${USER}-kuscia-autonomy-alice:/home/kuscia/var/certs/
 ```
 
-在 bob 里添加 alice 的证书等信息：
+在 alice 里添加 bob 的证书等信息：
 
 ```bash
-# [bob 机器] 添加 alice 的证书等信息
+# [alice 机器] 添加 alice 的证书等信息
 docker exec -it ${USER}-kuscia-autonomy-alice scripts/deploy/add_domain.sh bob p2p
 ```