Merge branch 'master' into upgrade-kernel

pingcap · Sep 20, 2024 · 4615834 · 4615834
2 parents d6bf614 + ff467a6
commit 4615834
Show file tree

Hide file tree

Showing 62 changed files with 766 additions and 260 deletions.
diff --git a/cmd/backup-manager/app/backup/manager.go b/cmd/backup-manager/app/backup/manager.go
@@ -35,6 +35,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	errorutils "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/json"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/klog/v2"
 )
@@ -101,6 +102,13 @@ func (bm *Manager) ProcessBackup() error {
 		return errorutils.NewAggregate(errs)
 	}
 
+	crData, err := json.Marshal(backup)
+	if err != nil {
+		klog.Errorf("failed to marshal backup %v to json, err: %v", backup, err)
+	} else {
+		klog.Infof("start to process backup: %s", string(crData))
+	}
+
 	// we treat snapshot backup as restarted if its status is not scheduled when backup pod just start to run
 	// we will clean backup data before run br command
 	if backup.Spec.Mode == v1alpha1.BackupModeSnapshot && (backup.Status.Phase != v1alpha1.BackupScheduled || v1alpha1.IsBackupRestart(backup)) {
@@ -132,6 +140,9 @@ func (bm *Manager) ProcessBackup() error {
 		return bm.performBackup(ctx, backup.DeepCopy(), nil)
 	}
 
+	klog.Infof("start to connect to tidb server (%s:%d) as the .spec.from field is specified",
+		backup.Spec.From.Host, backup.Spec.From.Port)
+
 	// validate and create from db
 	var db *sql.DB
 	db, err = bm.validateAndCreateFromDB(ctx, backup.DeepCopy())

diff --git a/cmd/backup-manager/app/export/manager.go b/cmd/backup-manager/app/export/manager.go
@@ -32,6 +32,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	errorutils "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/json"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/klog/v2"
 )
@@ -99,6 +100,13 @@ func (bm *BackupManager) ProcessBackup() error {
 		return errorutils.NewAggregate(errs)
 	}
 
+	crData, err := json.Marshal(backup)
+	if err != nil {
+		klog.Errorf("failed to marshal backup %v to json, err: %v", backup, err)
+	} else {
+		klog.Infof("start to process backup: %s", string(crData))
+	}
+
 	reason, err := bm.setOptions(backup)
 	if err != nil {
 		errs = append(errs, err)

diff --git a/cmd/backup-manager/app/import/manager.go b/cmd/backup-manager/app/import/manager.go
@@ -27,6 +27,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	errorutils "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/json"
 	"k8s.io/klog/v2"
 )
 
@@ -87,6 +88,13 @@ func (rm *RestoreManager) ProcessRestore() error {
 		return errorutils.NewAggregate(errs)
 	}
 
+	crData, err := json.Marshal(restore)
+	if err != nil {
+		klog.Errorf("failed to marshal restore %v to json, err: %s", restore, err)
+	} else {
+		klog.Infof("start to process restore: %s", string(crData))
+	}
+
 	rm.setOptions(restore)
 
 	return rm.performRestore(ctx, restore.DeepCopy())

diff --git a/cmd/backup-manager/app/restore/manager.go b/cmd/backup-manager/app/restore/manager.go
@@ -30,6 +30,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	errorutils "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/json"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/klog/v2"
 )
@@ -96,12 +97,22 @@ func (rm *Manager) ProcessRestore() error {
 		return fmt.Errorf("no br config in %s", rm)
 	}
 
+	crData, err := json.Marshal(restore)
+	if err != nil {
+		klog.Errorf("failed to marshal restore %v to json, err: %s", restore, err)
+	} else {
+		klog.Infof("start to process restore: %s", string(crData))
+	}
+
 	if restore.Spec.To == nil {
 		return rm.performRestore(ctx, restore.DeepCopy(), nil)
 	}
 
 	rm.setOptions(restore)
 
+	klog.Infof("start to connect to tidb server (%s:%d) as the .spec.to field is specified",
+		restore.Spec.To.Host, restore.Spec.To.Port)
+
 	var db *sql.DB
 	var dsn string
 	err = wait.PollImmediate(constants.PollInterval, constants.CheckTimeout, func() (done bool, err error) {

diff --git a/cmd/http-service/Dockerfile b/cmd/http-service/Dockerfile
@@ -1,4 +1,4 @@
-FROM pingcap/pingcap-base:v1
+FROM ghcr.io/pingcap-qe/bases/pingcap-base:v1.9.1
 
 ARG TARGETARCH
 RUN dnf install -y tzdata bind-utils && dnf clean all

diff --git a/docs/api-references/docs.md b/docs/api-references/docs.md
@@ -3394,6 +3394,30 @@ azblob service account credentials.</p>
 </tr>
 <tr>
 <td>
+<code>storageAccount</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<p>StorageAccount is the storage account of the azure blob storage
+If this field is set, then use this to set backup-manager env
+Otherwise retrieve the storage account from secret</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>sasToken</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<p>SasToken is the sas token of the storage account</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>prefix</code></br>
 <em>
 string
@@ -12321,6 +12345,18 @@ int
 </tr>
 <tr>
 <td>
+<code>initWaitTime</code></br>
+<em>
+int
+</em>
+</td>
+<td>
+<p>Wait time before pd get started. This wait time is to allow the new DNS record to propagate,
+ensuring that the PD DNS resolves to the same IP address as the pod.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>mode</code></br>
 <em>
 string

diff --git a/examples/backup/backup-azblob.yaml b/examples/backup/backup-azblob.yaml
@@ -6,12 +6,6 @@ metadata:
 spec:
   cleanPolicy: Delete
   # backupType: full
-  # Only needed for TiDB Operator < v1.1.7 or TiDB < v4.0.8
-  # from:
-  #   host: ${tidb-host}
-  #   port: ${tidb-port}
-  #   user: ${tidb-user}
-  #   secretName: backup-basic-tidb-secret
   br:
     cluster: basic
     clusterNamespace: default

diff --git a/examples/backup/backup-ebs-local.yaml b/examples/backup/backup-ebs-local.yaml
@@ -7,12 +7,6 @@ spec:
   cleanPolicy: Delete
   backupType: full
   backupMode: volume-snapshot
-  # Only needed for TiDB Operator < v1.1.7 or TiDB < v4.0.8
-  # from:
-  #   host: ${tidb-host}
-  #   port: ${tidb-port}
-  #   user: ${tidb-user}
-  #   secretName: backup-basic-tidb-secret
   toolImage: localhost:5000/pingcap/br:latest
   br:
     cluster: basic

diff --git a/examples/backup/backup-ebs-minio.yaml b/examples/backup/backup-ebs-minio.yaml
@@ -7,12 +7,6 @@ spec:
   cleanPolicy: Delete
   backupType: full
   backupMode: volume-snapshot
-  # Only needed for TiDB Operator < v1.1.7 or TiDB < v4.0.8
-  # from:
-  #   host: ${tidb-host}
-  #   port: ${tidb-port}
-  #   user: ${tidb-user}
-  #   secretName: backup-basic-tidb-secret
   toolImage: localhost:5000/pingcap/br:latest
   br:
     cluster: basic

diff --git a/examples/backup/backup-local.yaml b/examples/backup/backup-local.yaml
@@ -6,12 +6,6 @@ metadata:
 spec:
   cleanPolicy: Delete
   # backupType: full
-  # Only needed for TiDB Operator < v1.1.7 or TiDB < v4.0.8
-  # from:
-  #   host: ${tidb-host}
-  #   port: ${tidb-port}
-  #   user: ${tidb-user}
-  #   secretName: backup-basic-tidb-secret
   br:
     cluster: basic
     clusterNamespace: default

diff --git a/examples/backup/backup-nfs.yaml b/examples/backup/backup-nfs.yaml
@@ -6,12 +6,6 @@ metadata:
 spec:
   cleanPolicy: Delete
   # backupType: full
-  # Only needed for TiDB Operator < v1.1.7 or TiDB < v4.0.8
-  # from:
-  #   host: ${tidb-host}
-  #   port: ${tidb-port}
-  #   user: ${tidb-user}
-  #   secretName: backup-basic-tidb-secret
   br:
     cluster: basic
     clusterNamespace: default

diff --git a/examples/backup/backup-schedule-azblob.yaml b/examples/backup/backup-schedule-azblob.yaml
@@ -9,12 +9,6 @@ spec:
   # maxReservedTime: "2m"
   schedule: "*/1 * * * *"
   backupTemplate:
-    # Only needed for TiDB Operator < v1.1.7 or TiDB < v4.0.8
-    # from:
-    #   host: ${tidb_host}
-    #   port: ${tidb_port}
-    #   user: ${tidb_user}
-    #   secretName: backup-demo1-tidb-secret
     cleanPolicy: Delete
     br:
       cluster: basic

diff --git a/examples/backup/backup-schedule-nfs.yaml b/examples/backup/backup-schedule-nfs.yaml
@@ -9,12 +9,6 @@ spec:
   # maxReservedTime: "2m"
   schedule: "*/1 * * * *"
   backupTemplate:
-    # Only needed for TiDB Operator < v1.1.7 or TiDB < v4.0.8
-    # from:
-    #   host: ${tidb_host}
-    #   port: ${tidb_port}
-    #   user: ${tidb_user}
-    #   secretName: backup-demo1-tidb-secret
     cleanPolicy: Delete
     br:
       cluster: basic

diff --git a/examples/backup/restore-azblob.yaml b/examples/backup/restore-azblob.yaml
@@ -5,12 +5,6 @@ metadata:
   namespace: default
 spec:
   # backupType: full
-  # Only needed for TiDB Operator < v1.1.7 or TiDB < v4.0.8
-  # to:
-  #   host: ${tidb_host}
-  #   port: ${tidb_port}
-  #   user: ${tidb_user}
-  #   secretName: restore-demo2-tidb-secret
   br:
     cluster: basic
     clusterNamespace: default

diff --git a/examples/backup/restore-ebs-minio.yaml b/examples/backup/restore-ebs-minio.yaml
@@ -6,12 +6,6 @@ metadata:
 spec:
   backupType: full
   backupMode: volume-snapshot
-  # Only needed for TiDB Operator < v1.1.7 or TiDB < v4.0.8
-  # from:
-  #   host: ${tidb-host}
-  #   port: ${tidb-port}
-  #   user: ${tidb-user}
-  #   secretName: backup-basic-tidb-secret
   toolImage: localhost:5000/pingcap/br:latest
   br:
     cluster: basic

diff --git a/examples/backup/restore-local.yaml b/examples/backup/restore-local.yaml
@@ -5,12 +5,6 @@ metadata:
   namespace: default
 spec:
   # backupType: full
-  # Only needed for TiDB Operator < v1.1.7 or TiDB < v4.0.8
-  # from:
-  #   host: ${tidb-host}
-  #   port: ${tidb-port}
-  #   user: ${tidb-user}
-  #   secretName: backup-basic-tidb-secret
   br:
     cluster: basic
     clusterNamespace: default

diff --git a/examples/backup/restore-nfs.yaml b/examples/backup/restore-nfs.yaml
@@ -5,12 +5,6 @@ metadata:
   namespace: default
 spec:
   # backupType: full
-  # Only needed for TiDB Operator < v1.1.7 or TiDB < v4.0.8
-  # to:
-  #   host: ${tidb_host}
-  #   port: ${tidb_port}
-  #   user: ${tidb_user}
-  #   secretName: restore-demo2-tidb-secret
   br:
     cluster: basic
     clusterNamespace: default

diff --git a/examples/basic/pd-micro-service-cluster.yaml b/examples/basic/pd-micro-service-cluster.yaml
@@ -16,8 +16,9 @@ spec:
   helper:
     image: alpine:3.16.0
   pd:
-    baseImage: pingcap/pd
-    version: v8.1.0
+    # TODO: replaced v8.3.0 after v8.3.0 released
+    baseImage: hub.pingcap.net/devbuild/pd
+    version: v8.3.0-5427
     maxFailoverCount: 0
     replicas: 1
     # if storageClassName is not set, the default Storage Class of the Kubernetes cluster will be used
@@ -28,13 +29,13 @@ spec:
     mode: "ms"
   pdms:
     - name: "tso"
-      baseImage: pingcap/pd
-      version: v8.1.0
+      baseImage: hub.pingcap.net/devbuild/pd
+      version: v8.3.0-5427
       replicas: 2
     - name: "scheduling"
-      baseImage: pingcap/pd
-      version: v8.1.0
-      replicas: 1
+      baseImage: hub.pingcap.net/devbuild/pd
+      version: v8.3.0-5427
+      replicas: 2
   tikv:
     baseImage: pingcap/tikv
     version: v8.1.0

diff --git a/images/br-federation-manager/Dockerfile b/images/br-federation-manager/Dockerfile
@@ -1,4 +1,4 @@
-FROM pingcap/pingcap-base:v1
+FROM ghcr.io/pingcap-qe/bases/pingcap-base:v1.9.1
 ARG TARGETARCH
 RUN dnf install -y bind-utils tzdata && dnf clean all
 ADD bin/${TARGETARCH}/br-federation-manager /usr/local/bin/br-federation-manager
diff --git a/images/br-federation-manager/Dockerfile.e2e b/images/br-federation-manager/Dockerfile.e2e
@@ -1,4 +1,4 @@
-FROM pingcap/pingcap-base:v1
+FROM ghcr.io/pingcap-qe/bases/pingcap-base:v1.9.1
 
 ARG TARGETARCH
 RUN dnf install -y tzdata bind-utils && dnf clean all

diff --git a/images/tidb-backup-manager/Dockerfile b/images/tidb-backup-manager/Dockerfile
@@ -1,4 +1,4 @@
-FROM pingcap/pingcap-base:v1
+FROM ghcr.io/pingcap-qe/bases/pingcap-base:v1.9.1
 ARG TARGETARCH
 ARG RCLONE_VERSION=v1.57.0
 ARG SHUSH_VERSION=v1.4.0

diff --git a/images/tidb-backup-manager/Dockerfile.e2e b/images/tidb-backup-manager/Dockerfile.e2e
@@ -1,4 +1,4 @@
-FROM pingcap/pingcap-base:v1
+FROM ghcr.io/pingcap-qe/bases/pingcap-base:v1.9.1
 ARG TARGETARCH=amd64
 ARG RCLONE_VERSION=v1.57.0
 ARG SHUSH_VERSION=v1.4.0

diff --git a/images/tidb-operator/Dockerfile b/images/tidb-operator/Dockerfile
@@ -1,4 +1,4 @@
-FROM pingcap/pingcap-base:v1
+FROM ghcr.io/pingcap-qe/bases/pingcap-base:v1.9.1
 
 ARG TARGETARCH
 RUN dnf install -y tzdata bind-utils && dnf clean all

diff --git a/images/tidb-operator/Dockerfile.e2e b/images/tidb-operator/Dockerfile.e2e
@@ -1,4 +1,4 @@
-FROM pingcap/pingcap-base:v1
+FROM ghcr.io/pingcap-qe/bases/pingcap-base:v1.9.1
 
 RUN dnf install -y tzdata bash bind-utils && dnf clean all
 

diff --git a/manifests/backup/backup-aws-s3-br.yaml b/manifests/backup/backup-aws-s3-br.yaml
@@ -21,12 +21,6 @@ spec:
     # timeAgo: <time>
     # checksum: true
     # sendCredToTikv: true
-  from:
-    host: 172.30.6.56
-    secretName: mySecret
-    # port: 4000
-    # user: root
-    # tlsClientSecretName: <backup-tls-secretname>
   s3:
     provider: aws
     region: us-west-2

diff --git a/manifests/backup/backup-gcs-br.yaml b/manifests/backup/backup-gcs-br.yaml
@@ -18,12 +18,6 @@ spec:
     # rateLimit: 0
     # timeAgo: <time>
     # checksum: true
-  from:
-    host: 172.30.6.56
-    secretName: my-secret
-    # port: 4000
-    # user: root
-    # tlsClientSecretName: <backup-tls-secretname>
   gcs:
     projectId: gcp
     location: us-west2