*: add remote source from xtrabackup

acekingke · acekingke · commit 62a7e0f018d9 · 2023-10-11T14:40:57.000+08:00
diff --git a/Dockerfile.sidecar b/Dockerfile.sidecar
@@ -52,7 +52,7 @@ RUN set -ex; \
 ARG XTRABACKUP_PKG=percona-xtrabackup-24
 RUN set -ex; \
     apt-get update; \
-    apt-get install -y --no-install-recommends gnupg2 wget lsb-release curl bc; \
+    apt-get install -y --no-install-recommends gnupg2 wget lsb-release curl bc openssh-server  sshpass;\
     wget -P /tmp --no-check-certificate https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb; \
     dpkg -i /tmp/percona-release_latest.$(lsb_release -sc)_all.deb; \
     apt-get update; \
diff --git a/api/v1alpha1/mysqlcluster_types.go b/api/v1alpha1/mysqlcluster_types.go
@@ -113,6 +113,10 @@ type MysqlClusterSpec struct {
 	// Containing CA (ca.crt) and server cert (tls.crt), server private key (tls.key) for SSL
 	// +optional
 	TlsSecretName string `json:"tlsSecretName,omitempty"`
+
+	// Bootstraping from remote data source
+	// +optional
+	SourceConfig *corev1.SecretProjection `json:"sourceConfig,omitempty"`
 }
 
 // ReadOnly define the ReadOnly pods
@@ -345,7 +349,8 @@ const (
 	// ClusterUpdateState indicates whether the cluster is being updated.
 	ClusterUpdateState ClusterState = "Updating"
 	// ClusterReadyState indicates whether all containers in the pod are ready.
-	ClusterReadyState ClusterState = "Ready"
+	ClusterReadyState      ClusterState = "Ready"
+	ClusterRemoteCompleted ClusterState = "RemoteCompleted"
 	// ClusterCloseState indicates whether the cluster is closed.
 	ClusterCloseState ClusterState = "Closed"
 	// ClusterScaleInState indicates whether the cluster replicas is decreasing.
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/api/v1beta1/mysqlcluster_conversion.go b/api/v1beta1/mysqlcluster_conversion.go
@@ -95,7 +95,9 @@ func Convert_v1alpha1_MysqlClusterSpec_To_v1beta1_MysqlClusterSpec(in *v1alpha1.
 		out.DataSource.S3Backup.SecretName = in.BackupSecretName
 		out.DataSource.RestorePoint = in.RestorePoint
 	}
-
+	if in.SourceConfig != nil {
+		out.DataSource.Remote.SourceConfig = in.SourceConfig
+	}
 	if len(in.NFSServerAddress) != 0 {
 		ipStr := strings.Split(in.NFSServerAddress, ":")
 		out.DataSource.RestorePoint = in.RestorePoint
@@ -160,6 +162,9 @@ func Convert_v1beta1_MysqlClusterSpec_To_v1alpha1_MysqlClusterSpec(in *MysqlClus
 	out.PodPolicy.PriorityClassName = in.PriorityClassName
 	// in.DataSource in.Standby
 	out.XenonOpts.EnableAutoRebuild = in.EnableAutoRebuild
+	if in.DataSource.Remote.SourceConfig != nil {
+		out.SourceConfig = in.DataSource.Remote.SourceConfig
+	}
 	if len(in.DataSource.S3Backup.Name) != 0 {
 		out.RestoreFrom = in.DataSource.S3Backup.Name
 		out.BackupSecretName = in.DataSource.S3Backup.SecretName
diff --git a/api/v1beta1/zz_generated.conversion.go b/api/v1beta1/zz_generated.conversion.go
diff --git a/charts/mysql-operator/templates/mysql.radondb.com_mysqlclusters.yaml b/charts/mysql-operator/templates/mysql.radondb.com_mysqlclusters.yaml
@@ -2278,6 +2278,53 @@ spec:
                 description: RestorePoint is the target date and time to restore data.
                   The format is "2006-01-02 15:04:05"
                 type: string
+              sourceConfig:
+                description: Bootstraping from remote data source
+                properties:
+                  items:
+                    description: If unspecified, each key-value pair in the Data field
+                      of the referenced Secret will be projected into the volume as
+                      a file whose name is the key and content is the value. If specified,
+                      the listed keys will be projected into the specified paths,
+                      and unlisted keys will not be present. If a key is specified
+                      which is not present in the Secret, the volume setup will error
+                      unless it is marked optional. Paths must be relative and may
+                      not contain the '..' path or start with '..'.
+                    items:
+                      description: Maps a string key to a path within a volume.
+                      properties:
+                        key:
+                          description: The key to project.
+                          type: string
+                        mode:
+                          description: 'Optional: mode bits used to set permissions
+                            on this file. Must be an octal value between 0000 and
+                            0777 or a decimal value between 0 and 511. YAML accepts
+                            both octal and decimal values, JSON requires decimal values
+                            for mode bits. If not specified, the volume defaultMode
+                            will be used. This might be in conflict with other options
+                            that affect the file mode, like fsGroup, and the result
+                            can be other mode bits set.'
+                          format: int32
+                          type: integer
+                        path:
+                          description: The relative path of the file to map the key
+                            to. May not be an absolute path. May not contain the path
+                            element '..'. May not start with the string '..'.
+                          type: string
+                      required:
+                      - key
+                      - path
+                      type: object
+                    type: array
+                  name:
+                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                      TODO: Add other useful fields. apiVersion, kind, uid?'
+                    type: string
+                  optional:
+                    description: Specify whether the Secret or its key must be defined
+                    type: boolean
+                type: object
               tlsSecretName:
                 description: Containing CA (ca.crt) and server cert (tls.crt), server
                   private key (tls.key) for SSL
diff --git a/config/crd/bases/mysql.radondb.com_mysqlclusters.yaml b/config/crd/bases/mysql.radondb.com_mysqlclusters.yaml
@@ -2260,6 +2260,53 @@ spec:
                 description: RestorePoint is the target date and time to restore data.
                   The format is "2006-01-02 15:04:05"
                 type: string
+              sourceConfig:
+                description: Bootstraping from remote data source
+                properties:
+                  items:
+                    description: If unspecified, each key-value pair in the Data field
+                      of the referenced Secret will be projected into the volume as
+                      a file whose name is the key and content is the value. If specified,
+                      the listed keys will be projected into the specified paths,
+                      and unlisted keys will not be present. If a key is specified
+                      which is not present in the Secret, the volume setup will error
+                      unless it is marked optional. Paths must be relative and may
+                      not contain the '..' path or start with '..'.
+                    items:
+                      description: Maps a string key to a path within a volume.
+                      properties:
+                        key:
+                          description: The key to project.
+                          type: string
+                        mode:
+                          description: 'Optional: mode bits used to set permissions
+                            on this file. Must be an octal value between 0000 and
+                            0777 or a decimal value between 0 and 511. YAML accepts
+                            both octal and decimal values, JSON requires decimal values
+                            for mode bits. If not specified, the volume defaultMode
+                            will be used. This might be in conflict with other options
+                            that affect the file mode, like fsGroup, and the result
+                            can be other mode bits set.'
+                          format: int32
+                          type: integer
+                        path:
+                          description: The relative path of the file to map the key
+                            to. May not be an absolute path. May not contain the path
+                            element '..'. May not start with the string '..'.
+                          type: string
+                      required:
+                      - key
+                      - path
+                      type: object
+                    type: array
+                  name:
+                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                      TODO: Add other useful fields. apiVersion, kind, uid?'
+                    type: string
+                  optional:
+                    description: Specify whether the Secret or its key must be defined
+                    type: boolean
+                type: object
               tlsSecretName:
                 description: Containing CA (ca.crt) and server cert (tls.crt), server
                   private key (tls.key) for SSL
diff --git a/config/samples/mysql_v1beta1_mysqlcluster.yaml b/config/samples/mysql_v1beta1_mysqlcluster.yaml
@@ -14,7 +14,15 @@ spec:
     S3backup:
       name: ""
       secretName: ""
-    remote: {}
+    remote: 
+      # sourceConfig:
+      #     name: remotesecret
+      #     items:
+      #     - key: passwd
+      #       path: passwd
+      #     - key: host
+      #       path: host
+    
   image: percona/percona-server:8.0.25
   imagePullPolicy: Always
   logOpts:
diff --git a/docs/en-us/backup_and_restoration_nfs.md b/docs/en-us/backup_and_restoration_nfs.md
@@ -47,13 +47,13 @@ You can use `ClusterIp` to perform NFS backup. The cluster IP address in the exa
 ### 1. Configure the NFS server address
 
 ```yaml
-# config/samples/mysql_v1alpha1_backup.yaml
+# config/samples/mysql_v1beta1_backup.yaml
 nfsServerAddress: "10.96.253.82"
 ```
 
 ### 2. Create a backup
 ```shell
-kubectl apply -f config/samples/mysql_v1alpha1_backup.yaml
+kubectl apply -f config/samples/mysql_v1beta1_backup.yaml
 ```
 > Note: The backup CRD and MySQL cluster CRD must be in the same namespace.
 
@@ -67,7 +67,7 @@ index.html  initbackup  sample_2022419101946
 
  ## Restore the cluster from the NFS backup
 
-Configure the `nfsServerAddress` attribute to the NFS server address in the `mysql_v1alpha1_cluster.yaml` file.
+Configure the `nfsServerAddress` attribute to the NFS server address in the `mysql_v1alpha1_cluster.yaml` file or `mysql_v1beta1_cluster.yaml` file.
 
  ```yaml
  ...
diff --git a/docs/en-us/backup_and_restoration_s3.md b/docs/en-us/backup_and_restoration_s3.md
@@ -46,7 +46,7 @@ kubectl create -f config/samples/backup_secret.yaml
 ```
 
 ### Step 2: Configure the backup Secret for the Operator cluster
-Configure the `backupSecretName` property in `mysql_v1alpha1_mysqlcluster.yaml`, for example, `sample-backup-secre`.
+Configure the `backupSecretName` property in `mysql_v1beta1_mysqlcluster.yaml`, for example, `sample-backup-secre`.
 
 ```yaml
 spec:
diff --git a/docs/en-us/backup_cron.md b/docs/en-us/backup_cron.md
@@ -1,4 +1,4 @@
-Contents
+Contents(Deprecated)
 =============
 - [Overview](#overview)
 - [Configuration of scheduled backups](#configuration-of-scheduled-backups)
diff --git a/docs/en-us/how_to_use_nfsbcp.md b/docs/en-us/how_to_use_nfsbcp.md
@@ -1,5 +1,5 @@
 English | [简体中文](../en-cn/how_to_use_nfsbcp.md)
-# Quickstart for nfsbcp
+# Quickstart for nfsbcp （Deprecated）
 
 ## Contents
 * [Introduction](#Introduction)
diff --git a/docs/en-us/rebuild.md b/docs/en-us/rebuild.md
@@ -1,4 +1,4 @@
-English | [简体中文](../zh-cn/rebuild.md)
+English | [简体中文](../zh-cn/rebuild.md) (Deprecated)
 
 # Reason for rebuilding Pod
 RadonDB cluster implements MySQL semisynchronous replication. Semisynchronous replication may cause the replica nodes to update more data than the source node. So, the Pod needs to be rebuilt when it is detected as invalid by Xenon. 
diff --git a/docs/remotesource.md b/docs/remotesource.md
@@ -0,0 +1,37 @@
+English 
+
+# Quickstart remote migration from remote source
+
+## 1. perpare the remote source node
+
+We suppose you have got a existence mysql database, such as percona server, or other mysql release.
+and mysql version is 8.0.25 or mysql 5.7, this document use 8.0.25 for example. we suppose the ip of the node is `172.16.0.29` , and the root's password is `rootpass`. Make sure the sshd is ok, and it can login by `ssh` 
+
+## 2. create secret in k8s
+Now, what's need you to do is that create secret file in k8s, and it must contain the keys ,`host` and `passwd`. for example, we can create a secret named `remotesecret` as follow:
+```
+ kubectl create secret generic remotesecret  --from-literal=host=172.16.0.29    --from-literal=passwd=rootpass
+```
+## 3. fill the fields in mysqlcluster.yaml
+fill the dataSouce's `remote` field, you should fill the name which is the secret's name, in this example, name is `remotesecret`, and fill `items` as same as below:
+
+```yaml
+spec:
+  ...
+  dataSource:
+    ...
+    remote: 
+      sourceConfig:
+          name: remotesecret
+          items:
+          - key: passwd
+            path: passwd
+          - key: host
+            path: host
+    
+```
+## 4. apply the yaml file, run it in k8s
+
+```sh
+kubectl apply -f mysql_v1beta1_mysqlcluster.yaml
+```
diff --git a/mysqlcluster/container/init_sidecar.go b/mysqlcluster/container/init_sidecar.go
@@ -147,6 +147,12 @@ func (c *initSidecar) getEnvVars() []corev1.EnvVar {
 			Value: "1",
 		})
 	}
+	if c.Spec.SourceConfig != nil {
+		envs = append(envs, corev1.EnvVar{
+			Name:  "REMOTESRC",
+			Value: "1",
+		})
+	}
 	return envs
 }
 
@@ -252,6 +258,11 @@ func (c *initSidecar) getVolumeMounts() []corev1.VolumeMount {
 			},
 		)
 	}
-
+	if c.Spec.SourceConfig != nil {
+		volumeMounts = append(volumeMounts, corev1.VolumeMount{
+			Name:      utils.RemoteSourceVolume,
+			MountPath: utils.RemoteSourcePath,
+		})
+	}
 	return volumeMounts
 }
diff --git a/mysqlcluster/container/mysql.go b/mysqlcluster/container/mysql.go
@@ -190,5 +190,12 @@ func (c *mysql) getVolumeMounts() []corev1.VolumeMount {
 			},
 		)
 	}
+	// Just for test
+	// if c.Spec.SourceConfig != nil {
+	// 	volumeMounts = append(volumeMounts, corev1.VolumeMount{
+	// 		Name:      utils.RemoteSourceVolume,
+	// 		MountPath: utils.RemoteSourcePath,
+	// 	})
+	// }
 	return volumeMounts
 }
diff --git a/mysqlcluster/mysqlcluster.go b/mysqlcluster/mysqlcluster.go
@@ -286,6 +286,20 @@ func (c *MysqlCluster) EnsureVolumes() []corev1.Volume {
 			EmptyDir: &corev1.EmptyDirVolumeSource{},
 		},
 	})
+	if c.Spec.SourceConfig != nil {
+		volumes = append(volumes, corev1.Volume{
+			Name: utils.RemoteSourceVolume,
+			VolumeSource: corev1.VolumeSource{
+				Projected: &corev1.ProjectedVolumeSource{
+					Sources: []corev1.VolumeProjection{
+						{
+							Secret: c.Spec.SourceConfig,
+						},
+					},
+				},
+			},
+		})
+	}
 	return volumes
 }
 
diff --git a/sidecar/config.go b/sidecar/config.go
diff --git a/sidecar/init.go b/sidecar/init.go
diff --git a/utils/constants.go b/utils/constants.go
