support spdk tgt

Author: asignmisspast

Makefile

@@ -73,4 +73,3 @@ upload:
 lint:
 	go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCILINT_VERSION)
 	$(GOBIN_GOLANGCILINT) run -v
-

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629
 	github.com/kpango/glg v1.6.14
-	github.com/mattn/go-sqlite3 v1.14.16
+	github.com/mattn/go-sqlite3 v1.14.17
 	github.com/mcuadros/go-defaults v1.2.0
 	github.com/melbahja/goph v1.3.0
 	github.com/mitchellh/hashstructure/v2 v2.0.2

go.sum

@@ -331,6 +331,8 @@ github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh
 github.com/mattn/go-sqlite3 v1.6.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
 github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
+github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=

internal/common/common.go

@@ -120,6 +120,10 @@ const (
 	KEY_TARGET_OPTIONS = "TARGET_OPTIONS"
 	KEY_ALL_TARGETS    = "ALL_TARGETS"
 
+	// spdk_target
+	KEY_SPDK_TARGET_OPTIONS = "SPDK_TARGET_OPTIONS"
+	KEY_ALL_SPDK_TARGETS    = "ALL_SPDK_TARGETS"
+
 	// playground
 	KEY_ALL_PLAYGROUNDS_STATUS = "ALL_PLAYGROUNDS_STATUS"
 	PLAYGROUDN_STATUS_LOSED    = "Losed"

internal/errno/errno.go

@@ -419,15 +419,17 @@ var (
 	ERR_CLIENT_ID_NOT_FOUND                  = EC(410022, "client id not found")
 
 	// 420: common (curvebs client)
-	ERR_VOLUME_ALREADY_MAPPED             = EC(420000, "volume already mapped")
-	ERR_VOLUME_CONTAINER_LOSED            = EC(420001, "volume container is losed")
-	ERR_VOLUME_CONTAINER_ABNORMAL         = EC(420002, "volume container is abnormal")
-	ERR_CREATE_VOLUME_FAILED              = EC(420003, "create volume failed")
-	ERR_MAP_VOLUME_FAILED                 = EC(420004, "map volume to NBD device failed")
-	ERR_ENCODE_VOLUME_INFO_TO_JSON_FAILED = EC(420005, "encode volume info to json failed")
-	ERR_UNMAP_VOLUME_FAILED               = EC(420006, "unmap volume failed")
-	ERR_OLD_TARGET_DAEMON_IS_ABNORMAL     = EC(420007, "old target daemon is abnormal")
-	ERR_TARGET_DAEMON_IS_ABNORMAL         = EC(420008, "target daemon is abnormal")
+	ERR_VOLUME_ALREADY_MAPPED              = EC(420000, "volume already mapped")
+	ERR_VOLUME_CONTAINER_LOSED             = EC(420001, "volume container is losed")
+	ERR_VOLUME_CONTAINER_ABNORMAL          = EC(420002, "volume container is abnormal")
+	ERR_CREATE_VOLUME_FAILED               = EC(420003, "create volume failed")
+	ERR_MAP_VOLUME_FAILED                  = EC(420004, "map volume to NBD device failed")
+	ERR_ENCODE_VOLUME_INFO_TO_JSON_FAILED  = EC(420005, "encode volume info to json failed")
+	ERR_UNMAP_VOLUME_FAILED                = EC(420006, "unmap volume failed")
+	ERR_OLD_TARGET_DAEMON_IS_ABNORMAL      = EC(420007, "old target daemon is abnormal")
+	ERR_TARGET_DAEMON_IS_ABNORMAL          = EC(420008, "target daemon is abnormal")
+	ERR_OLD_SPDK_TARGET_DAEMON_IS_ABNORMAL = EC(420008, "old spdk target daemon is abnormal")
+	ERR_SPDK_TARGET_DAEMON_IS_ABNORMAL     = EC(420009, "spdk target daemon is abnormal")
 
 	// 430: common (curvefs client)
 	ERR_FS_PATH_ALREADY_MOUNTED  = EC(430000, "path already mounted")

internal/playbook/factory.go

@@ -121,6 +121,13 @@ const (
 	DELETE_TARGET
 	LIST_TARGETS
 
+	// bs/target
+	START_SPDK_TARGET_DAEMON
+	STOP_SPDK_TARGET_DAEMON
+	ADD_SPDK_TARGET
+	DELETE_SPDK_TARGET
+	LIST_SPDK_TARGETS
+
 	// fs
 	CHECK_CLIENT_S3
 	MOUNT_FILESYSTEM
@@ -275,6 +282,7 @@ func (p *Playbook) createTasks(step *PlaybookStep) (*tasks.Tasks, error) {
 			t, err = bs.NewMapTask(curveadm, config.GetCC(i))
 		case UNMAP_IMAGE:
 			t, err = bs.NewUnmapTask(curveadm, nil)
+
 		// bs/target
 		case START_TARGET_DAEMON:
 			t, err = bs.NewStartTargetDaemonTask(curveadm, config.GetCC(i))
@@ -286,6 +294,19 @@ func (p *Playbook) createTasks(step *PlaybookStep) (*tasks.Tasks, error) {
 			t, err = bs.NewDeleteTargetTask(curveadm, nil)
 		case LIST_TARGETS:
 			t, err = bs.NewListTargetsTask(curveadm, nil)
+
+		// bs/target
+		case START_SPDK_TARGET_DAEMON:
+			t, err = bs.NewStartSpdkTargetDaemonTask(curveadm, config.GetCC(i))
+		case STOP_SPDK_TARGET_DAEMON:
+			t, err = bs.NewStopSpdkTargetDaemonTask(curveadm, nil)
+		case ADD_SPDK_TARGET:
+			t, err = bs.NewAddSpdkTargetTask(curveadm, config.GetCC(i))
+		case DELETE_SPDK_TARGET:
+			t, err = bs.NewDeleteSpdkTargetTask(curveadm, nil)
+		case LIST_SPDK_TARGETS:
+			t, err = bs.NewListSpdkTargetsTask(curveadm, nil)
+
 		// fs
 		case CHECK_CLIENT_S3:
 			t, err = checker.NewClientS3ConfigureTask(curveadm, config.GetCC(i))

internal/task/scripts/add_spdk_target.go

@@ -0,0 +1,79 @@
+/*
+ *  Copyright (c) 2021 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: CurveAdm
+ * Created Date: 2022-02-08
+ * Author: Jingli Chen (Wine93)
+ */
+
+package scripts
+
+/*
+ * Usage: target USER VOLUME CREATE SIZE
+ * Example: target curve test true 10
+ * See Also: https://linux.die.net/man/8/tgtadm
+ */
+
+var ADD__SPDK_TARGET = `
+#!/usr/bin/env bash
+
+g_sockname=$1
+g_volume=$2
+g_originbdev=$3
+g_curvebdev=$4
+g_rpcpath=$5
+g_blocksize=$6
+g_cdbpath=$7
+g_isexclusive=$8
+g_targetip=$9
+g_targetport=$10
+g_initiatorip=$11
+g_initiatormask=$12
+g_user=$13
+g_create=$14
+g_size=%15
+g_volumename=%16
+
+mkdir -p /curvebs/nebd/data/lock
+touch /etc/curve/curvetab
+
+mkdir -p /curvebs/nebd/data/lock
+touch /etc/curve/curvetab
+if [ $g_create == "true" ]; then
+    output=$(curve_ops_tool create -userName=$g_user -fileName=$g_volume -fileLength=$g_size)
+    if [ $? -ne 0 ]; then
+        if [ "$output" != "CreateFile fail with errCode: 101" ]; then
+            exit 1
+        fi
+    fi
+fi
+if ${g_rpcpath} -s ${g_sockname} iscsi_get_target_nodes >/dev/null | grep "$g_curvebdev" | grep name >/dev/null; then
+    echo "spdk target is already created, please try anther name"
+    exit 1
+fi
+${g_rpcpath} -s ${g_sockname} bdev_aio_create ${g_volume} ${g_originbdev} 512 >/dev/null
+${g_rpcpath} -s ${g_sockname} bdev_cdb_create -b ${g_curvebdev} --cbd ${g_cbdpath} --exclusive=${g_isexclusive} --blocksize=${g_blocksize} >/dev/null
+${g_rpcpath} -s ${g_sockname} bdev_ocf_create  bdev_ocf_create myocf wb ${g_originbdev} > /dev/null
+${g_rpcpath} -s ${g_sockname} iscsi_create_portal_group 1 ${g_targetip}:${g_targetport} > /dev/null
+${g_rpcpath} -s ${g_sockname} iscsi_create_initiator_group 2 ANY ${g_initiatorip}/${g_initiatormask} > /dev/null
+
+${g_rpcpath} -s ${g_sockname} iscsi_create_target_node ${g_volumename} "Data Disk1" "myocf:0 ${g_originbdev}:1" 1:2 64 -d > /dev/null
+
+
+exit 0
+
+`

internal/task/scripts/spdk_target.go

@@ -0,0 +1,76 @@
+/*
+ *  Copyright (c) 2021 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: CurveAdm
+ * Created Date: 2022-02-08
+ * Author: Jingli Chen (Wine93)
+ */
+
+package scripts
+
+/*
+ * Usage: target USER VOLUME CREATE SIZE
+ * Example: target curve test true 10
+ * See Also: https://linux.die.net/man/8/tgtadm
+ */
+
+var SPDK_TARGET = `
+#!/usr/bin/env bash
+
+g_binarypath=$1
+g_cpumask=$2
+g_sockname=$3
+g_iscsi_log=$4
+
+mkdir -p /curvebs/nebd/data/lock
+touch /etc/curve/curvetab
+
+process_name="iscsi_tgt"
+
+if ps aux | grep -v grep | grep "$process_name" >/dev/null; then
+   echo "spdk iscsi_tgt has already been started, now exit!"
+   exit 1
+fi
+
+nohup ${g_binarypath} -m ${g_cpumask} -r ${g_sockname} > ${g_iscsi_log} 2>&1 &
+
+i=1
+while true
+do
+    result=$(cat $VSDK_LOG_FILE | grep "please run rpc")
+    if [[ "$result" != "" ]]; then
+        i=0
+        break
+    else
+        sleep 0.1
+        i++
+    fi
+    if [ "$i" -ge 50 ]; then
+        break
+    fi
+done
+
+if [ "$i" -eq 0]; then
+    echo "spdk iscsi_tgt started success!"
+else
+    echo "wait for 5s, spdk iscsi_tgt has not launched, please try again!"
+    exit 1
+fi
+
+exit 0
+
+`

internal/task/step/daemon.go

@@ -77,6 +77,9 @@ type AddDaemonTask struct {
 
 type DelDaemonTask struct {
 	ContainerId *string
+	Cmd         string
+	Args        []string
+	TaskName    string
 	Tid         string
 	MemStorage  *utils.SafeMap
 	module.ExecOptions