Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -204,3 +204,9 @@ PLUGIN_RUNTIME_MAX_BUFFER_SIZE=5242880
DIFY_BACKWARDS_INVOCATION_WRITE_TIMEOUT=5000
# dify backwards invocation read timeout in milliseconds
DIFY_BACKWARDS_INVOCATION_READ_TIMEOUT=240000


TEMP_DIR = E:\\Temp
PYTHONIOENCODING = utf-8
LANG=en_US.UTF-8
LC_ALL=en_US.UTF-8
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ require (
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0
go.opentelemetry.io/otel/sdk v1.39.0
go.opentelemetry.io/otel/sdk/metric v1.39.0
go.opentelemetry.io/otel/trace v1.39.0
golang.org/x/tools v0.38.0
gorm.io/driver/mysql v1.5.7
gorm.io/gorm v1.30.0
Expand Down Expand Up @@ -146,7 +147,6 @@ require (
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 // indirect
go.opentelemetry.io/otel/metric v1.39.0 // indirect
go.opentelemetry.io/otel/trace v1.39.0 // indirect
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
golang.org/x/mod v0.29.0 // indirect
golang.org/x/oauth2 v0.32.0 // indirect
Expand Down
24 changes: 21 additions & 3 deletions internal/core/local_runtime/setup_python_environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"os/exec"
"path"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync"
Expand All @@ -18,6 +19,7 @@ import (
routinepkg "github.com/langgenius/dify-plugin-daemon/pkg/routine"
"github.com/langgenius/dify-plugin-daemon/pkg/utils/log"
"github.com/langgenius/dify-plugin-daemon/pkg/utils/routine"
"github.com/langgenius/dify-plugin-daemon/pkg/utils/system"
gootel "go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/propagation"
Expand Down Expand Up @@ -139,6 +141,9 @@ func (p *LocalPluginRuntime) installDependencies(
) error {
baseCtx, parent := p.startSpan("python.install_deps", attribute.String("plugin.identity", p.Config.Identity()))
defer parent.End()
if runtime.GOOS == "windows" {
baseCtx = context.WithoutCancel(baseCtx)
}
ctx, cancel := context.WithTimeout(baseCtx, 10*time.Minute)
defer cancel()

Expand Down Expand Up @@ -175,6 +180,10 @@ func (p *LocalPluginRuntime) installDependencies(
if p.appConfig.NoProxy != "" {
cmd.Env = append(cmd.Env, fmt.Sprintf("NO_PROXY=%s", p.appConfig.NoProxy))
}
if p.appConfig.TempDir != "" {
cmd.Env = append(cmd.Env, fmt.Sprintf("TEMP=%s", p.appConfig.TempDir))
cmd.Env = append(cmd.Env, fmt.Sprintf("TMP=%s", p.appConfig.TempDir))
}
cmd.Dir = p.State.WorkingPath

// get stdout and stderr
Expand Down Expand Up @@ -299,12 +308,18 @@ const (
requirementsTxtFile PythonDependencyFileType = "requirements.txt"
)

var envPythonPath string
var envValidFlagFile string

const (
envPath = ".venv"
envPythonPath = envPath + "/bin/python"
envValidFlagFile = envPath + "/dify/plugin.json"
envPath = ".venv"
)

func init() {
envPythonPath = system.GetEnvPythonPath(envPath)
envValidFlagFile = envPath + "/dify/plugin.json"
}

func (p *LocalPluginRuntime) checkPythonVirtualEnvironment() (*PythonVirtualEnvironment, error) {
_, span := p.startSpan("python.check_venv")
defer span.End()
Expand Down Expand Up @@ -420,6 +435,9 @@ func (p *LocalPluginRuntime) preCompile(
) error {
baseCtx, span := p.startSpan("python.precompile")
defer span.End()
if runtime.GOOS == "windows" {
baseCtx = context.WithoutCancel(baseCtx)
}
ctx, cancel := context.WithTimeout(baseCtx, 10*time.Minute)
defer cancel()

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (

"github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities"
"github.com/langgenius/dify-plugin-daemon/pkg/utils/log"
"github.com/langgenius/dify-plugin-daemon/pkg/utils/system"
)

type InstalledBucket struct {
Expand Down Expand Up @@ -73,9 +74,15 @@ func (b *InstalledBucket) List() ([]plugin_entities.PluginUniqueIdentifier, erro
if strings.HasPrefix(path.Path, ".") {
continue
}

convertPath := system.ConvertPath(path.Path)
// windows path start with "/"
if after, ok := strings.CutPrefix(convertPath, "/"); ok {
convertPath = after
}
// remove prefix
identifier, err := plugin_entities.NewPluginUniqueIdentifier(
strings.TrimPrefix(path.Path, b.installedPath),
strings.TrimPrefix(convertPath, b.installedPath),
)
if err != nil {
log.Error("failed to create PluginUniqueIdentifier from path", "path", path.Path, "error", err)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package media_transport

import (
"runtime"
"testing"

"github.com/langgenius/dify-cloud-kit/oss"
"github.com/langgenius/dify-cloud-kit/oss/factory"
"github.com/langgenius/dify-plugin-daemon/internal/types/app"
"github.com/stretchr/testify/assert"
)

func TestPluginListWindonws(t *testing.T) {
if runtime.GOOS != "windows" {
return
}
config := &app.Config{
PluginStorageLocalRoot: "testdata",
PluginInstalledPath: "plugin",
PluginStorageType: "local",
}
var storage oss.OSS
var err error
storage, err = factory.Load(config.PluginStorageType, oss.OSSArgs{
Local: &oss.Local{
Path: config.PluginStorageLocalRoot,
},
})
if err != nil {
t.Fatal("failed to create storage")
}
installedBucket := NewInstalledBucket(storage, config.PluginInstalledPath)
identifiers, err := installedBucket.List()
if err != nil {
t.Error(err)
}
assert.Equal(t, len(identifiers), 1)
assert.Equal(t, identifiers[0].String(), "langgenius/github#0.3.2@1cb2f90ea05bbc7987fd712aff0a07594073816269125603dc2fa5b4229eb122")
}
Binary file not shown.
1 change: 1 addition & 0 deletions internal/types/app/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,7 @@ type Config struct {
PipPreferBinary bool `envconfig:"PIP_PREFER_BINARY" default:"true"`
PipVerbose bool `envconfig:"PIP_VERBOSE" default:"true"`
PipExtraArgs string `envconfig:"PIP_EXTRA_ARGS"`
TempDir string `envconfig:"TEMP_DIR"`

// Runtime buffer configuration (applies to both local and serverless runtimes)
// These are the new generic names that should be used going forward
Expand Down
9 changes: 5 additions & 4 deletions pkg/entities/plugin_entities/identity.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/go-playground/validator/v10"
"github.com/google/uuid"
"github.com/langgenius/dify-plugin-daemon/pkg/entities/manifest_entities"
"github.com/langgenius/dify-plugin-daemon/pkg/utils/system"
"github.com/langgenius/dify-plugin-daemon/pkg/validators"
)

Expand All @@ -20,7 +21,7 @@ var (
// for checksum, it must be a 32-character hexadecimal string.
// the author part is optional, if not specified, it will be empty.
pluginUniqueIdentifierRegexp = regexp.MustCompile(
`^(?:([a-z0-9_-]{1,64})\/)?([a-z0-9_-]{1,255}):([0-9]{1,4})(\.[0-9]{1,4}){1,3}(-\w{1,16})?@[a-f0-9]{32,64}$`,
`^(?:([a-z0-9_-]{1,64})\/)?([a-z0-9_-]{1,255})` + system.DelimiterFLag + `([0-9]{1,4})(\.[0-9]{1,4}){1,3}(-\w{1,16})?@[a-f0-9]{32,64}$`,
)
)

Expand All @@ -33,7 +34,7 @@ func NewPluginUniqueIdentifier(identifier string) (PluginUniqueIdentifier, error

func (p PluginUniqueIdentifier) PluginID() string {
// try find :
split := strings.Split(p.String(), ":")
split := strings.Split(p.String(), system.DelimiterFLag)
if len(split) == 2 {
return split[0]
}
Expand All @@ -44,7 +45,7 @@ func (p PluginUniqueIdentifier) Version() manifest_entities.Version {
// extract version part from the string
split := strings.Split(p.String(), "@")
if len(split) == 2 {
split = strings.Split(split[0], ":")
split = strings.Split(split[0], system.DelimiterFLag)
if len(split) == 2 {
return manifest_entities.Version(split[1])
}
Expand All @@ -60,7 +61,7 @@ func (p PluginUniqueIdentifier) RemoteLike() bool {

func (p PluginUniqueIdentifier) Author() string {
// extract author part from the string
split := strings.Split(p.String(), ":")
split := strings.Split(p.String(), system.DelimiterFLag)
if len(split) == 2 {
split = strings.Split(split[0], "/")
if len(split) == 2 {
Expand Down
10 changes: 6 additions & 4 deletions pkg/entities/plugin_entities/identity_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@ package plugin_entities

import (
"testing"

"github.com/langgenius/dify-plugin-daemon/pkg/utils/system"
)

func TestPluginUniqueIdentifier(t *testing.T) {
i, err := NewPluginUniqueIdentifier("langgenius/test:1.0.0@1234567890abcdef1234567890abcdef1234567890abcdef")
i, err := NewPluginUniqueIdentifier("langgenius/test" + system.DelimiterFLag + "1.0.0@1234567890abcdef1234567890abcdef1234567890abcdef")
if err != nil {
t.Fatalf("NewPluginUniqueIdentifier() returned an error: %v", err)
}
Expand All @@ -22,7 +24,7 @@ func TestPluginUniqueIdentifier(t *testing.T) {
t.Fatalf("Checksum() = %s; want 1234567890abcdef1234567890abcdef1234567890abcdef", i.Checksum())
}

_, err = NewPluginUniqueIdentifier("test:1.0.0@1234567890abcdef1234567890abcdef1234567890abcdef")
_, err = NewPluginUniqueIdentifier("test" + system.DelimiterFLag + "1.0.0@1234567890abcdef1234567890abcdef1234567890abcdef")
if err != nil {
t.Fatalf("NewPluginUniqueIdentifier() returned an error: %v", err)
}
Expand All @@ -37,12 +39,12 @@ func TestPluginUniqueIdentifier(t *testing.T) {
t.Fatalf("NewPluginUniqueIdentifier() returned nil error for invalid identifier")
}

_, err = NewPluginUniqueIdentifier("langgenius/test:1.0.0@123456")
_, err = NewPluginUniqueIdentifier("langgenius/test" + system.DelimiterFLag + "1.0.0@123456")
if err == nil {
t.Fatalf("NewPluginUniqueIdentifier() returned nil error for invalid identifier")
}

_, err = NewPluginUniqueIdentifier("langgenius/test:1.0.0")
_, err = NewPluginUniqueIdentifier("langgenius/test" + system.DelimiterFLag + "1.0.0")
if err == nil {
t.Fatalf("NewPluginUniqueIdentifier() returned nil error for invalid identifier")
}
Expand Down
Binary file not shown.
3 changes: 2 additions & 1 deletion pkg/plugin_packager/decoder/zip.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/langgenius/dify-plugin-daemon/pkg/entities/plugin_entities"
"github.com/langgenius/dify-plugin-daemon/pkg/plugin_packager/consts"
"github.com/langgenius/dify-plugin-daemon/pkg/utils/parser"
"github.com/langgenius/dify-plugin-daemon/pkg/utils/system"
)

type ZipPluginDecoder struct {
Expand Down Expand Up @@ -304,7 +305,7 @@ func (z *ZipPluginDecoder) ExtractTo(dst string) error {
return err
}

bytes, err := z.ReadFile(filepath.Join(dir, filename))
bytes, err := z.ReadFile(system.GetZipReadPath(dir, filename))
if err != nil {
return err
}
Expand Down
33 changes: 33 additions & 0 deletions pkg/plugin_packager/decoder/zip_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package decoder

import (
"os"
"path"
"testing"
)

func TestExtractFile(t *testing.T) {
pluginFile, err := os.ReadFile("testdata/github#0.3.2@1cb2f90ea05bbc7987fd712aff0a07594073816269125603dc2fa5b4229eb122")
if err != nil {
t.Fatalf("read file error: %v", err)
}
decoder, err := NewZipPluginDecoder(pluginFile)
if err != nil {
t.Fatalf("create new zip decoder error: %v", err)
}
extractPath := "testdata/cwd"
err = os.Mkdir(extractPath, 0755)
if err != nil {
t.Fatalf("mk dir error: %v", err)
}
defer os.RemoveAll(extractPath)

err = decoder.ExtractTo(extractPath)
if err != nil {
t.Fatalf("extract file error: %v", err)
}
_, err = os.Stat(path.Join(extractPath, "provider", "github.yaml"))
if err != nil {
t.Fatalf("extract file not exists: %v", err)
}
}
10 changes: 7 additions & 3 deletions pkg/utils/parser/identity.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
package parser

import "fmt"
import (
"fmt"

"github.com/langgenius/dify-plugin-daemon/pkg/utils/system"
)

func MarshalPluginID(author string, name string, version string) string {
if author == "" {
return fmt.Sprintf("%s:%s", name, version)
return fmt.Sprintf("%s%s%s", name, system.DelimiterFLag, version)
}
return fmt.Sprintf("%s/%s:%s", author, name, version)
return fmt.Sprintf("%s/%s%s%s", author, name, system.DelimiterFLag, version)
}
39 changes: 39 additions & 0 deletions pkg/utils/system/system.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package system

import (
"path"
"path/filepath"
"runtime"
"strings"
)

var DelimiterFLag string

func init() {
if runtime.GOOS == "windows" {
DelimiterFLag = "#"
} else {
DelimiterFLag = ":"
}
}

func ConvertPath(input string) string {
if runtime.GOOS == "windows" {
return strings.ReplaceAll(input, "\\", "/")
}
return input
}

func GetZipReadPath(dir string, filename string) string {
if runtime.GOOS == "windows" {
return path.Join(dir, filename)
}
return filepath.Join(dir, filename)
}

func GetEnvPythonPath(envPath string) string {
if runtime.GOOS == "windows" {
return envPath + "/Scripts/python.exe"
}
return envPath + "/bin/python"
}
25 changes: 25 additions & 0 deletions pkg/utils/system/system_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package system

import (
"path"
"path/filepath"
"runtime"
"testing"

"github.com/stretchr/testify/assert"
)

func TestFilePath(t *testing.T) {
filepathJoinRes := filepath.Join("foo", "bar.txt")
pathJoinRs := path.Join("foo", "bar.txt")
if runtime.GOOS == "windows" {
assert.Equal(t, "foo\\bar.txt", filepathJoinRes)
}
assert.Equal(t, "foo/bar.txt", pathJoinRs)
}

func TestConvertPath(t *testing.T) {
filepathJoinRes := filepath.Join("foo", "bar.txt")
res := ConvertPath(filepathJoinRes)
assert.Equal(t, "foo/bar.txt", res)
}