Skip to content

Commit

Permalink
chore: remove deprecated LoaderSplitter (#39)
Browse files Browse the repository at this point in the history
  • Loading branch information
shentongmartin authored Jan 15, 2025
1 parent e118e45 commit 4d92ca5
Show file tree
Hide file tree
Showing 11 changed files with 8 additions and 159 deletions.
6 changes: 0 additions & 6 deletions components/document/interface.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,6 @@ type Source struct {

//go:generate mockgen -destination ../../internal/mock/components/document/document_mock.go --package document -source interface.go

// LoaderSplitter is a document loader and splitter.
// Deprecated: use Loader instead.
type LoaderSplitter interface {
LoadAndSplit(ctx context.Context, src Source, opts ...LoaderSplitterOption) ([]*schema.Document, error)
}

// Loader is a document loader.
type Loader interface {
Load(ctx context.Context, src Source, opts ...LoaderOption) ([]*schema.Document, error)
Expand Down
56 changes: 0 additions & 56 deletions components/document/option.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,46 +16,13 @@

package document

// LoaderSplitterOption defines call option for LoaderSplitter component, which is part of the component interface signature.
// Each LoaderSplitter implementation could define its own options struct and option funcs within its own package,
// then wrap the impl specific option funcs into this type, before passing to LoadAndSplit.
// Deprecated: use LoaderOption instead.
type LoaderSplitterOption struct {
implSpecificOptFn any
}

// LoaderOption defines call option for Loader component, which is part of the component interface signature.
// Each Loader implementation could define its own options struct and option funcs within its own package,
// then wrap the impl specific option funcs into this type, before passing to Load.
type LoaderOption struct {
implSpecificOptFn any
}

// WrapImplSpecificOptFn wraps the impl specific option functions into LoaderSplitterOption type.
// T: the type of the impl specific options struct.
// LoaderSplitter implementations are required to use this function to convert its own option functions into the unified LoaderSplitterOption type.
// For example, if the LoaderSplitter impl defines its own options struct:
//
// type customOptions struct {
// conf string
// }
//
// Then the impl needs to provide an option function as such:
//
// func WithConf(conf string) Option {
// return WrapImplSpecificOptFn(func(o *customOptions) {
// o.conf = conf
// }
// }
//
// .
// Deprecated: use WrapLoaderImplSpecificOptFn instead.
func WrapImplSpecificOptFn[T any](optFn func(*T)) LoaderSplitterOption {
return LoaderSplitterOption{
implSpecificOptFn: optFn,
}
}

// WrapLoaderImplSpecificOptFn wraps the impl specific option functions into LoaderOption type.
// T: the type of the impl specific options struct.
// Loader implementations are required to use this function to convert its own option functions into the unified LoaderOption type.
Expand All @@ -78,29 +45,6 @@ func WrapLoaderImplSpecificOptFn[T any](optFn func(*T)) LoaderOption {
}
}

// GetImplSpecificOptions provides LoaderSplitter author the ability to extract their own custom options from the unified LoaderSplitterOption type.
// T: the type of the impl specific options struct.
// This function should be used within the LoaderSplitter implementation's LoadAndSplit function.
// It is recommended to provide a base T as the first argument, within which the LoaderSplitter author can provide default values for the impl specific options.
// Deprecated: use GetLoaderImplSpecificOptions instead.
func GetImplSpecificOptions[T any](base *T, opts ...LoaderSplitterOption) *T {
if base == nil {
base = new(T)
}

for i := range opts {
opt := opts[i]
if opt.implSpecificOptFn != nil {
s, ok := opt.implSpecificOptFn.(func(*T))
if ok {
s(base)
}
}
}

return base
}

// GetLoaderImplSpecificOptions provides Loader author the ability to extract their own custom options from the unified LoaderOption type.
// T: the type of the impl specific options struct.
// This function should be used within the Loader implementation's Load function.
Expand Down
17 changes: 8 additions & 9 deletions components/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,12 @@ func IsCallbacksEnabled(i any) bool {
type Component string

const (
ComponentOfPrompt Component = "ChatTemplate"
ComponentOfChatModel Component = "ChatModel"
ComponentOfEmbedding Component = "Embedding"
ComponentOfIndexer Component = "Indexer"
ComponentOfRetriever Component = "Retriever"
ComponentOfLoaderSplitter Component = "LoaderSplitter"
ComponentOfLoader Component = "Loader"
ComponentOfTransformer Component = "DocumentTransformer"
ComponentOfTool Component = "Tool"
ComponentOfPrompt Component = "ChatTemplate"
ComponentOfChatModel Component = "ChatModel"
ComponentOfEmbedding Component = "Embedding"
ComponentOfIndexer Component = "Indexer"
ComponentOfRetriever Component = "Retriever"
ComponentOfLoader Component = "Loader"
ComponentOfTransformer Component = "DocumentTransformer"
ComponentOfTool Component = "Tool"
)
8 changes: 0 additions & 8 deletions compose/chain.go
Original file line number Diff line number Diff line change
Expand Up @@ -259,14 +259,6 @@ func (c *Chain[I, O]) AppendRetriever(node retriever.Retriever, opts ...GraphAdd
return c
}

// AppendLoaderSplitter add a LoaderSplitter node to the chain.
// Deprecated: use AppendLoader instead.
func (c *Chain[I, O]) AppendLoaderSplitter(node document.LoaderSplitter, opts ...GraphAddNodeOpt) *Chain[I, O] {
gNode, options := toLoaderSplitterNode(node, opts...)
c.addNode(gNode, options)
return c
}

// AppendLoader adds a Loader node to the chain.
// e.g.
//
Expand Down
7 changes: 0 additions & 7 deletions compose/chain_branch.go
Original file line number Diff line number Diff line change
Expand Up @@ -180,13 +180,6 @@ func (cb *ChainBranch) AddRetriever(key string, node retriever.Retriever, opts .
return cb.addNode(key, gNode, options)
}

// AddLoaderSplitter adds a LoaderSplitter node to the branch.
// Deprecated: use AddLoader instead.
func (cb *ChainBranch) AddLoaderSplitter(key string, node document.LoaderSplitter, opts ...GraphAddNodeOpt) *ChainBranch {
gNode, options := toLoaderSplitterNode(node, opts...)
return cb.addNode(key, gNode, options)
}

// AddLoader adds a Loader node to the branch.
// eg.
//
Expand Down
7 changes: 0 additions & 7 deletions compose/chain_parallel.go
Original file line number Diff line number Diff line change
Expand Up @@ -134,13 +134,6 @@ func (p *Parallel) AddRetriever(outputKey string, node retriever.Retriever, opts
return p.addNode(outputKey, gNode, options)
}

// AddLoaderSplitter adds a loader splitter node to the parallel.
// Deprecated: use AddLoader instead.
func (p *Parallel) AddLoaderSplitter(outputKey string, node document.LoaderSplitter, opts ...GraphAddNodeOpt) *Parallel {
gNode, options := toLoaderSplitterNode(node, append(opts, WithOutputKey(outputKey))...)
return p.addNode(outputKey, gNode, options)
}

// AddLoader adds a loader node to the parallel.
// eg.
//
Expand Down
11 changes: 0 additions & 11 deletions compose/component_to_graph_node.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,17 +68,6 @@ func toRetrieverNode(node retriever.Retriever, opts ...GraphAddNodeOpt) (*graphN
opts...)
}

func toLoaderSplitterNode(node document.LoaderSplitter, opts ...GraphAddNodeOpt) (*graphNode, *graphAddNodeOpts) {
return toComponentNode(
node,
components.ComponentOfLoaderSplitter,
node.LoadAndSplit,
nil,
nil,
nil,
opts...)
}

func toLoaderNode(node document.Loader, opts ...GraphAddNodeOpt) (*graphNode, *graphAddNodeOpts) {
return toComponentNode(
node,
Expand Down
6 changes: 0 additions & 6 deletions compose/graph.go
Original file line number Diff line number Diff line change
Expand Up @@ -354,12 +354,6 @@ func (g *graph) AddRetrieverNode(key string, node retriever.Retriever, opts ...G
return g.addNode(key, gNode, options)
}

// Deprecated: use AddLoaderNode instead.
func (g *graph) AddLoaderSplitterNode(key string, node document.LoaderSplitter, opts ...GraphAddNodeOpt) error {
gNode, options := toLoaderSplitterNode(node, opts...)
return g.addNode(key, gNode, options)
}

// AddLoaderNode adds a node that implements document.Loader.
// e.g.
//
Expand Down
5 changes: 0 additions & 5 deletions compose/graph_call_options.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,11 +98,6 @@ func WithRetrieverOption(opts ...retriever.Option) Option {
return withComponentOption(opts...)
}

// Deprecated: use WithLoaderOption instead.
func WithLoaderSplitterOption(opts ...document.LoaderSplitterOption) Option {
return withComponentOption(opts...)
}

// WithLoaderOption is a functional option type for loader component.
// e.g.
//
Expand Down
43 changes: 0 additions & 43 deletions internal/mock/components/document/document_mock.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion utils/callbacks/template_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,6 @@ func TestNewComponentTemplate(t *testing.T) {

types := []components.Component{
components.ComponentOfPrompt,
components.ComponentOfLoaderSplitter,
components.ComponentOfChatModel,
components.ComponentOfEmbedding,
components.ComponentOfRetriever,
Expand Down

0 comments on commit 4d92ca5

Please sign in to comment.