From fc25af8c5b39cd369450217de729231e8338e640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Mon, 28 Oct 2024 12:11:33 +0100 Subject: [PATCH] Rework props --- internal/js/esbuild/batch.go | 486 +++++++++++++++++++++-------------- 1 file changed, 293 insertions(+), 193 deletions(-) diff --git a/internal/js/esbuild/batch.go b/internal/js/esbuild/batch.go index 6dac294bb88..35443bbc518 100644 --- a/internal/js/esbuild/batch.go +++ b/internal/js/esbuild/batch.go @@ -65,9 +65,37 @@ func (g *getOnce[T]) Get() T { return v } -func newOptions(name string) *options { +type optionsOptions struct { + name string + defaultExport string +} + +type optionsCompiler[C any] interface { + compile(*options) (C, error) +} + +type optionsCompiled[C optionsCompiler[C]] struct { + opts *options + compiled C +} + +func (o *optionsCompiled[C]) compile() error { + var c C + c, err := c.compile(o.opts) + if err != nil { + return err + } + o.compiled = c + if err := o.opts.v.checkStale(); err != nil { + return err + } + + return nil +} + +func newOptions(opts optionsOptions) *options { return &options{getOnce[*optionsSetter]{ - v: &optionsSetter{name: name}, + v: &optionsSetter{opts: opts}, }} } @@ -131,7 +159,7 @@ func (c *BatcherClient) New(id string) (Batcher, error) { scriptGroups: make(map[string]*scriptGroup), dependencyManager: dependencyManager, client: c, - configOptions: newOptions("config"), + configOptions: &optionsCompiled[configOptions]{opts: newOptions(optionsOptions{name: "config"})}, }, nil } @@ -183,36 +211,31 @@ func (p *Package) MarkStale() { p.origin.reset() } -func (p *Package) calculateStaleVersion() uint32 { - var sv uint32 +func (p *Package) isStale() bool { for _, v := range p.b.scriptGroups { - if v.instancesOptionsOld != nil { - if len(v.instancesOptions) != len(v.instancesOptionsOld) { - fmt.Println("==> len mismatch") - return 1 - } - for kk, vv := range v.instancesOptions { - if ov, found := v.instancesOptionsOld[kk]; found { - if !reflect.DeepEqual(vv.v, ov.v) { - fmt.Println("==> deep mismatch", "\n", vv, "\n", ov) - return 1 - } - } else { - fmt.Println("==> not found") - return 1 + for _, vv := range v.instancesOptions { + curr := vv.opts.v.optsCurr + prev := vv.opts.v.optsPrev + if prev != nil { + if len(curr) != len(prev) { + return true + } + if !reflect.DeepEqual(curr, prev) { + return true } } } } + var isStale bool p.b.forEeachStaleInfo(func(v resource.StaleInfo) bool { if i := v.StaleVersion(); i > 0 { - sv = i - return true + isStale = true } - return false + return isStale }) - return sv + + return isStale } // You should not depend on the invocation order when calling this. @@ -254,10 +277,49 @@ func (b *batcher) forEeachStaleInfo(f func(si resource.StaleInfo) bool) { } } +type configOptions struct { + Options ExternalOptions +} + +func (s configOptions) compile(o *options) (configOptions, error) { + m := o.commit().optsCurr + + config, err := DecodeExternalOptions(m) + if err != nil { + return configOptions{}, err + } + + return configOptions{ + Options: config, + }, nil +} + +// TODO1 unexport these. type ParamsOptions struct { Params json.RawMessage } +func (s ParamsOptions) compile(o *options) (ParamsOptions, error) { + v := struct { + Params map[string]any + }{} + + m := o.commit().optsCurr + + if err := mapstructure.WeakDecode(m, &v); err != nil { + return ParamsOptions{}, err + } + + paramsJSON, err := json.Marshal(v.Params) + if err != nil { + return ParamsOptions{}, err + } + + return ParamsOptions{ + Params: paramsJSON, + }, nil +} + type ScriptOptions struct { // The script to build. // TODO1 handle stale. @@ -275,6 +337,60 @@ type ScriptOptions struct { Params json.RawMessage } +func (s ScriptOptions) IsZero() bool { + return s.Resource == nil +} + +func (s ScriptOptions) compile(o *options) (ScriptOptions, error) { + v := struct { + Resource resource.Resource + ImportContext any + Export string + Params map[string]any + }{} + + m := o.commit().optsCurr + + if err := mapstructure.WeakDecode(m, &v); err != nil { + panic(err) + } + + var paramsJSON []byte + if v.Params != nil { + var err error + paramsJSON, err = json.Marshal(v.Params) + if err != nil { + panic(err) + } + } + + if v.Export == "" { + v.Export = o.v.opts.defaultExport + } + + importContext := o.v.compiledImportContext + if importContext == nil { + importContext = resource.NewCachedResourceGetter(v.ImportContext) + } + + compiled := ScriptOptions{ + Resource: v.Resource, + Export: v.Export, + ImportContext: importContext, + Params: paramsJSON, + } + + if compiled.Resource == nil { + return ScriptOptions{}, fmt.Errorf("resource not set") + } + + // TODO1 improve. + o.v.compiledImportContext = compiled.ImportContext + + return compiled, nil +} + +// TODO1 check usage. func (o ScriptOptions) Compile(m map[string]any) (*ScriptOptions, error) { var s optionsGetSet // TODO1 type. if err := mapstructure.WeakDecode(m, &s); err != nil { @@ -318,14 +434,11 @@ type batcher struct { client *BatcherClient dependencyManager identity.Manager - configOptions *options + configOptions *optionsCompiled[configOptions] // The last successfully built package. // If this is non-nil and not stale, we can reuse it (e.g. on server rebuilds) prevBuild *Package - - // Compiled. - config ExternalOptions } func (b *batcher) Build(ctx context.Context) (*Package, error) { @@ -345,7 +458,7 @@ func (b *batcher) Build(ctx context.Context) (*Package, error) { } func (b *batcher) Config() OptionsSetter { - return b.configOptions.Get() + return b.configOptions.opts.Get() } func (b *batcher) Group(id string) BatcherGroup { @@ -356,9 +469,9 @@ func (b *batcher) Group(id string) BatcherGroup { if !found { group = &scriptGroup{ id: id, b: b, - scriptsOptions: make(map[string]*options), - instancesOptions: make(map[instanceID]*options), - runnersOptions: make(map[string]*options), + scriptsOptions: make(optionsCompiledMap[string, ScriptOptions]), + instancesOptions: make(optionsCompiledMap[instanceID, ParamsOptions]), + runnersOptions: make(optionsCompiledMap[string, ScriptOptions]), } b.scriptGroups[id] = group } @@ -380,7 +493,7 @@ func (b *batcher) build(ctx context.Context) (*Package, error) { }() if b.prevBuild != nil { - if b.prevBuild.calculateStaleVersion() == 0 { + if b.prevBuild.isStale() { return b.prevBuild, nil } b.removeStale() @@ -395,9 +508,7 @@ func (b *batcher) build(ctx context.Context) (*Package, error) { } func (b *batcher) compile() error { - var err error - b.config, err = DecodeExternalOptions(b.configOptions.commit().opts) - if err != nil { + if err := b.configOptions.compile(); err != nil { return err } @@ -424,7 +535,7 @@ func (b *batcher) doBuild(ctx context.Context) (*Package, error) { type importContext struct { name string resourceGetter resource.ResourceGetter - scriptOptions *ScriptOptions // TODO1 remove resourceGetter? + scriptOptions ScriptOptions } state := struct { @@ -457,11 +568,11 @@ func (b *batcher) doBuild(ctx context.Context) (*Package, error) { for k, v := range b.scriptGroups { keyPath := keyPath + "_" + k var runners []scriptRunnerTemplateContext - for _, vv := range v.runners.Sorted() { - runnerKeyPath := keyPath + "_" + vv.ID - runnerImpPath := paths.AddLeadingSlash(runnerKeyPath + "_runner" + vv.Resource.MediaType().FirstSuffix.FullSuffix) - runners = append(runners, scriptRunnerTemplateContext{script: vv, Import: runnerImpPath}) - addResource(k, runnerImpPath, vv.Resource, false) + for _, vv := range v.runnersOptions.Sorted() { + runnerKeyPath := keyPath + "_" + vv.key + runnerImpPath := paths.AddLeadingSlash(runnerKeyPath + "_runner" + vv.compiled.Resource.MediaType().FirstSuffix.FullSuffix) + runners = append(runners, scriptRunnerTemplateContext{keyOpts: vv, Import: runnerImpPath}) + addResource(k, runnerImpPath, vv.compiled.Resource, false) } t := &batchTemplateContext{ @@ -470,11 +581,11 @@ func (b *batcher) doBuild(ctx context.Context) (*Package, error) { Runners: runners, } - instances := v.instances.Sorted() + instances := v.instancesOptions.Sorted() - for _, vv := range v.scripts.Sorted() { - keyPath := keyPath + "_" + vv.ID - opts := vv.ScriptOptions + for _, vv := range v.scriptsOptions.Sorted() { + keyPath := keyPath + "_" + vv.key + opts := vv.compiled impPath := path.Join(PrefixHugoVirtual, opts.Dir(), keyPath+opts.Resource.MediaType().FirstSuffix.FullSuffix) impCtx := opts.ImportContext @@ -485,13 +596,15 @@ func (b *batcher) doBuild(ctx context.Context) (*Package, error) { }) bt := scriptBatchTemplateContext{ - script: vv, - Import: impPath, + keyOpts: vv, + Import: impPath, } - - state.importResource.Set(bt.Import, vv.Resource) - for _, vvv := range instances.ByScriptID(vv.ID) { - bt.Instances = append(bt.Instances, scriptInstanceBatchTemplateContext{instance: vvv}) + state.importResource.Set(bt.Import, vv.compiled.Resource) + predicate := func(k instanceID) bool { + return k.scriptID == vv.key + } + for _, vvv := range instances.Filter(predicate) { + bt.Instances = append(bt.Instances, scriptInstanceBatchTemplateContext{keyOpts: vvv}) } t.Scripts = append(t.Scripts, bt) @@ -505,7 +618,6 @@ func (b *batcher) doBuild(ctx context.Context) (*Package, error) { state.importerImportContext.Set(s, importContext{ name: s, resourceGetter: nil, - scriptOptions: nil, }) addResource(v.id, s, r, true) @@ -520,7 +632,7 @@ func (b *batcher) doBuild(ctx context.Context) (*Package, error) { return nil, err } - externalOptions := b.config + externalOptions := b.configOptions.compiled.Options if externalOptions.Format == "" { externalOptions.Format = "esm" } @@ -576,7 +688,7 @@ func (b *batcher) doBuild(ctx context.Context) (*Package, error) { }, ImportParamsOnLoadFunc: func(args api.OnLoadArgs) json.RawMessage { if importContext, found := state.importerImportContext.Get(args.Path); found { - if importContext.scriptOptions != nil { + if !importContext.scriptOptions.IsZero() { return importContext.scriptOptions.Params } } @@ -777,26 +889,9 @@ type instanceID struct { } type ( - instanceMap map[instanceID]*ParamsOptions - instances []*instance + instances []*instance ) -func (p instanceMap) Sorted() instances { - var a []*instance - for k, v := range p { - a = append(a, &instance{instanceID: k, ParamsOptions: v}) - } - sort.Slice(a, func(i, j int) bool { - ai := a[i] - aj := a[j] - if ai.instanceID.scriptID != aj.instanceID.scriptID { - return ai.instanceID.scriptID < aj.instanceID.scriptID - } - return ai.instanceID.instanceID < aj.instanceID.instanceID - }) - return a -} - func (i instances) ByScriptID(id string) instances { var a instances for _, v := range i { @@ -813,7 +908,7 @@ type options struct { func (o *options) Reset() { mu := o.once.ResetWithLock() - deb(o.v.name, "reset", o.v.opts) + deb(o.v.opts.name, "reset", o.v.optsCurr) o.v.staleVersion.Store(0) mu.Unlock() } @@ -859,69 +954,64 @@ func (s *optionsGetSet) SetOptions(m map[string]any) string { } type optionsSetter struct { - name string + opts optionsOptions + staleVersion atomic.Uint32 - opts map[string]any - // Compiled early so we can compare for equality. + optsCurr map[string]any + // Preserve one generation of options so we can detect changes. + optsPrev map[string]any + + // Compiled early so we can compare for equality. TODO1 remove. compiledImportContext resource.ResourceGetter } -func (o *optionsSetter) SetOptions(m map[string]any) string { - if o.opts != nil { - if reflect.DeepEqual(o.opts, m) { - return "" - } - var isStale bool - for k, v := range m { - vv, found := o.opts[k] - if !found { - isStale = true - } else { - if si, ok := vv.(resource.StaleInfo); ok { - isStale = si.StaleVersion() > 0 - if isStale { - deb2("stale info", k) - } - } else if strings.EqualFold(k, "ImportContext") { - newImportContext := resource.NewCachedResourceGetter(v) - isStale = !o.compiledImportContext.(resource.IsProbablySameResourceGetter).IsProbablySameResourceGetter(newImportContext) - if isStale { - deb2("stale importContext", k) - } +func (o *optionsSetter) checkStale() error { + if o.optsPrev != nil { + isStale := func() bool { + if len(o.optsCurr) != len(o.optsPrev) { + return true + } + if reflect.DeepEqual(o.optsCurr, o.optsPrev) { + return false + } + for k, v := range o.optsPrev { + vv, found := o.optsCurr[k] + if !found { + return true } else { - isStale = !reflect.DeepEqual(v, vv) - if isStale { - deb2("stale reflect", k) + if si, ok := vv.(resource.StaleInfo); ok { + if si.StaleVersion() > 0 { + return true + } + } else if strings.EqualFold(k, "ImportContext") { + newImportContext := resource.NewCachedResourceGetter(v) + if !o.compiledImportContext.(resource.IsProbablySameResourceGetter).IsProbablySameResourceGetter(newImportContext) { + return true + } + } else { + if !reflect.DeepEqual(v, vv) { + return true + } } } } - if isStale { - break - } - } + return false + }() - if !isStale { - return "" + if isStale { + o.staleVersion.Add(1) } - - o.staleVersion.Add(1) } - o.opts = m - - // Compile the import context so we can compare for equality. - v := struct { - ImportContext any - }{} - - if err := mapstructure.WeakDecode(m, &v); err != nil { - panic(err) - } + o.optsPrev = o.optsCurr - o.compiledImportContext = resource.NewCachedResourceGetter(v.ImportContext) + return nil +} +func (o *optionsSetter) SetOptions(m map[string]any) string { + o.optsCurr = m return "" } @@ -931,17 +1021,21 @@ type script struct { } type scriptBatchTemplateContext struct { - *script + keyOpts[string, ScriptOptions] Import string Instances []scriptInstanceBatchTemplateContext } +func (s *scriptBatchTemplateContext) Export() string { + return s.keyOpts.compiled.Export +} + func (c scriptBatchTemplateContext) MarshalJSON() (b []byte, err error) { return json.Marshal(&struct { ID string `json:"id"` Instances []scriptInstanceBatchTemplateContext `json:"instances"` }{ - ID: c.ID, + ID: c.key, Instances: c.Instances, }) } @@ -956,7 +1050,7 @@ func (b scriptBatchTemplateContext) RunnerJSON(i int) string { Binding string `json:"binding"` Instances []scriptInstanceBatchTemplateContext `json:"instances"` }{ - b.ID, + b.key, script, b.Instances, } @@ -973,6 +1067,42 @@ func (b scriptBatchTemplateContext) RunnerJSON(i int) string { return s } +type optionsCompiledMap[K comparable, T optionsCompiler[T]] map[K]*optionsCompiled[T] + +type keyOpts[K comparable, T optionsCompiler[T]] struct { + key K + *optionsCompiled[T] +} + +type keyOptss[K comparable, T optionsCompiler[T]] []keyOpts[K, T] + +func (ko keyOptss[K, T]) Filter(predicate func(K) bool) keyOptss[K, T] { + var a keyOptss[K, T] + for _, v := range ko { + if predicate(v.key) { + a = append(a, v) + } + } + return a +} + +func (o optionsCompiledMap[K, T]) Sorted() keyOptss[K, T] { + var keys []K + for k := range o { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + // TODO1 + return fmt.Sprintf("%v", keys[i]) < fmt.Sprintf("%v", keys[j]) + }) + + var ko []keyOpts[K, T] + for _, k := range keys { + ko = append(ko, keyOpts[K, T]{key: k, optionsCompiled: o[k]}) + } + return ko +} + type scriptGroup struct { mu sync.Mutex @@ -980,18 +1110,10 @@ type scriptGroup struct { b *batcher - scriptsOptions map[string]*options - instancesOptions map[instanceID]*options - instancesOptionsOld map[instanceID]*options + scriptsOptions optionsCompiledMap[string, ScriptOptions] + instancesOptions optionsCompiledMap[instanceID, ParamsOptions] instancesOptionsBuildID int - runnersOptions map[string]*options - - // Compiled. - scripts scriptMap - - // TODO1. For instances, for rebuilds, clear instance map for script on first new set. That should solve the "shortcode ordinal key issue". - instances instanceMap - runners scriptMap + runnersOptions optionsCompiledMap[string, ScriptOptions] } func (s *scriptGroup) Instance(sid, iid string) OptionsSetter { @@ -1000,39 +1122,40 @@ func (s *scriptGroup) Instance(sid, iid string) OptionsSetter { if s.instancesOptionsBuildID != s.b.buildCount { // New build, reset the instances. // Store the old map so we can compare for equality. - s.instancesOptionsOld = s.instancesOptions + /*s.instancesOptionsOld = s.instancesOptions s.instancesOptions = make(map[instanceID]*options) s.instances = make(map[instanceID]*ParamsOptions) s.instancesOptionsBuildID = s.b.buildCount + */ } id := instanceID{scriptID: sid, instanceID: iid} if v, found := s.instancesOptions[id]; found { - return v.Get() + return v.opts.Get() } - s.instancesOptions[id] = newOptions("instance") - return s.instancesOptions[id].Get() + s.instancesOptions[id] = &optionsCompiled[ParamsOptions]{opts: newOptions(optionsOptions{name: "instance"})} + return s.instancesOptions[id].opts.Get() } func (g *scriptGroup) Reset() { for _, v := range g.scriptsOptions { - v.Reset() + v.opts.Reset() } for _, v := range g.instancesOptions { - v.Reset() + v.opts.Reset() } for _, v := range g.runnersOptions { - v.Reset() + v.opts.Reset() } } func (g *scriptGroup) removeStale() bool { for k, v := range g.scriptsOptions { - if v.StaleVersion() > 0 { - deb("remove script", v.v.name, k) - delete(g.scripts, k) + if v.opts.StaleVersion() > 0 { + deb("remove script", v.opts.v.opts.name, k) + delete(g.scriptsOptions, k) for kk := range g.instancesOptions { if kk.scriptID == k { - delete(g.instances, kk) + delete(g.scriptsOptions, kk.scriptID) } } } @@ -1045,20 +1168,20 @@ func (s *scriptGroup) Runner(id string) OptionsSetter { s.mu.Lock() defer s.mu.Unlock() if v, found := s.runnersOptions[id]; found { - return v.Get() + return v.opts.Get() } - s.runnersOptions[id] = newOptions("runner") - return s.runnersOptions[id].Get() + s.runnersOptions[id] = &optionsCompiled[ScriptOptions]{opts: newOptions(optionsOptions{name: "runner", defaultExport: "default"})} + return s.runnersOptions[id].opts.Get() } func (s *scriptGroup) Script(id string) OptionsSetter { s.mu.Lock() defer s.mu.Unlock() if v, found := s.scriptsOptions[id]; found { - return v.Get() + return v.opts.Get() } - s.scriptsOptions[id] = newOptions("script") - return s.scriptsOptions[id].Get() + s.scriptsOptions[id] = &optionsCompiled[ScriptOptions]{opts: newOptions(optionsOptions{name: "script", defaultExport: "*"})} + return s.scriptsOptions[id].opts.Get() } func (s *scriptGroup) errFailedToCompile(what, id string, err error) error { @@ -1066,33 +1189,22 @@ func (s *scriptGroup) errFailedToCompile(what, id string, err error) error { } func (s *scriptGroup) compile() error { - // TODO1 lock? - s.scripts = make(map[string]*ScriptOptions) - s.instances = make(map[instanceID]*ParamsOptions) - s.runners = make(map[string]*ScriptOptions) - for k, v := range s.scriptsOptions { - compiled, err := compileScriptOptions(v, "*") - if err != nil { + if err := v.compile(); err != nil { return s.errFailedToCompile("Script", k, err) } - s.scripts[k] = compiled } for k, v := range s.instancesOptions { - compiled, err := compileParamsOptions(v) - if err != nil { - return err + if err := v.compile(); err != nil { + return s.errFailedToCompile("Instance", k.instanceID, err) // TODO1 } - s.instances[k] = compiled } for k, v := range s.runnersOptions { - compiled, err := compileScriptOptions(v, "default") - if err != nil { - return s.errFailedToCompile("Runner", k, err) + if err := v.compile(); err != nil { + return s.errFailedToCompile("Runner", k, err) // TODO1 } - s.runners[k] = compiled } return nil @@ -1112,11 +1224,11 @@ func (s scriptGroups) Sorted() []*scriptGroup { } type scriptInstanceBatchTemplateContext struct { - *instance + keyOpts[instanceID, ParamsOptions] } func (c scriptInstanceBatchTemplateContext) ID() string { - return c.instanceID.instanceID + return c.key.instanceID } func (c scriptInstanceBatchTemplateContext) MarshalJSON() (b []byte, err error) { @@ -1124,8 +1236,8 @@ func (c scriptInstanceBatchTemplateContext) MarshalJSON() (b []byte, err error) ID string `json:"id"` Params json.RawMessage `json:"params"` }{ - ID: c.instanceID.instanceID, - Params: c.Params, + ID: c.key.instanceID, + Params: c.compiled.Params, }) } @@ -1143,42 +1255,30 @@ func (s scriptMap) Sorted() []*script { } type scriptRunnerTemplateContext struct { - *script + keyOpts[string, ScriptOptions] Import string } +func (s *scriptRunnerTemplateContext) Export() string { + return s.keyOpts.compiled.Export +} + func (c scriptRunnerTemplateContext) MarshalJSON() (b []byte, err error) { return json.Marshal(&struct { ID string `json:"id"` }{ - ID: c.ID, + ID: c.key, }) } func (g *getOnce[T]) commit() T { - g.once.Do(func() {}) + g.once.Do(func() { + }) return g.v } func compileParamsOptions(o *options) (*ParamsOptions, error) { - v := struct { - Params map[string]any - }{} - - m := o.commit().opts - - if err := mapstructure.WeakDecode(m, &v); err != nil { - return nil, err - } - - paramsJSON, err := json.Marshal(v.Params) - if err != nil { - return nil, err - } - - return &ParamsOptions{ - Params: paramsJSON, - }, nil + return nil, nil // TODO1 remove } func compileScriptOptions(o *options, defaultExport string) (*ScriptOptions, error) { @@ -1189,7 +1289,7 @@ func compileScriptOptions(o *options, defaultExport string) (*ScriptOptions, err Params map[string]any }{} - m := o.commit().opts + m := o.commit().optsCurr if err := mapstructure.WeakDecode(m, &v); err != nil { panic(err)