bug: 大文件上传接口引起内存溢出

[q343959872]

Current Behavior

网关在转发大附件上传的接口时，超过500M的文件就出现内存溢出，实际服务器可用内存大于6G。

需要帮助排查下是设计代码上的缺陷还是怎么样

Expected Behavior

No response

Error Logs

fatal error: runtime: out of memory

runtime stack:
runtime.throw({0x27b8152?, 0x2030?})
/opt/hostedtoolcache/go/1.21.1/x64/src/runtime/panic.go:1077 +0x5c fp=0xc000093df8 sp=0xc000093dc8 pc=0x43e4bc
runtime.sysMapOS(0xc191400000, 0x36c00000?)
/opt/hostedtoolcache/go/1.21.1/x64/src/runtime/mem_linux.go:167 +0x116 fp=0xc000093e40 sp=0xc000093df8 pc=0x41d036
runtime.sysMap(0x4252b20?, 0x4322e0?, 0x4262ce8?)
/opt/hostedtoolcache/go/1.21.1/x64/src/runtime/mem.go:155 +0x34
fp=0xc000093e70 sp=0xc000093e40 pc=0x41ca14
runtime.(*mheap).grow(0x4252b20, 0x1b5bb?)
/opt/hostedtoolcache/go/1.21.1/x64/src/runtime/mheap.go:1533 +0x236 fp=0xc000093ee0 sp=0xc000093e70 pc=0x42f116
runtime.(*mheap).allocSpan(0x4252b20, 0x1b5bb, 0x0, 0x3f?)
/opt/hostedtoolcache/go/1.21.1/x64/src/runtime/mheap.go:1250 +0x1b0 fp=0xc000093f80 sp=0xc000093ee0 pc=0x42e830
runtime.(*mheap).alloc.func1()
/opt/hostedtoolcache/go/1.21.1/x64/src/runtime/mheap.go:968 +0x5c fp=0xc000093fc8
sp=0xc000093f80 pc=0x42e2bc
runtime.systemstack
()
/opt/hostedtoolcache/go/1.21.1/x64/src/runtime/asm_amd64.s:509
+0x4a fp=0xc000093fd8 sp=0xc000093fc8 pc=0x4747ea

goroutine 217
[running]:
runtime.systemstack_switch()

/opt/hostedtoolcache/go/1.21.1/x64/src/runtime/asm_amd64.s:474 +0x8
fp=0xc000692d48 sp=0xc000692d38 pc=0x474788

runtime.(*mheap).alloc(0x36b76000?, 0x1b5bb?
, 0xc8?)
/opt/hostedtoolcache/go/1.21.1/x64/src/runtime/mheap.go:962 +0x5b fp=0xc000692d90 sp=0xc000692d48 pc=0x42e21b
runtime.(*mcache).allocLarge(0x7f267d77c108
?, 0x36b76000, 0x0?)
/opt/hostedtoolcache/go/1.21.1/x64/src/runtime/mcache.go:234 +0x85 fp=
0xc000692dd8 sp=0xc000692d90 pc=0x41bbe5
runtime.mallocgc(0x36b76000, 0x0, 0x0)
/opt/hostedtoolcache/go/1.21.1/x64/src/runtime/malloc.go:1123 +0x4f6 fp=0xc000692e40 sp=0xc000692dd8 pc=0x412df6
runtime.growslice(0xc1616ce000, 0x2bc5e001?, 0x2bc5e000?
, 0x1?, 0x21c6e20?)
/opt/hostedtoolcache/go/1.21.1/x64/src/runtime/slice.go:266 +0x4cf
fp=0xc000692eb0 sp=0xc000692e40 pc=0x4579af
io.ReadAll(
{0x7f26366f5028, 0xc000282288})
/opt/hostedtoolcache/go/1.21.1/x64/src/io/io.go:715 +
0xd2 fp=0xc000692f28 sp=0xc000692eb0 pc=
0x4cc572
github.com/eolinker/apinto/node/http-context.(*BodyRequestHandler).resetFile(0xc00011da08)
/home/runner/work/apinto/apinto/node/http-context/body.go:231 +
0x4f4 fp=0xc0006930f0 sp=0xc000692f28 pc=0x1a0edf4
github.com/eolinker/apinto/node/http-context.(*BodyRequestHandler).MultipartForm(0xc00011da08)
/home/runner/work/apinto/apinto/node/http-context/body.go
:55 +0xd5 fp=0xc000693148
sp=0xc0006930f0 pc=0x1a0db55
github.com/eolinker/apinto/node/http-context.(*BodyRequestHandler).BodyForm
(0xc00011da08)
/home/runner/work/apinto/apinto/node/http-context/body.go:111
+0xdf fp=0xc000693188 sp=0xc000693148 pc=
0x1a0deff
github.com/eolinker/apinto/drivers/app.parseBodyParams
({0x2d488d0, 0xc00011da00
})
/home/runner/work/apinto/apinto/drivers/app/extend-param.go:141
+0x1f7 fp=0xc0006931d0 sp=0xc000693188
pc=0x1b6f8d7
github.com/eolinker/apinto/drivers/app.(*additionalParam).Execute
(0xc000c219b0, {0x2d488d0,
0xc00011da00})
/home/runner/work/apinto/apinto/drivers/app/extend-param.go:
31 +0x7b fp=0xc000693318 sp=0xc0006931d0
pc=0x1b6e5fb
github.com/eolinker/apinto/drivers/app.(*executor).Execute(0x10?,
{0x2d488d0, 0xc00011da00})
/home/runner/work/apinto/apinto/drivers/app/executor.go:28 +0x64
fp=0xc000693358 sp=0xc000693318 pc=0x1b6e524
github.com/eolinker/apinto/drivers/app.(*app).Execute(0x2d488d0
?, {0x2d488d0?, 0xc00011da00
?})
/home/runner/work/apinto/apinto/drivers/app/app.go:
26 +0x2f fp=0xc000693380 sp=
0xc000693358 pc=0x1b6ce6f
github.com/eolinker/apinto/drivers/plugins/app.(*App).auth(0xc000bbc000, {
0x2d488d0, 0xc00011da00})
/home/runner/work/apinto/apinto/drivers/plugins/app/app.go
:111 +0x698 fp=0xc0006934a8 sp=0xc000693380
pc=0x1b91d38
github.com/eolinker/apinto/drivers/plugins/app.(*App).DoHttpFilter
(0x18?,
{0x2d488d0?, 0xc00011da00},
{0x2cfc958, 0xc000c8eac8}
)
/home/runner/work/apinto/apinto/drivers/plugins/app/app.go:32 +0x90
fp=0xc000693510 sp=0xc0006934a8 pc=0x1b91110
github.com/eolinker/eosc/eocontext/http-context.DoHttpFilter({0x2ceb160, 0xc000bbc000}, {0x2d447b0, 0xc00011da00}
, {0x2cfc958, 0xc000c8eac8})
/home/runner/go/pkg/mod/github.com/eolinker/[email protected]/eocontext/http-context/filter.go:19 +0xb8 fp=0xc000693550 sp=0xc000693510 pc=0xe376b8
github.com/eolinker/apinto/drivers/plugins/app.(*App).DoFilter(0xc000c8c0e0?, {0x2d447b0?, 0xc00011da00?}, {0x2cfc958?, 0xc000c8eac8?})
/home/runner/work/apinto/apinto/drivers/plugins/app/app.go:24 +0x3d fp=0xc000693590 sp=0xc000693550 pc=0x1b90ffd

github.com/eolinker/eosc/eocontext.Filters.DoChain(
{0xc000c8c0d0?, 0x410d8c?,
0x2603000?}, {0x2d447b0,
0xc00011da00})
/home/runner/go/pkg/mod/github.com/eolinker/[email protected]/eocontext/filter.go:
27 +0x82 fp=0xc0006935d8 sp=
0xc000693590 pc=0xe36f62
github.com/eolinker/eosc/eocontext.(*Filters).DoChain(
0xc000a89ce0?, {0x2d447b0?
, 0xc00011da00?})
:1 +0x45 fp=0xc000693610 sp=0xc0006935d8 pc=0xe37465
github.com/eolinker/apinto/drivers/plugins/proxy_rewrite_v2.(*ProxyRewrite).DoHttpFilter(0xc000a89ce0, {0x2d488d0, 0xc00011da00}
, {0x2cfc958, 0xc000c8eab0})

/home/runner/work/apinto/apinto/drivers/plugins/proxy_rewrite_v2/proxy-rewrite2.go:60 +0x19a fp=0xc000693678 sp=0xc000693610
pc=0x2033dba
github.com/eolinker/eosc/eocontext/http-context.DoHttpFilter(
{0x2ceb980, 0xc000a89ce0}, {0x2d447b0
, 0xc00011da00}, {0x2cfc958, 0xc000c8eab0
})
/home/runner/go/pkg/mod/github.com/eolinker/[email protected]/eocontext/http-context/filter.go:19 +
0xb8 fp=0xc0006936b8 sp=0xc000693678 pc=
0xe376b8
github.com/eolinker/apinto/drivers/plugins/proxy_rewrite_v2.(*ProxyRewrite).DoFilter(0xc000c8c0d0?,
{0x2d447b0?, 0xc00011da00?
}, {0x2cfc958?, 0xc000c8eab0?
})
/home/runner/work/apinto/apinto/drivers/plugins/proxy_rewrite_v2/proxy-rewrite2.go:48 +
0x3d fp=0xc0006936f8 sp=0xc0006936b8 pc=0x2033bbd
github.com/eolinker/eosc/eocontext.Filters.DoChain({0xc000c8c0c0?,
0x410d8c?, 0x2603000?},
{0x2d447b0, 0xc00011da00})
/home/runner/go/pkg/mod/github.com/eolinker/[email protected]/eocontext/filter.go:27 +0x82 fp=0xc000693740 sp=0xc0006936f8 pc=0xe36f62
github.com/eolinker/eosc/eocontext.(*Filters).DoChain(0xc0008cade0?, {0x2d447b0?, 0xc00011da00?})
:1 +0x45 fp=0xc000693778 sp=0xc000693740 pc=0xe37465
github.com/eolinker/apinto/drivers/plugins/monitor.(*worker).DoHttpFilter(0xc000b96c80, {
0x2d488d0, 0xc00011da00}, {0x2cfc958, 0xc000c8ea98})
/home/runner/work/apinto/apinto/drivers/plugins/monitor/monitor.go:28 +
0x103 fp=0xc0006937f8 sp=0xc000693778 pc=0x202a123

github.com/eolinker/eosc/eocontext/http-context.DoHttpFilter({0x2ceb840,
0xc000b96c80}, {0x2d447b0,
0xc00011da00}, {0x2cfc958,
0xc000c8ea98})
/home/runner/go/pkg/mod/github.com/eolinker/[email protected]/eocontext/http-context/filter.go:
19 +0xb8 fp=0xc000693838 sp=0xc0006937f8 pc=
0xe376b8
github.com/eolinker/apinto/drivers/plugins/monitor.(*worker).DoFilter(0xc000c8c0c0?,
{0x2d447b0?, 0xc00011da00?}
, {0x2cfc958?, 0xc000c8ea98
?})
/home/runner/work/apinto/apinto/drivers/plugins/monitor/monitor.go:21 +0x3d fp=0xc000693878 sp=0xc000693838 pc=0x2029fbd
github.com/eolinker/eosc/eocontext.Filters.DoChain(
{0xc000c8c0b0?, 0x410d8c?, 0x2603000?
}, {0x2d447b0, 0xc00011da00}
)
/home/runner/go/pkg/mod/github.com/eolinker/[email protected]/eocontext/filter.go:27 +0x82
fp=0xc0006938c0 sp=0xc000693878 pc=0xe36f62
github.com/eolinker/eosc/eocontext.(*Filters).DoChain(0x10?, {
0x2d447b0?, 0xc00011da00?}
)
:1 +0x45
fp=0xc0006938f8 sp=0xc0006938c0 pc=
0xe37465
github.com/eolinker/apinto/drivers/plugins/access-log.(*accessLog).DoHttpFilter(0xc000babec0
, {0x2d488d0?, 0xc00011da00}
, {0x2cfc958, 0xc000c8ea80}
)
/home/runner/work/apinto/apinto/drivers/plugins/access-log/log.go:28 +0x69 fp=
0xc000693968 sp=0xc0006938f8 pc=0x2000029
github.com/eolinker/eosc/eocontext/http-context.DoHttpFilter({0x2ceb4a0, 0xc000babec0},
{0x2d447b0, 0xc00011da00}, {0x2cfc958
, 0xc000c8ea80})
/home/runner/go/pkg/mod/github.com/eolinker/[email protected]/eocontext/http-context/filter.go:
19 +0xb8 fp=0xc0006939a8 sp=0xc000693968
pc=0xe376b8
github.com/eolinker/apinto/drivers/plugins/access-log.(*accessLog).DoFilter(
0xc000c8c0b0?, {0x2d447b0?
, 0xc00011da00?}, {
0x2cfc958?, 0xc000c8ea80?}
)
/home/runner/work/apinto/apinto/drivers/plugins/access-log/log.go:24 +
0x3d fp=0xc0006939e8 sp=0xc0006939a8 pc=
0x1ffff5d
github.com/eolinker/eosc/eocontext.Filters.DoChain(
{0xc000c8c0a0?, 0x412f05
?, 0xc000ab6220?},
{0x2d447b0, 0xc00011da00}
)
/home/runner/go/pkg/mod/github.com/eolinker/[email protected]/eocontext/filter.go:27 +0x82 fp=
0xc000693a30 sp=0xc0006939e8 pc=0xe36f62
github.com/eolinker/eosc/eocontext.DoChain({0x2d447b0, 0xc00011da00}, {0xc000ae4800?, 0x9, 0x413285?}, {0xc000ab6220?, 0x1, 0x1?})
/home/runner/go/pkg/mod/github.com/eolinker/[email protected]/eocontext/filter.go:
57 +0x10a fp=0xc000693a80 sp=0xc000693a30 pc=0xe3714a
github.com/eolinker/apinto/drivers/plugin-manager.(*PluginObj).Chain(0xc00011da00?, {0x2d447b0?, 0xc00011da00?}, {0xc000ab6220?, 0xc?, 0x4?})
/home/runner/work/apinto/apinto/drivers/plugin-manager/obj.go:21 +0x45 fp=0xc000693ad0 sp=0xc000693a80 pc=0x161b925
github.com/eolinker/apinto/drivers/router/http-router.(*httpHandler).Serve(0xc000b98bd0, {0x2d447b0, 0xc00011da00})
/home/runner/work/apinto/apinto/drivers/router/http-router/http-handler.go:83 +0x6ae
fp=0xc000693bb0 sp=0xc000693ad0 pc=0x1a2202e
github.com/eolinker/apinto/drivers/router/http-router/manager.(*Manager).FastHandler
(0x41ef7c0, 0xc000c89c48?, 0x4e5bef?)
/home/runner/work/apinto/apinto/drivers/router/http-router/manager/manager.go:102 +0x404 fp=0xc000693c30
sp=0xc000693bb0 pc=0x1a1d9a4
github.com/eolinker/apinto/drivers/router/http-router/manager.init.0.func1.1(0x0?)

/home/runner/work/apinto/apinto/drivers/router/http-router/manager/init.go:29 +0x25 fp=0xc000693c58 sp=
0xc000693c30 pc=0x1a1e845
github.com/valyala/fasthttp.(*Server).serveConn(0xc000c80000, {0x2d225d8?, 0xc0000faf50})
/home/runner/go/pkg/mod/github.com/valyala/[email protected]/server.go
:2365 +0x11d4 fp=0xc000693ec8 sp=0xc000693c58 pc=0x19bbaf4

github.com/valyala/fasthttp.(*Server).serveConn-fm({0x2d225d8?,
0xc0000faf50?})
:1 +0x33 fp=0xc000693ef0 sp=0xc000693ec8 pc=0x19cb653
github.com/valyala/fasthttp.(*workerPool).workerFunc(0xc000c8c000, 0xc000989880)
/home/runner/go/pkg/mod/github.com/valyala/[email protected]/workerpool.go:224 +0xa4 fp=0xc000693fa0 sp=0xc000693ef0 pc=0x19c85e4

Steps to Reproduce

1、配置API转发附件上传接口
2、通过HTTP请求工具配置接口，选择附件大于500M
3、发起请求
4、等待一阵会出现请求失败
5、查看网关异常日志，错误如上：Error Logs

Environment

• APINTO version (run apinto version): 0.19.4
• Operating system (run uname -a): Linux redgpts04 3.10.0-1160.el7.x86_64 init #1 SMP Mon Oct 19 16:18:59 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux

[modiao2018]

我也遇到这个问题了，这个应该是设计上的缺陷，附件的转发不应该全丢到内存的。

[Dot-Liu]

好的，我们先研究下，尽快对该问题进行处理