Allow calls to vararg functions to be inlined if the number of arguments are known at compile-time

Compiler/src/Compiler.cpp

@@ -294,7 +294,7 @@ struct Compiler
         f.upvals = upvals;
 
         // record information for inlining
-        if (options.optimizationLevel >= 2 && !func->vararg && !func->self && !getfenvUsed && !setfenvUsed)
+        if (options.optimizationLevel >= 2 && !func->self && !getfenvUsed && !setfenvUsed)
         {
             f.canInline = true;
             f.stackSize = stackSize;
@@ -403,7 +403,41 @@ struct Compiler
 
         setDebugLine(expr); // normally compileExpr sets up line info, but compileExprVarargs can be called directly
 
-        bytecode.emitABC(LOP_GETVARARGS, target, multRet ? 0 : uint8_t(targetCount + 1), 0);
+        /*
+        bool doVararg = true;
+        if (inlineFrames.size() > 0)
+        {
+            bool foundArg = false;
+
+            for (AstNode* arg : inlineFrames.back().call->args)
+                if (arg == expr)
+                {
+                    foundArg = true;
+                    break;
+                }
+
+            if (!foundArg)
+                doVararg = false;
+        }
+        */
+
+        if (inlineFrames.size() == 0)
+            bytecode.emitABC(LOP_GETVARARGS, target, multRet ? 0 : uint8_t(targetCount + 1), 0);
+        else
+        {
+            const InlineFrame& frame = inlineFrames.back();
+            LUAU_ASSERT(frame.call);
+
+            // we need to compile the arguments themselves instead of a vararg
+
+            for (size_t index = 0; index < frame.call->args.size; index++)
+            {
+                AstExpr* arg = frame.call->args.data[index]->asExpr();
+                LUAU_ASSERT(arg);
+
+                compileExpr(arg, target + index);
+            }
+        }
     }
 
     void compileExprSelectVararg(AstExprCall* expr, uint8_t target, uint8_t targetCount, bool targetTop, bool multRet, uint8_t regs)
@@ -590,10 +624,28 @@ struct Compiler
         // - additionally, we can't easily compile multret expressions into designated target as computed call arguments will get clobbered
         if (multRet)
         {
-            bytecode.addDebugRemark("inlining failed: can't convert fixed returns to multret");
+            bytecode.addDebugRemark("inlining failed: can't convert multret to fixed returns");
             return false;
         }
 
+        if (func->vararg)
+        {
+            if (func->args.size > expr->args.size)
+            {
+                bytecode.addDebugRemark("inlining failed: not enough arguments");
+                return false;
+            }
+
+            for (AstExpr* arg : expr->args)
+            {
+                if (isExprMultRet(arg))
+                {
+                    bytecode.addDebugRemark("inlining failed: can't inline vararg function calls with multret arguments");
+                    return false;
+                }
+            }
+        }
+
         // compute constant bitvector for all arguments to feed the cost model
         bool varc[8] = {};
         for (size_t i = 0; i < func->args.size && i < expr->args.size && i < 8; ++i)
@@ -731,7 +783,7 @@ struct Compiler
         }
 
         // the inline frame will be used to compile return statements as well as to reject recursive inlining attempts
-        inlineFrames.push_back({func, oldLocals, target, targetCount});
+        inlineFrames.push_back({func, expr, oldLocals, target, targetCount});
 
         // fold constant values updated above into expressions in the function body
         foldConstants(constants, variables, locstants, builtinsFold, builtinsFoldLibraryK, options.libraryMemberConstantCb, func->body, names);
@@ -813,9 +865,7 @@ struct Compiler
             // add a debug remark for cases when we didn't even call tryCompileInlinedCall
             if (func && !(fi && fi->canInline))
             {
-                if (func->vararg)
-                    bytecode.addDebugRemark("inlining failed: function is variadic");
-                else if (!fi)
+                if (!fi)
                     bytecode.addDebugRemark("inlining failed: can't inline recursive calls");
                 else if (getfenvUsed || setfenvUsed)
                     bytecode.addDebugRemark("inlining failed: module uses getfenv/setfenv");
@@ -979,7 +1029,7 @@ struct Compiler
                 CompileError::raise(expr->func->location, "Exceeded jump distance limit; simplify the code to compile");
         }
 
-        bytecode.emitABC(LOP_CALL, regs, multCall ? 0 : uint8_t(expr->self + expr->args.size + 1), multRet ? 0 : uint8_t(targetCount + 1));
+        bytecode.emitABC(LOP_CALL, regs, (multCall && inlineFrames.size() == 0) ? 0 : uint8_t(expr->self + expr->args.size + 1), multRet ? 0 : uint8_t(targetCount + 1));
 
         // if we didn't output results directly to target, we need to move them
         if (!targetTop)
@@ -4126,6 +4176,7 @@ struct Compiler
     struct InlineFrame
     {
         AstExprFunction* func;
+        AstExprCall* call;
 
         size_t localOffset;
 

tests/Compiler.test.cpp

@@ -6289,7 +6289,7 @@ RETURN R1 1
 
 TEST_CASE("InlineProhibited")
 {
-    // we can't inline variadic functions
+    // we can inline variadic functions with fixedret arguments
     CHECK_EQ(
         "\n" + compileFunction(
                    R"(
@@ -6305,12 +6305,35 @@ return x
                ),
         R"(
 DUPCLOSURE R0 K0 ['foo']
-MOVE R1 R0
-CALL R1 0 1
+LOADN R1 42
 RETURN R1 1
 )"
     );
 
+    // we can't inline variadic functions with multret arguments
+        CHECK_EQ(
+        "\n" + compileFunction(
+                   R"(
+local function foo(...)
+    return 42
+end
+
+function bar(...)
+    local x = foo(...)
+    return x
+end
+)",
+                   1,
+                   2
+               ),
+        R"(
+GETUPVAL R0 0
+GETVARARGS R1 -1
+CALL R0 -1 1
+RETURN R0 1
+)"
+    );
+
     // we can't inline any functions in modules with getfenv/setfenv
     CHECK_EQ(
         "\n" + compileFunction(

tests/conformance/vararg.luau

@@ -167,5 +167,11 @@ f = loadstring[[
 assert(f("a", "b", nil, {}, assert))
 assert(f())
 
+-- pass to another variadic function
+local function passmc(...)
+  assert(true, ...)
+end
+passmc(nil)
+
 return('OK')