Start introducing grading styles

Only batch is supported.
Also, NoOutput() must be accompanied with CustomScorer().

CMakeLists.txt

@@ -58,6 +58,8 @@ set(SOURCES
         include/tcframe/grader/GraderConfig.hpp
         include/tcframe/grader/GraderLogger.hpp
         include/tcframe/grader/TestCaseGrader.hpp
+        include/tcframe/grading_style.hpp
+        include/tcframe/grading_style/GradingStyle.hpp
         include/tcframe/io_manipulator.hpp
         include/tcframe/io_manipulator/GridIOSegmentManipulator.hpp
         include/tcframe/io_manipulator/IOManipulator.hpp
@@ -148,6 +150,7 @@ set(SOURCES
         test/tcframe/generator/MockGeneratorLogger.hpp
         test/tcframe/generator/MockTestCaseGenerator.hpp
         test/tcframe/generator/TestCaseGeneratorTests.cpp
+        test/tcframe/grading_style/MockGradingStyleFactory.hpp
         test/tcframe/io_manipulator/GridIOSegmentManipulatorTests.cpp
         test/tcframe/io_manipulator/IOManipulatorTests.cpp
         test/tcframe/io_manipulator/LineIOSegmentManipulatorTests.cpp

ete-test/resources/multi-no-output/spec.cpp

@@ -13,6 +13,7 @@ class ProblemSpec : public BaseProblemSpec {
     }
 
     void StyleConfig() {
+        CustomScorer();
         NoOutput();
     }
 

ete-test/resources/normal-custom-scorer/scorer.cpp

@@ -0,0 +1,25 @@
+#include <bits/stdc++.h>
+using namespace std;
+
+int main(int argc, char* argv[]) {
+    ifstream tc_in(argv[1]);
+    ifstream tc_out(argv[2]);
+    ifstream con_out(argv[3]);
+
+    int A, B;
+    tc_in >> A >> B;
+
+    double tc_answer;
+    tc_out >> tc_answer;
+
+    double con_answer;
+    con_out >> con_answer;
+
+    if (B == 4) {
+        return 1 / (B - 4);
+    } else if (abs(tc_answer - con_answer) < 1e-1) {
+        cout << "AC" << endl;
+    } else {
+        cout << "WA" << endl;
+    }
+}

ete-test/resources/normal-custom-scorer/solution.cpp

@@ -0,0 +1,8 @@
+#include <cstdio>
+
+int A, B;
+
+int main() {
+    scanf("%d %d", &A, &B);
+    printf("%.0lf\n", (double)(A + B));
+}

ete-test/resources/normal-custom-scorer/solution_alt.cpp

@@ -0,0 +1,14 @@
+#include <cstdio>
+
+int A, B;
+
+int main() {
+    scanf("%d %d", &A, &B);
+    if (B == 3) {
+        // WA
+        printf("%.2lf\n", A + B + 0.2);
+    } else {
+        // AC
+        printf("%.2lf\n", A + B + 0.02);
+    }
+}

ete-test/resources/normal-custom-scorer/spec.cpp

@@ -0,0 +1,45 @@
+#include <tcframe/spec.hpp>
+
+using namespace tcframe;
+
+class ProblemSpec : public BaseProblemSpec {
+protected:
+    int A, B;
+    double res;
+
+    void InputFormat() {
+        LINE(A, B);
+    }
+
+    void StyleConfig() {
+        CustomScorer();
+    }
+
+    void GradingConfig() {
+        TimeLimit(2);
+        MemoryLimit(64);
+    }
+
+    void Constraints() {
+        CONS(1 <= A && A <= 10);
+        CONS(1 <= B && B <= 10);
+    }
+};
+
+class TestSpec : public BaseTestSpec<ProblemSpec> {
+protected:
+    void SampleTestCase1() {
+        Input({
+            "1 5"
+        });
+        Output({
+            "6.09"
+        });
+    }
+
+    void TestCases() {
+        CASE(A = 1, B = 3);
+        CASE(A = 2, B = 4);
+        CASE(A = 5, B = 6);
+    }
+};

ete-test/resources/normal-no-output/spec.cpp

@@ -12,6 +12,7 @@ class ProblemSpec : public BaseProblemSpec {
     }
 
     void StyleConfig() {
+        CustomScorer();
         NoOutput();
     }
 

ete-test/resources/scripts/generate-with-custom-scorer.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -ex
+
+export TCFRAME_HOME=../../tcframe
+
+g++ -o solution solution.cpp
+g++ -o scorer scorer.cpp
+$TCFRAME_HOME/scripts/tcframe build
+./runner --solution=./solution --scorer=./scorer

ete-test/resources/scripts/grade-with-custom-scorer.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -ex
+
+export TCFRAME_HOME=../../tcframe
+
+g++ -o solution_alt solution_alt.cpp
+g++ -o scorer scorer.cpp
+./runner grade --solution=./solution_alt --scorer=./scorer

ete-test/tcframe/GenerationEteTests.cpp

@@ -47,6 +47,21 @@ TEST_F(GenerationEteTests, Normal_ComplexFormats) {
     ));
 }
 
+TEST_F(GenerationEteTests, Normal_CustomScorer) {
+    ASSERT_THAT(execStatus("cd ete/normal-custom-scorer && ../scripts/generate-with-custom-scorer.sh"), Eq(0));
+
+    EXPECT_THAT(ls("ete/normal-custom-scorer/tc"), UnorderedElementsAre(
+            "normal-custom-scorer_sample_1.in",
+            "normal-custom-scorer_sample_1.out",
+            "normal-custom-scorer_1.in",
+            "normal-custom-scorer_1.out",
+            "normal-custom-scorer_2.in",
+            "normal-custom-scorer_2.out",
+            "normal-custom-scorer_3.in",
+            "normal-custom-scorer_3.out"
+    ));
+}
+
 TEST_F(GenerationEteTests, Subtasks) {
     ASSERT_THAT(execStatus("cd ete/subtasks && ../scripts/generate.sh"), Eq(0));
 

ete-test/tcframe/GradingEteTests.cpp

@@ -19,4 +19,13 @@ TEST_F(GradingEteTests, Normal) {
             HasSubstr("normal_3: Time Limit Exceeded")));
 }
 
+TEST_F(GradingEteTests, Normal_CustomScorer) {
+    string result = exec("cd ete/normal-custom-scorer && ../scripts/grade-with-custom-scorer.sh");
+    EXPECT_THAT(result, AllOf(
+            HasSubstr("normal-custom-scorer_sample_1: Accepted"),
+            HasSubstr("normal-custom-scorer_1: Wrong Answer"),
+            HasSubstr("normal-custom-scorer_2: Internal Error"),
+            HasSubstr("normal-custom-scorer_3: Accepted")));
+}
+
 }

include/tcframe/grading_style.hpp

@@ -0,0 +1,3 @@
+#pragma once
+
+#include "tcframe/grading_style/GradingStyle.hpp"

include/tcframe/grading_style/GradingStyle.hpp

@@ -0,0 +1,51 @@
+#pragma once
+
+#include <string>
+
+#include "tcframe/evaluator.hpp"
+#include "tcframe/os.hpp"
+#include "tcframe/scorer.hpp"
+#include "tcframe/util.hpp"
+
+using std::string;
+
+namespace tcframe {
+
+struct GradingStyle {
+private:
+    Evaluator* evaluator_;
+    Scorer* scorer_;
+
+public:
+    GradingStyle(Evaluator* evaluator, Scorer* scorer)
+            : evaluator_(evaluator)
+            , scorer_(scorer) {}
+
+    Evaluator* evaluator() const {
+        return evaluator_;
+    }
+
+    Scorer* scorer() const {
+        return scorer_;
+    }
+};
+
+class GradingStyleFactory {
+public:
+    virtual ~GradingStyleFactory() {}
+
+    virtual GradingStyle createBatch(OperatingSystem* os, const optional<string>& scorerCommand) {
+        Evaluator* evaluator = new BatchEvaluator(os);
+
+        Scorer* scorer;
+        if (scorerCommand) {
+            scorer = new CustomScorer(os, scorerCommand.value());
+        } else {
+            scorer = new DiffScorer(os);
+        }
+
+        return GradingStyle(evaluator, scorer);
+    }
+};
+
+}

include/tcframe/runner/Runner.hpp

@@ -10,6 +10,7 @@
 #include "tcframe/evaluator.hpp"
 #include "tcframe/generator.hpp"
 #include "tcframe/grader.hpp"
+#include "tcframe/grading_style.hpp"
 #include "tcframe/os.hpp"
 #include "tcframe/scorer.hpp"
 #include "tcframe/spec.hpp"
@@ -32,6 +33,7 @@ class Runner {
     OperatingSystem* os_;
 
     RunnerLoggerFactory* runnerLoggerFactory_;
+    GradingStyleFactory* gradingStyleFactory_;
     GeneratorFactory* generatorFactory_;
     GraderFactory* graderFactory_;
 
@@ -42,20 +44,21 @@ class Runner {
             LoggerEngine* loggerEngine,
             OperatingSystem* os,
             RunnerLoggerFactory* runnerLoggerFactory,
+            GradingStyleFactory* gradingStyleFactory,
             GeneratorFactory* generatorFactory,
             GraderFactory* graderFactory)
             : specPath_(specPath)
             , testSpec_(testSpec)
             , loggerEngine_(loggerEngine)
             , os_(os)
             , runnerLoggerFactory_(runnerLoggerFactory)
+            , gradingStyleFactory_(gradingStyleFactory)
             , generatorFactory_(generatorFactory)
             , graderFactory_(graderFactory) {}
 
     int run(int argc, char* argv[]) {
         auto runnerLogger = runnerLoggerFactory_->create(loggerEngine_);
 
-
         try {
             string slug = parseSlug();
             Args args = parseArgs(argc, argv);
@@ -116,8 +119,9 @@ class Runner {
 
         auto ioManipulator = new IOManipulator(spec.ioFormat());
         auto verifier = new Verifier(spec.constraintSuite());
-        auto evaluator = new BatchEvaluator(os_);
-        auto scorer = new DiffScorer(os_);
+        auto gradingStyle = getGradingStyle(args, spec);
+        auto evaluator = gradingStyle.evaluator();
+        auto scorer = gradingStyle.scorer();
         auto logger = new GeneratorLogger(loggerEngine_);
         auto testCaseGenerator = new TestCaseGenerator(verifier, ioManipulator, os_, evaluator, scorer, logger);
         auto generator = generatorFactory_->create(spec.seedSetter(), testCaseGenerator, verifier, os_, logger);
@@ -144,15 +148,24 @@ class Runner {
         GraderConfig graderConfig = configBuilder.build();
 
         auto logger = new GraderLogger(loggerEngine_);
-        auto evaluator = new BatchEvaluator(os_);
-        auto scorer = new DiffScorer(os_);
+        auto gradingStyle = getGradingStyle(args, spec);
+        auto evaluator = gradingStyle.evaluator();
+        auto scorer = gradingStyle.scorer();
         auto testCaseGrader = new TestCaseGrader(evaluator, scorer, logger);
         auto grader = graderFactory_->create(testCaseGrader, logger);
 
         grader->grade(spec.testSuite(), spec.constraintSuite(), graderConfig);
         return 0;
     }
 
+    GradingStyle getGradingStyle(const Args& args, const Spec& spec) {
+        optional<string> scorerCommand;
+        if (spec.styleConfig().needsCustomScorer()) {
+            scorerCommand = optional<string>(args.scorer().value_or(CommonConfig::scorerCommand()));
+        }
+        return gradingStyleFactory_->createBatch(os_, scorerCommand);
+    }
+
     void cleanUp() {
         os_->execute(ExecutionRequestBuilder().setCommand("rm _*.out").build());
     }

include/tcframe/spec/core/StyleConfig.hpp

@@ -1,11 +1,13 @@
 #pragma once
 
+#include <stdexcept>
 #include <tuple>
 #include <utility>
 
 #include "CommonConfig.hpp"
 
 using std::move;
+using std::runtime_error;
 using std::tie;
 
 namespace tcframe {
@@ -14,9 +16,14 @@ struct StyleConfig {
     friend class StyleConfigBuilder;
 
 private:
+    bool needsCustomScorer_;
     bool needsOutput_;
 
 public:
+    bool needsCustomScorer() const {
+        return needsCustomScorer_;
+    }
+
     bool needsOutput() const {
         return needsOutput_;
     }
@@ -35,6 +42,12 @@ class StyleConfigBuilder {
 
     StyleConfigBuilder() {
         subject_.needsOutput_ = CommonConfig::needsOutput();
+        subject_.needsCustomScorer_ = CommonConfig::needsCustomScorer();
+    }
+
+    StyleConfigBuilder& CustomScorer() {
+        subject_.needsCustomScorer_ = true;
+        return *this;
     }
 
     StyleConfigBuilder& NoOutput() {
@@ -43,6 +56,10 @@ class StyleConfigBuilder {
     }
 
     StyleConfig build() {
+        if (!subject_.needsOutput_ && !subject_.needsCustomScorer_) {
+            throw runtime_error("Problem that does not need output must need custom scorer");
+        }
+
         return move(subject_);
     }
 };

src/tcframe/runner.cpp

@@ -1,5 +1,6 @@
 #include "tcframe/generator.hpp"
 #include "tcframe/grader.hpp"
+#include "tcframe/grading_style.hpp"
 #include "tcframe/os.hpp"
 #include "tcframe/runner.hpp"
 #include "tcframe/spec.hpp"
@@ -13,6 +14,7 @@ int main(int argc, char* argv[]) {
     auto loggerEngine = new SimpleLoggerEngine();
     auto os = new UnixOperatingSystem();
     auto runnerLoggerFactory = new RunnerLoggerFactory();
+    auto gradingStyleFactory = new GradingStyleFactory();
     auto generatorFactory = new GeneratorFactory();
     auto graderFactory = new GraderFactory();
 
@@ -22,6 +24,7 @@ int main(int argc, char* argv[]) {
             loggerEngine,
             os,
             runnerLoggerFactory,
+            gradingStyleFactory,
             generatorFactory,
             graderFactory);