Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow symlinks to test cases with other output validator flags and… #276

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

Allow symlinks to test cases with other output validator flags and… #276

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 12 additions & 11 deletions problemtools/verifyproblem.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,8 +183,8 @@ def __init__(self, problem: Problem, base: str, testcasegroup: TestCaseGroup) ->
self._problem = problem
self.testcasegroup = testcasegroup
self.reuse_result_from: TestCase|None = None
self.counter = len(problem.testcase_by_infile)
problem.testcase_by_infile[self.infile] = self
self.counter = len(problem.testcase_by_input)
problem.testcase_by_input[(self.infile, self.testcasegroup.config['output_validator_flags'])] = self

def check_newlines(self, filename: str) -> None:
with open(filename, 'rb') as f:
Expand Down Expand Up @@ -248,9 +248,9 @@ def matches_filter(self, filter_re: Pattern[str]) -> bool:
def set_symlinks(self) -> None:
if not os.path.islink(self.infile):
return
target = os.path.realpath(self.infile)
if target in self._problem.testcase_by_infile:
self.reuse_result_from = self._problem.testcase_by_infile[target]
target = os.path.realpath(self.infile), self.testcasegroup.config['output_validator_flags']
if target in self._problem.testcase_by_input:
self.reuse_result_from = self._problem.testcase_by_input[target]

def _check_symlinks(self) -> bool:
if not os.path.islink(self.infile):
Expand All @@ -264,12 +264,9 @@ def _check_symlinks(self) -> bool:
if ans_target != f'{in_target[:-3]}.ans':
self.error(f"Symbolic link '{nicepath}' must have a corresponding link for answer file")
return False
if self.reuse_result_from is None:
if not nicepath.startswith('data'):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Isn't the previous check more accurate?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How so?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry for late reply, but:

  • this covers e.g. data2 too
  • nicepath is the path prior to symlink resolution

self.error(f"Symbolic link points outside data/ directory for file '{nicepath}'")
return False
if self.testcasegroup.config['output_validator_flags'] != self.reuse_result_from.testcasegroup.config['output_validator_flags']:
self.error(f"Symbolic link '{nicepath}' points to testcase with different output validator flags")
return False
return True

def run_submission(self, sub, runner: Runner, context: Context) -> Result:
Expand Down Expand Up @@ -1661,7 +1658,8 @@ def validate(self, testcase: TestCase, submission_output: str) -> SubmissionResu
val_memlim = self._problem.config.get('limits')['validation_memory']
flags = self._problem.config.get('validator_flags').split() + testcase.testcasegroup.config['output_validator_flags'].split()
for val in self._actual_validators():
if val.compile()[0]:
compile_res = val.compile()
if compile_res[0]:
feedbackdir = tempfile.mkdtemp(prefix='feedback', dir=self._problem.tmpdir)
validator_output = tempfile.mkdtemp(prefix='checker_out', dir=self._problem.tmpdir)
outfile = validator_output + "/out.txt"
Expand All @@ -1670,6 +1668,7 @@ def validate(self, testcase: TestCase, submission_output: str) -> SubmissionResu
args=[testcase.infile, testcase.ansfile, feedbackdir] + flags,
timelim=val_timelim, memlim=val_memlim,
outfile=outfile, errfile=errfile)

if self.log.isEnabledFor(logging.DEBUG):
try:
with open(outfile, mode="rt") as f:
Expand All @@ -1687,6 +1686,8 @@ def validate(self, testcase: TestCase, submission_output: str) -> SubmissionResu
shutil.rmtree(validator_output)
if res.verdict != 'AC':
return res
else:
self.warning(f"Compilation failed for {val.name}: {compile_res[1]}")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Won't this result in a lot of warning spam (one log line per submission per test case)? IIRC we already give an error in an earlier phase.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I had an error and got no information before adding this. It was very confusing.
I will double check though.


# TODO: check that all output validators give same result
return res
Expand Down Expand Up @@ -1982,7 +1983,7 @@ def __enter__(self) -> Problem:
self.input_validators = InputValidators(self)
self.output_validators = OutputValidators(self)
self.graders = Graders(self)
self.testcase_by_infile: dict[str, TestCase] = {}
self.testcase_by_input: dict[tuple[str, str], TestCase] = {}
self.testdata = TestCaseGroup(self, os.path.join(self.probdir, 'data'))
self.submissions = Submissions(self)
self.generators = Generators(self)
Expand Down