From 6fc8840fced9a031c3e7f8a160e02673f9257cae Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Wed, 13 Mar 2013 18:06:05 -0400 Subject: [PATCH] question: Question.check() now returns (correct, details) When the answer is correct, details will be None. When the answer is incorrect, details may be None or an explanatory message that provides details about why the answer is incorrect. This is mostly useful for `ScriptQuestion`s, where the difference between the answered and expected output streams may help diagnose errors. The details are returned as a string (instead of, for example, logging them in invocation_difference) so they will be easy to display in user interfaces that aren't built around the command line. --- quizzer/question.py | 53 +++++++++++++++++++++++++++++------------- quizzer/ui/__init__.py | 4 ++-- quizzer/ui/cli.py | 9 +++++-- quizzer/util.py | 4 ---- 4 files changed, 46 insertions(+), 24 deletions(-) diff --git a/quizzer/question.py b/quizzer/question.py index f7fa269..3e69abb 100644 --- a/quizzer/question.py +++ b/quizzer/question.py @@ -71,7 +71,12 @@ class Question (object): self.__dict__.update(state) def check(self, answer): - return answer == self.answer + correct = answer == self.answer + details = None + if not correct: + details = 'answer ({}) does not match expected value'.format( + answer) + return (correct, details) def _format_attribute(self, attribute, newline='\n'): value = getattr(self, attribute) @@ -91,12 +96,23 @@ class NormalizedStringQuestion (Question): return string.strip().lower() def check(self, answer): - return self.normalize(answer) == self.normalize(self.answer) + normalized_answer = self.normalize(answer) + correct = normalized_answer == self.normalize(self.answer) + details = None + if not correct: + details = ('normalized answer ({}) does not match expected value' + ).format(normalized_answer) + return (correct, details) class ChoiceQuestion (Question): def check(self, answer): - return answer in self.answer + correct = answer in self.answer + details = None + if not correct: + details = 'answer ({}) is not in list of expected values'.format( + answer) + return (correct, details) class ScriptQuestion (Question): @@ -208,6 +224,7 @@ class ScriptQuestion (Question): Arguments are passed through to ._invoke() for calculating the user's response. """ + details = None # figure out the expected values (ea_status,ea_stdout,ea_stderr, et_status,et_stdout,et_stderr) = self._invoke(answer=self.answer) @@ -218,25 +235,29 @@ class ScriptQuestion (Question): answer=answer, tempdir=tempdir) except (KeyboardInterrupt, _error.CommandError) as e: if isinstance(e, KeyboardInterrupt): - LOG.warning('KeyboardInterrupt') + details = 'KeyboardInterrupt' else: - LOG.warning(e) - return False + details = str(e) + return (False, details) # compare user-generated output with expected values if answer: if self.compare_answers: - if _util.invocation_difference( # compare answers - ea_status, ea_stdout, ea_stderr, - ua_status, ua_stdout, ua_stderr): - return False + difference = _util.invocation_difference( # compare answers + ea_status, ea_stdout, ea_stderr, + ua_status, ua_stdout, ua_stderr) + if difference: + details = _util.format_invocation_difference(*difference) + return (False, details) elif ua_stderr: LOG.warning(ua_stderr) - if _util.invocation_difference( # compare teardown - et_status, et_stdout, et_stderr, - ut_status, ut_stdout, ut_stderr): - return False - return True - + difference = _util.invocation_difference( # compare teardown + et_status, et_stdout, et_stderr, + ut_status, ut_stdout, ut_stderr) + if difference: + details = _util.format_invocation_difference(*difference) + return (False, details) + return (True, None) + for name,obj in list(locals().items()): if name.startswith('_'): diff --git a/quizzer/ui/__init__.py b/quizzer/ui/__init__.py index 333c6c0..011a84b 100644 --- a/quizzer/ui/__init__.py +++ b/quizzer/ui/__init__.py @@ -42,13 +42,13 @@ class UserInterface (object): return self.stack.pop(0) def process_answer(self, question, answer, **kwargs): - correct = question.check(answer=answer, **kwargs) + correct,details = question.check(answer=answer, **kwargs) self.answers.add(question=question, answer=answer, correct=correct) if not correct: self.stack.insert(0, question) for qid in reversed(question.dependencies): self.stack.insert(0, self.quiz.get(id=qid)) - return correct + return (correct, details) def get_ui(name): diff --git a/quizzer/ui/cli.py b/quizzer/ui/cli.py index 85f95f0..60d649f 100644 --- a/quizzer/ui/cli.py +++ b/quizzer/ui/cli.py @@ -98,12 +98,17 @@ class QuestionCommandLine (_cmd.Cmd): kwargs = {} if self._tempdir: kwargs['tempdir'] = self._tempdir - correct = self.ui.process_answer( + correct,details = self.ui.process_answer( question=self.question, answer=answer, **kwargs) if correct: print(_colorize(self.ui.colors['correct'], 'correct\n')) else: - print(_colorize(self.ui.colors['incorrect'], 'incorrect\n')) + print(_colorize(self.ui.colors['incorrect'], 'incorrect')) + if details: + print(_colorize( + self.ui.colors['incorrect'], '{}\n'.format(details))) + else: + print('') return self.get_question() def do_answer(self, arg): diff --git a/quizzer/util.py b/quizzer/util.py index 9f9b6dc..6b6f1a1 100644 --- a/quizzer/util.py +++ b/quizzer/util.py @@ -22,9 +22,6 @@ import tempfile as _tempfile from . import error as _error -LOG = _logging.getLogger(__name__) - - def invoke(args, stdin=None, stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, universal_newlines=False, timeout=None, expect=None, **kwargs): if stdin: @@ -63,7 +60,6 @@ def invocation_difference(a_status, a_stdout, a_stderr, ('stdout', a_stdout, b_stdout), ]: if a != b: - LOG.info(format_invocation_difference(name=name, a=a, b=b)) return (name, a, b) def format_invocation_difference(name, a, b): -- 2.26.2