question: Question.check() now returns (correct, details)

When the answer is correct, details will be None.  When the answer is
incorrect, details may be None or an explanatory message that provides
details about why the answer is incorrect.  This is mostly useful for
`ScriptQuestion`s, where the difference between the answered and
expected output streams may help diagnose errors.

The details are returned as a string (instead of, for example, logging
them in invocation_difference) so they will be easy to display in user
interfaces that aren't built around the command line.

diff --git a/quizzer/question.py b/quizzer/question.py
index f7fa269f4b78a3f1dbb04b25ec526f07c3ce77aa..3e69abb1541bf8d4b8069291747fbc7b2600fc29 100644 (file)
--- a/quizzer/question.py
+++ b/quizzer/question.py
@@ -71,7 +71,12 @@ class Question (object):
         self.__dict__.update(state)
 
     def check(self, answer):
-        return answer == self.answer
+        correct = answer == self.answer
+        details = None
+        if not correct:
+            details = 'answer ({}) does not match expected value'.format(
+                answer)
+        return (correct, details)
 
     def _format_attribute(self, attribute, newline='\n'):
         value = getattr(self, attribute)
@@ -91,12 +96,23 @@ class NormalizedStringQuestion (Question):
         return string.strip().lower()
 
     def check(self, answer):
-        return self.normalize(answer) == self.normalize(self.answer)
+        normalized_answer = self.normalize(answer)
+        correct = normalized_answer == self.normalize(self.answer)
+        details = None
+        if not correct:
+            details = ('normalized answer ({}) does not match expected value'
+                       ).format(normalized_answer)
+        return (correct, details)
 
 
 class ChoiceQuestion (Question):
     def check(self, answer):
-        return answer in self.answer
+        correct = answer in self.answer
+        details = None
+        if not correct:
+            details = 'answer ({}) is not in list of expected values'.format(
+                answer)
+        return (correct, details)
 
 
 class ScriptQuestion (Question):
@@ -208,6 +224,7 @@ class ScriptQuestion (Question):
         Arguments are passed through to ._invoke() for calculating the
         user's response.
         """
+        details = None
         # figure out the expected values
         (ea_status,ea_stdout,ea_stderr,
          et_status,et_stdout,et_stderr) = self._invoke(answer=self.answer)
@@ -218,25 +235,29 @@ class ScriptQuestion (Question):
                 answer=answer, tempdir=tempdir)
         except (KeyboardInterrupt, _error.CommandError) as e:
             if isinstance(e, KeyboardInterrupt):
-                LOG.warning('KeyboardInterrupt')
+                details = 'KeyboardInterrupt'
             else:
-                LOG.warning(e)
-            return False
+                details = str(e)
+            return (False, details)
         # compare user-generated output with expected values
         if answer:
             if self.compare_answers:
-                if _util.invocation_difference(  # compare answers
-                        ea_status, ea_stdout, ea_stderr,
-                        ua_status, ua_stdout, ua_stderr):
-                    return False
+                difference = _util.invocation_difference(  # compare answers
+                    ea_status, ea_stdout, ea_stderr,
+                    ua_status, ua_stdout, ua_stderr)
+                if difference:
+                    details = _util.format_invocation_difference(*difference)
+                    return (False, details)
             elif ua_stderr:
                 LOG.warning(ua_stderr)
-        if _util.invocation_difference(  # compare teardown
-                et_status, et_stdout, et_stderr,
-                ut_status, ut_stdout, ut_stderr):
-            return False
-        return True
-        
+        difference = _util.invocation_difference(  # compare teardown
+            et_status, et_stdout, et_stderr,
+            ut_status, ut_stdout, ut_stderr)
+        if difference:
+            details = _util.format_invocation_difference(*difference)
+            return (False, details)
+        return (True, None)
+
 
 for name,obj in list(locals().items()):
     if name.startswith('_'):

diff --git a/quizzer/ui/__init__.py b/quizzer/ui/__init__.py
index 333c6c02c24cc9fb76f92ca4f54abb0e5dc40090..011a84ba41b0ee773e690fe91225ed75df7dfcd3 100644 (file)
--- a/quizzer/ui/__init__.py
+++ b/quizzer/ui/__init__.py
@@ -42,13 +42,13 @@ class UserInterface (object):
             return self.stack.pop(0)
 
     def process_answer(self, question, answer, **kwargs):
-        correct = question.check(answer=answer, **kwargs)
+        correct,details = question.check(answer=answer, **kwargs)
         self.answers.add(question=question, answer=answer, correct=correct)
         if not correct:
             self.stack.insert(0, question)
             for qid in reversed(question.dependencies):
                 self.stack.insert(0, self.quiz.get(id=qid))
-        return correct
+        return (correct, details)
 
 
 def get_ui(name):

diff --git a/quizzer/ui/cli.py b/quizzer/ui/cli.py
index 85f95f0e82b188e4822cdbfec6b6da8b60bfceeb..60d649fdc6f2b332a35de1b4d0f8fb570446a60b 100644 (file)
--- a/quizzer/ui/cli.py
+++ b/quizzer/ui/cli.py
@@ -98,12 +98,17 @@ class QuestionCommandLine (_cmd.Cmd):
         kwargs = {}
         if self._tempdir:
             kwargs['tempdir'] = self._tempdir
-        correct = self.ui.process_answer(
+        correct,details = self.ui.process_answer(
             question=self.question, answer=answer, **kwargs)
         if correct:
             print(_colorize(self.ui.colors['correct'], 'correct\n'))
         else:
-            print(_colorize(self.ui.colors['incorrect'], 'incorrect\n'))
+            print(_colorize(self.ui.colors['incorrect'], 'incorrect'))
+            if details:
+                print(_colorize(
+                        self.ui.colors['incorrect'], '{}\n'.format(details)))
+            else:
+                print('')
         return self.get_question()
 
     def do_answer(self, arg):

diff --git a/quizzer/util.py b/quizzer/util.py
index 9f9b6dca8c0299bec9c672c1c584f42d7f1adcd9..6b6f1a1b3c9acde7ea39432816ec1a866b8cd23d 100644 (file)
--- a/quizzer/util.py
+++ b/quizzer/util.py
@@ -22,9 +22,6 @@ import tempfile as _tempfile
 from . import error as _error
 
 
-LOG = _logging.getLogger(__name__)
-
-
 def invoke(args, stdin=None, stdout=_subprocess.PIPE, stderr=_subprocess.PIPE,
            universal_newlines=False, timeout=None, expect=None, **kwargs):
     if stdin:
@@ -63,7 +60,6 @@ def invocation_difference(a_status, a_stdout, a_stderr,
         ('stdout', a_stdout, b_stdout),
         ]:
         if a != b:
-            LOG.info(format_invocation_difference(name=name, a=a, b=b))
             return (name, a, b)
 
 def format_invocation_difference(name, a, b):