Merge pull request #131 from lambda-feedback/tr102-rtol-slow

Tr102 rtol slow

Author: KarlLundengaard

app/evaluation_tests.py

@@ -42,4 +42,4 @@ def test_eval_function_can_handle_latex_input(self):
  assert result["is_correct"] is True
 
 if __name__ == "__main__":
- pytest.main(['-xsk not slow', "--tb=line", os.path.abspath(__file__)])
+ pytest.main(['-xsk not slow', '--tb=line', '--durations=10', os.path.abspath(__file__)])

app/expression_utilities.py

@@ -578,7 +578,7 @@ def parse_expression(expr, parsing_params):
  substitutions.sort(key=lambda x: -len(x[0]))
  expr = substitute(expr, substitutions)
  can_split = lambda x: False if x in unsplittable_symbols else _token_splittable(x)
- if strict_syntax:
+ if strict_syntax is True:
  transformations = parser_transformations[0:4]+extra_transformations
  else:
  transformations = parser_transformations[0:5, 6]+extra_transformations+(split_symbols_custom(can_split),)+parser_transformations[8]

app/feedback/symbolic_comparison.py

@@ -36,5 +36,6 @@
  "EXPRESSION_NOT_EQUALITY": "The response was an expression but was expected to be an equality.",
  "EQUALITY_NOT_EXPRESSION": "The response was an equality but was expected to be an expression.",
  "WITHIN_TOLERANCE": "", # "The difference between the response the answer is within specified error tolerance.",
- "SYMBOLICALLY_EQUAL": "The response and answer are symbolically equal.",
+ "NOT_NUMERICAL": "The expression cannot be evaluated numerically.",
+# "SYMBOLICALLY_EQUAL": "The response and answer are symbolically equal.",
 }

app/symbolic_comparison_evaluation.py

@@ -255,8 +255,20 @@ def symbolic_comparison(response, answer, params, eval_response) -> dict:
  eval_response.is_correct = ((res.args[0]-res.args[1])/(ans.args[0]-ans.args[1])).simplify().is_constant()
  return eval_response
 
- error_below_atol = False
- error_below_rtol = False
+ is_correct = True
+ parameters_dict = {
+ "parsing_params": parsing_params,
+ "reserved_expressions": reserved_expressions,
+ "reference_criteria_strings": reference_criteria_strings,
+ "symbolic_comparison_criteria": symbolic_comparison_criteria,
+ "eval_response": eval_response,
+ }
+ for criterion in criteria_parsed:
+ is_correct = is_correct and check_criterion(criterion, parameters_dict)
+ eval_response.is_correct = is_correct
+
+ error_below_atol = None
+ error_below_rtol = None
 
  if params.get("numerical", False) or params.get("rtol", False) or params.get("atol", False):
  # REMARK: 'pi' should be a reserved symbol but it is sometimes not treated as one, possibly because of input symbols.
@@ -271,32 +283,28 @@ def replace_pi(expr):
  ans = replace_pi(ans)
  res = replace_pi(res)
  if "atol" in params.keys():
- absolute_error = abs(ans-res)
- if isinstance(absolute_error, float) or absolute_error.is_constant():
- error_below_atol = bool(float(absolute_error) < float(params["atol"]))
+ try:
+ absolute_error = abs(float(ans-res))
+ error_below_atol = bool(absolute_error < float(params["atol"]))
+ except TypeError:
+ error_below_atol = None
  else:
  error_below_atol = True
  if "rtol" in params.keys():
- relative_error = abs(((ans-res)/ans).simplify())
- if isinstance(relative_error, float) or relative_error.is_constant():
- error_below_rtol = bool(float(relative_error) < float(params["rtol"]))
+ try:
+ relative_error = abs(float((ans-res)/ans)) # TODO: capture error here and see if you can rewrite this in a faster way
+ error_below_rtol = bool(relative_error < float(params["rtol"]))
+ except TypeError:
+ error_below_rtol = None
  else:
  error_below_rtol = True
- if error_below_atol and error_below_rtol:
+ if error_below_atol is None or error_below_rtol is None:
+ eval_response.is_correct = False
+ tag = "NOT_NUMERICAL"
+ eval_response.add_feedback((tag, symbolic_comparison_internal_messages[tag]))
+ elif error_below_atol is True and error_below_rtol is True:
  eval_response.is_correct = True
  tag = "WITHIN_TOLERANCE"
  eval_response.add_feedback((tag, symbolic_comparison_internal_messages[tag]))
- return eval_response
 
- is_correct = True
- parameters_dict = {
- "parsing_params": parsing_params,
- "reserved_expressions": reserved_expressions,
- "reference_criteria_strings": reference_criteria_strings,
- "symbolic_comparison_criteria": symbolic_comparison_criteria,
- "eval_response": eval_response,
- }
- for criterion in criteria_parsed:
- is_correct = is_correct and check_criterion(criterion, parameters_dict)
- eval_response.is_correct = is_correct
  return eval_response

app/symbolic_comparison_evaluation_tests.py

@@ -548,71 +548,81 @@ def test_empty_input_symbols_codes_and_alternatives(self):
  assert result["is_correct"] is True
 
  @pytest.mark.parametrize(
- "description,response,answer,tolerance,outcome",
+ "description,response,answer,tolerance,tags,outcome",
  [
  (
  "Correct response, tolerance specified with atol",
  "6.73",
  "sqrt(3)+5",
  {"atol": 0.005},
+ ["WITHIN_TOLERANCE"],
  True
  ),
  (
  "Incorrect response, tolerance specified with atol",
  "6.7",
  "sqrt(3)+5",
  {"atol": 0.005},
+ [],
  False
  ),
  (
  "Correct response, tolerance specified with rtol",
  "6.73",
  "sqrt(3)+5",
  {"rtol": 0.0005},
+ ["WITHIN_TOLERANCE"],
  True
  ),
  (
  "Incorrect response, tolerance specified with rtol",
  "6.7",
  "sqrt(3)+5",
  {"rtol": 0.0005},
+ [],
  False
  ),
  (
  "Response is not constant, tolerance specified with atol",
  "6.7+x",
  "sqrt(3)+5",
  {"atol": 0.005},
+ ["NOT_NUMERICAL"],
  False
  ),
  (
  "Answer is not constant, tolerance specified with atol",
  "6.73",
  "sqrt(3)+x",
  {"atol": 0.005},
+ ["NOT_NUMERICAL"],
  False
  ),
  (
  "Response is not constant, tolerance specified with rtol",
  "6.7+x",
  "sqrt(3)+5",
  {"rtol": 0.0005},
+ ["NOT_NUMERICAL"],
  False
  ),
  (
  "Answer is not constant, tolerance specified with rtol",
  "6.73",
  "sqrt(3)+x",
  {"rtol": 0.0005},
+ ["NOT_NUMERICAL"],
  False
  ),
  ]
  )
- def test_numerical_comparison_problem(self, description, response, answer, tolerance, outcome):
+ def test_numerical_comparison_problem(self, description, response, answer, tolerance, tags, outcome):
  params = {"numerical": True}
  params.update(tolerance)
- result = evaluation_function(response, answer, params)
+ result = evaluation_function(response, answer, params, include_test_data=True)
  assert result["is_correct"] is outcome
+ for tag in tags:
+ tag in result["tags"]
 
  @pytest.mark.parametrize(
  "description,response,answer,tolerance,outcome",
@@ -633,7 +643,7 @@ def test_numerical_comparison_problem(self, description, response, answer, toler
  ),
  ]
  )
- def test_numerical_comparison(self, description, response, answer, tolerance, outcome):
+ def test_numerical_comparison_AERO4007(self, description, response, answer, tolerance, outcome):
  params = {
  "strict_syntax": False,
  "elementary_functions": True,
@@ -1052,4 +1062,4 @@ def test_exclamation_mark_for_factorial(self):
  assert result["is_correct"] is True
 
 if __name__ == "__main__":
- pytest.main(['-xsk not slow', "--tb=line", os.path.abspath(__file__)])
+ pytest.main(['-xsk not slow', "--tb=line", '--durations=10', os.path.abspath(__file__)])