fix: check if prefill

Author: terryyz

bigcodebench/provider/hf.py

@@ -41,7 +41,8 @@ def __init__(
  if self.is_direct_completion(): # no chat template
  self.eos += extra_eos_for_direct_completion(dataset)
  else: # with chat template
- self.eos += ["\n```\n"]
+ if self.prefill and "```" in self.response_prefix:
+ self.eos += ["\n```\n"]
 
  print(f"{self.eos = }")
  self.model = AutoModelForCausalLM.from_pretrained(name, **kwargs)

bigcodebench/provider/vllm.py

@@ -27,7 +27,8 @@ def __init__(self, name: str, dataset: str, tp: int, **kwargs) -> None:
  if self.is_direct_completion():
  self.eos += extra_eos_for_direct_completion(dataset)
  else:
- self.eos += ["\n```\n"]
+ if self.prefill:
+ self.eos += ["\n```\n"]
  self.llm = LLM(model=name, max_model_len=self.max_new_tokens, **kwargs)
  self.llm.set_tokenizer(tokenizer=self.tokenizer)