Lots of CLI changes (openai#22)

* Add CLI option to download files (openai#34) * Option to check if file has been uploaded in the past before uploading (openai#33) The check is done based on filename, file purpose and file size * Add fine-tuning hparams directly into the fine-tunes CLI (openai#35) * update fine_tunes cli use_packing argument (openai#38) * A file verification and remediation tool. It applies the following validations: - prints the number of examples, and warns if it's lower than 100 - ensures prompt and completion columns are present - optionally removes any additional columns - ensures all completions are non-empty - infers which type of fine-tuning the data is most likely in (classification, conditional generation and open-ended generation) - optionally removes duplicate rows - infers the existence of a common suffix, and if there is none, suggests one for classification and conditional generation - optionally prepends a space to each completion, to make tokenization better - optionally splits into training and validation set for the classification use case - optionally ensures there's an ending string for all completions - optionally lowercases completions or prompts if more than a 1/3 of alphanumeric characters are upper case It interactively asks the user to accept or reject recommendations. If the user is happy, then it saves the modified output file as a jsonl, which is ready for being used in fine-tuning with the printed command. * Completion: remove from kwargs before passing to EngineAPI (openai#37) * Version bump before pushing to external Co-authored-by: Todor Markov <todor.m.markov@gmail.com> Co-authored-by: Boris Power <81998504+BorisPower@users.noreply.github.com> Co-authored-by: Dave Cummings <dave@openai.com>

Author: 4 people

bin/openai

@@ -1,13 +1,11 @@
 #!/usr/bin/env python
 import argparse
-import json
 import logging
-import os
 import sys
 
 import openai
 from openai.cli import display_error
-from openai.cli import register as api_register
+from openai.cli import api_register, tools_register
 
 logger = logging.getLogger()
 formatter = logging.Formatter("[%(asctime)s] %(message)s")
@@ -40,9 +38,11 @@ def main():
  parser.set_defaults(func=help)
 
  subparsers = parser.add_subparsers()
- sub = subparsers.add_parser("api", help="Direct API calls")
+ sub_api = subparsers.add_parser("api", help="Direct API calls")
+ sub_tools = subparsers.add_parser("tools", help="Client side tools for convenience")
 
- api_register(sub)
+ api_register(sub_api)
+ tools_register(sub_tools)
 
  args = parser.parse_args()
  if args.verbosity == 1:

openai/api_resources/completion.py

@@ -19,7 +19,7 @@ def create(cls, *args, **kwargs):
  of valid parameters.
  """
  start = time.time()
- timeout = kwargs.get("timeout", None)
+ timeout = kwargs.pop("timeout", None)
  if kwargs.get("model", None) is None and kwargs.get("engine", None) is None:
  raise InvalidRequestError(
  "Must provide an 'engine' or 'model' parameter to create a Completion.",

openai/api_resources/file.py

@@ -1,5 +1,8 @@
 from __future__ import absolute_import, division, print_function
 
+import json
+import os
+
 import openai
 from openai import api_requestor, util
 from openai.api_resources.abstract import (
@@ -29,3 +32,59 @@ def create(
  return util.convert_to_openai_object(
  response, api_key, api_version, organization
  )
+
+ @classmethod
+ def download(
+ cls, id, api_key=None, api_base=None, api_version=None, organization=None
+ ):
+ requestor = api_requestor.APIRequestor(
+ api_key,
+ api_base=api_base or openai.file_api_base or openai.api_base,
+ api_version=api_version,
+ organization=organization,
+ )
+ url = f"{cls.class_url()}/{id}/content"
+ rbody, rcode, rheaders, _, _ = requestor.request_raw("get", url)
+ if not 200 <= rcode < 300:
+ raise requestor.handle_error_response(
+ rbody, rcode, json.loads(rbody), rheaders, stream_error=False
+ )
+ return rbody
+
+ @classmethod
+ def find_matching_files(
+ cls,
+ api_key=None,
+ api_base=None,
+ api_version=None,
+ organization=None,
+ file=None,
+ purpose=None,
+ ):
+ if file is None:
+ raise openai.error.InvalidRequestError(
+ "'file' is a required property", "file"
+ )
+ if purpose is None:
+ raise openai.error.InvalidRequestError(
+ "'purpose' is a required property", "purpose"
+ )
+ all_files = cls.list(
+ api_key=api_key,
+ api_base=api_base or openai.file_api_base or openai.api_base,
+ api_version=api_version,
+ organization=organization,
+ ).get("data", [])
+ matching_files = []
+ for f in all_files:
+ if f["purpose"] != purpose:
+ continue
+ if not hasattr(file, "name") or f["filename"] != file.name:
+ continue
+ file.seek(0, os.SEEK_END)
+ if f["bytes"] != file.tell():
+ file.seek(0)
+ continue
+ file.seek(0)
+ matching_files.append(f)
+ return matching_files