CG-10739: Open Hands Semantic Edit Implimentation (#476)

- Can now specify line start and end point for edit - For large files, it will replace the portion of the file from start and end using gpt-4o-mini (the draft editor) - For smaller files (< 300 lines), it will resort to modifying the entire file - added better system prompts with more information about codegen --------- Co-authored-by: jayhack <jayhack.0@gmail.com> Co-authored-by: Jay Hack <jayhack@users.noreply.github.com>

Author: tawsifkamal

src/codegen/extensions/langchain/__init__.py

@@ -0,0 +1,54 @@
+"""Langchain tools for workspace operations."""
+
+from langchain_core.tools.base import BaseTool
+
+from codegen import Codebase
+
+from .tools import (
+ CommitTool,
+ CreateFileTool,
+ DeleteFileTool,
+ EditFileTool,
+ ListDirectoryTool,
+ RevealSymbolTool,
+ SearchTool,
+ SemanticEditTool,
+ ViewFileTool,
+)
+
+__all__ = [
+ # Tool classes
+ "CommitTool",
+ "CreateFileTool",
+ "DeleteFileTool",
+ "EditFileTool",
+ "ListDirectoryTool",
+ "RevealSymbolTool",
+ "SearchTool",
+ "SemanticEditTool",
+ "ViewFileTool",
+ # Helper functions
+ "get_workspace_tools",
+]
+
+
+def get_workspace_tools(codebase: Codebase) -> list[BaseTool]:
+ """Get all workspace tools initialized with a codebase.
+
+ Args:
+ codebase: The codebase to operate on
+
+ Returns:
+ List of initialized Langchain tools
+ """
+ return [
+ ViewFileTool(codebase),
+ ListDirectoryTool(codebase),
+ SearchTool(codebase),
+ EditFileTool(codebase),
+ CreateFileTool(codebase),
+ DeleteFileTool(codebase),
+ CommitTool(codebase),
+ RevealSymbolTool(codebase),
+ SemanticEditTool(codebase),
+ ]

src/codegen/extensions/langchain/agent.py

@@ -6,6 +6,7 @@
 from langchain.tools import BaseTool
 from langchain_core.chat_history import InMemoryChatMessageHistory
 from langchain_core.messages import BaseMessage
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
 from langchain_core.runnables.history import RunnableWithMessageHistory
 from langchain_openai import ChatOpenAI
 
@@ -75,8 +76,62 @@ def create_codebase_agent(
  GithubCreatePRReviewCommentTool(codebase),
  ]
 
- # Get the prompt to use
- prompt = pull("hwchase17/openai-functions-agent")
+ prompt = ChatPromptTemplate.from_messages(
+ [
+ (
+ "system",
+ """
+ You are an expert software engineer with deep knowledge of code analysis, refactoring, and development best practices.
+ You have access to a powerful set of tools from codegen that allow you to analyze and modify codebases:
+
+ Core Capabilities:
+ 1. Code Analysis & Navigation:
+ - Search codebases using text or regex patterns
+ - View file contents and metadata (functions, classes, imports)
+ - Analyze code structure and dependencies
+ - Reveal symbol definitions and usages
+
+ 2. File Operations:
+ - View, create, edit, and delete files
+ - Rename files while updating all imports
+ - Move symbols between files
+ - Commit changes to disk
+
+ 3. Semantic Editing:
+ - Make precise, context-aware code edits
+ - Analyze affected code structures
+ - Preview changes before applying
+ - Ensure code quality with linting
+
+
+ 4. Code Search:
+ - Text-based and semantic search
+ - Search within specific directories
+ - Filter by file extensions
+ - Get paginated results
+
+ Best Practices:
+ - Always analyze code structure before making changes
+ - Preview edits to understand their impact
+ - Update imports and dependencies when moving code
+ - Use semantic edits for complex changes
+ - Commit changes after significant modifications
+ - Maintain code quality and consistency
+
+ Remember: You can combine these tools to perform complex refactoring
+ and development tasks. Always explain your approach before making changes.
+ Important rules: If you are asked to make any edits to a file, always
+ first view the file to understand its context and make sure you understand
+ the impact of the changes. Only then make the changes.
+ Ensure if specifiying line numbers, it's chosen with room (around 20
+ lines before and 20 lines after the edit range)
+ """,
+ ),
+ MessagesPlaceholder("chat_history", optional=True),
+ ("human", "{input}"),
+ MessagesPlaceholder("agent_scratchpad"),
+ ]
+ )
 
  # Create the agent
  agent = OpenAIFunctionsAgent(

src/codegen/extensions/langchain/tools.py

@@ -3,7 +3,7 @@
 import json
 from typing import Callable, ClassVar, Literal, Optional
 
-from langchain.tools import BaseTool
+from langchain_core.tools.base import BaseTool
 from pydantic import BaseModel, Field
 
 from codegen import Codebase
@@ -18,6 +18,10 @@
  linear_search_issues_tool,
 )
 from codegen.extensions.tools.link_annotation import add_links_to_message
+from codegen.extensions.tools.reveal_symbol import reveal_symbol
+from codegen.extensions.tools.search import search
+from codegen.extensions.tools.semantic_edit import semantic_edit
+from codegen.extensions.tools.semantic_search import semantic_search
 
 from ..tools import (
  commit,
@@ -30,13 +34,10 @@
  list_directory,
  move_symbol,
  rename_file,
- reveal_symbol,
- search,
- semantic_edit,
- semantic_search,
  view_file,
  view_pr,
 )
+from ..tools.tool_prompts import _FILE_EDIT_DESCRIPTION
 
 
 class ViewFileInput(BaseModel):
@@ -118,7 +119,7 @@ class EditFileTool(BaseTool):
  """Tool for editing files."""
 
  name: ClassVar[str] = "edit_file"
- description: ClassVar[str] = "Edit a file by replacing its entire content"
+ description: ClassVar[str] = "Edit a file by replacing its entire content. This tool should only be used for replacing entire file contents."
  args_schema: ClassVar[type[BaseModel]] = EditFileInput
  codebase: Codebase = Field(exclude=True)
 
@@ -233,36 +234,35 @@ def _run(
  return json.dumps(result, indent=2)
 
 
+_SEMANTIC_EDIT_BRIEF = """Tool for semantic editing of files.
+* Allows editing files by providing a draft of the new content
+* For large files, specify line ranges to edit
+* Will intelligently handle unchanged sections of code. Also supports appending to the end of a file."""
+
+
 class SemanticEditInput(BaseModel):
  """Input for semantic editing."""
 
- filepath: str = Field(..., description="Path to the file to edit")
- edit_spec: str = Field(
- ...,
- description="""The edit specification showing desired changes.
-Must contain code blocks between '# ... existing code ...' markers.
-Example:
-# ... existing code ...
-def new_function():
- print("Hello")
-# ... existing code ...
-""",
- )
+ filepath: str = Field(..., description="Path of the file relative to workspace root")
+ edit_content: str = Field(..., description=_FILE_EDIT_DESCRIPTION)
+ start: int = Field(default=1, description="Starting line number (1-indexed, inclusive). Default is 1.")
+ end: int = Field(default=-1, description="Ending line number (1-indexed, inclusive). Default is -1 (end of file).")
 
 
 class SemanticEditTool(BaseTool):
  """Tool for semantic editing of files."""
 
  name: ClassVar[str] = "semantic_edit"
- description: ClassVar[str] = "Edit a file using a semantic edit specification with code blocks"
+ description: ClassVar[str] = _SEMANTIC_EDIT_BRIEF # Short description
  args_schema: ClassVar[type[BaseModel]] = SemanticEditInput
  codebase: Codebase = Field(exclude=True)
 
  def __init__(self, codebase: Codebase) -> None:
  super().__init__(codebase=codebase)
 
- def _run(self, filepath: str, edit_spec: str) -> str:
- result = semantic_edit(self.codebase, filepath, edit_spec)
+ def _run(self, filepath: str, edit_content: str, start: int = 1, end: int = -1) -> str:
+ # Create the the draft editor mini llm
+ result = semantic_edit(self.codebase, filepath, edit_content, start=start, end=end)
  return json.dumps(result, indent=2)
 
 

src/codegen/extensions/tools/file_operations.py

@@ -6,6 +6,36 @@
 from codegen.sdk.core.directory import Directory
 
 
+def _add_line_numbers(content: str) -> str:
+ """Add line numbers to source code.
+
+ Args:
+ source: The source code as a string
+ line_range: Optional tuple of (start_line, end_line) to only show specific lines.
+ Line numbers are 1-indexed.
+
+ Returns:
+ Source code with line numbers prefixed
+ """
+ lines = content.splitlines()
+ total_lines = len(lines)
+
+ # Calculate padding for line numbers based on max line number
+ max_line_num = total_lines
+ padding = len(str(max_line_num))
+
+ # Get the line range to display
+ start = 0
+ end = total_lines
+
+ # Add line numbers
+ numbered_lines = []
+ for i, line in enumerate(lines[start:end], start=start + 1):
+ numbered_lines.append(f"{i:>{padding}}|{line}")
+
+ return "\n".join(numbered_lines)
+
+
 def view_file(codebase: Codebase, filepath: str) -> dict[str, Any]:
  """View the contents and metadata of a file.
 
@@ -28,7 +58,7 @@ def view_file(codebase: Codebase, filepath: str) -> dict[str, Any]:
 
  return {
  "filepath": file.filepath,
- "content": file.content,
+ "content": _add_line_numbers(file.content),
  }