Update Document segmentation for CLI

Author: Zongwei9888

cli/cli_app.py

@@ -37,6 +37,8 @@ def __init__(self):
  self.app = None # Will be initialized by workflow adapter
  self.logger = None
  self.context = None
+ # Document segmentation configuration
+ self.segmentation_config = {"enabled": True, "size_threshold_chars": 50000}
 
  async def initialize_mcp_app(self):
  """初始化MCP应用 - 使用工作流适配器"""
@@ -47,9 +49,51 @@ async def cleanup_mcp_app(self):
  """清理MCP应用 - 使用工作流适配器"""
  await self.workflow_adapter.cleanup_mcp_app()
 
+ def update_segmentation_config(self):
+ """Update document segmentation configuration in mcp_agent.config.yaml"""
+ import yaml
+ import os
+
+ config_path = os.path.join(
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+ "mcp_agent.config.yaml",
+ )
+
+ try:
+ # Read current config
+ with open(config_path, "r", encoding="utf-8") as f:
+ config = yaml.safe_load(f)
+
+ # Update document segmentation settings
+ if "document_segmentation" not in config:
+ config["document_segmentation"] = {}
+
+ config["document_segmentation"]["enabled"] = self.segmentation_config[
+ "enabled"
+ ]
+ config["document_segmentation"]["size_threshold_chars"] = (
+ self.segmentation_config["size_threshold_chars"]
+ )
+
+ # Write updated config
+ with open(config_path, "w", encoding="utf-8") as f:
+ yaml.dump(config, f, default_flow_style=False, allow_unicode=True)
+
+ self.cli.print_status(
+ "📄 Document segmentation configuration updated", "success"
+ )
+
+ except Exception as e:
+ self.cli.print_status(
+ f"⚠️ Failed to update segmentation config: {str(e)}", "warning"
+ )
+
  async def process_input(self, input_source: str, input_type: str):
  """处理输入源（URL或文件）- 使用升级版智能体编排引擎"""
  try:
+ # Update segmentation configuration before processing
+ self.update_segmentation_config()
+
  self.cli.print_separator()
  self.cli.print_status(
  "🚀 Starting intelligent agent orchestration...", "processing"
@@ -237,8 +281,20 @@ async def run_interactive_session(self):
  self.cli.show_history()
 
  elif choice in ["c", "config", "configure"]:
+ # Sync current segmentation config from CLI interface
+ self.segmentation_config["enabled"] = self.cli.segmentation_enabled
+ self.segmentation_config["size_threshold_chars"] = (
+ self.cli.segmentation_threshold
+ )
+
  self.cli.show_configuration_menu()
 
+ # Sync back from CLI interface after configuration changes
+ self.segmentation_config["enabled"] = self.cli.segmentation_enabled
+ self.segmentation_config["size_threshold_chars"] = (
+ self.cli.segmentation_threshold
+ )
+
  else:
  self.cli.print_status(
  "Invalid choice. Please select U, F, T, C, H, or Q.", "warning"

cli/cli_interface.py

@@ -40,6 +40,8 @@ def __init__(self):
  self.is_running = True
  self.processing_history = []
  self.enable_indexing = True # Default configuration
+ self.segmentation_enabled = True # Default to smart segmentation
+ self.segmentation_threshold = 50000 # Default threshold
 
  # Check tkinter availability for file dialogs
  self.tkinter_available = True
@@ -125,6 +127,9 @@ def create_menu(self):
  # Display current configuration
  pipeline_mode = "🧠 COMPREHENSIVE" if self.enable_indexing else "⚡ OPTIMIZED"
  index_status = "✅ Enabled" if self.enable_indexing else "🔶 Disabled"
+ segmentation_mode = (
+ "📄 SMART" if self.segmentation_enabled else "📋 TRADITIONAL"
+ )
 
  menu = f"""
 {Colors.BOLD}{Colors.CYAN}╔═══════════════════════════════════════════════════════════════════════════════╗
@@ -135,6 +140,7 @@ def create_menu(self):
 ║ ║
 ║ {Colors.BOLD}🤖 Current Pipeline Mode: {pipeline_mode}{Colors.CYAN} ║
 ║ {Colors.BOLD}🗂️ Codebase Indexing: {index_status}{Colors.CYAN} ║
+║ {Colors.BOLD}📄 Document Processing: {segmentation_mode}{Colors.CYAN} ║
 ║ ║
 ║ {Colors.YELLOW}📝 URL Processing:{Colors.CYAN} ║
 ║ {Colors.YELLOW} ▶ Enter research paper URL (arXiv, IEEE, ACM, etc.) {Colors.CYAN}║
@@ -693,6 +699,11 @@ def show_history(self):
  def show_configuration_menu(self):
  """Show configuration options menu"""
  self.clear_screen()
+
+ # Get segmentation config status
+ segmentation_enabled = getattr(self, "segmentation_enabled", True)
+ segmentation_threshold = getattr(self, "segmentation_threshold", 50000)
+
  print(f"""
 {Colors.BOLD}{Colors.CYAN}╔═══════════════════════════════════════════════════════════════════════════════╗
 ║ CONFIGURATION MENU ║
@@ -716,9 +727,23 @@ def show_configuration_menu(self):
 ║ ✗ Repository Acquisition (Skipped) ║
 ║ ✗ Codebase Intelligence Orchestration (Skipped) ║
 ║ ║
-║ {Colors.YELLOW}Current Setting:{Colors.CYAN} {'🧠 Comprehensive Mode' if self.enable_indexing else '⚡ Optimized Mode'} ║
+║ {Colors.OKCYAN}[2] Document Processing:{Colors.CYAN} ║
+║ {Colors.BOLD}📄 Smart Segmentation{Colors.CYAN} - Intelligent document analysis (Default) ║
+║ ✓ Semantic boundary detection ║
+║ ✓ Algorithm integrity preservation ║
+║ ✓ Formula chain recognition ║
+║ ✓ Adaptive character limits ║
+║ ║
+║ {Colors.BOLD}📋 Traditional Processing{Colors.CYAN} - Full document reading ║
+║ ✓ Complete document analysis ║
+║ ✗ Smart segmentation (Disabled) ║
 ║ ║
-║ {Colors.OKGREEN}[T] Toggle Pipeline Mode {Colors.CYAN}│ {Colors.FAIL}[B] Back to Main Menu{Colors.CYAN} ║
+║ {Colors.YELLOW}Current Settings:{Colors.CYAN} ║
+║ Pipeline: {'🧠 Comprehensive Mode' if self.enable_indexing else '⚡ Optimized Mode'} ║
+║ Document: {'📄 Smart Segmentation' if segmentation_enabled else '📋 Traditional Processing'} ║
+║ Threshold: {segmentation_threshold} characters ║
+║ ║
+║ {Colors.OKGREEN}[T] Toggle Pipeline {Colors.BLUE}[S] Toggle Segmentation {Colors.FAIL}[B] Back{Colors.CYAN} ║
 ╚═══════════════════════════════════════════════════════════════════════════════╝{Colors.ENDC}
 """)
 
@@ -737,8 +762,25 @@ def show_configuration_menu(self):
  self.show_configuration_menu()
  return
 
+ elif choice in ["s", "segmentation"]:
+ current_state = getattr(self, "segmentation_enabled", True)
+ self.segmentation_enabled = not current_state
+ seg_mode = (
+ "📄 Smart Segmentation"
+ if self.segmentation_enabled
+ else "📋 Traditional Processing"
+ )
+ self.print_status(
+ f"Document processing switched to: {seg_mode}", "success"
+ )
+ time.sleep(1)
+ self.show_configuration_menu()
+ return
+
  elif choice in ["b", "back"]:
  return
 
  else:
- self.print_status("Invalid choice. Please enter 'T' or 'B'.", "warning")
+ self.print_status(
+ "Invalid choice. Please enter 'T', 'S', or 'B'.", "warning"
+ )

cli/main_cli.py

@@ -94,15 +94,21 @@ def parse_arguments():
  formatter_class=argparse.RawDescriptionHelpFormatter,
  epilog=f"""
 {Colors.BOLD}Examples:{Colors.ENDC}
- {Colors.CYAN}python main_cli.py{Colors.ENDC} # Interactive mode
- {Colors.CYAN}python main_cli.py --file paper.pdf{Colors.ENDC} # Process file directly
- {Colors.CYAN}python main_cli.py --url https://...{Colors.ENDC} # Process URL directly
- {Colors.CYAN}python main_cli.py --chat "Build a web app..."{Colors.ENDC} # Process chat requirements
- {Colors.CYAN}python main_cli.py --optimized{Colors.ENDC} # Use optimized mode
+ {Colors.CYAN}python main_cli.py{Colors.ENDC} # Interactive mode
+ {Colors.CYAN}python main_cli.py --file paper.pdf{Colors.ENDC} # Process file directly
+ {Colors.CYAN}python main_cli.py --url https://...{Colors.ENDC} # Process URL directly
+ {Colors.CYAN}python main_cli.py --chat "Build a web app..."{Colors.ENDC} # Process chat requirements
+ {Colors.CYAN}python main_cli.py --optimized{Colors.ENDC} # Use optimized mode
+ {Colors.CYAN}python main_cli.py --disable-segmentation{Colors.ENDC} # Disable document segmentation
+ {Colors.CYAN}python main_cli.py --segmentation-threshold 30000{Colors.ENDC} # Custom segmentation threshold
 
 {Colors.BOLD}Pipeline Modes:{Colors.ENDC}
  {Colors.GREEN}Comprehensive{Colors.ENDC}: Full intelligence analysis with indexing
  {Colors.YELLOW}Optimized{Colors.ENDC}: Fast processing without indexing
+
+{Colors.BOLD}Document Processing:{Colors.ENDC}
+ {Colors.BLUE}Smart Segmentation{Colors.ENDC}: Intelligent document segmentation for large papers
+ {Colors.MAGENTA}Supported Formats{Colors.ENDC}: PDF, DOCX, DOC, PPT, PPTX, XLS, XLSX, HTML, TXT, MD
  """,
  )
 
@@ -128,6 +134,19 @@ def parse_arguments():
  help="Use optimized mode (skip indexing for faster processing)",
  )
 
+ parser.add_argument(
+ "--disable-segmentation",
+ action="store_true",
+ help="Disable intelligent document segmentation (use traditional full-document processing)",
+ )
+
+ parser.add_argument(
+ "--segmentation-threshold",
+ type=int,
+ default=50000,
+ help="Document size threshold (characters) to trigger segmentation (default: 50000)",
+ )
+
  parser.add_argument(
  "--verbose", "-v", action="store_true", help="Enable verbose output"
  )
@@ -206,6 +225,24 @@ async def main():
  f"\n{Colors.GREEN}🧠 Comprehensive mode enabled - full intelligence analysis{Colors.ENDC}"
  )
 
+ # Configure document segmentation settings
+ if hasattr(args, "disable_segmentation") and args.disable_segmentation:
+ print(
+ f"\n{Colors.MAGENTA}📄 Document segmentation disabled - using traditional processing{Colors.ENDC}"
+ )
+ app.segmentation_config = {
+ "enabled": False,
+ "size_threshold_chars": args.segmentation_threshold,
+ }
+ else:
+ print(
+ f"\n{Colors.BLUE}📄 Smart document segmentation enabled (threshold: {args.segmentation_threshold} chars){Colors.ENDC}"
+ )
+ app.segmentation_config = {
+ "enabled": True,
+ "size_threshold_chars": args.segmentation_threshold,
+ }
+
  # 检查是否为直接处理模式
  if args.file or args.url or args.chat:
  if args.file: