Add the ability to filter by max file size

Author: cyclotruc

src/ingest.py

@@ -269,6 +269,45 @@ def generate_token_string(context_string: str) -> str:
  formatted_tokens = f"{total_gpt_tokens}"
  return formatted_tokens
 
+
+def ingest_single_file(path: str, query: dict) -> Dict:
+ if not os.path.isfile(path):
+ raise ValueError(f"Path {path} is not a file")
+ 
+ file_size = os.path.getsize(path)
+ is_text = is_text_file(path)
+ if not is_text:
+ raise ValueError(f"File {path} is not a text file")
+ 
+ content = read_file_content(path)
+ if file_size > query['max_file_size']:
+ content = "[Content ignored: file too large]"
+ 
+ file_info = {
+ "path": path.replace(query['local_path'], ""),
+ "content": content,
+ "size": file_size
+ }
+ 
+ summary = (
+ f"Repository: {query['user_name']}/{query['repo_name']}\n"
+ f"File: {os.path.basename(path)}\n"
+ f"Size: {file_size:,} bytes\n"
+ f"Lines: {len(content.splitlines()):,}\n"
+ )
+ 
+ 
+ 
+ files_content = create_file_content_string([file_info])
+ tree = "Directory structure:\n└── " + os.path.basename(path)
+
+
+ formatted_tokens = generate_token_string(files_content)
+ if formatted_tokens:
+ summary += f"\nEstimated tokens: {formatted_tokens}"
+ return (summary, tree, files_content)
+
+
 def ingest_from_query(query: dict, ignore_patterns: List[str] = DEFAULT_IGNORE_PATTERNS) -> Dict:
  """Main entry point for analyzing a codebase directory or single file."""
 
@@ -277,43 +316,7 @@ def ingest_from_query(query: dict, ignore_patterns: List[str] = DEFAULT_IGNORE_P
  raise ValueError(f"{query['slug']} cannot be found, make sure the repository is public")
 
  if query.get('type') == 'blob':
- if not os.path.isfile(path):
- raise ValueError(f"Path {path} is not a file")
- 
- file_size = os.path.getsize(path)
- is_text = is_text_file(path)
- if not is_text:
- raise ValueError(f"File {path} is not a text file")
- 
- content = read_file_content(path)
- if file_size > query['max_file_size']:
- content = "[Content ignored: file too large]"
- 
- file_info = {
- "path": path.replace(query['local_path'], ""),
- "content": content,
- "size": file_size
- }
- 
- summary = (
- f"Repository: {query['user_name']}/{query['repo_name']}\n"
- f"File: {os.path.basename(path)}\n"
- f"Size: {file_size:,} bytes\n"
- f"Lines: {len(content.splitlines()):,}\n"
- )
- 
- 
- 
- files_content = create_file_content_string([file_info])
- tree = "Directory structure:\n└── " + os.path.basename(path)
-
-
- print(files_content)
- formatted_tokens = generate_token_string(files_content)
- if formatted_tokens:
- summary += f"\nEstimated tokens: {formatted_tokens}"
- return (summary, tree, files_content)
- 
+ return ingest_single_file(path, query)
  else:
  nodes = scan_directory(path, ignore_patterns, query['local_path'])
  files = extract_files_content(query, nodes, query['max_file_size'])

src/routers/dynamic.py

@@ -6,7 +6,7 @@
 from process_query import process_query
 from config import MAX_DISPLAY_SIZE
 from utils.limiter import limiter
-
+from utils.log_convert import logSliderToSize
 
 router = APIRouter()
 templates = Jinja2Templates(directory="templates")
@@ -21,15 +21,27 @@ async def catch_all(request: Request, full_path: str):
  "result": False,
  "loading": True,
  "github_url": f"https://github.com/{full_path}",
+ "default_file_size": 243
  }
  )
 
 @router.post("/{full_path:path}", response_class=HTMLResponse)
 @limiter.limit("10/minute") 
-async def process_catch_all(request: Request, input_text: str = Form(...)):
+async def process_catch_all(
+ request: Request, 
+ input_text: str = Form(...),
+ max_file_size: int = Form(...)
+):
+
+ slider_position = max_file_size
+ size_in_kb = logSliderToSize(max_file_size)
  try:
- parsed_url = parse_url(input_text)
- summary, tree, content = await process_query(parsed_url)
+ parsed_query = parse_url(input_text, size_in_kb)
+ 
+ summary, tree, content = await process_query(
+ parsed_query,
+ 
+ )
  except Exception as e:
  print(e)
  return templates.TemplateResponse(
@@ -38,11 +50,13 @@ async def process_catch_all(request: Request, input_text: str = Form(...)):
  "request": request, 
  "result": False, 
  "loading": False,
- "error_message": f"Error: \n {e}"
+ "github_url": input_text,
+ "error_message": f"Error: \n {e}",
+ "default_file_size": slider_position,
+ 
  }
  )
 
- 
  if len(content) > MAX_DISPLAY_SIZE:
  content = f"(Files content cropped to {int(MAX_DISPLAY_SIZE/1000)}k characters, download full digest to see more)\n" + content[:MAX_DISPLAY_SIZE]
 
@@ -55,6 +69,8 @@ async def process_catch_all(request: Request, input_text: str = Form(...)):
  "summary": summary,
  "tree": tree, 
  "content": content,
- "ingest_id": parsed_url["id"]
+ "ingest_id": parsed_query["id"],
+ "github_url": input_text,
+ "default_file_size": max_file_size 
  }
  )

src/routers/index.py

@@ -1,27 +1,19 @@
 from fastapi import APIRouter, Request, Form
 from fastapi.responses import HTMLResponse, JSONResponse
 from fastapi.templating import Jinja2Templates
-import math
 
 from utils.parse_url import parse_url
 from utils.limiter import limiter
 from process_query import process_query
 from config import MAX_DISPLAY_SIZE, EXAMPLE_REPOS
+from utils.log_convert import logSliderToSize
 
 
 router = APIRouter()
 templates = Jinja2Templates(directory="templates")
 
 
-def logSliderToSize(position):
- """Convert slider position to file size in KB"""
- minp = 0
- maxp = 500
- minv = math.log(1)
- maxv = math.log(102400)
- 
- scale = (maxv - minv) / (maxp - minp)
- return round(math.exp(minv + scale * (position - minp)))
+
 
 
 @router.get("/", response_class=HTMLResponse)
@@ -31,7 +23,7 @@ async def home(request: Request):
  {
  "request": request,
  "examples": EXAMPLE_REPOS,
- "default_file_size": 250
+ "default_file_size": 243
  }
  )
 
@@ -43,12 +35,8 @@ async def index_post(
  input_text: str = Form(...),
  max_file_size: int = Form(...)
 ):
- # Store original slider position (0-500)
  slider_position = max_file_size
- 
- # Convert to KB for processing
  size_in_kb = logSliderToSize(max_file_size)
- print(f"Processing repository with max file size: {size_in_kb}kb")
 
  try:
  parsed_query = parse_url(input_text, size_in_kb)
@@ -69,7 +57,6 @@ async def index_post(
  if len(content) > MAX_DISPLAY_SIZE:
  content = f"(Files content cropped to {int(MAX_DISPLAY_SIZE/1000)}k characters, download full ingest to see more)\n" + content[:MAX_DISPLAY_SIZE]
 
- print(f"Slider position: {(max_file_size)}")
  return templates.TemplateResponse(
  "index.jinja.html", 
  {

src/static/js/utils.js

@@ -129,8 +129,7 @@ function logSliderToSize(position) {
  const minv = Math.log(1);
  const maxv = Math.log(102400);
 
- const scale = (maxv - minv) / (maxp - minp);
- const value = Math.exp(minv + scale * (position - minp));
+ const value = Math.exp(minv + (maxv - minv) * Math.pow(position / maxp, 1.5));
  return Math.round(value);
 }
 

src/templates/components/github_form.jinja.html

@@ -31,10 +31,10 @@
  max="500" 
  value="{{ default_file_size }}"
  class="w-full h-3 
- bg-[#FE4A60] <!-- This defines the left (filled) part color -->
+ bg-[#FE4A60]
  bg-no-repeat 
  bg-[length:50%_100%] 
- bg-[#FFF4DA] <!-- This defines the right (unfilled) part color -->
+ bg-[#FFF4DA]
  appearance-none 
  border-[3px] 
  border-gray-900 
@@ -70,63 +70,4 @@
  </div>
  {% endif %}
  </div>
-</div> 
-
-<script>
-document.addEventListener('DOMContentLoaded', function() {
- const slider = document.getElementById('file_size');
- const sizeValue = document.getElementById('size_value');
- const form = document.getElementById('ingestForm');
-
- // Convert linear slider value (0-500) to logarithmic size (1KB to 100MB)
- function logSliderToSize(position) {
- const minp = 0;
- const maxp = 500;
- const minv = Math.log(1);
- const maxv = Math.log(102400);
-
- const scale = (maxv - minv) / (maxp - minp);
- const value = Math.exp(minv + scale * (position - minp));
- return Math.round(value);
- }
-
- function formatSize(sizeInKB) {
- if (sizeInKB >= 1024) {
- return Math.round(sizeInKB / 1024) + 'mb';
- }
- return Math.round(sizeInKB) + 'kb';
- }
-
- function updateSlider() {
- const value = logSliderToSize(slider.value);
- sizeValue.textContent = formatSize(value);
- // Update the background size
- slider.style.backgroundSize = `${(slider.value / slider.max) * 100}% 100%`;
- }
-
- // Initialize slider
- if (slider) {
- updateSlider(); // This will now update both the label and background on load
- }
-
- // Update on slider change
- slider.addEventListener('input', updateSlider);
-
- // Add form submit handler to convert size before sending
- form.addEventListener('submit', function(e) {
- // Get the actual KB value from the slider
- const kbValue = logSliderToSize(slider.value);
- 
- // Create a hidden input to send the actual KB value
- const hiddenInput = document.createElement('input');
- hiddenInput.type = 'hidden';
- hiddenInput.name = 'max_file_size';
- hiddenInput.value = kbValue;
- 
- // Replace the original slider input in the form submission
- form.appendChild(hiddenInput);
- });
-});
-
-
-</script> 
+</div> 

src/utils/log_convert.py

@@ -0,0 +1,9 @@
+import math
+
+def logSliderToSize(position):
+ """Convert slider position to file size in KB"""
+ maxp = 500
+ minv = math.log(1)
+ maxv = math.log(102400)
+ 
+ return round(math.exp(minv + (maxv - minv) * pow(position / maxp, 1.5)))