Merge pull request #5 from moe-mizrak/feat/add-chunk-logic-to-filter-methods

Feat/add chunk logic to filter methods

Author: moe-mizrak

README.md

@@ -20,6 +20,8 @@ You can publish and run the migrations with:
 > - [ ] Modify LOG_PATTERN in FileLogReader, check ppssible patterns in real world cases so that LOG_PATTERN will cover them
 > - [ ] We might end up adding some limits to the file and db log reading (even though it might against what we are achieving here), where when a limit passes cetain size, it does not add anymore e.g. while (! feof($handle)) will be someting like while (!feof($handle) && $resultCount < $limit); also some logic can be added to db log reader for that.
 > - [ ] we might add user_id, requiest_id and ip_address columns to config and the logic, atm we use extra column for them. But those fields/columns could be added specifically. 
+> - [ ] Maybe first get the count/size of the result before doing anything, if it is big then ask for more filter from user, or use streaming etc. So basically first send the count/size request to make sure result is not too big, and then proceed.
+> - [ ] Maybe to be able to work with big log data, another approach might be: The log_insights table acts as a normalized, summarized, and semantically searchable index over all log sources — optimized for MCP-style natural language queries. This way even thoug we use different log mechanisms, we normalized them into a single canonical format, so that mcp tool will perform better, also instead of working with big log data, we do have a summarized log table (with index) so that performance will be increased drastically. We need a system where periodically it adds up new log data to the table. Will it lack some info in the process of summarizing the logs, so that MCP tool might not find the answer? (Since we store semantic results/summaries, we might miss some asnwers for the prompts like "Show me the exact error messages user 433 got when their login failed.", "What was the stack trace for the 500 errors yesterday?")
 
 ## Contributing
 

src/Readers/DatabaseLogReader.php

@@ -28,6 +28,9 @@ public function __construct(
  $this->searchableColumns = config('laravel-log-reader.db.searchable_columns', []);
  }
 
+ /**
+ * {@inheritDoc}
+ */
  public function search(string $query, bool $chunk = false): array
  {
  if (empty($query)) {
@@ -40,32 +43,17 @@ public function search(string $query, bool $chunk = false): array
  ->whereAny($columns, 'like', '%' . $query . '%')
  ->orderByDesc($this->getColumn(LogTableColumnType::TIMESTAMP->value));
 
- if ($chunk) {
- $chunkSize = (int) config('laravel-log-reader.db.chunk_size', 500);
- $results = [];
-
- $builder->chunk($chunkSize, function ($chunk) use (&$results) {
- $results = array_merge($results, $chunk->all());
- });
-
- return $this->convertToLogData($results);
- }
-
- return $this->convertToLogData($builder->get()->all());
+ return $this->executeQuery($builder, $chunk);
  }
 
- public function filter(array $filters = []): array
+ /**
+ * {@inheritDoc}
+ */
+ public function filter(array $filters = [], bool $chunk = false): array
  {
  $builder = $this->getQueryBuilder();
 
- if (empty($filters)) {
- return $this->convertToLogData(
- $builder->orderByDesc($this->getColumn(LogTableColumnType::TIMESTAMP->value))->get()->all()
- );
- }
-
  foreach ($filters as $key => $value) {
- // Apply specific filters based on known keys
  match ($key) {
  FilterKeyType::LEVEL->value => $builder->where($this->getColumn(LogTableColumnType::LEVEL->value), mb_strtolower((string) $value)),
  FilterKeyType::DATE_FROM->value => $builder->where($this->getColumn(LogTableColumnType::TIMESTAMP->value), '>=', $value),
@@ -75,10 +63,31 @@ public function filter(array $filters = []): array
  };
  }
 
- // Return results ordered by creation date descending
- return $this->convertToLogData(
- $builder->orderByDesc($this->getColumn(LogTableColumnType::TIMESTAMP->value))->get()->all()
- );
+ $builder->orderByDesc($this->getColumn(LogTableColumnType::TIMESTAMP->value));
+
+ return $this->executeQuery($builder, $chunk);
+ }
+
+ /**
+ * Executes the query builder and returns LogData array.
+ * Optionally processes results in chunks for memory efficiency.
+ *
+ * @return array<LogData>
+ */
+ protected function executeQuery(Builder $builder, bool $chunk = false): array
+ {
+ if ($chunk) {
+ $chunkSize = (int) config('laravel-log-reader.db.chunk_size', 500);
+ $results = [];
+
+ $builder->chunk($chunkSize, function ($chunk) use (&$results) {
+ $results = array_merge($results, $chunk->all());
+ });
+
+ return $this->convertToLogData($results);
+ }
+
+ return $this->convertToLogData($builder->get()->all());
  }
 
  protected function applyCustomFilter(Builder $builder, string $key, mixed $value): void

src/Readers/FileLogReader.php

@@ -20,84 +20,96 @@
 
  public function __construct(protected string $filePath) {}
 
+ /**
+ * {@inheritDoc}
+ */
  public function search(string $query, bool $chunk = false): array
  {
- if (empty($query)) {
+ if (empty($query) || ! file_exists($this->filePath)) {
  return [];
  }
 
  $searchTerm = mb_strtolower($query);
 
- if ($chunk) {
- $chunkSize = (int) config('laravel-log-reader.file.chunk_size', 512 * 1024); // default 512KB
- $handle = @fopen($this->filePath, 'r');
-
- if (! $handle) {
- return [];
- }
+ return $this->processFileChunks($chunk, function (array $logs) use ($searchTerm): array {
+ return array_filter(
+ $logs,
+ fn(LogData $log) => str_contains(mb_strtolower($log->message ?? ''), $searchTerm) ||
+ str_contains(mb_strtolower($log->context ?? ''), $searchTerm)
+ );
+ });
+ }
 
- $results = [];
- $buffer = '';
+ /**
+ * {@inheritDoc}
+ */
+ public function filter(array $filters = [], bool $chunk = false): array
+ {
+ if (! file_exists($this->filePath)) {
+ return [];
+ }
 
- while (! feof($handle)) {
- $buffer .= fread($handle, $chunkSize);
+ if (empty($filters)) {
+ return $this->parseLogFile();
+ }
 
- if (! feof($handle)) {
- $lastNewLinePos = strrpos($buffer, PHP_EOL);
+ return $this->processFileChunks($chunk, function (array $logs) use ($filters): array {
+ foreach ($filters as $key => $value) {
+ $logs = array_filter($logs, fn(LogData $log) => $this->matchesFilter($log, $key, $value));
+ }
 
- if ($lastNewLinePos === false) {
- // no newline yet — read more before parsing
- continue;
- }
+ return $logs;
+ });
+ }
 
- $contentChunk = substr($buffer, 0, $lastNewLinePos);
- $buffer = substr($buffer, $lastNewLinePos + 1);
- } else {
- $contentChunk = $buffer;
- $buffer = '';
- }
+ /**
+ * @param callable(array<LogData>)
+ *
+ * @return array<LogData>
+ */
+ protected function processFileChunks(bool $chunk, callable $callback): array
+ {
+ if (! $chunk) {
+ $logs = $this->parseLogFile();
 
-  // Parse and filter this chunk
-  $logs = $this->convertToLogData($this->extractLogsFromContent($contentChunk));
+ return array_values($callback($logs));
+ }
 
- $filtered = array_filter(
- $logs,
- fn(LogData $log) => str_contains(mb_strtolower($log->message ?? ''), $searchTerm)
- || str_contains(mb_strtolower($log->context ?? ''), $searchTerm)
- );
+ $chunkSize = (int) config('laravel-log-reader.file.chunk_size', 512 * 1024);
+ $handle = @fopen($this->filePath, 'r');
 
- $results = array_merge($results, array_values($filtered));
- }
+ if (! $handle) {
+ return [];
+ }
 
- fclose($handle);
+ $results = [];
+ $buffer = '';
 
-  return $results;
- }
+ while (! feof($handle)) {
+  $buffer .= fread($handle, $chunkSize);
 
- // Non-chunked (default)
- $logs = $this->parseLogFile();
+  if (! feof($handle)) {
+  $lastNewLinePos = strrpos($buffer, PHP_EOL);
 
- return array_values(array_filter(
- $logs,
- fn(LogData $log) => str_contains(mb_strtolower($log->message ?? ''), $searchTerm) ||
- str_contains(mb_strtolower($log->context ?? ''), $searchTerm)
- ));
- }
+ if ($lastNewLinePos === false) {
+ continue;
+ }
 
- // todo add chunking/stream logic to filter method for both database filter method and here
- public function filter(array $filters = []): array
- {
- $logs = $this->parseLogFile();
+ $contentChunk = substr($buffer, 0, $lastNewLinePos);
+ $buffer = substr($buffer, $lastNewLinePos + 1);
+ } else {
+ $contentChunk = $buffer;
+ $buffer = '';
+ }
 
- if (empty($filters)) {
- return $logs;
+ $logs = $this->convertToLogData($this->extractLogsFromContent($contentChunk));
+ $filtered = $callback($logs);
+ $results = array_merge($results, array_values($filtered));
  }
 
- foreach ($filters as $key => $value) {
- $logs = array_filter($logs, fn(LogData $log) => $this->matchesFilter($log, $key, $value));
- }
+ fclose($handle);
 
- return array_values($logs);
+ return $results;
  }
 
  /**

src/Readers/LogReaderInterface.php

@@ -11,11 +11,15 @@ interface LogReaderInterface
 {
  /**
  * Search logs based on a query string.
+ *
+ * @return array<LogData>
  */
  public function search(string $query, bool $chunk = false): array;
 
  /**
  * Filter logs based on filter criteria.
+ *
+ * @return array<LogData>
  */
- public function filter(array $filters = []): array;
+ public function filter(array $filters = [], bool $chunk = false): array;
 }

tests/Readers/DatabaseLogReaderTest.php

@@ -319,7 +319,6 @@ public function it_decodes_extra_with_user_and_request_metadata(): void
  /* ASSERT */
  $this->assertCount(1, $results);
  $log = $results[0];
-
  $this->assertSame('User metadata logged', $log->message);
  $this->assertIsArray($log->extra);
  $this->assertSame($userId, $log->extra['user_id']);
@@ -433,6 +432,66 @@ public function it_searches_logs_with_chunk_size_greater_than_one(): void
  $this->assertSame($expected, $messages);
  }
 
+ #[Test]
+ public function it_filters_logs_using_chunking(): void
+ {
+ /* SETUP */
+ config(['laravel-log-reader.db.chunk_size' => 1]);
+
+ /* EXECUTE */
+ $results = $this->databaseLogReader->filter([FilterKeyType::LEVEL->value => 'error'], true);
+
+ /* ASSERT */
+ $this->assertCount(1, $results);
+ $this->assertSame('Payment failed', $results[0]->message);
+ }
+
+ #[Test]
+ public function it_filters_returns_same_results_with_and_without_chunking(): void
+ {
+ /* SETUP */
+ config(['laravel-log-reader.db.chunk_size' => 1]);
+
+ /* EXECUTE */
+ $nonChunked = $this->databaseLogReader->filter([FilterKeyType::CHANNEL->value => 'payment'], false);
+ $chunked = $this->databaseLogReader->filter([FilterKeyType::CHANNEL->value => 'payment'], true);
+
+ /* ASSERT */
+ $this->assertCount(1, $nonChunked);
+ $this->assertCount(1, $chunked);
+ $this->assertSame($nonChunked[0]->message, $chunked[0]->message);
+ }
+
+ #[Test]
+ public function it_filters_logs_with_chunk_size_greater_than_one(): void
+ {
+ /* SETUP */
+ config(['laravel-log-reader.db.chunk_size' => 2]);
+ $count = 5;
+ for ($i = 1; $i <= $count; $i++) {
+ DB::table($this->table)->insert([
+ 'level' => 'warning',
+ 'message' => "Warning {$i}",
+ 'channel' => 'bulk',
+ 'context' => '{}',
+ 'extra' => '{}',
+ 'created_at' => now()->subSeconds($count - $i),
+ ]);
+ }
+
+ /* EXECUTE */
+ $results = $this->databaseLogReader->filter([FilterKeyType::LEVEL->value => 'warning'], true);
+
+ /* ASSERT */
+ $this->assertCount($count, $results);
+ $expected = [];
+ for ($i = $count; $i >= 1; $i--) {
+ $expected[] = "Warning {$i}";
+ }
+ $messages = array_map(fn($r) => $r->message, $results);
+ $this->assertSame($expected, $messages);
+ }
+
  /**
  * Create the logs table for testing.
  */

tests/Readers/FileLogReaderTest.php

@@ -237,8 +237,6 @@ public function it_returns_same_results_with_and_without_chunking(): void
 
  /* ASSERT */
  $this->assertCount(count($nonChunked), $chunked);
- 
- /* ASSERT */
  $nonChunkedMessages = array_map(fn($r) => $r->message, $nonChunked);
  $chunkedMessages = array_map(fn($r) => $r->message, $chunked);
  $this->assertSame($nonChunkedMessages, $chunkedMessages);
@@ -278,6 +276,40 @@ public function it_reads_all_logs_correctly_when_chunking_with_unique_data(): vo
  $this->assertSame($expectedMessages, $messages);
  }
 
+ #[Test]
+ public function it_filters_logs_using_chunking(): void
+ {
+ /* SETUP */
+ config(['laravel-log-reader.file.chunk_size' => 64]);
+
+ /* EXECUTE */
+ $results = $this->fileReader->filter([
+ FilterKeyType::LEVEL->value => 'error',
+ ], true);
+
+ /* ASSERT */
+ $this->assertCount(1, $results);
+ $this->assertSame('ERROR', $results[0]->level);
+ $this->assertStringContainsString('undefined method', $results[0]->message);
+ }
+
+ #[Test]
+ public function it_filters_using_chunking_and_checks_same_results_with_and_without_chunking(): void
+ {
+ /* SETUP */
+ config(['laravel-log-reader.file.chunk_size' => 64]);
+
+ /* EXECUTE */
+ $nonChunked = $this->fileReader->filter([FilterKeyType::LEVEL->value => 'info'], false);
+ $chunked = $this->fileReader->filter([FilterKeyType::LEVEL->value => 'info'], true);
+
+ /* ASSERT */
+ $this->assertCount(count($nonChunked), $chunked);
+ $nonChunkedMessages = array_map(fn($r) => $r->message, $nonChunked);
+ $chunkedMessages = array_map(fn($r) => $r->message, $chunked);
+ $this->assertSame($nonChunkedMessages, $chunkedMessages);
+ }
+
  /**
  * Creates a temporary log file with sample log entries for testing.
  */