feat: add logger integration

Author: VinciGit00

scrapegraphai/nodes/fetch_node.py

@@ -1,4 +1,4 @@
-""" 
+""""
 FetchNode Module
 """
 
@@ -13,7 +13,7 @@
 from ..docloaders import ChromiumLoader
 from .base_node import BaseNode
 from ..utils.cleanup_html import cleanup_html
-
+from ..utils.logging import get_logger
 
 class FetchNode(BaseNode):
  """
@@ -74,7 +74,8 @@ def execute(self, state):
  necessary information to perform the operation is missing.
  """
  if self.verbose:
- print(f"--- Executing {self.node_name} Node ---")
+ logger = get_logger("fetch node")
+ logger.info(f"--- Executing {self.node_name} Node ---")
 
  # Interpret input keys based on the provided input expression
  input_keys = self.get_input_keys(state)
@@ -128,7 +129,7 @@ def execute(self, state):
  cleanedup_html = cleanup_html(response.text, source)
  compressed_document = [Document(page_content=cleanedup_html)]
  else:
- print(f"Failed to retrieve contents from the webpage at url: {source}")
+ logger.warning(f"Failed to retrieve contents from the webpage at url: {source}")
 
  else:
  loader_kwargs = {}
@@ -144,4 +145,4 @@ def execute(self, state):
  ]
 
  state.update({self.output[0]: compressed_document})
- return state
+ return state

scrapegraphai/nodes/generate_answer_csv_node.py

@@ -9,6 +9,7 @@
 from langchain.prompts import PromptTemplate
 from langchain_core.output_parsers import JsonOutputParser
 from langchain_core.runnables import RunnableParallel
+from ..utils.logging import get_logger
 
 # Imports from the library
 from .base_node import BaseNode
@@ -72,7 +73,8 @@ def execute(self, state):
  """
 
  if self.verbose:
- print(f"--- Executing {self.node_name} Node ---")
+ logger = get_logger("generate_answer csv node")
+ logger.info(f"--- Executing {self.node_name} Node ---")
 
  # Interpret input keys based on the provided input expression
  input_keys = self.get_input_keys(state)

scrapegraphai/nodes/generate_answer_node.py

@@ -10,6 +10,7 @@
 from langchain.prompts import PromptTemplate
 from langchain_core.output_parsers import JsonOutputParser
 from langchain_core.runnables import RunnableParallel
+from ..utils.logging import get_logger
 
 # Imports from the library
 from .base_node import BaseNode
@@ -59,7 +60,8 @@ def execute(self, state: dict) -> dict:
  """
 
  if self.verbose:
- print(f"--- Executing {self.node_name} Node ---")
+ logger = get_logger("generate answer node")
+ logger.info(f"--- Executing {self.node_name} Node ---")
 
  # Interpret input keys based on the provided input expression
  input_keys = self.get_input_keys(state)

scrapegraphai/nodes/generate_answer_pdf_node.py

@@ -9,6 +9,7 @@
 from langchain.prompts import PromptTemplate
 from langchain_core.output_parsers import JsonOutputParser
 from langchain_core.runnables import RunnableParallel
+from ..utils.logging import get_logger
 
 # Imports from the library
 from .base_node import BaseNode
@@ -72,7 +73,8 @@ def execute(self, state):
  """
 
  if self.verbose:
- print(f"--- Executing {self.node_name} Node ---")
+ logger = get_logger("generate answer pdf node")
+ logger.info(f"--- Executing {self.node_name} Node ---")
 
  # Interpret input keys based on the provided input expression
  input_keys = self.get_input_keys(state)

scrapegraphai/nodes/generate_scraper_node.py

@@ -10,6 +10,7 @@
 from langchain.prompts import PromptTemplate
 from langchain_core.output_parsers import StrOutputParser
 from langchain_core.runnables import RunnableParallel
+from ..utils.logging import get_logger
 
 # Imports from the library
 from .base_node import BaseNode
@@ -63,7 +64,8 @@ def execute(self, state: dict) -> dict:
  """
 
  if self.verbose:
- print(f"--- Executing {self.node_name} Node ---")
+ logger = get_logger("generate scraper node")
+ logger.info(f"--- Executing {self.node_name} Node ---")
 
  # Interpret input keys based on the provided input expression
  input_keys = self.get_input_keys(state)

scrapegraphai/nodes/get_probable_tags_node.py

@@ -6,7 +6,7 @@
 from langchain.output_parsers import CommaSeparatedListOutputParser
 from langchain.prompts import PromptTemplate
 from .base_node import BaseNode
-
+from ..utils.logging import get_logger
 
 class GetProbableTagsNode(BaseNode):
  """
@@ -25,11 +25,12 @@ class GetProbableTagsNode(BaseNode):
  node_name (str): The unique identifier name for the node, defaulting to "GetProbableTags".
  """
 
- def __init__(self, input: str, output: List[str], model_config: dict,
+ def __init__(self, input: str, output: List[str], node_config: dict,
  node_name: str = "GetProbableTags"):
- super().__init__(node_name, "node", input, output, 2, model_config)
+ super().__init__(node_name, "node", input, output, 2, node_config)
 
- self.llm_model = model_config["llm_model"]
+ self.llm_model = node_config["llm_model"]
+ self.verbose = False if node_config is None else node_config.get("verbose", False)
 
  def execute(self, state: dict) -> dict:
  """
@@ -49,7 +50,9 @@ def execute(self, state: dict) -> dict:
  necessary information for generating tag predictions is missing.
  """
 
- print(f"--- Executing {self.node_name} Node ---")
+ if self.verbose:
+ logger = get_logger("get probable tags node")
+ logger.info(f"--- Executing {self.node_name} Node ---")
 
  # Interpret input keys based on the provided input expression
  input_keys = self.get_input_keys(state)

scrapegraphai/nodes/graph_iterator_node.py

@@ -5,7 +5,7 @@
 import asyncio
 import copy
 from typing import List, Optional
-
+from ..utils.logging import get_logger
 from tqdm.asyncio import tqdm
 
 from .base_node import BaseNode

scrapegraphai/nodes/image_to_text_node.py

@@ -4,6 +4,7 @@
 
 from typing import List, Optional
 from .base_node import BaseNode
+from ..utils.logging import get_logger
 
 
 class ImageToTextNode(BaseNode):
@@ -42,7 +43,8 @@ def execute(self, state: dict) -> dict:
  """
 
  if self.verbose:
- print("---GENERATING TEXT FROM IMAGE---")
+ logger = get_logger("image to text node")
+ logger.info(f"--- Executing {self.node_name} Node ---")
 
  input_keys = self.get_input_keys(state)
  input_data = [state[key] for key in input_keys]

scrapegraphai/nodes/merge_answers_node.py

@@ -5,7 +5,7 @@
 # Imports from standard library
 from typing import List, Optional
 from tqdm import tqdm
-
+from ..utils.logging import get_logger
 # Imports from Langchain
 from langchain.prompts import PromptTemplate
 from langchain_core.output_parsers import JsonOutputParser
@@ -54,7 +54,8 @@ def execute(self, state: dict) -> dict:
  """
 
  if self.verbose:
- print(f"--- Executing {self.node_name} Node ---")
+ logger = get_logger("fetch node")
+ logger.info(f"--- Executing {self.node_name} Node ---")
 
  # Interpret input keys based on the provided input expression
  input_keys = self.get_input_keys(state)

scrapegraphai/nodes/parse_node.py

@@ -6,7 +6,7 @@
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.document_transformers import Html2TextTransformer
 from .base_node import BaseNode
-
+from ..utils.logging import get_logger
 
 class ParseNode(BaseNode):
  """
@@ -49,7 +49,8 @@ def execute(self, state: dict) -> dict:
  """
 
  if self.verbose:
- print(f"--- Executing {self.node_name} Node ---")
+ logger = get_logger("fetch node")
+ logger.info(f"--- Executing {self.node_name} Node ---")
 
  # Interpret input keys based on the provided input expression
  input_keys = self.get_input_keys(state)