Ace Your Exams: Automated Question Generation for the Diligent Student

######################## # LOAD LIBRARIES ######################## import spacy import warnings import time import logging from langchain.llms import Ollama from langchain.prompts import PromptTemplate from pdfminer.high_level import extract_text from tqdm import tqdm ################################################################################### # SUPPRESS WARNINGS THAT CAN CLUTTER THE OUTPUT, SUCH AS DEPRECATION WARNINGS, ETC. ################################################################################### warnings.filterwarnings("ignore") ##################################### # LOAD THE SPACY LANGUAGE MODEL. ##################################### nlp = spacy.load("en_core_web_sm") def nest_sentences(document, max_length=4096): """ Break down a document into manageable chunks of sentences where each chunk is under a specified length. Parameters: - document (str): The input text document to be processed. - max_length (int): The maximum character length for each chunk. Returns: - list: A list where each element is a group of sentences that together are less than max_length characters. """ nested = [] # List to hold all chunks of sentences  sent = [] # Temporary list to hold sentences for a current chunk  length = 0 # Counter to keep track of the character length of the current chunk  doc = nlp(document) # Process the document using Spacy to tokenize into sentences  for sentence in doc.sents: length += len(sentence.text) if length < max_length: sent.append(sentence.text) else: nested.append(' '.join(sent)) # Join sentences in the chunk and add to the nested list  sent = [sentence.text] # Start a new chunk with the current sentence  length = len(sentence.text) # Reset the length counter to the length of the current sentence  if sent: # Don't forget to add the last chunk if it's not empty  nested.append(' '.join(sent)) return nested def generate_summary(text, llm, max_length=4096): """ Generate a summary for provided text using the specified large language model (LLM). Parameters: - text (str): Text to summarize. - llm (LLMChain): The large language model to use for generating summaries. - max_length (int): The maximum character length for each summary chunk. Returns: - str: A single string that is the concatenated summary of all processed chunks. """ sentences = nest_sentences(text, max_length) summaries = [] # List to hold summaries of each chunk  seen_questions = set() # Set to track unique questions  prompt_template = PromptTemplate( input_variables=["text"], template="Generate diverse multiple-choice questions/answer are one sentence long based on the context here: {text}. " "Ensure each question is unique and not repetitive. " "Format:\nQuestion: Question?\n- A) Option A.\n- B) Option B.\n- C) Option C.\n- D) Option D.\nAnswer: Answer\n***\n" ) for chunk in tqdm(sentences, desc="Generating summaries"): # Use the LLM to generate the summary based on the prompt.  prompt = prompt_template.format(text=chunk) result = llm.invoke(prompt) result_lines = result.strip().split("\n") for line in result_lines: if line.startswith("Question:"): question = line.strip() if question not in seen_questions: summaries.append(question) seen_questions.add(question) answer = next((l.strip() for l in result_lines if l.startswith("Answer:")), "") summaries.append(answer) # Optionally print each generated summary.  print(result.strip()) # Join all summaries into a single string with spaces in between.  return "\n".join(summaries) def main_loop(delay): """ Run the main loop, which generates summaries periodically, for 30 minutes. Parameters: - delay (int): The delay in seconds between each iteration of the loop. """ end_time = time.time() + 30 * 60 # 30 minutes from now  while time.time() < end_time: try: # Extract text from a PDF file.  text = extract_text("/home/roomal/Desktop/PSY-3180/pdfs/Book 1/13 - Reproductive Behavior.pdf") # Generate and print the summary for the extracted text.  summary = generate_summary(text, llm) print(summary) except Exception as e: logging.error(f"An error occurred: {e}") # Pause for the specified delay before the next iteration.  time.sleep(delay) ##################################### # CONFIGURATION FOR THE LANGUAGE MODEL. ##################################### llm = Ollama(model="llama3:latest", temperature=0.9) ####################################### # RUN THE MAIN LOOP FOR 30 MINUTES ####################################### if __name__ == '__main__': logging.basicConfig(level=logging.INFO) logging.info("Starting the main loop for 30 minutes... Or whatever.") delay = int(input("Enter the delay time in seconds between each iteration: ")) main_loop(delay) logging.info("Main loop completed.") 

# pip install wikipedia-api bertopic umap-learn pandas nltk pdfminer.six tqdm rich langchain  import wikipediaapi import pandas as pd import concurrent.futures from tqdm import tqdm import json import nltk import re import time import umap import logging from bertopic import BERTopic from bertopic.representation import KeyBERTInspired from langchain import PromptTemplate from langchain.llms import Ollama from nltk.corpus import stopwords from nltk.tokenize import sent_tokenize, word_tokenize from rich.progress import Progress from sklearn.feature_extraction.text import CountVectorizer nltk.download('punkt') nltk.download('stopwords') logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') def scrape_wikipedia(name_topic, verbose=True, max_workers=5): def link_to_wikipedia(link): try: page = api_wikipedia.page(link) if page.exists(): return { "page": link, "text": page.text, "link": page.fullurl, "categories": list(page.categories.keys()), } except Exception as e: print(f"Error processing {link}: {e}") return None api_wikipedia = wikipediaapi.Wikipedia( language="en", user_agent="YourUserAgentHere", extract_format=wikipediaapi.ExtractFormat.WIKI, ) name_of_page = api_wikipedia.page(name_topic) if not name_of_page.exists(): print(f"Page {name_topic} is not present") return links_to_page = list(name_of_page.links.keys()) procceed = tqdm(desc="Scraped links", unit="", total=len(links_to_page)) if verbose else None origin = [{ "page": name_topic, "text": name_of_page.text, "link": name_of_page.fullurl, "categories": list(name_of_page.categories.keys()), }] with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: links_future = {executor.submit(link_to_wikipedia, link): link for link in links_to_page} for future in concurrent.futures.as_completed(links_future): info = future.result() if info: origin.append(info) if verbose: procceed.update(1) if verbose: procceed.close() # Define namespaces to exclude  namespaces = ( "Wikipedia", "Special", "Talk", "LyricWiki", "File", "MediaWiki", "Template", "Help", "User", "Category talk", "Portal talk" ) # Create DataFrame and filter based on text length and namespaces  origin_df = pd.DataFrame(origin) origin_df = origin_df[ (origin_df["text"].str.len() > 20) & (~origin_df["page"].str.startswith(namespaces, na=True)) ] # Process categories to remove 'Category:' prefix  origin_df["categories"] = origin_df["categories"].apply(lambda cats: [cat[9:] for cat in cats]) origin_df["topic"] = name_topic print("Scraped pages:", len(origin_df)) return origin_df def clean_text(text): text = text.lower() text = re.sub(r'\s+', ' ', text) # Remove extra whitespace  text = re.sub(r'[^a-z0-9\s.]', '', text) # Remove non-alphanumeric characters except periods  return text def tokenize_text(text): tokens = word_tokenize(text) return tokens def remove_stopwords(tokens): filtered_tokens = [token for token in tokens if token not in stopwords.words('english')] return filtered_tokens def preprocess_text(text): cleaned_text = clean_text(text) tokens = tokenize_text(cleaned_text) filtered_tokens = remove_stopwords(tokens) return ' '.join(filtered_tokens) def generate_questions(context, llm, prompt_template): formatted_prompt = prompt_template.format(context=context) response = llm.invoke(formatted_prompt) return response.strip() # Start the process logging.info("Starting the process...") # 1. Scrape Wikipedia for the specified topic topic = input("Enter the Wikipedia topic to scrape: ") data = scrape_wikipedia(topic) if data is None or data.empty: logging.error(f"No data found for topic: {topic}") exit() # Save the scraped data to a CSV file data.to_csv("/home/roomal/Desktop/scraped_data.csv", index=False) logging.info("Scraped data saved to CSV.") # Convert the scraped text data into a single text format text_data = " ".join(data["text"].tolist()) # 2. Preprocess the text processed_text = preprocess_text(text_data) logging.info("Text preprocessing complete.") # 3. Tokenize the text into sentences sentences = sent_tokenize(processed_text) logging.info(f"Number of sentences extracted: {len(sentences)}") # 4. Configure UMAP and vectorizer parameters umap_model = umap.UMAP(n_neighbors=10, n_components=5, min_dist=0.1, metric='cosine') vectorizer_model = CountVectorizer(min_df=1, max_df=0.95) # 5. Fit BERTopic model logging.info("Fitting BERTopic model...") topic_model = BERTopic( umap_model=umap_model, vectorizer_model=vectorizer_model, representation_model=KeyBERTInspired() ) with Progress() as progress: task = progress.add_task("Fitting BERTopic model...", total=len(sentences)) topics, probs = topic_model.fit_transform(sentences) progress.update(task, advance=len(sentences)) logging.info("BERTopic model fitted successfully.") topic_info = topic_model.get_topic_info() # Save the topic information to a CSV file topic_info.to_csv("/home/roomal/Desktop/topic_info.csv", index=False) logging.info("Topic information saved to CSV.") # 6. Fine-tune topic representations with Ollama logging.info("Fine-tuning topic representations with Ollama...") topic_representation_model = Ollama(model="mistral:latest", temperature=0.8) topic_model = BERTopic( representation_model=topic_representation_model, umap_model=umap_model, vectorizer_model=vectorizer_model ) with Progress() as progress: task = progress.add_task("Fitting BERTopic with Ollama...", total=len(sentences)) topic_model.fit(sentences) progress.update(task, advance=len(sentences)) logging.info("Topic representations fine-tuned successfully.") # 7. Get representative documents for each topic docs_by_topic = topic_model.get_representative_docs() # 8. Generate questions using LangChain llm_for_questions = Ollama(model="llama3:latest", temperature=0.5) # Define a more varied prompt template for generating questions prompt_template = PromptTemplate( input_variables=['context'], template='''Generate several insightful and varied questions based on the context below: Context: {context} Questions:\n 1. ''' ) # Generate questions for each topic questions = [] with Progress() as progress: task = progress.add_task("Generating questions...", total=len(docs_by_topic)) for topic, docs in docs_by_topic.items(): context = " ".join(docs) question = generate_questions(context, llm_for_questions, prompt_template) questions.append(question) progress.update(task, advance=1) logging.info("Questions generated successfully.") # Print the generated questions for i, question in enumerate(questions): print(f"Topic {i+1}: {question}") # Save the questions to a text file with open("/home/roomal/Desktop/generated_questions.txt", "w") as f: for i, question in enumerate(questions): f.write(f"Topic {i+1}:\n{question}\n\n") logging.info("Generated questions saved to text file.") 

#################### # LOAD LIBRARIES #################### import json import nltk import re import time import umap import logging from bertopic import BERTopic from bertopic.representation import KeyBERTInspired from langchain import PromptTemplate from langchain.llms import Ollama from nltk.corpus import stopwords from nltk.tokenize import sent_tokenize, word_tokenize from pdfminer.high_level import extract_text from rich.progress import Progress from sklearn.feature_extraction.text import CountVectorizer #################### # CONFIGURE LOGGING #################### logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') ################################################ # ENSURE NECESSARY NLTK RESOURCES ARE DOWNLOADED ################################################ nltk.download('punkt') nltk.download('stopwords') ###################### # PATH TO THE PDF FILE ###################### file_path = "C:\\Users\\sefer\\OneDrive\\Desktop\\PSY-3180\\pdfs\\Book 1\\13 - Reproductive Behavior.pdf" ############################ # EXTRACT TEXT FROM THE PDF ############################ logging.info("Extracting text from PDF...") text = extract_text(file_path) def clean_text(text): """Clean the input text using various preprocessing steps.""" text = text.lower() text = re.sub(r'\s+', ' ', text) # Remove extra whitespace  text = re.sub(r'[^a-z0-9\s.]', '', text) # Remove non-alphanumeric characters except periods  return text def tokenize_text(text): """Tokenize text into individual words.""" tokens = word_tokenize(text) return tokens def remove_stopwords(tokens): """Remove stopwords from the list of tokenized words.""" filtered_tokens = [token for token in tokens if token not in stopwords.words('english')] return filtered_tokens def preprocess_text(text): """Full preprocessing pipeline integrating all the steps.""" cleaned_text = clean_text(text) tokens = tokenize_text(cleaned_text) filtered_tokens = remove_stopwords(tokens) return ' '.join(filtered_tokens) def generate_questions(context, llm, prompt_template): formatted_prompt = prompt_template.format(context=context) response = llm.invoke(formatted_prompt) return response.strip() def main_loop(duration_minutes): end_time = time.time() + duration_minutes * 60 while time.time() < end_time: logging.info("Starting new iteration of text processing and question generation...") # Preprocess the text  processed_text = preprocess_text(text) logging.info("Text preprocessing complete.") # Tokenize the text into sentences  sentences = sent_tokenize(processed_text) logging.info(f"Number of sentences extracted: {len(sentences)}") # Configure UMAP parameters  umap_model = umap.UMAP(n_neighbors=10, n_components=5, min_dist=0.1, metric='cosine') vectorizer_model = CountVectorizer(min_df=1, max_df=0.95) # BERTopic model  logging.info("Fitting BERTopic model...") topic_model = BERTopic( umap_model=umap_model, vectorizer_model=vectorizer_model, representation_model=KeyBERTInspired() ) with Progress() as progress: task = progress.add_task("Fitting BERTopic model...", total=len(sentences)) topics, probs = topic_model.fit_transform(sentences) progress.update(task, advance=len(sentences)) logging.info("BERTopic model fitted successfully.") topic_model.get_topic_info() # Fine-tune topic representations with Ollama  logging.info("Fine-tuning topic representations with Ollama...") topic_representation_model = Ollama(model="llama3:latest", temperature=0.8) topic_model = BERTopic( representation_model=topic_representation_model, umap_model=umap_model, vectorizer_model=vectorizer_model ) with Progress() as progress: task = progress.add_task("Fitting BERTopic with Ollama...", total=len(sentences)) topic_model.fit(sentences) progress.update(task, advance=len(sentences)) logging.info("Topic representations fine-tuned successfully.") # Get representative documents for each topic  docs_by_topic = topic_model.get_representative_docs() # Generate questions using LangChain  llm_for_questions = Ollama(model="llama3:latest", temperature=0.5) # Define a more varied prompt template for generating questions  prompt_template = PromptTemplate( input_variables=['context'], template='''Generate several insightful and varied questions based on the context below: Context: {context} Questions:\n 1. ''' ) # Generate questions for each topic  questions = [] with Progress() as progress: task = progress.add_task("Generating questions...", total=len(docs_by_topic)) for topic, docs in docs_by_topic.items(): context = " ".join(docs) question = generate_questions(context, llm_for_questions, prompt_template) questions.append(question) progress.update(task, advance=1) logging.info("Questions generated successfully.") # Print the generated questions  for i, question in enumerate(questions): print(f"Topic {i+1}: {question}") # Pause for a moment before the next iteration  time.sleep(1) ####################################### # RUN THE MAIN LOOP FOR 30 MINUTES ####################################### if __name__ == '__main__': logging.info("Starting the main loop for 30 minutes...Or whatever.") main_loop(30) logging.info("Main loop completed.")