Added the ability to delete a datasource

app/api/data_source.py

@@ -70,11 +70,11 @@ async def delete_data_source(data_source_id: int):
 
 
 @router.post("")
-async def add_integration(dto: AddDataSource, background_tasks: BackgroundTasks):
+async def add_integration(dto: AddDataSource, background_tasks: BackgroundTasks) -> int:
  data_source = DataSourceContext.create_data_source(name=dto.name, config=dto.config)
 
  # in main.py we have a background task that runs every 5 minutes and indexes the data source
  # but here we want to index the data source immediately
  background_tasks.add_task(data_source.index)
 
- return {"success": "Data source added successfully"}
+ return data_source.get_id()

app/data_source/api/base_data_source.py

@@ -83,6 +83,9 @@ def __init__(self, config: Dict, data_source_id: int, last_index_time: datetime
  self._last_index_time = last_index_time
  self._last_task_time = None
 
+ def get_id(self):
+ return self._data_source_id
+
  def _save_index_time_in_db(self) -> None:
  """
  Sets the index time in the database, to be now

app/indexing/bm25_index.py

@@ -1,16 +1,14 @@
 import os
 import pickle
+from typing import List
+
 import nltk
 import numpy as np
 from rank_bm25 import BM25Okapi
-from sqlalchemy import Connection
-from typing import List
 
 from db_engine import Session
-from schemas import Paragraph
 from paths import BM25_INDEX_PATH
-
-
+from schemas import Paragraph
 
 
 def _add_metadata_for_indexing(paragraph: Paragraph) -> str:
@@ -60,13 +58,13 @@ def _update(self, session):
  self.index = BM25Okapi(corpus)
  self.id_map = id_map
 
- def update(self, session = None):
+ def update(self, session=None):
  if session is None:
  with Session() as session:
  self._update(session)
  else:
  self._update(session)
- 
+
  self._save()
 
  def search(self, query: str, top_k: int) -> List[int]:
@@ -87,4 +85,4 @@ def clear(self):
 
  def _save(self):
  with open(BM25_INDEX_PATH, 'wb') as f:
- pickle.dump(self, f)
+ pickle.dump(self, f)

ui/src/App.tsx

@@ -22,15 +22,15 @@ import 'react-toastify/dist/ReactToastify.css';
 import { ClipLoader } from "react-spinners";
 import { FiSettings } from "react-icons/fi";
 import {AiFillWarning} from "react-icons/ai";
-import { ConnectedDataSourceType, DataSourceType } from "./data-source";
+import { ConnectedDataSource, DataSourceType } from "./data-source";
 
 export interface AppState {
  query: string
  results: SearchResultDetails[]
  searchDuration: number
  dataSourceTypes: DataSourceType[]
  dataSourceTypesDict: { [key: string]: DataSourceType }
- connectedDataSources: string[]
+ connectedDataSources: ConnectedDataSource[]
  isLoading: boolean
  isNoResults: boolean
  isModalOpen: boolean
@@ -137,9 +137,8 @@ export default class App extends React.Component <{}, AppState>{
 
  async listConnectedDataSources() {
  try {
- const response = await api.get<ConnectedDataSourceType[]>('/data-sources/connected');
- let nameList = response.data.map((dataSource) => dataSource.name);
- this.setState({ connectedDataSources: nameList })
+ const response = await api.get<ConnectedDataSource[]>('/data-sources/connected');
+ this.setState({ connectedDataSources: response.data });
  } catch (error) {
  }
  }
@@ -168,7 +167,7 @@ export default class App extends React.Component <{}, AppState>{
  }
 
  this.setState({isServerDown: false, docsLeftToIndex: res.data.docs_left_to_index,
-  docsInIndexing: res.data.docs_in_indexing, isPreparingIndexing: isPreparingIndexing});
+ docsInIndexing: res.data.docs_in_indexing, isPreparingIndexing: isPreparingIndexing});
 
  let timeToSleep = isPreparingIndexing ? 1000 : successSleepSeconds * 1000;
  setTimeout(() => this.fetchStatsusForever(), timeToSleep);
@@ -184,7 +183,7 @@ export default class App extends React.Component <{}, AppState>{
  }) 
  }
 
- shouldShowIndexingStatus() {
+ inIndexing() {
  return this.state.isPreparingIndexing || this.state.docsInIndexing > 0 || this.state.docsLeftToIndex > 0;
  }
 
@@ -261,8 +260,8 @@ export default class App extends React.Component <{}, AppState>{
  toast.success("Code accepted. Welcome!", {autoClose: 3000});
  }
 
- dataSourcesAdded = (dataSourceType: string) => {
- this.setState({isPreparingIndexing: true, connectedDataSources: [...this.state.connectedDataSources, dataSourceType]});
+ dataSourcesAdded = (newlyConnected: ConnectedDataSource) => {
+ this.setState({isPreparingIndexing: true, connectedDataSources: [...this.state.connectedDataSources, newlyConnected]});
  // if had no data from server, show toast after 30 seconds
  setTimeout(() => {
  if (this.state.isPreparingIndexing) {
@@ -272,6 +271,10 @@ export default class App extends React.Component <{}, AppState>{
  }, 1000 * 120);
  }
 
+ dataSourceRemoved = (removed: ConnectedDataSource) => {
+ this.setState({connectedDataSources: this.state.connectedDataSources.filter((ds) => ds.id !== removed.id)});
+ }
+
  render() {
  return (
  <div>
@@ -280,7 +283,7 @@ export default class App extends React.Component <{}, AppState>{
  className="absolute right-0 z-30 float-right mr-6 mt-6 text-[42px] hover:cursor-pointer hover:rotate-90 transition-all duration-300 hover:drop-shadow-2xl">
  </FiSettings>
  {
- this.shouldShowIndexingStatus() &&
+ this.inIndexing() &&
  <div className="absolute mx-auto left-0 right-0 w-fit z-20 top-6">
  <div className="text-xs bg-[#191919] border-[#4F4F4F] border-[.8px] rounded-full inline-block px-3 py-1">
  <div className="text-[#E4E4E4] font-medium font-inter text-sm flex flex-row justify-center items-center">
@@ -348,7 +351,8 @@ export default class App extends React.Component <{}, AppState>{
  contentLabel="Example Modal"
  style={customStyles}>
  <DataSourcePanel onClose={this.closeModal} connectedDataSources={this.state.connectedDataSources}
- onAdded={this.dataSourcesAdded} dataSourceTypesDict={this.state.dataSourceTypesDict}/>
+ inIndexing={this.inIndexing()}
+ onAdded={this.dataSourcesAdded} dataSourceTypesDict={this.state.dataSourceTypesDict} onRemoved={this.dataSourceRemoved} />
  </Modal>
 
  {/* front search page*/}