refactoring Translator to remove pandas library dependency

Author: pitchmuc

docs/releases.md

@@ -3,6 +3,9 @@
 This page gathered the changes made between version of the launchpy module.\
 This has been started after the 0.3.0 release.\
 
+## 0.4.3
+* Refactor `Translator` class for removing `pandas` dependency
+
 ## 0.4.2
 * Supporting the Oauth Token V2 authentication
 

docs/translator.md

@@ -9,7 +9,7 @@ Also it is possible to use it not only between 2 properties but between 2 Organi
 ## Starting with translator class
 
 The instantiation of the translator class is very basic as you would just need to call for it.
-What is really required is the base property extensions and rules. The id of the elements from your “mother” property.
+What is really required is the base property extensions and rules. The id of the elements from your "mother" property.
 This is an assumption that I have when I created this facilitator, you would have a blueprint that you will use to copy elements from.
 The translator class provide 2 methods to add base rule and base extension ids:
 
@@ -25,7 +25,7 @@ The translator class provide 2 methods to add base rule and base extension ids:
 
 ## Set Target Rule and Extensions ids
 
-Once you have loaded the base Extension and Rules ids, the method will start to expand the table into multiple columns for matching. Underlying is the use of pandas dataframe to realize that matching.
+Once you have loaded the base Extension and Rules ids, the method will start to expand the table into multiple columns for matching.
 You will need to be consistent with the naming of your target in order for the method to work properly.
 The methods that allow extension of your table are the following:
 

launchpy/__version__.py

@@ -1 +1 @@
-__version__ = "0.4.2-1"
+__version__ = "0.4.3"

launchpy/launchpy.py

@@ -410,56 +410,42 @@ class Translator:
  """
 
  def __init__(self):
- self.rules = pd.DataFrame()
- self.extensions = pd.DataFrame()
+ self.rules = {}
+ self.baseRuleIdName = {}
+ self.extensions = {}
+ self.baseExtensionIdName = {}
 
- def setBaseExtensions(self, extensions: list=None, property_name: str=None)->None:
+ def setBaseExtensions(self, base_property_extensions: list, property_name: str):
  """
  Pass all the extensions from the base property to start building the table. 
  Arguments: 
- extensions_base : REQUIRED : list of all extensions retrieve through getExtensions method
+ base_property_extensions : REQUIRED : list of all extensions retrieve through getExtensions method
  property_name : REQUIRED : name of your base property.
  """
- if extensions is None or type(extensions)!= list:
- raise ValueError("Require a list of extensions to be loaded")
- if property_name is None:
- raise ValueError("Require the main property name")
- df = pd.DataFrame(extensionsInfo(extensions)).T
- df = pd.DataFrame(df['id'])
- df.columns = [property_name]
- self.extensions = df
+ self.baseExtensionIdName = {ext['id'] : ext['attributes']['name'] for ext in base_property_extensions}
+ self.extensions = {ext['attributes']['name']:{property_name:ext['id']} for ext in base_property_extensions}
 
- def extendExtensions(self, extensions: list=None, property_name: str=None)-> None:
+ def extendExtensions(self, new_property_extensions: list, new_prop_name: str)-> None:
  """
  Add the extensions id from a target property.
  Arguments: 
- extensions: REQUIRED : the extension list from your target property. 
- property_name : REQUIRED : target property name. 
+ new_property_extensions: REQUIRED : the extension list from your target property. 
+ new_prop_name : REQUIRED : target property name. 
  """
- if extensions is None or type(extensions)!= list:
- raise ValueError("Require a list of extensions to be loaded")
- if property_name is None:
- raise ValueError("Require the main property name")
- df = pd.DataFrame(extensionsInfo(extensions)).T
- df = pd.DataFrame(df['id'])
- self.extensions[property_name] = df
+ for ext in new_property_extensions:
+ if ext['attributes']['name'] in list(self.extensions.keys()):
+ self.extensions[ext['attributes']['name']][new_prop_name] = ext['id']
  return self.extensions
 
- def setBaseRules(self, rules: list=None, property_name: str=None)->None:
+ def setBaseRules(self, base_property_rules: list, property_name: str):
  """
  Pass all the rules from the base property to start building the table. 
  Arguments: 
- rules : REQUIRED : list of all rules retrieve through getExtensions method
+ base_property : REQUIRED : list of all rules retrieve through getExtensions method
  property_name : REQUIRED : name of your base property.
  """
- if rules is None or type(rules)!= list:
- raise ValueError("Require a list of rules to be loaded")
- if property_name is None:
- raise ValueError("Require the main property name")
- df = pd.DataFrame(rulesInfo(rules)).T
- df = pd.DataFrame(df['id'])
- df.columns = [property_name]
- self.rules = df
+ self.baseRuleIdName = {rule['id']:rule['attributes']['name'] for rule in base_property_rules}
+ self.rules = {rule['attributes']['name']:{property_name:rule['id']} for rule in base_property_rules}
 
  def extendBaseRules(self,ruleName:str=None,ruleId:str=None,property_name: str=None)->None:
  """
@@ -474,20 +460,20 @@ def extendBaseRules(self,ruleName:str=None,ruleId:str=None,property_name: str=No
  raise ValueError("Require a rule name to be loaded")
  if property_name is None:
  raise ValueError("Require the main property name")
- temp_df = pd.DataFrame.from_dict({property_name:{ruleName:ruleId}})
- self.rules = self.rules.append(temp_df)
+ self.baseRuleIdName[ruleId] = ruleName
+ self.rules[ruleName][property_name] = ruleId
 
 
- def extendRules(self, rules: list=None, property_name: str=None):
+ def extendRules(self, new_property_rules: list, new_prop_name: str):
  """
- Add the rule id from a target property.
+ Add the extensions id from a target property.
  Arguments: 
- rules: REQUIRED : the rules list from your target property. 
- property_name : REQUIRED : target property name. 
+ new_property_rules: REQUIRED : the rules list from your target property. 
+ new_prop_name : REQUIRED : target property name. 
  """
- df = pd.DataFrame(rulesInfo(rules)).T
- df = pd.DataFrame(df['id'])
- self.rules[property_name] = df
+ for rule in new_property_rules:
+  if rule['attributes']['name'] in list(self.rules.keys()):
+  self.rules[rule['attributes']['name']][new_prop_name] = rule['id']
  return self.rules
 
  def extendTargetRules(self,ruleName:str=None,ruleId:str=None,property_name: str=None)->None:
@@ -504,10 +490,9 @@ def extendTargetRules(self,ruleName:str=None,ruleId:str=None,property_name: str=
  raise ValueError("Require a rule name to be loaded")
  if property_name is None:
  raise ValueError("Require the main property name")
- temp_df = pd.DataFrame.from_dict({property_name:{ruleName:ruleId}})
- self.rules.update(temp_df)
+ self.rules[ruleName][property_name] = ruleId
 
- def translate(self, target_property: str=None, data_element: dict = None, rule_component: dict = None)->dict:
+ def translate(self, target_property: str, data_element: dict = None, rule_component: dict = None)->dict:
  """
  change the id from the base element to the new property. 
  Pre checked should be done beforehands (updating Extension & Rules elements)
@@ -516,39 +501,32 @@ def translate(self, target_property: str=None, data_element: dict = None, rule_c
  data_element : OPTIONAL : if the elements passed are data elements
  rule_component : OPTIONAL : if the elements passed are rule components
  """
- if target_property is None:
- raise ValueError("Require the target property name")
- if data_element is None and rule_component is None:
- raise AttributeError("Require at least data element or rule component to be passed")
- if self.extensions.empty == True:
+ if len(self.extensions) == 0:
  raise AttributeError(
  "You didn't import the base extensions or the target extensions")
  if data_element is not None:
  new_de = deepcopy(data_element)
  base_id = new_de['extension']['id']
- row = self.extensions[self.extensions.iloc[:, 0]
- == base_id].index.values[0]
- new_value = self.extensions.loc[row, target_property]
- new_de['extension']['id'] = new_value
+ based_ext_name = self.baseExtensionIdName[base_id]
+ new_extension_id = self.extensions[based_ext_name][target_property]
+ new_de['extension']['id'] = new_extension_id
  return new_de
  elif rule_component is not None:
- if self.rules.empty == True:
+ if len(self.rules) == 0:
  raise AttributeError(
  "The rules have not been imported, the rule id needs to be changed")
  new_rc = deepcopy(rule_component)
  base_id = new_rc['extension']['id']
- row = self.extensions[self.extensions.eq(
- base_id).any(1)].index.values[0]
- new_value = self.extensions.loc[row, target_property]
- new_rc['extension']['id'] = new_value
- if self.rules.empty == False:
+ based_ext_name = self.baseExtensionIdName[base_id]
+ new_extension_id = self.extensions[based_ext_name][target_property]
+ new_rc['extension']['id'] = new_extension_id
+ if len(self.rules) > 0:
  new_rc['rule_setting'] = {
  'data': [{
- 'id': self.rules.loc[rule_component['rule_name'], target_property],
+ 'id': self.rules[rule_component['rule_name']][target_property],
  'type':'rules'}
  ]}
- new_rc['rule_id'] = self.rules.loc[rule_component['rule_name'],
- target_property]
+ new_rc['rule_id'] = self.rules[rule_component['rule_name']][target_property]
  else:
  print(
  "You didn't load the rules. Please use setExtensions and setRules, and extendExtensions and extendRules")

launchpy/property.py

@@ -1492,118 +1492,6 @@ def copySettings(data: object)->object:
  return obj
 
 
-class Translator:
- """
- A class to store the translator dataframe for extensions ids. 
- It has multiple methods, you should set the Extensions and the Rules. 
- 1. setBaseExtensions
- 2. extendExtensions
- --> You can use the translate method to translate Data Element settings 
- 3. setBaseRules
- 4. extendRules
- --> You can use the translate method to translate Rule Components settings 
- """
-
- def __init__(self):
- self.rules = pd.DataFrame()
- self.extensions = pd.DataFrame()
-
- def setBaseExtensions(self, base_property_extensions: object, property_name: str):
- """
- Pass all the extensions from the base property to start building the table. 
- Arguments: 
- base_property : REQUIRED : list of all extensions retrieve through getExtensions method
- property_name : REQUIRED : name of your base property.
- """
- df = pd.DataFrame(extensionsInfo(base_property_extensions)).T
- df = pd.DataFrame(df['id'])
- df.columns = [property_name]
- self.extensions = df
-
- def extendExtensions(self, new_property_extensions: object, new_prop_name: str)-> None:
- """
- Add the extensions id from a target property.
- Arguments: 
- new_property_extensions: REQUIRED : the extension list from your target property. 
- new_prop_name : REQUIRED : target property name. 
- """
- df = pd.DataFrame(extensionsInfo(new_property_extensions)).T
- df = pd.DataFrame(df['id'])
- self.extensions[new_prop_name] = df
- return self.extensions
-
- def setBaseRules(self, base_property_rules: object, property_name: str):
- """
- Pass all the rules from the base property to start building the table. 
- Arguments: 
- base_property : REQUIRED : list of all rules retrieve through getExtensions method
- property_name : REQUIRED : name of your base property.
- """
- df = pd.DataFrame(rulesInfo(base_property_rules)).T
- df = pd.DataFrame(df['id'])
- df.columns = [property_name]
- self.rules = df
-
- def extendRules(self, new_property_rules: object, new_prop_name: str):
- """
- Add the extensions id from a target property.
- Arguments: 
- new_property_rules: REQUIRED : the rules list from your target property. 
- new_prop_name : REQUIRED : target property name. 
- """
- df = pd.DataFrame(rulesInfo(new_property_rules)).T
- df = pd.DataFrame(df['id'])
- self.rules[new_prop_name] = df
- return self.rules
-
- def translate(self, target_property: str, data_element: dict = None, rule_component: dict = None)->dict:
- """
- change the id from the base element to the new property. 
- Pre checked should be done beforehands (updating Extension & Rules elements)
- Arguments: 
- target_property : REQUIRED : property that is targeted to translate the element to
- data_element : OPTIONAL : if the elements passed are data elements
- rule_component : OPTIONAL : if the elements passed are rule components
- """
- if self.extensions.empty == True:
- raise AttributeError(
- "You didn't import the base extensions or the target extensions")
- if data_element is not None:
- new_de = deepcopy(data_element)
- base_id = new_de['extension']['id']
- row = self.extensions[self.extensions.iloc[:, 0]
- == base_id].index.values[0]
- new_value = self.extensions.loc[row, target_property]
- new_de['extension']['id'] = new_value
- return new_de
- elif rule_component is not None:
- if self.rules.empty == True:
- print(
- "The rules have not been imported, the rule id needs to be changed")
- new_rc = deepcopy(rule_component)
- base_id = new_rc['extension']['id']
- row = self.extensions[self.extensions.eq(
- base_id).any(1)].index.values[0]
- new_value = self.extensions.loc[row, target_property]
- # print(f"name : {rule_component['rule_name']}")
- # print(f"old_id : {base_id}")
- # print(f"new_id : {new_value}")
- new_rc['extension']['id'] = new_value
- if self.rules.empty == False:
- new_rc['rule_setting'] = {
- 'data': [{
- 'id': self.rules.loc[rule_component['rule_name'], target_property],
- 'type':'rules'}
- ]}
- new_rc['rule_id'] = self.rules.loc[rule_component['rule_name'],
- target_property]
- else:
- print(
- "You didn't load the rules. Please use setExtensions and setRules, and extendExtensions and extendRules")
- del new_rc['rules']
- return new_rc
-
-
 def extractAnalyticsCode(rcSettings: str, save: bool = False, filename: str = None,encoding:str='utf-8')->None:
  """
  Extract the custom code of the rule and save it in a file.

launchpy/synchronizer.py

@@ -159,7 +159,7 @@ def syncComponent(self,componentName:str=None,componentId:str=None,publishedVers
  if len(self.target_configs.get(target,{}).get('inclComponents',[]))==0 or flagAllowList:
  translatedComponent = self.translator.translate(target,data_element=cmp_baseDict['copy'])
  ## if it does not exist
- if cmp_baseDict['name'] not in [de['attributes']['name'] for de in self.targets[target]['dataElements']]:
+ if cmp_baseDict['name'] not in [de.get('attributes',{}).get('name') for de in self.targets[target]['dataElements']]:
  comp = self.targets[target]['api'].createDataElement(
  name=cmp_baseDict['name'],
  descriptor= translatedComponent['descriptor'],