Merge pull request #35 from PayloadSecurity/v2

Refactor application and add broad support for API V2

Author: Jan Miller

.gitignore

@@ -5,3 +5,5 @@ api_classes/__pycache__
 cli_classes/__pycache__/*.pyc
 /output
 *.pyc
+/cli.log
+/cache

README.md

@@ -13,6 +13,28 @@ The Falcon Sandbox Python API Connector (e.g. for https://www.hybrid-analysis.co
 > Using Debian/Ubuntu OS, this can be done by calling `sudo apt-get install python3-pip`. It will then be available via `pip3`
 > Using Windows, this can be done automatically when installing `python` (proper checkbox on the installer has to be checked). It should be available via `pip` 
 
+Versions
+---
+
+### V2
+
+This version has broad support for all capabilities of VxWebService APIv2 and much more. New features include:
+
+- support for APIv2
+- improved application performance
+- unified and simplified CLI schema
+- bulk quick scan and sandbox submissions
+- improved file handling
+- test coverage
+
+Example: `python3 vxapi.py scan_file C:\file-repo all`
+
+![Alt text](/img/scan_example.png?raw=true "Falcon Sandbox API CLI Example Bulk Quick Scan")
+
+### V1
+
+The legacy app utilizing the APIv1 is not supported anymore. For backward compatibility, it is still available in the `v1` branch.
+
 Usage
 ---
 
@@ -22,12 +44,11 @@ Copy the `config_tpl.py` and name it `config.py`.
 
 The configuration file specifies a triplet of api key/secret and server:
 
-- api_key
-- api_secret
-- server - full url of the WebService e.g. `https://www.hybrid-analysis.com`
+- api_key (should be compatible with API v2 - should contains at least 60 chars)
+- server - full url of the WebService instance e.g. `https://www.hybrid-analysis.com`
 
 Please fill them with the appropriate data. You can generate a public (restricted) API key by following these instructions:
-https://www.hybrid-analysis.com/apikeys/info
+https://www.hybrid-analysis.com/knowledge-base/issuing-self-signed-api-key
 
 If you have the full version of Falcon Sandbox, create any kind of API key in the admin area:
 https://www.hybrid-analysis.com/apikeys
@@ -80,6 +101,13 @@ After choosing the `action_name`
 > To ensure that the program will work correctly, please use `python3`.
 > In Windows after having installed `python`, please add the parent folder to `PATH` environment variable. Now use `python` to callout the script.
 
+### FAQ
+
+##### My API Key authorization level was updated, but VxApi is still showing the old value.
+
+VxApi is caching key data response. To get the fresh one, please remove `cache` directory content and try to use application once again.
+
+
 ### License
 
 Licensed GNU GENERAL PUBLIC LICENSE, Version 3, 29 June 2007

_version.py

@@ -1 +1 @@
-__version__ = '1.2.2'
+__version__ = '2.0.0'

api_classes/api_caller.py renamed to api/callers/api_caller.py

@@ -2,7 +2,7 @@
 from exceptions import ResponseObjectNotExistError
 from exceptions import UrlBuildError
 from exceptions import JsonParseError
-from requests.auth import HTTPBasicAuth
+from exceptions import ConfigError
 import json
 
 
@@ -20,7 +20,6 @@ class ApiCaller:
  CONST_API_AUTH_LEVEL_SUPER = 1000
 
  api_key = ''
- api_secret = ''
 
  server = ''
  endpoint_url = ''
@@ -34,16 +33,14 @@ class ApiCaller:
  api_unexpected_error_msg = 'Unexpected error has occurred (HTTP code: {}). Please try again later or contact with the support'
  api_unexpected_error_404_msg = 'Unexpected error has occurred (HTTP code: {}). This error is mostly occurring when called webservice is outdated and so does not support current action. If you believe it is an error, please contact with the support'
  api_success_msg = 'Your request was successfully processed by Falcon Sandbox'
- api_expected_error_msg = 'API error has occurred. HTTP code: {}, API error code: {}, message: \'{}\''
- response_msg_success_nature = False
+ api_expected_error_msg = 'API error has occurred. HTTP code: {}, message: \'{}\''
 
- api_response = None
+ api_response = None # TODO - rebuild the way how we set those values to use abstract methods instead
  api_expected_data_type = CONST_EXPECTED_DATA_TYPE_JSON
  api_response_json = {}
 
- def __init__(self, api_key: str, api_secret: str, server: str):
+ def __init__(self, api_key: str, server: str):
  self.api_key = api_key
- self.api_secret = api_secret
  self.server = server
  self.check_class_options()
 
@@ -62,11 +59,16 @@ def check_class_options(self):
  raise OptionNotDeclaredError('Value for \'{}\' should be declared in class \'{}\'.'.format(requested_field, self.__class__.__name__))
 
  def call(self, request_handler, headers={'User-agent': 'VxApi Connector'}):
- if ':' in self.endpoint_url:
+ if '$' in self.endpoint_url:
  raise UrlBuildError('Can\'t call API endpoint with url \'{}\', when some placeholders are still not filled.'.format(self.endpoint_url))
 
  caller_function = getattr(request_handler, self.request_method_name)
- self.api_response = caller_function(self.server + self.endpoint_url, data=self.data, params=self.params, files=self.files, headers=headers, verify=False, auth=HTTPBasicAuth(self.api_key, self.api_secret))
+ headers['api-key'] = self.api_key
+
+ self.api_response = caller_function(self.get_full_endpoint_url(), data=self.data, params=self.params, files=self.files, headers=headers, verify=False, allow_redirects=False)
+ if self.if_request_redirect():
+ raise ConfigError('Got redirect while trying to reach server URL. Please try to update it and pass the same URL base which is visible in the browser URL bar. '
+ 'For example: it should be \'https://www.hybrid-analysis.com\', instead of \'http://www.hybrid-analysis.com\' or \'https://hybrid-analysis.com\'')
  self.api_result_msg = self.prepare_response_msg()
 
  def attach_data(self, options):
@@ -80,49 +82,30 @@ def attach_params(self, params):
  def attach_files(self, files):
  self.files = files
 
+ def if_request_success(self):
+ return str(self.api_response.status_code).startswith('2') # 20x status code
+
+ def if_request_redirect(self):
+ return str(self.api_response.status_code).startswith('3') # 30x status code
+
  def prepare_response_msg(self) -> str:
  if self.api_response is None:
  raise ResponseObjectNotExistError('It\'s not possible to get response message since API was not called.')
 
- '''
- Unfortunately some instances can return valid json with text/html content type. 
- So we're assuming that that case it's not 'not expected error' and in other steps,
- we will try to get proper error message from that json.
- '''
- if self.api_response.headers['Content-Type'].startswith('text/html') and self.api_response.status_code != 200:
- if self.api_response.status_code == 404:
- self.api_result_msg = self.api_unexpected_error_404_msg.format(self.api_response.status_code)
- else:
- self.api_result_msg = self.api_unexpected_error_msg.format(self.api_response.status_code)
+ if self.if_request_success() is True:
+ if self.api_expected_data_type == self.CONST_EXPECTED_DATA_TYPE_JSON:
+ self.api_response_json = self.get_response_json()
+
+ self.api_result_msg = self.api_success_msg
  else:
- if self.api_response.status_code == 200:
- if self.api_expected_data_type == self.CONST_EXPECTED_DATA_TYPE_JSON:
- self.api_response_json = self.get_response_json()
- if 'response_code' in self.api_response_json: # Unfortunately not all of endpoints return the unified json.
- if self.api_response_json['response_code'] == 0:
- self.api_result_msg = self.api_success_msg
- self.response_msg_success_nature = True
- else:
- self.api_result_msg = self.api_expected_error_msg.format(self.api_response.status_code, self.api_response_json['response_code'], self.api_response_json['response']['error'])
- else:
- self.api_result_msg = self.api_success_msg
- self.response_msg_success_nature = True
- else: # Few endpoints can return files
- # Endpoint which is returning file, can also return json file. Then we need to find if we get error msg or that json file.
- if 'response_code' in self.get_response_json():
- self.api_result_msg = self.api_expected_error_msg.format(self.api_response.status_code, self.api_response_json['response_code'], self.api_response_json['response']['error'])
- else:
- self.api_result_msg = self.api_success_msg
- self.response_msg_success_nature = True
+ if self.api_response.headers['Content-Type'] == 'application/json':
+ self.api_response_json = self.api_response.json()
+ self.api_result_msg = self.api_expected_error_msg.format(self.api_response.status_code, self.api_response_json['message'])
  else:
- if self.api_expected_data_type == self.CONST_EXPECTED_DATA_TYPE_JSON and bool(self.api_response.json()) is True: # Sometimes response can has status code different than 200 and store json with error msg
- self.api_response_json = self.api_response.json()
- self.api_result_msg = self.api_expected_error_msg.format(self.api_response.status_code, self.api_response_json['response_code'], self.api_response_json['response']['error'])
+ if self.api_response.status_code == 404:
+ self.api_result_msg = self.api_unexpected_error_404_msg.format(self.api_response.status_code)
  else:
- if self.api_response.status_code == 404:
- self.api_result_msg = self.api_unexpected_error_404_msg.format(self.api_response.status_code)
- else:
- self.api_result_msg = self.api_unexpected_error_msg.format(self.api_response.status_code)
+ self.api_result_msg = self.api_unexpected_error_msg.format(self.api_response.status_code)
 
  return self.api_result_msg
 
@@ -132,12 +115,6 @@ def get_api_response(self):
 
  return self.api_response
 
- def get_response_msg_success_nature(self):
- if self.api_response is None:
- raise ResponseObjectNotExistError('It\'s not possible to define api msg nature since API was not called.')
-
- return self.response_msg_success_nature
-
  def get_response_status_code(self):
  if self.api_response is None:
  raise ResponseObjectNotExistError('It\'s not possible to get response code since API was not called.')
@@ -179,14 +156,20 @@ def get_response_json(self):
 
  return self.api_response_json
 
+ def get_headers(self):
+ if self.api_response is None:
+ raise ResponseObjectNotExistError('It\'s not possible to get response headers since API was not called.')
+
+ return self.api_response.headers
+
  def build_url(self, params):
- if ':' in self.endpoint_url:
+ if '$' in self.endpoint_url:
  url_data = params
  url_data_copy = url_data.copy()
  for key, value in url_data.items():
- searched_key = ':' + key
+ searched_key = '$' + key
  if searched_key in self.endpoint_url:
- self.endpoint_url = self.endpoint_url.replace(':' + key, value)
+ self.endpoint_url = self.endpoint_url.replace('$' + key, str(value))
  del url_data_copy[key] # Working on copy, since it's not possible to manipulate dict size, during iteration
 
  if self.request_method_name == self.CONST_REQUEST_METHOD_GET:
@@ -195,4 +178,4 @@ def build_url(self, params):
  self.data = url_data_copy
 
  def get_full_endpoint_url(self):
- return self.server + self.endpoint_url
+ return '{}/api/v2{}'.format(self.server, self.endpoint_url)

api/callers/feed/__init__.py

@@ -0,0 +1,2 @@
+from api.callers.feed.api_feed import ApiFeed
+from api.callers.feed.api_feed_latest import ApiFeedLatest

api/callers/feed/api_feed.py

@@ -0,0 +1,7 @@
+from api.callers.api_caller import ApiCaller
+
+
+class ApiFeed(ApiCaller):
+ endpoint_url = '/feed/$days/days'
+ endpoint_auth_level = ApiCaller.CONST_API_AUTH_LEVEL_ELEVATED
+ request_method_name = ApiCaller.CONST_REQUEST_METHOD_GET

api_classes/api_api_query_limits.py renamed to api/callers/feed/api_feed_latest.py

@@ -1,7 +1,7 @@
-from api_classes.api_caller import ApiCaller
+from api.callers.api_caller import ApiCaller
 
 
-class ApiApiQueryLimits(ApiCaller):
- endpoint_url = '/api/api-limits'
- request_method_name = ApiCaller.CONST_REQUEST_METHOD_GET
+class ApiFeedLatest(ApiCaller):
+ endpoint_url = '/feed/latest'
  endpoint_auth_level = ApiCaller.CONST_API_AUTH_LEVEL_RESTRICTED
+ request_method_name = ApiCaller.CONST_REQUEST_METHOD_GET

api/callers/key/__init__.py

@@ -0,0 +1,2 @@
+from api.callers.key.api_key_create import ApiKeyCreate
+from api.callers.key.api_key_current import ApiKeyCurrent

api_classes/api_nssf_list.py renamed to api/callers/key/api_key_create.py

@@ -1,7 +1,7 @@
-from api_classes.api_caller import ApiCaller
+from api.callers.api_caller import ApiCaller
 
 
-class ApiNssfList(ApiCaller):
- endpoint_url = '/api/nssf/list'
- request_method_name = ApiCaller.CONST_REQUEST_METHOD_POST
+class ApiKeyCreate(ApiCaller):
+ endpoint_url = '/key/create'
  endpoint_auth_level = ApiCaller.CONST_API_AUTH_LEVEL_ELEVATED
+ request_method_name = ApiCaller.CONST_REQUEST_METHOD_POST

api_classes/api_environments.py renamed to api/callers/key/api_key_current.py

@@ -1,7 +1,7 @@
-from api_classes.api_caller import ApiCaller
+from api.callers.api_caller import ApiCaller
 
 
-class ApiEnvironments(ApiCaller):
- endpoint_url = '/api/environments'
- request_method_name = ApiCaller.CONST_REQUEST_METHOD_GET
+class ApiKeyCurrent(ApiCaller):
+ endpoint_url = '/key/current'
  endpoint_auth_level = ApiCaller.CONST_API_AUTH_LEVEL_RESTRICTED
+ request_method_name = ApiCaller.CONST_REQUEST_METHOD_GET