reduce table size

* use lower-case keys * pre-establish key set as lower-case to avoid repeated .toLoweCase calls

Author: efd6

packages/mimecast/data_stream/siem_logs/elasticsearch/ingest_pipeline/default.yml

@@ -29,113 +29,101 @@ processors:
  lang: painless
  params:
  definite_positive:
- Action: receipt
- Attempt: delivery
- AttNames: process
- CustomerIP: avlog
- CustomName: impersonation-protect
- CustomThreatDictionary: impersonation-protect
- Definition: impersonation-protect
- Delivered: delivery
- Err: delivery
- Error: receipt
- fileName: attachment-protect
- Hits: impersonation-protect
- Hld: process
- InternalName: impersonation-protect
- IPInternalName: process
- IPNewDomain: process
- IPReplyMismatch: process
- IPSimilarDomain: process
- IPThreadDict: process
- Latency: delivery
- MimecastIP: avlog
- MsgSize: process
- NewDomain: impersonation-protect
+ action: receipt
+ attempt: delivery
+ attnames: process
+ customerip: avlog
+ customname: impersonation-protect
+ customthreatdictionary: impersonation-protect
+ definition: impersonation-protect
+ delivered: delivery
+ err: delivery
+ error: receipt
+ filename: attachment-protect
+ hits: impersonation-protect
+ hld: process
+ internalname: impersonation-protect
+ ipinternalname: process
+ ipnewdomain: process
+ ipreplymismatch: process
+ ipsimilardomain: process
+ ipthreaddict: process
+ latency: delivery
+ mimecastip: avlog
+ msgsize: process
+ newdomain: impersonation-protect
  reason: url-protect
- ReceiptAck: delivery
- ReplyMismatch: impersonation-protect
- RcptActType: journal
- ScanResultInfo: internal-email-protect
- SenderDomainInternal: avlog
- SimilarCustomExternalDomain: impersonation-protect
- SimilarInternalDomain: impersonation-protect
- SimilarMimecastExternalDomain: impersonation-protect
- Snt: delivery
- SpamInfo: receipt
- SpamLimit: receipt
- SpamProcessingDetail: receipt
- SpamScore: receipt
- TaggedExternal: impersonation-protect
- TaggedMalicious: impersonation-protect
- ThreatDictionary: impersonation-protect
- UseTls: delivery
+ receiptack: delivery
+ replymismatch: impersonation-protect
+ rcptacttype: journal
+ scanresultinfo: internal-email-protect
+ senderdomaininternal: avlog
+ similarcustomexternaldomain: impersonation-protect
+ similarinternaldomain: impersonation-protect
+ similarmimecastexternaldomain: impersonation-protect
+ snt: delivery
+ spaminfo: receipt
+ spamlimit: receipt
+ spamprocessingdetail: receipt
+ spamscore: receipt
+ taggedexternal: impersonation-protect
+ taggedmalicious: impersonation-protect
+ threatdictionary: impersonation-protect
+ usetls: delivery
  negative:
- aCode: [avlog, url-protect, attachment-protect]
- Act: [delivery, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, journal]
- AttCnt: [receipt, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, journal]
- AttSize: [receipt, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, journal]
- Cphr: [process,avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, journal]
- Dir: [process, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect]
- fileExt: [receipt, process, delivery, spam, internal-email-protect, impersonation-protect, url-protect, journal]
- fileMime: [receipt, process, delivery, spam, internal-email-protect, impersonation-protect, url-protect, journal]
- headerFrom: [process, delivery, avlog, internal-email-protect, impersonation-protect, url-protect, attachment-protect, journal]
- IP: [process, spam, internal-email-protect, url-protect, journal]
+ acode: [avlog, url-protect, attachment-protect]
+ act: [delivery, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, journal]
+ attcnt: [receipt, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, journal]
+ attsize: [receipt, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, journal]
+ cphr: [process,avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, journal]
+ dir: [process, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect]
+ fileext: [receipt, process, delivery, spam, internal-email-protect, impersonation-protect, url-protect, journal]
+ filemime: [receipt, process, delivery, spam, internal-email-protect, impersonation-protect, url-protect, journal]
+ headerfrom: [process, delivery, avlog, internal-email-protect, impersonation-protect, url-protect, attachment-protect, journal]
+ ip: [process, spam, internal-email-protect, url-protect, journal]
  md5: [receipt, process, delivery, spam, internal-email-protect, impersonation-protect, url-protect, journal]
- Rcpt: [process, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect]
+ rcpt: [process, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect]
  recipient: [receipt, process, delivery, journal]
- Recipient: [receipt, process, delivery, journal]
- RejCode: [process, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, journal]
- RejInfo: [process, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, journal]
- RejType: [process, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, journal]
+ rejcode: [process, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, journal]
+ rejinfo: [process, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, journal]
+ rejtype: [process, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, journal]
  route: [receipt, process, journal]
- Route: [receipt, process, journal]
- senderDomain: [receipt, process, delivery, internal-email-protect, impersonation-protect, journal]
- SenderDomain: [receipt, process, delivery, internal-email-protect, impersonation-protect, journal]
+ senderdomain: [receipt, process, delivery, internal-email-protect, impersonation-protect, journal]
  sha1: [receipt, process, delivery, spam, internal-email-protect, impersonation-protect, url-protect, journal]
  sha256: [receipt, process, delivery, spam, internal-email-protect, impersonation-protect, url-protect, journal]
- Size: [receipt, process, delivery, spam, internal-email-protect, impersonation-protect, url-protect, journal]
- sourceIp: [receipt, process, delivery, avlog, internal-email-protect, impersonation-protect, attachment-protect, journal]
- SourceIP: [receipt, process, delivery, avlog, internal-email-protect, impersonation-protect, attachment-protect, journal]
- TlsVer: [process, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, journal]
+ size: [receipt, process, delivery, spam, internal-email-protect, impersonation-protect, url-protect, journal]
+ sourceip: [receipt, process, delivery, avlog, internal-email-protect, impersonation-protect, attachment-protect, journal]
+ tlsver: [process, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, journal]
  url: [receipt, process, delivery, avlog, spam, impersonation-protect, attachment-protect, journal]
- URL: [receipt, process, delivery, avlog, spam, impersonation-protect, attachment-protect, journal]
- urlCategory: [receipt, process, delivery, avlog, spam, impersonation-protect, attachment-protect, journal]
- UrlCategory: [receipt, process, delivery, avlog, spam, impersonation-protect, attachment-protect, journal]
- Virus: [process, delivery, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, journal]
+ urlcategory: [receipt, process, delivery, avlog, spam, impersonation-protect, attachment-protect, journal]
+ virus: [process, delivery, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect, journal]
  positive:
- aCode: [receipt, process, delivery, spam, internal-email-protect, impersonation-protect, journal]
- Act: [receipt, process]
- AttCnt: [process, delivery]
- AttSize: [process, delivery]
- Cphr: [receipt, delivery]
- Dir: [receipt, delivery, journal]
- fileExt: [avlog, attachment-protect]
- fileMime: [avlog, attachment-protect]
- headerFrom: [receipt, spam]
- IP: [receipt, delivery, avlog, impersonation-protect, attachment-protect]
+ acode: [receipt, process, delivery, spam, internal-email-protect, impersonation-protect, journal]
+ act: [receipt, process]
+ attcnt: [process, delivery]
+ attsize: [process, delivery]
+ cphr: [receipt, delivery]
+ dir: [receipt, delivery, journal]
+ fileext: [avlog, attachment-protect]
+ filemime: [avlog, attachment-protect]
+ headerfrom: [receipt, spam]
+ ip: [receipt, delivery, avlog, impersonation-protect, attachment-protect]
  md5: [avlog, attachment-protect]
- Rcpt: [receipt, delivery, journal]
+ rcpt: [receipt, delivery, journal]
  recipient: [avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect]
- Recipient: [avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect]
- RejCode: [receipt, delivery]
- RejInfo: [receipt, delivery]
- RejType: [receipt, delivery]
+ rejcode: [receipt, delivery]
+ rejinfo: [receipt, delivery]
+ rejtype: [receipt, delivery]
  route: [delivery, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect]
- Route: [delivery, avlog, spam, internal-email-protect, impersonation-protect, url-protect, attachment-protect]
- senderDomain: [avlog, spam, url-protect, attachment-protect]
- SenderDomain: [avlog, spam, url-protect, attachment-protect]
+ senderdomain: [avlog, spam, url-protect, attachment-protect]
  sha1: [avlog, attachment-protect]
  sha256: [avlog, attachment-protect]
- Size: [avlog, attachment-protect]
- sourceIp: [spam, url-protect]
- SourceIP: [spam, url-protect]
- TlsVer: [receipt, delivery]
+ size: [avlog, attachment-protect]
+ sourceip: [spam, url-protect]
+ tlsver: [receipt, delivery]
  url: [internal-email-protect, url-protect]
- URL: [internal-email-protect, url-protect]
- urlCategory: [internal-email-protect, url-protect]
- UrlCategory: [internal-email-protect, url-protect]
- Virus: [receipt, avlog]
+ urlcategory: [internal-email-protect, url-protect]
+ virus: [receipt, avlog]
  candidates:
  receipt: 0
  process: 0
@@ -149,7 +137,14 @@ processors:
  attachment-protect: 0
  if: ctx.mimecast instanceof Map
  source: |
+ // Canonicalise keys to lowercase. If this causes issues in future
+ // because case becomes significant, this table space optimisation
+ // will need to be reverted.
+ def keys = new HashSet();
  for (def k: ctx.mimecast.keySet()) {
+ keys.add(k.toLowerCase());
+ } 
+ for (def k: keys) {
  def typ = params.definite_positive.get(k);
  if (typ != null) {
  // We have a definitive known stage.
@@ -158,7 +153,7 @@ processors:
  }
  }
  def score = params.candidates.clone();
- for (def k: ctx.mimecast.keySet()) {
+ for (def k: keys) {
  def typ = params.negative.get(k);
  if (typ == null) {
  continue;
@@ -174,7 +169,7 @@ processors:
  }
  // Find best remaining and list all co-equal winners.
  int max = 0;
- for (def k: ctx.mimecast.keySet()) {
+ for (def k: keys) {
  def typ = params.positive.get(k);
  if (typ == null) {
  continue;