update

.gitignore

@@ -4,3 +4,14 @@ ctmArtifacts.tar
 .DS_Store
 /.pydevproject
 3-infrastructure-as-code-examples/kubernetes-agent-application-pods/Images/runKjobConnectionProfile.png
+/*.vsix
+
+.gitignore
+
+6-ide-integrations/integration-via-marketplace/vscode/node_modules
+6-ide-integrations/integration-via-marketplace/vscode/package-lock.json
+6-ide-integrations/integration-via-marketplace/vscode/.vscodeignore
+6-ide-integrations/integration-via-marketplace/vscode/out/*
+6-ide-integrations/integration-via-marketplace/vscode/*.vsix
+6-ide-integrations/integration-via-marketplace/vscode/src/samples/*
+6-ide-integrations/integration-via-marketplace/vscode/src/samples/ctm.basic.json

1-general-examples/maintenance-window-order-jobs/README.md

@@ -58,7 +58,7 @@ After any unwanted jobs have been cleaned up, deactivate the policy that prevent
 
 **ctm run workloadpolicy::deactivate SAP_STOP**
 
-Now that the maintenanc window has been completed, there amay be jobs that you wish to order manually. Use the script in the next section to accomplish that without having to manually select the desired jobs.
+Now that the maintenance window has been completed, there may be jobs that you wish to order manually. Use the script in the next section to accomplish that without having to manually select the desired jobs.
 
 ## **Powershell Script**
 **OrderManualJobs.ps1**

1-general-examples/powershell-job-manipulation-tool/bhcDJ.ps1

@@ -1,3 +1,8 @@
+# Copyright © BMC Software, Inc. All rights reserved.
+# Your access and use of the following is governed by the terms and conditions set forth in License in the project root.
+# Changes may cause incorrect behavior and will be lost if the code is regenerated.
+
+
 function Select-Environment 
 {
 $envString = ""
@@ -58,7 +63,7 @@ function Select-Environment
 #	but it is not yet available in SaaS so checking for 'saas' in endpoint
 #	as a temporary kludge.
 #--------------------------------------------------------------------------
-if ($endPoint.IndexOf('saas') -gt 0) {
+if (($endPoint.IndexOf('saas') -gt 0) -Or ($endPoint.IndexOf('controlm.com') -gt 0)) {
 Write-Host Assuming SaaS environment based on endpoint
 $subVersion = "20"
 $monthly = "40"
@@ -165,12 +170,14 @@ function Do-One-Job
 {
 $jobId = $jobHash.statuses.jobId[$jobSelect - 1]
 
-$function = Read-Host "Select action: b (Bypass), c (confirm), d (details), k (Kill), l (Log), n (Rerun now), o (Output), r (Rerun) or q (quit)"
+$function = Read-Host "Select action: b (Bypass), c (confirm), d (details), f (free), h (hold, k (Kill), l (Log), n (Rerun now), o (Output), r (Rerun) or q (quit)"
 Switch ($function)
 {
 b {ctm run job::runNow $jobId -e $envName}
 c {ctm run job::confirm $jobId -e $envName}
 d {ctm run job:status::get $jobId -e $envName}
+f {ctm run job::free $jobId -e $envName}
+h {ctm run job::hold $jobId -e $envName}
 k {ctm run job::kill $jobId -e $envName}
 l {ctm run job:log::get $jobId -e $envName}
 n { 
@@ -225,7 +232,7 @@ function Do-Jobs
 $jobCtr++
 Add-Member -InputObject $job -MemberType NoteProperty -Name sq -Value $jobCtr
 }
-$jobHash.statuses | Format-Table -RepeatHeader -Property sq, name, folder, status, jobId, application, subApplication, startTime, endTime, host, cyclic
+$jobHash.statuses | Format-Table -RepeatHeader -Property sq, name, folder, status, jobId, application, subApplication, orderDate, startTime, endTime, host, cyclic
 
 $jobSelect = 0
 

1-general-examples/use-case-azure-smart-building/Production/JSON/SMB_Pipeline.json

@@ -11,8 +11,7 @@
  "Command": "pwsh \"C:\\Production\\scripts\\SetCtmVars.ps1\"",
  "eventsToAdd": {
  "Type": "AddEvents",
- "Events": [{"Event": "SMB_Set_Variable-TO-SMB_Blob_Watch_Incoming"},
- {"Event": "SMB_Set_Variables-TO-SMB_SOR_DataExtracts"}]
+ "Events": [{"Event": "SMB_Set_Variables-TO-SMB_SOR_DataExtracts"}]
  }
  },
 
@@ -23,79 +22,28 @@
  "AI-BlobWatch Container": "smartbuilding",
  "AI-BlobWatch Blob Name": "files/buildingdata.csv",
  "AI-BlobWatch Sleep Interval": "30",
- "ConnectionProfile": "FY21DEMO",
- "eventsToWaitFor": {
- "Type": "WaitForEvents",
- "Events": [{"Event": "SMB_Set_Variable-TO-SMB_Blob_Watch_Incoming"}]
- },
- "eventsToDelete": {
- "Type": "DeleteEvents",
- "Events": [{"Event": "SMB_Set_Variable-TO-SMB_Blob_Watch_Incoming"}]
- },
- "eventsToAdd": {
- "Type": "AddEvents",
- "Events": [{"Event": "SMB_Blob_Watch_Incoming-TO-SMB_ADF_SparkPipeline"}]
- }
+ "ConnectionProfile": "FY21DEMO"
  },
 
  "SMB_ADF_SparkPipeline": {
  "Type": "Job:ApplicationIntegrator:AI Azure Data Factory",
  "AI-Azure Data Factory": "%%AZDF",
  "AI-Resource Group": "%%AZRG",
  "AI-Pipeline": "SmartBuilding_SparkPipeline",
- "ConnectionProfile": "ADF",
- "eventsToWaitFor": {
- "Type": "WaitForEvents",
- "Events": [{"Event": "SMB_Blob_Watch_Incoming-TO-SMB_ADF_SparkPipeline"},
- {"Event": "SMB_SOR_DataExtract-TO-SMB_ADF_SparkPipeline"}]
- },
- "eventsToAdd": {
- "Type": "AddEvents",
- "Events": [{"Event": "SMB_ADF_SparkPipeline-TO-SMB_SMS_Notification"},
- {"Event": "SMB_ADF_SparkPipeline-TO-SMB_Email_Notification"}]
- },
- "eventsToDelete": {
- "Type": "DeleteEvents",
- "Events": [{"Event": "SMB_Blob_Watch_Incoming-TO-SMB_ADF_SparkPipeline"},
- {"Event": "SMB_SOR_DataExtract-TO-SMB_ADF_SparkPipeline"}]
- }
+ "ConnectionProfile": "ADF"
  },
 
  "SMB_SMS_Notification": {
  "Type": "Job:Command",
  "RunAs": "ctmadmin",
- "Command": "python c:\\Production\\scripts\\twilio_send.py -m \"some text\" -n \"+14167225672\" -f \"C:\\Production\\Data\\TwilioCreds.txt\"",
- "eventsToWaitFor": {
- "Type": "WaitForEvents",
- "Events": [{"Event": "SMB_ADF_SparkPipeline-TO-SMB_SMS_Notification"}]
- },
- "eventsToAdd": {
- "Type": "AddEvents",
- "Events": [{"Event": "SMB_SMS_Notification-TO-SMB_Blob_Delete"}]
- },
- "eventsToDelete": {
- "Type": "DeleteEvents",
- "Events": [{"Event": "SMB_ADF_SparkPipeline-TO-SMB_SMS_Notification"}]
- }
+ "Command": "python c:\\Production\\scripts\\twilio_send.py -m \"some text\" -n \"+14167225672\" -f \"C:\\Production\\Data\\TwilioCreds.txt\""
  },
  "SMB_Email_Notification": {
  "Type": "Job:ApplicationIntegrator:AI AzureLogicApps",
  "AI-Resource Group": "%%AZRG",
  "AI-TriggerName": "manual",
  "AI-Workflow": "SendEmail",
- "ConnectionProfile": "AZ_LOGICAPPS",
- "eventsToWaitFor": {
- "Type": "WaitForEvents",
- "Events": [{"Event": "SMB_ADF_SparkPipeline-TO-SMB_Email_Notification"}]
- },
- "eventsToAdd": {
- "Type": "AddEvents",
- "Events": [{"Event": "SMB_Email_Notification-TO-SMB_Blob_Delete"}]
- },
- "eventsToDelete": {
- "Type": "DeleteEvents",
- "Events": [{"Event": "SMB_ADF_SparkPipeline-TO-SMB_Email_Notification"}]
- }
+ "ConnectionProfile": "AZ_LOGICAPPS"
  },
 
 "SMB_Blob_Delete": {
@@ -104,16 +52,15 @@
  "AI-Blob name (Delete)": "files/buildingdata.csv",
  "AI-Public Access": "Off",
  "AI-Container (Delete)": "smartbuilding",
- "ConnectionProfile": "FY21DEMO",
- "eventsToWaitFor": {
- "Type": "WaitForEvents",
- "Events": [{"Event": "SMB_Email_Notification-TO-SMB_Blob_Delete"}, 
- {"Event": "SMB_SMS_Notification-TO-SMB_Blob_Delete"}]
- },
- "eventsToDelete": {
- "Type": "DeleteEvents",
- "Events": [{"Event": "SMB_Email_Notification-TO-SMB_Blob_Delete"},
- {"Event": "SMB_SMS_Notification-TO-SMB_Blob_Delete"}]
+ "ConnectionProfile": "FY21DEMO"
+ },
+
+"SMB_RunPipeline_Service": {
+ "Type": "Job:SLAManagement",
+ "ServiceName": "Smart Building Data Pipeline",
+ "ServicePriority": "1", "RunAs": "DUMMYUSR",
+ "CompleteIn": {
+ "Time": "00:30"
  }
  },
 
@@ -156,6 +103,9 @@
  "Type": "DeleteEvents",
  "Events": [{"Event": "SMB_Set_Variables-TO-SMB_SOR_DataExtracts"}]
  }
- }
+ },
+
+"flow00" : {"Type":"Flow", "Sequence":["SMB_Set_Variables", "SMB_Blob_Watch_Incoming", "SMB_ADF_SparkPipeline", "SMB_SMS_Notification", "SMB_Blob_Delete"]},
+ "flow01" : {"Type":"Flow", "Sequence":["SMB_ADF_SparkPipeline", "SMB_Email_Notification", "SMB_Blob_Delete", "SMB_RunPipeline_Service"]}
  }
 }

1-general-examples/use-case-azure-smart-building/README.md

@@ -12,7 +12,7 @@ This is a view of the application components:
 
 ![Architecture](Images/SmartBuildingArchitecture.png)
 
- - **HDInsight** - the Azure Hadoop offering is dynamically launched based upin demand
+ - **HDInsight** - the Azure Hadoop offering is dynamically launched based upon demand
  - **Control-M Agent** - orchestration worker deployed onto the HDInsight cluster during instantiation
  - **Transfer files** - input IOT data is pushed to the HDInsight cluster after the cluster has initialized
  - **Spark** - Scala Machine Learning algorithms run in Spark
@@ -21,10 +21,10 @@ This is a view of the application components:
  - **Terminate HDInsight** - decommission cluster to manage costs
 
 ### Data
-IOT data from vehicle-mounted sensors are collected by a Telematics provider and dropped into an S3 bucket periodically. Control-M watches the S3 bucket for data arrival. That event triggers the workflow you see in the architecture above.
+IOT data from buildings is collected and dropped into a Blob container periodically. Control-M watches the container for data arrival. That event triggers the workflow you see in the architecture above.
 ### HDInsight Cluster 
-The HDInsight service launches a Hadoop cluster pre-configured with the selected components/versions selected during the instantiation request. A bootstrap mechanism is provided to add custom components. This facility is used to deploy a Control-M agent to manage workflow operations that run on the HDInsight cluster.
+The HDInsight service launches a Hadoop cluster with the selected components/versions as defined in the dynamic Linked HDI service within the Azure Data Factory (ADF) pipeline. The Spark analytics execution is managed by ADF.
 ### Other Processing
 Data movement is performed by Control-M and its Managed File Transfer facility. 
-The ML portion including creation and training of models is [available in this repo](https://github.com/werowe/preventiveMaintenanceLogitReg).
+The ML portion including the analytics is [available in this repo](https://github.com/werowe/preventiveMaintenanceLogitReg).
 Notification to interested parties is performed via text messages using Twilio.

1-general-examples/use-case-gcp-fraud-detection/README.md

@@ -0,0 +1,18 @@
+# **Google Cloud Servicess with Control-M**
+
+This example illustrates usage of Control-M to orchestrate a data pipeline in a Google Cloud PLatform environment. 
+
+## Details
+The components include data arriving in Google Storage, processed with Google Dataflow and inserted into BiqQuery.
+
+### **Artifacts**
+A brief description of the contents:
+
+ - **cp_DFLOW-sample.json** - a connection profile for Google Dataflow.
+ - **cp_jobgcp.json** - a connection profile for Google Storage, used by Control-M's Managed File Transfer.
+ - **cp_jog-BQ.json** - connection profile for BgQuery.
+ - **define-sless-queues.sh** - connection profile for Google Dataflow using a default service account of the virtual machine on which the Control-M job executes
+ - **driver-BigQuery.json** - JDBC driver provided by Google, for SQL access to BigQuery.
+ - **jog-mc-gcp-fraud.json** - the definition for the tasks that that make up this orchestration. 
+
+ For any questions, please contact the author at joe_goldberg@bmc.com.

1-general-examples/use-case-gcp-fraud-detection/cp-jog-dflow-midentity.json

@@ -0,0 +1,10 @@
+{
+ "JOG-DFLOW-MIDENTITY": {
+ "Type": "ConnectionProfile:ApplicationIntegrator:AI Google DataFlow",
+ "AI-Identity Type": "os_user",
+ "AI-DataFlow URL": "https://dataflow.googleapis.com",
+ "AI-Service Account Key (JSON Format)": "{ }",
+ "Description": "",
+ "Centralized": true
+ }
+}

1-general-examples/use-case-gcp-fraud-detection/cp_DFLOW-sample.json

@@ -0,0 +1,10 @@
+{
+ "DFLOW-631873438236": {
+ "Type": "ConnectionProfile:ApplicationIntegrator:AI Google DataFlow",
+ "AI-Identity Type": "service_account",
+ "AI-DataFlow URL": "https://dataflow.googleapis.com",
+ "AI-Service Account Key (JSON Format)": "<service account key in json format goes here>",
+ "TargetAgent": "ip-172-31-16-53",
+ "TargetCTM": "smprod"
+ }
+}

1-general-examples/use-case-gcp-fraud-detection/cp_jobgcp.json

@@ -0,0 +1,16 @@
+{
+ "joggcp": {
+ "Type": "ConnectionProfile:FileTransfer:S3:Compatible",
+ "AdditionalParameters": [
+ {
+ "Name": "s3.set.api.version",
+ "Value": "1"
+ }
+ ],
+ "SecretAccessKey": "<AWS Secret Access Key goes here>",
+ "RestEndPoint": "storage.googleapis.com",
+ "AccessKey": "<AWS access key id goes here>",
+ "Description": "Google Cloud Storage Interoperability credentials",
+ "Centralized": true
+ }
+}