Definitions for @turbot/azure-datafactory

dataFactory

{
"allOf": [
{
"$ref": "turbot#/definitions/service"
},
{
"type": "object",
"properties": {
"name": {
"const": "Data Factory"
},
"namespace": {
"const": "Microsoft.DataFactory"
},
"registrationState": {
"type": "string"
},
"turbot": {
"type": "object",
"properties": {
"akas": {
"type": "array",
"items": {
"$ref": "#/definitions/dataFactoryAka"
}
},
"title": {
"const": "Data Factory"
},
"custom": {
"type": "object",
"properties": {
"azure": {
"$ref": "azure#/definitions/azureMetadata"
}
}
}
}
}
},
"required": [
"name",
"turbot"
],
"additionalProperties": true,
"tests": [
{
"description": "All details provided",
"input": {
"name": "Data Factory",
"turbot": {
"akas": [
"azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/providers/Microsoft.DataFactory/services/dataFactory"
],
"title": "Data Factory",
"custom": {
"azure": {
"subscriptionId": "9e3548cf-17e2-4751-b87e-b72bdd2c77f7"
}
}
}
}
},
{
"description": "Invalid - Name is not as declared",
"input": {
"name": "DataFactory",
"turbot": {
"akas": [
"azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/providers/Microsoft.DataFactory/services/dataFactory"
],
"title": "Data Factory",
"custom": {
"azure": {
"subscriptionId": "9e3548cf-17e2-4751-b87e-b72bdd2c77f7"
}
}
}
},
"expected": false
},
{
"description": "Invalid - Service name missing",
"input": {
"turbot": null,
"akas": [
"azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/providers/Microsoft.DataFactory/services/dataFactory"
],
"title": "Data Factory",
"custom": {
"azure": {
"subscriptionId": "9e3548cf-17e2-4751-b87e-b72bdd2c77f7"
}
}
},
"expected": false
}
]
}
],
".turbot": {
"uri": "tmod:@turbot/azure-datafactory#/definitions/dataFactory",
"modUri": "tmod:@turbot/azure-datafactory"
}
}

dataFactoryAka

{
"type": "string",
"pattern": "^azure:///subscriptions/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/providers/Microsoft.DataFactory/services/dataFactory$",
"tests": [
{
"description": "base",
"input": "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/providers/Microsoft.DataFactory/services/dataFactory"
},
{
"description": "invalid service name",
"input": "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/providers/Microsoft.DataFactoryr/services/dataFactory",
"expected": false
},
{
"description": "invalid subscription id",
"input": "azure:///subscriptions/9e3548cf-17e2-4751-12345-b72bdd2c77f7/providers/Microsoft.DataFactory/services/dataFactory",
"expected": false
}
],
".turbot": {
"uri": "tmod:@turbot/azure-datafactory#/definitions/dataFactoryAka",
"modUri": "tmod:@turbot/azure-datafactory"
}
}

dataset

{
"type": "object",
"properties": {
"name": {
"$ref": "#/definitions/datasetName"
},
"id": {
"$ref": "#/definitions/datasetId"
},
"etag": {
"type": "string"
},
"properties": {
"type": "object"
},
"turbot": {
"type": "object",
"properties": {
"title": {
"$ref": "#/definitions/factoryName"
},
"akas": {
"type": "array",
"items": {
"$ref": "#/definitions/datasetAka"
}
},
"custom": {
"type": "object",
"properties": {
"createTimestamp": {
"$ref": "turbot#/definitions/isoTimestamp"
},
"azure": {
"$ref": "azure#/definitions/azureMetadata"
},
"factoryName": {
"$ref": "#/definitions/factoryName"
}
}
}
}
}
},
"additionalProperties": true,
"tests": [
{
"description": "Valid - All properties given",
"input": {
"name": "test0001",
"id": "/subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test90/datasets/testing",
"etag": "00000a61-0000-0100-0000-5d5155ec0000",
"properties": {
"description": "testing",
"linkedServiceName": {
"type": "LinkedServiceReference",
"referenceName": "AzureBlobStorage1"
},
"parameters": {
"nkml": {
"type": "string",
"defaultValue": "value"
}
},
"annotations": [],
"type": "DelimitedText",
"typeProperties": {
"location": {
"type": "AzureBlobStorageLocation"
},
"columnDelimiter": ",",
"escapeChar": "\\",
"quoteChar": "\""
},
"schema": []
},
"turbot": {
"title": "test0001",
"akas": [
"azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test90/datasets/testing"
],
"custom": {
"azure": {
"subscriptionId": "9e3548cf-17e2-4751-b87e-b72bdd2c77f7",
"regionName": "centralindia",
"resourceGroupName": "test"
}
}
}
}
}
],
".turbot": {
"uri": "tmod:@turbot/azure-datafactory#/definitions/dataset",
"modUri": "tmod:@turbot/azure-datafactory"
}
}

datasetAka

{
"type": "string",
"pattern": "^azure:///subscriptions/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/resourceGroups/[A-Za-z0-9_.-]{0,89}[A-Za-z0-9_-]/providers/Microsoft.DataFactory/factories/([a-z0-9]|[a-z0-9][a-z0-9-]{0,61}[a-z0-9])/datasets/[a-zA-Z0-9][a-zA-Z0-9_]+$",
"tests": [
{
"description": "base",
"input": "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012/datasets/test09"
},
{
"description": "invalid service name",
"input": "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactorys/factoriesr/test/datasets/test09",
"expected": false
}
],
".turbot": {
"uri": "tmod:@turbot/azure-datafactory#/definitions/datasetAka",
"modUri": "tmod:@turbot/azure-datafactory"
}
}

datasetId

{
"type": "string",
"pattern": "^/subscriptions/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/resourceGroups/[A-Za-z0-9_.-]{0,89}[A-Za-z0-9_-]/providers/Microsoft.DataFactory/factories/([a-z0-9]|[a-z0-9][a-z0-9-]{0,61}[a-z0-9])/datasets/[a-zA-Z0-9][a-zA-Z0-9_]+$",
"tests": [
{
"description": "base",
"input": "/subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012/datasets/test09"
},
{
"description": "invalid service name",
"input": "//subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test/datasets/test09",
"expected": false
}
],
".turbot": {
"uri": "tmod:@turbot/azure-datafactory#/definitions/datasetId",
"modUri": "tmod:@turbot/azure-datafactory"
}
}

datasetName

{
"type": "string",
"pattern": "^[a-zA-Z0-9][a-zA-Z0-9_]+$",
"tests": [
{
"input": "ab"
},
{
"input": "a_a"
},
{
"description": "invalid - too short",
"input": "",
"expected": false
},
{
"description": "invalid - no spaces",
"input": "Test me please",
"expected": false
}
],
".turbot": {
"uri": "tmod:@turbot/azure-datafactory#/definitions/datasetName",
"modUri": "tmod:@turbot/azure-datafactory"
}
}

factory

{
"type": "object",
"properties": {
"name": {
"$ref": "#/definitions/factoryName"
},
"id": {
"$ref": "#/definitions/factoryId"
},
"tags": {
"$ref": "azure#/definitions/tags"
},
"location": {
"type": "string"
},
"eTag": {
"type": "string"
},
"identity": {
"type": "object"
},
"provisioningState": {
"type": "string"
},
"createTime": {
"type": "string"
},
"version": {
"type": "string"
},
"turbot": {
"type": "object",
"properties": {
"title": {
"$ref": "#/definitions/factoryName"
},
"tags": {
"$ref": "azure#/definitions/tags"
},
"akas": {
"type": "array",
"items": {
"$ref": "#/definitions/factoryAka"
}
},
"custom": {
"type": "object",
"properties": {
"createTimestamp": {
"$ref": "turbot#/definitions/isoTimestamp"
},
"azure": {
"$ref": "azure#/definitions/azureMetadata"
}
}
}
}
}
},
"tests": [
{
"description": "Valid - All properties given",
"input": {
"name": "test0001",
"id": "/subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012",
"type": "Microsoft.DataFactory/factories",
"location": "EastUS",
"tags": {},
"eTag": "\"0c008ca0-0000-0100-0000-5d4c2a0e0000\"",
"identity": {
"type": "SystemAssigned",
"principalId": "cb216664-666e-4b1a-ab42-5f12d8c13862",
"tenantId": "0fae5fbb-c4cf-4d04-9e13-a1260e9fd8f8"
},
"provisioningState": "Succeeded",
"createTime": "2019-08-08T13:56:30.026Z",
"version": "2018-06-01",
"turbot": {
"title": "test0001",
"akas": [
"azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012"
],
"custom": {
"azure": {
"subscriptionId": "9e3548cf-17e2-4751-b87e-b72bdd2c77f7",
"regionName": "centralindia",
"resourceGroupName": "test"
}
}
}
}
},
{
"description": "Invalid - name missing",
"input": {
"turbot": {
"title": "test0001",
"akas": [
"azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012"
],
"custom": {
"azure": {
"subscriptionId": "9e3548cf-17e2-4751-b87e-b72bdd2c77f7",
"regionName": "centralindia",
"resourceGroupName": "test"
}
}
}
}
},
{
"description": "Invalid - servername missing",
"input": {
"id": "/subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012",
"turbot": {
"title": "test0001",
"akas": [
"azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012"
],
"custom": {
"azure": {
"subscriptionId": "9e3548cf-17e2-4751-b87e-b72bdd2c77f7",
"regionName": "centralindia",
"resourceGroupName": "test"
}
}
}
}
}
],
".turbot": {
"uri": "tmod:@turbot/azure-datafactory#/definitions/factory",
"modUri": "tmod:@turbot/azure-datafactory"
}
}

factoryAka

{
"type": "string",
"pattern": "^azure:///subscriptions/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/resourceGroups/[A-Za-z0-9_.-]{0,89}[A-Za-z0-9_-]/providers/Microsoft.DataFactory/factories/([a-z0-9]|[a-z0-9][a-z0-9-]{0,61}[a-z0-9])$",
"tests": [
{
"description": "base",
"input": "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012"
},
{
"description": "invalid service name",
"input": "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactorys/factoriesr/test",
"expected": false
}
],
".turbot": {
"uri": "tmod:@turbot/azure-datafactory#/definitions/factoryAka",
"modUri": "tmod:@turbot/azure-datafactory"
}
}

factoryId

{
"type": "string",
"pattern": "^/subscriptions/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/resourceGroups/[A-Za-z0-9_.-]{0,89}[A-Za-z0-9_-]/providers/Microsoft.DataFactory/factories/([a-z0-9]|[a-z0-9][a-z0-9-]{0,61}[a-z0-9])$",
"tests": [
{
"description": "base",
"input": "/subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012"
},
{
"description": "invalid service name",
"input": "//subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test",
"expected": false
}
],
".turbot": {
"uri": "tmod:@turbot/azure-datafactory#/definitions/factoryId",
"modUri": "tmod:@turbot/azure-datafactory"
}
}

factoryName

{
"type": "string",
"pattern": "^([a-z0-9]|[a-z0-9][a-z0-9-]{0,61}[a-z0-9])$",
"tests": [
{
"input": "a"
},
{
"input": "a-a"
},
{
"description": "invalid - too short",
"input": "",
"expected": false
},
{
"description": "invalid - too long",
"input": "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzjklmnopqrstuvwxyzabcdefghijklmnopqrstuv",
"expected": false
},
{
"description": "invalid - no spaces",
"input": "Test me please",
"expected": false
}
],
".turbot": {
"uri": "tmod:@turbot/azure-datafactory#/definitions/factoryName",
"modUri": "tmod:@turbot/azure-datafactory"
}
}

pipeline

{
"type": "object",
"properties": {
"name": {
"$ref": "#/definitions/pipelineName"
},
"id": {
"$ref": "#/definitions/pipelineId"
},
"etag": {
"type": "string"
},
"activities": {
"type": "array"
},
"parameters": {
"type": "object"
},
"annotations": {
"type": "array"
},
"turbot": {
"type": "object",
"properties": {
"title": {
"$ref": "#/definitions/pipelineName"
},
"akas": {
"type": "array",
"items": {
"$ref": "#/definitions/pipelineAka"
}
},
"custom": {
"type": "object",
"properties": {
"createTimestamp": {
"$ref": "turbot#/definitions/isoTimestamp"
},
"azure": {
"$ref": "azure#/definitions/azureMetadata"
},
"factoryName": {
"$ref": "#/definitions/factoryName"
}
}
}
}
}
},
"tests": [
{
"description": "Valid - All properties given",
"input": {
"name": "test0001",
"id": "/subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012/pipelines/test0001",
"etag": "01004305-0000-0100-0000-5d529b1b0000",
"description": "Copy huge amount of data in bulk from database using external control table to store source table list with partitions for each table.\n\nWhen you want to migrate data from your data warehouse like Oracle server, Netezza server, Teradata server or SQL Server to Azure, you have to load huge amount of data from multiple tables in data sources. In most cases, data has to be further partitioned in each table so that you can load rows with multiple threads in parallel from single table.",
"activities": [
{
"name": "GetPartitionList",
"description": "Lookup activity to retrieve the list of partitions stored in the external control table.",
"dependsOn": [],
"userProperties": [],
"type": "Lookup",
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureInput": false,
"secureOutput": false
},
"source": {
"type": "SqlSource",
"sqlReaderQuery": {
"value": "select @{pipeline().parameters.Control_Table_Schema_PartitionID} as PartitionID, @{pipeline().parameters.Control_Table_Schema_SourceTableName} as SourceTableName, @{pipeline().parameters.Control_Table_Schema_FilterQuery} as FilterQuery from @{pipeline().parameters.Control_Table_Name}",
"type": "Expression"
}
},
"dataset": {
"type": "DatasetReference",
"referenceName": "AzureSqlDatabaseExternal_ControlTable1",
"parameters": {
"TableName": {
"value": "@pipeline().parameters.Control_Table_Name",
"type": "Expression"
}
}
},
"firstRowOnly": false
},
{
"name": "ForEachPartition",
"description": "ForEach activity to get the partition list from Lookup activity and then iterate each of them to Copy activity. ",
"dependsOn": [
{
"activity": "GetPartitionList",
"dependencyConditions": [
"Succeeded"
]
}
],
"userProperties": [],
"type": "ForEach",
"batchCount": 5,
"items": {
"type": "Expression",
"value": "@activity('GetPartitionList').output.value"
},
"activities": [
{
"name": "CopyOnePartition",
"description": "Copy activity to copy each partition from source database store to destination store.",
"dependsOn": [],
"userProperties": [],
"type": "Copy",
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureInput": false,
"secureOutput": false
},
"source": {
"type": "SqlSource",
"sqlReaderQuery": {
"value": "@item().FilterQuery",
"type": "Expression"
}
},
"sink": {
"type": "BlobSink"
},
"enableStaging": false,
"dataIntegrationUnits": 0,
"inputs": [
{
"type": "DatasetReference",
"referenceName": "AzureSqlDatabaseDataSource1",
"parameters": {}
}
],
"outputs": [
{
"type": "DatasetReference",
"referenceName": "AzureBlobStorageDataDestination1",
"parameters": {
"FolderPath": {
"value": "@pipeline().parameters.Data_Destination_Folder_Path",
"type": "Expression"
},
"FileName": {
"value": "@concat('Partition_',item().PartitionID,'_',item().SourceTableName)",
"type": "Expression"
}
}
}
]
}
]
}
],
"parameters": {
"Control_Table_Name": {
"type": "String"
},
"Control_Table_Schema_PartitionID": {
"type": "String"
},
"Control_Table_Schema_SourceTableName": {
"type": "String"
},
"Control_Table_Schema_FilterQuery": {
"type": "String"
},
"Data_Destination_Folder_Path": {
"type": "String"
}
},
"annotations": [],
"turbot": {
"title": "test0001",
"akas": [
"azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012/pipelines/test0001"
],
"custom": {
"azure": {
"subscriptionId": "9e3548cf-17e2-4751-b87e-b72bdd2c77f7",
"regionName": "centralindia",
"resourceGroupName": "test"
}
}
}
}
},
{
"description": "Invalid - name missing",
"input": {
"turbot": {
"title": "test0001",
"akas": [
"azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012/pipelines/test0001"
],
"custom": {
"azure": {
"subscriptionId": "9e3548cf-17e2-4751-b87e-b72bdd2c77f7",
"regionName": "centralindia",
"resourceGroupName": "test"
}
}
}
}
},
{
"description": "Invalid - servername missing",
"input": {
"id": "/subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012/pipelines/test0001",
"turbot": {
"title": "test0001",
"akas": [
"azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012/pipelines/test0001"
],
"custom": {
"azure": {
"subscriptionId": "9e3548cf-17e2-4751-b87e-b72bdd2c77f7",
"regionName": "centralindia",
"resourceGroupName": "test"
}
}
}
}
}
],
".turbot": {
"uri": "tmod:@turbot/azure-datafactory#/definitions/pipeline",
"modUri": "tmod:@turbot/azure-datafactory"
}
}

pipelineAka

{
"type": "string",
"pattern": "^azure:///subscriptions/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/resourceGroups/[A-Za-z0-9_.-]{0,89}[A-Za-z0-9_-]/providers/Microsoft.DataFactory/factories/([a-z0-9]|[a-z0-9][a-z0-9-]{0,61}[a-z0-9])/pipelines/[a-zA-Z0-9][a-zA-Z-_ 0-9]+[a-zA-Z-_0-9]{1,140}$",
"tests": [
{
"description": "base",
"input": "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012/pipelines/test06"
},
{
"description": "invalid service name",
"input": "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactorys/factoriesr/test/pipelines/test60",
"expected": false
}
],
".turbot": {
"uri": "tmod:@turbot/azure-datafactory#/definitions/pipelineAka",
"modUri": "tmod:@turbot/azure-datafactory"
}
}

pipelineId

{
"type": "string",
"pattern": "^/subscriptions/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/resourceGroups/[A-Za-z0-9_.-]{0,89}[A-Za-z0-9_-]/providers/Microsoft.DataFactory/factories/([a-z0-9]|[a-z0-9][a-z0-9-]{0,61}[a-z0-9])/pipelines/[a-zA-Z0-9][a-zA-Z-_ 0-9]+[a-zA-Z-_0-9]{1,140}$",
"tests": [
{
"description": "base",
"input": "/subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012/pipelines/test08"
},
{
"description": "invalid service name",
"input": "//subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test/pipelines/test08",
"expected": false
}
],
".turbot": {
"uri": "tmod:@turbot/azure-datafactory#/definitions/pipelineId",
"modUri": "tmod:@turbot/azure-datafactory"
}
}

pipelineName

{
"type": "string",
"pattern": "^[a-zA-Z0-9][a-zA-Z-_ 0-9]+[a-zA-Z-_0-9]{1,140}$",
"tests": [
{
"input": "a-a"
},
{
"description": "invalid - too short",
"input": "",
"expected": false
},
{
"description": "with spaces",
"input": "Test me please"
}
],
".turbot": {
"uri": "tmod:@turbot/azure-datafactory#/definitions/pipelineName",
"modUri": "tmod:@turbot/azure-datafactory"
}
}