Definitions for @turbot/azure-datafactory
- dataFactory
- dataFactoryAka
- dataset
- datasetAka
- datasetId
- datasetName
- factory
- factoryAka
- factoryId
- factoryName
- pipeline
- pipelineAka
- pipelineId
- pipelineName
dataFactory
{ "allOf": [ { "$ref": "turbot#/definitions/service" }, { "type": "object", "properties": { "name": { "const": "Data Factory" }, "namespace": { "const": "Microsoft.DataFactory" }, "registrationState": { "type": "string" }, "turbot": { "type": "object", "properties": { "akas": { "type": "array", "items": { "$ref": "#/definitions/dataFactoryAka" } }, "title": { "const": "Data Factory" }, "custom": { "type": "object", "properties": { "azure": { "$ref": "azure#/definitions/azureMetadata" } } } } } }, "required": [ "name", "turbot" ], "additionalProperties": true, "tests": [ { "description": "All details provided", "input": { "name": "Data Factory", "turbot": { "akas": [ "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/providers/Microsoft.DataFactory/services/dataFactory" ], "title": "Data Factory", "custom": { "azure": { "subscriptionId": "9e3548cf-17e2-4751-b87e-b72bdd2c77f7" } } } } }, { "description": "Invalid - Name is not as declared", "input": { "name": "DataFactory", "turbot": { "akas": [ "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/providers/Microsoft.DataFactory/services/dataFactory" ], "title": "Data Factory", "custom": { "azure": { "subscriptionId": "9e3548cf-17e2-4751-b87e-b72bdd2c77f7" } } } }, "expected": false }, { "description": "Invalid - Service name missing", "input": { "turbot": null, "akas": [ "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/providers/Microsoft.DataFactory/services/dataFactory" ], "title": "Data Factory", "custom": { "azure": { "subscriptionId": "9e3548cf-17e2-4751-b87e-b72bdd2c77f7" } } }, "expected": false } ] } ], ".turbot": { "uri": "tmod:@turbot/azure-datafactory#/definitions/dataFactory", "modUri": "tmod:@turbot/azure-datafactory" }}
dataFactoryAka
{ "type": "string", "pattern": "^azure:///subscriptions/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/providers/Microsoft.DataFactory/services/dataFactory$", "tests": [ { "description": "base", "input": "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/providers/Microsoft.DataFactory/services/dataFactory" }, { "description": "invalid service name", "input": "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/providers/Microsoft.DataFactoryr/services/dataFactory", "expected": false }, { "description": "invalid subscription id", "input": "azure:///subscriptions/9e3548cf-17e2-4751-12345-b72bdd2c77f7/providers/Microsoft.DataFactory/services/dataFactory", "expected": false } ], ".turbot": { "uri": "tmod:@turbot/azure-datafactory#/definitions/dataFactoryAka", "modUri": "tmod:@turbot/azure-datafactory" }}
dataset
{ "type": "object", "properties": { "name": { "$ref": "#/definitions/datasetName" }, "id": { "$ref": "#/definitions/datasetId" }, "etag": { "type": "string" }, "properties": { "type": "object" }, "turbot": { "type": "object", "properties": { "title": { "$ref": "#/definitions/factoryName" }, "akas": { "type": "array", "items": { "$ref": "#/definitions/datasetAka" } }, "custom": { "type": "object", "properties": { "createTimestamp": { "$ref": "turbot#/definitions/isoTimestamp" }, "azure": { "$ref": "azure#/definitions/azureMetadata" }, "factoryName": { "$ref": "#/definitions/factoryName" } } } } } }, "additionalProperties": true, "tests": [ { "description": "Valid - All properties given", "input": { "name": "test0001", "id": "/subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test90/datasets/testing", "etag": "00000a61-0000-0100-0000-5d5155ec0000", "properties": { "description": "testing", "linkedServiceName": { "type": "LinkedServiceReference", "referenceName": "AzureBlobStorage1" }, "parameters": { "nkml": { "type": "string", "defaultValue": "value" } }, "annotations": [], "type": "DelimitedText", "typeProperties": { "location": { "type": "AzureBlobStorageLocation" }, "columnDelimiter": ",", "escapeChar": "\\", "quoteChar": "\"" }, "schema": [] }, "turbot": { "title": "test0001", "akas": [ "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test90/datasets/testing" ], "custom": { "azure": { "subscriptionId": "9e3548cf-17e2-4751-b87e-b72bdd2c77f7", "regionName": "centralindia", "resourceGroupName": "test" } } } } } ], ".turbot": { "uri": "tmod:@turbot/azure-datafactory#/definitions/dataset", "modUri": "tmod:@turbot/azure-datafactory" }}
datasetAka
{ "type": "string", "pattern": "^azure:///subscriptions/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/resourceGroups/[A-Za-z0-9_.-]{0,89}[A-Za-z0-9_-]/providers/Microsoft.DataFactory/factories/([a-z0-9]|[a-z0-9][a-z0-9-]{0,61}[a-z0-9])/datasets/[a-zA-Z0-9][a-zA-Z0-9_]+$", "tests": [ { "description": "base", "input": "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012/datasets/test09" }, { "description": "invalid service name", "input": "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactorys/factoriesr/test/datasets/test09", "expected": false } ], ".turbot": { "uri": "tmod:@turbot/azure-datafactory#/definitions/datasetAka", "modUri": "tmod:@turbot/azure-datafactory" }}
datasetId
{ "type": "string", "pattern": "^/subscriptions/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/resourceGroups/[A-Za-z0-9_.-]{0,89}[A-Za-z0-9_-]/providers/Microsoft.DataFactory/factories/([a-z0-9]|[a-z0-9][a-z0-9-]{0,61}[a-z0-9])/datasets/[a-zA-Z0-9][a-zA-Z0-9_]+$", "tests": [ { "description": "base", "input": "/subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012/datasets/test09" }, { "description": "invalid service name", "input": "//subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test/datasets/test09", "expected": false } ], ".turbot": { "uri": "tmod:@turbot/azure-datafactory#/definitions/datasetId", "modUri": "tmod:@turbot/azure-datafactory" }}
datasetName
{ "type": "string", "pattern": "^[a-zA-Z0-9][a-zA-Z0-9_]+$", "tests": [ { "input": "ab" }, { "input": "a_a" }, { "description": "invalid - too short", "input": "", "expected": false }, { "description": "invalid - no spaces", "input": "Test me please", "expected": false } ], ".turbot": { "uri": "tmod:@turbot/azure-datafactory#/definitions/datasetName", "modUri": "tmod:@turbot/azure-datafactory" }}
factory
{ "type": "object", "properties": { "name": { "$ref": "#/definitions/factoryName" }, "id": { "$ref": "#/definitions/factoryId" }, "tags": { "$ref": "azure#/definitions/tags" }, "location": { "type": "string" }, "eTag": { "type": "string" }, "identity": { "type": "object" }, "provisioningState": { "type": "string" }, "createTime": { "type": "string" }, "version": { "type": "string" }, "turbot": { "type": "object", "properties": { "title": { "$ref": "#/definitions/factoryName" }, "tags": { "$ref": "azure#/definitions/tags" }, "akas": { "type": "array", "items": { "$ref": "#/definitions/factoryAka" } }, "custom": { "type": "object", "properties": { "createTimestamp": { "$ref": "turbot#/definitions/isoTimestamp" }, "azure": { "$ref": "azure#/definitions/azureMetadata" } } } } } }, "tests": [ { "description": "Valid - All properties given", "input": { "name": "test0001", "id": "/subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012", "type": "Microsoft.DataFactory/factories", "location": "EastUS", "tags": {}, "eTag": "\"0c008ca0-0000-0100-0000-5d4c2a0e0000\"", "identity": { "type": "SystemAssigned", "principalId": "cb216664-666e-4b1a-ab42-5f12d8c13862", "tenantId": "0fae5fbb-c4cf-4d04-9e13-a1260e9fd8f8" }, "provisioningState": "Succeeded", "createTime": "2019-08-08T13:56:30.026Z", "version": "2018-06-01", "turbot": { "title": "test0001", "akas": [ "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012" ], "custom": { "azure": { "subscriptionId": "9e3548cf-17e2-4751-b87e-b72bdd2c77f7", "regionName": "centralindia", "resourceGroupName": "test" } } } } }, { "description": "Invalid - name missing", "input": { "turbot": { "title": "test0001", "akas": [ "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012" ], "custom": { "azure": { "subscriptionId": "9e3548cf-17e2-4751-b87e-b72bdd2c77f7", "regionName": "centralindia", "resourceGroupName": "test" } } } } }, { "description": "Invalid - servername missing", "input": { "id": "/subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012", "turbot": { "title": "test0001", "akas": [ "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012" ], "custom": { "azure": { "subscriptionId": "9e3548cf-17e2-4751-b87e-b72bdd2c77f7", "regionName": "centralindia", "resourceGroupName": "test" } } } } } ], ".turbot": { "uri": "tmod:@turbot/azure-datafactory#/definitions/factory", "modUri": "tmod:@turbot/azure-datafactory" }}
factoryAka
{ "type": "string", "pattern": "^azure:///subscriptions/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/resourceGroups/[A-Za-z0-9_.-]{0,89}[A-Za-z0-9_-]/providers/Microsoft.DataFactory/factories/([a-z0-9]|[a-z0-9][a-z0-9-]{0,61}[a-z0-9])$", "tests": [ { "description": "base", "input": "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012" }, { "description": "invalid service name", "input": "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactorys/factoriesr/test", "expected": false } ], ".turbot": { "uri": "tmod:@turbot/azure-datafactory#/definitions/factoryAka", "modUri": "tmod:@turbot/azure-datafactory" }}
factoryId
{ "type": "string", "pattern": "^/subscriptions/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/resourceGroups/[A-Za-z0-9_.-]{0,89}[A-Za-z0-9_-]/providers/Microsoft.DataFactory/factories/([a-z0-9]|[a-z0-9][a-z0-9-]{0,61}[a-z0-9])$", "tests": [ { "description": "base", "input": "/subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012" }, { "description": "invalid service name", "input": "//subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test", "expected": false } ], ".turbot": { "uri": "tmod:@turbot/azure-datafactory#/definitions/factoryId", "modUri": "tmod:@turbot/azure-datafactory" }}
factoryName
{ "type": "string", "pattern": "^([a-z0-9]|[a-z0-9][a-z0-9-]{0,61}[a-z0-9])$", "tests": [ { "input": "a" }, { "input": "a-a" }, { "description": "invalid - too short", "input": "", "expected": false }, { "description": "invalid - too long", "input": "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzjklmnopqrstuvwxyzabcdefghijklmnopqrstuv", "expected": false }, { "description": "invalid - no spaces", "input": "Test me please", "expected": false } ], ".turbot": { "uri": "tmod:@turbot/azure-datafactory#/definitions/factoryName", "modUri": "tmod:@turbot/azure-datafactory" }}
pipeline
{ "type": "object", "properties": { "name": { "$ref": "#/definitions/pipelineName" }, "id": { "$ref": "#/definitions/pipelineId" }, "etag": { "type": "string" }, "activities": { "type": "array" }, "parameters": { "type": "object" }, "annotations": { "type": "array" }, "turbot": { "type": "object", "properties": { "title": { "$ref": "#/definitions/pipelineName" }, "akas": { "type": "array", "items": { "$ref": "#/definitions/pipelineAka" } }, "custom": { "type": "object", "properties": { "createTimestamp": { "$ref": "turbot#/definitions/isoTimestamp" }, "azure": { "$ref": "azure#/definitions/azureMetadata" }, "factoryName": { "$ref": "#/definitions/factoryName" } } } } } }, "tests": [ { "description": "Valid - All properties given", "input": { "name": "test0001", "id": "/subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012/pipelines/test0001", "etag": "01004305-0000-0100-0000-5d529b1b0000", "description": "Copy huge amount of data in bulk from database using external control table to store source table list with partitions for each table.\n\nWhen you want to migrate data from your data warehouse like Oracle server, Netezza server, Teradata server or SQL Server to Azure, you have to load huge amount of data from multiple tables in data sources. In most cases, data has to be further partitioned in each table so that you can load rows with multiple threads in parallel from single table.", "activities": [ { "name": "GetPartitionList", "description": "Lookup activity to retrieve the list of partitions stored in the external control table.", "dependsOn": [], "userProperties": [], "type": "Lookup", "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureInput": false, "secureOutput": false }, "source": { "type": "SqlSource", "sqlReaderQuery": { "value": "select @{pipeline().parameters.Control_Table_Schema_PartitionID} as PartitionID, @{pipeline().parameters.Control_Table_Schema_SourceTableName} as SourceTableName, @{pipeline().parameters.Control_Table_Schema_FilterQuery} as FilterQuery from @{pipeline().parameters.Control_Table_Name}", "type": "Expression" } }, "dataset": { "type": "DatasetReference", "referenceName": "AzureSqlDatabaseExternal_ControlTable1", "parameters": { "TableName": { "value": "@pipeline().parameters.Control_Table_Name", "type": "Expression" } } }, "firstRowOnly": false }, { "name": "ForEachPartition", "description": "ForEach activity to get the partition list from Lookup activity and then iterate each of them to Copy activity. ", "dependsOn": [ { "activity": "GetPartitionList", "dependencyConditions": [ "Succeeded" ] } ], "userProperties": [], "type": "ForEach", "batchCount": 5, "items": { "type": "Expression", "value": "@activity('GetPartitionList').output.value" }, "activities": [ { "name": "CopyOnePartition", "description": "Copy activity to copy each partition from source database store to destination store.", "dependsOn": [], "userProperties": [], "type": "Copy", "policy": { "timeout": "7.00:00:00", "retry": 0, "retryIntervalInSeconds": 30, "secureInput": false, "secureOutput": false }, "source": { "type": "SqlSource", "sqlReaderQuery": { "value": "@item().FilterQuery", "type": "Expression" } }, "sink": { "type": "BlobSink" }, "enableStaging": false, "dataIntegrationUnits": 0, "inputs": [ { "type": "DatasetReference", "referenceName": "AzureSqlDatabaseDataSource1", "parameters": {} } ], "outputs": [ { "type": "DatasetReference", "referenceName": "AzureBlobStorageDataDestination1", "parameters": { "FolderPath": { "value": "@pipeline().parameters.Data_Destination_Folder_Path", "type": "Expression" }, "FileName": { "value": "@concat('Partition_',item().PartitionID,'_',item().SourceTableName)", "type": "Expression" } } } ] } ] } ], "parameters": { "Control_Table_Name": { "type": "String" }, "Control_Table_Schema_PartitionID": { "type": "String" }, "Control_Table_Schema_SourceTableName": { "type": "String" }, "Control_Table_Schema_FilterQuery": { "type": "String" }, "Data_Destination_Folder_Path": { "type": "String" } }, "annotations": [], "turbot": { "title": "test0001", "akas": [ "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012/pipelines/test0001" ], "custom": { "azure": { "subscriptionId": "9e3548cf-17e2-4751-b87e-b72bdd2c77f7", "regionName": "centralindia", "resourceGroupName": "test" } } } } }, { "description": "Invalid - name missing", "input": { "turbot": { "title": "test0001", "akas": [ "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012/pipelines/test0001" ], "custom": { "azure": { "subscriptionId": "9e3548cf-17e2-4751-b87e-b72bdd2c77f7", "regionName": "centralindia", "resourceGroupName": "test" } } } } }, { "description": "Invalid - servername missing", "input": { "id": "/subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012/pipelines/test0001", "turbot": { "title": "test0001", "akas": [ "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012/pipelines/test0001" ], "custom": { "azure": { "subscriptionId": "9e3548cf-17e2-4751-b87e-b72bdd2c77f7", "regionName": "centralindia", "resourceGroupName": "test" } } } } } ], ".turbot": { "uri": "tmod:@turbot/azure-datafactory#/definitions/pipeline", "modUri": "tmod:@turbot/azure-datafactory" }}
pipelineAka
{ "type": "string", "pattern": "^azure:///subscriptions/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/resourceGroups/[A-Za-z0-9_.-]{0,89}[A-Za-z0-9_-]/providers/Microsoft.DataFactory/factories/([a-z0-9]|[a-z0-9][a-z0-9-]{0,61}[a-z0-9])/pipelines/[a-zA-Z0-9][a-zA-Z-_ 0-9]+[a-zA-Z-_0-9]{1,140}$", "tests": [ { "description": "base", "input": "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012/pipelines/test06" }, { "description": "invalid service name", "input": "azure:///subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactorys/factoriesr/test/pipelines/test60", "expected": false } ], ".turbot": { "uri": "tmod:@turbot/azure-datafactory#/definitions/pipelineAka", "modUri": "tmod:@turbot/azure-datafactory" }}
pipelineId
{ "type": "string", "pattern": "^/subscriptions/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/resourceGroups/[A-Za-z0-9_.-]{0,89}[A-Za-z0-9_-]/providers/Microsoft.DataFactory/factories/([a-z0-9]|[a-z0-9][a-z0-9-]{0,61}[a-z0-9])/pipelines/[a-zA-Z0-9][a-zA-Z-_ 0-9]+[a-zA-Z-_0-9]{1,140}$", "tests": [ { "description": "base", "input": "/subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test0012/pipelines/test08" }, { "description": "invalid service name", "input": "//subscriptions/9e3548cf-17e2-4751-b87e-b72bdd2c77f7/resourceGroups/test/providers/Microsoft.DataFactory/factories/test/pipelines/test08", "expected": false } ], ".turbot": { "uri": "tmod:@turbot/azure-datafactory#/definitions/pipelineId", "modUri": "tmod:@turbot/azure-datafactory" }}
pipelineName
{ "type": "string", "pattern": "^[a-zA-Z0-9][a-zA-Z-_ 0-9]+[a-zA-Z-_0-9]{1,140}$", "tests": [ { "input": "a-a" }, { "description": "invalid - too short", "input": "", "expected": false }, { "description": "with spaces", "input": "Test me please" } ], ".turbot": { "uri": "tmod:@turbot/azure-datafactory#/definitions/pipelineName", "modUri": "tmod:@turbot/azure-datafactory" }}