none
Azure Data Factory Copy Activity RRS feed

  • Question

  • I have been working on this for a couple days and cannot get past this error. I have 2 activities in this pipeline. The first activity copies data from an ODBC connection to an Azure database, which is successful. The 2nd activity transfers the data from Azure table to another Azure table and keeps failing.

    The error message is: Copy activity met invalid parameters: 'UnknownParameterName', Detailed message: An item with the same key has already been added..

    I do not see any invalid parameters or unknown parameter names. I have rewritten this multiple times using their add activity code template and by myself, but do not receive any errors when deploying on when it is running. Below is the JSON pipeline code.

    Only the 2nd activity is receiving an error.

    Thanks.

    Source Data set

    {
    "name": "AnalyticsDB-SHIPUPS_06shp-01src_AZ-915PM",
    "properties": {
        "structure": [
            {
                "name": "UPSD_BOL",
                "type": "String"
            },
            {
                "name": "UPSD_ORDN",
                "type": "String"
            }
        ],
        "published": false,
        "type": "AzureSqlTable",
        "linkedServiceName": "Source-SQLAzure",
        "typeProperties": {},
        "availability": {
            "frequency": "Day",
            "interval": 1,
            "offset": "04:15:00"
        },
        "external": true,
        "policy": {}
    }

    Destination Data set

    {
    "name": "AnalyticsDB-SHIPUPS_06shp-02dst_AZ-915PM",
    "properties": {
        "structure": [
            {
                "name": "SHIP_SYS_TRACK_NUM",
                "type": "String"
            },
            {
                "name": "SHIP_TRACK_NUM",
                "type": "String"
            }
        ],
        "published": false,
        "type": "AzureSqlTable",
        "linkedServiceName": "Destination-Azure-AnalyticsDB",
        "typeProperties": {
            "tableName": "[olcm].[SHIP_Tracking]"
        },
        "availability": {
            "frequency": "Day",
            "interval": 1,
            "offset": "04:15:00"
        },
        "external": false,
        "policy": {}
    }

    Pipeline

    {
    "name": "SHIPUPS_FC_COPY-915PM",
    "properties": {
        "description": "copy shipments ",
        "activities": [
            {
                "type": "Copy",
                "typeProperties": {
                    "source": {
                        "type": "RelationalSource",
                        "query": "$$Text.Format('SELECT COMPANY, UPSD_ORDN, UPSD_BOL FROM \"orupsd - UPS interface Dtl\" WHERE COMPANY = \\'01\\'', WindowStart, WindowEnd)"
                    },
                    "sink": {
                        "type": "SqlSink",
                        "sqlWriterCleanupScript": "$$Text.Format('delete imp_fc.SHIP_UPS_IntDtl_Tracking', WindowStart, WindowEnd)",
                        "writeBatchSize": 0,
                        "writeBatchTimeout": "00:00:00"
                    },
                    "translator": {
                        "type": "TabularTranslator",
                        "columnMappings": "COMPANY:COMPANY, UPSD_ORDN:UPSD_ORDN, UPSD_BOL:UPSD_BOL"
                    }
                },
                "inputs": [
                    {
                        "name": "AnalyticsDB-SHIPUPS_03shp-01src_FC-915PM"
                    }
                ],
                "outputs": [
                    {
                        "name": "AnalyticsDB-SHIPUPS_03shp-02dst_AZ-915PM"
                    }
                ],
                "policy": {
                    "timeout": "1.00:00:00",
                    "concurrency": 1,
                    "executionPriorityOrder": "NewestFirst",
                    "style": "StartOfInterval",
                    "retry": 3,
                    "longRetry": 0,
                    "longRetryInterval": "00:00:00"
                },
                "scheduler": {
                    "frequency": "Day",
                    "interval": 1,
                    "offset": "04:15:00"
                },
                "name": "915PM-SHIPUPS-fc-copy->[imp_fc]_[SHIP_UPS_IntDtl_Tracking]"
            },
            {
                "type": "Copy",
                "typeProperties": {
                    "source": {
                        "type": "SqlSource",
                        "sqlReaderQuery": "$$Text.Format('select distinct ups.UPSD_BOL, ups.UPSD_BOL from imp_fc.SHIP_UPS_IntDtl_Tracking ups LEFT JOIN olcm.SHIP_Tracking st ON ups.UPSD_BOL = st.SHIP_SYS_TRACK_NUM WHERE st.SHIP_SYS_TRACK_NUM IS NULL', WindowStart, WindowEnd)"
                    },
                    "sink": {
                        "type": "SqlSink",
                        "writeBatchSize": 0,
                        "writeBatchTimeout": "00:00:00"
                    },
                    "translator": {
                        "type": "TabularTranslator",
                        "columnMappings": "UPSD_BOL:SHIP_SYS_TRACK_NUM, UPSD_BOL:SHIP_TRACK_NUM"
                    }
                },
                "inputs": [
                    {
                        "name": "AnalyticsDB-SHIPUPS_06shp-01src_AZ-915PM"
                    }
                ],
                "outputs": [
                    {
                        "name": "AnalyticsDB-SHIPUPS_06shp-02dst_AZ-915PM"
                    }
                ],
                "policy": {
                    "timeout": "1.00:00:00",
                    "concurrency": 1,
                    "executionPriorityOrder": "NewestFirst",
                    "style": "StartOfInterval",
                    "retry": 3,
                    "longRetryInterval": "00:00:00"
                },
                "scheduler": {
                    "frequency": "Day",
                    "interval": 1,
                    "offset": "04:15:00"
                },
                "name": "915PM-SHIPUPS-AZ-update->[olcm]_[SHIP_Tracking]"
            }
        ],
        "start": "2017-08-22T03:00:00Z",
        "end": "2099-12-31T08:00:00Z",
        "isPaused": false,
        "hubName": "adf-tm-prod-01_hub",
        "pipelineMode": "Scheduled"
    }
    }




    • Edited by matt134679 Wednesday, August 23, 2017 5:47 PM
    Wednesday, August 23, 2017 5:46 PM