locked
Azure Data Lake Gen 2 Patch , action Flush return400 Bad Request, MissingRequiredHeader. RRS feed

  • Question

  • I am trying to create files in azure data lake Gen 2 from service fabric stateless /actor service with REST API's. The same code works fine with console application (both .NET Core as well as .NET Framework).

    From Service Fabric statless or actor service i get response as System.Net.Http.NoWriteNoSeekStreamContent when i create a file , do a patch operation.

    Patch Operation fails for Flush Action.

    a. Append Action Succeeds with REST call. Response code : 202 Accepted. With Response status : NoWriteNoSeekStreamContent.
    b. Action Flush fails with 400 Bad Request, MissingRequiredHeader, "An HTTP header that's mandatory for this request is not specified."


    Patch code:

    ublic async Task<long> UpdateFileAsync(string fileSystem, string path,  byte[] dataToUpload, CancellationToken cancellationToken = default(CancellationToken))
    {

    long filesize = await GetFileContentLengthAsync(fileSystem, path, cancellationToken);

    string uri = $"{_fileEndpoint}/{fileSystem}/{path}?action=append&position={filesize}&timeout={timeoutDuration}";
    using (var httpRequestMessage = new HttpRequestMessage(new HttpMethod("PATCH"), uri))
    {
    DLStorageRestAPIAuthHelper.SetUpHttpRequestMessage(httpRequestMessage, _storageAccountName, _storageAccountKey, dataToUpload);
    var response = await SingletonClient.GetClient(uri).SendAsync(httpRequestMessage, cancellationToken);
    CheckResponse(response);
    }

    filesize += dataToUpload.Length;
    uri = $"{_fileEndpoint}/{fileSystem}/{path}?action=flush&position={filesize}&timeout={timeoutDuration}";
    using (var httpRequestMessage = new HttpRequestMessage(new HttpMethod("PATCH"), uri))
    {
    DLStorageRestAPIAuthHelper.SetUpHttpRequestMessage(httpRequestMessage, _storageAccountName, _storageAccountKey);
    var response = await SingletonClient.GetClient(httpRequestMessage.RequestUri.AbsoluteUri).SendAsync(httpRequestMessage, cancellationToken);
    CheckResponse(response);
    }

    return filesize;
    }

    public static void SetUpHttpRequestMessage(HttpRequestMessage httpRequestMessage, 
    string storageAccountName, string storageAccountKey, byte[] dataToUpload = null)
    {
    DateTime now = DateTime.UtcNow;
    httpRequestMessage.Headers.Add("x-ms-date", now.ToString("R", CultureInfo.InvariantCulture));
    httpRequestMessage.Headers.Add("x-ms-version", "2018-11-09");
    if (dataToUpload != null)
    {
    httpRequestMessage.Content = new ByteArrayContent(dataToUpload);
    }

    httpRequestMessage.Headers.Authorization = GetAuthorizationHeader(httpRequestMessage , storageAccountName , storageAccountKey);
    }

    private static  AuthenticationHeaderValue GetAuthorizationHeader(HttpRequestMessage httpRequestMessage, string storageAccountName,
    string storageAccountKey)
    {
    // This is the raw representation of the message signature.
    string canonicalizedHeaders = GetCanonicalizedHeaders(httpRequestMessage);
    string canonicalizedResource = GetCanonicalizedResource(httpRequestMessage.RequestUri, storageAccountName);
    string template = $"{httpRequestMessage.Method}\n\n\n" +
      $"{(httpRequestMessage.Content == null ? string.Empty : httpRequestMessage.Content.Headers.ContentLength.ToString())}\n" +
      "\n\n\n\n\n\n\n\n" +
      $"{canonicalizedHeaders}" +
      $"{canonicalizedResource}";

    byte[] signatureBytes = Encoding.UTF8.GetBytes(template);
    HMACSHA256 sha256 = new HMACSHA256(Convert.FromBase64String(storageAccountKey));
    string signature = Convert.ToBase64String(sha256.ComputeHash(signatureBytes));
    return new AuthenticationHeaderValue("SharedKey", storageAccountName + ":" + signature);
    }


    private static string GetCanonicalizedHeaders(HttpRequestMessage httpRequestMessage)
    {
    var headers = from kvp in httpRequestMessage.Headers
      where kvp.Key.StartsWith("x-ms-", StringComparison.OrdinalIgnoreCase)
      orderby kvp.Key
      select new { Key = kvp.Key.ToLowerInvariant(), kvp.Value };

    StringBuilder headersBuilder = new StringBuilder();

    // Create the string in the right format; this is what makes the headers "canonicalized" --
    //   it means put in a standard format. https://en.wikipedia.org/wiki/Canonicalization
    foreach (var kvp in headers)
    {
    headersBuilder.Append(kvp.Key);
    char separator = ':';

    // Get the value for each header, strip out \r\n if found, then append it with the key.
    foreach (string headerValue in kvp.Value)
    {
    string trimmedValue = headerValue.TrimStart().Replace("\r\n", string.Empty);
    headersBuilder.Append(separator).Append(trimmedValue);

    // Set this to a comma; this will only be used
    // if there are multiple values for one of the headers.
    separator = ',';
    }

    headersBuilder.Append("\n");
    }

    return headersBuilder.ToString();
    }

    private static string GetCanonicalizedResource(Uri address, string storageAccountName)
    {
    // The absolute path will be "/" because for we're getting a list of containers.
    StringBuilder sb = new StringBuilder("/").Append(storageAccountName).Append(address.AbsolutePath);

    // Address.Query is the resource, such as "?comp=list".
    // This ends up with a NameValueCollection with 1 entry having key=comp, value=list.
    // It will have more entries if you have more query parameters.
    NameValueCollection values = HttpUtility.ParseQueryString(address.Query);

    foreach (var item in values.AllKeys.OrderBy(k => k))
    {
    sb.Append('\n').Append(item).Append(':').Append(values[item]);
    }

    return sb.ToString();
    }


    Any help is greatly appreciated!

    • Edited by Santosh999 Friday, May 31, 2019 10:25 AM
    Friday, May 31, 2019 10:15 AM

Answers

All replies

  • Hi Santosh,

    Please make sure that you are setting all the required headers. For reference :

    curl -i -X PATCH -H "x-ms-version: 2018-11-09" -H "content-length: 0" -H "Authorization: Bearer $ACCESS_TOKEN" "https://$STORAGE_ACCOUNT_NAME.dfs.core.windows.net/mydata/data/file1?action=flush&position=44"

    https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/update#request-headers


    Hope this helps.




    Friday, May 31, 2019 12:28 PM
  • Hi Santosh,

    Just wanted to check - Did you try adding Content-Length : 0 to the headers? Did the above suggestion help? If yes, do consider upvoting and/or marking it as answer as it would help other community members reading this thread.

    Tuesday, June 4, 2019 4:14 AM
  • I added the header you mentioned but also the error

    Monday, September 23, 2019 6:32 AM
  • Hi Santhosh and Sakurai_db,

    I just checked internally with the team. The only header you are missing is Content-Type : text/plain.

    Can you please try including this header? You can skip x-ms-date as well.

    Hope this helps.

    Monday, September 23, 2019 9:21 AM
  • still no luck

    actually for flush there should be no "body"

    because suppose we already patch with action append with body .

    Monday, September 23, 2019 2:49 PM
  • which is a json body

    Monday, September 23, 2019 2:56 PM
  • anyone has idea ?

    I tried so many combination of the header but still not ok

    Tuesday, September 24, 2019 3:02 PM
  • anyone has the idea ?
    Thursday, September 26, 2019 1:19 AM