none
Uploading blob by blocks using shared access signature

    Question

  • Hi.

    I am trying to upload large file by blocks of 4MB using operation PUT. General idea is to give to user permission to upload file direct to blob within the hour using shared access signature.

    This is my sample code:

    using System;
    using System.Collections.Generic;
    using System.ComponentModel;
    using System.Data;
    using System.Drawing;
    using System.Linq;
    using System.Text;
    using System.Windows.Forms;
    using System.Net;
    using System.IO;
    using System.Threading;
    
    namespace WindowsFormsApplication1 {
        public partial class Form1 : Form
        {
            public Form1()
            {
                InitializeComponent();
            }
    
            private void sendRequest(String url, byte[] content, int contentLength, String contentType, String[] additionalHeaders)
            {
                WebRequest request = HttpWebRequest.Create(url);
    
                request.Method = "PUT";
                request.ContentLength = contentLength;
                request.ContentType = contentType;
    
                if (additionalHeaders != null)
                {
                    foreach (String additionalHeader in additionalHeaders)
                    {
                        request.Headers.Add(additionalHeader);
                    }
                }
    
                request.Headers.Add("x-ms-version: 2009-09-19");
                request.Headers.Add("x-ms-date: " + DateTime.Now.ToUniversalTime().ToString("r"));
    
                Stream dataStream = request.GetRequestStream();
                dataStream.Write(content, 0, contentLength);
                dataStream.Close();
    
                WebResponse response = request.GetResponse();
    
                /*this.Invoke((MethodInvoker)delegate
                {
                    WebHeaderCollection headers = response.Headers;
    
                    textBox1.Text += "Url=" + url + "\r\n";
                    textBox1.Text += "Status=" + ((HttpWebResponse)response).StatusCode + "\r\n";
                    textBox1.Text += "Status desc=" + ((HttpWebResponse)response).StatusDescription + "\r\n";
    
                    for (int i = 0; i < headers.Count; i++)
                    {
                        textBox1.Text += headers.GetKey(i) + " = " + headers.Get(i) + "\r\n";
                    }
    
                    textBox1.Text += "\r\n\r\n";
                });*/
            }
    
            public void sendFile(object obj)
            {
                SendData data = (SendData)obj;
    
                String url = "http://127.0.0.1:10000/devstoreaccount1/test2/1.mp4?st=2012-03-19T06%3A27%3A12.0000000Z&se=2012-03-19T07%3A26%3A12.0000000Z&sr=b&sp=w&sig=3rX7g7kAcxu4oZi6n%2BDgo1EY8rlBO8omFOqz4Irb8%2Fg%3D";
                FileStream file = new FileStream(data.fileName, FileMode.Open);
    
                int page = 4194304;
                int parts = (int)Math.Ceiling((double)file.Length / (double)page);
    
                this.Invoke((MethodInvoker)delegate
                {
                    progressBar1.Minimum = 0;
                    progressBar1.Maximum = parts + 1;
                    progressBar1.Value = 0;
                    label1.Text = "Starting...";
                    label1.ForeColor = Color.Blue;
                });
    
                int part = 0;
                byte[] buffer = new byte[page];
                String[] blocks = new String[parts];
    
                while (part < parts)
                {
                    int bytes = file.Read(buffer, 0, page);
                    if (bytes == 0) break;
    
                    String blockId = Convert.ToBase64String(Encoding.UTF8.GetBytes(Guid.NewGuid().ToString()));
    
                    try
                    {
                        sendRequest(url + "&comp=block&blockid=" + blockId, buffer, bytes, MIMEAssistant.getMimeType(data.fileName), null);
                    }
                    catch (Exception ex)
                    {
                        this.Invoke((MethodInvoker)delegate
                        {
                            progressBar1.Value = 0;
                            label1.Text = ex.Message;
                            label1.ForeColor = Color.Red;
                        });
    
                        file.Close();
    
                        return;
                    }
    
                    blocks[part] = blockId;
                    part++;
    
                    this.Invoke((MethodInvoker)delegate
                    {
                        progressBar1.Increment(1);
                        label1.Text = "Block " + part.ToString() + " from " + parts.ToString() + " finished...";
                    });
                }
    
                file.Close();
    
                String content = "<?xml version=\"1.0\" encoding=\"utf-8\"?><BlockList>";
    
                foreach (String blockId in blocks)
                {
                    content += "<Latest>" + blockId + "</Latest>";
                }
    
                content += "</BlockList>";
    
                byte[] contentBytes = Encoding.UTF8.GetBytes(content);
    
                sendRequest(url + "&comp=blocklist", contentBytes, contentBytes.Length, MIMEAssistant.getMimeType(data.fileName), null);
    
                this.Invoke((MethodInvoker)delegate
                {
                    progressBar1.Increment(1);
                    label1.Text = "Finished";
                });
            }
    
            private void button1_Click(object sender, EventArgs e)
            {
                if (openFileDialog1.ShowDialog() == System.Windows.Forms.DialogResult.OK)
                {
                    Thread sendThread = new Thread(sendFile);
    
                    sendThread.Start(new SendData(openFileDialog1.FileName));
                }
            }
        }
    
        public static class MIMEAssistant
        {
            private static readonly Dictionary<string, string> MIMETypesDictionary = new Dictionary<string, string>
            {
                {"mp4", "video/mp4"},
                {"txt", "text/plain"}
            };
    
            public static string getMimeType(string fileName)
            {
                if (MIMETypesDictionary.ContainsKey(Path.GetExtension(fileName).Remove(0, 1)))
                {
                    return MIMETypesDictionary[Path.GetExtension(fileName).Remove(0, 1)];
                }
    
                return "";
            }
        }
    
        public class SendData
        {
            public String fileName;
    
            public SendData(String fileName)
            {
                this.fileName = fileName;
            }
        }
    }


    You can see this url:

    String url = "http://127.0.0.1:10000/devstoreaccount1/test2/1.mp4?st=2012-03-19T06%3A27%3A12.0000000Z&se=2012-03-19T07%3A26%3A12.0000000Z&sr=b&sp=w&sig=3rX7g7kAcxu4oZi6n%2BDgo1EY8rlBO8omFOqz4Irb8%2Fg%3D";

    This url is generated on server to give user permission to write to blob within next hour.

    It works correctly on my local storage emulator.

    Hovewer, on real account inside cloud I have problem: after successfully uploaded first and second blocks I receive error: Connection timeout.

    It is not important which is block size: 4MB or 100KB. I always get this error message after second request.

    I don't known exactly should I sign this part in request too:

    "&comp=block&blockid=" + blockId
    I appreciate any help. Thanks.
    Monday, March 19, 2012 7:01 AM

Answers

All replies

  • Hi,

    Since you’re using .NET, it is recommended to use Windows Azure Storage Library instead of creating the SAS manually. Using the library, we simply need to use the SharedAccessPolicies to work with SAS. I would like to suggest you to check http://msdn.microsoft.com/en-us/library/windowsazure/hh508996.aspx for more information.

    Best Regards,

    Ming Xu.


    Please mark the replies as answers if they help or unmark if not.
    If you have any feedback about my replies, please contact msdnmg@microsoft.com.
    Microsoft One Code Framework

    Monday, March 19, 2012 12:50 PM
    Moderator
  • An url you see is generated by Azure SDK for PHP on server.

    String url = "http://127.0.0.1:10000/devstoreaccount1/test2/1.mp4?st=2012-03-19T06%3A27%3A12.0000000Z&se=2012-03-19T07%3A26%3A12.0000000Z&sr=b&sp=w&sig=3rX7g7kAcxu4oZi6n%2BDgo1EY8rlBO8omFOqz4Irb8%2Fg%3D";
    I don't want to use Storage Library, becouse just want to see possibility to do what I want and then translate alghoritm to other language, Javascript, maybe.
    Tuesday, March 20, 2012 9:23 AM
  • Hi,

    The behavior sounds to be related to concurrent HTTP requests. The default is 2 concurrent requests:

    http://msdn.microsoft.com/en-us/library/system.net.servicepointmanager.defaultconnectionlimit.aspx

    Does it work if you set it to larger value? If it still doesn't work could you post the callstack of this exception?


    Allen Chen [MSFT]
    MSDN Community Support | Feedback to us
    Get or Request Code Sample from Microsoft
    Please remember to mark the replies as answers if they help and unmark them if they provide no help.

    Monday, March 26, 2012 9:41 AM
    Moderator
  • The behavior sounds to be related to concurrent HTTP requests. The default is 2 concurrent requests:

    Thank you very much. You are right.

    After receiving response from server:

    WebResponse response = request.GetResponse();

    I must close it:

    response.Close();

    Now the behavior is like I want.

    Actually, now it is not Azure Storage question.

    Thank you.

    Tuesday, March 27, 2012 8:34 AM