none
Unable to download a file when proxy is mentioned in WebClient or HttpWebRequest RRS feed

  • Question

  • Hi,
    I am trying to download the belo URLs 

    http://tsp.finra.org/finra_org/ticksizepilot/TSPrePilotSecurities.txt

    http://tsp.finra.org/finra_org/ticksizepilot/TSPrePilotChanges.txt

    I am able to download the above URLs with out mentioning the Proxy.
    If I set the proxy I am getting the error message "Error Too many automatic redirections were attempted" .

    Please let me know if you need more inforamtion.
    Thanks,
    Siddi.

    What I have tried:

    Below are the different options I have tried:
    using System;
    using System.Collections.Generic;
    using System.Linq;
    using System.Net;
    using System.Text;
    using System.Threading.Tasks;
    using System.Diagnostics;
    using System.IO;

    namespace AutoRedirect
    {
    class Program
    {
    static string pilotSecuritiesFilePath = @"C:\FINRA\pilotSecuritiesProxy.txt";//GetPilotSecuritiesFilePath();
    static string pilotChangesFilePath = @"C:\FINRA\pilotChangesProxy.txt";//GetPilotChangesFilePath();
    static string masterURL = "http://tsp.finra.org/finra_org/ticksizepilot/TSPrePilotSecurities.txt";
    static string changeURL = "http://tsp.finra.org/finra_org/ticksizepilot/TSPrePilotChanges.txt";
    static string sSource;
    static string sLog;
    static string sEvent;
    static string filePath = @"C:\FINRA\Log.txt";

    static void Main(string[] args)
    {
    sSource = "AutoRedirectApp";
    sLog = "Application";
    sEvent = "AutoRedirect";

    if (File.Exists(filePath))
    {
    File.Delete(filePath);
    }

    //if (!EventLog.SourceExists(sSource))
    // EventLog.CreateEventSource(sSource, sLog);
    DownloadPilotSecuritiesWithoutProxy();
    DownloadPilotSecurities();
    DownloadUsingWebRequestWithoutProxy();
    DownloadUsingWebRequestWithProxy();
    DownloadUsingWebRequestWithProxyMaxRedirections();
    DownloadUsingWebRequestWithProxyWithBaseURL();
    }







    private static void DownloadPilotSecuritiesWithoutProxy()
    {
    //LogMessage("Download using WebClient Without Proxy: Start"); 
    try
    {
    using (WebClient webClient = new WebClient())

    //webClient.BaseAddress = "http://www.finra.org/";
    webClient.DownloadFile(masterURL, pilotSecuritiesFilePath);
    webClient.DownloadFile(changeURL, pilotChangesFilePath);
    }
    }
    catch (Exception ex)
    {
    EventLog.WriteEntry(sSource, ex.Message);
    //LogMessage(ex.Message);
    LogMessage("Download using WebClient Without Proxy: Error " + ex.Message); 
    }
    LogMessage("Download using WebClient Without Proxy: Completed"); 
    }

    private static void DownloadPilotSecurities()
    {
    // LogMessage("Download using WebClient with Proxy: Started"); 
    try
    {
    using (WebClient webClient = new WebClient())
    {
    WebProxy wp = new WebProxy("http://SomeProxy.pac");
    webClient.Proxy = wp;
    //webClient.BaseAddress = "http://www.finra.org/";
    webClient.DownloadFile(masterURL, pilotSecuritiesFilePath);
    webClient.DownloadFile(changeURL, pilotChangesFilePath);
    }
    LogMessage("Download using WebClient with Proxy: Completed");
    }
    catch (Exception ex)
    {
    //EventLog.WriteEntry(sSource, ex.Message);
    Console.WriteLine(ex.Message);
    //LogMessage(ex.Message);
    LogMessage("Download using WebClient with Proxy: Error " + ex.Message); 
    }

    }

    private static void DownloadUsingWebRequestWithoutProxy()
    {
    // LogMessage("Download Using WebRequest without Proxy: Started");
    HttpWebRequest webReq = (HttpWebRequest)WebRequest.Create(masterURL);
    string res;
    try

    webReq.Method = "GET"; 

    using (WebResponse response = webReq.GetResponse())
    {
    using (Stream stream = response.GetResponseStream())
    {
    StreamReader reader = new StreamReader(stream);
    res = reader.ReadToEnd();

    }

    File.AppendAllText(pilotSecuritiesFilePath, res);
    LogMessage("Download Using WebRequest without Proxy: Completed");
    }
    catch (Exception ex)
    {
    LogMessage("Download Using WebRequest without Proxy: Error " + ex.Message);
    }
    }

    private static void DownloadUsingWebRequestWithProxy()
    {
    // LogMessage("Download Using WebRequest without Proxy: Started");
    HttpWebRequest webReq = (HttpWebRequest)WebRequest.Create(masterURL);
    CookieContainer cookieContainer = new CookieContainer();
    string res;
    try
    {
    webReq.Method = "GET";
    WebProxy wp = new WebProxy("http://SomeProxy.pac");
    webReq.Proxy = wp;
    webReq.CookieContainer = cookieContainer; 

    using (WebResponse response = webReq.GetResponse())
    {
    using (Stream stream = response.GetResponseStream())
    {
    StreamReader reader = new StreamReader(stream);
    res = reader.ReadToEnd();
    }
    }

    File.AppendAllText(pilotSecuritiesFilePath, res);
    LogMessage("Download Using WebRequest with Proxy: Completed");
    }
    catch (Exception ex)
    {
    LogMessage("Download Using WebRequest with Proxy: Error " + ex.Message);
    }
    }

    private static void DownloadUsingWebRequestWithProxyMaxRedirections()
    {
    // LogMessage("Download Using WebRequest without Proxy: Started");
    HttpWebRequest webReq = (HttpWebRequest)WebRequest.Create(masterURL);
    CookieContainer cookieContainer = new CookieContainer();
    string res;
    try
    {
    webReq.Method = "GET";
    WebProxy wp = new WebProxy("http://SomeProxy.pac");
    webReq.Proxy = wp;
    webReq.CookieContainer = cookieContainer;
    webReq.MaximumAutomaticRedirections = 10000;

    using (WebResponse response = webReq.GetResponse())
    {
    using (Stream stream = response.GetResponseStream())
    {
    StreamReader reader = new StreamReader(stream);
    res = reader.ReadToEnd();
    }
    }

    File.AppendAllText(pilotSecuritiesFilePath, res);
    LogMessage("Download Using WebRequest with Proxy with Max Redirections: Completed");
    }
    catch (Exception ex)
    {
    LogMessage("Download Using WebRequest with Proxy with Max Redirections: Error " + ex.Message);
    }
    }

    private static void LogMessage(string message)
    {

    File.AppendAllText(filePath, Environment.NewLine+message);

    }

    private static void DownloadUsingWebRequestWithProxyWithBaseURL()
    {
    HttpWebRequest webReq = (HttpWebRequest)WebRequest.Create("http://www.finra.org/");
    CookieContainer cookieContainer = new CookieContainer();
    string res;
    try
    {
    webReq.Method = "GET";

    //webReq.Proxy = wp;
    webReq.CookieContainer = cookieContainer;
    //webReq.MaximumAutomaticRedirections = 10000;

    using (WebResponse response = webReq.GetResponse())
    {
    using (Stream stream = response.GetResponseStream())
    {
    StreamReader reader = new StreamReader(stream);
    res = reader.ReadToEnd();
    }
    }

    webReq = (HttpWebRequest)WebRequest.Create("http://tsp.finra.org/finra_org/ticksizepilot/TSPrePilotSecurities.txt");
    webReq.CookieContainer = cookieContainer;
    webReq.Method = "GET";
    WebProxy wp = new WebProxy("http://SomeProxy.pac");
    webReq.Proxy = wp;
    using(WebResponse response2 = webReq.GetResponse())
    {
    using (Stream stream = response2.GetResponseStream())
    {
    StreamReader reader = new StreamReader(stream);
    res = reader.ReadToEnd();
    }
    }

    File.AppendAllText(pilotSecuritiesFilePath, res);
    LogMessage("Download Using WebRequest with Proxy with Base URL: Completed");
    }
    catch (Exception ex)
    {
    LogMessage("Download Using WebRequest with Proxy with Base URL : Error " + ex.Message);
    }
    }


    }
    }

    Siddi Khadar Vali Pothuganti


    • Edited by siddi4 Wednesday, May 25, 2016 11:00 AM Removing the sensitive information
    Thursday, May 19, 2016 8:02 AM

Answers

  • Hi siddi4,

    >>"WebProxy wp = new WebProxy("http://proxyconfig.schwab.com/svcproxy.pac");
    webClient.Proxy = wp"

    According to the document of WebProxy method. WebProxy method need a proxy server url as input parameter and what you input is a pac file url. It can't automatically detect the proxy server in the pac file, you have to parse the file by yourself and get the proxy server. For how to parse the pac file, please take a look at this article.

    Using PAC files proxy

    Best Regards,
    Li Wang

    Note: This response contains a reference to a third party World Wide Web site. Microsoft is providing this information as a convenience to you. Microsoft does not control these sites and has not tested any software or information found on these sites; therefore, Microsoft cannot make any representations regarding the quality, safety, or suitability of any software or information found there. There are inherent dangers in the use of any software found on the Internet, and Microsoft cautions you to make sure that you completely understand the risk before retrieving any software from the Internet.


    We are trying to better understand customer views on social support experience, so your participation in this interview project would be greatly appreciated if you have time. Thanks for helping make community forums a great place.
    Click HERE to participate the survey.

    • Marked as answer by siddi4 Wednesday, May 25, 2016 10:51 AM
    Tuesday, May 24, 2016 7:09 AM
    Moderator

All replies

  • I am curious about how do you define the following proxy?

    webReq.Method = "GET";
    WebProxy wp = new WebProxy("http://proxyconfig.schwab.com/svcproxy.pac");
    webReq.Proxy = wp;
    webReq.CookieContainer = cookieContainer; 
    

    Maybe the following link using the default proxy is enough.

    http://tsp.finra.org/finra_org/ticksizepilot/TSPrePilotSecurities.txt
    
    http://tsp.finra.org/finra_org/ticksizepilot/TSPrePilotChanges.txt
    

    Friday, May 20, 2016 8:58 AM
  • Hi Linki Tan,

    Thanks for looking into this.

    In our servers there will be no internet connectivity. So to download the URLs

    http://tsp.finra.org/finra_org/ticksizepilot/TSPrePilotSecurities.txt
    
    http://tsp.finra.org/finra_org/ticksizepilot/TSPrePilotChanges.txt

    we were given the proxy "http://proxyconfig.schwab.com/svcproxy.pac", to set it in the c# code while downloading the file.

    If we set the above proxy in Tools==>Internet Options==>LANSettings==>Automatic Configuration Section==>

    Check Use Automatic Configuration script, And enter "http://proxyconfig.schwab.com/svcproxy.pac" for address field, then  we are able to browse the above URLs and able to download,

    But the proxy need to be set in the code  while downloading the above URLs, instead of mentioning in the Tools==>Internet Options==>LAN Settings.

    Please let me know if you need more information.

    Thanks,

    Siddi.


    Siddi Khadar Vali Pothuganti

    Friday, May 20, 2016 9:49 AM
  • Hi siddi4,

    >>"WebProxy wp = new WebProxy("http://proxyconfig.schwab.com/svcproxy.pac");
    webClient.Proxy = wp"

    According to the document of WebProxy method. WebProxy method need a proxy server url as input parameter and what you input is a pac file url. It can't automatically detect the proxy server in the pac file, you have to parse the file by yourself and get the proxy server. For how to parse the pac file, please take a look at this article.

    Using PAC files proxy

    Best Regards,
    Li Wang

    Note: This response contains a reference to a third party World Wide Web site. Microsoft is providing this information as a convenience to you. Microsoft does not control these sites and has not tested any software or information found on these sites; therefore, Microsoft cannot make any representations regarding the quality, safety, or suitability of any software or information found there. There are inherent dangers in the use of any software found on the Internet, and Microsoft cautions you to make sure that you completely understand the risk before retrieving any software from the Internet.


    We are trying to better understand customer views on social support experience, so your participation in this interview project would be greatly appreciated if you have time. Thanks for helping make community forums a great place.
    Click HERE to participate the survey.

    • Marked as answer by siddi4 Wednesday, May 25, 2016 10:51 AM
    Tuesday, May 24, 2016 7:09 AM
    Moderator
  • Thank you Wang  for helping me on this.

    Thanks,

    Siddi.


    Siddi Khadar Vali Pothuganti

    Wednesday, May 25, 2016 10:52 AM
  • Wang Can you please replace all the occurrences of the URL "http://proxyconfig.schwab.com/svcproxy.pac" ,  in this post with some dummy value "http://SomeProxy.pac". since it is sensitive information.

    I came to know that Moderators can edit the posts, I am able to my comment.  If possible can you please edit in other comments also. Please let me know if you need more information.

    Thanks,

    Siddi.


    Siddi Khadar Vali Pothuganti

    Wednesday, May 25, 2016 10:56 AM
  • Hi siddi4,

    >>"WebProxy wp = new WebProxy("http://proxyconfig.schwab.com/svcproxy.pac");
    webClient.Proxy = wp"

    According to the document of WebProxy method. WebProxy method need a proxy server url as input parameter and what you input is a pac file url. It can't automatically detect the proxy server in the pac file, you have to parse the file by yourself and get the proxy server. For how to parse the pac file, please take a look at this article.

    Using PAC files proxy

    Best Regards,
    Li Wang

    Note: This response contains a reference to a third party World Wide Web site. Microsoft is providing this information as a convenience to you. Microsoft does not control these sites and has not tested any software or information found on these sites; therefore, Microsoft cannot make any representations regarding the quality, safety, or suitability of any software or information found there. There are inherent dangers in the use of any software found on the Internet, and Microsoft cautions you to make sure that you completely understand the risk before retrieving any software from the Internet.


    We are trying to better understand customer views on social support experience, so your participation in this interview project would be greatly appreciated if you have time. Thanks for helping make community forums a great place.
    Click HERE to participate the survey.

    Wang Can you please replace all the occurrences of the URL "http://proxyconfig.schwab.com/svcproxy.pac" ,  in this post with some dummy value "http://SomeProxy.pac". since it is sensitive information.

    I came to know that Moderators can edit the posts, I am able to my comment.  If possible can you please edit in other comments also. Please let me know if you need more information.

    Thanks,

    Siddi.


    Siddi Khadar Vali Pothuganti

    Wednesday, May 25, 2016 11:16 AM