c# - C# winforms 一次循环一个 4096 kB 的文件，并将 HTTPPosts 排队到互联网

Question

感谢您的阅读，我真的很感激我问了很多读过这篇文章的人。

我必须将一个物理文件切成 4096 kB 的块进行上传。我可以很容易地从“另一端”拿起它们并重新制作它们。

我很欣赏这篇文章和这篇文章中有很多信息，但我正在努力如何优雅地将这些信息组合在一起

这个想法只是将大于 4096kB 的文件切割成更小的包裹。更糟糕的是：如果有一种方法可以同步排队发送文件而不必将它们放在本地磁盘上（即仅在内存中工作）我真的很喜欢......

 //Pseudocode:
 Queue queue = new Queue();
 byte[] currentChunk ;
 int i =0;
 while(currentChunk = openFileinChunks(filename, 4194304)) //4096 kB
 {
      HTTPPoster mypost = new HTTPPoster();
      mypost.add(currentChunk);
      //need comparison here - to know when last loop is happening to use different name :-/ for this next bit
      mypost.uploadedName(filename+"."+ i.ToString().PadLeft(6, '0');
      queue.Enqueue(mypost);
      i++;
 }

 do
 {
    HTTPPoster Post = (HTTPPoster) queue.Dequeue();
    Post.Run();
 } while (queue.Count != 0);

 class HTTPPoster{
     byte[] sendable;
     string tmpname;
     public void add(byte[] toPost)
     {
          sendable = toPost;
     }
     public void uploadedName(string name)
     {
          tmpname = name;
     }
     public void Run()
     {
         //[preferably some condensed reworked code to replace this] **
     }
 }

**：使用 HTTPWebrequest 上传文件（multipart/form-data）

Answer 1

I found this article http://blogs.msdn.com/b/johan/archive/2009/03/30/1080526.aspx very handy when trying to do something similar.

The approach in that article is relatively straightforward: you set up the web request then write chunks of data to the stream. However, by setting AllowWriteStreamBuffering to false, you prevent the request from buffering the whole file in memory.

The problem with your current approach is that it appears to keep everything in memory - this might cause you problems with larger files.

What does the server component of this look like? That will probably dictate the precise method you use to send the file.

Edit:

What about this?

int bytesRead;
int bufferSize = 4194304;
byte[] buffer = new byte[bufferSize];

using (FileStream fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read))
{
    int i = 0;

    while ((bytesRead = fileStream.Read(buffer, 0, buffer.Length)) != 0)
    {
        string postFileName = Path.GetFileName(filePath) + "." + i++.ToString().PadLeft(10, '0');

        if (bytesRead < bufferSize)
        {
            postFileName = Path.GetFileName(filePath) + ".terminus";
        }

        PostFileChunk(buffer, bytesRead, postFileName);
    }
}

This will read the file in blocks of 4MB, then call the PostFileChunk method (your HTTPPoster could come in to play here) to send the data to the server.

Keep in mind that when you are writing the chunk to the post in your Run method, use the bytesRead - because we are reusing the buffer, if the last read is less than bufferSize, it will still contain some of the data from the previous read. bytesRead will ensure that these characters don't get written.

Answer 2

我的解决方案：

void LikeThat(byte[] file)
    {

        var ms = new System.IO.MemoryStream(file);
        var b=  new byte[4096];
        while (ms.Length > 0)
        {
            ms.Read(b, 0, 4096);
            //now b contains your chunk, just transmit it!
        }
    }

Answer 3

这行得通，我已将其减少到 4096 B（不是 4096kB）“块”，但原理是一样的

注意：有少量的对象引用是我自己的——我使用的一个叫做 jsonFolder 和 jsonDomain 的东西，它基本上保存了用户名、密码和 url 以及文件信息。虽然你应该能够遵循这个逻辑。我会尽量抽出时间将其发布到教程中（包括 PHP 服务器端代码和 incron 重建器）

/******
*   the terminus file format is weird
*   basically it goes like this
*       tmp/myfilename.ext.00000000001  4kB
*       tmp/myfilename.ext.00000000002  4kB
*       tmp/myfilename.ext.00000000003  4kB
*       tmp/myfilename.ext.00000000004  4kB
*       tmp/myfilename.ext.terminus     <=4kB
*   this will re-built here to 
*       dir/myfilename.ext              >16kB <20kB
*/


class classActionType_clientPush : classActionType
{
    string relpath = "";
    jsonFolder myFolder;
    jsonDomain myDomain;
    Queue<HTTPPoster> queue = new Queue<HTTPPoster>();

    /// <param name="relativeLocalFilePath">
    /// this "relativeLocalFilePath" refers to path RE: the ~localFolder~ - this class will cut the file and upload it in 4096kB chunks
    /// </param>
    /// <param name="folder">
    /// use this for the folder pathing
    /// </param>
    /// <param name="domain">
    /// this is for the credentials and the url
    /// </param>
    public void setPath(string relativeLocalFilePath, jsonFolder folder, jsonDomain domain)
    {
        string tmppath = folder.localFolder + relativeLocalFilePath;
        if (File.Exists(tmppath))
        {
            relpath = relativeLocalFilePath;
            myFolder = folder;
            myDomain = domain;
        }
        else
        {
            throw new Exception("classActionType_clientPull.setPath():tmppath \"" + tmppath + "\" does not already exist");
        }
    }

    public override void action()
    {
        if (relpath == "")
        {
            throw new Exception("classActionType_clientPull.action():relpath cannot be \"\"");
        }
        /***
         * split it into chunks and copy file to server
         */
        try
        {

            int bytesRead;
            int bufferSize = 4096;
            byte[] buffer = new byte[bufferSize];
            string filePath = myFolder.localFolder + relpath;
            using (FileStream fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read))
            {
                int i = 0;

                while ((bytesRead = fileStream.Read(buffer, 0, buffer.Length)) != 0)
                {
                    string postFileName = Path.GetFileName(filePath) + "." + i++.ToString().PadLeft(10, '0');

                    if (bytesRead < bufferSize)
                    {
                        postFileName = Path.GetFileName(filePath) + ".terminus";
                    }

                    HTTPPoster mypost = new HTTPPoster();
                    mypost.add(buffer);
                    mypost.setBytes(bytesRead);
                    mypost.setDomain(this.myDomain);
                    mypost.uploadedName(postFileName);
                    queue.Enqueue(mypost);

                    Debug.WriteLine("   nof: HTTPPoster.action() loop counter " + i + "");
                }
            }

            do
            {
                HTTPPoster Post = (HTTPPoster)queue.Dequeue();
                Post.Run();
            } while (queue.Count != 0);
        }
        catch (Exception ex)
        {
            Debug.WriteLine("   nof: HTTPPoster.action() failed\r\n" + ex.Message + "\r\n" + ex.StackTrace);
        }
    }
}

class HTTPPoster
{
    byte[] sendable;
    string filename;
    int bytes;
    jsonDomain myDomain;

    public void add(byte[] b)
    {
        sendable = b;
    }

    public void uploadedName(string p)
    {
        filename = p;
    }

    public void setDomain(jsonDomain domain)
    {
        myDomain = domain;
    }

    public void setBytes(int bytesRead)
    {
        bytes = bytesRead;
    }

    public void Run()
    {
        try
        {
            /***
             * this does the actual post (!gulp)
             */ 
            string boundary = "---------------------------" + DateTime.Now.Ticks.ToString("x");
            byte[] boundarybytes = System.Text.Encoding.ASCII.GetBytes("\r\n--" + boundary + "\r\n");

            HttpWebRequest wr = (HttpWebRequest)WebRequest.Create(myDomain.url + "/io/clientPush.php");
            ServicePointManager.ServerCertificateValidationCallback = delegate { return true; };
            wr.ContentType = "multipart/form-data; boundary=" + boundary;
            wr.Method = "POST";
            wr.KeepAlive = true;
            wr.Credentials = System.Net.CredentialCache.DefaultCredentials;

            Stream rs = wr.GetRequestStream();

            string formdataTemplate = "Content-Disposition: form-data; name=\"{0}\"\r\n\r\n{1}";

            string formitem;
            byte[] formitembytes;
            classHasher hash = new classHasher();

            rs.Write(boundarybytes, 0, boundarybytes.Length);
            formitem = string.Format(formdataTemplate, "username", myDomain.user);
            formitembytes = System.Text.Encoding.UTF8.GetBytes(formitem);
            rs.Write(formitembytes, 0, formitembytes.Length);

            rs.Write(boundarybytes, 0, boundarybytes.Length);
            formitem = string.Format(formdataTemplate, "password", hash.Decrypt(myDomain.password, "saltysaltsalt"));
            formitembytes = System.Text.Encoding.UTF8.GetBytes(formitem);
            rs.Write(formitembytes, 0, formitembytes.Length);

            rs.Write(boundarybytes, 0, boundarybytes.Length);

            string headerTemplate = "Content-Disposition: form-data; name=\"{0}\"; filename=\"{1}\"\r\nContent-Type: {2}\r\n\r\n";
            string header = string.Format(headerTemplate, "file", filename, "multipart/mixed");
            byte[] headerbytes = System.Text.Encoding.UTF8.GetBytes(header);
            rs.Write(headerbytes, 0, headerbytes.Length);

             rs.Write(sendable, 0, bytes);

            byte[] trailer = System.Text.Encoding.ASCII.GetBytes("\r\n--" + boundary + "--\r\n");
            rs.Write(trailer, 0, trailer.Length);
            rs.Close();

            WebResponse wresp = null;
            try
            {
                wresp = wr.GetResponse();
                Stream myStream = wresp.GetResponseStream();
                StreamReader myReader = new StreamReader(myStream);
                Debug.WriteLine("   nof: HTTPPoster.Run() all ok \r\n" + myReader.ReadToEnd());
            }
            catch (Exception ex)
            {
                Debug.WriteLine("   nof: HTTPPoster.Run() finished\r\n" + ex.Message + "\r\n" + ex.StackTrace);
                if (wresp != null)
                {
                    wresp.Close();
                    wresp = null;
                }
            }
            finally
            {
                wr = null;
            }
        }
        catch (Exception ex)
        {
            Debug.WriteLine("   nof: HTTPPoster.Run() !ERROR! \r\n" + ex.Message + "\r\n" + ex.StackTrace);               
        }
    }
}