Skip to content

Instantly share code, notes, and snippets.

@isauran

isauran/readme.md

Last active October 12, 2025 05:04
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save isauran/64e6587d85d42d202e0744212fe92bc9 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save isauran/64e6587d85d42d202e0744212fe92bc9 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. isauran revised this gist Oct 12, 2025. 1 changed file with 2 additions and 22 deletions.
    24 changes: 2 additions & 22 deletions readme.md
    View file
    Original file line number Diff line number Diff line change
    @@ -1,22 +1,2 @@
    Go HTTP Client: Example Multipart Streaming (from server1 to server2) behavior using only standard Go libraries (net/http, mime/multipart, io.Pipe)

    ### Streaming & Memory Efficiency
    ```go
    // Stream large files without loading into memory
    sourceResp, err := http.Get("https://example.com/large-file.zip")
    if err != nil {
    return err
    }
    defer sourceResp.Body.Close()

    // Stream directly to upload endpoint
    uploadResp, err := client.Multipart(ctx, "/upload").
    File("archive", "backup.zip", sourceResp.Body). // Stream without buffering
    Param("category", "backups").
    Send()
    ```

    ## Examples
    See the [github.com/nativebpm/http-client/blob/master/examples](https://github.com/nativebpm/http-client/blob/master/examples) directory.
    - [Streaming multipart](https://github.com/nativebpm/http-client/blob/master/examples/multipart_streaming_example)
    - [Standard library comparison](https://github.com/nativebpm/http-client/blob/master/examples/multipart_streaming_example/multipart_straming_without_fluent_api)
    - [Streaming multipart upload](https://github.com/nativebpm/httpstream/tree/main/examples/multipart_streaming_example)
    - [Without fluent API (for comparison)](https://github.com/nativebpm/httpstream/tree/main/examples/multipart_streaming_example/multipart_streaming_without_fluent_api)
  2. isauran revised this gist Oct 6, 2025. 1 changed file with 2 additions and 0 deletions.
    2 changes: 2 additions & 0 deletions readme.md
    View file
    Original file line number Diff line number Diff line change
    @@ -1,3 +1,5 @@
    Go HTTP Client: Example Multipart Streaming (from server1 to server2) behavior using only standard Go libraries (net/http, mime/multipart, io.Pipe)

    ### Streaming & Memory Efficiency
    ```go
    // Stream large files without loading into memory
  3. isauran revised this gist Oct 6, 2025. 4 changed files with 20 additions and 215 deletions.
    123 changes: 0 additions & 123 deletions main.go
    View file
    Original file line number Diff line number Diff line change
    @@ -1,123 +0,0 @@
    package main

    import (
    "context"
    "fmt"
    "io"
    "log/slog"
    "mime"
    "mime/multipart"
    "net/http"
    "runtime"
    "time"
    // "github.com/nativebpm/http-client/examples/multipart_streaming_example/middleware"
    )

    func main() {
    var m runtime.MemStats
    runtime.ReadMemStats(&m)
    fmt.Printf("Before streaming: Alloc=%d KB, TotalAlloc=%d KB\n", m.Alloc/1024, m.TotalAlloc/1024)

    httpClient := &http.Client{Timeout: 30 * time.Second}
    logger := slog.Default()

    // server1Client: standard client with progress middleware for GET
    server1Client := *httpClient
    // server1Client.Transport = middleware.ProgressMiddleware(logger.WithGroup("server1"))(http.DefaultTransport)

    // server2Client: standard client with upload progress middleware for POST
    server2Client := *httpClient
    // server2Client.Transport = middleware.UploadProgressMiddleware(logger.WithGroup("server2"))(http.DefaultTransport)

    // GET /file from server1
    ctx1, cancel1 := context.WithTimeout(context.Background(), 30*time.Second)
    defer cancel1()
    req1, err := http.NewRequestWithContext(ctx1, "GET", "http://localhost:8080/file", nil)
    if err != nil {
    logger.Error("Failed to create GET request", "error", err)
    return
    }
    server1Resp, err := server1Client.Do(req1)
    if err != nil {
    logger.Error("Failed to get file from server1", "error", err)
    return
    }
    defer server1Resp.Body.Close()

    if server1Resp.StatusCode != http.StatusOK {
    logger.Error("Server1 returned status", "status", server1Resp.Status)
    return
    }

    runtime.ReadMemStats(&m)
    fmt.Printf("After GET (before upload): Alloc=%d KB, TotalAlloc=%d KB\n", m.Alloc/1024, m.TotalAlloc/1024)

    ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
    defer cancel()

    // Streaming multipart upload to server2
    pr, pw := io.Pipe()
    mw := multipart.NewWriter(pw)

    go func() {
    defer pw.Close()
    defer mw.Close()

    filename := filename(server1Resp.Header, "default_filename")
    part, err := mw.CreateFormFile("file", filename)
    if err != nil {
    pw.CloseWithError(err)
    return
    }
    _, err = io.Copy(part, server1Resp.Body)
    if err != nil {
    pw.CloseWithError(err)
    }
    }()

    req2, err := http.NewRequestWithContext(ctx, "POST", "http://localhost:8081/upload", pr)
    if err != nil {
    logger.Error("Failed to create POST request", "error", err)
    return
    }
    req2.Header.Set("Content-Type", mw.FormDataContentType())

    server2Resp, err := server2Client.Do(req2)
    if err != nil {
    logger.Error("Failed to upload file", "error", err)
    return
    }
    defer server2Resp.Body.Close()

    if server2Resp.StatusCode != http.StatusOK {
    logger.Error("Upload failed with status", "status", server2Resp.Status)
    return
    }

    runtime.ReadMemStats(&m)
    fmt.Printf("After upload (before reading response): Alloc=%d KB, TotalAlloc=%d KB\n", m.Alloc/1024, m.TotalAlloc/1024)

    body, err := io.ReadAll(server2Resp.Body)
    if err != nil {
    logger.Error("Failed to read response", "error", err)
    return
    }

    runtime.ReadMemStats(&m)
    fmt.Printf("After streaming: Alloc=%d KB, TotalAlloc=%d KB\n", m.Alloc/1024, m.TotalAlloc/1024)

    logger.Info("Upload successful", "server2Resp response", string(body))
    }

    // filename extracts filename from Content-Disposition header.
    func filename(headers http.Header, defaultName string) string {
    if v := headers.Get("Content-Disposition"); v != "" {
    _, params, err := mime.ParseMediaType(v)
    if err == nil {
    if fn, ok := params["filename"]; ok {
    return fn
    }
    }
    }
    return defaultName
    }
    20 changes: 20 additions & 0 deletions readme.md
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,20 @@
    ### Streaming & Memory Efficiency
    ```go
    // Stream large files without loading into memory
    sourceResp, err := http.Get("https://example.com/large-file.zip")
    if err != nil {
    return err
    }
    defer sourceResp.Body.Close()

    // Stream directly to upload endpoint
    uploadResp, err := client.Multipart(ctx, "/upload").
    File("archive", "backup.zip", sourceResp.Body). // Stream without buffering
    Param("category", "backups").
    Send()
    ```

    ## Examples
    See the [github.com/nativebpm/http-client/blob/master/examples](https://github.com/nativebpm/http-client/blob/master/examples) directory.
    - [Streaming multipart](https://github.com/nativebpm/http-client/blob/master/examples/multipart_streaming_example)
    - [Standard library comparison](https://github.com/nativebpm/http-client/blob/master/examples/multipart_streaming_example/multipart_straming_without_fluent_api)
    31 changes: 0 additions & 31 deletions server1.go
    View file
    Original file line number Diff line number Diff line change
    @@ -1,31 +0,0 @@
    package main

    import (
    "fmt"
    "io"
    "net/http"
    "strings"
    )

    func main() {
    http.HandleFunc("/file", func(w http.ResponseWriter, r *http.Request) {
    // Generate a large file
    w.Header().Set("Content-Type", "text/plain")
    w.Header().Set("Content-Disposition", "attachment; filename=large.txt")

    // Create a reader that generates data on the fly with line numbering
    var builder strings.Builder
    for i := 1; i <= 10000000; i++ {
    builder.WriteString(fmt.Sprintf("Line %d: This is a line in the large file.\n", i))
    }
    reader := strings.NewReader(builder.String())

    _, err := io.Copy(w, reader)
    if err != nil {
    http.Error(w, "Failed to generate file", http.StatusInternalServerError)
    }
    })

    fmt.Println("Server 1 running on :8080")
    http.ListenAndServe(":8080", nil)
    }
    61 changes: 0 additions & 61 deletions server2.go
    View file
    Original file line number Diff line number Diff line change
    @@ -1,61 +0,0 @@
    package main

    import (
    "fmt"
    "io"
    "log/slog"
    "net/http"
    "os"
    )

    type progressWriter struct {
    writer io.Writer
    logger *slog.Logger
    total int64
    reported int64
    }

    func main() {
    logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
    Level: slog.LevelInfo,
    }))

    http.HandleFunc("/upload", func(w http.ResponseWriter, r *http.Request) {
    // Parse multipart
    err := r.ParseMultipartForm(32 << 20) // 32MB max
    if err != nil {
    logger.Error("Failed to parse multipart", "error", err)
    http.Error(w, err.Error(), http.StatusBadRequest)
    return
    }

    file, header, err := r.FormFile("file")
    if err != nil {
    logger.Error("Failed to get form file", "error", err)
    http.Error(w, err.Error(), http.StatusBadRequest)
    return
    }
    defer file.Close()

    dst, err := os.Create(header.Filename)
    if err != nil {
    logger.Error("Failed to create file", "error", err)
    http.Error(w, err.Error(), http.StatusInternalServerError)
    return
    }
    defer dst.Close()

    _, err = io.Copy(dst, file)
    if err != nil {
    logger.Error("Failed to copy file", "error", err)
    http.Error(w, err.Error(), http.StatusInternalServerError)
    return
    }

    logger.Info("File saved successfully", "filename", header.Filename)
    fmt.Fprintf(w, "File %s uploaded and save", header.Filename)
    })

    fmt.Println("Server 2 running on :8081")
    http.ListenAndServe(":8081", nil)
    }
  4. isauran revised this gist Oct 6, 2025. 1 changed file with 61 additions and 0 deletions.
    61 changes: 61 additions & 0 deletions server2.go
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,61 @@
    package main

    import (
    "fmt"
    "io"
    "log/slog"
    "net/http"
    "os"
    )

    type progressWriter struct {
    writer io.Writer
    logger *slog.Logger
    total int64
    reported int64
    }

    func main() {
    logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
    Level: slog.LevelInfo,
    }))

    http.HandleFunc("/upload", func(w http.ResponseWriter, r *http.Request) {
    // Parse multipart
    err := r.ParseMultipartForm(32 << 20) // 32MB max
    if err != nil {
    logger.Error("Failed to parse multipart", "error", err)
    http.Error(w, err.Error(), http.StatusBadRequest)
    return
    }

    file, header, err := r.FormFile("file")
    if err != nil {
    logger.Error("Failed to get form file", "error", err)
    http.Error(w, err.Error(), http.StatusBadRequest)
    return
    }
    defer file.Close()

    dst, err := os.Create(header.Filename)
    if err != nil {
    logger.Error("Failed to create file", "error", err)
    http.Error(w, err.Error(), http.StatusInternalServerError)
    return
    }
    defer dst.Close()

    _, err = io.Copy(dst, file)
    if err != nil {
    logger.Error("Failed to copy file", "error", err)
    http.Error(w, err.Error(), http.StatusInternalServerError)
    return
    }

    logger.Info("File saved successfully", "filename", header.Filename)
    fmt.Fprintf(w, "File %s uploaded and save", header.Filename)
    })

    fmt.Println("Server 2 running on :8081")
    http.ListenAndServe(":8081", nil)
    }
  5. isauran revised this gist Oct 6, 2025. 1 changed file with 31 additions and 0 deletions.
    31 changes: 31 additions & 0 deletions server1.go
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,31 @@
    package main

    import (
    "fmt"
    "io"
    "net/http"
    "strings"
    )

    func main() {
    http.HandleFunc("/file", func(w http.ResponseWriter, r *http.Request) {
    // Generate a large file
    w.Header().Set("Content-Type", "text/plain")
    w.Header().Set("Content-Disposition", "attachment; filename=large.txt")

    // Create a reader that generates data on the fly with line numbering
    var builder strings.Builder
    for i := 1; i <= 10000000; i++ {
    builder.WriteString(fmt.Sprintf("Line %d: This is a line in the large file.\n", i))
    }
    reader := strings.NewReader(builder.String())

    _, err := io.Copy(w, reader)
    if err != nil {
    http.Error(w, "Failed to generate file", http.StatusInternalServerError)
    }
    })

    fmt.Println("Server 1 running on :8080")
    http.ListenAndServe(":8080", nil)
    }
  6. isauran created this gist Oct 6, 2025.
    123 changes: 123 additions & 0 deletions main.go
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,123 @@
    package main

    import (
    "context"
    "fmt"
    "io"
    "log/slog"
    "mime"
    "mime/multipart"
    "net/http"
    "runtime"
    "time"
    // "github.com/nativebpm/http-client/examples/multipart_streaming_example/middleware"
    )

    func main() {
    var m runtime.MemStats
    runtime.ReadMemStats(&m)
    fmt.Printf("Before streaming: Alloc=%d KB, TotalAlloc=%d KB\n", m.Alloc/1024, m.TotalAlloc/1024)

    httpClient := &http.Client{Timeout: 30 * time.Second}
    logger := slog.Default()

    // server1Client: standard client with progress middleware for GET
    server1Client := *httpClient
    // server1Client.Transport = middleware.ProgressMiddleware(logger.WithGroup("server1"))(http.DefaultTransport)

    // server2Client: standard client with upload progress middleware for POST
    server2Client := *httpClient
    // server2Client.Transport = middleware.UploadProgressMiddleware(logger.WithGroup("server2"))(http.DefaultTransport)

    // GET /file from server1
    ctx1, cancel1 := context.WithTimeout(context.Background(), 30*time.Second)
    defer cancel1()
    req1, err := http.NewRequestWithContext(ctx1, "GET", "http://localhost:8080/file", nil)
    if err != nil {
    logger.Error("Failed to create GET request", "error", err)
    return
    }
    server1Resp, err := server1Client.Do(req1)
    if err != nil {
    logger.Error("Failed to get file from server1", "error", err)
    return
    }
    defer server1Resp.Body.Close()

    if server1Resp.StatusCode != http.StatusOK {
    logger.Error("Server1 returned status", "status", server1Resp.Status)
    return
    }

    runtime.ReadMemStats(&m)
    fmt.Printf("After GET (before upload): Alloc=%d KB, TotalAlloc=%d KB\n", m.Alloc/1024, m.TotalAlloc/1024)

    ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
    defer cancel()

    // Streaming multipart upload to server2
    pr, pw := io.Pipe()
    mw := multipart.NewWriter(pw)

    go func() {
    defer pw.Close()
    defer mw.Close()

    filename := filename(server1Resp.Header, "default_filename")
    part, err := mw.CreateFormFile("file", filename)
    if err != nil {
    pw.CloseWithError(err)
    return
    }
    _, err = io.Copy(part, server1Resp.Body)
    if err != nil {
    pw.CloseWithError(err)
    }
    }()

    req2, err := http.NewRequestWithContext(ctx, "POST", "http://localhost:8081/upload", pr)
    if err != nil {
    logger.Error("Failed to create POST request", "error", err)
    return
    }
    req2.Header.Set("Content-Type", mw.FormDataContentType())

    server2Resp, err := server2Client.Do(req2)
    if err != nil {
    logger.Error("Failed to upload file", "error", err)
    return
    }
    defer server2Resp.Body.Close()

    if server2Resp.StatusCode != http.StatusOK {
    logger.Error("Upload failed with status", "status", server2Resp.Status)
    return
    }

    runtime.ReadMemStats(&m)
    fmt.Printf("After upload (before reading response): Alloc=%d KB, TotalAlloc=%d KB\n", m.Alloc/1024, m.TotalAlloc/1024)

    body, err := io.ReadAll(server2Resp.Body)
    if err != nil {
    logger.Error("Failed to read response", "error", err)
    return
    }

    runtime.ReadMemStats(&m)
    fmt.Printf("After streaming: Alloc=%d KB, TotalAlloc=%d KB\n", m.Alloc/1024, m.TotalAlloc/1024)

    logger.Info("Upload successful", "server2Resp response", string(body))
    }

    // filename extracts filename from Content-Disposition header.
    func filename(headers http.Header, defaultName string) string {
    if v := headers.Get("Content-Disposition"); v != "" {
    _, params, err := mime.ParseMediaType(v)
    if err == nil {
    if fn, ok := params["filename"]; ok {
    return fn
    }
    }
    }
    return defaultName
    }