FAQ
I have a problem with http.Get(): it becomes slow :(

My program:
- do an http request on a local image file (40ko, from apache)
- store the result (body) in memory (slice of byte)
- do a system call ("ls")
- loop

The first request is "fast": 1 ms
After 10-20 sec: 40 ms
After 30-40 sec: 200 ms

If you comment the code related to the system call, the http request is
very fast: < 1 ms

If you comment the comment related to the cache/store, the http request
takes 4-5 ms


package main

import (
"fmt"
"io/ioutil"
"net/http"
"os/exec"
"time"
)

func main() {
concurrency := 10
iteration := 1000
done := make(chan bool)
for i := 0; i < concurrency; i++ {
go func() {
worker(iteration, done)
}()
}
for i := 0; i < concurrency; i++ {
<- done
}
}

func worker(iteration int, done chan bool) {
store := make([][]byte, 0)
for j := 0; j < iteration; j++ {
var err error

store, err = doRequest(store)
if err != nil {
fmt.Println(err)
continue
}

err = doCommand()
if err != nil {
fmt.Println(err)
continue
}
}
done <- true
}

func doRequest(storeIn [][]byte) (store [][]byte, err error) {
store = storeIn

t := time.Now()
response, err := http.Get("http://localhost/image.jpg")
fmt.Println(time.Now().Sub(t))
if err != nil {
return
}
defer response.Body.Close()

data, err := ioutil.ReadAll(response.Body)
if err != nil {
return
}
store = append(store, data)

return
}

func doCommand() (err error) {
cmd := exec.Command("ls")
err = cmd.Run()
return
}

--
You received this message because you are subscribed to the Google Groups "golang-nuts" group.
To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
For more options, visit https://groups.google.com/groups/opt_out.

Search Discussions

  • Brandon Chen at Jun 14, 2013 at 1:51 pm
    if you store all results in memory that will allocate a lot of space and
    make garbage collector to scan memory frequently.
    and you should not use ioutil.ReadAll to read response.Body in benchmark.
      reuse a byte.buffer in worker is more memory efficiency.

    you may like to see my project for the same purpose :)
    http://github.com/parkghost/gohttpbench

    On Friday, June 14, 2013 9:08:18 PM UTC+8, Pierre Durand wrote:

    I have a problem with http.Get(): it becomes slow :(

    My program:
    - do an http request on a local image file (40ko, from apache)
    - store the result (body) in memory (slice of byte)
    - do a system call ("ls")
    - loop

    The first request is "fast": 1 ms
    After 10-20 sec: 40 ms
    After 30-40 sec: 200 ms

    If you comment the code related to the system call, the http request is
    very fast: < 1 ms

    If you comment the comment related to the cache/store, the http request
    takes 4-5 ms


    package main

    import (
    "fmt"
    "io/ioutil"
    "net/http"
    "os/exec"
    "time"
    )

    func main() {
    concurrency := 10
    iteration := 1000
    done := make(chan bool)
    for i := 0; i < concurrency; i++ {
    go func() {
    worker(iteration, done)
    }()
    }
    for i := 0; i < concurrency; i++ {
    <- done
    }
    }

    func worker(iteration int, done chan bool) {
    store := make([][]byte, 0)
    for j := 0; j < iteration; j++ {
    var err error

    store, err = doRequest(store)
    if err != nil {
    fmt.Println(err)
    continue
    }

    err = doCommand()
    if err != nil {
    fmt.Println(err)
    continue
    }
    }
    done <- true
    }

    func doRequest(storeIn [][]byte) (store [][]byte, err error) {
    store = storeIn

    t := time.Now()
    response, err := http.Get("http://localhost/image.jpg")
    fmt.Println(time.Now().Sub(t))
    if err != nil {
    return
    }
    defer response.Body.Close()

    data, err := ioutil.ReadAll(response.Body)
    if err != nil {
    return
    }
    store = append(store, data)

    return
    }

    func doCommand() (err error) {
    cmd := exec.Command("ls")
    err = cmd.Run()
    return
    }
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Dave Cheney at Jun 15, 2013 at 9:30 am
    Can you please run your program with GOGCTRACE=1 and post the results.
    This may help to confirm the suspicion that gc pauses are the problem.
    On Fri, Jun 14, 2013 at 11:08 PM, Pierre Durand wrote:
    I have a problem with http.Get(): it becomes slow :(

    My program:
    - do an http request on a local image file (40ko, from apache)
    - store the result (body) in memory (slice of byte)
    - do a system call ("ls")
    - loop

    The first request is "fast": 1 ms
    After 10-20 sec: 40 ms
    After 30-40 sec: 200 ms

    If you comment the code related to the system call, the http request is very
    fast: < 1 ms

    If you comment the comment related to the cache/store, the http request
    takes 4-5 ms


    package main

    import (
    "fmt"
    "io/ioutil"
    "net/http"
    "os/exec"
    "time"
    )

    func main() {
    concurrency := 10
    iteration := 1000
    done := make(chan bool)
    for i := 0; i < concurrency; i++ {
    go func() {
    worker(iteration, done)
    }()
    }
    for i := 0; i < concurrency; i++ {
    <- done
    }
    }

    func worker(iteration int, done chan bool) {
    store := make([][]byte, 0)
    for j := 0; j < iteration; j++ {
    var err error

    store, err = doRequest(store)
    if err != nil {
    fmt.Println(err)
    continue
    }

    err = doCommand()
    if err != nil {
    fmt.Println(err)
    continue
    }
    }
    done <- true
    }

    func doRequest(storeIn [][]byte) (store [][]byte, err error) {
    store = storeIn

    t := time.Now()
    response, err := http.Get("http://localhost/image.jpg")
    fmt.Println(time.Now().Sub(t))
    if err != nil {
    return
    }
    defer response.Body.Close()

    data, err := ioutil.ReadAll(response.Body)
    if err != nil {
    return
    }
    store = append(store, data)

    return
    }

    func doCommand() (err error) {
    cmd := exec.Command("ls")
    err = cmd.Run()
    return
    }

    --
    You received this message because you are subscribed to the Google Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an
    email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Pierre Durand at Jun 16, 2013 at 1:32 pm
    Ok, new version of my code:

    package main
    import (
    "fmt"
    "net/http"
    "os/exec"
    "time"
    )
    var fakeData = make([][]byte, 100 * 1000 * 1000)
    func main() {
    concurrency := 10
    iteration := 1000
    done := make(chan bool)
    for i := 0; i < concurrency; i++ {
    go func() {
    worker(iteration, done)
    }()
    }
    for i := 0; i < concurrency; i++ {
    <- done
    }
    }
    func worker(iteration int, done chan bool) {
    for j := 0; j < iteration; j++ {
    var err error

    err = doRequest()
    if err != nil {
    fmt.Println(err)
    continue
    }

    err = doCommand()
    if err != nil {
    fmt.Println(err)
    continue
    }
    }
    done <- true
    }
    func doRequest() (err error) {
    t := time.Now()
    response, err := http.Get("http://localhost/image.jpg")
    fmt.Println("http", time.Now().Sub(t))
    if err != nil {
    return
    }
    defer response.Body.Close()
    return
    }
    func doCommand() (err error) {
    t := time.Now()
    cmd := exec.Command("ls")
    err = cmd.Run()
    fmt.Println("cmd", time.Now().Sub(t))
    return
    }


    *When i run my code with GOGCTRACE=1*

    gc1(1): 0+0+0 ms, 0 -> 0 MB 24 -> 26 (27-1) objects, 0(0) handoff, 0(0)
    steal, 0/0/0 yields
    gc2(1): 0+0+0 ms, 1 -> 1 MB 4762 -> 2260 (4832-2572) objects, 0(0) handoff,
    0(0) steal, 0/0/0 yields
    gc3(1): 0+0+0 ms, 2 -> 1 MB 10268 -> 2430 (12864-10434) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc4(1): 0+0+0 ms, 2 -> 1 MB 12317 -> 3071 (22767-19696) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc5(1): 0+0+0 ms, 2 -> 1 MB 14594 -> 3897 (34342-30445) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc6(1): 0+0+0 ms, 2 -> 1 MB 16364 -> 5052 (46893-41841) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc7(1): 0+0+0 ms, 2 -> 1 MB 18531 -> 6806 (60477-53671) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc8(1): 0+0+0 ms, 3 -> 1 MB 20477 -> 7704 (74291-66587) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc1(1): 0+0+0 ms, 0 -> 0 MB 24 -> 26 (27-1) objects, 0(0) handoff, 0(0)
    steal, 0/0/0 yields
    gc2(1): 942+0+0 ms, 2289 -> 2289 MB 362 -> 335 (390-55) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    http 2.568601ms
    cmd 31.761521ms
    http 34.672019ms
    cmd 21.826463ms
    http 56.653344ms
    cmd 24.871041ms
    http 81.6895ms
    cmd 23.93777ms
    http 105.932605ms
    cmd 24.456854ms
    *http 131.966949ms*
    cmd 32.726602ms
    http 164.865881ms
    cmd 24.962569ms
    http 189.902009ms
    cmd 26.716692ms
    *http 216.74655ms*
    *http 247.244051ms*

    Le samedi 15 juin 2013 11:30:25 UTC+2, Dave Cheney a écrit :
    Can you please run your program with GOGCTRACE=1 and post the results.
    This may help to confirm the suspicion that gc pauses are the problem.
    On Fri, Jun 14, 2013 at 11:08 PM, Pierre Durand wrote:
    I have a problem with http.Get(): it becomes slow :(

    My program:
    - do an http request on a local image file (40ko, from apache)
    - store the result (body) in memory (slice of byte)
    - do a system call ("ls")
    - loop

    The first request is "fast": 1 ms
    After 10-20 sec: 40 ms
    After 30-40 sec: 200 ms

    If you comment the code related to the system call, the http request is very
    fast: < 1 ms

    If you comment the comment related to the cache/store, the http request
    takes 4-5 ms


    package main

    import (
    "fmt"
    "io/ioutil"
    "net/http"
    "os/exec"
    "time"
    )

    func main() {
    concurrency := 10
    iteration := 1000
    done := make(chan bool)
    for i := 0; i < concurrency; i++ {
    go func() {
    worker(iteration, done)
    }()
    }
    for i := 0; i < concurrency; i++ {
    <- done
    }
    }

    func worker(iteration int, done chan bool) {
    store := make([][]byte, 0)
    for j := 0; j < iteration; j++ {
    var err error

    store, err = doRequest(store)
    if err != nil {
    fmt.Println(err)
    continue
    }

    err = doCommand()
    if err != nil {
    fmt.Println(err)
    continue
    }
    }
    done <- true
    }

    func doRequest(storeIn [][]byte) (store [][]byte, err error) {
    store = storeIn

    t := time.Now()
    response, err := http.Get("http://localhost/image.jpg")
    fmt.Println(time.Now().Sub(t))
    if err != nil {
    return
    }
    defer response.Body.Close()

    data, err := ioutil.ReadAll(response.Body)
    if err != nil {
    return
    }
    store = append(store, data)

    return
    }

    func doCommand() (err error) {
    cmd := exec.Command("ls")
    err = cmd.Run()
    return
    }

    --
    You received this message because you are subscribed to the Google Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an
    email to golang-nuts...@googlegroups.com <javascript:>.
    For more options, visit https://groups.google.com/groups/opt_out.
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Pierre Durand at Jun 16, 2013 at 1:35 pm
    If I disable the code related to "fakeData" OR "os/exec", the http requests
    are fast (< 1ms)

    Le dimanche 16 juin 2013 15:32:36 UTC+2, Pierre Durand a écrit :
    Ok, new version of my code:

    package main
    import (
    "fmt"
    "net/http"
    "os/exec"
    "time"
    )
    var fakeData = make([][]byte, 100 * 1000 * 1000)
    func main() {
    concurrency := 10
    iteration := 1000
    done := make(chan bool)
    for i := 0; i < concurrency; i++ {
    go func() {
    worker(iteration, done)
    }()
    }
    for i := 0; i < concurrency; i++ {
    <- done
    }
    }
    func worker(iteration int, done chan bool) {
    for j := 0; j < iteration; j++ {
    var err error

    err = doRequest()
    if err != nil {
    fmt.Println(err)
    continue
    }

    err = doCommand()
    if err != nil {
    fmt.Println(err)
    continue
    }
    }
    done <- true
    }
    func doRequest() (err error) {
    t := time.Now()
    response, err := http.Get("http://localhost/image.jpg")
    fmt.Println("http", time.Now().Sub(t))
    if err != nil {
    return
    }
    defer response.Body.Close()
    return
    }
    func doCommand() (err error) {
    t := time.Now()
    cmd := exec.Command("ls")
    err = cmd.Run()
    fmt.Println("cmd", time.Now().Sub(t))
    return
    }


    *When i run my code with GOGCTRACE=1*

    gc1(1): 0+0+0 ms, 0 -> 0 MB 24 -> 26 (27-1) objects, 0(0) handoff, 0(0)
    steal, 0/0/0 yields
    gc2(1): 0+0+0 ms, 1 -> 1 MB 4762 -> 2260 (4832-2572) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc3(1): 0+0+0 ms, 2 -> 1 MB 10268 -> 2430 (12864-10434) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc4(1): 0+0+0 ms, 2 -> 1 MB 12317 -> 3071 (22767-19696) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc5(1): 0+0+0 ms, 2 -> 1 MB 14594 -> 3897 (34342-30445) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc6(1): 0+0+0 ms, 2 -> 1 MB 16364 -> 5052 (46893-41841) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc7(1): 0+0+0 ms, 2 -> 1 MB 18531 -> 6806 (60477-53671) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc8(1): 0+0+0 ms, 3 -> 1 MB 20477 -> 7704 (74291-66587) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc1(1): 0+0+0 ms, 0 -> 0 MB 24 -> 26 (27-1) objects, 0(0) handoff, 0(0)
    steal, 0/0/0 yields
    gc2(1): 942+0+0 ms, 2289 -> 2289 MB 362 -> 335 (390-55) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    http 2.568601ms
    cmd 31.761521ms
    http 34.672019ms
    cmd 21.826463ms
    http 56.653344ms
    cmd 24.871041ms
    http 81.6895ms
    cmd 23.93777ms
    http 105.932605ms
    cmd 24.456854ms
    *http 131.966949ms*
    cmd 32.726602ms
    http 164.865881ms
    cmd 24.962569ms
    http 189.902009ms
    cmd 26.716692ms
    *http 216.74655ms*
    *http 247.244051ms*

    Le samedi 15 juin 2013 11:30:25 UTC+2, Dave Cheney a écrit :
    Can you please run your program with GOGCTRACE=1 and post the results.
    This may help to confirm the suspicion that gc pauses are the problem.

    On Fri, Jun 14, 2013 at 11:08 PM, Pierre Durand <pierre...@gmail.com>
    wrote:
    I have a problem with http.Get(): it becomes slow :(

    My program:
    - do an http request on a local image file (40ko, from apache)
    - store the result (body) in memory (slice of byte)
    - do a system call ("ls")
    - loop

    The first request is "fast": 1 ms
    After 10-20 sec: 40 ms
    After 30-40 sec: 200 ms

    If you comment the code related to the system call, the http request is very
    fast: < 1 ms

    If you comment the comment related to the cache/store, the http request
    takes 4-5 ms


    package main

    import (
    "fmt"
    "io/ioutil"
    "net/http"
    "os/exec"
    "time"
    )

    func main() {
    concurrency := 10
    iteration := 1000
    done := make(chan bool)
    for i := 0; i < concurrency; i++ {
    go func() {
    worker(iteration, done)
    }()
    }
    for i := 0; i < concurrency; i++ {
    <- done
    }
    }

    func worker(iteration int, done chan bool) {
    store := make([][]byte, 0)
    for j := 0; j < iteration; j++ {
    var err error

    store, err = doRequest(store)
    if err != nil {
    fmt.Println(err)
    continue
    }

    err = doCommand()
    if err != nil {
    fmt.Println(err)
    continue
    }
    }
    done <- true
    }

    func doRequest(storeIn [][]byte) (store [][]byte, err error) {
    store = storeIn

    t := time.Now()
    response, err := http.Get("http://localhost/image.jpg")
    fmt.Println(time.Now().Sub(t))
    if err != nil {
    return
    }
    defer response.Body.Close()

    data, err := ioutil.ReadAll(response.Body)
    if err != nil {
    return
    }
    store = append(store, data)

    return
    }

    func doCommand() (err error) {
    cmd := exec.Command("ls")
    err = cmd.Run()
    return
    }

    --
    You received this message because you are subscribed to the Google Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an
    email to golang-nuts...@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Pierre Durand at Jun 16, 2013 at 1:42 pm
    With GOMAXPROCS=1
    http 253.717278*ms*
    cmd 24.485308ms

    With GOMAXPROCS=2
    http 503.618*us*
    cmd 294.062177ms

    With GOMAXPROCS=8
    http 687.168*us*
    cmd 274.374382ms

    Le dimanche 16 juin 2013 15:35:50 UTC+2, Pierre Durand a écrit :
    If I disable the code related to "fakeData" OR "os/exec", the http
    requests are fast (< 1ms)

    Le dimanche 16 juin 2013 15:32:36 UTC+2, Pierre Durand a écrit :
    Ok, new version of my code:

    package main
    import (
    "fmt"
    "net/http"
    "os/exec"
    "time"
    )
    var fakeData = make([][]byte, 100 * 1000 * 1000)
    func main() {
    concurrency := 10
    iteration := 1000
    done := make(chan bool)
    for i := 0; i < concurrency; i++ {
    go func() {
    worker(iteration, done)
    }()
    }
    for i := 0; i < concurrency; i++ {
    <- done
    }
    }
    func worker(iteration int, done chan bool) {
    for j := 0; j < iteration; j++ {
    var err error

    err = doRequest()
    if err != nil {
    fmt.Println(err)
    continue
    }

    err = doCommand()
    if err != nil {
    fmt.Println(err)
    continue
    }
    }
    done <- true
    }
    func doRequest() (err error) {
    t := time.Now()
    response, err := http.Get("http://localhost/image.jpg")
    fmt.Println("http", time.Now().Sub(t))
    if err != nil {
    return
    }
    defer response.Body.Close()
    return
    }
    func doCommand() (err error) {
    t := time.Now()
    cmd := exec.Command("ls")
    err = cmd.Run()
    fmt.Println("cmd", time.Now().Sub(t))
    return
    }


    *When i run my code with GOGCTRACE=1*

    gc1(1): 0+0+0 ms, 0 -> 0 MB 24 -> 26 (27-1) objects, 0(0) handoff, 0(0)
    steal, 0/0/0 yields
    gc2(1): 0+0+0 ms, 1 -> 1 MB 4762 -> 2260 (4832-2572) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc3(1): 0+0+0 ms, 2 -> 1 MB 10268 -> 2430 (12864-10434) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc4(1): 0+0+0 ms, 2 -> 1 MB 12317 -> 3071 (22767-19696) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc5(1): 0+0+0 ms, 2 -> 1 MB 14594 -> 3897 (34342-30445) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc6(1): 0+0+0 ms, 2 -> 1 MB 16364 -> 5052 (46893-41841) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc7(1): 0+0+0 ms, 2 -> 1 MB 18531 -> 6806 (60477-53671) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc8(1): 0+0+0 ms, 3 -> 1 MB 20477 -> 7704 (74291-66587) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc1(1): 0+0+0 ms, 0 -> 0 MB 24 -> 26 (27-1) objects, 0(0) handoff, 0(0)
    steal, 0/0/0 yields
    gc2(1): 942+0+0 ms, 2289 -> 2289 MB 362 -> 335 (390-55) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    http 2.568601ms
    cmd 31.761521ms
    http 34.672019ms
    cmd 21.826463ms
    http 56.653344ms
    cmd 24.871041ms
    http 81.6895ms
    cmd 23.93777ms
    http 105.932605ms
    cmd 24.456854ms
    *http 131.966949ms*
    cmd 32.726602ms
    http 164.865881ms
    cmd 24.962569ms
    http 189.902009ms
    cmd 26.716692ms
    *http 216.74655ms*
    *http 247.244051ms*

    Le samedi 15 juin 2013 11:30:25 UTC+2, Dave Cheney a écrit :
    Can you please run your program with GOGCTRACE=1 and post the results.
    This may help to confirm the suspicion that gc pauses are the problem.

    On Fri, Jun 14, 2013 at 11:08 PM, Pierre Durand <pierre...@gmail.com>
    wrote:
    I have a problem with http.Get(): it becomes slow :(

    My program:
    - do an http request on a local image file (40ko, from apache)
    - store the result (body) in memory (slice of byte)
    - do a system call ("ls")
    - loop

    The first request is "fast": 1 ms
    After 10-20 sec: 40 ms
    After 30-40 sec: 200 ms

    If you comment the code related to the system call, the http request is very
    fast: < 1 ms

    If you comment the comment related to the cache/store, the http request
    takes 4-5 ms


    package main

    import (
    "fmt"
    "io/ioutil"
    "net/http"
    "os/exec"
    "time"
    )

    func main() {
    concurrency := 10
    iteration := 1000
    done := make(chan bool)
    for i := 0; i < concurrency; i++ {
    go func() {
    worker(iteration, done)
    }()
    }
    for i := 0; i < concurrency; i++ {
    <- done
    }
    }

    func worker(iteration int, done chan bool) {
    store := make([][]byte, 0)
    for j := 0; j < iteration; j++ {
    var err error

    store, err = doRequest(store)
    if err != nil {
    fmt.Println(err)
    continue
    }

    err = doCommand()
    if err != nil {
    fmt.Println(err)
    continue
    }
    }
    done <- true
    }

    func doRequest(storeIn [][]byte) (store [][]byte, err error) {
    store = storeIn

    t := time.Now()
    response, err := http.Get("http://localhost/image.jpg")
    fmt.Println(time.Now().Sub(t))
    if err != nil {
    return
    }
    defer response.Body.Close()

    data, err := ioutil.ReadAll(response.Body)
    if err != nil {
    return
    }
    store = append(store, data)

    return
    }

    func doCommand() (err error) {
    cmd := exec.Command("ls")
    err = cmd.Run()
    return
    }

    --
    You received this message because you are subscribed to the Google Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an
    email to golang-nuts...@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Dave Cheney at Jun 16, 2013 at 2:23 pm
    I think you are using go run, and this is mixing in the trace data from your program and the go run tool itself. Please use go build, the try again.


    On 16/06/2013, at 23:42, Pierre Durand wrote:

    With GOMAXPROCS=1
    http 253.717278ms
    cmd 24.485308ms

    With GOMAXPROCS=2
    http 503.618us
    cmd 294.062177ms

    With GOMAXPROCS=8
    http 687.168us
    cmd 274.374382ms

    Le dimanche 16 juin 2013 15:35:50 UTC+2, Pierre Durand a écrit :
    If I disable the code related to "fakeData" OR "os/exec", the http requests are fast (< 1ms)

    Le dimanche 16 juin 2013 15:32:36 UTC+2, Pierre Durand a écrit :
    Ok, new version of my code:

    package main
    import (
    "fmt"
    "net/http"
    "os/exec"
    "time"
    )
    var fakeData = make([][]byte, 100 * 1000 * 1000)
    func main() {
    concurrency := 10
    iteration := 1000
    done := make(chan bool)
    for i := 0; i < concurrency; i++ {
    go func() {
    worker(iteration, done)
    }()
    }
    for i := 0; i < concurrency; i++ {
    <- done
    }
    }
    func worker(iteration int, done chan bool) {
    for j := 0; j < iteration; j++ {
    var err error

    err = doRequest()
    if err != nil {
    fmt.Println(err)
    continue
    }

    err = doCommand()
    if err != nil {
    fmt.Println(err)
    continue
    }
    }
    done <- true
    }
    func doRequest() (err error) {
    t := time.Now()
    response, err := http.Get("http://localhost/image.jpg")
    fmt.Println("http", time.Now().Sub(t))
    if err != nil {
    return
    }
    defer response.Body.Close()
    return
    }
    func doCommand() (err error) {
    t := time.Now()
    cmd := exec.Command("ls")
    err = cmd.Run()
    fmt.Println("cmd", time.Now().Sub(t))
    return
    }


    When i run my code with GOGCTRACE=1

    gc1(1): 0+0+0 ms, 0 -> 0 MB 24 -> 26 (27-1) objects, 0(0) handoff, 0(0) steal, 0/0/0 yields
    gc2(1): 0+0+0 ms, 1 -> 1 MB 4762 -> 2260 (4832-2572) objects, 0(0) handoff, 0(0) steal, 0/0/0 yields
    gc3(1): 0+0+0 ms, 2 -> 1 MB 10268 -> 2430 (12864-10434) objects, 0(0) handoff, 0(0) steal, 0/0/0 yields
    gc4(1): 0+0+0 ms, 2 -> 1 MB 12317 -> 3071 (22767-19696) objects, 0(0) handoff, 0(0) steal, 0/0/0 yields
    gc5(1): 0+0+0 ms, 2 -> 1 MB 14594 -> 3897 (34342-30445) objects, 0(0) handoff, 0(0) steal, 0/0/0 yields
    gc6(1): 0+0+0 ms, 2 -> 1 MB 16364 -> 5052 (46893-41841) objects, 0(0) handoff, 0(0) steal, 0/0/0 yields
    gc7(1): 0+0+0 ms, 2 -> 1 MB 18531 -> 6806 (60477-53671) objects, 0(0) handoff, 0(0) steal, 0/0/0 yields
    gc8(1): 0+0+0 ms, 3 -> 1 MB 20477 -> 7704 (74291-66587) objects, 0(0) handoff, 0(0) steal, 0/0/0 yields
    gc1(1): 0+0+0 ms, 0 -> 0 MB 24 -> 26 (27-1) objects, 0(0) handoff, 0(0) steal, 0/0/0 yields
    gc2(1): 942+0+0 ms, 2289 -> 2289 MB 362 -> 335 (390-55) objects, 0(0) handoff, 0(0) steal, 0/0/0 yields
    http 2.568601ms
    cmd 31.761521ms
    http 34.672019ms
    cmd 21.826463ms
    http 56.653344ms
    cmd 24.871041ms
    http 81.6895ms
    cmd 23.93777ms
    http 105.932605ms
    cmd 24.456854ms
    http 131.966949ms
    cmd 32.726602ms
    http 164.865881ms
    cmd 24.962569ms
    http 189.902009ms
    cmd 26.716692ms
    http 216.74655ms
    http 247.244051ms

    Le samedi 15 juin 2013 11:30:25 UTC+2, Dave Cheney a écrit :
    Can you please run your program with GOGCTRACE=1 and post the results.
    This may help to confirm the suspicion that gc pauses are the problem.
    On Fri, Jun 14, 2013 at 11:08 PM, Pierre Durand wrote:
    I have a problem with http.Get(): it becomes slow :(

    My program:
    - do an http request on a local image file (40ko, from apache)
    - store the result (body) in memory (slice of byte)
    - do a system call ("ls")
    - loop

    The first request is "fast": 1 ms
    After 10-20 sec: 40 ms
    After 30-40 sec: 200 ms

    If you comment the code related to the system call, the http request is very
    fast: < 1 ms

    If you comment the comment related to the cache/store, the http request
    takes 4-5 ms


    package main

    import (
    "fmt"
    "io/ioutil"
    "net/http"
    "os/exec"
    "time"
    )

    func main() {
    concurrency := 10
    iteration := 1000
    done := make(chan bool)
    for i := 0; i < concurrency; i++ {
    go func() {
    worker(iteration, done)
    }()
    }
    for i := 0; i < concurrency; i++ {
    <- done
    }
    }

    func worker(iteration int, done chan bool) {
    store := make([][]byte, 0)
    for j := 0; j < iteration; j++ {
    var err error

    store, err = doRequest(store)
    if err != nil {
    fmt.Println(err)
    continue
    }

    err = doCommand()
    if err != nil {
    fmt.Println(err)
    continue
    }
    }
    done <- true
    }

    func doRequest(storeIn [][]byte) (store [][]byte, err error) {
    store = storeIn

    t := time.Now()
    response, err := http.Get("http://localhost/image.jpg")
    fmt.Println(time.Now().Sub(t))
    if err != nil {
    return
    }
    defer response.Body.Close()

    data, err := ioutil.ReadAll(response.Body)
    if err != nil {
    return
    }
    store = append(store, data)

    return
    }

    func doCommand() (err error) {
    cmd := exec.Command("ls")
    err = cmd.Run()
    return
    }

    --
    You received this message because you are subscribed to the Google Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an
    email to golang-nuts...@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Dmitry Vyukov at Jun 16, 2013 at 2:25 pm
    the program's trace is:
    gc1(1): 0+0+0 ms, 0 -> 0 MB 24 -> 26 (27-1) objects, 0(0) handoff,
    0(0) steal, 0/0/0 yields
    gc2(1): 942+0+0 ms, 2289 -> 2289 MB 362 -> 335 (390-55) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields

    On Sun, Jun 16, 2013 at 6:22 PM, Dave Cheney wrote:
    I think you are using go run, and this is mixing in the trace data from your
    program and the go run tool itself. Please use go build, the try again.



    On 16/06/2013, at 23:42, Pierre Durand wrote:

    With GOMAXPROCS=1
    http 253.717278ms
    cmd 24.485308ms

    With GOMAXPROCS=2
    http 503.618us
    cmd 294.062177ms

    With GOMAXPROCS=8
    http 687.168us
    cmd 274.374382ms

    Le dimanche 16 juin 2013 15:35:50 UTC+2, Pierre Durand a écrit :
    If I disable the code related to "fakeData" OR "os/exec", the http
    requests are fast (< 1ms)

    Le dimanche 16 juin 2013 15:32:36 UTC+2, Pierre Durand a écrit :
    Ok, new version of my code:

    package main
    import (
    "fmt"
    "net/http"
    "os/exec"
    "time"
    )
    var fakeData = make([][]byte, 100 * 1000 * 1000)
    func main() {
    concurrency := 10
    iteration := 1000
    done := make(chan bool)
    for i := 0; i < concurrency; i++ {
    go func() {
    worker(iteration, done)
    }()
    }
    for i := 0; i < concurrency; i++ {
    <- done
    }
    }
    func worker(iteration int, done chan bool) {
    for j := 0; j < iteration; j++ {
    var err error

    err = doRequest()
    if err != nil {
    fmt.Println(err)
    continue
    }

    err = doCommand()
    if err != nil {
    fmt.Println(err)
    continue
    }
    }
    done <- true
    }
    func doRequest() (err error) {
    t := time.Now()
    response, err := http.Get("http://localhost/image.jpg")
    fmt.Println("http", time.Now().Sub(t))
    if err != nil {
    return
    }
    defer response.Body.Close()
    return
    }
    func doCommand() (err error) {
    t := time.Now()
    cmd := exec.Command("ls")
    err = cmd.Run()
    fmt.Println("cmd", time.Now().Sub(t))
    return
    }


    When i run my code with GOGCTRACE=1

    gc1(1): 0+0+0 ms, 0 -> 0 MB 24 -> 26 (27-1) objects, 0(0) handoff, 0(0)
    steal, 0/0/0 yields
    gc2(1): 0+0+0 ms, 1 -> 1 MB 4762 -> 2260 (4832-2572) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc3(1): 0+0+0 ms, 2 -> 1 MB 10268 -> 2430 (12864-10434) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc4(1): 0+0+0 ms, 2 -> 1 MB 12317 -> 3071 (22767-19696) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc5(1): 0+0+0 ms, 2 -> 1 MB 14594 -> 3897 (34342-30445) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc6(1): 0+0+0 ms, 2 -> 1 MB 16364 -> 5052 (46893-41841) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc7(1): 0+0+0 ms, 2 -> 1 MB 18531 -> 6806 (60477-53671) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc8(1): 0+0+0 ms, 3 -> 1 MB 20477 -> 7704 (74291-66587) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc1(1): 0+0+0 ms, 0 -> 0 MB 24 -> 26 (27-1) objects, 0(0) handoff, 0(0)
    steal, 0/0/0 yields
    gc2(1): 942+0+0 ms, 2289 -> 2289 MB 362 -> 335 (390-55) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    http 2.568601ms
    cmd 31.761521ms
    http 34.672019ms
    cmd 21.826463ms
    http 56.653344ms
    cmd 24.871041ms
    http 81.6895ms
    cmd 23.93777ms
    http 105.932605ms
    cmd 24.456854ms
    http 131.966949ms
    cmd 32.726602ms
    http 164.865881ms
    cmd 24.962569ms
    http 189.902009ms
    cmd 26.716692ms
    http 216.74655ms
    http 247.244051ms

    Le samedi 15 juin 2013 11:30:25 UTC+2, Dave Cheney a écrit :
    Can you please run your program with GOGCTRACE=1 and post the results.
    This may help to confirm the suspicion that gc pauses are the problem.

    On Fri, Jun 14, 2013 at 11:08 PM, Pierre Durand <pierre...@gmail.com>
    wrote:
    I have a problem with http.Get(): it becomes slow :(

    My program:
    - do an http request on a local image file (40ko, from apache)
    - store the result (body) in memory (slice of byte)
    - do a system call ("ls")
    - loop

    The first request is "fast": 1 ms
    After 10-20 sec: 40 ms
    After 30-40 sec: 200 ms

    If you comment the code related to the system call, the http request
    is very
    fast: < 1 ms

    If you comment the comment related to the cache/store, the http
    request
    takes 4-5 ms


    package main

    import (
    "fmt"
    "io/ioutil"
    "net/http"
    "os/exec"
    "time"
    )

    func main() {
    concurrency := 10
    iteration := 1000
    done := make(chan bool)
    for i := 0; i < concurrency; i++ {
    go func() {
    worker(iteration, done)
    }()
    }
    for i := 0; i < concurrency; i++ {
    <- done
    }
    }

    func worker(iteration int, done chan bool) {
    store := make([][]byte, 0)
    for j := 0; j < iteration; j++ {
    var err error

    store, err = doRequest(store)
    if err != nil {
    fmt.Println(err)
    continue
    }

    err = doCommand()
    if err != nil {
    fmt.Println(err)
    continue
    }
    }
    done <- true
    }

    func doRequest(storeIn [][]byte) (store [][]byte, err error) {
    store = storeIn

    t := time.Now()
    response, err := http.Get("http://localhost/image.jpg")
    fmt.Println(time.Now().Sub(t))
    if err != nil {
    return
    }
    defer response.Body.Close()

    data, err := ioutil.ReadAll(response.Body)
    if err != nil {
    return
    }
    store = append(store, data)

    return
    }

    func doCommand() (err error) {
    cmd := exec.Command("ls")
    err = cmd.Run()
    return
    }

    --
    You received this message because you are subscribed to the Google
    Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send
    an
    email to golang-nuts...@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
    --
    You received this message because you are subscribed to the Google Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an
    email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.



    --
    You received this message because you are subscribed to the Google Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an
    email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Pierre Durand at Jun 16, 2013 at 2:27 pm
    go build, then go run:

    gc1(1): 0+0+0 ms, 0 -> 0 MB 24 -> 26 (27-1) objects, 0(0) handoff, 0(0)
    steal, 0/0/0 yields
    gc2(1): 947+0+0 ms, 2289 -> 2289 MB 363 -> 336 (395-59) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    http 3.670492ms
    cmd 26.157017ms
    http 30.348704ms
    cmd 21.675649ms
    http 52.101314ms
    cmd 23.950119ms
    http 76.474504ms
    cmd 25.243252ms
    http 101.77158ms
    cmd 28.280721ms
    http 130.16794ms
    cmd 27.922582ms
    http 159.31109ms
    cmd 25.562088ms
    http 184.945917ms
    cmd 24.008813ms
    http 209.081216ms
    cmd 27.923421ms
    http 237.084015ms
    cmd 27.188528ms
    http 236.102743ms
    cmd 25.990072ms
    http *240.024721ms*
    cmd 24.148968ms
    http 240.259026ms
    cmd 25.784031ms
    http 212.348542ms
    cmd 20.525545ms
    http 261.431806ms
    cmd 24.736508ms
    http *229.777385ms*
    cmd 22.080401ms
    .....
    .....
    .....

    *After 30 seconds:*
    http 213.985908ms
    cmd 23.536825ms
    http 216.377399ms
    cmd 24.003831ms
    http 215.227435ms
    cmd 21.57304ms
    http 216.112566ms
    cmd 24.78603ms
    http 216.989756ms
    cmd 22.474236ms
    http 216.569515ms
    cmd 23.54612ms
    http 217.574992ms
    cmd 25.349937ms
    http 219.915975ms
    cmd 18.20254ms
    http 213.143903ms
    cmd 26.40005ms
    ....
    ....

    Le dimanche 16 juin 2013 16:22:54 UTC+2, Dave Cheney a écrit :
    I think you are using go run, and this is mixing in the trace data from
    your program and the go run tool itself. Please use go build, the try
    again.



    On 16/06/2013, at 23:42, Pierre Durand <pierre...@gmail.com <javascript:>>
    wrote:

    With GOMAXPROCS=1
    http 253.717278*ms*
    cmd 24.485308ms

    With GOMAXPROCS=2
    http 503.618*us*
    cmd 294.062177ms

    With GOMAXPROCS=8
    http 687.168*us*
    cmd 274.374382ms

    Le dimanche 16 juin 2013 15:35:50 UTC+2, Pierre Durand a écrit :
    If I disable the code related to "fakeData" OR "os/exec", the http
    requests are fast (< 1ms)

    Le dimanche 16 juin 2013 15:32:36 UTC+2, Pierre Durand a écrit :
    Ok, new version of my code:

    package main
    import (
    "fmt"
    "net/http"
    "os/exec"
    "time"
    )
    var fakeData = make([][]byte, 100 * 1000 * 1000)
    func main() {
    concurrency := 10
    iteration := 1000
    done := make(chan bool)
    for i := 0; i < concurrency; i++ {
    go func() {
    worker(iteration, done)
    }()
    }
    for i := 0; i < concurrency; i++ {
    <- done
    }
    }
    func worker(iteration int, done chan bool) {
    for j := 0; j < iteration; j++ {
    var err error

    err = doRequest()
    if err != nil {
    fmt.Println(err)
    continue
    }

    err = doCommand()
    if err != nil {
    fmt.Println(err)
    continue
    }
    }
    done <- true
    }
    func doRequest() (err error) {
    t := time.Now()
    response, err := http.Get("http://localhost/image.jpg")
    fmt.Println("http", time.Now().Sub(t))
    if err != nil {
    return
    }
    defer response.Body.Close()
    return
    }
    func doCommand() (err error) {
    t := time.Now()
    cmd := exec.Command("ls")
    err = cmd.Run()
    fmt.Println("cmd", time.Now().Sub(t))
    return
    }


    *When i run my code with GOGCTRACE=1*

    gc1(1): 0+0+0 ms, 0 -> 0 MB 24 -> 26 (27-1) objects, 0(0) handoff, 0(0)
    steal, 0/0/0 yields
    gc2(1): 0+0+0 ms, 1 -> 1 MB 4762 -> 2260 (4832-2572) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc3(1): 0+0+0 ms, 2 -> 1 MB 10268 -> 2430 (12864-10434) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc4(1): 0+0+0 ms, 2 -> 1 MB 12317 -> 3071 (22767-19696) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc5(1): 0+0+0 ms, 2 -> 1 MB 14594 -> 3897 (34342-30445) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc6(1): 0+0+0 ms, 2 -> 1 MB 16364 -> 5052 (46893-41841) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc7(1): 0+0+0 ms, 2 -> 1 MB 18531 -> 6806 (60477-53671) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc8(1): 0+0+0 ms, 3 -> 1 MB 20477 -> 7704 (74291-66587) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc1(1): 0+0+0 ms, 0 -> 0 MB 24 -> 26 (27-1) objects, 0(0) handoff, 0(0)
    steal, 0/0/0 yields
    gc2(1): 942+0+0 ms, 2289 -> 2289 MB 362 -> 335 (390-55) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    http 2.568601ms
    cmd 31.761521ms
    http 34.672019ms
    cmd 21.826463ms
    http 56.653344ms
    cmd 24.871041ms
    http 81.6895ms
    cmd 23.93777ms
    http 105.932605ms
    cmd 24.456854ms
    *http 131.966949ms*
    cmd 32.726602ms
    http 164.865881ms
    cmd 24.962569ms
    http 189.902009ms
    cmd 26.716692ms
    *http 216.74655ms*
    *http 247.244051ms*

    Le samedi 15 juin 2013 11:30:25 UTC+2, Dave Cheney a écrit :
    Can you please run your program with GOGCTRACE=1 and post the results.
    This may help to confirm the suspicion that gc pauses are the problem.

    On Fri, Jun 14, 2013 at 11:08 PM, Pierre Durand <pierre...@gmail.com>
    wrote:
    I have a problem with http.Get(): it becomes slow :(

    My program:
    - do an http request on a local image file (40ko, from apache)
    - store the result (body) in memory (slice of byte)
    - do a system call ("ls")
    - loop

    The first request is "fast": 1 ms
    After 10-20 sec: 40 ms
    After 30-40 sec: 200 ms

    If you comment the code related to the system call, the http request is very
    fast: < 1 ms

    If you comment the comment related to the cache/store, the http request
    takes 4-5 ms


    package main

    import (
    "fmt"
    "io/ioutil"
    "net/http"
    "os/exec"
    "time"
    )

    func main() {
    concurrency := 10
    iteration := 1000
    done := make(chan bool)
    for i := 0; i < concurrency; i++ {
    go func() {
    worker(iteration, done)
    }()
    }
    for i := 0; i < concurrency; i++ {
    <- done
    }
    }

    func worker(iteration int, done chan bool) {
    store := make([][]byte, 0)
    for j := 0; j < iteration; j++ {
    var err error

    store, err = doRequest(store)
    if err != nil {
    fmt.Println(err)
    continue
    }

    err = doCommand()
    if err != nil {
    fmt.Println(err)
    continue
    }
    }
    done <- true
    }

    func doRequest(storeIn [][]byte) (store [][]byte, err error) {
    store = storeIn

    t := time.Now()
    response, err := http.Get("http://localhost/image.jpg")
    fmt.Println(time.Now().Sub(t))
    if err != nil {
    return
    }
    defer response.Body.Close()

    data, err := ioutil.ReadAll(response.Body)
    if err != nil {
    return
    }
    store = append(store, data)

    return
    }

    func doCommand() (err error) {
    cmd := exec.Command("ls")
    err = cmd.Run()
    return
    }

    --
    You received this message because you are subscribed to the Google Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an
    email to golang-nuts...@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
    --
    You received this message because you are subscribed to the Google Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an
    email to golang-nuts...@googlegroups.com <javascript:>.
    For more options, visit https://groups.google.com/groups/opt_out.


    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Pierre Durand at Jun 16, 2013 at 3:06 pm
    The results are completely different if I run with GOMAXPROCS=1 or
    GOMAXPROCS=2

    Le dimanche 16 juin 2013 16:27:10 UTC+2, Pierre Durand a écrit :
    go build, then go run:

    gc1(1): 0+0+0 ms, 0 -> 0 MB 24 -> 26 (27-1) objects, 0(0) handoff, 0(0)
    steal, 0/0/0 yields
    gc2(1): 947+0+0 ms, 2289 -> 2289 MB 363 -> 336 (395-59) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    http 3.670492ms
    cmd 26.157017ms
    http 30.348704ms
    cmd 21.675649ms
    http 52.101314ms
    cmd 23.950119ms
    http 76.474504ms
    cmd 25.243252ms
    http 101.77158ms
    cmd 28.280721ms
    http 130.16794ms
    cmd 27.922582ms
    http 159.31109ms
    cmd 25.562088ms
    http 184.945917ms
    cmd 24.008813ms
    http 209.081216ms
    cmd 27.923421ms
    http 237.084015ms
    cmd 27.188528ms
    http 236.102743ms
    cmd 25.990072ms
    http *240.024721ms*
    cmd 24.148968ms
    http 240.259026ms
    cmd 25.784031ms
    http 212.348542ms
    cmd 20.525545ms
    http 261.431806ms
    cmd 24.736508ms
    http *229.777385ms*
    cmd 22.080401ms
    .....
    .....
    .....

    *After 30 seconds:*
    http 213.985908ms
    cmd 23.536825ms
    http 216.377399ms
    cmd 24.003831ms
    http 215.227435ms
    cmd 21.57304ms
    http 216.112566ms
    cmd 24.78603ms
    http 216.989756ms
    cmd 22.474236ms
    http 216.569515ms
    cmd 23.54612ms
    http 217.574992ms
    cmd 25.349937ms
    http 219.915975ms
    cmd 18.20254ms
    http 213.143903ms
    cmd 26.40005ms
    ....
    ....

    Le dimanche 16 juin 2013 16:22:54 UTC+2, Dave Cheney a écrit :
    I think you are using go run, and this is mixing in the trace data from
    your program and the go run tool itself. Please use go build, the try
    again.



    On 16/06/2013, at 23:42, Pierre Durand wrote:

    With GOMAXPROCS=1
    http 253.717278*ms*
    cmd 24.485308ms

    With GOMAXPROCS=2
    http 503.618*us*
    cmd 294.062177ms

    With GOMAXPROCS=8
    http 687.168*us*
    cmd 274.374382ms

    Le dimanche 16 juin 2013 15:35:50 UTC+2, Pierre Durand a écrit :
    If I disable the code related to "fakeData" OR "os/exec", the http
    requests are fast (< 1ms)

    Le dimanche 16 juin 2013 15:32:36 UTC+2, Pierre Durand a écrit :
    Ok, new version of my code:

    package main
    import (
    "fmt"
    "net/http"
    "os/exec"
    "time"
    )
    var fakeData = make([][]byte, 100 * 1000 * 1000)
    func main() {
    concurrency := 10
    iteration := 1000
    done := make(chan bool)
    for i := 0; i < concurrency; i++ {
    go func() {
    worker(iteration, done)
    }()
    }
    for i := 0; i < concurrency; i++ {
    <- done
    }
    }
    func worker(iteration int, done chan bool) {
    for j := 0; j < iteration; j++ {
    var err error

    err = doRequest()
    if err != nil {
    fmt.Println(err)
    continue
    }

    err = doCommand()
    if err != nil {
    fmt.Println(err)
    continue
    }
    }
    done <- true
    }
    func doRequest() (err error) {
    t := time.Now()
    response, err := http.Get("http://localhost/image.jpg")
    fmt.Println("http", time.Now().Sub(t))
    if err != nil {
    return
    }
    defer response.Body.Close()
    return
    }
    func doCommand() (err error) {
    t := time.Now()
    cmd := exec.Command("ls")
    err = cmd.Run()
    fmt.Println("cmd", time.Now().Sub(t))
    return
    }


    *When i run my code with GOGCTRACE=1*

    gc1(1): 0+0+0 ms, 0 -> 0 MB 24 -> 26 (27-1) objects, 0(0) handoff, 0(0)
    steal, 0/0/0 yields
    gc2(1): 0+0+0 ms, 1 -> 1 MB 4762 -> 2260 (4832-2572) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc3(1): 0+0+0 ms, 2 -> 1 MB 10268 -> 2430 (12864-10434) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc4(1): 0+0+0 ms, 2 -> 1 MB 12317 -> 3071 (22767-19696) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc5(1): 0+0+0 ms, 2 -> 1 MB 14594 -> 3897 (34342-30445) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc6(1): 0+0+0 ms, 2 -> 1 MB 16364 -> 5052 (46893-41841) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc7(1): 0+0+0 ms, 2 -> 1 MB 18531 -> 6806 (60477-53671) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc8(1): 0+0+0 ms, 3 -> 1 MB 20477 -> 7704 (74291-66587) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    gc1(1): 0+0+0 ms, 0 -> 0 MB 24 -> 26 (27-1) objects, 0(0) handoff, 0(0)
    steal, 0/0/0 yields
    gc2(1): 942+0+0 ms, 2289 -> 2289 MB 362 -> 335 (390-55) objects, 0(0)
    handoff, 0(0) steal, 0/0/0 yields
    http 2.568601ms
    cmd 31.761521ms
    http 34.672019ms
    cmd 21.826463ms
    http 56.653344ms
    cmd 24.871041ms
    http 81.6895ms
    cmd 23.93777ms
    http 105.932605ms
    cmd 24.456854ms
    *http 131.966949ms*
    cmd 32.726602ms
    http 164.865881ms
    cmd 24.962569ms
    http 189.902009ms
    cmd 26.716692ms
    *http 216.74655ms*
    *http 247.244051ms*

    Le samedi 15 juin 2013 11:30:25 UTC+2, Dave Cheney a écrit :
    Can you please run your program with GOGCTRACE=1 and post the results.
    This may help to confirm the suspicion that gc pauses are the problem.

    On Fri, Jun 14, 2013 at 11:08 PM, Pierre Durand <pierre...@gmail.com>
    wrote:
    I have a problem with http.Get(): it becomes slow :(

    My program:
    - do an http request on a local image file (40ko, from apache)
    - store the result (body) in memory (slice of byte)
    - do a system call ("ls")
    - loop

    The first request is "fast": 1 ms
    After 10-20 sec: 40 ms
    After 30-40 sec: 200 ms

    If you comment the code related to the system call, the http request is very
    fast: < 1 ms

    If you comment the comment related to the cache/store, the http request
    takes 4-5 ms


    package main

    import (
    "fmt"
    "io/ioutil"
    "net/http"
    "os/exec"
    "time"
    )

    func main() {
    concurrency := 10
    iteration := 1000
    done := make(chan bool)
    for i := 0; i < concurrency; i++ {
    go func() {
    worker(iteration, done)
    }()
    }
    for i := 0; i < concurrency; i++ {
    <- done
    }
    }

    func worker(iteration int, done chan bool) {
    store := make([][]byte, 0)
    for j := 0; j < iteration; j++ {
    var err error

    store, err = doRequest(store)
    if err != nil {
    fmt.Println(err)
    continue
    }

    err = doCommand()
    if err != nil {
    fmt.Println(err)
    continue
    }
    }
    done <- true
    }

    func doRequest(storeIn [][]byte) (store [][]byte, err error) {
    store = storeIn

    t := time.Now()
    response, err := http.Get("http://localhost/image.jpg")
    fmt.Println(time.Now().Sub(t))
    if err != nil {
    return
    }
    defer response.Body.Close()

    data, err := ioutil.ReadAll(response.Body)
    if err != nil {
    return
    }
    store = append(store, data)

    return
    }

    func doCommand() (err error) {
    cmd := exec.Command("ls")
    err = cmd.Run()
    return
    }

    --
    You received this message because you are subscribed to the Google Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an
    email to golang-nuts...@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
    --
    You received this message because you are subscribed to the Google Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an
    email to golang-nuts...@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.


    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Rémy Oudompheng at Jun 16, 2013 at 3:09 pm

    On 2013/6/16 Pierre Durand wrote:
    The results are completely different if I run with GOMAXPROCS=1 or
    GOMAXPROCS=2
    Can you use the 'time' command in order to measure how much
    User/Kernel CPU time is used by your program in each case?

    Rémy.

    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Pierre Durand at Jun 16, 2013 at 3:18 pm
    I reduced "iteration" to 100

    GOMAXPROCS=1
    http 316.95901ms
    cmd 25.901617ms
    go run test.go 1,40s user 22,77s system 92% cpu 26,201 total

    GOMAXPROCS=2
    http 798.128us
    cmd 280.51061ms
    go run test.go 1,70s user 37,11s system 133% cpu 29,140 total

    GOMAXPROCS=8
    http 673.332us
    cmd 275.273538ms
    go run test.go 1,64s user 35,89s system 133% cpu 28,029 total

    Le dimanche 16 juin 2013 17:09:10 UTC+2, Rémy Oudompheng a écrit :
    On 2013/6/16 Pierre Durand <pierre...@gmail.com <javascript:>> wrote:
    The results are completely different if I run with GOMAXPROCS=1 or
    GOMAXPROCS=2
    Can you use the 'time' command in order to measure how much
    User/Kernel CPU time is used by your program in each case?

    Rémy.
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Rémy Oudompheng at Jun 16, 2013 at 3:20 pm

    2013/6/16 Pierre Durand <pierredurand@gmail.com>:
    I reduced "iteration" to 100

    GOMAXPROCS=1
    http 316.95901ms
    cmd 25.901617ms
    go run test.go 1,40s user 22,77s system 92% cpu 26,201 total

    GOMAXPROCS=2
    http 798.128us
    cmd 280.51061ms
    go run test.go 1,70s user 37,11s system 133% cpu 29,140 total

    GOMAXPROCS=8
    http 673.332us
    cmd 275.273538ms
    go run test.go 1,64s user 35,89s system 133% cpu 28,029 total
    Can you run them under "strace -c -f" ?

    Rémy.

    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Dmitry Vyukov at Jun 16, 2013 at 3:24 pm
    The problem is with fork() call with 2.2GB heap size. It takes
    ~10-20ms and is called with RawSyscall, so nothing can happen
    meanwhile.

    On Sun, Jun 16, 2013 at 7:20 PM, Rémy Oudompheng
    wrote:
    2013/6/16 Pierre Durand <pierredurand@gmail.com>:
    I reduced "iteration" to 100

    GOMAXPROCS=1
    http 316.95901ms
    cmd 25.901617ms
    go run test.go 1,40s user 22,77s system 92% cpu 26,201 total

    GOMAXPROCS=2
    http 798.128us
    cmd 280.51061ms
    go run test.go 1,70s user 37,11s system 133% cpu 29,140 total

    GOMAXPROCS=8
    http 673.332us
    cmd 275.273538ms
    go run test.go 1,64s user 35,89s system 133% cpu 28,029 total
    Can you run them under "strace -c -f" ?

    Rémy.

    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Dave Cheney at Jun 16, 2013 at 3:25 pm
    Wow. Great analysis Dmitry.


    On 17/06/2013, at 1:23, Dmitry Vyukov wrote:

    The problem is with fork() call with 2.2GB heap size. It takes
    ~10-20ms and is called with RawSyscall, so nothing can happen
    meanwhile.

    On Sun, Jun 16, 2013 at 7:20 PM, Rémy Oudompheng
    wrote:
    2013/6/16 Pierre Durand <pierredurand@gmail.com>:
    I reduced "iteration" to 100

    GOMAXPROCS=1
    http 316.95901ms
    cmd 25.901617ms
    go run test.go 1,40s user 22,77s system 92% cpu 26,201 total

    GOMAXPROCS=2
    http 798.128us
    cmd 280.51061ms
    go run test.go 1,70s user 37,11s system 133% cpu 29,140 total

    GOMAXPROCS=8
    http 673.332us
    cmd 275.273538ms
    go run test.go 1,64s user 35,89s system 133% cpu 28,029 total
    Can you run them under "strace -c -f" ?

    Rémy.

    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Rémy Oudompheng at Jun 16, 2013 at 3:27 pm

    2013/6/16 Dmitry Vyukov <dvyukov@google.com>:
    The problem is with fork() call with 2.2GB heap size. It takes
    ~10-20ms and is called with RawSyscall, so nothing can happen
    meanwhile.
    Speaking about fork(), do you think anything can help me with issue 5517 ?

    Rémy.

    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Dmitry Vyukov at Jun 16, 2013 at 3:32 pm

    On Sun, Jun 16, 2013 at 7:27 PM, Rémy Oudompheng wrote:
    2013/6/16 Dmitry Vyukov <dvyukov@google.com>:
    The problem is with fork() call with 2.2GB heap size. It takes
    ~10-20ms and is called with RawSyscall, so nothing can happen
    meanwhile.
    Speaking about fork(), do you think anything can help me with issue 5517 ?

    Looks like the same issue.
    Fork can take significant time, currently with GOMAXPROCS=1 this
    completely blocks the process.
    I don't know why it hang completely with Go1.0.3. Probably just one of
    the fixed bugs.

    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Dmitry Vyukov at Jun 16, 2013 at 3:27 pm
    One solution may be to carefully use vfork, it should be much faster.
    Another solution may be to carefully use entersyscall/exitsyscall
    around fork, but the child process must not execute exitsyscall
    because it can deadlock.

    On Sun, Jun 16, 2013 at 7:23 PM, Dmitry Vyukov wrote:
    The problem is with fork() call with 2.2GB heap size. It takes
    ~10-20ms and is called with RawSyscall, so nothing can happen
    meanwhile.

    On Sun, Jun 16, 2013 at 7:20 PM, Rémy Oudompheng
    wrote:
    2013/6/16 Pierre Durand <pierredurand@gmail.com>:
    I reduced "iteration" to 100

    GOMAXPROCS=1
    http 316.95901ms
    cmd 25.901617ms
    go run test.go 1,40s user 22,77s system 92% cpu 26,201 total

    GOMAXPROCS=2
    http 798.128us
    cmd 280.51061ms
    go run test.go 1,70s user 37,11s system 133% cpu 29,140 total

    GOMAXPROCS=8
    http 673.332us
    cmd 275.273538ms
    go run test.go 1,64s user 35,89s system 133% cpu 28,029 total
    Can you run them under "strace -c -f" ?

    Rémy.

    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Dave Cheney at Jun 16, 2013 at 3:31 pm
    Could one of you please codify this discussion in an issue.

    Cheers

    Dave


    On 17/06/2013, at 1:27, Dmitry Vyukov wrote:

    One solution may be to carefully use vfork, it should be much faster.
    Another solution may be to carefully use entersyscall/exitsyscall
    around fork, but the child process must not execute exitsyscall
    because it can deadlock.

    On Sun, Jun 16, 2013 at 7:23 PM, Dmitry Vyukov wrote:
    The problem is with fork() call with 2.2GB heap size. It takes
    ~10-20ms and is called with RawSyscall, so nothing can happen
    meanwhile.

    On Sun, Jun 16, 2013 at 7:20 PM, Rémy Oudompheng
    wrote:
    2013/6/16 Pierre Durand <pierredurand@gmail.com>:
    I reduced "iteration" to 100

    GOMAXPROCS=1
    http 316.95901ms
    cmd 25.901617ms
    go run test.go 1,40s user 22,77s system 92% cpu 26,201 total

    GOMAXPROCS=2
    http 798.128us
    cmd 280.51061ms
    go run test.go 1,70s user 37,11s system 133% cpu 29,140 total

    GOMAXPROCS=8
    http 673.332us
    cmd 275.273538ms
    go run test.go 1,64s user 35,89s system 133% cpu 28,029 total
    Can you run them under "strace -c -f" ?

    Rémy.

    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Pierre Durand at Jun 16, 2013 at 4:46 pm
    https://code.google.com/p/go/issues/detail?id=5717

    Le dimanche 16 juin 2013 17:31:25 UTC+2, Dave Cheney a écrit :
    Could one of you please codify this discussion in an issue.

    Cheers

    Dave



    On 17/06/2013, at 1:27, Dmitry Vyukov <dvy...@google.com <javascript:>>
    wrote:
    One solution may be to carefully use vfork, it should be much faster.
    Another solution may be to carefully use entersyscall/exitsyscall
    around fork, but the child process must not execute exitsyscall
    because it can deadlock.

    On Sun, Jun 16, 2013 at 7:23 PM, Dmitry Vyukov wrote:
    The problem is with fork() call with 2.2GB heap size. It takes
    ~10-20ms and is called with RawSyscall, so nothing can happen
    meanwhile.

    On Sun, Jun 16, 2013 at 7:20 PM, Rémy Oudompheng
    <remyoud...@gmail.com <javascript:>> wrote:
    2013/6/16 Pierre Durand <pierre...@gmail.com <javascript:>>:
    I reduced "iteration" to 100

    GOMAXPROCS=1
    http 316.95901ms
    cmd 25.901617ms
    go run test.go 1,40s user 22,77s system 92% cpu 26,201 total

    GOMAXPROCS=2
    http 798.128us
    cmd 280.51061ms
    go run test.go 1,70s user 37,11s system 133% cpu 29,140 total

    GOMAXPROCS=8
    http 673.332us
    cmd 275.273538ms
    go run test.go 1,64s user 35,89s system 133% cpu 28,029 total
    Can you run them under "strace -c -f" ?

    Rémy.

    --
    You received this message because you are subscribed to the Google
    Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send
    an email to golang-nuts...@googlegroups.com <javascript:>.
    --
    You received this message because you are subscribed to the Google
    Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send
    an email to golang-nuts...@googlegroups.com <javascript:>.
    For more options, visit https://groups.google.com/groups/opt_out.
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Jim Robinson at Jun 18, 2013 at 3:51 pm

    On Sunday, June 16, 2013 8:23:35 AM UTC-7, Dmitry Vyukov wrote:

    The problem is with fork() call with 2.2GB heap size. It takes
    ~10-20ms and is called with RawSyscall, so nothing can happen
    meanwhile.

    Urgh. I was planning on doing something similar with a server I'm
    writing, where I would periodically make calls to uptime(1) to get
    ahold of the load average. Sounds like that won't work so well!


    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Dmitry Vyukov at Jun 18, 2013 at 4:04 pm

    On Tue, Jun 18, 2013 at 7:51 PM, Jim Robinson wrote:
    On Sunday, June 16, 2013 8:23:35 AM UTC-7, Dmitry Vyukov wrote:

    The problem is with fork() call with 2.2GB heap size. It takes
    ~10-20ms and is called with RawSyscall, so nothing can happen
    meanwhile.

    Urgh. I was planning on doing something similar with a server I'm
    writing, where I would periodically make calls to uptime(1) to get
    ahold of the load average. Sounds like that won't work so well!

    Please add this to issue.
    As you noted things are not that bad with GOMAXPROCS>1, fork block
    only 1 thread.

    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • James Bardin at Jun 18, 2013 at 7:56 pm

    On Tuesday, June 18, 2013 11:51:31 AM UTC-4, Jim Robinson wrote:

    Urgh. I was planning on doing something similar with a server I'm
    writing, where I would periodically make calls to uptime(1) to get
    ahold of the load average. Sounds like that won't work so well!
    Slightly OT, but on linux you can get around this easily by reading
    directly from /proc/loadavg.
    You could also use syscall.SysInfo to get the data, but I can't really the
    details of how to use that off the top of my head.

    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Mikael Gustavsson at Jun 19, 2013 at 8:57 am

    On Tuesday, June 18, 2013 5:51:31 PM UTC+2, Jim Robinson wrote:
    Urgh. I was planning on doing something similar with a server I'm
    writing, where I would periodically make calls to uptime(1) to get
    ahold of the load average. Sounds like that won't work so well!
    I'm far from an expert on this matter, but I think the situation is not as
    bad as you conclude it to be.
    Spawning a new process on every request is going to be problematic if you
    want to handle 100s or 1000s of requests per second.
    But periodically calling uptime a couple of times a second is probably ok.

    If I understand Dmitry correctly, forking is also faster if the parent
    process has a small heap. (?)
    If you expect your parent process will have a large heap, i think you can
    do the uptime checking from a different process that does nothing else but
    this and communicate with it through sockets.

    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Dave Cheney at Jun 19, 2013 at 9:00 am
    There are much less expensive ways to the the loadavg.


    On 19/06/2013, at 18:57, Mikael Gustavsson wrote:
    On Tuesday, June 18, 2013 5:51:31 PM UTC+2, Jim Robinson wrote:

    Urgh. I was planning on doing something similar with a server I'm
    writing, where I would periodically make calls to uptime(1) to get
    ahold of the load average. Sounds like that won't work so well!
    I'm far from an expert on this matter, but I think the situation is not as bad as you conclude it to be.
    Spawning a new process on every request is going to be problematic if you want to handle 100s or 1000s of requests per second.
    But periodically calling uptime a couple of times a second is probably ok.

    If I understand Dmitry correctly, forking is also faster if the parent process has a small heap. (?)
    If you expect your parent process will have a large heap, i think you can do the uptime checking from a different process that does nothing else but this and communicate with it through sockets.

    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Pierre Durand at Jun 19, 2013 at 9:18 am
    In my project, I need to call an external executable :/
    https://github.com/pierrre/imageserver/blob/master/processor/graphicsmagick/graphicsmagick.go#L68

    Le mercredi 19 juin 2013 10:59:56 UTC+2, Dave Cheney a écrit :
    There are much less expensive ways to the the loadavg.



    On 19/06/2013, at 18:57, Mikael Gustavsson <slv...@gmail.com <javascript:>>
    wrote:
    On Tuesday, June 18, 2013 5:51:31 PM UTC+2, Jim Robinson wrote:


    Urgh. I was planning on doing something similar with a server I'm
    writing, where I would periodically make calls to uptime(1) to get
    ahold of the load average. Sounds like that won't work so well!
    I'm far from an expert on this matter, but I think the situation is not as
    bad as you conclude it to be.
    Spawning a new process on every request is going to be problematic if you
    want to handle 100s or 1000s of requests per second.
    But periodically calling uptime a couple of times a second is probably ok.

    If I understand Dmitry correctly, forking is also faster if the parent
    process has a small heap. (?)
    If you expect your parent process will have a large heap, i think you can
    do the uptime checking from a different process that does nothing else but
    this and communicate with it through sockets.

    --
    You received this message because you are subscribed to the Google Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an
    email to golang-nuts...@googlegroups.com <javascript:>.
    For more options, visit https://groups.google.com/groups/opt_out.


    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Atomly at Jun 20, 2013 at 4:49 am
    Well, exactly what kind of scale are you expecting with your GraphicsMagick
    stuff? Chances are you won't be using gigs of memory and you won't be doing
    hundreds of requests per second. If this is the case, you can probably just
    use it as-is.

    Otherwise, you have a couple of options with GraphicsMagick. Several years
    ago, I was the architect at Fotolog.com, at the time one of the most
    popular social/photo sites in the world. We had millions of photos a day
    to process (we used GraphicsMagick) and during peak time it could get to
    hundreds a second. Our app was in Java and its forking was godawful as
    well. The problem was exacerbated by the fact that we were running on
    SunFire T1000 CMT (64-core) chips where synchronization and blocking were
    the deathknell for any sort of performance.

    Since you're working from Go, you might be able to write a CGO interface to
    GraphicsMagick's code.. I haven't looked into the feasibility of this at
    all, but it might be worth at least investigating.

    What we ended up doing was a gross hack-- a C++ program that used the
    Magick++ (http://www.graphicsmagick.org/Magick++/) C++ library from
    GraphicsMagick, which we then wrapped in a cgi-bin that existed behind
    Aphache. Since Apache has excellent fork/exec performance, this worked
    great for us. We passed a query string with the image name and whatever
    parameters we needed for the conversion and got back our output filenames
    in some standard spot. It sounds really hackish, but it's essentially just
    building a service oriented architecture like all the cool kids once talked
    about, but using cgi-bin instead of some sort of fancy RPC. Heh.


    :: atomly ::

    [ atomly@atomly.com : www.atomly.com : http://blog.atomly.com/ ...
    [ atomiq records : new york city : +1.347.692.8661 ...
    [ e-mail atomly-news-subscribe@atomly.com for atomly info and updates ...

    On Wed, Jun 19, 2013 at 5:18 AM, Pierre Durand wrote:

    In my project, I need to call an external executable :/

    https://github.com/pierrre/imageserver/blob/master/processor/graphicsmagick/graphicsmagick.go#L68

    Le mercredi 19 juin 2013 10:59:56 UTC+2, Dave Cheney a écrit :
    There are much less expensive ways to the the loadavg.



    On 19/06/2013, at 18:57, Mikael Gustavsson wrote:
    On Tuesday, June 18, 2013 5:51:31 PM UTC+2, Jim Robinson wrote:


    Urgh. I was planning on doing something similar with a server I'm
    writing, where I would periodically make calls to uptime(1) to get
    ahold of the load average. Sounds like that won't work so well!
    I'm far from an expert on this matter, but I think the situation is not
    as bad as you conclude it to be.
    Spawning a new process on every request is going to be problematic if you
    want to handle 100s or 1000s of requests per second.
    But periodically calling uptime a couple of times a second is probably ok.

    If I understand Dmitry correctly, forking is also faster if the parent
    process has a small heap. (?)
    If you expect your parent process will have a large heap, i think you can
    do the uptime checking from a different process that does nothing else but
    this and communicate with it through sockets.

    --
    You received this message because you are subscribed to the Google Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an
    email to golang-nuts...@**googlegroups.com.

    For more options, visit https://groups.google.com/**groups/opt_out<https://groups.google.com/groups/opt_out>
    .



    --
    You received this message because you are subscribed to the Google Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an
    email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.

    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Pierre Durand at Jun 20, 2013 at 9:41 am
    @ atomly:
    My project is an image server https://github.com/pierrre/imageserver
    It processes (resize / convert) images and stores them in a cache.
    You can use it with an http server.
    The architecture is modular, you can replace the components with your own
    implementations. (
    https://github.com/pierrre/imageserver/blob/master/example/main.go)

    Actually, I use GraphicsMagick with the command line executable, because
    it's a prototype (easier and faster to implement).
    I plan to use CGO of course.
    (If you know how to do it, your help will be welcome :D )
    With the current implementation, it can resize/convert 100-150 images/s
    with my quad core laptop

    It uses a lot of memory because you can use an "in memory" cache instead of
    redis/memcache
    https://github.com/pierrre/imageserver/blob/master/cache/memory/memory.go
    It's faster: 20-30k req/s vs 10k req/s

    Le jeudi 20 juin 2013 06:48:40 UTC+2, atomly a écrit :
    Well, exactly what kind of scale are you expecting with your
    GraphicsMagick stuff? Chances are you won't be using gigs of memory and you
    won't be doing hundreds of requests per second. If this is the case, you
    can probably just use it as-is.

    Otherwise, you have a couple of options with GraphicsMagick. Several years
    ago, I was the architect at Fotolog.com, at the time one of the most
    popular social/photo sites in the world. We had millions of photos a day
    to process (we used GraphicsMagick) and during peak time it could get to
    hundreds a second. Our app was in Java and its forking was godawful as
    well. The problem was exacerbated by the fact that we were running on
    SunFire T1000 CMT (64-core) chips where synchronization and blocking were
    the deathknell for any sort of performance.

    Since you're working from Go, you might be able to write a CGO interface
    to GraphicsMagick's code.. I haven't looked into the feasibility of this at
    all, but it might be worth at least investigating.

    What we ended up doing was a gross hack-- a C++ program that used the
    Magick++ (http://www.graphicsmagick.org/Magick++/) C++ library from
    GraphicsMagick, which we then wrapped in a cgi-bin that existed behind
    Aphache. Since Apache has excellent fork/exec performance, this worked
    great for us. We passed a query string with the image name and whatever
    parameters we needed for the conversion and got back our output filenames
    in some standard spot. It sounds really hackish, but it's essentially just
    building a service oriented architecture like all the cool kids once talked
    about, but using cgi-bin instead of some sort of fancy RPC. Heh.


    :: atomly ::

    [ ato...@atomly.com <javascript:> : www.atomly.com :
    http://blog.atomly.com/ ...
    [ atomiq records : new york city : +1.347.692.8661 ...
    [ e-mail atomly-new...@atomly.com <javascript:> for atomly info and
    updates ...


    On Wed, Jun 19, 2013 at 5:18 AM, Pierre Durand <pierre...@gmail.com<javascript:>
    wrote:
    In my project, I need to call an external executable :/

    https://github.com/pierrre/imageserver/blob/master/processor/graphicsmagick/graphicsmagick.go#L68

    Le mercredi 19 juin 2013 10:59:56 UTC+2, Dave Cheney a écrit :
    There are much less expensive ways to the the loadavg.



    On 19/06/2013, at 18:57, Mikael Gustavsson wrote:
    On Tuesday, June 18, 2013 5:51:31 PM UTC+2, Jim Robinson wrote:


    Urgh. I was planning on doing something similar with a server I'm
    writing, where I would periodically make calls to uptime(1) to get
    ahold of the load average. Sounds like that won't work so well!
    I'm far from an expert on this matter, but I think the situation is not
    as bad as you conclude it to be.
    Spawning a new process on every request is going to be problematic if
    you want to handle 100s or 1000s of requests per second.
    But periodically calling uptime a couple of times a second is probably
    ok.

    If I understand Dmitry correctly, forking is also faster if the parent
    process has a small heap. (?)
    If you expect your parent process will have a large heap, i think you
    can do the uptime checking from a different process that does nothing else
    but this and communicate with it through sockets.

    --
    You received this message because you are subscribed to the Google
    Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send
    an email to golang-nuts...@**googlegroups.com.

    For more options, visit https://groups.google.com/**groups/opt_out<https://groups.google.com/groups/opt_out>
    .



    --
    You received this message because you are subscribed to the Google Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an
    email to golang-nuts...@googlegroups.com <javascript:>.
    For more options, visit https://groups.google.com/groups/opt_out.

    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Jim Robinson at Jun 19, 2013 at 12:34 pm

    On Wednesday, June 19, 2013 1:59:56 AM UTC-7, Dave Cheney wrote:

    There are much less expensive ways to the the loadavg.
    Ok, I hadn't been aware of a cross platform method. Were you referring to
    the SysInfo data that was mentioned earlier? I had assumed I'd have to
    replicate the logic used on each platform to convert those numbers to
    the same sorts of values reported by uptime on each platform.

    Jim

    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Dave Cheney at Jun 19, 2013 at 12:38 pm
    There are only 5 platforms that Go supports that have a concept of
    LoadAvg. Tell me which ones you're targeting and I'll write you a
    function to get the loadavg.
    On Wed, Jun 19, 2013 at 10:34 PM, Jim Robinson wrote:
    On Wednesday, June 19, 2013 1:59:56 AM UTC-7, Dave Cheney wrote:

    There are much less expensive ways to the the loadavg.

    Ok, I hadn't been aware of a cross platform method. Were you referring to
    the SysInfo data that was mentioned earlier? I had assumed I'd have to
    replicate the logic used on each platform to convert those numbers to
    the same sorts of values reported by uptime on each platform.

    Jim

    --
    You received this message because you are subscribed to the Google Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an
    email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • James A. Robinson at Jun 19, 2013 at 12:44 pm

    On Wed, Jun 19, 2013 at 5:38 AM, Dave Cheney wrote:
    There are only 5 platforms that Go supports that have a concept of
    LoadAvg. Tell me which ones you're targeting and I'll write you a
    function to get the loadavg.
    I had wanted a functioning server on Linux and Darwin. Are you in fact
    suggesting the use of SysInfo? I don't have any objection to doing the
    work myself, I just hadn't known of the option.

    Jim

    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Dave Cheney at Jun 19, 2013 at 9:20 pm
    I don't know the precise mechnism, i'm just going to look at the
    source for uptime(1). On darwin it is a sysctl (from memory), on linux
    I think you have a choice.

    On Wed, Jun 19, 2013 at 10:43 PM, James A. Robinson
    wrote:
    On Wed, Jun 19, 2013 at 5:38 AM, Dave Cheney wrote:
    There are only 5 platforms that Go supports that have a concept of
    LoadAvg. Tell me which ones you're targeting and I'll write you a
    function to get the loadavg.
    I had wanted a functioning server on Linux and Darwin. Are you in fact
    suggesting the use of SysInfo? I don't have any objection to doing the
    work myself, I just hadn't known of the option.

    Jim
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Carlos Castillo at Jun 20, 2013 at 12:45 am
    Unfortunately, you can't use the sysctl method from go because the value
    you're looking to read "vm.loadavg", is a struct type, but go only supports
    string and int32 values. You could use CGO to do it correctly...
    On Wednesday, June 19, 2013 2:20:27 PM UTC-7, Dave Cheney wrote:

    I don't know the precise mechnism, i'm just going to look at the
    source for uptime(1). On darwin it is a sysctl (from memory), on linux
    I think you have a choice.

    On Wed, Jun 19, 2013 at 10:43 PM, James A. Robinson
    <jim.ro...@gmail.com <javascript:>> wrote:
    On Wed, Jun 19, 2013 at 5:38 AM, Dave Cheney wrote:
    There are only 5 platforms that Go supports that have a concept of
    LoadAvg. Tell me which ones you're targeting and I'll write you a
    function to get the loadavg.
    I had wanted a functioning server on Linux and Darwin. Are you in fact
    suggesting the use of SysInfo? I don't have any objection to doing the
    work myself, I just hadn't known of the option.

    Jim
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Dave Cheney at Jun 20, 2013 at 1:21 am
    https://github.com/davecheney/loadavg
    On Thu, Jun 20, 2013 at 10:45 AM, Carlos Castillo wrote:
    Unfortunately, you can't use the sysctl method from go because the value
    you're looking to read "vm.loadavg", is a struct type, but go only supports
    string and int32 values. You could use CGO to do it correctly...

    On Wednesday, June 19, 2013 2:20:27 PM UTC-7, Dave Cheney wrote:

    I don't know the precise mechnism, i'm just going to look at the
    source for uptime(1). On darwin it is a sysctl (from memory), on linux
    I think you have a choice.

    On Wed, Jun 19, 2013 at 10:43 PM, James A. Robinson
    wrote:
    On Wed, Jun 19, 2013 at 5:38 AM, Dave Cheney wrote:
    There are only 5 platforms that Go supports that have a concept of
    LoadAvg. Tell me which ones you're targeting and I'll write you a
    function to get the loadavg.
    I had wanted a functioning server on Linux and Darwin. Are you in fact
    suggesting the use of SysInfo? I don't have any objection to doing the
    work myself, I just hadn't known of the option.

    Jim
    --
    You received this message because you are subscribed to the Google Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an
    email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Carlos Castillo at Jun 20, 2013 at 1:32 am
    My mistake, the "string" returned by syscall.Sysctl() is not just meant for
    text, but to return a byte sequence, which can be (unsafe-ly) converted to
    a struct of our choosing.

    On Wed, Jun 19, 2013 at 6:21 PM, Dave Cheney wrote:

    https://github.com/davecheney/loadavg
    On Thu, Jun 20, 2013 at 10:45 AM, Carlos Castillo wrote:
    Unfortunately, you can't use the sysctl method from go because the value
    you're looking to read "vm.loadavg", is a struct type, but go only supports
    string and int32 values. You could use CGO to do it correctly...

    On Wednesday, June 19, 2013 2:20:27 PM UTC-7, Dave Cheney wrote:

    I don't know the precise mechnism, i'm just going to look at the
    source for uptime(1). On darwin it is a sysctl (from memory), on linux
    I think you have a choice.

    On Wed, Jun 19, 2013 at 10:43 PM, James A. Robinson
    wrote:
    On Wed, Jun 19, 2013 at 5:38 AM, Dave Cheney wrote:
    There are only 5 platforms that Go supports that have a concept of
    LoadAvg. Tell me which ones you're targeting and I'll write you a
    function to get the loadavg.
    I had wanted a functioning server on Linux and Darwin. Are you in
    fact
    suggesting the use of SysInfo? I don't have any objection to doing
    the
    work myself, I just hadn't known of the option.

    Jim
    --
    You received this message because you are subscribed to the Google Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an
    email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.


    --
    Carlos Castillo

    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Dave Cheney at Jun 20, 2013 at 1:34 am
    Yup, this is a bit of a bodge, but it'll work well enough for the OP
    to validate their hypothesis that the loadavg is useful as a
    throttling metric.
    On Thu, Jun 20, 2013 at 11:32 AM, Carlos Castillo wrote:
    My mistake, the "string" returned by syscall.Sysctl() is not just meant for
    text, but to return a byte sequence, which can be (unsafe-ly) converted to a
    struct of our choosing.

    On Wed, Jun 19, 2013 at 6:21 PM, Dave Cheney wrote:

    https://github.com/davecheney/loadavg

    On Thu, Jun 20, 2013 at 10:45 AM, Carlos Castillo <cookieo9@gmail.com>
    wrote:
    Unfortunately, you can't use the sysctl method from go because the value
    you're looking to read "vm.loadavg", is a struct type, but go only
    supports
    string and int32 values. You could use CGO to do it correctly...

    On Wednesday, June 19, 2013 2:20:27 PM UTC-7, Dave Cheney wrote:

    I don't know the precise mechnism, i'm just going to look at the
    source for uptime(1). On darwin it is a sysctl (from memory), on linux
    I think you have a choice.

    On Wed, Jun 19, 2013 at 10:43 PM, James A. Robinson
    wrote:
    On Wed, Jun 19, 2013 at 5:38 AM, Dave Cheney <da...@cheney.net>
    wrote:
    There are only 5 platforms that Go supports that have a concept of
    LoadAvg. Tell me which ones you're targeting and I'll write you a
    function to get the loadavg.
    I had wanted a functioning server on Linux and Darwin. Are you in
    fact
    suggesting the use of SysInfo? I don't have any objection to doing
    the
    work myself, I just hadn't known of the option.

    Jim
    --
    You received this message because you are subscribed to the Google
    Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send
    an
    email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.



    --
    Carlos Castillo
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Carlos Castillo at Jun 20, 2013 at 1:40 am
    Will this code also work for *BSD?
    On 2013-06-19, at 6:34 PM, Dave Cheney wrote:

    Yup, this is a bit of a bodge, but it'll work well enough for the OP
    to validate their hypothesis that the loadavg is useful as a
    throttling metric.
    On Thu, Jun 20, 2013 at 11:32 AM, Carlos Castillo wrote:
    My mistake, the "string" returned by syscall.Sysctl() is not just meant for
    text, but to return a byte sequence, which can be (unsafe-ly) converted to a
    struct of our choosing.

    On Wed, Jun 19, 2013 at 6:21 PM, Dave Cheney wrote:

    https://github.com/davecheney/loadavg

    On Thu, Jun 20, 2013 at 10:45 AM, Carlos Castillo <cookieo9@gmail.com>
    wrote:
    Unfortunately, you can't use the sysctl method from go because the value
    you're looking to read "vm.loadavg", is a struct type, but go only
    supports
    string and int32 values. You could use CGO to do it correctly...

    On Wednesday, June 19, 2013 2:20:27 PM UTC-7, Dave Cheney wrote:

    I don't know the precise mechnism, i'm just going to look at the
    source for uptime(1). On darwin it is a sysctl (from memory), on linux
    I think you have a choice.

    On Wed, Jun 19, 2013 at 10:43 PM, James A. Robinson
    wrote:
    On Wed, Jun 19, 2013 at 5:38 AM, Dave Cheney <da...@cheney.net>
    wrote:
    There are only 5 platforms that Go supports that have a concept of
    LoadAvg. Tell me which ones you're targeting and I'll write you a
    function to get the loadavg.
    I had wanted a functioning server on Linux and Darwin. Are you in
    fact
    suggesting the use of SysInfo? I don't have any objection to doing
    the
    work myself, I just hadn't known of the option.

    Jim
    --
    You received this message because you are subscribed to the Google
    Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send
    an
    email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.



    --
    Carlos Castillo
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Dave Cheney at Jun 20, 2013 at 1:43 am
    Probably, patches welcome.

    http://godoc.org/github.com/davecheney/loadavg
    On Thu, Jun 20, 2013 at 11:40 AM, Carlos Castillo wrote:
    Will this code also work for *BSD?
    On 2013-06-19, at 6:34 PM, Dave Cheney wrote:

    Yup, this is a bit of a bodge, but it'll work well enough for the OP
    to validate their hypothesis that the loadavg is useful as a
    throttling metric.
    On Thu, Jun 20, 2013 at 11:32 AM, Carlos Castillo wrote:
    My mistake, the "string" returned by syscall.Sysctl() is not just meant for
    text, but to return a byte sequence, which can be (unsafe-ly) converted to a
    struct of our choosing.

    On Wed, Jun 19, 2013 at 6:21 PM, Dave Cheney wrote:

    https://github.com/davecheney/loadavg

    On Thu, Jun 20, 2013 at 10:45 AM, Carlos Castillo <cookieo9@gmail.com>
    wrote:
    Unfortunately, you can't use the sysctl method from go because the value
    you're looking to read "vm.loadavg", is a struct type, but go only
    supports
    string and int32 values. You could use CGO to do it correctly...

    On Wednesday, June 19, 2013 2:20:27 PM UTC-7, Dave Cheney wrote:

    I don't know the precise mechnism, i'm just going to look at the
    source for uptime(1). On darwin it is a sysctl (from memory), on linux
    I think you have a choice.

    On Wed, Jun 19, 2013 at 10:43 PM, James A. Robinson
    wrote:
    On Wed, Jun 19, 2013 at 5:38 AM, Dave Cheney <da...@cheney.net>
    wrote:
    There are only 5 platforms that Go supports that have a concept of
    LoadAvg. Tell me which ones you're targeting and I'll write you a
    function to get the loadavg.
    I had wanted a functioning server on Linux and Darwin. Are you in
    fact
    suggesting the use of SysInfo? I don't have any objection to doing
    the
    work myself, I just hadn't known of the option.

    Jim
    --
    You received this message because you are subscribed to the Google
    Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send
    an
    email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.



    --
    Carlos Castillo
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Glen Newton at Jun 20, 2013 at 3:29 am
    I would suggest that you cache the load average. Have a variable that
    records the last time the loadaverage was calculated and another for the
    loadaverage for that time.
    If <N seconds (where N might be 2-20seconds, perhaps settable), then the
    memoized value is returned. If >=N seconds, get a new loadavg value and
    reset the time.
    As the time granularity of loadavg is relatively large, this should have
    minimal impact on client applications and would avoid reading too often the
    /proc/loadavg file.

    -Glen
    On Wednesday, June 19, 2013 9:43:17 PM UTC-4, Dave Cheney wrote:

    Probably, patches welcome.

    http://godoc.org/github.com/davecheney/loadavg
    On Thu, Jun 20, 2013 at 11:40 AM, Carlos Castillo wrote:
    Will this code also work for *BSD?

    On 2013-06-19, at 6:34 PM, Dave Cheney <da...@cheney.net <javascript:>>
    wrote:
    Yup, this is a bit of a bodge, but it'll work well enough for the OP
    to validate their hypothesis that the loadavg is useful as a
    throttling metric.
    On Thu, Jun 20, 2013 at 11:32 AM, Carlos Castillo wrote:
    My mistake, the "string" returned by syscall.Sysctl() is not just
    meant for
    text, but to return a byte sequence, which can be (unsafe-ly)
    converted to a
    struct of our choosing.

    On Wed, Jun 19, 2013 at 6:21 PM, Dave Cheney wrote:

    https://github.com/davecheney/loadavg

    On Thu, Jun 20, 2013 at 10:45 AM, Carlos Castillo <cook...@gmail.com<javascript:>>
    wrote:
    Unfortunately, you can't use the sysctl method from go because the
    value
    you're looking to read "vm.loadavg", is a struct type, but go only
    supports
    string and int32 values. You could use CGO to do it correctly...

    On Wednesday, June 19, 2013 2:20:27 PM UTC-7, Dave Cheney wrote:

    I don't know the precise mechnism, i'm just going to look at the
    source for uptime(1). On darwin it is a sysctl (from memory), on
    linux
    I think you have a choice.

    On Wed, Jun 19, 2013 at 10:43 PM, James A. Robinson
    wrote:
    On Wed, Jun 19, 2013 at 5:38 AM, Dave Cheney <da...@cheney.net>
    wrote:
    There are only 5 platforms that Go supports that have a concept
    of
    LoadAvg. Tell me which ones you're targeting and I'll write you a
    function to get the loadavg.
    I had wanted a functioning server on Linux and Darwin. Are you in
    fact
    suggesting the use of SysInfo? I don't have any objection to
    doing
    the
    work myself, I just hadn't known of the option.

    Jim
    --
    You received this message because you are subscribed to the Google
    Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it,
    send
    an
    email to golang-nuts...@googlegroups.com <javascript:>.
    For more options, visit https://groups.google.com/groups/opt_out.



    --
    Carlos Castillo
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Carlos Castillo at Jun 20, 2013 at 3:53 am
    I think such behaviour is beyond the scope of a simple low-level package.
    If a programmer wants such behaviour they can write the code themselves.

    BTW, reading a /proc file is very cheap, as the "file" is really data from
    the kernel, so no disk access is needed, and you are essentially copying a
    value from kernel space to user space.

    On Wed, Jun 19, 2013 at 8:29 PM, Glen Newton wrote:

    I would suggest that you cache the load average. Have a variable that
    records the last time the loadaverage was calculated and another for the
    loadaverage for that time.
    If <N seconds (where N might be 2-20seconds, perhaps settable), then the
    memoized value is returned. If >=N seconds, get a new loadavg value and
    reset the time.
    As the time granularity of loadavg is relatively large, this should have
    minimal impact on client applications and would avoid reading too often the
    /proc/loadavg file.

    -Glen

    On Wednesday, June 19, 2013 9:43:17 PM UTC-4, Dave Cheney wrote:

    Probably, patches welcome.

    http://godoc.org/github.com/**davecheney/loadavg<http://godoc.org/github.com/davecheney/loadavg>

    On Thu, Jun 20, 2013 at 11:40 AM, Carlos Castillo <cook...@gmail.com>
    wrote:
    Will this code also work for *BSD?
    On 2013-06-19, at 6:34 PM, Dave Cheney wrote:

    Yup, this is a bit of a bodge, but it'll work well enough for the OP
    to validate their hypothesis that the loadavg is useful as a
    throttling metric.

    On Thu, Jun 20, 2013 at 11:32 AM, Carlos Castillo <cook...@gmail.com>
    wrote:
    My mistake, the "string" returned by syscall.Sysctl() is not just
    meant for
    text, but to return a byte sequence, which can be (unsafe-ly)
    converted to a
    struct of our choosing.


    On Wed, Jun 19, 2013 at 6:21 PM, Dave Cheney <da...@cheney.net>
    wrote:
    https://github.com/davecheney/**loadavg<https://github.com/davecheney/loadavg>

    On Thu, Jun 20, 2013 at 10:45 AM, Carlos Castillo <cook...@gmail.com>
    wrote:
    Unfortunately, you can't use the sysctl method from go because the
    value
    you're looking to read "vm.loadavg", is a struct type, but go only
    supports
    string and int32 values. You could use CGO to do it correctly...

    On Wednesday, June 19, 2013 2:20:27 PM UTC-7, Dave Cheney wrote:

    I don't know the precise mechnism, i'm just going to look at the
    source for uptime(1). On darwin it is a sysctl (from memory), on
    linux
    I think you have a choice.

    On Wed, Jun 19, 2013 at 10:43 PM, James A. Robinson
    wrote:
    On Wed, Jun 19, 2013 at 5:38 AM, Dave Cheney <da...@cheney.net>
    wrote:
    There are only 5 platforms that Go supports that have a concept
    of
    LoadAvg. Tell me which ones you're targeting and I'll write you
    a
    function to get the loadavg.
    I had wanted a functioning server on Linux and Darwin. Are you
    in
    fact
    suggesting the use of SysInfo? I don't have any objection to
    doing
    the
    work myself, I just hadn't known of the option.

    Jim
    --
    You received this message because you are subscribed to the Google
    Groups
    "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it,
    send
    an
    email to golang-nuts...@**googlegroups.com.
    For more options, visit https://groups.google.com/**groups/opt_out<https://groups.google.com/groups/opt_out>.



    --
    Carlos Castillo

    --
    Carlos Castillo

    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • James A. Robinson at Jun 20, 2013 at 1:22 pm

    On Wed, Jun 19, 2013 at 6:21 PM, Dave Cheney wrote:
    https://github.com/davecheney/loadavg
    Thank you, that looks useful and, as you said, much less expensive
    than a fork based call. I'll try plugging into my code today!

    Jim

    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Jim Robinson at Jun 19, 2013 at 12:40 pm

    On Wednesday, June 19, 2013 1:57:05 AM UTC-7, Mikael Gustavsson wrote:

    I'm far from an expert on this matter, but I think the situation is not as
    bad as you conclude it to be.
    Spawning a new process on every request is going to be problematic if you
    want to handle 100s or 1000s of requests per second.
    But periodically calling uptime a couple of times a second is probably ok.

    If I understand Dmitry correctly, forking is also faster if the parent
    process has a small heap. (?)
    If you expect your parent process will have a large heap, i think you can
    do the uptime checking from a different process that does nothing else but
    this and communicate with it through sockets.
    Sure, what I was doing was running a goroutine that would periodically
    execute the
    command and store the load average numbers into local state. Then incoming
    channel requests could fetch the saved state. So it would be executing
    once a
    minute or something.

    Yes, I could set up a distinct server but that seems like overkill. I'll
    measure what
    happens when the command is being executed and see what, if anything, it
    does
    in my situation.

    Several people have also pointed out that they think I could get at the
    data without
    using an exec. Depending on what they are thinking about, I might be able
    to
    rework the logic.


    Jim

    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.
  • Pierre Durand at Jun 16, 2013 at 3:25 pm
    With GOMAXPROCS=1

    % time seconds usecs/call calls errors syscall
    ------ ----------- ----------- --------- --------- ----------------
      65.27 79.024087 65363 1209 1014 futex
      15.20 18.406029 3578 5144 6 select
      10.10 12.228117 12228 1000 3 fork
       5.93 7.174781 7168 1001 execve
       3.24 3.919311 3919 1000 wait4
       0.23 0.281858 20 14029 1000 read
       0.03 0.030559 2778 11 clone
       0.00 0.001768 0 20031 close
       0.00 0.001052 0 27109 mmap
       0.00 0.001034 0 14018 open
       0.00 0.000853 0 4000 write
       0.00 0.000745 0 16022 mprotect
       0.00 0.000559 0 9005 9005 access
       0.00 0.000338 0 4020 munmap
       0.00 0.000335 0 7994 6000 stat
       0.00 0.000264 0 3003 brk
       0.00 0.000250 0 2011 1011 connect
       0.00 0.000202 0 2000 getdents
       0.00 0.000154 0 12017 fstat
       0.00 0.000139 0 2000 1000 statfs
       0.00 0.000110 0 1000 2 rt_sigreturn
       0.00 0.000055 0 1000 pipe2
       0.00 0.000050 0 3000 3000 ioctl
       0.00 0.000032 0 1013 socket
       0.00 0.000024 0 2061 rt_sigaction
       0.00 0.000016 0 1000 openat
       0.00 0.000014 0 2000 epoll_ctl
       0.00 0.000013 0 1000 getpeername
       0.00 0.000013 0 1001 getrlimit
       0.00 0.000010 0 2002 setsockopt
       0.00 0.000008 0 1001 arch_prctl
       0.00 0.000000 0 1026 rt_sigprocmask
       0.00 0.000000 0 3000 dup2
       0.00 0.000000 0 2 bind
       0.00 0.000000 0 1000 getsockname
       0.00 0.000000 0 10 uname
       0.00 0.000000 0 9 sigaltstack
       0.00 0.000000 0 1 sched_getaffinity
       0.00 0.000000 0 1001 set_tid_address
       0.00 0.000000 0 260 epoll_wait
       0.00 0.000000 0 1009 set_robust_list
       0.00 0.000000 0 1 epoll_create1
    ------ ----------- ----------- --------- --------- ----------------
    100.00 121.072780 170021 22041 total

    With GOMAXPROCS=2

    % time seconds usecs/call calls errors syscall
    ------ ----------- ----------- --------- --------- ----------------
      40.47 78.726553 25396 3100 1019 futex
      13.67 26.600789 2395 11106 5 select
      13.40 26.063026 4383 5946 epoll_wait
      10.50 20.423422 1276464 16 16 restart_syscall
       9.12 17.741893 17742 1000 880 fork
       4.88 9.487761 10684 888 clone
       4.28 8.320102 593 14035 1002 read
       3.66 7.118892 7112 1001 execve
       0.02 0.038843 39 1000 wait4
       0.00 0.002213 1 2012 1012 connect
       0.00 0.001709 0 20034 close
       0.00 0.001497 0 4000 write
       0.00 0.000947 0 27114 mmap
       0.00 0.000576 0 7993 6000 stat
       0.00 0.000463 0 1000 pipe2
       0.00 0.000400 0 14020 open
       0.00 0.000372 0 16023 mprotect
       0.00 0.000363 0 4025 munmap
       0.00 0.000343 0 9005 9005 access
       0.00 0.000231 19 12 uname
       0.00 0.000215 0 1014 socket
       0.00 0.000213 0 2000 epoll_ctl
       0.00 0.000168 0 1000 getsockname
       0.00 0.000119 0 12019 fstat
       0.00 0.000094 0 2000 getdents
       0.00 0.000063 0 2000 1000 statfs
       0.00 0.000061 0 3003 brk
       0.00 0.000055 0 2002 setsockopt
       0.00 0.000038 0 1000 6 rt_sigreturn
       0.00 0.000038 0 1000 getpeername
       0.00 0.000023 0 1001 set_tid_address
       0.00 0.000017 0 1000 openat
       0.00 0.000011 0 1001 arch_prctl
       0.00 0.000005 0 1026 rt_sigprocmask
       0.00 0.000000 0 2061 rt_sigaction
       0.00 0.000000 0 3000 3000 ioctl
       0.00 0.000000 0 23 sched_yield
       0.00 0.000000 0 3746 dup2
       0.00 0.000000 0 2 bind
       0.00 0.000000 0 746 fcntl
       0.00 0.000000 0 1001 getrlimit
       0.00 0.000000 0 9 sigaltstack
       0.00 0.000000 0 1 sched_getaffinity
       0.00 0.000000 0 1009 set_robust_list
       0.00 0.000000 0 1 epoll_create1
    ------ ----------- ----------- --------- --------- ----------------
    100.00 194.531515 185995 22945 total

    Le dimanche 16 juin 2013 17:20:54 UTC+2, Rémy Oudompheng a écrit :
    2013/6/16 Pierre Durand <pierre...@gmail.com <javascript:>>:
    I reduced "iteration" to 100

    GOMAXPROCS=1
    http 316.95901ms
    cmd 25.901617ms
    go run test.go 1,40s user 22,77s system 92% cpu 26,201 total

    GOMAXPROCS=2
    http 798.128us
    cmd 280.51061ms
    go run test.go 1,70s user 37,11s system 133% cpu 29,140 total

    GOMAXPROCS=8
    http 673.332us
    cmd 275.273538ms
    go run test.go 1,64s user 35,89s system 133% cpu 28,029 total
    Can you run them under "strace -c -f" ?

    Rémy.
    --
    You received this message because you are subscribed to the Google Groups "golang-nuts" group.
    To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts+unsubscribe@googlegroups.com.
    For more options, visit https://groups.google.com/groups/opt_out.

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupgolang-nuts @
categoriesgo
postedJun 14, '13 at 1:08p
activeJun 20, '13 at 1:22p
posts43
users11
websitegolang.org

People

Translate

site design / logo © 2021 Grokbase