Go: One producer many consumers

Question

So I have seen a lot of ways of implementing one consumer and many producers in Go - the classic fanIn function from the Concurrency in Go talk.

What I want is a fanOut function. It takes as a parameter a channel it reads a value from and returns a slice of channels that it writes copies of this value to.

Is there a correct/recommended way of implementing this?

Cruces

Answer 1

You pretty much described the best way to do it but here is a small sample of code that does it.

Go playground: https://play.golang.org/p/jwdtDXVHJk

package main

import (
    "fmt"
    "time"
)

func producer(iters int) <-chan int {
    c := make(chan int)
    go func() {
        for i := 0; i < iters; i++ {
            c <- i
            time.Sleep(1 * time.Second)
        }
        close(c)
    }()
    return c
}

func consumer(cin <-chan int) {
    for i := range cin {
        fmt.Println(i)
    }
}

func fanOut(ch <-chan int, size, lag int) []chan int {
    cs := make([]chan int, size)
    for i, _ := range cs {
        // The size of the channels buffer controls how far behind the recievers
        // of the fanOut channels can lag the other channels.
        cs[i] = make(chan int, lag)
    }
    go func() {
        for i := range ch {
            for _, c := range cs {
                c <- i
            }
        }
        for _, c := range cs {
            // close all our fanOut channels when the input channel is exhausted.
            close(c)
        }
    }()
    return cs
}

func fanOutUnbuffered(ch <-chan int, size int) []chan int {
    cs := make([]chan int, size)
    for i, _ := range cs {
        // The size of the channels buffer controls how far behind the recievers
        // of the fanOut channels can lag the other channels.
        cs[i] = make(chan int)
    }
    go func() {
        for i := range ch {
            for _, c := range cs {
                c <- i
            }
        }
        for _, c := range cs {
            // close all our fanOut channels when the input channel is exhausted.
            close(c)
        }
    }()
    return cs
}

func main() {
    c := producer(10)
    chans := fanOutUnbuffered(c, 3)
    go consumer(chans[0])
    go consumer(chans[1])
    consumer(chans[2])
}

The important part to note is how we close the output channels once the input channel has been exhausted. Also if one of the output channels blocks on the send it will hold up the send on the other output channels. We control the amount of lag by setting the buffer size of the channels.

Profusion

Excellent! Thank you. It was the closing of the channels that was messing me up. As a thank you and a quick reference to those needing this in the future, here's a running version: play.golang.org/p/jwdtDXVHJk – Cruces

Answer 2

This solution below is a bit contrived, but it works for me:

package main

import (
    "fmt"
    "time"
    "crypto/rand"
    "encoding/binary"
)

func handleNewChannels(arrchangen chan [](chan uint32),
                       intchangen chan (chan uint32)) {
    currarr := []chan uint32{}
    arrchangen <- currarr
    for {
        newchan := <-intchangen
        currarr = append(currarr, newchan)
        arrchangen <- currarr
    }
}

func sendToChannels(arrchangen chan [](chan uint32)) {
    tick := time.Tick(1 * time.Second)
    currarr := <-arrchangen
    for {
        select {
        case <-tick:
            sent := false
            var n uint32
            binary.Read(rand.Reader, binary.LittleEndian, &n)
            for i := 0 ; i < len(currarr) ; i++ {
                currarr[i] <- n
                sent = true
            }
            if sent {
                fmt.Println("Sent generated ", n)
            }
        case newarr := <-arrchangen:
            currarr = newarr
        }
    }
}
func handleChannel(tchan chan uint32) {
    for {
        val := <-tchan
        fmt.Println("Got the value ", val)
    }
}

func createChannels(intchangen chan (chan uint32)) {
    othertick := time.Tick(5 * time.Second)
    for {
        <-othertick
        fmt.Println("Creating new channel! ")
        newchan := make(chan uint32)
        intchangen <- newchan
        go handleChannel(newchan)
    }
}

func main() {
    arrchangen := make(chan [](chan uint32))
    intchangen := make(chan (chan uint32))
    go handleNewChannels(arrchangen, intchangen)
    go sendToChannels(arrchangen)
    createChannels(intchangen)
}

Ravid

Answer 3

First, see related question What is the neatest idiom for producer/consumer in Go? and One thread showing interest in another thread (consumer / producer). Also, take a look to producer-consumer problem. About concurrency see how to achieve concurrency In Google Go.

Neurogenic

Answer 4

We can handle multiple consumers without making the copy of channel data for each consumer.

Go playground: https://play.golang.org/p/yOKindnqiZv

package main

import (
    "fmt"
    "sync"
)

type data struct {
    msg string
    consumers int
}

func main() {
    ch := make(chan *data) // both block or non-block are ok
    var wg sync.WaitGroup
    consumerCount := 3 // specify no. of consumers

    producer := func() {
        obj := &data {
            msg: "hello everyone!",
            consumers: consumerCount,
        }
        ch <- obj
    }
    consumer := func(idx int) {
        defer wg.Done()
        obj := <-ch
        fmt.Printf("consumer %d received data %v\n", idx, obj)
        obj.consumers--
        if obj.consumers > 0 {
            ch <- obj // forward to others
        } else {
            fmt.Printf("last receiver: %d\n", idx)
        }
    }

    go producer()
    for i:=1; i<=consumerCount; i++ {
        wg.Add(1)
        go consumer(i)
    }

    wg.Wait()
}

Guardianship