elasticstream/source/elastic/worker.go

package elastic

import (
	"log"
	"time"
)

type Worker struct {
	client *Client
	index  string
	offset int
}

func NewWorker(client *Client, index string, offset int) {
	w := &Worker{
		client: client,
		index:  index,
		offset: offset,
	}

	go w.start()
}

func (w *Worker) start() {

	for {
		log.Printf("worker index=%s offset=%d size=%d\n", w.index, w.offset, w.client.cfg.BatchSize)

		dataArray, err := search(w.client.es, w.index, &w.offset, &w.client.cfg.BatchSize)
		if err != nil {
			log.Println("search() err:", err)
			time.Sleep(1 * time.Second)
			continue
		}

		for _, data := range dataArray {
			w.client.ch <- data
			w.offset++
		}
	}

}