package elastic import ( "fmt" "log" "time" "elasticstream/opencdc" ) type Worker struct { client *Client index string offset int } func NewWorker(client *Client, index string, offset int) { w := &Worker{ client: client, index: index, offset: offset, } go w.start() } func (w *Worker) start() { defer w.client.wg.Done() for { log.Printf("worker index=%s offset=%d size=%d\n", w.index, w.offset, w.client.cfg.BatchSize) searchResponse, err := search(w.client.es, w.index, &w.offset, &w.client.cfg.BatchSize) if err != nil || len(searchResponse.Hits.Hits) == 0 { // log.Println("search() err:", err) select { case <-w.client.shutdown: fmt.Println("shuting donw..") return case <-time.After(time.Second): continue } } for _, hit := range searchResponse.Hits.Hits { data := opencdc.Data{ Header: opencdc.Header{ ID: hit.ID, Index: hit.Index, Position: w.offset + 1, }, Payload: hit.Source, } select { case w.client.ch <- data: w.offset++ case <-w.client.shutdown: fmt.Println("Stopping worker...") return } } } }