elasticstream/source/elastic/worker.go

package elastic

import (
	"fmt"
	"log"
	"time"

	"elasticstream/opencdc"
)

type Worker struct {
	client *Client
	index  string
	offset int
}

func NewWorker(client *Client, index string, offset int) {
	w := &Worker{
		client: client,
		index:  index,
		offset: offset,
	}

	go w.start()
}

func (w *Worker) start() {
	defer w.client.wg.Done()

	for {
		log.Printf("worker index=%s offset=%d size=%d\n", w.index, w.offset, w.client.cfg.BatchSize)

		searchResponse, err := search(w.client.es, w.index, &w.offset, &w.client.cfg.BatchSize)
		if err != nil || len(searchResponse.Hits.Hits) == 0 {
			// log.Println("search() err:", err)
			select {
			case <-w.client.shutdown:
				fmt.Println("shuting donw..")
				return

			case <-time.After(time.Second):
				continue
			}
		}

		for _, hit := range searchResponse.Hits.Hits {
			data := opencdc.Data{
				Header: opencdc.Header{
					ID:       hit.ID,
					Index:    hit.Index,
					Position: w.offset + 1,
				},
				Payload: hit.Source,
			}

			select {
			case w.client.ch <- data:
				w.offset++

			case <-w.client.shutdown:
				fmt.Println("Stopping worker...")
				return
			}
		}
	}
}