67 lines
1.1 KiB
Go
67 lines
1.1 KiB
Go
package elastic
|
|
|
|
import (
|
|
"fmt"
|
|
"log"
|
|
"time"
|
|
|
|
"elasticstream/opencdc"
|
|
)
|
|
|
|
type Worker struct {
|
|
client *Client
|
|
index string
|
|
offset int
|
|
}
|
|
|
|
func NewWorker(client *Client, index string, offset int) {
|
|
w := &Worker{
|
|
client: client,
|
|
index: index,
|
|
offset: offset,
|
|
}
|
|
|
|
go w.start()
|
|
}
|
|
|
|
func (w *Worker) start() {
|
|
defer w.client.wg.Done()
|
|
|
|
for {
|
|
log.Printf("worker index=%s offset=%d size=%d\n", w.index, w.offset, w.client.cfg.BatchSize)
|
|
|
|
searchResponse, err := search(w.client.es, w.index, &w.offset, &w.client.cfg.BatchSize)
|
|
if err != nil || len(searchResponse.Hits.Hits) == 0 {
|
|
// log.Println("search() err:", err)
|
|
select {
|
|
case <-w.client.shutdown:
|
|
fmt.Println("shuting donw..")
|
|
return
|
|
|
|
case <-time.After(time.Second):
|
|
continue
|
|
}
|
|
}
|
|
|
|
for _, hit := range searchResponse.Hits.Hits {
|
|
data := opencdc.Data{
|
|
Header: opencdc.Header{
|
|
ID: hit.ID,
|
|
Index: hit.Index,
|
|
Position: w.offset + 1,
|
|
},
|
|
Payload: hit.Source,
|
|
}
|
|
|
|
select {
|
|
case w.client.ch <- data:
|
|
w.offset++
|
|
|
|
case <-w.client.shutdown:
|
|
fmt.Println("Stopping worker...")
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|