package docsyncer import ( "context" "errors" "fmt" "sync/atomic" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/bsontype" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" conf "github.com/alibaba/MongoShake/v2/collector/configure" "github.com/alibaba/MongoShake/v2/collector/filter" utils "github.com/alibaba/MongoShake/v2/common" l "github.com/alibaba/MongoShake/v2/pkg/log" ) // DocumentSplitter pre-split the big collection into several pieces type DocumentSplitter struct { src string // source mongo address url sslRootCaFile string // source root ca ssl ns utils.NS // namespace client *utils.MongoCommunityConn readerChan chan *DocumentReader // reader chan pieceByteSize uint64 // each piece max byte size count uint64 // total document number pieceNumber int // how many piece isTimeSeries bool // ns is a time-series collection, which cannot be split } // SplitVectorResult is the result of splitVector cmd, contains multi split points. type SplitVectorResult struct { SplitKeys []bson.Raw `bson:"splitKeys"` } func NewDocumentSplitter(src, sslRootCaFile string, ns utils.NS) *DocumentSplitter { ds := &DocumentSplitter{ src: src, sslRootCaFile: sslRootCaFile, ns: ns, } // create connection var err error // disable timeout ds.client, err = utils.NewMongoCommunityConn(ds.src, conf.Options.MongoConnectMode, false, utils.ReadWriteConcernLocal, utils.ReadWriteConcernDefault, sslRootCaFile) if err != nil { l.Logger.Errorf("splitter[%s] connection mongo[%v] failed[%v]", ds, utils.BlockMongoUrlPassword(ds.src, "***"), err) return nil } // get total count var res struct { Count int64 `bson:"count"` Size float64 `bson:"size"` StorageSize float64 `bson:"storageSize"` TimeSeries *bson.Raw `bson:"timeseries"` } if err := ds.client.Client.Database(ds.ns.Database).RunCommand(nil, bson.D{{"collStats", ds.ns.Collection}}).Decode(&res); err != nil { l.Logger.Errorf("splitter[%s] connection mongo[%v] failed[%v]", ds, utils.BlockMongoUrlPassword(ds.src, "***"), err) return nil } ds.count = uint64(res.Count) ds.isTimeSeries = res.TimeSeries != nil // max pieceByteSize should be limited by maxChunkSize of splitVector cmd ds.pieceByteSize = uint64(res.Size / float64(conf.Options.FullSyncReaderParallelThread)) if ds.pieceByteSize > 1024*utils.MB { // at most 8GB per chunk ds.pieceByteSize = 1024 * utils.MB } l.Logger.Infof("NewDocumentSplitter db[%v] col[%v] res[%v], pieceByteSize[%v]", ds.ns.Database, ds.ns.Collection, res, ds.pieceByteSize) if conf.Options.FullSyncReaderParallelThread <= 1 { ds.readerChan = make(chan *DocumentReader, 1) } else { ds.readerChan = make(chan *DocumentReader, 4196) } go func() { if err := ds.Run(); err != nil { l.Logger.Panicf("%v", err) } }() return ds } func (ds *DocumentSplitter) Close() { ds.client.Close() } func (ds *DocumentSplitter) String() string { return fmt.Sprintf("DocumentSplitter src[%s] ns[%s] count[%v] pieceByteSize[%v MB] pieceNumber[%v]", utils.BlockMongoUrlPassword(ds.src, "***"), ds.ns, ds.count, ds.pieceByteSize/utils.MB, ds.pieceNumber) } func (ds *DocumentSplitter) Run() error { // close channel defer close(ds.readerChan) // disable split if conf.Options.FullSyncReaderParallelThread <= 1 { l.Logger.Infof("splitter[%s] disable split or no need", ds) ds.readerChan <- NewDocumentReader(0, ds.src, ds.ns, "", nil, nil, ds.sslRootCaFile) l.Logger.Infof("splitter[%s] exits", ds) return nil } // A time-series logical view has no _id index, so neither the min/max probe // below nor splitVector can be served from one: both end up as a blocking sort // over every unpacked measurement, which spills to disk and can exhaust the // source's free space (#993). Read the whole collection as a single piece. if ds.isTimeSeries { l.Logger.Infof("splitter[%s] is a time-series collection, give up parallel fetching", ds) ds.readerChan <- NewDocumentReader(0, ds.src, ds.ns, "", nil, nil, ds.sslRootCaFile) l.Logger.Infof("splitter[%s] exits", ds) return nil } // check index key type consistency before parallel split if !ds.checkIndexKeyTypeConsistency() { l.Logger.Warnf("splitter[%s] detected mixed types on index key [%s], give up parallel fetching", ds, conf.Options.FullSyncReaderParallelIndex) ds.readerChan <- NewDocumentReader(0, ds.src, ds.ns, "", nil, nil, ds.sslRootCaFile) return nil } maxChunkSize := conf.Options.FullSyncReaderSplitMaxChunkSize // default 1024MB, could be set by user if uint64(maxChunkSize*utils.MB) > ds.pieceByteSize { maxChunkSize = int(ds.pieceByteSize / uint64(utils.MB)) } splitVectorCmd := bson.D{ {"splitVector", ds.ns.Str()}, {"keyPattern", bson.M{conf.Options.FullSyncReaderParallelIndex: 1}}, // {_id:1} or {shardKey:1} // {"maxSplitPoints", ds.pieceNumber - 1}, {"maxChunkSize", maxChunkSize}, } l.Logger.Infof("splitter[%s] splitVector cmd: %v, waiting splitVector return...", ds, splitVectorCmd) res := SplitVectorResult{} err := ds.client.Client.Database(ds.ns.Database).RunCommand(context.Background(), splitVectorCmd).Decode(&res) // if failed, do not panic, run single thread fetching if err != nil { l.Logger.Warnf("splitter[%s] run splitVector failed[%v], give up parallel fetching", ds, err) ds.readerChan <- NewDocumentReader(0, ds.src, ds.ns, "", nil, nil, ds.sslRootCaFile) l.Logger.Infof("splitter[%s] exits", ds) return nil } // only print if len(splitKeys) < 100 if len(res.SplitKeys) > 100 { l.Logger.Infof("splitter[%s] run splitVector result len(splitKeys): %v", ds, len(res.SplitKeys)) } else { l.Logger.Infof("splitter[%s] run splitVector result splitKeys: %v", ds, res) } if len(res.SplitKeys) > 0 { // return list is sorted ds.pieceNumber = len(res.SplitKeys) + 1 var start interface{} cnt := 0 for i, keyDoc := range res.SplitKeys { // check key == conf.Options.FullSyncReaderParallelIndex doc := bson.M{} err := bson.Unmarshal(keyDoc, &doc) if err != nil { l.Logger.Panicf("splitter[%s] unmarshal doc [%v] failed: %v", ds, keyDoc, err) } key, val, err := parseDocKeyValue(doc) if err != nil { l.Logger.Panicf("splitter[%s] parse doc key failed: %v", ds, err) } if key != conf.Options.FullSyncReaderParallelIndex { l.Logger.Panicf("splitter[%s] parse doc invalid key: %v", ds, key) } l.Logger.Infof("splitter[%s] piece[%d] create reader with boundary(%v, %v]", ds, cnt, start, val) // inject new DocumentReader into channel ds.readerChan <- NewDocumentReader(cnt, ds.src, ds.ns, key, start, val, ds.sslRootCaFile) // new start start = val cnt++ // last one if i == len(res.SplitKeys)-1 { l.Logger.Infof("splitter[%s] piece[%d] create reader with boundary(%v, INF)", ds, cnt, start) // inject new DocumentReader into channel ds.readerChan <- NewDocumentReader(cnt, ds.src, ds.ns, key, start, nil, ds.sslRootCaFile) } } return nil } else { l.Logger.Warnf("splitter[%s] run splitVector return null result[%v]", ds, res) } l.Logger.Warnf("splitter[%s] give up parallel fetching", ds) ds.readerChan <- NewDocumentReader(0, ds.src, ds.ns, "", nil, nil, ds.sslRootCaFile) l.Logger.Infof("splitter[%s] exits", ds) l.Logger.Infof("splitter[%s] exits", ds) return nil } // isNumericBsonType returns true if the BSON type is a numeric type. // MongoDB treats int32, int64, double, and decimal128 as comparable in sort order, // so range queries ($gt/$lte) across these types will not lose data. func isNumericBsonType(t bsontype.Type) bool { return t == bson.TypeInt32 || t == bson.TypeInt64 || t == bson.TypeDouble || t == bson.TypeDecimal128 } // checkIndexKeyTypeConsistency checks whether the min and max values of the parallel index key // share the same BSON type. If they differ (excluding numeric type normalization), parallel // fetching with range queries could silently miss data. // Returns true if types are consistent (safe to split), false otherwise. func (ds *DocumentSplitter) checkIndexKeyTypeConsistency() bool { indexKey := conf.Options.FullSyncReaderParallelIndex coll := ds.client.Client.Database(ds.ns.Database).Collection(ds.ns.Collection) // find min value: sort ascending, limit 1 minOpts := options.FindOne().SetSort(bson.D{{indexKey, 1}}).SetProjection(bson.D{{indexKey, 1}}) minResult := coll.FindOne(context.Background(), bson.D{}, minOpts) if minResult.Err() != nil { if errors.Is(minResult.Err(), mongo.ErrNoDocuments) { // empty collection, safe to proceed return true } l.Logger.Warnf("splitter[%s] check type consistency: find min failed: %v", ds, minResult.Err()) // on error, be conservative and allow parallel (splitVector will fail anyway if there's a real problem) return true } // find max value: sort descending, limit 1 maxOpts := options.FindOne().SetSort(bson.D{{indexKey, -1}}).SetProjection(bson.D{{indexKey, 1}}) maxResult := coll.FindOne(context.Background(), bson.D{}, maxOpts) if maxResult.Err() != nil { l.Logger.Warnf("splitter[%s] check type consistency: find max failed: %v", ds, maxResult.Err()) return true } // decode raw documents minRaw, err := minResult.Raw() if err != nil { l.Logger.Warnf("splitter[%s] check type consistency: decode min failed: %v", ds, err) return true } maxRaw, err := maxResult.Raw() if err != nil { l.Logger.Warnf("splitter[%s] check type consistency: decode max failed: %v", ds, err) return true } // extract the index key field type minVal, err := minRaw.LookupErr(indexKey) if err != nil { l.Logger.Warnf("splitter[%s] check type consistency: lookup min key [%s] failed: %v", ds, indexKey, err) return true } maxVal, err := maxRaw.LookupErr(indexKey) if err != nil { l.Logger.Warnf("splitter[%s] check type consistency: lookup max key [%s] failed: %v", ds, indexKey, err) return true } minType := minVal.Type maxType := maxVal.Type // normalize numeric types if isNumericBsonType(minType) && isNumericBsonType(maxType) { l.Logger.Infof("splitter[%s] index key [%s] type check passed: min=%v, max=%v (both numeric)", ds, indexKey, minType, maxType) return true } if minType == maxType { l.Logger.Infof("splitter[%s] index key [%s] type check passed: type=%v", ds, indexKey, minType) return true } l.Logger.Warnf("splitter[%s] index key [%s] has inconsistent types: min=%v, max=%v", ds, indexKey, minType, maxType) return false } func parseDocKeyValue(x interface{}) (string, interface{}, error) { keyDocM := x.(bson.M) if len(keyDocM) > 1 { return "", nil, fmt.Errorf("invalid key doc[%v]", keyDocM) } var key string var val interface{} for key, val = range keyDocM { } return key, val, nil } // DocumentReader is the reader for single piece type DocumentReader struct { // source mongo address url src string ns utils.NS sslRootCaFile string // source root ca ssl // mongo document reader client *utils.MongoCommunityConn docCursor *mongo.Cursor ctx context.Context rebuild int // rebuild times concurrency int32 // for test only // query statement and current max cursor query bson.M key string id int // lastReadID tracks the _id of the last document successfully read // from the cursor. Used to resume from the correct position when // the cursor is rebuilt after a transient error (e.g. QueryPlanKilled). // RawValue owns a copy of the BSON value because Cursor.Current is reused // by the MongoDB driver on the next call to Next. lastReadID bson.RawValue } // NewDocumentReader creates reader with mongodb url func NewDocumentReader(id int, src string, ns utils.NS, key string, start, end interface{}, sslRootCaFile string) *DocumentReader { q := make(bson.M) if start != nil || end != nil { innerQ := make(bson.M) if start != nil { innerQ["$gt"] = start } if end != nil { innerQ["$lte"] = end } q[key] = innerQ } ctx := context.Background() return &DocumentReader{ id: id, src: src, ns: ns, sslRootCaFile: sslRootCaFile, query: q, key: key, ctx: ctx, } } func (reader *DocumentReader) String() string { ret := fmt.Sprintf("DocumentReader id[%v], src[%v] ns[%s] query[%v]", reader.id, utils.BlockMongoUrlPassword(reader.src, "***"), reader.ns, reader.query) if reader.docCursor != nil { ret = fmt.Sprintf("%s docCursorId[%v]", ret, reader.docCursor.ID()) } return ret } // NextDoc returns a document by raw bytes which is []byte // reader.docCursor.Current is valid only before next docCursor.Next(), So must be copy func (reader *DocumentReader) NextDoc() (doc bson.Raw, err error) { if err := reader.ensureNetwork(); err != nil { return nil, err } atomic.AddInt32(&reader.concurrency, 1) defer atomic.AddInt32(&reader.concurrency, -1) if !reader.docCursor.Next(reader.ctx) { if err := reader.docCursor.Err(); err != nil { reader.releaseCursor() return nil, err } else { l.Logger.Infof("reader[%s] finish", reader.String()) return nil, nil } } doc = reader.docCursor.Current reader.setLastReadID(doc) // A successful read breaks a sequence of rebuild failures. Keep the // rebuild limit for consecutive connection/cursor failures only. reader.rebuild = 0 return doc, nil } func (reader *DocumentReader) setLastReadID(doc bson.Raw) { id := doc.Lookup("_id") if id.Type == 0 { l.Logger.Warnf("reader[%s] read document without _id; cannot advance cursor resume point", reader) return } value := make([]byte, len(id.Value)) copy(value, id.Value) reader.lastReadID = bson.RawValue{Type: id.Type, Value: value} } // resumeQuery preserves the piece range in query and, after a cursor rebuild, // adds an _id lower bound. Readers are always sorted by _id, even when the // piece itself is split by a shard key, so the continuation must use _id too. func (reader *DocumentReader) resumeQuery() bson.M { resumeQuery := make(bson.M, len(reader.query)+1) for key, value := range reader.query { resumeQuery[key] = value } if reader.lastReadID.Type == 0 { return resumeQuery } // Do not mutate reader.query: its _id condition can be used again if a // reader is reused or a later rebuild needs to retain its upper boundary. idCondition := make(bson.M) if original, ok := resumeQuery["_id"].(bson.M); ok { for operator, value := range original { idCondition[operator] = value } } idCondition["$gt"] = reader.lastReadID resumeQuery["_id"] = idCondition return resumeQuery } // ensureNetwork establish the mongodb connection at first // if current connection is not ready or disconnected func (reader *DocumentReader) ensureNetwork() (err error) { if reader.docCursor != nil { return nil } if reader.client == nil { l.Logger.Infof("reader[%s] client is empty, create one", reader.String()) reader.client, err = utils.NewMongoCommunityConn(reader.src, conf.Options.MongoConnectMode, true, utils.ReadWriteConcernLocal, utils.ReadWriteConcernDefault, conf.Options.MongoSslRootCaFile) if err != nil { return err } } reader.rebuild += 1 if reader.rebuild > 10 { return fmt.Errorf("reader[%s] rebuild too many times(%d)", reader.String(), reader.rebuild) } resumeQuery := reader.resumeQuery() findOptions := new(options.FindOptions) findOptions.SetBatchSize(int32(conf.Options.FullSyncReaderFetchBatchSize)) // set big for test // A time-series logical view has no _id index, so an _id hint is rejected // outright (#897) and an _id sort becomes a blocking sort over every unpacked // measurement — that spills to disk and can exhaust the source's free space on // large collections (#993). Full sync does not depend on _id order, so // time-series collections are scanned in natural order with neither sort nor // hint. noCursorTimeout is unsupported on them as well (#784). if !reader.client.IsTimeSeriesCollection(reader.ns.Database, reader.ns.Collection) { findOptions.SetSort(map[string]interface{}{ "_id": 1, }) // enable noCursorTimeout anyway! #451 findOptions.SetNoCursorTimeout(true) } findOptions.SetComment(fmt.Sprintf("mongo-shake full sync: ns[%v] query[%v] rebuild-times[%v]", reader.ns, resumeQuery, reader.rebuild)) // field-level whitelist: server-side inclusion projection (top-level fields) if fields := conf.Options.FullSyncFieldWhitelistMap[reader.ns.Str()]; len(fields) > 0 { findOptions.SetProjection(filter.BuildInclusionProjection(fields)) } reader.docCursor, err = reader.client.Client.Database(reader.ns.Database).Collection(reader.ns.Collection, nil). Find(nil, resumeQuery, findOptions) if err != nil { return fmt.Errorf("run find failed: %w", err) } l.Logger.Infof("reader[%s] generates new cursor with resume query[%v]", reader.String(), resumeQuery) return nil } func (reader *DocumentReader) releaseCursor() { if reader.docCursor != nil { l.Logger.Infof("reader[%s] closes cursor[%v]", reader, reader.docCursor.ID()) err := reader.docCursor.Close(reader.ctx) if err != nil { l.Logger.Errorf("release cursor fail: %v", err) } } reader.docCursor = nil } func (reader *DocumentReader) Close() { l.Logger.Infof("reader[%s] close", reader) if reader.docCursor != nil { reader.docCursor.Close(reader.ctx) } if reader.client != nil { reader.client.Client.Disconnect(reader.ctx) reader.client = nil } }