lmflow.pipeline.dpov2_aligner#

Attributes#

`logger`
`ReferenceModelArguments`

Classes#

`DPOv2Aligner`
`MemorySafeDPOv2Aligner`

Module Contents#

lmflow.pipeline.dpov2_aligner.logger[source]#

lmflow.pipeline.dpov2_aligner.ReferenceModelArguments[source]#

class lmflow.pipeline.dpov2_aligner.DPOv2Aligner(model_args: lmflow.args.ModelArguments, data_args: lmflow.args.DatasetArguments, aligner_args: lmflow.args.DPOv2AlignerArguments, ref_model_args: lmflow.args.ModelArguments)[source]#

Bases: lmflow.pipeline.base_aligner.BaseAligner

model_args[source]#

ref_model_args[source]#

data_args[source]#

aligner_args[source]#

align(model: lmflow.models.hf_decoder_model.HFDecoderModel, ref_model: lmflow.models.hf_decoder_model.HFDecoderModel, train_dataset: lmflow.datasets.dataset.Dataset, eval_dataset: lmflow.datasets.dataset.Dataset, transform_dataset_in_place: bool = True)[source]#

convert_to_paired_dataset(source_dataset: lmflow.datasets.dataset.Dataset, sampling_paired_method: str = 'random', length_penalty: float = 0.0, margin_scale: float = 1.0, use_fast: bool = False) → lmflow.datasets.dataset.Dataset[source]#: Convert a scored one to multiple (text_to_scored_textlist) to a paired dataset by rejection sampling.

sampling_paired_idx_from_rewards(rewards: list[float], sampling_paired_method: str = 'random', use_fast: bool = False) → tuple[int, int][source]#: Prepare the dataset for DPO training by rejection sampling. We implement different strategies to select pairs, including random: randomly select two instances max_min: best v.s. worst max_max: best v.s. second best max_random: best v.s. random from the remaining

class lmflow.pipeline.dpov2_aligner.MemorySafeDPOv2Aligner(model_args: lmflow.args.ModelArguments, data_args: lmflow.args.DatasetArguments, aligner_args: lmflow.args.DPOv2AlignerArguments, ref_model_args: lmflow.args.ModelArguments)[source]#

model_args[source]#

ref_model_args[source]#

data_args[source]#

aligner_args[source]#

aligner_file_path[source]#

align()[source]#