# Copyright 2020 Google LLC. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. Neither the name of the copyright holder nor the names of its # contributors may be used to endorse or promote products derived from this # software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. """Runs all 3 steps to go from input DNA reads to output VCF/gVCF files. This script currently provides the most common use cases and standard models. If you want to access more flags that are available in `make_examples`, `call_variants`, and `postprocess_variants`, you can also call them separately using the binaries in the Docker image. For more details, see: https://github.com/google/deepvariant/blob/r1.10/docs/deepvariant-quick-start.md """ import enum import json import os import re import subprocess import sys import tempfile from typing import Any, Optional from absl import app from absl import flags from absl import logging import tensorflow as tf FLAGS = flags.FLAGS class ModelType(enum.Enum): WGS = 'WGS' WES = 'WES' PACBIO = 'PACBIO' ONT_R104 = 'ONT_R104' HYBRID_PACBIO_ILLUMINA = 'HYBRID_PACBIO_ILLUMINA' MASSEQ = 'MASSEQ' RNASEQ = 'RNASEQ' # Required flags. _MODEL_TYPE = flags.DEFINE_enum( 'model_type', None, [m.value for m in ModelType], ( 'Required. Type of model to use for variant calling. Set this flag to' ' use the default model associated with each type, and it will set' ' necessary flags corresponding to each model. If you want to use a' ' customized model, add --customized_model flag in addition to this' ' flag.' ), ) _REF = flags.DEFINE_string( 'ref', None, ( 'Required. Genome reference to use. Must have an associated FAI index' ' as well. Supports text or gzipped references. Should match the' ' reference used to align the BAM file provided to --reads.' ), ) _READS = flags.DEFINE_string( 'reads', None, ( 'Required. Aligned, sorted, indexed BAM file containing the reads we' ' want to call. Should be aligned to a reference genome compatible with' ' --ref.' ), ) _OUTPUT_VCF = flags.DEFINE_string( 'output_vcf', None, 'Required. Path where we should write VCF file.' ) # Optional flags. _HAPLOID_CONTIGS = flags.DEFINE_string( 'haploid_contigs', None, ( 'Optional list of non autosomal chromosomes. For all listed' ' chromosomes, HET probabilities are not considered. For samples with' ' XY karyotype it is expected to set --haploid_contigs="chrX,chrY" for' ' GRCh38 and --haploid_contigs="X,Y" for GRCh37. For samples with' ' XX karyotype --haploid_contigs flag should not be used.' ), ) _PAR_REGIONS = flags.DEFINE_string( 'par_regions_bed', None, ( 'Optional BED file containing Human Pseudoautosomal Region (PAR)' ' regions. This should be specific to the reference used. For example' ' GRCh38 PAR bed file would be different from GRCh37 bed file. Regions' ' in this bed file are treated as diploid, effectively subtracting them' ' from the --haploid_contigs.' ), ) _DRY_RUN = flags.DEFINE_boolean( 'dry_run', False, 'Optional. If True, only prints out commands without executing them.', ) _INTERMEDIATE_RESULTS_DIR = flags.DEFINE_string( 'intermediate_results_dir', None, ( 'Optional. If specified, this should be an existing ' 'directory that is visible insider docker, and will be ' 'used to to store intermediate outputs.' ), ) _LOGGING_DIR = flags.DEFINE_string( 'logging_dir', None, ( 'Optional. Directory where we should write log files ' 'for each stage and optionally runtime reports.' ), ) _RUNTIME_REPORT = flags.DEFINE_boolean( 'runtime_report', False, ( 'Output make_examples runtime metrics ' 'and create a visual runtime report using runtime_by_region_vis. ' 'Only works with --logging_dir.' ), ) _VERSION = flags.DEFINE_boolean( 'version', None, 'Optional. If true, print out version number and exit.', allow_hide_cpp=True, ) # Optional flags for call_variants and make_examples. _CUSTOMIZED_MODEL = flags.DEFINE_string( 'customized_model', None, ( 'Optional. A path to a model checkpoint to load for the `call_variants`' ' step. If not set, the default for each --model_type will be used' ), ) _CUSTOMIZED_MODEL_JSON = flags.DEFINE_string( 'customized_model_json', None, ( 'Optional. File path to a model.example_info.json to pair with' ' --customized_model. If not set, we will use the' ' model.example_info.json found in the model directory.' ), ) _DISABLE_SMALL_MODEL = flags.DEFINE_boolean( 'disable_small_model', False, 'Optional. Disable the use of the small model to call variants during ' 'the `make_examples` step.', ) _CUSTOMIZED_SMALL_MODEL = flags.DEFINE_string( 'customized_small_model', None, ( 'Optional. A path to a small model checkpoint to call variants during ' 'the `make_examples` step.' ), ) _NUM_SHARDS = flags.DEFINE_integer( 'num_shards', 1, 'Optional. Number of shards for make_examples step.' ) _REGIONS = flags.DEFINE_string( 'regions', None, ( 'Optional. Space-separated list of regions we want to process. Elements' ' can be region literals (e.g., chr20:10-20) or paths to BED/BEDPE' ' files.' ), ) _SAMPLE_NAME = flags.DEFINE_string( 'sample_name', None, ( 'Sample name to use instead of the sample name from the input reads BAM' ' (SM tag in the header). This flag is used for both make_examples and' ' postprocess_variants.' ), ) _USE_HP_INFORMATION = flags.DEFINE_boolean( 'use_hp_information', None, ( '(Deprecated in v1.4.0) Optional. If True, corresponding flags will be' ' set to properly use the HP information present in the BAM input.' ), ) _MAKE_EXAMPLES_EXTRA_ARGS = flags.DEFINE_string( 'make_examples_extra_args', None, ( 'A comma-separated list of flag_name=flag_value. "flag_name" has to be' ' valid flags for make_examples.py. If the flag_value is boolean, it' ' has to be flag_name=true or flag_name=false.' ), ) _CALL_VARIANTS_EXTRA_ARGS = flags.DEFINE_string( 'call_variants_extra_args', None, ( 'A comma-separated list of flag_name=flag_value. "flag_name" has to be' ' valid flags for call_variants.py. If the flag_value is boolean, it' ' has to be flag_name=true or flag_name=false.' ), ) # Optional flag for postprocess variants _POSTPROCESS_CPUS = flags.DEFINE_integer( 'postprocess_cpus', None, 'Optional. Number of cpus to use during' ' postprocess_variants. Set to 0 to disable multiprocessing. Default is' ' None which sets to num_shards.', ) _POSTPROCESS_VARIANTS_EXTRA_ARGS = flags.DEFINE_string( 'postprocess_variants_extra_args', None, ( 'A comma-separated list of flag_name=flag_value. "flag_name" has to be' ' valid flags for postprocess_variants.py. If the flag_value is' ' boolean, it has to be flag_name=true or flag_name=false.' ), ) _PHASE_VCF = flags.DEFINE_boolean( 'phase_vcf', None, 'Optional. If true, emit phasing information in the VCF output. Available' ' for long-read models (PacBio, ONT) only. Default is True for PacBio and' ' ONT models and False for others.', ) # Optional flags for postprocess_variants. _OUTPUT_GVCF = flags.DEFINE_string( 'output_gvcf', None, 'Optional. Path where we should write gVCF file.' ) # Optional flags for vcf_stats_report. _VCF_STATS_REPORT = flags.DEFINE_boolean( 'vcf_stats_report', False, ( 'Optional. Output a visual report (HTML) of ' 'statistics about the output VCF.' ), ) _REPORT_TITLE = flags.DEFINE_string( 'report_title', None, ( 'Optional. Title for the VCF stats report (HTML).' 'If not provided, the title will be the sample name.' ), ) _EMIT_VCF_BY_SMALL_MODEL_GQ_VALUES = flags.DEFINE_list( 'emit_vcf_by_small_model_gq_values', None, '[Experimental] When set as a list of integers, DeepVariant classifies all' ' candidates using both small and CNN models. It then generates separate' ' VCF files for each specified GQ (Genotype Quality) threshold by' ' initiating multiple post-processing jobs. This experimental feature aims' ' to analyze the combined accuracy of the two models across different GQ' ' thresholds for all samples.', ) MODEL_TYPE_MAP = { ModelType.WGS: '/opt/models/wgs', ModelType.WES: '/opt/models/wes', ModelType.PACBIO: '/opt/models/pacbio', ModelType.ONT_R104: '/opt/models/ont_r104', ModelType.HYBRID_PACBIO_ILLUMINA: '/opt/models/hybrid_pacbio_illumina', ModelType.MASSEQ: '/opt/models/masseq', ModelType.RNASEQ: '/opt/models/rnaseq', } # Current release version of DeepVariant. # Should be the same in dv_vcf_constants.py. DEEP_VARIANT_VERSION = '1.10.0' def _is_quoted(value): if value.startswith('"') and value.endswith('"'): return True if value.startswith("'") and value.endswith("'"): return True return False def _add_quotes(value): if isinstance(value, str) and _is_quoted(value): return value return '"{}"'.format(value) def trim_suffix(string: str, suffix: str) -> str: if string.endswith(suffix): return string[: -len(suffix)] else: return string def split_extra_args(input_string: str) -> list[str]: """Splits into strs that do not contain commas or are enclosed in quotes.""" pattern = r"[^,]+=[\"'][^\"']*[\"']|[^,]+" return re.findall(pattern, input_string) def _extra_args_to_dict(extra_args: str) -> dict[str, Any]: """Parses comma-separated list of flag_name=flag_value to dict.""" args_dict = {} if extra_args is None: return args_dict for extra_arg in split_extra_args(extra_args): (flag_name, flag_value) = extra_arg.split('=') flag_name = flag_name.strip('-') # Check for boolean values. if flag_value.lower() == 'true': flag_value = True elif flag_value.lower() == 'false': flag_value = False args_dict[flag_name] = flag_value return args_dict def _extend_command_by_args_dict(command, extra_args): """Adds `extra_args` to the command string.""" for key in sorted(extra_args): value = extra_args[key] if value is None: continue if isinstance(value, bool): added_arg = '' if value else 'no' added_arg += key command.extend(['--' + added_arg]) else: command.extend(['--' + key, _add_quotes(value)]) return command def _update_kwargs_with_warning(kwargs, extra_args): """Updates `kwargs` with `extra_args`; gives a warning if values changed.""" for k, v in extra_args.items(): if k in kwargs: if kwargs[k] != v: print( '\nWarning: --{} is previously set to {}, now to {}.'.format( k, kwargs[k], v ) ) kwargs[k] = v return kwargs def _use_small_model(model_ckpt_json: Optional[str] = None) -> bool: """Determines if the small model is enabled based on flags and model type.""" if _DISABLE_SMALL_MODEL.value: return False if _CUSTOMIZED_SMALL_MODEL.value: return True if model_ckpt_json and tf.io.gfile.exists(model_ckpt_json): with tf.io.gfile.GFile(model_ckpt_json) as f: info = json.load(f) if info.get('flags_for_calling', {}).get('trained_small_model_path'): return True return False def _set_small_model_config( special_args: dict[str, Any], customized_small_model: str | None, model_ckpt_json: str | None, ) -> None: """Sets small model config parameters.""" if not _use_small_model(model_ckpt_json): special_args['call_small_model_examples'] = False return special_args['call_small_model_examples'] = True if customized_small_model: special_args['trained_small_model_path'] = customized_small_model if _EMIT_VCF_BY_SMALL_MODEL_GQ_VALUES.value: special_args['small_model_emit_all_candidates'] = True def _rename_vcf_file_by_gq( vcf_file: str, gq_threshold: str | int, ) -> str: """Renames a VCF file by adding a suffix with the GQ threshold.""" if vcf_file.endswith('.vcf.gz'): return vcf_file.replace('.vcf.gz', f'_gq_{gq_threshold}.vcf.gz') else: return f'{vcf_file}_gq_{gq_threshold}.vcf.gz' def make_examples_command( ref, reads, examples, model_ckpt, model_ckpt_json, extra_args, runtime_by_region_path=None, output_local_read_phasing=None, **kwargs, ): """Returns a make_examples (command, logfile) for subprocess. Args: ref: Input FASTA file. reads: Input BAM file. examples: Output tfrecord file containing tensorflow.Example files. model_ckpt: Path to the TensorFlow model checkpoint. model_ckpt_json: Path to the model.example_info.json file, or None. extra_args: Comma-separated list of flag_name=flag_value. runtime_by_region_path: Output path for runtime by region metrics. output_local_read_phasing: Output path for local read phasing TSV files. **kwargs: Additional arguments to pass in for make_examples. Returns: (string, string) A command to run, and a log file to output to. """ command = [ 'time', 'seq 0 {} |'.format(_NUM_SHARDS.value - 1), 'parallel -q --halt 2 --line-buffer', '/opt/deepvariant/bin/make_examples', ] command.extend(['--mode', 'calling']) command.extend(['--ref', '"{}"'.format(ref)]) command.extend(['--reads', '"{}"'.format(reads)]) command.extend(['--examples', '"{}"'.format(examples)]) command.extend(['--checkpoint', '"{}"'.format(model_ckpt)]) if model_ckpt_json: command.extend(['--checkpoint_json', '"{}"'.format(model_ckpt_json)]) if runtime_by_region_path is not None: command.extend( ['--runtime_by_region', '"{}"'.format(runtime_by_region_path)] ) if output_local_read_phasing is not None: command.extend([ '--output_local_read_phasing', '"{}"'.format(output_local_read_phasing), ]) command.extend(['--output_phase_info']) special_args = {} # Starting from v1.10.0, the args for make_examples in calling mode are # defined in the model.example_info.json file corresponding to each model # type. # Small model args should be in the model.example_info.json file # as well. However, if the user specifies customized small model, we will # override the default small model args here. _set_small_model_config( special_args, _CUSTOMIZED_SMALL_MODEL.value, model_ckpt_json ) kwargs = _update_kwargs_with_warning(kwargs, special_args) # Extend the command with all items in kwargs and extra_args. kwargs = _update_kwargs_with_warning(kwargs, _extra_args_to_dict(extra_args)) command = _extend_command_by_args_dict(command, kwargs) command.extend(['--task {}']) logfile = None if _LOGGING_DIR.value: logfile = '{}/make_examples.log'.format(_LOGGING_DIR.value) return (' '.join(command), logfile) def call_variants_command( outfile: str, examples: str, model_ckpt: str, extra_args: str, ) -> tuple[str, Optional[str]]: """Returns a call_variants (command, logfile) for subprocess.""" binary_name = 'call_variants' command = ['time', f'/opt/deepvariant/bin/{binary_name}'] command.extend(['--outfile', '"{}"'.format(outfile)]) command.extend(['--examples', '"{}"'.format(examples)]) command.extend(['--checkpoint', '"{}"'.format(model_ckpt)]) if extra_args and 'use_openvino' in extra_args: raise RuntimeError( 'OpenVINO is not installed by default in DeepVariant ' 'Docker images. Please rerun without use_openvino flag.' ) # Extend the command with all items in extra_args. command = _extend_command_by_args_dict( command, _extra_args_to_dict(extra_args) ) logfile = None if _LOGGING_DIR.value: logfile = '{}/{}.log'.format(_LOGGING_DIR.value, binary_name) return (' '.join(command), logfile) def postprocess_variants_command( ref: str, infile: str, outfile: str, small_model_cvo_records: str, extra_args: str, model_ckpt_json: Optional[str] = None, **kwargs, ) -> tuple[str, Optional[str]]: """Returns a postprocess_variants (command, logfile) for subprocess.""" cpus = _POSTPROCESS_CPUS.value if cpus is None: cpus = _NUM_SHARDS.value # WES does not benefit from multiprocessing. if ModelType(_MODEL_TYPE.value) == ModelType.WES: cpus = 0 command = ['time', '/opt/deepvariant/bin/postprocess_variants'] command.extend(['--ref', '"{}"'.format(ref)]) command.extend(['--infile', '"{}"'.format(infile)]) command.extend(['--outfile', '"{}"'.format(outfile)]) command.extend(['--cpus', '"{}"'.format(cpus)]) if model_ckpt_json and tf.io.gfile.exists(model_ckpt_json): command.extend(['--checkpoint_json', '"{}"'.format(model_ckpt_json)]) if _use_small_model(model_ckpt_json): command.extend( ['--small_model_cvo_records', '"{}"'.format(small_model_cvo_records)] ) # Extend the command with all items in kwargs and extra_args. kwargs = _update_kwargs_with_warning(kwargs, _extra_args_to_dict(extra_args)) command = _extend_command_by_args_dict(command, kwargs) logfile = None if _LOGGING_DIR.value: logfile = '{}/postprocess_variants.log'.format(_LOGGING_DIR.value) return (' '.join(command), logfile) def vcf_stats_report_command( vcf_path: str, title: Optional[str] = None ) -> tuple[str, Optional[str]]: """Returns a vcf_stats_report (command, logfile) for subprocess. Args: vcf_path: Path to VCF, which will be passed to --input_vcf and suffix-trimmed for --outfile_base. title: Passed straight to command unless it's None. Returns: [command string for subprocess, optional log directory path] """ command = ['time', '/opt/deepvariant/bin/vcf_stats_report'] command.extend(['--input_vcf', '"{}"'.format(vcf_path)]) outfile_base = trim_suffix(trim_suffix(vcf_path, '.gz'), '.vcf') command.extend(['--outfile_base', '"{}"'.format(outfile_base)]) if title is not None: command.extend(['--title', '"{}"'.format(title)]) logfile = None if _LOGGING_DIR.value: logfile = '{}/vcf_stats_report.log'.format(_LOGGING_DIR.value) return (' '.join(command), logfile) def runtime_by_region_vis_command( runtime_by_region_path: str, title: str = 'DeepVariant' ) -> tuple[str, None]: """Returns a runtime_by_region_vis (command, logfile=None) for subprocess.""" runtime_report = os.path.join( _LOGGING_DIR.value, 'make_examples_runtime_by_region_report.html' ) command = ['time', '/opt/deepvariant/bin/runtime_by_region_vis'] command.extend(['--input', '"{}"'.format(runtime_by_region_path)]) command.extend(['--title', '"{}"'.format(title)]) command.extend(['--output', '"{}"'.format(runtime_report)]) return (' '.join(command), None) def _should_phase_vcf() -> bool: if _PHASE_VCF.value is not None: return _PHASE_VCF.value return _MODEL_TYPE.value in [ModelType.PACBIO.value, ModelType.ONT_R104.value] def check_or_create_intermediate_results_dir( intermediate_results_dir: Optional[str], ) -> str: """Checks or creates the path to the directory for intermediate results.""" if intermediate_results_dir is None: intermediate_results_dir = tempfile.mkdtemp() if not os.path.isdir(intermediate_results_dir): logging.info( 'Creating a directory for intermediate results in %s', intermediate_results_dir, ) os.makedirs(intermediate_results_dir) else: logging.info( 'Re-using the directory for intermediate results in %s', intermediate_results_dir, ) return intermediate_results_dir def check_flags(): """Additional logic to make sure flags are set appropriately.""" if _CUSTOMIZED_MODEL.value is not None: use_saved_model = tf.io.gfile.exists( _CUSTOMIZED_MODEL.value ) and tf.io.gfile.exists(f'{_CUSTOMIZED_MODEL.value}/saved_model.pb') if use_saved_model: logging.info('Using saved model: %s', str(use_saved_model)) elif not tf.io.gfile.exists( _CUSTOMIZED_MODEL.value + '.data-00000-of-00001' ) or not tf.io.gfile.exists(_CUSTOMIZED_MODEL.value + '.index'): raise RuntimeError( 'The model files {}* do not exist. Potentially ' 'relevant issue: ' 'https://github.com/google/deepvariant/blob/r1.10/docs/' 'FAQ.md#why-cant-it-find-one-of-the-input-files-eg-' 'could-not-open'.format(_CUSTOMIZED_MODEL.value) ) logging.info( ( 'You set --customized_model. Instead of using the default ' 'model for %s, `call_variants` step will load %s* ' 'instead.' ), _MODEL_TYPE.value, _CUSTOMIZED_MODEL.value, ) if tf.io.gfile.isdir(_CUSTOMIZED_MODEL.value): model_dir = _CUSTOMIZED_MODEL.value else: model_dir = os.path.dirname(_CUSTOMIZED_MODEL.value) # Check if model.example_info.json exists in the model directory # or if the user provided a customized json file. json_in_model_dir = tf.io.gfile.exists( f'{model_dir}/model.example_info.json' ) custom_json_exists = _CUSTOMIZED_MODEL_JSON.value and tf.io.gfile.exists( _CUSTOMIZED_MODEL_JSON.value ) if not json_in_model_dir and not custom_json_exists: raise RuntimeError( 'Starting from v1.10.0, the model file needs to have a corresponding' ' model.example_info.json file. You can see' f' gs://deepvariant/models/DeepVariant/{DEEP_VARIANT_VERSION}/savedmodels/deepvariant.*.savedmodel/model.example_info.json' ' as examples. The best way is to consult the people who trained the' ' model to understand what flags should be used for make_examples.' ) if _CUSTOMIZED_SMALL_MODEL.value is not None: logging.info( ( 'You set --customized_small_model. Instead of using the default' ' small model for %s, `make_examples` will load %s* instead. Make' ' sure you set the GQ thresholds explicitly via' ' make_examples_extra_args.' ), _MODEL_TYPE.value, _CUSTOMIZED_SMALL_MODEL.value, ) if _should_phase_vcf(): if ModelType(_MODEL_TYPE.value) not in [ ModelType.PACBIO, ModelType.ONT_R104, ]: raise RuntimeError( 'Emiting phasing information is only supported for PacBio and ONT' ' models.' ) def get_model_ckpt(model_type: str, customized_model: str) -> str: """Return the path to the model checkpoint based on the input args.""" if customized_model is not None: return customized_model else: return MODEL_TYPE_MAP[ModelType(model_type)] def create_all_commands_and_logfiles(intermediate_results_dir): """Creates 3 (command, logfile) to be executed later.""" check_flags() commands = [] # make_examples nonvariant_site_tfrecord_path = None if _OUTPUT_GVCF.value is not None: nonvariant_site_tfrecord_path = os.path.join( intermediate_results_dir, 'gvcf.tfrecord@{}.gz'.format(_NUM_SHARDS.value), ) examples = os.path.join( intermediate_results_dir, 'make_examples.tfrecord@{}.gz'.format(_NUM_SHARDS.value), ) small_model_cvo_records = os.path.join( intermediate_results_dir, 'make_examples_call_variant_outputs.tfrecord@{}.gz'.format( _NUM_SHARDS.value ), ) local_read_phasing_tsv_files = None if _should_phase_vcf(): local_read_phasing_tsv_files = os.path.join( intermediate_results_dir, f'read-phasing_debug@{_NUM_SHARDS.value}.tsv' ) if _LOGGING_DIR.value and _RUNTIME_REPORT.value: runtime_directory = os.path.join( _LOGGING_DIR.value, 'make_examples_runtime_by_region' ) if not os.path.isdir(runtime_directory): logging.info( 'Creating a make_examples runtime by region directory in %s', runtime_directory, ) os.makedirs(runtime_directory) # The path to runtime metrics output is sharded just like the examples. runtime_by_region_path = os.path.join( runtime_directory, 'make_examples_runtime@{}.tsv'.format(_NUM_SHARDS.value), ) else: runtime_by_region_path = None model_ckpt = get_model_ckpt(_MODEL_TYPE.value, _CUSTOMIZED_MODEL.value) model_ckpt_json = _CUSTOMIZED_MODEL_JSON.value if not model_ckpt_json: if tf.io.gfile.isdir(model_ckpt): model_dir = model_ckpt else: model_dir = os.path.dirname(model_ckpt) potential_json = f'{model_dir}/model.example_info.json' if tf.io.gfile.exists(potential_json): model_ckpt_json = potential_json commands.append( make_examples_command( ref=_REF.value, reads=_READS.value, examples=examples, model_ckpt=model_ckpt, model_ckpt_json=model_ckpt_json, runtime_by_region_path=runtime_by_region_path, extra_args=_MAKE_EXAMPLES_EXTRA_ARGS.value, # kwargs: gvcf=nonvariant_site_tfrecord_path, regions=_REGIONS.value, sample_name=_SAMPLE_NAME.value, haploid_contigs=_HAPLOID_CONTIGS.value, par_regions_bed=_PAR_REGIONS.value, output_local_read_phasing=local_read_phasing_tsv_files, ) ) # call_variants call_variants_output = os.path.join( intermediate_results_dir, 'call_variants_output.tfrecord.gz' ) commands.append( call_variants_command( outfile=call_variants_output, examples=examples, model_ckpt=model_ckpt, extra_args=_CALL_VARIANTS_EXTRA_ARGS.value, ) ) # postprocess_variants if _EMIT_VCF_BY_SMALL_MODEL_GQ_VALUES.value: for gq in _EMIT_VCF_BY_SMALL_MODEL_GQ_VALUES.value: commands.append( postprocess_variants_command( ref=_REF.value, infile=call_variants_output, outfile=_rename_vcf_file_by_gq(_OUTPUT_VCF.value, gq), small_model_cvo_records=small_model_cvo_records, extra_args=_POSTPROCESS_VARIANTS_EXTRA_ARGS.value, sample_name=_SAMPLE_NAME.value, haploid_contigs=_HAPLOID_CONTIGS.value, par_regions_bed=_PAR_REGIONS.value, regions=_REGIONS.value, resolve_call_variants_outputs_by_model=True, small_model_gq_threshold=gq, phased_reads_input_path=local_read_phasing_tsv_files, model_ckpt_json=model_ckpt_json, ) ) else: commands.append( postprocess_variants_command( ref=_REF.value, infile=call_variants_output, outfile=_OUTPUT_VCF.value, small_model_cvo_records=small_model_cvo_records, extra_args=_POSTPROCESS_VARIANTS_EXTRA_ARGS.value, nonvariant_site_tfrecord_path=nonvariant_site_tfrecord_path, gvcf_outfile=_OUTPUT_GVCF.value, sample_name=_SAMPLE_NAME.value, haploid_contigs=_HAPLOID_CONTIGS.value, par_regions_bed=_PAR_REGIONS.value, regions=_REGIONS.value, phased_reads_input_path=local_read_phasing_tsv_files, model_ckpt_json=model_ckpt_json, ) ) # vcf_stats_report if _VCF_STATS_REPORT.value: commands.append( vcf_stats_report_command( vcf_path=_OUTPUT_VCF.value, title=_REPORT_TITLE.value ) ) # runtime-by-region if _LOGGING_DIR.value and _RUNTIME_REPORT.value: commands.append( runtime_by_region_vis_command( runtime_by_region_path, title=_REPORT_TITLE.value ) ) return commands def main(_): if _USE_HP_INFORMATION.value: raise NotImplementedError( 'The --use_hp_information flag has been ' 'deprecated. DeepVariant now phases internally ' 'for PacBio mode.' ) if _VERSION.value: print('DeepVariant version {}'.format(DEEP_VARIANT_VERSION)) return for flag_key in ['model_type', 'ref', 'reads', 'output_vcf']: if FLAGS.get_flag_value(flag_key, None) is None: sys.stderr.write('--{} is required.\n'.format(flag_key)) sys.stderr.write('Pass --helpshort or --helpfull to see help on flags.\n') sys.exit(1) intermediate_results_dir = check_or_create_intermediate_results_dir( _INTERMEDIATE_RESULTS_DIR.value ) if _LOGGING_DIR.value and not os.path.isdir(_LOGGING_DIR.value): logging.info('Creating a directory for logs in %s', _LOGGING_DIR.value) os.makedirs(_LOGGING_DIR.value) commands_logfiles = create_all_commands_and_logfiles(intermediate_results_dir) print( '\n***** Intermediate results will be written to {} ' 'in docker. ****\n'.format(intermediate_results_dir) ) env = os.environ.copy() logging.info('env = %s', env) for command, logfile in commands_logfiles: print('\n***** Running the command:*****\n{}\n'.format(command)) if not _DRY_RUN.value: fp = open(logfile, 'w') if logfile is not None else None with subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, shell=True, executable='/bin/bash', universal_newlines=True, env=env, ) as proc: for line in proc.stdout: print(line, end='') if fp is not None: print(line, end='', file=fp) if fp is not None: fp.close() if proc.returncode != 0: sys.exit(proc.returncode) if __name__ == '__main__': app.run(main)