pythonpython-3.xargparse

How to get arguments from ArgParser to write to a textfile so it can be read with `fromfile_prefix_chars`


I want a script to save all arguments (whether provided or populate from defaults or read from a text file) to a text file in such a format that I can re-run the script using fromfile_prefix_chars with that file. This is to allow for easy reproducability and store the arguments that were used to generate certain data.

I have a very brittle and incomplete implementation. I'm sure there is a better way but have not found it via google.

Q: how to capture a) all arguments with b) clean and concise code ?

Context

ArgumentParser has a feature that allows the reading of arguments from a text file (fromfile_prefix_chars). There seems little point in reinventing the wheel to read e.g. a JSON file. While the reading of a text file is part of the ArgumentParser feature set, writing that file seems not to be. I am looking for an elegant solution to convert a arg: Namespace and a parser: ArgumentParser into a text file (containing all arguments, not only those passed in at the command line), that can be read using ArgumentParser

Workflow Example

  1. Run script python script.py --count 7 --channel-stdev 17 /somepath This should produce a text file called args_out.txt with the content
--count
1
--channel-mean
4.5
--channel-stdev
3
/somepath
  1. I can re-run the script using python script.py @args_out.txt

Minimal example

from pathlib import Path
import logging
import argparse
from argparse import Namespace, ArgumentParser
from typing import Optional, List, Tuple


def get_arg_parser() -> ArgumentParser:
    parser = ArgumentParser(description='Test',
                            fromfile_prefix_chars='@')
    parser.add_argument('--count', default=1, type=int,
                        help='Number of ')
    parser.add_argument('--channel-mean', default=4.5, type=float,
                        help='Mean number ')
    parser.add_argument('--channel-stdev', default=3, type=float,
                        help='Standard deviation ')

    # Logging control
    log_group = parser.add_mutually_exclusive_group()
    log_group.add_argument('-v', '--debug', action='store_true', help='Enable debug mode')
    log_group.add_argument('-q', '--quiet', action='store_true', help='Enable quiet mode')

    # I/O control
    parser.add_argument('input', nargs='+', help='Path to a single')
    return parser


def parse_args(arg_list: Optional[List[str]] = None) -> Namespace:
    parser = get_arg_parser()
    return parser.parse_args(arg_list)


def convert_args_to_text(args: Namespace, parser: ArgumentParser) -> str:
    """ Convert an argparse.Namespace object to a list of strings that can be written to a file and then
    read as arguments again. """
    required, optionals, optionals_store_true = get_arg_name_to_argument_dicts(parser)

    text = ''
    for k, v in vars(args).items():
        k_dash = k.replace('_', '-')
        if k_dash in optionals:
            text = add_key_and_value(text, key=k_dash, value=v)

        elif k_dash in optionals_store_true:
            if v:
                text = add_key_and_value(text, key=v, value=None)

        elif k_dash in required:
            text = add_key_and_value(text, key=None, value=v)
        else:
            logging.warning(f"skipping argument {k}")

    return text


def get_arg_name_to_argument_dicts(parser: ArgumentParser) -> Tuple[dict, dict, dict]:
    """ Here I try to extract the argument names and whether they are optional or required from the ArgParser
    This is very brittle FIXME """
    optionals = {}
    optionals_store_true = {}  # These are only added if they are true
    required = {}
    for optk, optv in parser._optionals._option_string_actions.items():
        if not optk.startswith('--'):
            # Skip short options
            continue

        if isinstance(optv, argparse._StoreAction):
            optionals[optk.strip('--')] = optv
        elif isinstance(optv, argparse._StoreTrueAction):
            optionals_store_true[optk.strip('--')] = optv

    for req_group_actions in parser._positionals._group_actions:
        name = req_group_actions.dest
        required[name] = req_group_actions

    return required, optionals, optionals_store_true


def add_key_and_value(text: str, key: Optional[str], value: Optional[Tuple[str, List]]) -> str:
    """ Add a argument key and value to a text block. """

    if key is not None:
        text += f"--{key}\n"

    if value is None:
        return text

    if isinstance(value, list):
        for vi in value:
            text += f'{vi}\n'
    else:
        text += f'{value}\n'

    return text


def main():
    args = parse_args()

    print(args)

    text = convert_args_to_text(args, get_arg_parser())
    with open('args_out.txt', 'w') as f:
        f.write(text)


if __name__ == '__main__':
    main()

Thank you for your help


Solution

  • I eventually went with this. Its imperfect but avoids having to define the arguments twice. I'm sure a better solution exists.

    def convert_args_to_text(args: Namespace, parser: ArgumentParser) -> str:
        """ Convert an argparse.Namespace object to a list of strings that can be written to a file and then
        read as arguments again using the `fromfile_prefix_chars` argument in the ArgumentParser.
    
        Currently, only supports optional `_StoreTrueAction` and optional and required `_StoreAction`.
        """
    
        dest_to_arg = get_arg_name_to_argument_dicts(parser)
    
        text = []
        for k, v in vars(args).items():
            action = dest_to_arg[k]
    
            if isinstance(action, argparse._StoreTrueAction):
                if v:
                    key = action.option_strings[-1]
                    text.append(key)
                continue
    
            if action.option_strings:  # This is an optional argument
                key = action.option_strings[-1]
                if isinstance(v, list):
                    for vi in v:
                        text += [key, vi]
                else:
                    text += [key, v]
    
            else:  # Required argument
                if isinstance(v, list):
                    text += v
                else:
                    text.append(v)
    
        return '\n'.join([str(x) for x in text])
    
    
    def get_arg_name_to_argument_dicts(parser: ArgumentParser) -> dict:
        dest_to_arg = {}
        for action in parser._actions:
            dest_to_arg[action.dest] = action
    
        return dest_to_arg
    
    
    def save_arguments(args: Namespace, parser: ArgumentParser, output_dir: str):
        """ Save the CLI arguments to a text file. """
        text = convert_args_to_text(args, parser)
    
        output_dir_path = Path(output_dir)
        output_dir_path.mkdir(parents=True, exist_ok=True)
        outfile = output_dir_path / f'args_{dt.now().isoformat()}.txt'
        with open(outfile, 'w') as f:
            f.write(text)