I want a script to save all arguments (whether provided or populate from defaults or read from a text file) to a text file in such a format that I can re-run the script using fromfile_prefix_chars with that file. This is to allow for easy reproducability and store the arguments that were used to generate certain data.
I have a very brittle and incomplete implementation. I'm sure there is a better way but have not found it via google.
Q: how to capture a) all arguments with b) clean and concise code ?
ArgumentParser has a feature that allows the reading of arguments from a text file (fromfile_prefix_chars). There seems little point in reinventing the wheel to read e.g. a JSON file. While the reading of a text file is part of the ArgumentParser feature set, writing that file seems not to be. I am looking for an elegant solution to convert a arg: Namespace and a parser: ArgumentParser into a text file (containing all arguments, not only those passed in at the command line), that can be read using ArgumentParser
python script.py --count 7 --channel-stdev 17 /somepath
This should produce a text file called args_out.txt with the content--count
1
--channel-mean
4.5
--channel-stdev
3
/somepath
python script.py @args_out.txtfrom pathlib import Path
import logging
import argparse
from argparse import Namespace, ArgumentParser
from typing import Optional, List, Tuple
def get_arg_parser() -> ArgumentParser:
parser = ArgumentParser(description='Test',
fromfile_prefix_chars='@')
parser.add_argument('--count', default=1, type=int,
help='Number of ')
parser.add_argument('--channel-mean', default=4.5, type=float,
help='Mean number ')
parser.add_argument('--channel-stdev', default=3, type=float,
help='Standard deviation ')
# Logging control
log_group = parser.add_mutually_exclusive_group()
log_group.add_argument('-v', '--debug', action='store_true', help='Enable debug mode')
log_group.add_argument('-q', '--quiet', action='store_true', help='Enable quiet mode')
# I/O control
parser.add_argument('input', nargs='+', help='Path to a single')
return parser
def parse_args(arg_list: Optional[List[str]] = None) -> Namespace:
parser = get_arg_parser()
return parser.parse_args(arg_list)
def convert_args_to_text(args: Namespace, parser: ArgumentParser) -> str:
""" Convert an argparse.Namespace object to a list of strings that can be written to a file and then
read as arguments again. """
required, optionals, optionals_store_true = get_arg_name_to_argument_dicts(parser)
text = ''
for k, v in vars(args).items():
k_dash = k.replace('_', '-')
if k_dash in optionals:
text = add_key_and_value(text, key=k_dash, value=v)
elif k_dash in optionals_store_true:
if v:
text = add_key_and_value(text, key=v, value=None)
elif k_dash in required:
text = add_key_and_value(text, key=None, value=v)
else:
logging.warning(f"skipping argument {k}")
return text
def get_arg_name_to_argument_dicts(parser: ArgumentParser) -> Tuple[dict, dict, dict]:
""" Here I try to extract the argument names and whether they are optional or required from the ArgParser
This is very brittle FIXME """
optionals = {}
optionals_store_true = {} # These are only added if they are true
required = {}
for optk, optv in parser._optionals._option_string_actions.items():
if not optk.startswith('--'):
# Skip short options
continue
if isinstance(optv, argparse._StoreAction):
optionals[optk.strip('--')] = optv
elif isinstance(optv, argparse._StoreTrueAction):
optionals_store_true[optk.strip('--')] = optv
for req_group_actions in parser._positionals._group_actions:
name = req_group_actions.dest
required[name] = req_group_actions
return required, optionals, optionals_store_true
def add_key_and_value(text: str, key: Optional[str], value: Optional[Tuple[str, List]]) -> str:
""" Add a argument key and value to a text block. """
if key is not None:
text += f"--{key}\n"
if value is None:
return text
if isinstance(value, list):
for vi in value:
text += f'{vi}\n'
else:
text += f'{value}\n'
return text
def main():
args = parse_args()
print(args)
text = convert_args_to_text(args, get_arg_parser())
with open('args_out.txt', 'w') as f:
f.write(text)
if __name__ == '__main__':
main()
Thank you for your help
I eventually went with this. Its imperfect but avoids having to define the arguments twice. I'm sure a better solution exists.
def convert_args_to_text(args: Namespace, parser: ArgumentParser) -> str:
""" Convert an argparse.Namespace object to a list of strings that can be written to a file and then
read as arguments again using the `fromfile_prefix_chars` argument in the ArgumentParser.
Currently, only supports optional `_StoreTrueAction` and optional and required `_StoreAction`.
"""
dest_to_arg = get_arg_name_to_argument_dicts(parser)
text = []
for k, v in vars(args).items():
action = dest_to_arg[k]
if isinstance(action, argparse._StoreTrueAction):
if v:
key = action.option_strings[-1]
text.append(key)
continue
if action.option_strings: # This is an optional argument
key = action.option_strings[-1]
if isinstance(v, list):
for vi in v:
text += [key, vi]
else:
text += [key, v]
else: # Required argument
if isinstance(v, list):
text += v
else:
text.append(v)
return '\n'.join([str(x) for x in text])
def get_arg_name_to_argument_dicts(parser: ArgumentParser) -> dict:
dest_to_arg = {}
for action in parser._actions:
dest_to_arg[action.dest] = action
return dest_to_arg
def save_arguments(args: Namespace, parser: ArgumentParser, output_dir: str):
""" Save the CLI arguments to a text file. """
text = convert_args_to_text(args, parser)
output_dir_path = Path(output_dir)
output_dir_path.mkdir(parents=True, exist_ok=True)
outfile = output_dir_path / f'args_{dt.now().isoformat()}.txt'
with open(outfile, 'w') as f:
f.write(text)