I want to parse an externally defined (and undocumented) file format in Python. It looks somewhat similar to TOML, but with different text styles, and no quoting. For example:
[Schedule_Step122]
m_nMaxCurrent=0
m_szAddIn=Relay OFF
m_szLabel=06 - End Charge
m_uLimitNum=2
[Schedule_Step122_Limit0]
Equation0_szCompareSign=>=
Equation0_szRight=F_05_Charge_Capacity
Equation0_szLeft=PV_CHAN_Charge_Capacity
m_bStepLimit=1
m_szGotoStep=End Test
[Schedule_Step122_Limit1]
Equation0_szCompareSign=>=
Equation0_szLeft=PV_CHAN_Voltage
Equation0_szRight=3
m_bStepLimit=1
m_szGotoStep=End Test
(This is Arbin's test schedule format.)
I would like the parsed structure to be something like:
"steps": [
{
"max_current": 0,
"add_in": RELAY_OFF,
"label": "09 - End Charge",
"limits": [
{
"equations": [
{
"left": PV_CHAN_CHARGE_CAPACITY,
"compare_sign": ">=",
"right": F_05_CHARGE_CAPACITY
}
],
"step_limit": 1,
"goto_step": END_TEST
},
{
"equations": [
{
"left": PV_CHAN_VOLTAGE,
"compare_sign": ">=",
"right": 6
}
],
"step_limit": 1,
"goto_step": END_TEST
}
]
}
]
The format seems superficially similar to TOML, including some of the nesting, but the string handling is different. I would also like to capture certain values as named constants.
I was also looking into defining a context-free grammar and using a lexer/parser like ANTLR, PLY, pyparsing, or Lark. I'm familiar with reading grammars in documentation, but haven't written or used one with a parser before. However, I don't know how one would represent the nesting structure (such as Schedule_Step122_Limit0 being a member of Schedule_Step122) or the lack of guaranteed order among related keys (like Equation0_szCompareSign, Equation0_szLeft`, etc).
Is there a generic parsing tool I could write a definition for, which would give me the parsed/structured output? Or is the best approach here to write custom parsing logic?
Tools like ANTLR, PLY, pyparsing, or Lark will give you almost no help with this problem. configparser might help a little, but I suspect it'd be more bother than it's worth.
The following code is close to what you want. You'll need to tweak it based on what you discover about the input-format, and what you'd like for the output-structure.
import re, json
def main():
obj = parse('input.txt')
print(json.dumps(obj, indent=2))
def parse(filename):
root_object = {}
current_object = None
for line in open(filename):
# trim trailing whitespace:
line = line.rstrip()
if line == '':
# blank line
pass
elif mo := re.fullmatch(r'\[(\w+)\]', line):
# header line
# This identifies, via a 'path' from the root object,
# the object that subsequent name-value lines are talking about.
header_path = mo.group(1)
header_pieces = header_path.split('_')
current_object = get_nested_object(root_object, header_pieces)
elif mo := re.fullmatch(r'([^=]+)=(.*)', line):
# name-value line
(name_part, value_str) = mo.groups()
# The {name_part} identifies a field in {current_object}
# or some object nested within {current_object}.
# The {value_str} encodes the value to be assigned to that field.
name_pieces = name_part.split('_')
prefix_pieces = name_pieces[:-1]
field_name_piece = name_pieces[-1]
if prefix_pieces == ['m']:
# This is an 'immediate' field of {current_object}
obj_w_field = current_object
else:
# This is a field of some object nested within {current_object}
obj_w_field = get_nested_object(current_object, prefix_pieces)
mo = re.fullmatch(r'([a-z]+)([A-Z][a-zA-Z]*)', field_name_piece)
(type_indicator, field_name_pc) = mo.groups()
field_name = to_snake_case(field_name_pc)
field_value = value_str
obj_w_field[field_name] = field_value
else:
assert 0, line
return root_object
def get_nested_object(base_object, header_pieces):
if header_pieces == []:
return base_object
else:
prefix_pieces = header_pieces[:-1]
last_piece = header_pieces[-1]
obj = get_nested_object(base_object, prefix_pieces)
if mo := re.fullmatch(r'[A-Za-z]+', last_piece):
# e.g. "Schedule"
# This identifies a field/property/member of {obj}
field_name = to_snake_case(last_piece)
# That field might or might not exist already.
if field_name not in obj:
# It doesn't exist yet.
# We assume that the value of the field is an object
obj[field_name] = {}
return obj[field_name]
elif mo := re.fullmatch(r'([A-Za-z]+)(\d+)', last_piece):
# e.g., "Step122", "Limit0"
# This identifies an element of an array that is a field of {obj}
# e.g., "Step122" implies that {obj} has a field named "steps",
# whose value is an array,
# and this identifies the element at index 122 in that array.
(array_field_name_pc, index_str) = mo.groups()
array_field_name = to_snake_case(array_field_name_pc) + 's'
index = int(index_str)
if array_field_name not in obj:
obj[array_field_name] = {}
# In practice, you might want to make this a list.
array = obj[array_field_name]
if index not in array:
array[index] = {}
return array[index]
else:
assert 0, last_piece
assert 0
# "_pc" suffix denotes a Pascal-cased name, e.g. "MaxCurrent"
def to_snake_case(name_pc):
assert '_' not in name_pc
def replfunc(mo):
cap_letter = mo.group(0)
low_letter = cap_letter.lower()
if mo.start() == 0:
return low_letter
else:
return '_' + low_letter
return re.sub(r'[A-Z]', replfunc, name_pc)
main()
For the example input, it prints:
{
"schedule": {
"steps": {
"122": {
"max_current": "0",
"add_in": "Relay OFF",
"label": "06 - End Charge",
"limit_num": "2",
"limits": {
"0": {
"equations": {
"0": {
"compare_sign": ">=",
"right": "F_05_Charge_Capacity",
"left": "PV_CHAN_Charge_Capacity"
}
},
"step_limit": "1",
"goto_step": "End Test"
},
"1": {
"equations": {
"0": {
"compare_sign": ">=",
"left": "PV_CHAN_Voltage",
"right": "3"
}
},
"step_limit": "1",
"goto_step": "End Test"
}
}
}
}
}
}