I am writing a data export/import feature in my application. The output is written as text to a csv file and this appears to be working ok, but the data import is skipping some of the exported lines (the "statistic" data in the output csv file below) and I can’t understand why this is happening.
So you can see how the file is built, the export routine looks like this:
//
// Default to read files from the documents director
final directory =
await ExternalPath.getExternalStoragePublicDirectory(
ExternalPath.DIRECTORY_DOWNLOADS);
//
// Build filename
String filename = 'frequentBackup';
filename = filename + DateTime.now().year.toString();
filename =
filename + DateTime.now().month.toString().padLeft(2, '0');
filename =
filename + DateTime.now().day.toString().padLeft(2, '0');
filename = '$filename.csv';
String fullname = '$directory/$filename'; // Add path
//
// Build output and save to disk
try {
//
// Export Instruments
File(fullname).writeAsStringSync(exportInstruments(),
mode: FileMode.write, flush: true);
//
// Export Range Names
File(fullname).writeAsStringSync(exportRanges(),
mode: FileMode.append, flush: true);
//
// Export Frequencies
File(fullname).writeAsStringSync(exportFrequencies(),
mode: FileMode.append, flush: true);
//
// Export FrequencySort
File(fullname).writeAsStringSync(
await exportFrequencySort(database: widget._db),
mode: FileMode.append,
flush: true);
//
// Export Statistics
File(fullname).writeAsStringSync(
await exportStatistics(database: widget._db),
mode: FileMode.append,
flush: true);
//
// Export AppSettings
await File(fullname).writeAsString(exportAppSettings(),
mode: FileMode.append, flush: true);
//
// Notify user that the export is complete
String message = 'Export to $filename complete.';
showSnackBar(message: message);
The individual functions to create the export rows look like this:
Instrument Data:
String exportInstruments() {
String output = '';
Map<int, String> instruments = Instruments.data();
instruments.forEach((id, name) {
output += 'instrument,';
output += 'id:$id,';
output += 'name:$name\n';
});
return output;
}
Range Data:
String exportRanges() {
String output = '';
Map<int, String> ranges = Ranges.data();
ranges.forEach((id, name) {
output += 'range name,';
output += 'id:$id,';
output += 'name:$name\n';
});
return output;
}
Frequency Data:
String exportFrequencies() {
String output = '';
List<Record> frequencies = Frequencies.master();
for (Record r in frequencies) {
output += 'frequency,';
output += 'fid:${r.id},';
output += 'iid:${r.instrumentID},';
output += 'iname:${r.instrument},';
output += 'rid:${r.rangeNameID},';
output += 'rname:${r.rangeName},';
output += 'rtype:${r.rangeType},';
output += 'fstart:${r.freqStart},';
output += 'fend:${r.freqEnd},';
output += 'fundamental:${r.fundamental},';
output += 'audiofile:${r.audioFile},';
output += 'audiopath:${r.audioPath} \n';
}
return output;
}
Frequency Sort Data:
Future<String> exportFrequencySort({required MyDatabase database}) async {
String output = '';
// Get statistics records from the database (Future)
await DataFrequencies.getSortOrder(db: database).then((order) {
order.forEach((position, fid) {
output += 'frequencyorder,';
output += 'position:$position,';
output += 'fid:$fid\n';
});
});
return output;
}
Statistic Data:
Future<String> exportStatistics({required MyDatabase database}) async {
String output = '';
// Get statistics records from the database (Future)
await DataStatistics.getAllStatistics(db: database).then((statistics) {
for (StatRecord s in statistics) {
output += 'statistic,';
output += 'type:${s.type.toString()},';
output += 'fid:${s.id},';
output += 'iid:${s.iid},';
output += 'rid:${s.rid},';
output += 'presented:${s.presented},';
output += 'correct:${s.correct}\n';
}
});
return output;
}
App Setting Data:
String exportAppSettings() {
String output = '';
Map<String, dynamic> settings = Settings.getAllSettings();
settings.forEach((name, setting) {
output += 'setting,';
output += 'name:$name,';
output += 'setting:$setting\n';
});
return output;
}
The output csv file produced by these functions looks like this:
instrument,id:1,name:banjo
instrument,id:2,name:piano
instrument,id:3,name:xylophone
range name,id:1,name:crump
range name,id:2,name:kapow
range name,id:3,name:whump
frequency,fid:1,iid:1,iname:banjo ,rid:1,rname:crump,rtype:R,fstart:30,fend:60,fundamental:1,audiofile:,audiopath:
frequency,fid:2,iid:1,iname:banjo ,rid:2,rname:kapow,rtype:R,fstart:100,fend:250,fundamental:0,audiofile:,audiopath:
frequency,fid:3,iid:1,iname:banjo ,rid:3,rname:whump,rtype:R,fstart:1000,fend:2000,fundamental:0,audiofile:,audiopath:
frequency,fid:4,iid:2,iname:piano,rid:1,rname:crump,rtype:R,fstart:1000,fend:1500,fundamental:0,audiofile:,audiopath:
frequency,fid:5,iid:2,iname:piano,rid:2,rname:kapow,rtype:R,fstart:2000,fend:3000,fundamental:1,audiofile:,audiopath:
frequency,fid:6,iid:2,iname:piano,rid:3,rname:whump,rtype:R,fstart:4000,fend:5000,fundamental:0,audiofile:,audiopath:
frequencyorder,position:0,fid:1
frequencyorder,position:1,fid:2
frequencyorder,position:2,fid:3
frequencyorder,position:3,fid:4
frequencyorder,position:4,fid:5
frequencyorder,position:5,fid:6
statistic,type:FlashType.frequency,fid:1,iid:1,rid:1,presented:0,correct:0
statistic,type:FlashType.frequency,fid:2,iid:1,rid:2,presented:0,correct:0
statistic,type:FlashType.frequency,fid:3,iid:1,rid:3,presented:0,correct:0
statistic,type:FlashType.frequency,fid:4,iid:2,rid:1,presented:0,correct:0
statistic,type:FlashType.frequency,fid:5,iid:2,rid:2,presented:0,correct:0
statistic,type:FlashType.frequency,fid:6,iid:2,rid:3,presented:0,correct:0
statistic,type:FlashType.ranges,fid:1,iid:1,rid:1,presented:0,correct:0
statistic,type:FlashType.ranges,fid:2,iid:1,rid:2,presented:0,correct:0
statistic,type:FlashType.ranges,fid:3,iid:1,rid:3,presented:0,correct:0
statistic,type:FlashType.ranges,fid:4,iid:2,rid:1,presented:0,correct:0
statistic,type:FlashType.ranges,fid:5,iid:2,rid:2,presented:0,correct:0
statistic,type:FlashType.ranges,fid:6,iid:2,rid:3,presented:0,correct:0
statistic,type:FlashType.fundamentals,fid:1,iid:1,rid:1,presented:0,correct:0
statistic,type:FlashType.fundamentals,fid:2,iid:1,rid:2,presented:0,correct:0
statistic,type:FlashType.fundamentals,fid:3,iid:1,rid:3,presented:0,correct:0
statistic,type:FlashType.fundamentals,fid:4,iid:2,rid:1,presented:0,correct:0
statistic,type:FlashType.fundamentals,fid:5,iid:2,rid:2,presented:0,correct:0
statistic,type:FlashType.fundamentals,fid:6,iid:2,rid:3,presented:0,correct:0
statistic,type:FlashType.sounds,fid:1,iid:1,rid:1,presented:0,correct:0
statistic,type:FlashType.sounds,fid:2,iid:1,rid:2,presented:0,correct:0
statistic,type:FlashType.sounds,fid:3,iid:1,rid:3,presented:0,correct:0
statistic,type:FlashType.sounds,fid:4,iid:2,rid:1,presented:0,correct:0
statistic,type:FlashType.sounds,fid:5,iid:2,rid:2,presented:0,correct:0
statistic,type:FlashType.sounds,fid:6,iid:2,rid:3,presented:0,correct:0
setting,name:colourOneEnabled,setting:false
setting,name:colourOne,setting:#fffff59d
setting,name:colourTwoEnabled,setting:false
setting,name:colourTwo,setting:#ff9dffb1
setting,name:fontColourEnabled,setting:false
setting,name:fontColour,setting:#ff000000
setting,name:fundamentalColourEnabled,setting:false
setting,name:fundamentalColour,setting:#ff4caf50
setting,name:disabledColourEnabled,setting:false
setting,name:disabledColour,setting:#ff9e9e9e
setting,name:themeMode,setting:ThemeMode.dark
setting,name:leftHanded,setting:false
The import routine that reads this file uses readAsStringSync (I have the same issue if I use readAsLinesSync) as follows:
String fullName = result.files.first.path as String;
dynamic temp = File(fullName).readAsStringSync();
print('$temp');
The debug console output looks like this:
I/flutter ( 6499): instrument,id:1,name:banjo
I/flutter ( 6499): instrument,id:2,name:piano
I/flutter ( 6499): instrument,id:3,name:xylophone
I/flutter ( 6499): range name,id:1,name:crump
I/flutter ( 6499): range name,id:2,name:kapow
I/flutter ( 6499): range name,id:3,name:whump
I/flutter ( 6499): frequency,fid:1,iid:1,iname:banjo ,rid:1,rname:crump,rtype:R,fstart:30,fend:60,fundamental:1,audiofile:,audiopath:
I/flutter ( 6499): frequency,fid:2,iid:1,iname:banjo ,rid:2,rname:kapow,rtype:R,fstart:100,fend:250,fundamental:0,audiofile:,audiopath:
I/flutter ( 6499): frequency,fid:3,iid:1,iname:banjo ,rid:3,rname:whump,rtype:R,fstart:1000,fend:2000,fundamental:0,audiofile:,audiopath:
I/flutter ( 6499): frequency,fid:4,iid:2,iname:piano,rid:1,rname:crump,rtype:R,fstart:1000,fend:1500,fundamental:0,audiofile:,audiopath:
I/flutter ( 6499): frequency,fid:5,iid:2,iname:piano,rid:2,rname:kapow,rtype:R,fstart:2000,fend:3000,fundamental:1,audiofile:,audiopath:
I/flutter ( 6499): frequency,fid:6,iid:2,iname:piano,rid:3,rname:whump,rtype:R,fstart:4000,fend:5000,fundamental:0,audiofile:,audiopath:
I/flutter ( 6499): frequencyorder,position:0,fid:1
I/flutter ( 6499): frequencyorder,position:1,fid:2
I/flutter ( 6499): frequencyorder,position:2,fid:3
I/flutter ( 6499): frequencyorder,position:3,fid:4
I/flutter ( 6499): frequencyorder,position:4,fid:5
I/flutter ( 6499): frequencyorder,position:5,fid:6
I/flutter ( 6499): setting,name:colourOneEnabled,setting:false
I/flutter ( 6499): setting,name:colourOne,setting:#fffff59d
I/flutter ( 6499): setting,name:colourTwoEnabled,setting:false
I/flutter ( 6499): setting,name:colourTwo,setting:#ff9dffb1
I/flutter ( 6499): setting,name:fontColourEnabled,setting:false
I/flutter ( 6499): setting,name:fontColour,setting:#ff000000
I/flutter ( 6499): setting,name:fundamentalColourEnabled,setting:false
I/flutter ( 6499): setting,name:fundamentalColour,setting:#ff4caf50
I/flutter ( 6499): setting,name:disabledColourEnabled,setting:false
I/flutter ( 6499): setting,name:disabledColour,setting:#ff9e9e9e
I/flutter ( 6499): setting,name:themeMode,setting:ThemeMode.system
I/flutter ( 6499): setting,name:leftHanded,setting:false
Note the missing “statistic” lines between the "frequencyorder" and "setting" lines.
I am running Flutter (Channel stable, 3.7.11, on Microsoft Windows [Version 10.0.22621.1555], locale en-GB).
Does anyone have any ideas why this is happening?
As per the post here: Special characters in Flutter, the solution is to convert the dot to unicode and then export the String. When this is read by readAsString or readAsLines, the unicode is converted back to a dot automatically. My export code now looks like this:
Future<String> exportStatistics({required MyDatabase database}) async {
String output = '';
// Get statistics records from the database (Future)
await DataStatistics.getAllStatistics(db: database).then((statistics) {
for (StatRecord s in statistics) {
output += 'statistic,';
dynamic temp = s.type.toString().replaceAll('.', '\u002e');
output += 'type:$temp,';
output += 'fid:${s.id},';
output += 'iid:${s.iid},';
output += 'rid:${s.rid},';
output += 'presented:${s.presented},';
output += 'correct:${s.correct}\n';
}
});
return output;
}
This wasn't immediately obvious in testing because I have also learned that, on my setup at least, when I am testing the app on a physically attached device I needed to clear the cache on the app before importing the new file. If I don't do this, then the app seems to be importing a previously cached version of the file even after changing the export routine, restarting the app and re-exporting the data to the export file. Because of this, it looks like changes haven't worked when they have.