javaspringspring-bootspring-batchspring-autoconfiguration

Spring Batch: How to specify a single header and footer line for ClassifierCompositeItemWriter writing into 1 file


StepConfig.java

package electronicdocumentdeliverybatch.config;

import electronicdocumentdeliverybatch.models.*;
import electronicdocumentdeliverybatch.models.properties.TestFileProperties;
import electronicdocumentdeliverybatch.models.stepModals.ElectronicDocumentsBatchOutput;
import electronicdocumentdeliverybatch.utils.FlatFileConstants;
import org.springframework.amqp.core.Message;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.ItemProcessor;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.file.FlatFileFooterCallback;
import org.springframework.batch.item.file.FlatFileHeaderCallback;
import org.springframework.batch.item.file.FlatFileItemWriter;
import org.springframework.batch.item.file.transform.BeanWrapperFieldExtractor;
import org.springframework.batch.item.file.transform.FormatterLineAggregator;
import org.springframework.batch.item.support.ClassifierCompositeItemWriter;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.classify.SubclassClassifier;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.FileSystemResource;

import java.util.HashMap;

@Configuration
public class StepConfig {

    private <T> FormatterLineAggregator<T> createLineAggregator(String[] fieldNames, String fieldFormat) {
        FormatterLineAggregator<T> lineAggregator = new FormatterLineAggregator<>();
        BeanWrapperFieldExtractor<T> fieldExtractor = new BeanWrapperFieldExtractor<>();
        fieldExtractor.setNames(fieldNames);
        lineAggregator.setFormat(fieldFormat);
        lineAggregator.setFieldExtractor(fieldExtractor);
        return lineAggregator;
    }

    @Bean(name = "NP01FlatFileItemWriter")
    public FlatFileItemWriter<NP01BatchOutput> np01FlatFileItemWriter(TestFileProperties testFileProperties, FlatFileFooterCallback footerCallback, FlatFileHeaderCallback headerCallback) {
        FlatFileItemWriter<NP01BatchOutput> writer = new FlatFileItemWriter<>();
        writer.setResource(new FileSystemResource(testFileProperties.getOutput().getResults()));
        writer.setAppendAllowed(true);
        writer.setLineAggregator(createLineAggregator(FlatFileConstants.NP01_CSV_FIELD_NAMES, FlatFileConstants.NP01_LINE_AGGREGATOR_FORMAT));
        writer.setFooterCallback(footerCallback);
        writer.setHeaderCallback(headerCallback);
        return writer;
    }

    @Bean(name = "FP01FlatFileItemWriter")
    public FlatFileItemWriter<FP01BatchOutput> fp01FlatFileItemWriter(TestFileProperties testFileProperties, FlatFileFooterCallback footerCallback, FlatFileHeaderCallback headerCallback) {
        FlatFileItemWriter<FP01BatchOutput> writer = new FlatFileItemWriter<>();
        writer.setResource(new FileSystemResource(testFileProperties.getOutput().getResults()));
        writer.setAppendAllowed(true);
        writer.setLineAggregator(createLineAggregator(FlatFileConstants.FP01_CSV_FIELD_NAMES, FlatFileConstants.FP01_LINE_AGGREGATOR_FORMAT));
        writer.setFooterCallback(footerCallback);
        writer.setHeaderCallback(headerCallback);
        return writer;
    }

    @Bean(name = "SJ01FlatFileItemWriter")
    public FlatFileItemWriter<SJ01BatchOutput> sj01FlatFileItemWriter(TestFileProperties testFileProperties, FlatFileFooterCallback footerCallback, FlatFileHeaderCallback headerCallback) {
        FlatFileItemWriter<SJ01BatchOutput> writer = new FlatFileItemWriter<>();
        writer.setResource(new FileSystemResource(testFileProperties.getOutput().getResults()));
        writer.setAppendAllowed(true);
        writer.setLineAggregator(createLineAggregator(FlatFileConstants.SJ01_CSV_FIELD_NAMES, FlatFileConstants.SJ01_LINE_AGGREGATOR_FORMAT));
        writer.setFooterCallback(footerCallback);
        writer.setHeaderCallback(headerCallback);
        return writer;
    }

    @Bean(name = "PR01FlatFileItemWriter")
    public FlatFileItemWriter<PR01BatchOutput> pr01FlatFileItemWriter(TestFileProperties testFileProperties, FlatFileFooterCallback footerCallback, FlatFileHeaderCallback headerCallback) {
        FlatFileItemWriter<PR01BatchOutput> writer = new FlatFileItemWriter<>();
        writer.setResource(new FileSystemResource(testFileProperties.getOutput().getResults()));
        writer.setAppendAllowed(true);
        writer.setLineAggregator(createLineAggregator(FlatFileConstants.PR01_CSV_FIELD_NAMES, FlatFileConstants.PR01_LINE_AGGREGATOR_FORMAT));
        writer.setFooterCallback(footerCallback);
        writer.setHeaderCallback(headerCallback);
        return writer;
    }

    @Bean(name = "FR01FlatFileItemWriter")
    public FlatFileItemWriter<FR01BatchOutput> fr01FlatFileItemWriter(TestFileProperties testFileProperties, FlatFileFooterCallback footerCallback, FlatFileHeaderCallback headerCallback) {
        FlatFileItemWriter<FR01BatchOutput> writer = new FlatFileItemWriter<>();
        writer.setResource(new FileSystemResource(testFileProperties.getOutput().getResults()));
        writer.setAppendAllowed(true);
        writer.setLineAggregator(createLineAggregator(FlatFileConstants.FR01_CSV_FIELD_NAMES, FlatFileConstants.FR01_LINE_AGGREGATOR_FORMAT));
        writer.setFooterCallback(footerCallback);
        writer.setHeaderCallback(headerCallback);
        return writer;
    }

    @Bean(name = "MR01FlatFileItemWriter")
    public FlatFileItemWriter<MR01BatchOutput> mr01FlatFileItemWriter(TestFileProperties testFileProperties, FlatFileFooterCallback footerCallback, FlatFileHeaderCallback headerCallback) {
        FlatFileItemWriter<MR01BatchOutput> writer = new FlatFileItemWriter<>();
        writer.setResource(new FileSystemResource(testFileProperties.getOutput().getResults()));
        writer.setAppendAllowed(true);
        writer.setLineAggregator(createLineAggregator(FlatFileConstants.MR01_CSV_FIELD_NAMES, FlatFileConstants.MR01_LINE_AGGREGATOR_FORMAT));
        writer.setFooterCallback(footerCallback);
        writer.setHeaderCallback(headerCallback);
        return writer;
    }

    @Bean(name = "DP01FlatFileItemWriter")
    public FlatFileItemWriter<DP01BatchOutput> dp01FlatFileItemWriter(TestFileProperties testFileProperties, FlatFileFooterCallback footerCallback, FlatFileHeaderCallback headerCallback) {
        FlatFileItemWriter<DP01BatchOutput> writer = new FlatFileItemWriter<>();
        writer.setResource(new FileSystemResource(testFileProperties.getOutput().getResults()));
        writer.setAppendAllowed(true);
        writer.setLineAggregator(createLineAggregator(FlatFileConstants.DP01_CSV_FIELD_NAMES, FlatFileConstants.DP01_LINE_AGGREGATOR_FORMAT));
        writer.setFooterCallback(footerCallback);
        writer.setHeaderCallback(headerCallback);
        return writer;
    }

    @Bean(name = "ClassifierCompositeItemWriter")
    public ClassifierCompositeItemWriter<BatchOutput> classifierCompositeItemWriter(SubclassClassifier subclassClassifier) {
        ClassifierCompositeItemWriter<BatchOutput> writer = new ClassifierCompositeItemWriter<>();
        writer.setClassifier(subclassClassifier);
        return writer;
    }

    @Bean
    public SubclassClassifier<Object, Object> subclassClassifier(
            FlatFileItemWriter<NP01BatchOutput> np01FlatFileItemWriter, FlatFileItemWriter<FP01BatchOutput> fp01FlatFileItemWriter,
            FlatFileItemWriter<SJ01BatchOutput> sj01FlatFileItemWriter, FlatFileItemWriter<PR01BatchOutput> pr01FlatFileItemWriter,
            FlatFileItemWriter<FR01BatchOutput> fr01FlatFileItemWriter, FlatFileItemWriter<MR01BatchOutput> mr01FlatFileItemWriter,
            FlatFileItemWriter<DP01BatchOutput> dp01FlatFileItemWriter) {
        SubclassClassifier<Object, Object> classifier = new SubclassClassifier<>();
        HashMap<Class<?>, Object> typeMap = new HashMap<>();
        typeMap.put(NP01BatchOutput.class, np01FlatFileItemWriter);
        typeMap.put(FP01BatchOutput.class, fp01FlatFileItemWriter);
        typeMap.put(SJ01BatchOutput.class, sj01FlatFileItemWriter);
        typeMap.put(PR01BatchOutput.class, pr01FlatFileItemWriter);
        typeMap.put(FR01BatchOutput.class, fr01FlatFileItemWriter);
        typeMap.put(MR01BatchOutput.class, mr01FlatFileItemWriter);
        typeMap.put(DP01BatchOutput.class, dp01FlatFileItemWriter);
        classifier.setTypeMap(typeMap);
        return classifier;
    }


    @Bean(name = "createFlatFileStep")
    public Step documentDeliveryStep(StepBuilderFactory stepBuilderFactory,
                                     @Qualifier("electronicDocumentsQueueItemReader") ItemReader<Message> reader,
                                     @Qualifier("electronicDocumentsQueueItemProcessor") ItemProcessor<Message, ElectronicDocumentsBatchOutput> itemProcessor,
                                     @Qualifier("electronicDocumentsQueueItemWriter") ItemWriter<ElectronicDocumentsBatchOutput> writer,
                                     FlatFileItemWriter<NP01BatchOutput> np01FlatFileItemWriter, FlatFileItemWriter<FP01BatchOutput> fp01FlatFileItemWriter,
                                     FlatFileItemWriter<SJ01BatchOutput> sj01FlatFileItemWriter, FlatFileItemWriter<PR01BatchOutput> pr01FlatFileItemWriter,
                                     FlatFileItemWriter<FR01BatchOutput> fr01FlatFileItemWriter, FlatFileItemWriter<MR01BatchOutput> mr01FlatFileItemWriter,
                                     FlatFileItemWriter<DP01BatchOutput> dp01FlatFileItemWriter) {
        return stepBuilderFactory.get("computeFlatFile")
                .<Message, ElectronicDocumentsBatchOutput>chunk(1)
                .reader(reader)
                .processor(itemProcessor)
                .writer(writer)
                .stream(np01FlatFileItemWriter)
                .stream(mr01FlatFileItemWriter)
                .stream(fp01FlatFileItemWriter)
                .stream(sj01FlatFileItemWriter)
                .stream(pr01FlatFileItemWriter)
                .stream(fr01FlatFileItemWriter)
                .stream(dp01FlatFileItemWriter)
                .build();
    }
}

I have a ClassifierCompositeItemWriter that contains multiple FlatFileItemWriters. The FlatFileItemWriters handle different object types, and output a single line into the SAME flat file. All of that works as of now. What I need to add is a single header and footer to the single output file from the batch. Using the current setup, I believe the header works, though not in as intentional of a way as I would prefer (First call to headerCallback will only happen once, from what I'm noticing). My current footer setup calls for the footer string multiple times, as in it calls the footerCallback on every single writer, not just 1. I think I could do this manually just fine, but I was hoping I could find a solution that is more associated to spring configurations if possible.

To summarize, is there a way to Spring Boot auto-configure header and footer lines for a flat file when writing from multiple writers into a single file?


Solution

  • I moved away from the design pattern I'm currently using, but I did find a way to make it work. Using the code from the question, my header already worked as is, since you can only write 1 header on a file. The issue was the footer. To mitigate the issues of writing multiple footers, I just set a boolean in the FooterItemWriter to determine if a single footer had been written yet.

    @Component
    public class ElectronicDocumentsFooterWriter implements FlatFileFooterCallback {
        private static boolean footerWritten = false;
        private final RecordService recordService;
    
        public ElectronicDocumentsFooterWriter(RecordService recordService) {
            this.recordService = recordService;
        }
    
        @Override
        public void writeFooter(Writer writer) throws IOException {
            if (!footerWritten) {
                writer.write(recordService.deriveFooter());
                footerWritten = true;
            }
        }
    }