How to programmatically run a jMeter benchmark from code?

I want to benchmark some remote API calls via code. I've been using JMH for it so far, but it doesnt quite fit my need as a stress test tool (JMH works well for micro benchmarking, where the snippet being benchmarks rusn really really fast). My remote APIs respond in tens of seconds, so I really need to run the tests with:

1. Multiple parameterized inputs
2. Multiple client threads (controlling the load)

I'm able achieve a lot via manual tests in JMeter UI, but I would like to write some java tests/benchmarks that use JMeter and does the same. Is there a way to do that?

You can either kick off an existing JMeter test or create a new one using JMeter API

1. Run existing test example code:

   import org.apache.jmeter.engine.StandardJMeterEngine;
   import org.apache.jmeter.reporters.ResultCollector;
   import org.apache.jmeter.reporters.Summariser;
   import org.apache.jmeter.save.SaveService;
   import org.apache.jmeter.util.JMeterUtils;
   import org.apache.jorphan.collections.HashTree;
   
   import java.io.File;
   
   public class RunExistingJMeterTest {
   
       public static void main(String[] args) throws Exception {
           // Initialize properties
           JMeterUtils.loadJMeterProperties("/path/to/your/jmeter/bin/jmeter.properties");        
   
           // JMeter Engine
           StandardJMeterEngine jmeter = new StandardJMeterEngine();
   
   
           // Initialize logging, locale, etc.
           JMeterUtils.setJMeterHome("/path/to/your/jmeter");
           JMeterUtils.initLocale();
   
           // Initialize JMeter SaveService
           SaveService.loadProperties();
   
           // Load existing .jmx Test Plan
           HashTree testPlanTree = SaveService.loadTree(new File("/path/to/your/jmeter/extras/Test.jmx"));
   
           Summariser summer = null;
           String summariserName = JMeterUtils.getPropDefault("summariser.name", "summary");
           if (summariserName.length() > 0) {
               summer = new Summariser(summariserName);
           }
   
   
           // Store execution results into a .jtl file
           String logFile = "/path/to/results/file.jtl";
           ResultCollector logger = new ResultCollector(summer);
           logger.setFilename(logFile);
           testPlanTree.add(testPlanTree.getArray()[0], logger);
   
           // Run JMeter Test
           jmeter.configure(testPlanTree);
           jmeter.run();
   
       }
   }
   

2. Create JMeter script programmatically:

   import org.apache.jmeter.config.Arguments;
   import org.apache.jmeter.config.gui.ArgumentsPanel;
   import org.apache.jmeter.control.LoopController;
   import org.apache.jmeter.control.gui.LoopControlPanel;
   import org.apache.jmeter.control.gui.TestPlanGui;
   import org.apache.jmeter.engine.StandardJMeterEngine;
   import org.apache.jmeter.protocol.http.control.gui.HttpTestSampleGui;
   import org.apache.jmeter.protocol.http.sampler.HTTPSamplerProxy;
   import org.apache.jmeter.reporters.ResultCollector;
   import org.apache.jmeter.reporters.Summariser;
   import org.apache.jmeter.save.SaveService;
   import org.apache.jmeter.testelement.TestElement;
   import org.apache.jmeter.testelement.TestPlan;
   import org.apache.jmeter.threads.ThreadGroup;
   import org.apache.jmeter.threads.gui.ThreadGroupGui;
   import org.apache.jmeter.util.JMeterUtils;
   import org.apache.jorphan.collections.HashTree;
   
   import java.io.FileOutputStream;
   
   public class CreateNewJMeterTest {
   
       public static void main(String[] args) throws Exception {
   
           StandardJMeterEngine jmeter = new StandardJMeterEngine();
   
           //JMeter initialization (properties, log levels, locale, etc)
           JMeterUtils.setJMeterHome("/path/to/your/jmeter/installation");
           JMeterUtils.loadJMeterProperties("/path/to/your/jmeter/bin/jmeter.properties");
           JMeterUtils.initLocale();
   
           // JMeter Test Plan, basically JOrphan HashTree
           HashTree testPlanTree = new HashTree();
   
           // First HTTP Sampler - open example.com
           HTTPSamplerProxy examplecomSampler = new HTTPSamplerProxy();
           examplecomSampler.setDomain("example.com");
           examplecomSampler.setPort(80);
           examplecomSampler.setPath("/");
           examplecomSampler.setMethod("GET");
           examplecomSampler.setName("Open example.com");
           examplecomSampler.setProperty(TestElement.TEST_CLASS, HTTPSamplerProxy.class.getName());
           examplecomSampler.setProperty(TestElement.GUI_CLASS, HttpTestSampleGui.class.getName());
   
   
           // Loop Controller
           LoopController loopController = new LoopController();
           loopController.setLoops(1);
           loopController.setFirst(true);
           loopController.setProperty(TestElement.TEST_CLASS, LoopController.class.getName());
           loopController.setProperty(TestElement.GUI_CLASS, LoopControlPanel.class.getName());
           loopController.initialize();
   
           // Thread Group
           ThreadGroup threadGroup = new ThreadGroup();
           threadGroup.setName("Example Thread Group");
           threadGroup.setNumThreads(1);
           threadGroup.setRampUp(1);
           threadGroup.setSamplerController(loopController);
           threadGroup.setProperty(TestElement.TEST_CLASS, ThreadGroup.class.getName());
           threadGroup.setProperty(TestElement.GUI_CLASS, ThreadGroupGui.class.getName());
   
           // Test Plan
           TestPlan testPlan = new TestPlan("Create JMeter Script From Java Code");
           testPlan.setProperty(TestElement.TEST_CLASS, TestPlan.class.getName());
           testPlan.setProperty(TestElement.GUI_CLASS, TestPlanGui.class.getName());
           testPlan.setUserDefinedVariables((Arguments) new ArgumentsPanel().createTestElement());
   
           // Construct Test Plan from previously initialized elements
           testPlanTree.add(testPlan);
           HashTree threadGroupHashTree = testPlanTree.add(testPlan, threadGroup);
           threadGroupHashTree.add(examplecomSampler);
   
           // save generated test plan to JMeter's .jmx file format
           SaveService.saveTree(testPlanTree, new FileOutputStream("/path/to/test.jmx"));
   
           //add Summarizer output to get test progress in stdout like:
           // summary =      2 in   1.3s =    1.5/s Avg:   631 Min:   290 Max:   973 Err:     0 (0.00%)
           Summariser summer = null;
           String summariserName = JMeterUtils.getPropDefault("summariser.name", "summary");
           if (summariserName.length() > 0) {
               summer = new Summariser(summariserName);
           }
   
   
           // Store execution results into a .jtl file
           String logFile = "/path/to/test/results.jtl";
           ResultCollector logger = new ResultCollector(summer);
           logger.setFilename(logFile);
           testPlanTree.add(testPlanTree.getArray()[0], logger);
   
           // Run Test Plan
           jmeter.configure(testPlanTree);
           jmeter.run();
   
           System.exit(0);
       }
   }
   

Check out Five Ways To Launch a JMeter Test without Using the JMeter GUI article for more information.