1
2
3
4
5
6
7
8
9
10
11
12
13 package com.eviware.soapui.tools;
14
15 import java.io.File;
16 import java.io.IOException;
17 import java.util.ArrayList;
18 import java.util.List;
19
20 import org.apache.commons.cli.CommandLine;
21
22 import com.eviware.soapui.SoapUI;
23 import com.eviware.soapui.impl.wsdl.WsdlProject;
24 import com.eviware.soapui.impl.wsdl.loadtest.WsdlLoadTest;
25 import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportLoadTestLogAction;
26 import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportStatisticsAction;
27 import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLog;
28 import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLogEntry;
29 import com.eviware.soapui.model.project.ProjectFactoryRegistry;
30 import com.eviware.soapui.model.testsuite.LoadTestRunContext;
31 import com.eviware.soapui.model.testsuite.LoadTestRunListener;
32 import com.eviware.soapui.model.testsuite.LoadTestRunner;
33 import com.eviware.soapui.model.testsuite.TestCase;
34 import com.eviware.soapui.model.testsuite.TestCaseRunContext;
35 import com.eviware.soapui.model.testsuite.TestCaseRunner;
36 import com.eviware.soapui.model.testsuite.TestStep;
37 import com.eviware.soapui.model.testsuite.TestStepResult;
38 import com.eviware.soapui.model.testsuite.TestSuite;
39 import com.eviware.soapui.model.testsuite.TestRunner.Status;
40 import com.eviware.soapui.settings.UISettings;
41 import com.eviware.soapui.support.SoapUIException;
42 import com.eviware.soapui.support.StringUtils;
43
44 /***
45 * Standalone test-runner used from maven-plugin, can also be used from
46 * command-line (see xdocs) or directly from other classes.
47 * <p>
48 * For standalone usage, set the project file (with setProjectFile) and other
49 * desired properties before calling run
50 * </p>
51 *
52 * @author Ole.Matzura
53 */
54
55 public class SoapUILoadTestRunner extends AbstractSoapUITestRunner implements LoadTestRunListener
56 {
57 private String testSuite;
58 private String testCase;
59 private String loadTest;
60 private boolean printReport;
61 private List<LoadTestRunner> failedTests = new ArrayList<LoadTestRunner>();
62 private int testCaseCount;
63 private int loadTestCount;
64 private int limit = -1;
65 private long threadCount = -1;
66 private boolean saveAfterRun;
67
68 public static String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " LoadTest Runner";
69
70 /***
71 * Runs the loadtests in the specified soapUI project file, see soapUI xdocs
72 * for details.
73 *
74 * @param args
75 * @throws Exception
76 */
77
78 public static void main( String[] args )
79 {
80 System.exit( new SoapUILoadTestRunner().runFromCommandLine( args ) );
81 }
82
83 protected boolean processCommandLine( CommandLine cmd )
84 {
85 String message = "";
86
87 if( cmd.hasOption( "e" ) )
88 setEndpoint( cmd.getOptionValue( "e" ) );
89
90 if( cmd.hasOption( "s" ) )
91 {
92 String testSuite = getCommandLineOptionSubstSpace( cmd, "s" );
93 setTestSuite( testSuite );
94 message += validateTestSuite();
95 }
96 if( cmd.hasOption( "c" ) )
97 {
98 String testCase = cmd.getOptionValue( "c" );
99 setTestCase( testCase );
100 message += validateTestCase();
101 }
102
103 if( cmd.hasOption( "l" ) )
104 setLoadTest( cmd.getOptionValue( "l" ) );
105
106 if( cmd.hasOption( "u" ) )
107 setUsername( cmd.getOptionValue( "u" ) );
108
109 if( cmd.hasOption( "p" ) )
110 setPassword( cmd.getOptionValue( "p" ) );
111
112 if( cmd.hasOption( "w" ) )
113 setWssPasswordType( cmd.getOptionValue( "w" ) );
114
115 if( cmd.hasOption( "d" ) )
116 setDomain( cmd.getOptionValue( "d" ) );
117
118 if( cmd.hasOption( "h" ) )
119 setHost( cmd.getOptionValue( "h" ) );
120
121 if( cmd.hasOption( "m" ) )
122 setLimit( Integer.parseInt( cmd.getOptionValue( "m" ) ) );
123
124 if( cmd.hasOption( "n" ) )
125 setThreadCount( Integer.parseInt( cmd.getOptionValue( "n" ) ) );
126
127 if( cmd.hasOption( "f" ) )
128 setOutputFolder( getCommandLineOptionSubstSpace( cmd, "f" ) );
129
130 if( cmd.hasOption( "t" ) )
131 setSettingsFile( getCommandLineOptionSubstSpace( cmd, "t" ) );
132
133 setPrintReport( cmd.hasOption( "r" ) );
134 setSaveAfterRun( cmd.hasOption( "S" ) );
135
136 if( cmd.hasOption( "x" ) )
137 {
138 setProjectPassword( cmd.getOptionValue( "x" ) );
139 }
140
141 if( cmd.hasOption( "v" ) )
142 {
143 setSoapUISettingsPassword( cmd.getOptionValue( "v" ) );
144 }
145
146 if( cmd.hasOption( "D" ) )
147 {
148 setSystemProperties( cmd.getOptionValues( "D" ) );
149 }
150
151 if( cmd.hasOption( "G" ) )
152 {
153 setGlobalProperties( cmd.getOptionValues( "G" ) );
154 }
155
156 if( cmd.hasOption( "P" ) )
157 {
158 setProjectProperties( cmd.getOptionValues( "P" ) );
159 }
160
161 if( message.length() > 0 )
162 {
163 log.error( message );
164 return false;
165 }
166
167 return true;
168 }
169
170 private String validateTestCase()
171 {
172 WsdlProject project = ( WsdlProject )ProjectFactoryRegistry.getProjectFactory( "wsdl" ).createNew(
173 getProjectFile(), getProjectPassword() );
174
175 if( project.getTestSuiteByName( testSuite ) == null )
176 return "Test Suite with name:'" + testSuite + "' is missing from project:'" + project.getName() + "' \n";
177
178 if( project.getTestSuiteByName( testSuite ).getTestCaseByName( testCase ) == null )
179 return "Test Case with name:'" + testCase + "' is missing from testSuite:'" + testSuite + "' \n";
180
181 return "";
182 }
183
184 private String validateTestSuite()
185 {
186 WsdlProject project = ( WsdlProject )ProjectFactoryRegistry.getProjectFactory( "wsdl" ).createNew(
187 getProjectFile(), getProjectPassword() );
188
189 if( project.getTestSuiteByName( testSuite ) == null )
190 return "Test Suite with name:'" + testSuite + "' is missing from project:'" + project.getName() + "' \n";
191
192 return "";
193
194 }
195
196 public void setLimit( int limit )
197 {
198 this.limit = limit;
199 }
200
201 public void setThreadCount( long threadCount )
202 {
203 this.threadCount = threadCount;
204 }
205
206 protected SoapUIOptions initCommandLineOptions()
207 {
208 SoapUIOptions options = new SoapUIOptions( "loadtestrunner" );
209 options.addOption( "e", true, "Sets the endpoint" );
210 options.addOption( "s", true, "Sets the testsuite" );
211 options.addOption( "c", true, "Sets the testcase" );
212 options.addOption( "l", true, "Sets the loadtest" );
213 options.addOption( "u", true, "Sets the username" );
214 options.addOption( "p", true, "Sets the password" );
215 options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
216 options.addOption( "d", true, "Sets the domain" );
217 options.addOption( "h", true, "Sets the host" );
218 options.addOption( "m", true, "Overrides the LoadTest Limit" );
219 options.addOption( "n", true, "Overrides the LoadTest ThreadCount" );
220 options.addOption( "r", false, "Exports statistics and testlogs for each LoadTest run" );
221 options.addOption( "f", true, "Sets the output folder to export to" );
222 options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
223 options.addOption( "x", true, "Sets project password for decryption if project is encrypted" );
224 options.addOption( "v", true, "Sets password for soapui-settings.xml file" );
225 options.addOption( "D", true, "Sets system property with name=value" );
226 options.addOption( "G", true, "Sets global property with name=value" );
227 options.addOption( "P", true, "Sets or overrides project property with name=value" );
228 options.addOption( "S", false, "Saves the project after running the tests" );
229
230 return options;
231 }
232
233 public SoapUILoadTestRunner()
234 {
235 this( TITLE );
236 }
237
238 public SoapUILoadTestRunner( String title )
239 {
240 super( title );
241 }
242
243 public void setLoadTest( String loadTest )
244 {
245 this.loadTest = loadTest;
246 }
247
248 public void setPrintReport( boolean printReport )
249 {
250 this.printReport = printReport;
251 }
252
253 public void setSaveAfterRun( boolean saveAfterRun )
254 {
255 this.saveAfterRun = saveAfterRun;
256 }
257
258 /***
259 * Runs the testcases as configured with setXXX methods
260 *
261 * @throws Exception
262 * thrown if any tests fail
263 */
264
265 public boolean runRunner() throws Exception
266 {
267 if( SoapUI.getSettings().getBoolean( UISettings.DONT_DISABLE_GROOVY_LOG ) )
268 {
269 initGroovyLog();
270 }
271
272 String projectFile = getProjectFile();
273
274
275
276 WsdlProject project = ( WsdlProject )ProjectFactoryRegistry.getProjectFactory( "wsdl" ).createNew( projectFile,
277 getProjectPassword() );
278
279 if( project.isDisabled() )
280 throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
281
282 initProjectProperties( project );
283
284 int suiteCount = 0;
285
286 for( int c = 0; c < project.getTestSuiteCount(); c++ )
287 {
288 if( testSuite == null || project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ) )
289 {
290 runSuite( project.getTestSuiteAt( c ) );
291 suiteCount++ ;
292 }
293 }
294
295 if( suiteCount == 0 )
296 {
297 log.warn( "No test-suites matched argument [" + testSuite + "]" );
298 }
299 else if( testCaseCount == 0 )
300 {
301 log.warn( "No test-cases matched argument [" + testCase + "]" );
302 }
303 else if( loadTestCount == 0 )
304 {
305 log.warn( "No load-tests matched argument [" + loadTest + "]" );
306 }
307 else
308 {
309 if( saveAfterRun && !project.isRemote() )
310 {
311 try
312 {
313 project.save();
314 }
315 catch( Throwable t )
316 {
317 log.error( "Failed to save project", t );
318 }
319 }
320
321 if( !failedTests.isEmpty() )
322 {
323 log.info( failedTests.size() + " load tests failed:" );
324 for( LoadTestRunner loadTestRunner : failedTests )
325 {
326 log.info( loadTestRunner.getLoadTest().getName() + ": " + loadTestRunner.getReason() );
327 }
328
329 throw new SoapUIException( "LoadTests failed" );
330 }
331 }
332
333 return true;
334 }
335
336 /***
337 * Run tests in the specified TestSuite
338 *
339 * @param suite
340 * the TestSuite to run
341 */
342
343 public void runSuite( TestSuite suite )
344 {
345 long start = System.currentTimeMillis();
346 for( int c = 0; c < suite.getTestCaseCount(); c++ )
347 {
348 String name = suite.getTestCaseAt( c ).getName();
349 if( testCase == null || name.equalsIgnoreCase( testCase ) )
350 {
351 runTestCase( suite.getTestCaseAt( c ) );
352 testCaseCount++ ;
353 }
354 else
355 log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]" );
356 }
357 log.info( "soapUI suite [" + suite.getName() + "] finished in " + ( System.currentTimeMillis() - start ) + "ms" );
358 }
359
360 /***
361 * Runs the specified TestCase
362 *
363 * @param testCase
364 * the testcase to run
365 */
366
367 private void runTestCase( TestCase testCase )
368 {
369 for( int c = 0; c < testCase.getLoadTestCount(); c++ )
370 {
371 String name = testCase.getLoadTestAt( c ).getName();
372 if( loadTest == null || loadTest.equalsIgnoreCase( name ) )
373 {
374 runWsdlLoadTest( ( WsdlLoadTest )testCase.getLoadTestAt( c ) );
375 loadTestCount++ ;
376 }
377 }
378 }
379
380 /***
381 * Runs the specified LoadTest
382 *
383 * @param loadTest
384 * the loadTest to run
385 */
386
387 protected void runWsdlLoadTest( WsdlLoadTest loadTest )
388 {
389 try
390 {
391 log.info( "Running LoadTest [" + loadTest.getName() + "]" );
392 if( limit >= 0 )
393 {
394 log.info( "Overriding limit [" + loadTest.getTestLimit() + "] with specified [" + limit + "]" );
395 loadTest.setTestLimit( limit );
396 }
397
398 if( threadCount >= 0 )
399 {
400 log
401 .info( "Overriding threadCount [" + loadTest.getThreadCount() + "] with specified [" + threadCount
402 + "]" );
403 loadTest.setThreadCount( threadCount );
404 }
405
406 loadTest.addLoadTestRunListener( this );
407 LoadTestRunner runner = loadTest.run();
408
409
410 while( !runner.hasStopped() )
411 {
412 if( runner.getStatus() == Status.RUNNING )
413 {
414 log.info( "LoadTest [" + loadTest.getName() + "] progress: " + runner.getProgress() + ", "
415 + runner.getRunningThreadCount() );
416 }
417 Thread.sleep( 1000 );
418 }
419
420 log.info( "LoadTest [" + loadTest.getName() + "] finished with status " + runner.getStatus().toString() );
421
422 if( printReport )
423 {
424 log.info( "Exporting log and statistics for LoadTest [" + loadTest.getName() + "]" );
425
426 loadTest.getStatisticsModel().finish();
427
428 exportLog( loadTest );
429 exportStatistics( loadTest );
430 }
431 }
432 catch( Exception e )
433 {
434 SoapUI.logError( e );
435 log.error( e );
436 }
437 }
438
439 private void exportStatistics( WsdlLoadTest loadTest ) throws IOException
440 {
441 ExportStatisticsAction exportStatisticsAction = new ExportStatisticsAction( loadTest.getStatisticsModel() );
442 String statisticsFileName = StringUtils.createFileName( loadTest.getName(), '_' ) + "-statistics.txt";
443 if( getOutputFolder() != null )
444 {
445 ensureOutputFolder( loadTest );
446 statisticsFileName = getAbsoluteOutputFolder( loadTest ) + File.separator + statisticsFileName;
447 }
448
449 int cnt = exportStatisticsAction.exportToFile( new File( statisticsFileName ) );
450 log.info( "Exported " + cnt + " statistics to [" + statisticsFileName + "]" );
451 }
452
453 private void exportLog( WsdlLoadTest loadTest ) throws IOException
454 {
455
456 LoadTestLog loadTestLog = loadTest.getLoadTestLog();
457 ExportLoadTestLogAction exportLoadTestLogAction = new ExportLoadTestLogAction( loadTestLog, null );
458 String logFileName = StringUtils.createFileName( loadTest.getName(), '_' ) + "-log.txt";
459 if( getOutputFolder() != null )
460 {
461 ensureOutputFolder( loadTest );
462 logFileName = getAbsoluteOutputFolder( loadTest ) + File.separator + logFileName;
463 }
464
465 int cnt = exportLoadTestLogAction.exportToFile( new File( logFileName ) );
466 log.info( "Exported " + cnt + " log items to [" + logFileName + "]" );
467
468 int errorCnt = 0;
469 for( int c = 0; c < loadTestLog.getSize(); c++ )
470 {
471 LoadTestLogEntry entry = ( LoadTestLogEntry )loadTestLog.getElementAt( c );
472
473 if( entry != null && entry.isError() )
474 {
475 String entryFileName = StringUtils.createFileName( loadTest.getName(), '_' ) + "-error-" + errorCnt++
476 + "-entry.txt";
477 if( getOutputFolder() != null )
478 {
479 ensureOutputFolder( loadTest );
480 entryFileName = getAbsoluteOutputFolder( loadTest ) + File.separator + entryFileName;
481 }
482
483 try
484 {
485 entry.exportToFile( entryFileName );
486 }
487 catch( Exception e )
488 {
489 SoapUI.logError( e );
490 }
491 }
492 }
493 log.info( "Exported " + errorCnt + " error results" );
494 }
495
496 /***
497 * Sets the testcase to run
498 *
499 * @param testCase
500 * the testcase to run
501 */
502
503 public void setTestCase( String testCase )
504 {
505 this.testCase = testCase;
506 }
507
508 /***
509 * Sets the TestSuite to run. If not set all TestSuites in the specified
510 * project file are run
511 *
512 * @param testSuite
513 * the testSuite to run.
514 */
515
516 public void setTestSuite( String testSuite )
517 {
518 this.testSuite = testSuite;
519 }
520
521 public void afterLoadTest( LoadTestRunner loadTestRunner, LoadTestRunContext context )
522 {
523 if( loadTestRunner.getStatus() == LoadTestRunner.Status.FAILED )
524 {
525 failedTests.add( loadTestRunner );
526 }
527 }
528
529 public void afterTestCase( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestCaseRunner testRunner,
530 TestCaseRunContext runContext )
531 {
532 }
533
534 public void afterTestStep( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestCaseRunner testRunner,
535 TestCaseRunContext runContext, TestStepResult testStepResult )
536 {
537 super.afterStep( testRunner, runContext, testStepResult );
538 }
539
540 public void beforeLoadTest( LoadTestRunner loadTestRunner, LoadTestRunContext context )
541 {
542 }
543
544 public void beforeTestCase( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestCaseRunner testRunner,
545 TestCaseRunContext runContext )
546 {
547 }
548
549 public void beforeTestStep( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestCaseRunner testRunner,
550 TestCaseRunContext runContext, TestStep testStep )
551 {
552 super.beforeStep( testRunner, runContext, testStep );
553 }
554
555 public void loadTestStarted( LoadTestRunner loadTestRunner, LoadTestRunContext context )
556 {
557 }
558
559 public void loadTestStopped( LoadTestRunner loadTestRunner, LoadTestRunContext context )
560 {
561 }
562 }