001/* 002 * Copyright 2008-2019 Ping Identity Corporation 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright (C) 2008-2019 Ping Identity Corporation 007 * 008 * This program is free software; you can redistribute it and/or modify 009 * it under the terms of the GNU General Public License (GPLv2 only) 010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 011 * as published by the Free Software Foundation. 012 * 013 * This program is distributed in the hope that it will be useful, 014 * but WITHOUT ANY WARRANTY; without even the implied warranty of 015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 016 * GNU General Public License for more details. 017 * 018 * You should have received a copy of the GNU General Public License 019 * along with this program; if not, see <http://www.gnu.org/licenses>. 020 */ 021package com.unboundid.ldap.sdk.examples; 022 023 024 025import java.io.IOException; 026import java.io.OutputStream; 027import java.io.Serializable; 028import java.text.ParseException; 029import java.util.ArrayList; 030import java.util.LinkedHashMap; 031import java.util.List; 032import java.util.Set; 033import java.util.concurrent.CyclicBarrier; 034import java.util.concurrent.atomic.AtomicBoolean; 035import java.util.concurrent.atomic.AtomicLong; 036 037import com.unboundid.ldap.sdk.Control; 038import com.unboundid.ldap.sdk.LDAPConnection; 039import com.unboundid.ldap.sdk.LDAPConnectionOptions; 040import com.unboundid.ldap.sdk.LDAPException; 041import com.unboundid.ldap.sdk.ResultCode; 042import com.unboundid.ldap.sdk.Version; 043import com.unboundid.ldap.sdk.controls.AssertionRequestControl; 044import com.unboundid.ldap.sdk.controls.PermissiveModifyRequestControl; 045import com.unboundid.ldap.sdk.controls.PostReadRequestControl; 046import com.unboundid.ldap.sdk.controls.PreReadRequestControl; 047import com.unboundid.util.ColumnFormatter; 048import com.unboundid.util.Debug; 049import com.unboundid.util.FixedRateBarrier; 050import com.unboundid.util.FormattableColumn; 051import com.unboundid.util.HorizontalAlignment; 052import com.unboundid.util.LDAPCommandLineTool; 053import com.unboundid.util.ObjectPair; 054import com.unboundid.util.OutputFormat; 055import com.unboundid.util.RateAdjustor; 056import com.unboundid.util.ResultCodeCounter; 057import com.unboundid.util.StaticUtils; 058import com.unboundid.util.ThreadSafety; 059import com.unboundid.util.ThreadSafetyLevel; 060import com.unboundid.util.ValuePattern; 061import com.unboundid.util.WakeableSleeper; 062import com.unboundid.util.args.ArgumentException; 063import com.unboundid.util.args.ArgumentParser; 064import com.unboundid.util.args.BooleanArgument; 065import com.unboundid.util.args.ControlArgument; 066import com.unboundid.util.args.FileArgument; 067import com.unboundid.util.args.FilterArgument; 068import com.unboundid.util.args.IntegerArgument; 069import com.unboundid.util.args.StringArgument; 070 071 072 073/** 074 * This class provides a tool that can be used to perform repeated modifications 075 * in an LDAP directory server using multiple threads. It can help provide an 076 * estimate of the modify performance that a directory server is able to 077 * achieve. The target entry DN may be a value pattern as described in the 078 * {@link ValuePattern} class. This makes it possible to modify a range of 079 * entries rather than repeatedly updating the same entry. 080 * <BR><BR> 081 * Some of the APIs demonstrated by this example include: 082 * <UL> 083 * <LI>Argument Parsing (from the {@code com.unboundid.util.args} 084 * package)</LI> 085 * <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util} 086 * package)</LI> 087 * <LI>LDAP Communication (from the {@code com.unboundid.ldap.sdk} 088 * package)</LI> 089 * <LI>Value Patterns (from the {@code com.unboundid.util} package)</LI> 090 * </UL> 091 * <BR><BR> 092 * All of the necessary information is provided using command line arguments. 093 * Supported arguments include those allowed by the {@link LDAPCommandLineTool} 094 * class, as well as the following additional arguments: 095 * <UL> 096 * <LI>"-b {entryDN}" or "--targetDN {baseDN}" -- specifies the DN of the 097 * entry to be modified. This must be provided. It may be a simple DN, 098 * or it may be a value pattern to express a range of entry DNs.</LI> 099 * <LI>"-A {name}" or "--attribute {name}" -- specifies the name of the 100 * attribute to modify. Multiple attributes may be modified by providing 101 * multiple instances of this argument. At least one attribute must be 102 * provided.</LI> 103 * <LI>"--valuePattern {pattern}" -- specifies the pattern to use to generate 104 * the value to use for each modification. If this argument is provided, 105 * then neither the "--valueLength" nor "--characterSet" arguments may be 106 * given.</LI> 107 * <LI>"-l {num}" or "--valueLength {num}" -- specifies the length in bytes to 108 * use for the values of the target attributes. If this is not provided, 109 * then a default length of 10 bytes will be used.</LI> 110 * <LI>"-C {chars}" or "--characterSet {chars}" -- specifies the set of 111 * characters that will be used to generate the values to use for the 112 * target attributes. It should only include ASCII characters. Values 113 * will be generated from randomly-selected characters from this set. If 114 * this is not provided, then a default set of lowercase alphabetic 115 * characters will be used.</LI> 116 * <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of 117 * concurrent threads to use when performing the modifications. If this 118 * is not provided, then a default of one thread will be used.</LI> 119 * <LI>"-i {sec}" or "--intervalDuration {sec}" -- specifies the length of 120 * time in seconds between lines out output. If this is not provided, 121 * then a default interval duration of five seconds will be used.</LI> 122 * <LI>"-I {num}" or "--numIntervals {num}" -- specifies the maximum number of 123 * intervals for which to run. If this is not provided, then it will 124 * run forever.</LI> 125 * <LI>"--iterationsBeforeReconnect {num}" -- specifies the number of modify 126 * iterations that should be performed on a connection before that 127 * connection is closed and replaced with a newly-established (and 128 * authenticated, if appropriate) connection.</LI> 129 * <LI>"-r {modifies-per-second}" or "--ratePerSecond {modifies-per-second}" 130 * -- specifies the target number of modifies to perform per second. It 131 * is still necessary to specify a sufficient number of threads for 132 * achieving this rate. If this option is not provided, then the tool 133 * will run at the maximum rate for the specified number of threads.</LI> 134 * <LI>"--variableRateData {path}" -- specifies the path to a file containing 135 * information needed to allow the tool to vary the target rate over time. 136 * If this option is not provided, then the tool will either use a fixed 137 * target rate as specified by the "--ratePerSecond" argument, or it will 138 * run at the maximum rate.</LI> 139 * <LI>"--generateSampleRateFile {path}" -- specifies the path to a file to 140 * which sample data will be written illustrating and describing the 141 * format of the file expected to be used in conjunction with the 142 * "--variableRateData" argument.</LI> 143 * <LI>"--warmUpIntervals {num}" -- specifies the number of intervals to 144 * complete before beginning overall statistics collection.</LI> 145 * <LI>"--timestampFormat {format}" -- specifies the format to use for 146 * timestamps included before each output line. The format may be one of 147 * "none" (for no timestamps), "with-date" (to include both the date and 148 * the time), or "without-date" (to include only time time).</LI> 149 * <LI>"-Y {authzID}" or "--proxyAs {authzID}" -- Use the proxied 150 * authorization v2 control to request that the operation be processed 151 * using an alternate authorization identity. In this case, the bind DN 152 * should be that of a user that has permission to use this control. The 153 * authorization identity may be a value pattern.</LI> 154 * <LI>"--suppressErrorResultCodes" -- Indicates that information about the 155 * result codes for failed operations should not be displayed.</LI> 156 * <LI>"-c" or "--csv" -- Generate output in CSV format rather than a 157 * display-friendly format.</LI> 158 * </UL> 159 */ 160@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 161public final class ModRate 162 extends LDAPCommandLineTool 163 implements Serializable 164{ 165 /** 166 * The serial version UID for this serializable class. 167 */ 168 private static final long serialVersionUID = 2709717414202815822L; 169 170 171 172 // Indicates whether a request has been made to stop running. 173 private final AtomicBoolean stopRequested; 174 175 // The argument used to indicate whether to generate output in CSV format. 176 private BooleanArgument csvFormat; 177 178 // Indicates that the tool should use the increment modification type instead 179 // of replace. 180 private BooleanArgument increment; 181 182 // Indicates that modify requests should include the permissive modify request 183 // control. 184 private BooleanArgument permissiveModify; 185 186 // The argument used to indicate whether to suppress information about error 187 // result codes. 188 private BooleanArgument suppressErrorsArgument; 189 190 // The argument used to indicate that a generic control should be included in 191 // the request. 192 private ControlArgument control; 193 194 // The argument used to specify a variable rate file. 195 private FileArgument sampleRateFile; 196 197 // The argument used to specify a variable rate file. 198 private FileArgument variableRateData; 199 200 // Indicates that modify requests should include the assertion request control 201 // with the specified filter. 202 private FilterArgument assertionFilter; 203 204 // The argument used to specify the collection interval. 205 private IntegerArgument collectionInterval; 206 207 // The increment amount to use when performing an increment instead of a 208 // replace. 209 private IntegerArgument incrementAmount; 210 211 // The argument used to specify the number of modify iterations on a 212 // connection before it is closed and re-established. 213 private IntegerArgument iterationsBeforeReconnect; 214 215 // The argument used to specify the number of intervals. 216 private IntegerArgument numIntervals; 217 218 // The argument used to specify the number of threads. 219 private IntegerArgument numThreads; 220 221 // The argument used to specify the seed to use for the random number 222 // generator. 223 private IntegerArgument randomSeed; 224 225 // The target rate of modifies per second. 226 private IntegerArgument ratePerSecond; 227 228 // The number of values to include in the replace modification. 229 private IntegerArgument valueCount; 230 231 // The argument used to specify the length of the values to generate. 232 private IntegerArgument valueLength; 233 234 // The number of warm-up intervals to perform. 235 private IntegerArgument warmUpIntervals; 236 237 // The argument used to specify the name of the attribute to modify. 238 private StringArgument attribute; 239 240 // The argument used to specify the set of characters to use when generating 241 // values. 242 private StringArgument characterSet; 243 244 // The argument used to specify the DNs of the entries to modify. 245 private StringArgument entryDN; 246 247 // Indicates that modify requests should include the post-read request control 248 // to request the specified attribute. 249 private StringArgument postReadAttribute; 250 251 // Indicates that modify requests should include the pre-read request control 252 // to request the specified attribute. 253 private StringArgument preReadAttribute; 254 255 // The argument used to specify the proxied authorization identity. 256 private StringArgument proxyAs; 257 258 // The argument used to specify the timestamp format. 259 private StringArgument timestampFormat; 260 261 // The argument used to specify the pattern to use to generate values. 262 private StringArgument valuePattern; 263 264 // The thread currently being used to run the searchrate tool. 265 private volatile Thread runningThread; 266 267 // A wakeable sleeper that will be used to sleep between reporting intervals. 268 private final WakeableSleeper sleeper; 269 270 271 272 /** 273 * Parse the provided command line arguments and make the appropriate set of 274 * changes. 275 * 276 * @param args The command line arguments provided to this program. 277 */ 278 public static void main(final String[] args) 279 { 280 final ResultCode resultCode = main(args, System.out, System.err); 281 if (resultCode != ResultCode.SUCCESS) 282 { 283 System.exit(resultCode.intValue()); 284 } 285 } 286 287 288 289 /** 290 * Parse the provided command line arguments and make the appropriate set of 291 * changes. 292 * 293 * @param args The command line arguments provided to this program. 294 * @param outStream The output stream to which standard out should be 295 * written. It may be {@code null} if output should be 296 * suppressed. 297 * @param errStream The output stream to which standard error should be 298 * written. It may be {@code null} if error messages 299 * should be suppressed. 300 * 301 * @return A result code indicating whether the processing was successful. 302 */ 303 public static ResultCode main(final String[] args, 304 final OutputStream outStream, 305 final OutputStream errStream) 306 { 307 final ModRate modRate = new ModRate(outStream, errStream); 308 return modRate.runTool(args); 309 } 310 311 312 313 /** 314 * Creates a new instance of this tool. 315 * 316 * @param outStream The output stream to which standard out should be 317 * written. It may be {@code null} if output should be 318 * suppressed. 319 * @param errStream The output stream to which standard error should be 320 * written. It may be {@code null} if error messages 321 * should be suppressed. 322 */ 323 public ModRate(final OutputStream outStream, final OutputStream errStream) 324 { 325 super(outStream, errStream); 326 327 stopRequested = new AtomicBoolean(false); 328 sleeper = new WakeableSleeper(); 329 } 330 331 332 333 /** 334 * Retrieves the name for this tool. 335 * 336 * @return The name for this tool. 337 */ 338 @Override() 339 public String getToolName() 340 { 341 return "modrate"; 342 } 343 344 345 346 /** 347 * Retrieves the description for this tool. 348 * 349 * @return The description for this tool. 350 */ 351 @Override() 352 public String getToolDescription() 353 { 354 return "Perform repeated modifications against " + 355 "an LDAP directory server."; 356 } 357 358 359 360 /** 361 * Retrieves the version string for this tool. 362 * 363 * @return The version string for this tool. 364 */ 365 @Override() 366 public String getToolVersion() 367 { 368 return Version.NUMERIC_VERSION_STRING; 369 } 370 371 372 373 /** 374 * Indicates whether this tool should provide support for an interactive mode, 375 * in which the tool offers a mode in which the arguments can be provided in 376 * a text-driven menu rather than requiring them to be given on the command 377 * line. If interactive mode is supported, it may be invoked using the 378 * "--interactive" argument. Alternately, if interactive mode is supported 379 * and {@link #defaultsToInteractiveMode()} returns {@code true}, then 380 * interactive mode may be invoked by simply launching the tool without any 381 * arguments. 382 * 383 * @return {@code true} if this tool supports interactive mode, or 384 * {@code false} if not. 385 */ 386 @Override() 387 public boolean supportsInteractiveMode() 388 { 389 return true; 390 } 391 392 393 394 /** 395 * Indicates whether this tool defaults to launching in interactive mode if 396 * the tool is invoked without any command-line arguments. This will only be 397 * used if {@link #supportsInteractiveMode()} returns {@code true}. 398 * 399 * @return {@code true} if this tool defaults to using interactive mode if 400 * launched without any command-line arguments, or {@code false} if 401 * not. 402 */ 403 @Override() 404 public boolean defaultsToInteractiveMode() 405 { 406 return true; 407 } 408 409 410 411 /** 412 * Indicates whether this tool should provide arguments for redirecting output 413 * to a file. If this method returns {@code true}, then the tool will offer 414 * an "--outputFile" argument that will specify the path to a file to which 415 * all standard output and standard error content will be written, and it will 416 * also offer a "--teeToStandardOut" argument that can only be used if the 417 * "--outputFile" argument is present and will cause all output to be written 418 * to both the specified output file and to standard output. 419 * 420 * @return {@code true} if this tool should provide arguments for redirecting 421 * output to a file, or {@code false} if not. 422 */ 423 @Override() 424 protected boolean supportsOutputFile() 425 { 426 return true; 427 } 428 429 430 431 /** 432 * Indicates whether this tool should default to interactively prompting for 433 * the bind password if a password is required but no argument was provided 434 * to indicate how to get the password. 435 * 436 * @return {@code true} if this tool should default to interactively 437 * prompting for the bind password, or {@code false} if not. 438 */ 439 @Override() 440 protected boolean defaultToPromptForBindPassword() 441 { 442 return true; 443 } 444 445 446 447 /** 448 * Indicates whether this tool supports the use of a properties file for 449 * specifying default values for arguments that aren't specified on the 450 * command line. 451 * 452 * @return {@code true} if this tool supports the use of a properties file 453 * for specifying default values for arguments that aren't specified 454 * on the command line, or {@code false} if not. 455 */ 456 @Override() 457 public boolean supportsPropertiesFile() 458 { 459 return true; 460 } 461 462 463 464 /** 465 * Indicates whether the LDAP-specific arguments should include alternate 466 * versions of all long identifiers that consist of multiple words so that 467 * they are available in both camelCase and dash-separated versions. 468 * 469 * @return {@code true} if this tool should provide multiple versions of 470 * long identifiers for LDAP-specific arguments, or {@code false} if 471 * not. 472 */ 473 @Override() 474 protected boolean includeAlternateLongIdentifiers() 475 { 476 return true; 477 } 478 479 480 481 /** 482 * {@inheritDoc} 483 */ 484 @Override() 485 protected boolean logToolInvocationByDefault() 486 { 487 return true; 488 } 489 490 491 492 /** 493 * Adds the arguments used by this program that aren't already provided by the 494 * generic {@code LDAPCommandLineTool} framework. 495 * 496 * @param parser The argument parser to which the arguments should be added. 497 * 498 * @throws ArgumentException If a problem occurs while adding the arguments. 499 */ 500 @Override() 501 public void addNonLDAPArguments(final ArgumentParser parser) 502 throws ArgumentException 503 { 504 String description = "The DN of the entry to modify. It may be a simple " + 505 "DN or a value pattern to specify a range of DN (e.g., " + 506 "\"uid=user.[1-1000],ou=People,dc=example,dc=com\"). See " + 507 ValuePattern.PUBLIC_JAVADOC_URL + " for complete details about the " + 508 "value pattern syntax. This must be provided."; 509 entryDN = new StringArgument('b', "entryDN", true, 1, "{dn}", description); 510 entryDN.setArgumentGroupName("Modification Arguments"); 511 entryDN.addLongIdentifier("entry-dn", true); 512 parser.addArgument(entryDN); 513 514 515 description = "The name of the attribute to modify. Multiple attributes " + 516 "may be specified by providing this argument multiple " + 517 "times. At least one attribute must be specified."; 518 attribute = new StringArgument('A', "attribute", true, 0, "{name}", 519 description); 520 attribute.setArgumentGroupName("Modification Arguments"); 521 parser.addArgument(attribute); 522 523 524 description = "The pattern to use to generate values for the replace " + 525 "modifications. If this is provided, then neither the " + 526 "--valueLength argument nor the --characterSet arguments " + 527 "may be provided."; 528 valuePattern = new StringArgument(null, "valuePattern", false, 1, 529 "{pattern}", description); 530 valuePattern.setArgumentGroupName("Modification Arguments"); 531 valuePattern.addLongIdentifier("value-pattern", true); 532 parser.addArgument(valuePattern); 533 534 535 description = "The length in bytes to use when generating values for the " + 536 "replace modifications. If this is not provided, then a " + 537 "default length of ten bytes will be used."; 538 valueLength = new IntegerArgument('l', "valueLength", false, 1, "{num}", 539 description, 1, Integer.MAX_VALUE); 540 valueLength.setArgumentGroupName("Modification Arguments"); 541 valueLength.addLongIdentifier("value-length", true); 542 parser.addArgument(valueLength); 543 544 545 description = "The number of values to include in replace " + 546 "modifications. If this is not provided, then a default " + 547 "of one value will be used."; 548 valueCount = new IntegerArgument(null, "valueCount", false, 1, "{num}", 549 description, 0, Integer.MAX_VALUE, 1); 550 valueCount.setArgumentGroupName("Modification Arguments"); 551 valueCount.addLongIdentifier("value-count", true); 552 parser.addArgument(valueCount); 553 554 555 description = "Indicates that the tool should use the increment " + 556 "modification type rather than the replace modification " + 557 "type."; 558 increment = new BooleanArgument(null, "increment", 1, description); 559 increment.setArgumentGroupName("Modification Arguments"); 560 parser.addArgument(increment); 561 562 563 description = "The amount by which to increment values when using the " + 564 "increment modification type. The amount may be negative " + 565 "if values should be decremented rather than incremented. " + 566 "If this is not provided, then a default increment amount " + 567 "of one will be used."; 568 incrementAmount = new IntegerArgument(null, "incrementAmount", false, 1, 569 null, description, Integer.MIN_VALUE, 570 Integer.MAX_VALUE, 1); 571 incrementAmount.setArgumentGroupName("Modification Arguments"); 572 incrementAmount.addLongIdentifier("increment-amount", true); 573 parser.addArgument(incrementAmount); 574 575 576 description = "The set of characters to use to generate the values for " + 577 "the modifications. It should only include ASCII " + 578 "characters. If this is not provided, then a default set " + 579 "of lowercase alphabetic characters will be used."; 580 characterSet = new StringArgument('C', "characterSet", false, 1, "{chars}", 581 description); 582 characterSet.setArgumentGroupName("Modification Arguments"); 583 characterSet.addLongIdentifier("character-set", true); 584 parser.addArgument(characterSet); 585 586 587 description = "Indicates that modify requests should include the " + 588 "assertion request control with the specified filter."; 589 assertionFilter = new FilterArgument(null, "assertionFilter", false, 1, 590 "{filter}", description); 591 assertionFilter.setArgumentGroupName("Request Control Arguments"); 592 assertionFilter.addLongIdentifier("assertion-filter", true); 593 parser.addArgument(assertionFilter); 594 595 596 description = "Indicates that modify requests should include the " + 597 "permissive modify request control."; 598 permissiveModify = new BooleanArgument(null, "permissiveModify", 1, 599 description); 600 permissiveModify.setArgumentGroupName("Request Control Arguments"); 601 permissiveModify.addLongIdentifier("permissive-modify", true); 602 parser.addArgument(permissiveModify); 603 604 605 description = "Indicates that modify requests should include the " + 606 "pre-read request control with the specified requested " + 607 "attribute. This argument may be provided multiple times " + 608 "to indicate that multiple requested attributes should be " + 609 "included in the pre-read request control."; 610 preReadAttribute = new StringArgument(null, "preReadAttribute", false, 0, 611 "{attribute}", description); 612 preReadAttribute.setArgumentGroupName("Request Control Arguments"); 613 preReadAttribute.addLongIdentifier("pre-read-attribute", true); 614 parser.addArgument(preReadAttribute); 615 616 617 description = "Indicates that modify requests should include the " + 618 "post-read request control with the specified requested " + 619 "attribute. This argument may be provided multiple times " + 620 "to indicate that multiple requested attributes should be " + 621 "included in the post-read request control."; 622 postReadAttribute = new StringArgument(null, "postReadAttribute", false, 0, 623 "{attribute}", description); 624 postReadAttribute.setArgumentGroupName("Request Control Arguments"); 625 postReadAttribute.addLongIdentifier("post-read-attribute", true); 626 parser.addArgument(postReadAttribute); 627 628 629 description = "Indicates that the proxied authorization control (as " + 630 "defined in RFC 4370) should be used to request that " + 631 "operations be processed using an alternate authorization " + 632 "identity. This may be a simple authorization ID or it " + 633 "may be a value pattern to specify a range of " + 634 "identities. See " + ValuePattern.PUBLIC_JAVADOC_URL + 635 " for complete details about the value pattern syntax."; 636 proxyAs = new StringArgument('Y', "proxyAs", false, 1, "{authzID}", 637 description); 638 proxyAs.setArgumentGroupName("Request Control Arguments"); 639 proxyAs.addLongIdentifier("proxy-as", true); 640 parser.addArgument(proxyAs); 641 642 643 description = "Indicates that modify requests should include the " + 644 "specified request control. This may be provided multiple " + 645 "times to include multiple request controls."; 646 control = new ControlArgument('J', "control", false, 0, null, description); 647 control.setArgumentGroupName("Request Control Arguments"); 648 parser.addArgument(control); 649 650 651 description = "The number of threads to use to perform the " + 652 "modifications. If this is not provided, a single thread " + 653 "will be used."; 654 numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}", 655 description, 1, Integer.MAX_VALUE, 1); 656 numThreads.setArgumentGroupName("Rate Management Arguments"); 657 numThreads.addLongIdentifier("num-threads", true); 658 parser.addArgument(numThreads); 659 660 661 description = "The length of time in seconds between output lines. If " + 662 "this is not provided, then a default interval of five " + 663 "seconds will be used."; 664 collectionInterval = new IntegerArgument('i', "intervalDuration", true, 1, 665 "{num}", description, 1, 666 Integer.MAX_VALUE, 5); 667 collectionInterval.setArgumentGroupName("Rate Management Arguments"); 668 collectionInterval.addLongIdentifier("interval-duration", true); 669 parser.addArgument(collectionInterval); 670 671 672 description = "The maximum number of intervals for which to run. If " + 673 "this is not provided, then the tool will run until it is " + 674 "interrupted."; 675 numIntervals = new IntegerArgument('I', "numIntervals", true, 1, "{num}", 676 description, 1, Integer.MAX_VALUE, 677 Integer.MAX_VALUE); 678 numIntervals.setArgumentGroupName("Rate Management Arguments"); 679 numIntervals.addLongIdentifier("num-intervals", true); 680 parser.addArgument(numIntervals); 681 682 description = "The number of modify iterations that should be processed " + 683 "on a connection before that connection is closed and " + 684 "replaced with a newly-established (and authenticated, if " + 685 "appropriate) connection. If this is not provided, then " + 686 "connections will not be periodically closed and " + 687 "re-established."; 688 iterationsBeforeReconnect = new IntegerArgument(null, 689 "iterationsBeforeReconnect", false, 1, "{num}", description, 0); 690 iterationsBeforeReconnect.setArgumentGroupName("Rate Management Arguments"); 691 iterationsBeforeReconnect.addLongIdentifier("iterations-before-reconnect", 692 true); 693 parser.addArgument(iterationsBeforeReconnect); 694 695 description = "The target number of modifies to perform per second. It " + 696 "is still necessary to specify a sufficient number of " + 697 "threads for achieving this rate. If neither this option " + 698 "nor --variableRateData is provided, then the tool will " + 699 "run at the maximum rate for the specified number of " + 700 "threads."; 701 ratePerSecond = new IntegerArgument('r', "ratePerSecond", false, 1, 702 "{modifies-per-second}", description, 703 1, Integer.MAX_VALUE); 704 ratePerSecond.setArgumentGroupName("Rate Management Arguments"); 705 ratePerSecond.addLongIdentifier("rate-per-second", true); 706 parser.addArgument(ratePerSecond); 707 708 final String variableRateDataArgName = "variableRateData"; 709 final String generateSampleRateFileArgName = "generateSampleRateFile"; 710 description = RateAdjustor.getVariableRateDataArgumentDescription( 711 generateSampleRateFileArgName); 712 variableRateData = new FileArgument(null, variableRateDataArgName, false, 1, 713 "{path}", description, true, true, true, 714 false); 715 variableRateData.setArgumentGroupName("Rate Management Arguments"); 716 variableRateData.addLongIdentifier("variable-rate-data", true); 717 parser.addArgument(variableRateData); 718 719 description = RateAdjustor.getGenerateSampleVariableRateFileDescription( 720 variableRateDataArgName); 721 sampleRateFile = new FileArgument(null, generateSampleRateFileArgName, 722 false, 1, "{path}", description, false, 723 true, true, false); 724 sampleRateFile.setArgumentGroupName("Rate Management Arguments"); 725 sampleRateFile.addLongIdentifier("generate-sample-rate-file", true); 726 sampleRateFile.setUsageArgument(true); 727 parser.addArgument(sampleRateFile); 728 parser.addExclusiveArgumentSet(variableRateData, sampleRateFile); 729 730 description = "The number of intervals to complete before beginning " + 731 "overall statistics collection. Specifying a nonzero " + 732 "number of warm-up intervals gives the client and server " + 733 "a chance to warm up without skewing performance results."; 734 warmUpIntervals = new IntegerArgument(null, "warmUpIntervals", true, 1, 735 "{num}", description, 0, Integer.MAX_VALUE, 0); 736 warmUpIntervals.setArgumentGroupName("Rate Management Arguments"); 737 warmUpIntervals.addLongIdentifier("warm-up-intervals", true); 738 parser.addArgument(warmUpIntervals); 739 740 description = "Indicates the format to use for timestamps included in " + 741 "the output. A value of 'none' indicates that no " + 742 "timestamps should be included. A value of 'with-date' " + 743 "indicates that both the date and the time should be " + 744 "included. A value of 'without-date' indicates that only " + 745 "the time should be included."; 746 final Set<String> allowedFormats = 747 StaticUtils.setOf("none", "with-date", "without-date"); 748 timestampFormat = new StringArgument(null, "timestampFormat", true, 1, 749 "{format}", description, allowedFormats, "none"); 750 timestampFormat.addLongIdentifier("timestamp-format", true); 751 parser.addArgument(timestampFormat); 752 753 description = "Indicates that information about the result codes for " + 754 "failed operations should not be displayed."; 755 suppressErrorsArgument = new BooleanArgument(null, 756 "suppressErrorResultCodes", 1, description); 757 suppressErrorsArgument.addLongIdentifier("suppress-error-result-codes", 758 true); 759 parser.addArgument(suppressErrorsArgument); 760 761 description = "Generate output in CSV format rather than a " + 762 "display-friendly format"; 763 csvFormat = new BooleanArgument('c', "csv", 1, description); 764 parser.addArgument(csvFormat); 765 766 description = "Specifies the seed to use for the random number generator."; 767 randomSeed = new IntegerArgument('R', "randomSeed", false, 1, "{value}", 768 description); 769 randomSeed.addLongIdentifier("random-seed", true); 770 parser.addArgument(randomSeed); 771 772 773 // The incrementAmount argument can only be used if the increment argument 774 // is provided. 775 parser.addDependentArgumentSet(incrementAmount, increment); 776 777 778 // None of the valueLength, valueCount, characterSet, or valuePattern 779 // arguments can be used if the increment argument is provided. 780 parser.addExclusiveArgumentSet(increment, valueLength); 781 parser.addExclusiveArgumentSet(increment, valueCount); 782 parser.addExclusiveArgumentSet(increment, characterSet); 783 parser.addExclusiveArgumentSet(increment, valuePattern); 784 785 786 // The valuePattern argument cannot be used with either the valueLength or 787 // characterSet arguments. 788 parser.addExclusiveArgumentSet(valuePattern, valueLength); 789 parser.addExclusiveArgumentSet(valuePattern, characterSet); 790 } 791 792 793 794 /** 795 * Indicates whether this tool supports creating connections to multiple 796 * servers. If it is to support multiple servers, then the "--hostname" and 797 * "--port" arguments will be allowed to be provided multiple times, and 798 * will be required to be provided the same number of times. The same type of 799 * communication security and bind credentials will be used for all servers. 800 * 801 * @return {@code true} if this tool supports creating connections to 802 * multiple servers, or {@code false} if not. 803 */ 804 @Override() 805 protected boolean supportsMultipleServers() 806 { 807 return true; 808 } 809 810 811 812 /** 813 * Retrieves the connection options that should be used for connections 814 * created for use with this tool. 815 * 816 * @return The connection options that should be used for connections created 817 * for use with this tool. 818 */ 819 @Override() 820 public LDAPConnectionOptions getConnectionOptions() 821 { 822 final LDAPConnectionOptions options = new LDAPConnectionOptions(); 823 options.setUseSynchronousMode(true); 824 return options; 825 } 826 827 828 829 /** 830 * Performs the actual processing for this tool. In this case, it gets a 831 * connection to the directory server and uses it to perform the requested 832 * modifications. 833 * 834 * @return The result code for the processing that was performed. 835 */ 836 @Override() 837 public ResultCode doToolProcessing() 838 { 839 runningThread = Thread.currentThread(); 840 841 try 842 { 843 return doToolProcessingInternal(); 844 } 845 finally 846 { 847 runningThread = null; 848 } 849 850 } 851 852 853 /** 854 * Performs the actual processing for this tool. In this case, it gets a 855 * connection to the directory server and uses it to perform the requested 856 * modifications. 857 * 858 * @return The result code for the processing that was performed. 859 */ 860 private ResultCode doToolProcessingInternal() 861 { 862 // If the sample rate file argument was specified, then generate the sample 863 // variable rate data file and return. 864 if (sampleRateFile.isPresent()) 865 { 866 try 867 { 868 RateAdjustor.writeSampleVariableRateFile(sampleRateFile.getValue()); 869 return ResultCode.SUCCESS; 870 } 871 catch (final Exception e) 872 { 873 Debug.debugException(e); 874 err("An error occurred while trying to write sample variable data " + 875 "rate file '", sampleRateFile.getValue().getAbsolutePath(), 876 "': ", StaticUtils.getExceptionMessage(e)); 877 return ResultCode.LOCAL_ERROR; 878 } 879 } 880 881 882 // Determine the random seed to use. 883 final Long seed; 884 if (randomSeed.isPresent()) 885 { 886 seed = Long.valueOf(randomSeed.getValue()); 887 } 888 else 889 { 890 seed = null; 891 } 892 893 // Create the value patterns for the target entry DN and proxied 894 // authorization identities. 895 final ValuePattern dnPattern; 896 try 897 { 898 dnPattern = new ValuePattern(entryDN.getValue(), seed); 899 } 900 catch (final ParseException pe) 901 { 902 Debug.debugException(pe); 903 err("Unable to parse the entry DN value pattern: ", pe.getMessage()); 904 return ResultCode.PARAM_ERROR; 905 } 906 907 final ValuePattern authzIDPattern; 908 if (proxyAs.isPresent()) 909 { 910 try 911 { 912 authzIDPattern = new ValuePattern(proxyAs.getValue(), seed); 913 } 914 catch (final ParseException pe) 915 { 916 Debug.debugException(pe); 917 err("Unable to parse the proxied authorization pattern: ", 918 pe.getMessage()); 919 return ResultCode.PARAM_ERROR; 920 } 921 } 922 else 923 { 924 authzIDPattern = null; 925 } 926 927 928 // Get the set of controls to include in modify requests. 929 final ArrayList<Control> controlList = new ArrayList<>(5); 930 if (assertionFilter.isPresent()) 931 { 932 controlList.add(new AssertionRequestControl(assertionFilter.getValue())); 933 } 934 935 if (permissiveModify.isPresent()) 936 { 937 controlList.add(new PermissiveModifyRequestControl()); 938 } 939 940 if (preReadAttribute.isPresent()) 941 { 942 final List<String> attrList = preReadAttribute.getValues(); 943 final String[] attrArray = new String[attrList.size()]; 944 attrList.toArray(attrArray); 945 controlList.add(new PreReadRequestControl(attrArray)); 946 } 947 948 if (postReadAttribute.isPresent()) 949 { 950 final List<String> attrList = postReadAttribute.getValues(); 951 final String[] attrArray = new String[attrList.size()]; 952 attrList.toArray(attrArray); 953 controlList.add(new PostReadRequestControl(attrArray)); 954 } 955 956 if (control.isPresent()) 957 { 958 controlList.addAll(control.getValues()); 959 } 960 961 final Control[] controlArray = new Control[controlList.size()]; 962 controlList.toArray(controlArray); 963 964 965 // Get the names of the attributes to modify. 966 final String[] attrs = new String[attribute.getValues().size()]; 967 attribute.getValues().toArray(attrs); 968 969 970 // If the --ratePerSecond option was specified, then limit the rate 971 // accordingly. 972 FixedRateBarrier fixedRateBarrier = null; 973 if (ratePerSecond.isPresent() || variableRateData.isPresent()) 974 { 975 // We might not have a rate per second if --variableRateData is specified. 976 // The rate typically doesn't matter except when we have warm-up 977 // intervals. In this case, we'll run at the max rate. 978 final int intervalSeconds = collectionInterval.getValue(); 979 final int ratePerInterval = 980 (ratePerSecond.getValue() == null) 981 ? Integer.MAX_VALUE 982 : ratePerSecond.getValue() * intervalSeconds; 983 fixedRateBarrier = 984 new FixedRateBarrier(1000L * intervalSeconds, ratePerInterval); 985 } 986 987 988 // If --variableRateData was specified, then initialize a RateAdjustor. 989 RateAdjustor rateAdjustor = null; 990 if (variableRateData.isPresent()) 991 { 992 try 993 { 994 rateAdjustor = RateAdjustor.newInstance(fixedRateBarrier, 995 ratePerSecond.getValue(), variableRateData.getValue()); 996 } 997 catch (final IOException | IllegalArgumentException e) 998 { 999 Debug.debugException(e); 1000 err("Initializing the variable rates failed: " + e.getMessage()); 1001 return ResultCode.PARAM_ERROR; 1002 } 1003 } 1004 1005 1006 // Determine whether to include timestamps in the output and if so what 1007 // format should be used for them. 1008 final boolean includeTimestamp; 1009 final String timeFormat; 1010 if (timestampFormat.getValue().equalsIgnoreCase("with-date")) 1011 { 1012 includeTimestamp = true; 1013 timeFormat = "dd/MM/yyyy HH:mm:ss"; 1014 } 1015 else if (timestampFormat.getValue().equalsIgnoreCase("without-date")) 1016 { 1017 includeTimestamp = true; 1018 timeFormat = "HH:mm:ss"; 1019 } 1020 else 1021 { 1022 includeTimestamp = false; 1023 timeFormat = null; 1024 } 1025 1026 1027 // Determine whether any warm-up intervals should be run. 1028 final long totalIntervals; 1029 final boolean warmUp; 1030 int remainingWarmUpIntervals = warmUpIntervals.getValue(); 1031 if (remainingWarmUpIntervals > 0) 1032 { 1033 warmUp = true; 1034 totalIntervals = 0L + numIntervals.getValue() + remainingWarmUpIntervals; 1035 } 1036 else 1037 { 1038 warmUp = true; 1039 totalIntervals = 0L + numIntervals.getValue(); 1040 } 1041 1042 1043 // Create the table that will be used to format the output. 1044 final OutputFormat outputFormat; 1045 if (csvFormat.isPresent()) 1046 { 1047 outputFormat = OutputFormat.CSV; 1048 } 1049 else 1050 { 1051 outputFormat = OutputFormat.COLUMNS; 1052 } 1053 1054 final ColumnFormatter formatter = new ColumnFormatter(includeTimestamp, 1055 timeFormat, outputFormat, " ", 1056 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 1057 "Mods/Sec"), 1058 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 1059 "Avg Dur ms"), 1060 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 1061 "Errors/Sec"), 1062 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall", 1063 "Mods/Sec"), 1064 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall", 1065 "Avg Dur ms")); 1066 1067 1068 // Create values to use for statistics collection. 1069 final AtomicLong modCounter = new AtomicLong(0L); 1070 final AtomicLong errorCounter = new AtomicLong(0L); 1071 final AtomicLong modDurations = new AtomicLong(0L); 1072 final ResultCodeCounter rcCounter = new ResultCodeCounter(); 1073 1074 1075 // Determine the length of each interval in milliseconds. 1076 final long intervalMillis = 1000L * collectionInterval.getValue(); 1077 1078 1079 // Create the threads to use for the modifications. 1080 final CyclicBarrier barrier = new CyclicBarrier(numThreads.getValue() + 1); 1081 final ModRateThread[] threads = new ModRateThread[numThreads.getValue()]; 1082 for (int i=0; i < threads.length; i++) 1083 { 1084 final LDAPConnection connection; 1085 try 1086 { 1087 connection = getConnection(); 1088 } 1089 catch (final LDAPException le) 1090 { 1091 Debug.debugException(le); 1092 err("Unable to connect to the directory server: ", 1093 StaticUtils.getExceptionMessage(le)); 1094 return le.getResultCode(); 1095 } 1096 1097 final String valuePatternString; 1098 if (valuePattern.isPresent()) 1099 { 1100 valuePatternString = valuePattern.getValue(); 1101 } 1102 else 1103 { 1104 final int length; 1105 if (valueLength.isPresent()) 1106 { 1107 length = valueLength.getValue(); 1108 } 1109 else 1110 { 1111 length = 10; 1112 } 1113 1114 final String charSet; 1115 if (characterSet.isPresent()) 1116 { 1117 charSet = 1118 characterSet.getValue().replace("]", "]]").replace("[", "[["); 1119 } 1120 else 1121 { 1122 charSet = "abcdefghijklmnopqrstuvwxyz"; 1123 } 1124 1125 valuePatternString = "[random:" + length + ':' + charSet + ']'; 1126 } 1127 1128 final ValuePattern parsedValuePattern; 1129 try 1130 { 1131 parsedValuePattern = new ValuePattern(valuePatternString); 1132 } 1133 catch (final ParseException e) 1134 { 1135 Debug.debugException(e); 1136 err(e.getMessage()); 1137 return ResultCode.PARAM_ERROR; 1138 } 1139 1140 threads[i] = new ModRateThread(this, i, connection, dnPattern, attrs, 1141 parsedValuePattern, valueCount.getValue(), increment.isPresent(), 1142 incrementAmount.getValue(), controlArray, authzIDPattern, 1143 iterationsBeforeReconnect.getValue(), barrier, 1144 modCounter, modDurations, errorCounter, rcCounter, fixedRateBarrier); 1145 threads[i].start(); 1146 } 1147 1148 1149 // Display the table header. 1150 for (final String headerLine : formatter.getHeaderLines(true)) 1151 { 1152 out(headerLine); 1153 } 1154 1155 1156 // Start the RateAdjustor before the threads so that the initial value is 1157 // in place before any load is generated unless we're doing a warm-up in 1158 // which case, we'll start it after the warm-up is complete. 1159 if ((rateAdjustor != null) && (remainingWarmUpIntervals <= 0)) 1160 { 1161 rateAdjustor.start(); 1162 } 1163 1164 1165 // Indicate that the threads can start running. 1166 try 1167 { 1168 barrier.await(); 1169 } 1170 catch (final Exception e) 1171 { 1172 Debug.debugException(e); 1173 } 1174 1175 long overallStartTime = System.nanoTime(); 1176 long nextIntervalStartTime = System.currentTimeMillis() + intervalMillis; 1177 1178 1179 boolean setOverallStartTime = false; 1180 long lastDuration = 0L; 1181 long lastNumErrors = 0L; 1182 long lastNumMods = 0L; 1183 long lastEndTime = System.nanoTime(); 1184 for (long i=0; i < totalIntervals; i++) 1185 { 1186 if (rateAdjustor != null) 1187 { 1188 if (! rateAdjustor.isAlive()) 1189 { 1190 out("All of the rates in " + variableRateData.getValue().getName() + 1191 " have been completed."); 1192 break; 1193 } 1194 } 1195 1196 final long startTimeMillis = System.currentTimeMillis(); 1197 final long sleepTimeMillis = nextIntervalStartTime - startTimeMillis; 1198 nextIntervalStartTime += intervalMillis; 1199 if (sleepTimeMillis > 0) 1200 { 1201 sleeper.sleep(sleepTimeMillis); 1202 } 1203 1204 if (stopRequested.get()) 1205 { 1206 break; 1207 } 1208 1209 final long endTime = System.nanoTime(); 1210 final long intervalDuration = endTime - lastEndTime; 1211 1212 final long numMods; 1213 final long numErrors; 1214 final long totalDuration; 1215 if (warmUp && (remainingWarmUpIntervals > 0)) 1216 { 1217 numMods = modCounter.getAndSet(0L); 1218 numErrors = errorCounter.getAndSet(0L); 1219 totalDuration = modDurations.getAndSet(0L); 1220 } 1221 else 1222 { 1223 numMods = modCounter.get(); 1224 numErrors = errorCounter.get(); 1225 totalDuration = modDurations.get(); 1226 } 1227 1228 final long recentNumMods = numMods - lastNumMods; 1229 final long recentNumErrors = numErrors - lastNumErrors; 1230 final long recentDuration = totalDuration - lastDuration; 1231 1232 final double numSeconds = intervalDuration / 1_000_000_000.0d; 1233 final double recentModRate = recentNumMods / numSeconds; 1234 final double recentErrorRate = recentNumErrors / numSeconds; 1235 1236 final double recentAvgDuration; 1237 if (recentNumMods > 0L) 1238 { 1239 recentAvgDuration = 1.0d * recentDuration / recentNumMods / 1_000_000; 1240 } 1241 else 1242 { 1243 recentAvgDuration = 0.0d; 1244 } 1245 1246 if (warmUp && (remainingWarmUpIntervals > 0)) 1247 { 1248 out(formatter.formatRow(recentModRate, recentAvgDuration, 1249 recentErrorRate, "warming up", "warming up")); 1250 1251 remainingWarmUpIntervals--; 1252 if (remainingWarmUpIntervals == 0) 1253 { 1254 out("Warm-up completed. Beginning overall statistics collection."); 1255 setOverallStartTime = true; 1256 if (rateAdjustor != null) 1257 { 1258 rateAdjustor.start(); 1259 } 1260 } 1261 } 1262 else 1263 { 1264 if (setOverallStartTime) 1265 { 1266 overallStartTime = lastEndTime; 1267 setOverallStartTime = false; 1268 } 1269 1270 final double numOverallSeconds = 1271 (endTime - overallStartTime) / 1_000_000_000.0d; 1272 final double overallAuthRate = numMods / numOverallSeconds; 1273 1274 final double overallAvgDuration; 1275 if (numMods > 0L) 1276 { 1277 overallAvgDuration = 1.0d * totalDuration / numMods / 1_000_000; 1278 } 1279 else 1280 { 1281 overallAvgDuration = 0.0d; 1282 } 1283 1284 out(formatter.formatRow(recentModRate, recentAvgDuration, 1285 recentErrorRate, overallAuthRate, overallAvgDuration)); 1286 1287 lastNumMods = numMods; 1288 lastNumErrors = numErrors; 1289 lastDuration = totalDuration; 1290 } 1291 1292 final List<ObjectPair<ResultCode,Long>> rcCounts = 1293 rcCounter.getCounts(true); 1294 if ((! suppressErrorsArgument.isPresent()) && (! rcCounts.isEmpty())) 1295 { 1296 err("\tError Results:"); 1297 for (final ObjectPair<ResultCode,Long> p : rcCounts) 1298 { 1299 err("\t", p.getFirst().getName(), ": ", p.getSecond()); 1300 } 1301 } 1302 1303 lastEndTime = endTime; 1304 } 1305 1306 // Shut down the RateAdjustor if we have one. 1307 if (rateAdjustor != null) 1308 { 1309 rateAdjustor.shutDown(); 1310 } 1311 1312 // Stop all of the threads. 1313 ResultCode resultCode = ResultCode.SUCCESS; 1314 for (final ModRateThread t : threads) 1315 { 1316 final ResultCode r = t.stopRunning(); 1317 if (resultCode == ResultCode.SUCCESS) 1318 { 1319 resultCode = r; 1320 } 1321 } 1322 1323 return resultCode; 1324 } 1325 1326 1327 1328 /** 1329 * Requests that this tool stop running. This method will attempt to wait 1330 * for all threads to complete before returning control to the caller. 1331 */ 1332 public void stopRunning() 1333 { 1334 stopRequested.set(true); 1335 sleeper.wakeup(); 1336 1337 final Thread t = runningThread; 1338 if (t != null) 1339 { 1340 try 1341 { 1342 t.join(); 1343 } 1344 catch (final Exception e) 1345 { 1346 Debug.debugException(e); 1347 1348 if (e instanceof InterruptedException) 1349 { 1350 Thread.currentThread().interrupt(); 1351 } 1352 } 1353 } 1354 } 1355 1356 1357 1358 /** 1359 * {@inheritDoc} 1360 */ 1361 @Override() 1362 public LinkedHashMap<String[],String> getExampleUsages() 1363 { 1364 final LinkedHashMap<String[],String> examples = 1365 new LinkedHashMap<>(StaticUtils.computeMapCapacity(2)); 1366 1367 String[] args = 1368 { 1369 "--hostname", "server.example.com", 1370 "--port", "389", 1371 "--bindDN", "uid=admin,dc=example,dc=com", 1372 "--bindPassword", "password", 1373 "--entryDN", "uid=user.[1-1000000],ou=People,dc=example,dc=com", 1374 "--attribute", "description", 1375 "--valueLength", "12", 1376 "--numThreads", "10" 1377 }; 1378 String description = 1379 "Test modify performance by randomly selecting entries across a set " + 1380 "of one million users located below 'ou=People,dc=example,dc=com' " + 1381 "with ten concurrent threads and replacing the values for the " + 1382 "description attribute with a string of 12 randomly-selected " + 1383 "lowercase alphabetic characters."; 1384 examples.put(args, description); 1385 1386 args = new String[] 1387 { 1388 "--generateSampleRateFile", "variable-rate-data.txt" 1389 }; 1390 description = 1391 "Generate a sample variable rate definition file that may be used " + 1392 "in conjunction with the --variableRateData argument. The sample " + 1393 "file will include comments that describe the format for data to be " + 1394 "included in this file."; 1395 examples.put(args, description); 1396 1397 return examples; 1398 } 1399}