001/* 002 * Copyright 2016-2019 Ping Identity Corporation 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright (C) 2016-2019 Ping Identity Corporation 007 * 008 * This program is free software; you can redistribute it and/or modify 009 * it under the terms of the GNU General Public License (GPLv2 only) 010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 011 * as published by the Free Software Foundation. 012 * 013 * This program is distributed in the hope that it will be useful, 014 * but WITHOUT ANY WARRANTY; without even the implied warranty of 015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 016 * GNU General Public License for more details. 017 * 018 * You should have received a copy of the GNU General Public License 019 * along with this program; if not, see <http://www.gnu.org/licenses>. 020 */ 021package com.unboundid.ldap.sdk.transformations; 022 023 024 025import java.io.File; 026import java.io.FileOutputStream; 027import java.io.InputStream; 028import java.io.OutputStream; 029import java.util.ArrayList; 030import java.util.Iterator; 031import java.util.LinkedHashMap; 032import java.util.List; 033import java.util.TreeMap; 034import java.util.concurrent.atomic.AtomicLong; 035import java.util.zip.GZIPOutputStream; 036 037import com.unboundid.ldap.sdk.Attribute; 038import com.unboundid.ldap.sdk.DN; 039import com.unboundid.ldap.sdk.Entry; 040import com.unboundid.ldap.sdk.LDAPException; 041import com.unboundid.ldap.sdk.ResultCode; 042import com.unboundid.ldap.sdk.Version; 043import com.unboundid.ldap.sdk.schema.Schema; 044import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils; 045import com.unboundid.ldif.AggregateLDIFReaderChangeRecordTranslator; 046import com.unboundid.ldif.AggregateLDIFReaderEntryTranslator; 047import com.unboundid.ldif.LDIFException; 048import com.unboundid.ldif.LDIFReader; 049import com.unboundid.ldif.LDIFReaderChangeRecordTranslator; 050import com.unboundid.ldif.LDIFReaderEntryTranslator; 051import com.unboundid.ldif.LDIFRecord; 052import com.unboundid.util.ByteStringBuffer; 053import com.unboundid.util.CommandLineTool; 054import com.unboundid.util.Debug; 055import com.unboundid.util.ObjectPair; 056import com.unboundid.util.PassphraseEncryptedOutputStream; 057import com.unboundid.util.StaticUtils; 058import com.unboundid.util.ThreadSafety; 059import com.unboundid.util.ThreadSafetyLevel; 060import com.unboundid.util.args.ArgumentException; 061import com.unboundid.util.args.ArgumentParser; 062import com.unboundid.util.args.BooleanArgument; 063import com.unboundid.util.args.DNArgument; 064import com.unboundid.util.args.FileArgument; 065import com.unboundid.util.args.FilterArgument; 066import com.unboundid.util.args.IntegerArgument; 067import com.unboundid.util.args.ScopeArgument; 068import com.unboundid.util.args.StringArgument; 069 070import static com.unboundid.ldap.sdk.transformations.TransformationMessages.*; 071 072 073 074/** 075 * This class provides a command-line tool that can be used to apply a number of 076 * transformations to an LDIF file. The transformations that can be applied 077 * include: 078 * <UL> 079 * <LI> 080 * It can scramble the values of a specified set of attributes in a manner 081 * that attempts to preserve the syntax and consistently scrambles the same 082 * value to the same representation. 083 * </LI> 084 * <LI> 085 * It can strip a specified set of attributes out of entries. 086 * </LI> 087 * <LI> 088 * It can redact the values of a specified set of attributes, to indicate 089 * that the values are there but providing no information about what their 090 * values are. 091 * </LI> 092 * <LI> 093 * It can replace the values of a specified attribute with a given set of 094 * values. 095 * </LI> 096 * <LI> 097 * It can add an attribute with a given set of values to any entry that does 098 * not contain that attribute. 099 * </LI> 100 * <LI> 101 * It can replace the values of a specified attribute with a value that 102 * contains a sequentially-incrementing counter. 103 * </LI> 104 * <LI> 105 * It can strip entries matching a given base DN, scope, and filter out of 106 * the LDIF file. 107 * </LI> 108 * <LI> 109 * It can perform DN mapping, so that entries that exist below one base DN 110 * are moved below a different base DN. 111 * </LI> 112 * <LI> 113 * It can perform attribute mapping, to replace uses of one attribute name 114 * with another. 115 * </LI> 116 * </UL> 117 */ 118@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 119public final class TransformLDIF 120 extends CommandLineTool 121 implements LDIFReaderEntryTranslator 122{ 123 /** 124 * The maximum length of any message to write to standard output or standard 125 * error. 126 */ 127 private static final int MAX_OUTPUT_LINE_LENGTH = 128 StaticUtils.TERMINAL_WIDTH_COLUMNS - 1; 129 130 131 132 // The arguments for use by this program. 133 private BooleanArgument addToExistingValues = null; 134 private BooleanArgument appendToTargetLDIF = null; 135 private BooleanArgument compressTarget = null; 136 private BooleanArgument encryptTarget = null; 137 private BooleanArgument excludeNonMatchingEntries = null; 138 private BooleanArgument flattenAddOmittedRDNAttributesToEntry = null; 139 private BooleanArgument flattenAddOmittedRDNAttributesToRDN = null; 140 private BooleanArgument hideRedactedValueCount = null; 141 private BooleanArgument processDNs = null; 142 private BooleanArgument sourceCompressed = null; 143 private BooleanArgument sourceContainsChangeRecords = null; 144 private BooleanArgument sourceFromStandardInput = null; 145 private BooleanArgument targetToStandardOutput = null; 146 private DNArgument addAttributeBaseDN = null; 147 private DNArgument excludeEntryBaseDN = null; 148 private DNArgument flattenBaseDN = null; 149 private DNArgument moveSubtreeFrom = null; 150 private DNArgument moveSubtreeTo = null; 151 private FileArgument encryptionPassphraseFile = null; 152 private FileArgument schemaPath = null; 153 private FileArgument sourceLDIF = null; 154 private FileArgument targetLDIF = null; 155 private FilterArgument addAttributeFilter = null; 156 private FilterArgument excludeEntryFilter = null; 157 private FilterArgument flattenExcludeFilter = null; 158 private IntegerArgument initialSequentialValue = null; 159 private IntegerArgument numThreads = null; 160 private IntegerArgument randomSeed = null; 161 private IntegerArgument sequentialValueIncrement = null; 162 private IntegerArgument wrapColumn = null; 163 private ScopeArgument addAttributeScope = null; 164 private ScopeArgument excludeEntryScope = null; 165 private StringArgument addAttributeName = null; 166 private StringArgument addAttributeValue = null; 167 private StringArgument excludeAttribute = null; 168 private StringArgument redactAttribute = null; 169 private StringArgument renameAttributeFrom = null; 170 private StringArgument renameAttributeTo = null; 171 private StringArgument replaceValuesAttribute = null; 172 private StringArgument replacementValue = null; 173 private StringArgument scrambleAttribute = null; 174 private StringArgument scrambleJSONField = null; 175 private StringArgument sequentialAttribute = null; 176 private StringArgument textAfterSequentialValue = null; 177 private StringArgument textBeforeSequentialValue = null; 178 179 // A set of thread-local byte stream buffers that will be used to construct 180 // the LDIF representations of records. 181 private final ThreadLocal<ByteStringBuffer> byteStringBuffers = 182 new ThreadLocal<>(); 183 184 185 186 /** 187 * Invokes this tool with the provided set of arguments. 188 * 189 * @param args The command-line arguments provided to this program. 190 */ 191 public static void main(final String... args) 192 { 193 final ResultCode resultCode = main(System.out, System.err, args); 194 if (resultCode != ResultCode.SUCCESS) 195 { 196 System.exit(resultCode.intValue()); 197 } 198 } 199 200 201 202 /** 203 * Invokes this tool with the provided set of arguments. 204 * 205 * @param out The output stream to use for standard output. It may be 206 * {@code null} if standard output should be suppressed. 207 * @param err The output stream to use for standard error. It may be 208 * {@code null} if standard error should be suppressed. 209 * @param args The command-line arguments provided to this program. 210 * 211 * @return A result code indicating whether processing completed 212 * successfully. 213 */ 214 public static ResultCode main(final OutputStream out, final OutputStream err, 215 final String... args) 216 { 217 final TransformLDIF tool = new TransformLDIF(out, err); 218 return tool.runTool(args); 219 } 220 221 222 223 /** 224 * Creates a new instance of this tool with the provided information. 225 * 226 * @param out The output stream to use for standard output. It may be 227 * {@code null} if standard output should be suppressed. 228 * @param err The output stream to use for standard error. It may be 229 * {@code null} if standard error should be suppressed. 230 */ 231 public TransformLDIF(final OutputStream out, final OutputStream err) 232 { 233 super(out, err); 234 } 235 236 237 238 /** 239 * {@inheritDoc} 240 */ 241 @Override() 242 public String getToolName() 243 { 244 return "transform-ldif"; 245 } 246 247 248 249 /** 250 * {@inheritDoc} 251 */ 252 @Override() 253 public String getToolDescription() 254 { 255 return INFO_TRANSFORM_LDIF_TOOL_DESCRIPTION.get(); 256 } 257 258 259 260 /** 261 * {@inheritDoc} 262 */ 263 @Override() 264 public String getToolVersion() 265 { 266 return Version.NUMERIC_VERSION_STRING; 267 } 268 269 270 271 /** 272 * {@inheritDoc} 273 */ 274 @Override() 275 public boolean supportsInteractiveMode() 276 { 277 return true; 278 } 279 280 281 282 /** 283 * {@inheritDoc} 284 */ 285 @Override() 286 public boolean defaultsToInteractiveMode() 287 { 288 return true; 289 } 290 291 292 293 /** 294 * {@inheritDoc} 295 */ 296 @Override() 297 public boolean supportsPropertiesFile() 298 { 299 return true; 300 } 301 302 303 304 /** 305 * {@inheritDoc} 306 */ 307 @Override() 308 public void addToolArguments(final ArgumentParser parser) 309 throws ArgumentException 310 { 311 // Add arguments pertaining to the source and target LDIF files. 312 sourceLDIF = new FileArgument('l', "sourceLDIF", false, 0, null, 313 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_LDIF.get(), true, true, true, 314 false); 315 sourceLDIF.addLongIdentifier("inputLDIF", true); 316 sourceLDIF.addLongIdentifier("source-ldif", true); 317 sourceLDIF.addLongIdentifier("input-ldif", true); 318 sourceLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 319 parser.addArgument(sourceLDIF); 320 321 sourceFromStandardInput = new BooleanArgument(null, 322 "sourceFromStandardInput", 1, 323 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_STD_IN.get()); 324 sourceFromStandardInput.addLongIdentifier("source-from-standard-input", 325 true); 326 sourceFromStandardInput.setArgumentGroupName( 327 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 328 parser.addArgument(sourceFromStandardInput); 329 parser.addRequiredArgumentSet(sourceLDIF, sourceFromStandardInput); 330 parser.addExclusiveArgumentSet(sourceLDIF, sourceFromStandardInput); 331 332 targetLDIF = new FileArgument('o', "targetLDIF", false, 1, null, 333 INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_LDIF.get(), false, true, true, 334 false); 335 targetLDIF.addLongIdentifier("outputLDIF", true); 336 targetLDIF.addLongIdentifier("target-ldif", true); 337 targetLDIF.addLongIdentifier("output-ldif", true); 338 targetLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 339 parser.addArgument(targetLDIF); 340 341 targetToStandardOutput = new BooleanArgument(null, "targetToStandardOutput", 342 1, INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_STD_OUT.get()); 343 targetToStandardOutput.addLongIdentifier("target-to-standard-output", true); 344 targetToStandardOutput.setArgumentGroupName( 345 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 346 parser.addArgument(targetToStandardOutput); 347 parser.addExclusiveArgumentSet(targetLDIF, targetToStandardOutput); 348 349 sourceContainsChangeRecords = new BooleanArgument(null, 350 "sourceContainsChangeRecords", 351 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_CONTAINS_CHANGE_RECORDS.get()); 352 sourceContainsChangeRecords.addLongIdentifier( 353 "source-contains-change-records", true); 354 sourceContainsChangeRecords.setArgumentGroupName( 355 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 356 parser.addArgument(sourceContainsChangeRecords); 357 358 appendToTargetLDIF = new BooleanArgument(null, "appendToTargetLDIF", 359 INFO_TRANSFORM_LDIF_ARG_DESC_APPEND_TO_TARGET.get()); 360 appendToTargetLDIF.addLongIdentifier("append-to-target-ldif", true); 361 appendToTargetLDIF.setArgumentGroupName( 362 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 363 parser.addArgument(appendToTargetLDIF); 364 parser.addExclusiveArgumentSet(targetToStandardOutput, appendToTargetLDIF); 365 366 wrapColumn = new IntegerArgument(null, "wrapColumn", false, 1, null, 367 INFO_TRANSFORM_LDIF_ARG_DESC_WRAP_COLUMN.get(), 5, Integer.MAX_VALUE); 368 wrapColumn.addLongIdentifier("wrap-column", true); 369 wrapColumn.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 370 parser.addArgument(wrapColumn); 371 372 sourceCompressed = new BooleanArgument('C', "sourceCompressed", 373 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_COMPRESSED.get()); 374 sourceCompressed.addLongIdentifier("inputCompressed", true); 375 sourceCompressed.addLongIdentifier("source-compressed", true); 376 sourceCompressed.addLongIdentifier("input-compressed", true); 377 sourceCompressed.setArgumentGroupName( 378 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 379 parser.addArgument(sourceCompressed); 380 381 compressTarget = new BooleanArgument('c', "compressTarget", 382 INFO_TRANSFORM_LDIF_ARG_DESC_COMPRESS_TARGET.get()); 383 compressTarget.addLongIdentifier("compressOutput", true); 384 compressTarget.addLongIdentifier("compress", true); 385 compressTarget.addLongIdentifier("compress-target", true); 386 compressTarget.addLongIdentifier("compress-output", true); 387 compressTarget.setArgumentGroupName( 388 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 389 parser.addArgument(compressTarget); 390 391 encryptTarget = new BooleanArgument(null, "encryptTarget", 392 INFO_TRANSFORM_LDIF_ARG_DESC_ENCRYPT_TARGET.get()); 393 encryptTarget.addLongIdentifier("encryptOutput", true); 394 encryptTarget.addLongIdentifier("encrypt", true); 395 encryptTarget.addLongIdentifier("encrypt-target", true); 396 encryptTarget.addLongIdentifier("encrypt-output", true); 397 encryptTarget.setArgumentGroupName( 398 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 399 parser.addArgument(encryptTarget); 400 401 encryptionPassphraseFile = new FileArgument(null, 402 "encryptionPassphraseFile", false, 1, null, 403 INFO_TRANSFORM_LDIF_ARG_DESC_ENCRYPTION_PW_FILE.get(), true, true, 404 true, false); 405 encryptionPassphraseFile.addLongIdentifier("encryptionPasswordFile", true); 406 encryptionPassphraseFile.addLongIdentifier("encryption-passphrase-file", 407 true); 408 encryptionPassphraseFile.addLongIdentifier("encryption-password-file", 409 true); 410 encryptionPassphraseFile.setArgumentGroupName( 411 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 412 parser.addArgument(encryptionPassphraseFile); 413 414 415 // Add arguments pertaining to attribute scrambling. 416 scrambleAttribute = new StringArgument('a', "scrambleAttribute", false, 0, 417 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 418 INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_ATTR.get()); 419 scrambleAttribute.addLongIdentifier("attributeName", true); 420 scrambleAttribute.addLongIdentifier("scramble-attribute", true); 421 scrambleAttribute.addLongIdentifier("attribute-name", true); 422 scrambleAttribute.setArgumentGroupName( 423 INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get()); 424 parser.addArgument(scrambleAttribute); 425 426 scrambleJSONField = new StringArgument(null, "scrambleJSONField", false, 0, 427 INFO_TRANSFORM_LDIF_PLACEHOLDER_FIELD_NAME.get(), 428 INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_JSON_FIELD.get( 429 scrambleAttribute.getIdentifierString())); 430 scrambleJSONField.addLongIdentifier("scramble-json-field", true); 431 scrambleJSONField.setArgumentGroupName( 432 INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get()); 433 parser.addArgument(scrambleJSONField); 434 parser.addDependentArgumentSet(scrambleJSONField, scrambleAttribute); 435 436 randomSeed = new IntegerArgument('s', "randomSeed", false, 1, null, 437 INFO_TRANSFORM_LDIF_ARG_DESC_RANDOM_SEED.get()); 438 randomSeed.addLongIdentifier("random-seed", true); 439 randomSeed.setArgumentGroupName( 440 INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get()); 441 parser.addArgument(randomSeed); 442 443 444 // Add arguments pertaining to replacing attribute values with a generated 445 // value using a sequential counter. 446 sequentialAttribute = new StringArgument('S', "sequentialAttribute", 447 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 448 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_ATTR.get( 449 sourceContainsChangeRecords.getIdentifierString())); 450 sequentialAttribute.addLongIdentifier("sequentialAttributeName", true); 451 sequentialAttribute.addLongIdentifier("sequential-attribute", true); 452 sequentialAttribute.addLongIdentifier("sequential-attribute-name", true); 453 sequentialAttribute.setArgumentGroupName( 454 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 455 parser.addArgument(sequentialAttribute); 456 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 457 sequentialAttribute); 458 459 initialSequentialValue = new IntegerArgument('i', "initialSequentialValue", 460 false, 1, null, 461 INFO_TRANSFORM_LDIF_ARG_DESC_INITIAL_SEQUENTIAL_VALUE.get( 462 sequentialAttribute.getIdentifierString())); 463 initialSequentialValue.addLongIdentifier("initial-sequential-value", true); 464 initialSequentialValue.setArgumentGroupName( 465 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 466 parser.addArgument(initialSequentialValue); 467 parser.addDependentArgumentSet(initialSequentialValue, sequentialAttribute); 468 469 sequentialValueIncrement = new IntegerArgument(null, 470 "sequentialValueIncrement", false, 1, null, 471 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_INCREMENT.get( 472 sequentialAttribute.getIdentifierString())); 473 sequentialValueIncrement.addLongIdentifier("sequential-value-increment", 474 true); 475 sequentialValueIncrement.setArgumentGroupName( 476 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 477 parser.addArgument(sequentialValueIncrement); 478 parser.addDependentArgumentSet(sequentialValueIncrement, 479 sequentialAttribute); 480 481 textBeforeSequentialValue = new StringArgument(null, 482 "textBeforeSequentialValue", false, 1, null, 483 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_BEFORE.get( 484 sequentialAttribute.getIdentifierString())); 485 textBeforeSequentialValue.addLongIdentifier("text-before-sequential-value", 486 true); 487 textBeforeSequentialValue.setArgumentGroupName( 488 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 489 parser.addArgument(textBeforeSequentialValue); 490 parser.addDependentArgumentSet(textBeforeSequentialValue, 491 sequentialAttribute); 492 493 textAfterSequentialValue = new StringArgument(null, 494 "textAfterSequentialValue", false, 1, null, 495 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_AFTER.get( 496 sequentialAttribute.getIdentifierString())); 497 textAfterSequentialValue.addLongIdentifier("text-after-sequential-value", 498 true); 499 textAfterSequentialValue.setArgumentGroupName( 500 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 501 parser.addArgument(textAfterSequentialValue); 502 parser.addDependentArgumentSet(textAfterSequentialValue, 503 sequentialAttribute); 504 505 506 // Add arguments pertaining to attribute value replacement. 507 replaceValuesAttribute = new StringArgument(null, "replaceValuesAttribute", 508 false, 1, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 509 INFO_TRANSFORM_LDIF_ARG_DESC_REPLACE_VALUES_ATTR.get( 510 sourceContainsChangeRecords.getIdentifierString())); 511 replaceValuesAttribute.addLongIdentifier("replace-values-attribute", true); 512 replaceValuesAttribute.setArgumentGroupName( 513 INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get()); 514 parser.addArgument(replaceValuesAttribute); 515 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 516 replaceValuesAttribute); 517 518 replacementValue = new StringArgument(null, "replacementValue", false, 0, 519 null, 520 INFO_TRANSFORM_LDIF_ARG_DESC_REPLACEMENT_VALUE.get( 521 replaceValuesAttribute.getIdentifierString())); 522 replacementValue.addLongIdentifier("replacement-value", true); 523 replacementValue.setArgumentGroupName( 524 INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get()); 525 parser.addArgument(replacementValue); 526 parser.addDependentArgumentSet(replaceValuesAttribute, replacementValue); 527 parser.addDependentArgumentSet(replacementValue, replaceValuesAttribute); 528 529 530 // Add arguments pertaining to adding missing attributes. 531 addAttributeName = new StringArgument(null, "addAttributeName", false, 1, 532 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 533 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_ATTR.get( 534 "--addAttributeValue", 535 sourceContainsChangeRecords.getIdentifierString())); 536 addAttributeName.addLongIdentifier("add-attribute-name", true); 537 addAttributeName.setArgumentGroupName( 538 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 539 parser.addArgument(addAttributeName); 540 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 541 addAttributeName); 542 543 addAttributeValue = new StringArgument(null, "addAttributeValue", false, 0, 544 null, 545 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_VALUE.get( 546 addAttributeName.getIdentifierString())); 547 addAttributeValue.addLongIdentifier("add-attribute-value", true); 548 addAttributeValue.setArgumentGroupName( 549 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 550 parser.addArgument(addAttributeValue); 551 parser.addDependentArgumentSet(addAttributeName, addAttributeValue); 552 parser.addDependentArgumentSet(addAttributeValue, addAttributeName); 553 554 addToExistingValues = new BooleanArgument(null, "addToExistingValues", 555 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_MERGE_VALUES.get( 556 addAttributeName.getIdentifierString(), 557 addAttributeValue.getIdentifierString())); 558 addToExistingValues.addLongIdentifier("add-to-existing-values", true); 559 addToExistingValues.setArgumentGroupName( 560 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 561 parser.addArgument(addToExistingValues); 562 parser.addDependentArgumentSet(addToExistingValues, addAttributeName); 563 564 addAttributeBaseDN = new DNArgument(null, "addAttributeBaseDN", false, 1, 565 null, 566 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_BASE_DN.get( 567 addAttributeName.getIdentifierString())); 568 addAttributeBaseDN.addLongIdentifier("add-attribute-base-dn", true); 569 addAttributeBaseDN.setArgumentGroupName( 570 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 571 parser.addArgument(addAttributeBaseDN); 572 parser.addDependentArgumentSet(addAttributeBaseDN, addAttributeName); 573 574 addAttributeScope = new ScopeArgument(null, "addAttributeScope", false, 575 null, 576 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_SCOPE.get( 577 addAttributeBaseDN.getIdentifierString(), 578 addAttributeName.getIdentifierString())); 579 addAttributeScope.addLongIdentifier("add-attribute-scope", true); 580 addAttributeScope.setArgumentGroupName( 581 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 582 parser.addArgument(addAttributeScope); 583 parser.addDependentArgumentSet(addAttributeScope, addAttributeName); 584 585 addAttributeFilter = new FilterArgument(null, "addAttributeFilter", false, 586 1, null, 587 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_FILTER.get( 588 addAttributeName.getIdentifierString())); 589 addAttributeFilter.addLongIdentifier("add-attribute-filter", true); 590 addAttributeFilter.setArgumentGroupName( 591 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 592 parser.addArgument(addAttributeFilter); 593 parser.addDependentArgumentSet(addAttributeFilter, addAttributeName); 594 595 596 // Add arguments pertaining to renaming attributes. 597 renameAttributeFrom = new StringArgument(null, "renameAttributeFrom", 598 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 599 INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_FROM.get()); 600 renameAttributeFrom.addLongIdentifier("rename-attribute-from", true); 601 renameAttributeFrom.setArgumentGroupName( 602 INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get()); 603 parser.addArgument(renameAttributeFrom); 604 605 renameAttributeTo = new StringArgument(null, "renameAttributeTo", 606 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 607 INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_TO.get( 608 renameAttributeFrom.getIdentifierString())); 609 renameAttributeTo.addLongIdentifier("rename-attribute-to", true); 610 renameAttributeTo.setArgumentGroupName( 611 INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get()); 612 parser.addArgument(renameAttributeTo); 613 parser.addDependentArgumentSet(renameAttributeFrom, renameAttributeTo); 614 parser.addDependentArgumentSet(renameAttributeTo, renameAttributeFrom); 615 616 617 // Add arguments pertaining to flattening subtrees. 618 flattenBaseDN = new DNArgument(null, "flattenBaseDN", false, 1, null, 619 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_BASE_DN.get()); 620 flattenBaseDN.addLongIdentifier("flatten-base-dn", true); 621 flattenBaseDN.setArgumentGroupName( 622 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 623 parser.addArgument(flattenBaseDN); 624 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 625 flattenBaseDN); 626 627 flattenAddOmittedRDNAttributesToEntry = new BooleanArgument(null, 628 "flattenAddOmittedRDNAttributesToEntry", 1, 629 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_ENTRY.get()); 630 flattenAddOmittedRDNAttributesToEntry.addLongIdentifier( 631 "flatten-add-omitted-rdn-attributes-to-entry", true); 632 flattenAddOmittedRDNAttributesToEntry.setArgumentGroupName( 633 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 634 parser.addArgument(flattenAddOmittedRDNAttributesToEntry); 635 parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToEntry, 636 flattenBaseDN); 637 638 flattenAddOmittedRDNAttributesToRDN = new BooleanArgument(null, 639 "flattenAddOmittedRDNAttributesToRDN", 1, 640 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_RDN.get()); 641 flattenAddOmittedRDNAttributesToRDN.addLongIdentifier( 642 "flatten-add-omitted-rdn-attributes-to-rdn", true); 643 flattenAddOmittedRDNAttributesToRDN.setArgumentGroupName( 644 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 645 parser.addArgument(flattenAddOmittedRDNAttributesToRDN); 646 parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToRDN, 647 flattenBaseDN); 648 649 flattenExcludeFilter = new FilterArgument(null, "flattenExcludeFilter", 650 false, 1, null, 651 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_EXCLUDE_FILTER.get()); 652 flattenExcludeFilter.addLongIdentifier("flatten-exclude-filter", true); 653 flattenExcludeFilter.setArgumentGroupName( 654 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 655 parser.addArgument(flattenExcludeFilter); 656 parser.addDependentArgumentSet(flattenExcludeFilter, flattenBaseDN); 657 658 659 // Add arguments pertaining to moving subtrees. 660 moveSubtreeFrom = new DNArgument(null, "moveSubtreeFrom", false, 0, null, 661 INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_FROM.get()); 662 moveSubtreeFrom.addLongIdentifier("move-subtree-from", true); 663 moveSubtreeFrom.setArgumentGroupName( 664 INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get()); 665 parser.addArgument(moveSubtreeFrom); 666 667 moveSubtreeTo = new DNArgument(null, "moveSubtreeTo", false, 0, null, 668 INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_TO.get( 669 moveSubtreeFrom.getIdentifierString())); 670 moveSubtreeTo.addLongIdentifier("move-subtree-to", true); 671 moveSubtreeTo.setArgumentGroupName( 672 INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get()); 673 parser.addArgument(moveSubtreeTo); 674 parser.addDependentArgumentSet(moveSubtreeFrom, moveSubtreeTo); 675 parser.addDependentArgumentSet(moveSubtreeTo, moveSubtreeFrom); 676 677 678 // Add arguments pertaining to redacting attribute values. 679 redactAttribute = new StringArgument(null, "redactAttribute", false, 0, 680 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 681 INFO_TRANSFORM_LDIF_ARG_DESC_REDACT_ATTR.get()); 682 redactAttribute.addLongIdentifier("redact-attribute", true); 683 redactAttribute.setArgumentGroupName( 684 INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get()); 685 parser.addArgument(redactAttribute); 686 687 hideRedactedValueCount = new BooleanArgument(null, "hideRedactedValueCount", 688 INFO_TRANSFORM_LDIF_ARG_DESC_HIDE_REDACTED_COUNT.get()); 689 hideRedactedValueCount.addLongIdentifier("hide-redacted-value-count", 690 true); 691 hideRedactedValueCount.setArgumentGroupName( 692 INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get()); 693 parser.addArgument(hideRedactedValueCount); 694 parser.addDependentArgumentSet(hideRedactedValueCount, redactAttribute); 695 696 697 // Add arguments pertaining to excluding attributes and entries. 698 excludeAttribute = new StringArgument(null, "excludeAttribute", false, 0, 699 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 700 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ATTR.get()); 701 excludeAttribute.addLongIdentifier("suppressAttribute", true); 702 excludeAttribute.addLongIdentifier("exclude-attribute", true); 703 excludeAttribute.addLongIdentifier("suppress-attribute", true); 704 excludeAttribute.setArgumentGroupName( 705 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 706 parser.addArgument(excludeAttribute); 707 708 excludeEntryBaseDN = new DNArgument(null, "excludeEntryBaseDN", false, 1, 709 null, 710 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_BASE_DN.get( 711 sourceContainsChangeRecords.getIdentifierString())); 712 excludeEntryBaseDN.addLongIdentifier("suppressEntryBaseDN", true); 713 excludeEntryBaseDN.addLongIdentifier("exclude-entry-base-dn", true); 714 excludeEntryBaseDN.addLongIdentifier("suppress-entry-base-dn", true); 715 excludeEntryBaseDN.setArgumentGroupName( 716 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 717 parser.addArgument(excludeEntryBaseDN); 718 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 719 excludeEntryBaseDN); 720 721 excludeEntryScope = new ScopeArgument(null, "excludeEntryScope", false, 722 null, 723 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_SCOPE.get( 724 sourceContainsChangeRecords.getIdentifierString())); 725 excludeEntryScope.addLongIdentifier("suppressEntryScope", true); 726 excludeEntryScope.addLongIdentifier("exclude-entry-scope", true); 727 excludeEntryScope.addLongIdentifier("suppress-entry-scope", true); 728 excludeEntryScope.setArgumentGroupName( 729 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 730 parser.addArgument(excludeEntryScope); 731 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 732 excludeEntryScope); 733 734 excludeEntryFilter = new FilterArgument(null, "excludeEntryFilter", false, 735 1, null, 736 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_FILTER.get( 737 sourceContainsChangeRecords.getIdentifierString())); 738 excludeEntryFilter.addLongIdentifier("suppressEntryFilter", true); 739 excludeEntryFilter.addLongIdentifier("exclude-entry-filter", true); 740 excludeEntryFilter.addLongIdentifier("suppress-entry-filter", true); 741 excludeEntryFilter.setArgumentGroupName( 742 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 743 parser.addArgument(excludeEntryFilter); 744 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 745 excludeEntryFilter); 746 747 excludeNonMatchingEntries = new BooleanArgument(null, 748 "excludeNonMatchingEntries", 749 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_NON_MATCHING.get()); 750 excludeNonMatchingEntries.addLongIdentifier("exclude-non-matching-entries", 751 true); 752 excludeNonMatchingEntries.setArgumentGroupName( 753 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 754 parser.addArgument(excludeNonMatchingEntries); 755 parser.addDependentArgumentSet(excludeNonMatchingEntries, 756 excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter); 757 758 759 // Add the remaining arguments. 760 schemaPath = new FileArgument(null, "schemaPath", false, 0, null, 761 INFO_TRANSFORM_LDIF_ARG_DESC_SCHEMA_PATH.get(), 762 true, true, false, false); 763 schemaPath.addLongIdentifier("schemaFile", true); 764 schemaPath.addLongIdentifier("schemaDirectory", true); 765 schemaPath.addLongIdentifier("schema-path", true); 766 schemaPath.addLongIdentifier("schema-file", true); 767 schemaPath.addLongIdentifier("schema-directory", true); 768 parser.addArgument(schemaPath); 769 770 numThreads = new IntegerArgument('t', "numThreads", false, 1, null, 771 INFO_TRANSFORM_LDIF_ARG_DESC_NUM_THREADS.get(), 1, Integer.MAX_VALUE, 772 1); 773 numThreads.addLongIdentifier("num-threads", true); 774 parser.addArgument(numThreads); 775 776 processDNs = new BooleanArgument('d', "processDNs", 777 INFO_TRANSFORM_LDIF_ARG_DESC_PROCESS_DNS.get()); 778 processDNs.addLongIdentifier("process-dns", true); 779 parser.addArgument(processDNs); 780 781 782 // Ensure that at least one kind of transformation was requested. 783 parser.addRequiredArgumentSet(scrambleAttribute, sequentialAttribute, 784 replaceValuesAttribute, addAttributeName, renameAttributeFrom, 785 flattenBaseDN, moveSubtreeFrom, redactAttribute, excludeAttribute, 786 excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter); 787 } 788 789 790 791 /** 792 * {@inheritDoc} 793 */ 794 @Override() 795 public void doExtendedArgumentValidation() 796 throws ArgumentException 797 { 798 // Ideally, exactly one of the targetLDIF and targetToStandardOutput 799 // arguments should always be provided. But in order to preserve backward 800 // compatibility with a legacy scramble-ldif tool, we will allow both to be 801 // omitted if either --scrambleAttribute or --sequentialArgument is 802 // provided. In that case, the path of the output file will be the path of 803 // the first input file with ".scrambled" appended to it. 804 if (! (targetLDIF.isPresent() || targetToStandardOutput.isPresent())) 805 { 806 if (! (scrambleAttribute.isPresent() || sequentialAttribute.isPresent())) 807 { 808 throw new ArgumentException(ERR_TRANSFORM_LDIF_MISSING_TARGET_ARG.get( 809 targetLDIF.getIdentifierString(), 810 targetToStandardOutput.getIdentifierString())); 811 } 812 } 813 814 815 // Make sure that the --renameAttributeFrom and --renameAttributeTo 816 // arguments were provided an equal number of times. 817 final int renameFromOccurrences = renameAttributeFrom.getNumOccurrences(); 818 final int renameToOccurrences = renameAttributeTo.getNumOccurrences(); 819 if (renameFromOccurrences != renameToOccurrences) 820 { 821 throw new ArgumentException( 822 ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get( 823 renameAttributeFrom.getIdentifierString(), 824 renameAttributeTo.getIdentifierString())); 825 } 826 827 828 // Make sure that the --moveSubtreeFrom and --moveSubtreeTo arguments were 829 // provided an equal number of times. 830 final int moveFromOccurrences = moveSubtreeFrom.getNumOccurrences(); 831 final int moveToOccurrences = moveSubtreeTo.getNumOccurrences(); 832 if (moveFromOccurrences != moveToOccurrences) 833 { 834 throw new ArgumentException( 835 ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get( 836 moveSubtreeFrom.getIdentifierString(), 837 moveSubtreeTo.getIdentifierString())); 838 } 839 } 840 841 842 843 /** 844 * {@inheritDoc} 845 */ 846 @Override() 847 public ResultCode doToolProcessing() 848 { 849 final Schema schema; 850 try 851 { 852 schema = getSchema(); 853 } 854 catch (final LDAPException le) 855 { 856 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, le.getMessage()); 857 return le.getResultCode(); 858 } 859 860 861 // If an encryption passphrase file is provided, then get the passphrase 862 // from it. 863 String encryptionPassphrase = null; 864 if (encryptionPassphraseFile.isPresent()) 865 { 866 try 867 { 868 encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile( 869 encryptionPassphraseFile.getValue()); 870 } 871 catch (final LDAPException e) 872 { 873 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, e.getMessage()); 874 return e.getResultCode(); 875 } 876 } 877 878 879 // Create the translators to use to apply the transformations. 880 final ArrayList<LDIFReaderEntryTranslator> entryTranslators = 881 new ArrayList<>(10); 882 final ArrayList<LDIFReaderChangeRecordTranslator> changeRecordTranslators = 883 new ArrayList<>(10); 884 885 final AtomicLong excludedEntryCount = new AtomicLong(0L); 886 createTranslators(entryTranslators, changeRecordTranslators, 887 schema, excludedEntryCount); 888 889 final AggregateLDIFReaderEntryTranslator entryTranslator = 890 new AggregateLDIFReaderEntryTranslator(entryTranslators); 891 final AggregateLDIFReaderChangeRecordTranslator changeRecordTranslator = 892 new AggregateLDIFReaderChangeRecordTranslator(changeRecordTranslators); 893 894 895 // Determine the path to the target file to be written. 896 final File targetFile; 897 if (targetLDIF.isPresent()) 898 { 899 targetFile = targetLDIF.getValue(); 900 } 901 else if (targetToStandardOutput.isPresent()) 902 { 903 targetFile = null; 904 } 905 else 906 { 907 targetFile = 908 new File(sourceLDIF.getValue().getAbsolutePath() + ".scrambled"); 909 } 910 911 912 // Create the LDIF reader. 913 final LDIFReader ldifReader; 914 try 915 { 916 final InputStream inputStream; 917 if (sourceLDIF.isPresent()) 918 { 919 final ObjectPair<InputStream,String> p = 920 ToolUtils.getInputStreamForLDIFFiles(sourceLDIF.getValues(), 921 encryptionPassphrase, getOut(), getErr()); 922 inputStream = p.getFirst(); 923 if ((encryptionPassphrase == null) && (p.getSecond() != null)) 924 { 925 encryptionPassphrase = p.getSecond(); 926 } 927 } 928 else 929 { 930 inputStream = System.in; 931 } 932 933 ldifReader = new LDIFReader(inputStream, numThreads.getValue(), 934 entryTranslator, changeRecordTranslator); 935 if (schema != null) 936 { 937 ldifReader.setSchema(schema); 938 } 939 } 940 catch (final Exception e) 941 { 942 Debug.debugException(e); 943 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 944 ERR_TRANSFORM_LDIF_ERROR_CREATING_LDIF_READER.get( 945 StaticUtils.getExceptionMessage(e))); 946 return ResultCode.LOCAL_ERROR; 947 } 948 949 950 ResultCode resultCode = ResultCode.SUCCESS; 951 OutputStream outputStream = null; 952processingBlock: 953 try 954 { 955 // Create the output stream to use to write the transformed data. 956 try 957 { 958 if (targetFile == null) 959 { 960 outputStream = getOut(); 961 } 962 else 963 { 964 outputStream = 965 new FileOutputStream(targetFile, appendToTargetLDIF.isPresent()); 966 } 967 968 if (encryptTarget.isPresent()) 969 { 970 if (encryptionPassphrase == null) 971 { 972 encryptionPassphrase = ToolUtils.promptForEncryptionPassphrase( 973 false, true, getOut(), getErr()); 974 } 975 976 outputStream = new PassphraseEncryptedOutputStream( 977 encryptionPassphrase, outputStream); 978 } 979 980 if (compressTarget.isPresent()) 981 { 982 outputStream = new GZIPOutputStream(outputStream); 983 } 984 } 985 catch (final Exception e) 986 { 987 Debug.debugException(e); 988 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 989 ERR_TRANSFORM_LDIF_ERROR_CREATING_OUTPUT_STREAM.get( 990 targetFile.getAbsolutePath(), 991 StaticUtils.getExceptionMessage(e))); 992 resultCode = ResultCode.LOCAL_ERROR; 993 break processingBlock; 994 } 995 996 997 // Read the source data one record at a time. The transformations will 998 // automatically be applied by the LDIF reader's translators, and even if 999 // there are multiple reader threads, we're guaranteed to get the results 1000 // in the right order. 1001 long entriesWritten = 0L; 1002 while (true) 1003 { 1004 final LDIFRecord ldifRecord; 1005 try 1006 { 1007 ldifRecord = ldifReader.readLDIFRecord(); 1008 } 1009 catch (final LDIFException le) 1010 { 1011 Debug.debugException(le); 1012 if (le.mayContinueReading()) 1013 { 1014 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1015 ERR_TRANSFORM_LDIF_RECOVERABLE_MALFORMED_RECORD.get( 1016 StaticUtils.getExceptionMessage(le))); 1017 if (resultCode == ResultCode.SUCCESS) 1018 { 1019 resultCode = ResultCode.PARAM_ERROR; 1020 } 1021 continue; 1022 } 1023 else 1024 { 1025 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1026 ERR_TRANSFORM_LDIF_UNRECOVERABLE_MALFORMED_RECORD.get( 1027 StaticUtils.getExceptionMessage(le))); 1028 if (resultCode == ResultCode.SUCCESS) 1029 { 1030 resultCode = ResultCode.PARAM_ERROR; 1031 } 1032 break processingBlock; 1033 } 1034 } 1035 catch (final Exception e) 1036 { 1037 Debug.debugException(e); 1038 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1039 ERR_TRANSFORM_LDIF_UNEXPECTED_READ_ERROR.get( 1040 StaticUtils.getExceptionMessage(e))); 1041 resultCode = ResultCode.LOCAL_ERROR; 1042 break processingBlock; 1043 } 1044 1045 1046 // If the LDIF record is null, then we've run out of records so we're 1047 // done. 1048 if (ldifRecord == null) 1049 { 1050 break; 1051 } 1052 1053 1054 // Write the record to the output stream. 1055 try 1056 { 1057 if (ldifRecord instanceof PreEncodedLDIFEntry) 1058 { 1059 outputStream.write( 1060 ((PreEncodedLDIFEntry) ldifRecord).getLDIFBytes()); 1061 } 1062 else 1063 { 1064 final ByteStringBuffer buffer = getBuffer(); 1065 if (wrapColumn.isPresent()) 1066 { 1067 ldifRecord.toLDIF(buffer, wrapColumn.getValue()); 1068 } 1069 else 1070 { 1071 ldifRecord.toLDIF(buffer, 0); 1072 } 1073 buffer.append(StaticUtils.EOL_BYTES); 1074 buffer.write(outputStream); 1075 } 1076 } 1077 catch (final Exception e) 1078 { 1079 Debug.debugException(e); 1080 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1081 ERR_TRANSFORM_LDIF_WRITE_ERROR.get(targetFile.getAbsolutePath(), 1082 StaticUtils.getExceptionMessage(e))); 1083 resultCode = ResultCode.LOCAL_ERROR; 1084 break processingBlock; 1085 } 1086 1087 1088 // If we've written a multiple of 1000 entries, print a progress 1089 // message. 1090 entriesWritten++; 1091 if ((! targetToStandardOutput.isPresent()) && 1092 ((entriesWritten % 1000L) == 0)) 1093 { 1094 final long numExcluded = excludedEntryCount.get(); 1095 if (numExcluded > 0L) 1096 { 1097 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1098 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_WITH_EXCLUDED.get( 1099 entriesWritten, numExcluded)); 1100 } 1101 else 1102 { 1103 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1104 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_NONE_EXCLUDED.get( 1105 entriesWritten)); 1106 } 1107 } 1108 } 1109 1110 1111 if (! targetToStandardOutput.isPresent()) 1112 { 1113 final long numExcluded = excludedEntryCount.get(); 1114 if (numExcluded > 0L) 1115 { 1116 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1117 INFO_TRANSFORM_LDIF_COMPLETE_WITH_EXCLUDED.get(entriesWritten, 1118 numExcluded)); 1119 } 1120 else 1121 { 1122 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1123 INFO_TRANSFORM_LDIF_COMPLETE_NONE_EXCLUDED.get(entriesWritten)); 1124 } 1125 } 1126 } 1127 finally 1128 { 1129 if (outputStream != null) 1130 { 1131 try 1132 { 1133 outputStream.close(); 1134 } 1135 catch (final Exception e) 1136 { 1137 Debug.debugException(e); 1138 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1139 ERR_TRANSFORM_LDIF_ERROR_CLOSING_OUTPUT_STREAM.get( 1140 targetFile.getAbsolutePath(), 1141 StaticUtils.getExceptionMessage(e))); 1142 if (resultCode == ResultCode.SUCCESS) 1143 { 1144 resultCode = ResultCode.LOCAL_ERROR; 1145 } 1146 } 1147 } 1148 1149 try 1150 { 1151 ldifReader.close(); 1152 } 1153 catch (final Exception e) 1154 { 1155 Debug.debugException(e); 1156 // We can ignore this. 1157 } 1158 } 1159 1160 1161 return resultCode; 1162 } 1163 1164 1165 1166 /** 1167 * Retrieves the schema that should be used for processing. 1168 * 1169 * @return The schema that was created. 1170 * 1171 * @throws LDAPException If a problem is encountered while retrieving the 1172 * schema. 1173 */ 1174 private Schema getSchema() 1175 throws LDAPException 1176 { 1177 // If any schema paths were specified, then load the schema only from those 1178 // paths. 1179 if (schemaPath.isPresent()) 1180 { 1181 final ArrayList<File> schemaFiles = new ArrayList<>(10); 1182 for (final File path : schemaPath.getValues()) 1183 { 1184 if (path.isFile()) 1185 { 1186 schemaFiles.add(path); 1187 } 1188 else 1189 { 1190 final TreeMap<String,File> fileMap = new TreeMap<>(); 1191 for (final File schemaDirFile : path.listFiles()) 1192 { 1193 final String name = schemaDirFile.getName(); 1194 if (schemaDirFile.isFile() && name.toLowerCase().endsWith(".ldif")) 1195 { 1196 fileMap.put(name, schemaDirFile); 1197 } 1198 } 1199 schemaFiles.addAll(fileMap.values()); 1200 } 1201 } 1202 1203 if (schemaFiles.isEmpty()) 1204 { 1205 throw new LDAPException(ResultCode.PARAM_ERROR, 1206 ERR_TRANSFORM_LDIF_NO_SCHEMA_FILES.get( 1207 schemaPath.getIdentifierString())); 1208 } 1209 else 1210 { 1211 try 1212 { 1213 return Schema.getSchema(schemaFiles); 1214 } 1215 catch (final Exception e) 1216 { 1217 Debug.debugException(e); 1218 throw new LDAPException(ResultCode.LOCAL_ERROR, 1219 ERR_TRANSFORM_LDIF_ERROR_LOADING_SCHEMA.get( 1220 StaticUtils.getExceptionMessage(e))); 1221 } 1222 } 1223 } 1224 else 1225 { 1226 // If the INSTANCE_ROOT environment variable is set and it refers to a 1227 // directory that has a config/schema subdirectory that has one or more 1228 // schema files in it, then read the schema from that directory. 1229 try 1230 { 1231 final String instanceRootStr = System.getenv("INSTANCE_ROOT"); 1232 if (instanceRootStr != null) 1233 { 1234 final File instanceRoot = new File(instanceRootStr); 1235 final File configDir = new File(instanceRoot, "config"); 1236 final File schemaDir = new File(configDir, "schema"); 1237 if (schemaDir.exists()) 1238 { 1239 final TreeMap<String,File> fileMap = new TreeMap<>(); 1240 for (final File schemaDirFile : schemaDir.listFiles()) 1241 { 1242 final String name = schemaDirFile.getName(); 1243 if (schemaDirFile.isFile() && 1244 name.toLowerCase().endsWith(".ldif")) 1245 { 1246 fileMap.put(name, schemaDirFile); 1247 } 1248 } 1249 1250 if (! fileMap.isEmpty()) 1251 { 1252 return Schema.getSchema(new ArrayList<>(fileMap.values())); 1253 } 1254 } 1255 } 1256 } 1257 catch (final Exception e) 1258 { 1259 Debug.debugException(e); 1260 } 1261 } 1262 1263 1264 // If we've gotten here, then just return null and the tool will try to use 1265 // the default standard schema. 1266 return null; 1267 } 1268 1269 1270 1271 /** 1272 * Creates the entry and change record translators that will be used to 1273 * perform the transformations. 1274 * 1275 * @param entryTranslators A list to which all created entry 1276 * translators should be written. 1277 * @param changeRecordTranslators A list to which all created change record 1278 * translators should be written. 1279 * @param schema The schema to use when processing. 1280 * @param excludedEntryCount A counter used to keep track of the number 1281 * of entries that have been excluded from 1282 * the result set. 1283 */ 1284 private void createTranslators( 1285 final List<LDIFReaderEntryTranslator> entryTranslators, 1286 final List<LDIFReaderChangeRecordTranslator> changeRecordTranslators, 1287 final Schema schema, final AtomicLong excludedEntryCount) 1288 { 1289 if (scrambleAttribute.isPresent()) 1290 { 1291 final Long seed; 1292 if (randomSeed.isPresent()) 1293 { 1294 seed = randomSeed.getValue().longValue(); 1295 } 1296 else 1297 { 1298 seed = null; 1299 } 1300 1301 final ScrambleAttributeTransformation t = 1302 new ScrambleAttributeTransformation(schema, seed, 1303 processDNs.isPresent(), scrambleAttribute.getValues(), 1304 scrambleJSONField.getValues()); 1305 entryTranslators.add(t); 1306 changeRecordTranslators.add(t); 1307 } 1308 1309 if (sequentialAttribute.isPresent()) 1310 { 1311 final long initialValue; 1312 if (initialSequentialValue.isPresent()) 1313 { 1314 initialValue = initialSequentialValue.getValue().longValue(); 1315 } 1316 else 1317 { 1318 initialValue = 0L; 1319 } 1320 1321 final long incrementAmount; 1322 if (sequentialValueIncrement.isPresent()) 1323 { 1324 incrementAmount = sequentialValueIncrement.getValue().longValue(); 1325 } 1326 else 1327 { 1328 incrementAmount = 1L; 1329 } 1330 1331 for (final String attrName : sequentialAttribute.getValues()) 1332 { 1333 1334 1335 final ReplaceWithCounterTransformation t = 1336 new ReplaceWithCounterTransformation(schema, attrName, 1337 initialValue, incrementAmount, 1338 textBeforeSequentialValue.getValue(), 1339 textAfterSequentialValue.getValue(), processDNs.isPresent()); 1340 entryTranslators.add(t); 1341 } 1342 } 1343 1344 if (replaceValuesAttribute.isPresent()) 1345 { 1346 final ReplaceAttributeTransformation t = 1347 new ReplaceAttributeTransformation(schema, 1348 replaceValuesAttribute.getValue(), 1349 replacementValue.getValues()); 1350 entryTranslators.add(t); 1351 } 1352 1353 if (addAttributeName.isPresent()) 1354 { 1355 final AddAttributeTransformation t = new AddAttributeTransformation( 1356 schema, addAttributeBaseDN.getValue(), addAttributeScope.getValue(), 1357 addAttributeFilter.getValue(), 1358 new Attribute(addAttributeName.getValue(), schema, 1359 addAttributeValue.getValues()), 1360 (! addToExistingValues.isPresent())); 1361 entryTranslators.add(t); 1362 } 1363 1364 if (renameAttributeFrom.isPresent()) 1365 { 1366 final Iterator<String> renameFromIterator = 1367 renameAttributeFrom.getValues().iterator(); 1368 final Iterator<String> renameToIterator = 1369 renameAttributeTo.getValues().iterator(); 1370 while (renameFromIterator.hasNext()) 1371 { 1372 final RenameAttributeTransformation t = 1373 new RenameAttributeTransformation(schema, 1374 renameFromIterator.next(), renameToIterator.next(), 1375 processDNs.isPresent()); 1376 entryTranslators.add(t); 1377 changeRecordTranslators.add(t); 1378 } 1379 } 1380 1381 if (flattenBaseDN.isPresent()) 1382 { 1383 final FlattenSubtreeTransformation t = new FlattenSubtreeTransformation( 1384 schema, flattenBaseDN.getValue(), 1385 flattenAddOmittedRDNAttributesToEntry.isPresent(), 1386 flattenAddOmittedRDNAttributesToRDN.isPresent(), 1387 flattenExcludeFilter.getValue()); 1388 entryTranslators.add(t); 1389 } 1390 1391 if (moveSubtreeFrom.isPresent()) 1392 { 1393 final Iterator<DN> moveFromIterator = 1394 moveSubtreeFrom.getValues().iterator(); 1395 final Iterator<DN> moveToIterator = moveSubtreeTo.getValues().iterator(); 1396 while (moveFromIterator.hasNext()) 1397 { 1398 final MoveSubtreeTransformation t = 1399 new MoveSubtreeTransformation(moveFromIterator.next(), 1400 moveToIterator.next()); 1401 entryTranslators.add(t); 1402 changeRecordTranslators.add(t); 1403 } 1404 } 1405 1406 if (redactAttribute.isPresent()) 1407 { 1408 final RedactAttributeTransformation t = new RedactAttributeTransformation( 1409 schema, processDNs.isPresent(), 1410 (! hideRedactedValueCount.isPresent()), redactAttribute.getValues()); 1411 entryTranslators.add(t); 1412 changeRecordTranslators.add(t); 1413 } 1414 1415 if (excludeAttribute.isPresent()) 1416 { 1417 final ExcludeAttributeTransformation t = 1418 new ExcludeAttributeTransformation(schema, 1419 excludeAttribute.getValues()); 1420 entryTranslators.add(t); 1421 changeRecordTranslators.add(t); 1422 } 1423 1424 if (excludeEntryBaseDN.isPresent() || excludeEntryScope.isPresent() || 1425 excludeEntryFilter.isPresent()) 1426 { 1427 final ExcludeEntryTransformation t = new ExcludeEntryTransformation( 1428 schema, excludeEntryBaseDN.getValue(), excludeEntryScope.getValue(), 1429 excludeEntryFilter.getValue(), 1430 (! excludeNonMatchingEntries.isPresent()), excludedEntryCount); 1431 entryTranslators.add(t); 1432 } 1433 1434 entryTranslators.add(this); 1435 } 1436 1437 1438 1439 /** 1440 * {@inheritDoc} 1441 */ 1442 @Override() 1443 public LinkedHashMap<String[],String> getExampleUsages() 1444 { 1445 final LinkedHashMap<String[],String> examples = 1446 new LinkedHashMap<>(StaticUtils.computeMapCapacity(4)); 1447 1448 examples.put( 1449 new String[] 1450 { 1451 "--sourceLDIF", "input.ldif", 1452 "--targetLDIF", "scrambled.ldif", 1453 "--scrambleAttribute", "givenName", 1454 "--scrambleAttribute", "sn", 1455 "--scrambleAttribute", "cn", 1456 "--numThreads", "10", 1457 "--schemaPath", "/ds/config/schema", 1458 "--processDNs" 1459 }, 1460 INFO_TRANSFORM_LDIF_EXAMPLE_SCRAMBLE.get()); 1461 1462 examples.put( 1463 new String[] 1464 { 1465 "--sourceLDIF", "input.ldif", 1466 "--targetLDIF", "sequential.ldif", 1467 "--sequentialAttribute", "uid", 1468 "--initialSequentialValue", "1", 1469 "--sequentialValueIncrement", "1", 1470 "--textBeforeSequentialValue", "user.", 1471 "--numThreads", "10", 1472 "--schemaPath", "/ds/config/schema", 1473 "--processDNs" 1474 }, 1475 INFO_TRANSFORM_LDIF_EXAMPLE_SEQUENTIAL.get()); 1476 1477 examples.put( 1478 new String[] 1479 { 1480 "--sourceLDIF", "input.ldif", 1481 "--targetLDIF", "added-organization.ldif", 1482 "--addAttributeName", "o", 1483 "--addAttributeValue", "Example Corp.", 1484 "--addAttributeFilter", "(objectClass=person)", 1485 "--numThreads", "10", 1486 "--schemaPath", "/ds/config/schema" 1487 }, 1488 INFO_TRANSFORM_LDIF_EXAMPLE_ADD.get()); 1489 1490 examples.put( 1491 new String[] 1492 { 1493 "--sourceLDIF", "input.ldif", 1494 "--targetLDIF", "rebased.ldif", 1495 "--moveSubtreeFrom", "o=example.com", 1496 "--moveSubtreeTo", "dc=example,dc=com", 1497 "--numThreads", "10", 1498 "--schemaPath", "/ds/config/schema" 1499 }, 1500 INFO_TRANSFORM_LDIF_EXAMPLE_REBASE.get()); 1501 1502 return examples; 1503 } 1504 1505 1506 1507 /** 1508 * {@inheritDoc} 1509 */ 1510 @Override() 1511 public Entry translate(final Entry original, final long firstLineNumber) 1512 throws LDIFException 1513 { 1514 final ByteStringBuffer buffer = getBuffer(); 1515 if (wrapColumn.isPresent()) 1516 { 1517 original.toLDIF(buffer, wrapColumn.getValue()); 1518 } 1519 else 1520 { 1521 original.toLDIF(buffer, 0); 1522 } 1523 buffer.append(StaticUtils.EOL_BYTES); 1524 1525 return new PreEncodedLDIFEntry(original, buffer.toByteArray()); 1526 } 1527 1528 1529 1530 /** 1531 * Retrieves a byte string buffer that can be used to perform LDIF encoding. 1532 * 1533 * @return A byte string buffer that can be used to perform LDIF encoding. 1534 */ 1535 private ByteStringBuffer getBuffer() 1536 { 1537 ByteStringBuffer buffer = byteStringBuffers.get(); 1538 if (buffer == null) 1539 { 1540 buffer = new ByteStringBuffer(); 1541 byteStringBuffers.set(buffer); 1542 } 1543 else 1544 { 1545 buffer.clear(); 1546 } 1547 1548 return buffer; 1549 } 1550}