001/*
002 * Copyright 2016-2019 Ping Identity Corporation
003 * All Rights Reserved.
004 */
005/*
006 * Copyright (C) 2016-2019 Ping Identity Corporation
007 *
008 * This program is free software; you can redistribute it and/or modify
009 * it under the terms of the GNU General Public License (GPLv2 only)
010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011 * as published by the Free Software Foundation.
012 *
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016 * GNU General Public License for more details.
017 *
018 * You should have received a copy of the GNU General Public License
019 * along with this program; if not, see <http://www.gnu.org/licenses>.
020 */
021package com.unboundid.ldap.sdk.transformations;
022
023
024
025import java.io.File;
026import java.io.FileOutputStream;
027import java.io.InputStream;
028import java.io.OutputStream;
029import java.util.ArrayList;
030import java.util.Iterator;
031import java.util.LinkedHashMap;
032import java.util.List;
033import java.util.TreeMap;
034import java.util.concurrent.atomic.AtomicLong;
035import java.util.zip.GZIPOutputStream;
036
037import com.unboundid.ldap.sdk.Attribute;
038import com.unboundid.ldap.sdk.DN;
039import com.unboundid.ldap.sdk.Entry;
040import com.unboundid.ldap.sdk.LDAPException;
041import com.unboundid.ldap.sdk.ResultCode;
042import com.unboundid.ldap.sdk.Version;
043import com.unboundid.ldap.sdk.schema.Schema;
044import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils;
045import com.unboundid.ldif.AggregateLDIFReaderChangeRecordTranslator;
046import com.unboundid.ldif.AggregateLDIFReaderEntryTranslator;
047import com.unboundid.ldif.LDIFException;
048import com.unboundid.ldif.LDIFReader;
049import com.unboundid.ldif.LDIFReaderChangeRecordTranslator;
050import com.unboundid.ldif.LDIFReaderEntryTranslator;
051import com.unboundid.ldif.LDIFRecord;
052import com.unboundid.util.ByteStringBuffer;
053import com.unboundid.util.CommandLineTool;
054import com.unboundid.util.Debug;
055import com.unboundid.util.ObjectPair;
056import com.unboundid.util.PassphraseEncryptedOutputStream;
057import com.unboundid.util.StaticUtils;
058import com.unboundid.util.ThreadSafety;
059import com.unboundid.util.ThreadSafetyLevel;
060import com.unboundid.util.args.ArgumentException;
061import com.unboundid.util.args.ArgumentParser;
062import com.unboundid.util.args.BooleanArgument;
063import com.unboundid.util.args.DNArgument;
064import com.unboundid.util.args.FileArgument;
065import com.unboundid.util.args.FilterArgument;
066import com.unboundid.util.args.IntegerArgument;
067import com.unboundid.util.args.ScopeArgument;
068import com.unboundid.util.args.StringArgument;
069
070import static com.unboundid.ldap.sdk.transformations.TransformationMessages.*;
071
072
073
074/**
075 * This class provides a command-line tool that can be used to apply a number of
076 * transformations to an LDIF file.  The transformations that can be applied
077 * include:
078 * <UL>
079 *   <LI>
080 *     It can scramble the values of a specified set of attributes in a manner
081 *     that attempts to preserve the syntax and consistently scrambles the same
082 *     value to the same representation.
083 *   </LI>
084 *   <LI>
085 *     It can strip a specified set of attributes out of entries.
086 *   </LI>
087 *   <LI>
088 *     It can redact the values of a specified set of attributes, to indicate
089 *     that the values are there but providing no information about what their
090 *     values are.
091 *   </LI>
092 *   <LI>
093 *     It can replace the values of a specified attribute with a given set of
094 *     values.
095 *   </LI>
096 *   <LI>
097 *     It can add an attribute with a given set of values to any entry that does
098 *     not contain that attribute.
099 *   </LI>
100 *   <LI>
101 *     It can replace the values of a specified attribute with a value that
102 *     contains a sequentially-incrementing counter.
103 *   </LI>
104 *   <LI>
105 *     It can strip entries matching a given base DN, scope, and filter out of
106 *     the LDIF file.
107 *   </LI>
108 *   <LI>
109 *     It can perform DN mapping, so that entries that exist below one base DN
110 *     are moved below a different base DN.
111 *   </LI>
112 *   <LI>
113 *     It can perform attribute mapping, to replace uses of one attribute name
114 *     with another.
115 *   </LI>
116 * </UL>
117 */
118@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
119public final class TransformLDIF
120       extends CommandLineTool
121       implements LDIFReaderEntryTranslator
122{
123  /**
124   * The maximum length of any message to write to standard output or standard
125   * error.
126   */
127  private static final int MAX_OUTPUT_LINE_LENGTH =
128       StaticUtils.TERMINAL_WIDTH_COLUMNS - 1;
129
130
131
132  // The arguments for use by this program.
133  private BooleanArgument addToExistingValues = null;
134  private BooleanArgument appendToTargetLDIF = null;
135  private BooleanArgument compressTarget = null;
136  private BooleanArgument encryptTarget = null;
137  private BooleanArgument excludeNonMatchingEntries = null;
138  private BooleanArgument flattenAddOmittedRDNAttributesToEntry = null;
139  private BooleanArgument flattenAddOmittedRDNAttributesToRDN = null;
140  private BooleanArgument hideRedactedValueCount = null;
141  private BooleanArgument processDNs = null;
142  private BooleanArgument sourceCompressed = null;
143  private BooleanArgument sourceContainsChangeRecords = null;
144  private BooleanArgument sourceFromStandardInput = null;
145  private BooleanArgument targetToStandardOutput = null;
146  private DNArgument addAttributeBaseDN = null;
147  private DNArgument excludeEntryBaseDN = null;
148  private DNArgument flattenBaseDN = null;
149  private DNArgument moveSubtreeFrom = null;
150  private DNArgument moveSubtreeTo = null;
151  private FileArgument encryptionPassphraseFile = null;
152  private FileArgument schemaPath = null;
153  private FileArgument sourceLDIF = null;
154  private FileArgument targetLDIF = null;
155  private FilterArgument addAttributeFilter = null;
156  private FilterArgument excludeEntryFilter = null;
157  private FilterArgument flattenExcludeFilter = null;
158  private IntegerArgument initialSequentialValue = null;
159  private IntegerArgument numThreads = null;
160  private IntegerArgument randomSeed = null;
161  private IntegerArgument sequentialValueIncrement = null;
162  private IntegerArgument wrapColumn = null;
163  private ScopeArgument addAttributeScope = null;
164  private ScopeArgument excludeEntryScope = null;
165  private StringArgument addAttributeName = null;
166  private StringArgument addAttributeValue = null;
167  private StringArgument excludeAttribute = null;
168  private StringArgument redactAttribute = null;
169  private StringArgument renameAttributeFrom = null;
170  private StringArgument renameAttributeTo = null;
171  private StringArgument replaceValuesAttribute = null;
172  private StringArgument replacementValue = null;
173  private StringArgument scrambleAttribute = null;
174  private StringArgument scrambleJSONField = null;
175  private StringArgument sequentialAttribute = null;
176  private StringArgument textAfterSequentialValue = null;
177  private StringArgument textBeforeSequentialValue = null;
178
179  // A set of thread-local byte stream buffers that will be used to construct
180  // the LDIF representations of records.
181  private final ThreadLocal<ByteStringBuffer> byteStringBuffers =
182       new ThreadLocal<>();
183
184
185
186  /**
187   * Invokes this tool with the provided set of arguments.
188   *
189   * @param  args  The command-line arguments provided to this program.
190   */
191  public static void main(final String... args)
192  {
193    final ResultCode resultCode = main(System.out, System.err, args);
194    if (resultCode != ResultCode.SUCCESS)
195    {
196      System.exit(resultCode.intValue());
197    }
198  }
199
200
201
202  /**
203   * Invokes this tool with the provided set of arguments.
204   *
205   * @param  out   The output stream to use for standard output.  It may be
206   *               {@code null} if standard output should be suppressed.
207   * @param  err   The output stream to use for standard error.  It may be
208   *               {@code null} if standard error should be suppressed.
209   * @param  args  The command-line arguments provided to this program.
210   *
211   * @return  A result code indicating whether processing completed
212   *          successfully.
213   */
214  public static ResultCode main(final OutputStream out, final OutputStream err,
215                                final String... args)
216  {
217    final TransformLDIF tool = new TransformLDIF(out, err);
218    return tool.runTool(args);
219  }
220
221
222
223  /**
224   * Creates a new instance of this tool with the provided information.
225   *
226   * @param  out  The output stream to use for standard output.  It may be
227   *              {@code null} if standard output should be suppressed.
228   * @param  err  The output stream to use for standard error.  It may be
229   *              {@code null} if standard error should be suppressed.
230   */
231  public TransformLDIF(final OutputStream out, final OutputStream err)
232  {
233    super(out, err);
234  }
235
236
237
238  /**
239   * {@inheritDoc}
240   */
241  @Override()
242  public String getToolName()
243  {
244    return "transform-ldif";
245  }
246
247
248
249  /**
250   * {@inheritDoc}
251   */
252  @Override()
253  public String getToolDescription()
254  {
255    return INFO_TRANSFORM_LDIF_TOOL_DESCRIPTION.get();
256  }
257
258
259
260  /**
261   * {@inheritDoc}
262   */
263  @Override()
264  public String getToolVersion()
265  {
266    return Version.NUMERIC_VERSION_STRING;
267  }
268
269
270
271  /**
272   * {@inheritDoc}
273   */
274  @Override()
275  public boolean supportsInteractiveMode()
276  {
277    return true;
278  }
279
280
281
282  /**
283   * {@inheritDoc}
284   */
285  @Override()
286  public boolean defaultsToInteractiveMode()
287  {
288    return true;
289  }
290
291
292
293  /**
294   * {@inheritDoc}
295   */
296  @Override()
297  public boolean supportsPropertiesFile()
298  {
299    return true;
300  }
301
302
303
304  /**
305   * {@inheritDoc}
306   */
307  @Override()
308  public void addToolArguments(final ArgumentParser parser)
309         throws ArgumentException
310  {
311    // Add arguments pertaining to the source and target LDIF files.
312    sourceLDIF = new FileArgument('l', "sourceLDIF", false, 0, null,
313         INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_LDIF.get(), true, true, true,
314         false);
315    sourceLDIF.addLongIdentifier("inputLDIF", true);
316    sourceLDIF.addLongIdentifier("source-ldif", true);
317    sourceLDIF.addLongIdentifier("input-ldif", true);
318    sourceLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
319    parser.addArgument(sourceLDIF);
320
321    sourceFromStandardInput = new BooleanArgument(null,
322         "sourceFromStandardInput", 1,
323         INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_STD_IN.get());
324    sourceFromStandardInput.addLongIdentifier("source-from-standard-input",
325         true);
326    sourceFromStandardInput.setArgumentGroupName(
327         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
328    parser.addArgument(sourceFromStandardInput);
329    parser.addRequiredArgumentSet(sourceLDIF, sourceFromStandardInput);
330    parser.addExclusiveArgumentSet(sourceLDIF, sourceFromStandardInput);
331
332    targetLDIF = new FileArgument('o', "targetLDIF", false, 1, null,
333         INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_LDIF.get(), false, true, true,
334         false);
335    targetLDIF.addLongIdentifier("outputLDIF", true);
336    targetLDIF.addLongIdentifier("target-ldif", true);
337    targetLDIF.addLongIdentifier("output-ldif", true);
338    targetLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
339    parser.addArgument(targetLDIF);
340
341    targetToStandardOutput = new BooleanArgument(null, "targetToStandardOutput",
342         1, INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_STD_OUT.get());
343    targetToStandardOutput.addLongIdentifier("target-to-standard-output", true);
344    targetToStandardOutput.setArgumentGroupName(
345         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
346    parser.addArgument(targetToStandardOutput);
347    parser.addExclusiveArgumentSet(targetLDIF, targetToStandardOutput);
348
349    sourceContainsChangeRecords = new BooleanArgument(null,
350         "sourceContainsChangeRecords",
351         INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_CONTAINS_CHANGE_RECORDS.get());
352    sourceContainsChangeRecords.addLongIdentifier(
353         "source-contains-change-records", true);
354    sourceContainsChangeRecords.setArgumentGroupName(
355         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
356    parser.addArgument(sourceContainsChangeRecords);
357
358    appendToTargetLDIF = new BooleanArgument(null, "appendToTargetLDIF",
359         INFO_TRANSFORM_LDIF_ARG_DESC_APPEND_TO_TARGET.get());
360    appendToTargetLDIF.addLongIdentifier("append-to-target-ldif", true);
361    appendToTargetLDIF.setArgumentGroupName(
362         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
363    parser.addArgument(appendToTargetLDIF);
364    parser.addExclusiveArgumentSet(targetToStandardOutput, appendToTargetLDIF);
365
366    wrapColumn = new IntegerArgument(null, "wrapColumn", false, 1, null,
367         INFO_TRANSFORM_LDIF_ARG_DESC_WRAP_COLUMN.get(), 5, Integer.MAX_VALUE);
368    wrapColumn.addLongIdentifier("wrap-column", true);
369    wrapColumn.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
370    parser.addArgument(wrapColumn);
371
372    sourceCompressed = new BooleanArgument('C', "sourceCompressed",
373         INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_COMPRESSED.get());
374    sourceCompressed.addLongIdentifier("inputCompressed", true);
375    sourceCompressed.addLongIdentifier("source-compressed", true);
376    sourceCompressed.addLongIdentifier("input-compressed", true);
377    sourceCompressed.setArgumentGroupName(
378         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
379    parser.addArgument(sourceCompressed);
380
381    compressTarget = new BooleanArgument('c', "compressTarget",
382         INFO_TRANSFORM_LDIF_ARG_DESC_COMPRESS_TARGET.get());
383    compressTarget.addLongIdentifier("compressOutput", true);
384    compressTarget.addLongIdentifier("compress", true);
385    compressTarget.addLongIdentifier("compress-target", true);
386    compressTarget.addLongIdentifier("compress-output", true);
387    compressTarget.setArgumentGroupName(
388         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
389    parser.addArgument(compressTarget);
390
391    encryptTarget = new BooleanArgument(null, "encryptTarget",
392         INFO_TRANSFORM_LDIF_ARG_DESC_ENCRYPT_TARGET.get());
393    encryptTarget.addLongIdentifier("encryptOutput", true);
394    encryptTarget.addLongIdentifier("encrypt", true);
395    encryptTarget.addLongIdentifier("encrypt-target", true);
396    encryptTarget.addLongIdentifier("encrypt-output", true);
397    encryptTarget.setArgumentGroupName(
398         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
399    parser.addArgument(encryptTarget);
400
401    encryptionPassphraseFile = new FileArgument(null,
402         "encryptionPassphraseFile", false, 1, null,
403         INFO_TRANSFORM_LDIF_ARG_DESC_ENCRYPTION_PW_FILE.get(), true, true,
404         true, false);
405    encryptionPassphraseFile.addLongIdentifier("encryptionPasswordFile", true);
406    encryptionPassphraseFile.addLongIdentifier("encryption-passphrase-file",
407         true);
408    encryptionPassphraseFile.addLongIdentifier("encryption-password-file",
409         true);
410    encryptionPassphraseFile.setArgumentGroupName(
411         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
412    parser.addArgument(encryptionPassphraseFile);
413
414
415    // Add arguments pertaining to attribute scrambling.
416    scrambleAttribute = new StringArgument('a', "scrambleAttribute", false, 0,
417         INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
418         INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_ATTR.get());
419    scrambleAttribute.addLongIdentifier("attributeName", true);
420    scrambleAttribute.addLongIdentifier("scramble-attribute", true);
421    scrambleAttribute.addLongIdentifier("attribute-name", true);
422    scrambleAttribute.setArgumentGroupName(
423         INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get());
424    parser.addArgument(scrambleAttribute);
425
426    scrambleJSONField = new StringArgument(null, "scrambleJSONField", false, 0,
427         INFO_TRANSFORM_LDIF_PLACEHOLDER_FIELD_NAME.get(),
428         INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_JSON_FIELD.get(
429              scrambleAttribute.getIdentifierString()));
430    scrambleJSONField.addLongIdentifier("scramble-json-field", true);
431    scrambleJSONField.setArgumentGroupName(
432         INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get());
433    parser.addArgument(scrambleJSONField);
434    parser.addDependentArgumentSet(scrambleJSONField, scrambleAttribute);
435
436    randomSeed = new IntegerArgument('s', "randomSeed", false, 1, null,
437         INFO_TRANSFORM_LDIF_ARG_DESC_RANDOM_SEED.get());
438    randomSeed.addLongIdentifier("random-seed", true);
439    randomSeed.setArgumentGroupName(
440         INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get());
441    parser.addArgument(randomSeed);
442
443
444    // Add arguments pertaining to replacing attribute values with a generated
445    // value using a sequential counter.
446    sequentialAttribute = new StringArgument('S', "sequentialAttribute",
447         false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
448         INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_ATTR.get(
449              sourceContainsChangeRecords.getIdentifierString()));
450    sequentialAttribute.addLongIdentifier("sequentialAttributeName", true);
451    sequentialAttribute.addLongIdentifier("sequential-attribute", true);
452    sequentialAttribute.addLongIdentifier("sequential-attribute-name", true);
453    sequentialAttribute.setArgumentGroupName(
454         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
455    parser.addArgument(sequentialAttribute);
456    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
457         sequentialAttribute);
458
459    initialSequentialValue = new IntegerArgument('i', "initialSequentialValue",
460         false, 1, null,
461         INFO_TRANSFORM_LDIF_ARG_DESC_INITIAL_SEQUENTIAL_VALUE.get(
462              sequentialAttribute.getIdentifierString()));
463    initialSequentialValue.addLongIdentifier("initial-sequential-value", true);
464    initialSequentialValue.setArgumentGroupName(
465         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
466    parser.addArgument(initialSequentialValue);
467    parser.addDependentArgumentSet(initialSequentialValue, sequentialAttribute);
468
469    sequentialValueIncrement = new IntegerArgument(null,
470         "sequentialValueIncrement", false, 1, null,
471         INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_INCREMENT.get(
472              sequentialAttribute.getIdentifierString()));
473    sequentialValueIncrement.addLongIdentifier("sequential-value-increment",
474         true);
475    sequentialValueIncrement.setArgumentGroupName(
476         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
477    parser.addArgument(sequentialValueIncrement);
478    parser.addDependentArgumentSet(sequentialValueIncrement,
479         sequentialAttribute);
480
481    textBeforeSequentialValue = new StringArgument(null,
482         "textBeforeSequentialValue", false, 1, null,
483         INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_BEFORE.get(
484              sequentialAttribute.getIdentifierString()));
485    textBeforeSequentialValue.addLongIdentifier("text-before-sequential-value",
486         true);
487    textBeforeSequentialValue.setArgumentGroupName(
488         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
489    parser.addArgument(textBeforeSequentialValue);
490    parser.addDependentArgumentSet(textBeforeSequentialValue,
491         sequentialAttribute);
492
493    textAfterSequentialValue = new StringArgument(null,
494         "textAfterSequentialValue", false, 1, null,
495         INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_AFTER.get(
496              sequentialAttribute.getIdentifierString()));
497    textAfterSequentialValue.addLongIdentifier("text-after-sequential-value",
498         true);
499    textAfterSequentialValue.setArgumentGroupName(
500         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
501    parser.addArgument(textAfterSequentialValue);
502    parser.addDependentArgumentSet(textAfterSequentialValue,
503         sequentialAttribute);
504
505
506    // Add arguments pertaining to attribute value replacement.
507    replaceValuesAttribute = new StringArgument(null, "replaceValuesAttribute",
508         false, 1, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
509         INFO_TRANSFORM_LDIF_ARG_DESC_REPLACE_VALUES_ATTR.get(
510              sourceContainsChangeRecords.getIdentifierString()));
511    replaceValuesAttribute.addLongIdentifier("replace-values-attribute", true);
512    replaceValuesAttribute.setArgumentGroupName(
513         INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get());
514    parser.addArgument(replaceValuesAttribute);
515    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
516         replaceValuesAttribute);
517
518    replacementValue = new StringArgument(null, "replacementValue", false, 0,
519         null,
520         INFO_TRANSFORM_LDIF_ARG_DESC_REPLACEMENT_VALUE.get(
521              replaceValuesAttribute.getIdentifierString()));
522    replacementValue.addLongIdentifier("replacement-value", true);
523    replacementValue.setArgumentGroupName(
524         INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get());
525    parser.addArgument(replacementValue);
526    parser.addDependentArgumentSet(replaceValuesAttribute, replacementValue);
527    parser.addDependentArgumentSet(replacementValue, replaceValuesAttribute);
528
529
530    // Add arguments pertaining to adding missing attributes.
531    addAttributeName = new StringArgument(null, "addAttributeName", false, 1,
532         INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
533         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_ATTR.get(
534              "--addAttributeValue",
535              sourceContainsChangeRecords.getIdentifierString()));
536    addAttributeName.addLongIdentifier("add-attribute-name", true);
537    addAttributeName.setArgumentGroupName(
538         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
539    parser.addArgument(addAttributeName);
540    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
541         addAttributeName);
542
543    addAttributeValue = new StringArgument(null, "addAttributeValue", false, 0,
544         null,
545         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_VALUE.get(
546              addAttributeName.getIdentifierString()));
547    addAttributeValue.addLongIdentifier("add-attribute-value", true);
548    addAttributeValue.setArgumentGroupName(
549         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
550    parser.addArgument(addAttributeValue);
551    parser.addDependentArgumentSet(addAttributeName, addAttributeValue);
552    parser.addDependentArgumentSet(addAttributeValue, addAttributeName);
553
554    addToExistingValues = new BooleanArgument(null, "addToExistingValues",
555         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_MERGE_VALUES.get(
556              addAttributeName.getIdentifierString(),
557              addAttributeValue.getIdentifierString()));
558    addToExistingValues.addLongIdentifier("add-to-existing-values", true);
559    addToExistingValues.setArgumentGroupName(
560         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
561    parser.addArgument(addToExistingValues);
562    parser.addDependentArgumentSet(addToExistingValues, addAttributeName);
563
564    addAttributeBaseDN = new DNArgument(null, "addAttributeBaseDN", false, 1,
565         null,
566         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_BASE_DN.get(
567              addAttributeName.getIdentifierString()));
568    addAttributeBaseDN.addLongIdentifier("add-attribute-base-dn", true);
569    addAttributeBaseDN.setArgumentGroupName(
570         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
571    parser.addArgument(addAttributeBaseDN);
572    parser.addDependentArgumentSet(addAttributeBaseDN, addAttributeName);
573
574    addAttributeScope = new ScopeArgument(null, "addAttributeScope", false,
575         null,
576         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_SCOPE.get(
577              addAttributeBaseDN.getIdentifierString(),
578              addAttributeName.getIdentifierString()));
579    addAttributeScope.addLongIdentifier("add-attribute-scope", true);
580    addAttributeScope.setArgumentGroupName(
581         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
582    parser.addArgument(addAttributeScope);
583    parser.addDependentArgumentSet(addAttributeScope, addAttributeName);
584
585    addAttributeFilter = new FilterArgument(null, "addAttributeFilter", false,
586         1, null,
587         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_FILTER.get(
588              addAttributeName.getIdentifierString()));
589    addAttributeFilter.addLongIdentifier("add-attribute-filter", true);
590    addAttributeFilter.setArgumentGroupName(
591         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
592    parser.addArgument(addAttributeFilter);
593    parser.addDependentArgumentSet(addAttributeFilter, addAttributeName);
594
595
596    // Add arguments pertaining to renaming attributes.
597    renameAttributeFrom = new StringArgument(null, "renameAttributeFrom",
598         false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
599         INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_FROM.get());
600    renameAttributeFrom.addLongIdentifier("rename-attribute-from", true);
601    renameAttributeFrom.setArgumentGroupName(
602         INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get());
603    parser.addArgument(renameAttributeFrom);
604
605    renameAttributeTo = new StringArgument(null, "renameAttributeTo",
606         false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
607         INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_TO.get(
608              renameAttributeFrom.getIdentifierString()));
609    renameAttributeTo.addLongIdentifier("rename-attribute-to", true);
610    renameAttributeTo.setArgumentGroupName(
611         INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get());
612    parser.addArgument(renameAttributeTo);
613    parser.addDependentArgumentSet(renameAttributeFrom, renameAttributeTo);
614    parser.addDependentArgumentSet(renameAttributeTo, renameAttributeFrom);
615
616
617    // Add arguments pertaining to flattening subtrees.
618    flattenBaseDN = new DNArgument(null, "flattenBaseDN", false, 1, null,
619         INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_BASE_DN.get());
620    flattenBaseDN.addLongIdentifier("flatten-base-dn", true);
621    flattenBaseDN.setArgumentGroupName(
622         INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
623    parser.addArgument(flattenBaseDN);
624    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
625         flattenBaseDN);
626
627    flattenAddOmittedRDNAttributesToEntry = new BooleanArgument(null,
628         "flattenAddOmittedRDNAttributesToEntry", 1,
629         INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_ENTRY.get());
630    flattenAddOmittedRDNAttributesToEntry.addLongIdentifier(
631         "flatten-add-omitted-rdn-attributes-to-entry", true);
632    flattenAddOmittedRDNAttributesToEntry.setArgumentGroupName(
633         INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
634    parser.addArgument(flattenAddOmittedRDNAttributesToEntry);
635    parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToEntry,
636         flattenBaseDN);
637
638    flattenAddOmittedRDNAttributesToRDN = new BooleanArgument(null,
639         "flattenAddOmittedRDNAttributesToRDN", 1,
640         INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_RDN.get());
641    flattenAddOmittedRDNAttributesToRDN.addLongIdentifier(
642         "flatten-add-omitted-rdn-attributes-to-rdn", true);
643    flattenAddOmittedRDNAttributesToRDN.setArgumentGroupName(
644         INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
645    parser.addArgument(flattenAddOmittedRDNAttributesToRDN);
646    parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToRDN,
647         flattenBaseDN);
648
649    flattenExcludeFilter = new FilterArgument(null, "flattenExcludeFilter",
650         false, 1, null,
651         INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_EXCLUDE_FILTER.get());
652    flattenExcludeFilter.addLongIdentifier("flatten-exclude-filter", true);
653    flattenExcludeFilter.setArgumentGroupName(
654         INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
655    parser.addArgument(flattenExcludeFilter);
656    parser.addDependentArgumentSet(flattenExcludeFilter, flattenBaseDN);
657
658
659    // Add arguments pertaining to moving subtrees.
660    moveSubtreeFrom = new DNArgument(null, "moveSubtreeFrom", false, 0, null,
661         INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_FROM.get());
662    moveSubtreeFrom.addLongIdentifier("move-subtree-from", true);
663    moveSubtreeFrom.setArgumentGroupName(
664         INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get());
665    parser.addArgument(moveSubtreeFrom);
666
667    moveSubtreeTo = new DNArgument(null, "moveSubtreeTo", false, 0, null,
668         INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_TO.get(
669              moveSubtreeFrom.getIdentifierString()));
670    moveSubtreeTo.addLongIdentifier("move-subtree-to", true);
671    moveSubtreeTo.setArgumentGroupName(
672         INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get());
673    parser.addArgument(moveSubtreeTo);
674    parser.addDependentArgumentSet(moveSubtreeFrom, moveSubtreeTo);
675    parser.addDependentArgumentSet(moveSubtreeTo, moveSubtreeFrom);
676
677
678    // Add arguments pertaining to redacting attribute values.
679    redactAttribute = new StringArgument(null, "redactAttribute", false, 0,
680         INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
681         INFO_TRANSFORM_LDIF_ARG_DESC_REDACT_ATTR.get());
682    redactAttribute.addLongIdentifier("redact-attribute", true);
683    redactAttribute.setArgumentGroupName(
684         INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get());
685    parser.addArgument(redactAttribute);
686
687    hideRedactedValueCount = new BooleanArgument(null, "hideRedactedValueCount",
688         INFO_TRANSFORM_LDIF_ARG_DESC_HIDE_REDACTED_COUNT.get());
689    hideRedactedValueCount.addLongIdentifier("hide-redacted-value-count",
690         true);
691    hideRedactedValueCount.setArgumentGroupName(
692         INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get());
693    parser.addArgument(hideRedactedValueCount);
694    parser.addDependentArgumentSet(hideRedactedValueCount, redactAttribute);
695
696
697    // Add arguments pertaining to excluding attributes and entries.
698    excludeAttribute = new StringArgument(null, "excludeAttribute", false, 0,
699         INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
700         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ATTR.get());
701    excludeAttribute.addLongIdentifier("suppressAttribute", true);
702    excludeAttribute.addLongIdentifier("exclude-attribute", true);
703    excludeAttribute.addLongIdentifier("suppress-attribute", true);
704    excludeAttribute.setArgumentGroupName(
705         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
706    parser.addArgument(excludeAttribute);
707
708    excludeEntryBaseDN = new DNArgument(null, "excludeEntryBaseDN", false, 1,
709         null,
710         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_BASE_DN.get(
711              sourceContainsChangeRecords.getIdentifierString()));
712    excludeEntryBaseDN.addLongIdentifier("suppressEntryBaseDN", true);
713    excludeEntryBaseDN.addLongIdentifier("exclude-entry-base-dn", true);
714    excludeEntryBaseDN.addLongIdentifier("suppress-entry-base-dn", true);
715    excludeEntryBaseDN.setArgumentGroupName(
716         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
717    parser.addArgument(excludeEntryBaseDN);
718    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
719         excludeEntryBaseDN);
720
721    excludeEntryScope = new ScopeArgument(null, "excludeEntryScope", false,
722         null,
723         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_SCOPE.get(
724              sourceContainsChangeRecords.getIdentifierString()));
725    excludeEntryScope.addLongIdentifier("suppressEntryScope", true);
726    excludeEntryScope.addLongIdentifier("exclude-entry-scope", true);
727    excludeEntryScope.addLongIdentifier("suppress-entry-scope", true);
728    excludeEntryScope.setArgumentGroupName(
729         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
730    parser.addArgument(excludeEntryScope);
731    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
732         excludeEntryScope);
733
734    excludeEntryFilter = new FilterArgument(null, "excludeEntryFilter", false,
735         1, null,
736         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_FILTER.get(
737              sourceContainsChangeRecords.getIdentifierString()));
738    excludeEntryFilter.addLongIdentifier("suppressEntryFilter", true);
739    excludeEntryFilter.addLongIdentifier("exclude-entry-filter", true);
740    excludeEntryFilter.addLongIdentifier("suppress-entry-filter", true);
741    excludeEntryFilter.setArgumentGroupName(
742         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
743    parser.addArgument(excludeEntryFilter);
744    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
745         excludeEntryFilter);
746
747    excludeNonMatchingEntries = new BooleanArgument(null,
748         "excludeNonMatchingEntries",
749         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_NON_MATCHING.get());
750    excludeNonMatchingEntries.addLongIdentifier("exclude-non-matching-entries",
751         true);
752    excludeNonMatchingEntries.setArgumentGroupName(
753         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
754    parser.addArgument(excludeNonMatchingEntries);
755    parser.addDependentArgumentSet(excludeNonMatchingEntries,
756         excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter);
757
758
759    // Add the remaining arguments.
760    schemaPath = new FileArgument(null, "schemaPath", false, 0, null,
761         INFO_TRANSFORM_LDIF_ARG_DESC_SCHEMA_PATH.get(),
762         true, true, false, false);
763    schemaPath.addLongIdentifier("schemaFile", true);
764    schemaPath.addLongIdentifier("schemaDirectory", true);
765    schemaPath.addLongIdentifier("schema-path", true);
766    schemaPath.addLongIdentifier("schema-file", true);
767    schemaPath.addLongIdentifier("schema-directory", true);
768    parser.addArgument(schemaPath);
769
770    numThreads = new IntegerArgument('t', "numThreads", false, 1, null,
771         INFO_TRANSFORM_LDIF_ARG_DESC_NUM_THREADS.get(), 1, Integer.MAX_VALUE,
772         1);
773    numThreads.addLongIdentifier("num-threads", true);
774    parser.addArgument(numThreads);
775
776    processDNs = new BooleanArgument('d', "processDNs",
777         INFO_TRANSFORM_LDIF_ARG_DESC_PROCESS_DNS.get());
778    processDNs.addLongIdentifier("process-dns", true);
779    parser.addArgument(processDNs);
780
781
782    // Ensure that at least one kind of transformation was requested.
783    parser.addRequiredArgumentSet(scrambleAttribute, sequentialAttribute,
784         replaceValuesAttribute, addAttributeName, renameAttributeFrom,
785         flattenBaseDN, moveSubtreeFrom, redactAttribute, excludeAttribute,
786         excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter);
787  }
788
789
790
791  /**
792   * {@inheritDoc}
793   */
794  @Override()
795  public void doExtendedArgumentValidation()
796         throws ArgumentException
797  {
798    // Ideally, exactly one of the targetLDIF and targetToStandardOutput
799    // arguments should always be provided.  But in order to preserve backward
800    // compatibility with a legacy scramble-ldif tool, we will allow both to be
801    // omitted if either --scrambleAttribute or --sequentialArgument is
802    // provided.  In that case, the path of the output file will be the path of
803    // the first input file with ".scrambled" appended to it.
804    if (! (targetLDIF.isPresent() || targetToStandardOutput.isPresent()))
805    {
806      if (! (scrambleAttribute.isPresent() || sequentialAttribute.isPresent()))
807      {
808        throw new ArgumentException(ERR_TRANSFORM_LDIF_MISSING_TARGET_ARG.get(
809             targetLDIF.getIdentifierString(),
810             targetToStandardOutput.getIdentifierString()));
811      }
812    }
813
814
815    // Make sure that the --renameAttributeFrom and --renameAttributeTo
816    // arguments were provided an equal number of times.
817    final int renameFromOccurrences = renameAttributeFrom.getNumOccurrences();
818    final int renameToOccurrences = renameAttributeTo.getNumOccurrences();
819    if (renameFromOccurrences != renameToOccurrences)
820    {
821      throw new ArgumentException(
822           ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get(
823                renameAttributeFrom.getIdentifierString(),
824                renameAttributeTo.getIdentifierString()));
825    }
826
827
828    // Make sure that the --moveSubtreeFrom and --moveSubtreeTo arguments were
829    // provided an equal number of times.
830    final int moveFromOccurrences = moveSubtreeFrom.getNumOccurrences();
831    final int moveToOccurrences = moveSubtreeTo.getNumOccurrences();
832    if (moveFromOccurrences != moveToOccurrences)
833    {
834      throw new ArgumentException(
835           ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get(
836                moveSubtreeFrom.getIdentifierString(),
837                moveSubtreeTo.getIdentifierString()));
838    }
839  }
840
841
842
843  /**
844   * {@inheritDoc}
845   */
846  @Override()
847  public ResultCode doToolProcessing()
848  {
849    final Schema schema;
850    try
851    {
852      schema = getSchema();
853    }
854    catch (final LDAPException le)
855    {
856      wrapErr(0, MAX_OUTPUT_LINE_LENGTH, le.getMessage());
857      return le.getResultCode();
858    }
859
860
861    // If an encryption passphrase file is provided, then get the passphrase
862    // from it.
863    String encryptionPassphrase = null;
864    if (encryptionPassphraseFile.isPresent())
865    {
866      try
867      {
868        encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile(
869             encryptionPassphraseFile.getValue());
870      }
871      catch (final LDAPException e)
872      {
873        wrapErr(0, MAX_OUTPUT_LINE_LENGTH, e.getMessage());
874        return e.getResultCode();
875      }
876    }
877
878
879    // Create the translators to use to apply the transformations.
880    final ArrayList<LDIFReaderEntryTranslator> entryTranslators =
881         new ArrayList<>(10);
882    final ArrayList<LDIFReaderChangeRecordTranslator> changeRecordTranslators =
883         new ArrayList<>(10);
884
885    final AtomicLong excludedEntryCount = new AtomicLong(0L);
886    createTranslators(entryTranslators, changeRecordTranslators,
887         schema, excludedEntryCount);
888
889    final AggregateLDIFReaderEntryTranslator entryTranslator =
890         new AggregateLDIFReaderEntryTranslator(entryTranslators);
891    final AggregateLDIFReaderChangeRecordTranslator changeRecordTranslator =
892         new AggregateLDIFReaderChangeRecordTranslator(changeRecordTranslators);
893
894
895    // Determine the path to the target file to be written.
896    final File targetFile;
897    if (targetLDIF.isPresent())
898    {
899      targetFile = targetLDIF.getValue();
900    }
901    else if (targetToStandardOutput.isPresent())
902    {
903      targetFile = null;
904    }
905    else
906    {
907      targetFile =
908           new File(sourceLDIF.getValue().getAbsolutePath() + ".scrambled");
909    }
910
911
912    // Create the LDIF reader.
913    final LDIFReader ldifReader;
914    try
915    {
916      final InputStream inputStream;
917      if (sourceLDIF.isPresent())
918      {
919        final ObjectPair<InputStream,String> p =
920             ToolUtils.getInputStreamForLDIFFiles(sourceLDIF.getValues(),
921                  encryptionPassphrase, getOut(), getErr());
922        inputStream = p.getFirst();
923        if ((encryptionPassphrase == null) && (p.getSecond() != null))
924        {
925          encryptionPassphrase = p.getSecond();
926        }
927      }
928      else
929      {
930        inputStream = System.in;
931      }
932
933      ldifReader = new LDIFReader(inputStream, numThreads.getValue(),
934           entryTranslator, changeRecordTranslator);
935      if (schema != null)
936      {
937        ldifReader.setSchema(schema);
938      }
939    }
940    catch (final Exception e)
941    {
942      Debug.debugException(e);
943      wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
944           ERR_TRANSFORM_LDIF_ERROR_CREATING_LDIF_READER.get(
945                StaticUtils.getExceptionMessage(e)));
946      return ResultCode.LOCAL_ERROR;
947    }
948
949
950    ResultCode resultCode = ResultCode.SUCCESS;
951    OutputStream outputStream = null;
952processingBlock:
953    try
954    {
955      // Create the output stream to use to write the transformed data.
956      try
957      {
958        if (targetFile == null)
959        {
960          outputStream = getOut();
961        }
962        else
963        {
964          outputStream =
965               new FileOutputStream(targetFile, appendToTargetLDIF.isPresent());
966        }
967
968        if (encryptTarget.isPresent())
969        {
970          if (encryptionPassphrase == null)
971          {
972            encryptionPassphrase = ToolUtils.promptForEncryptionPassphrase(
973                 false, true, getOut(), getErr());
974          }
975
976          outputStream = new PassphraseEncryptedOutputStream(
977               encryptionPassphrase, outputStream);
978        }
979
980        if (compressTarget.isPresent())
981        {
982          outputStream = new GZIPOutputStream(outputStream);
983        }
984      }
985      catch (final Exception e)
986      {
987        Debug.debugException(e);
988        wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
989             ERR_TRANSFORM_LDIF_ERROR_CREATING_OUTPUT_STREAM.get(
990                  targetFile.getAbsolutePath(),
991                  StaticUtils.getExceptionMessage(e)));
992        resultCode = ResultCode.LOCAL_ERROR;
993        break processingBlock;
994      }
995
996
997      // Read the source data one record at a time.  The transformations will
998      // automatically be applied by the LDIF reader's translators, and even if
999      // there are multiple reader threads, we're guaranteed to get the results
1000      // in the right order.
1001      long entriesWritten = 0L;
1002      while (true)
1003      {
1004        final LDIFRecord ldifRecord;
1005        try
1006        {
1007          ldifRecord = ldifReader.readLDIFRecord();
1008        }
1009        catch (final LDIFException le)
1010        {
1011          Debug.debugException(le);
1012          if (le.mayContinueReading())
1013          {
1014            wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1015                 ERR_TRANSFORM_LDIF_RECOVERABLE_MALFORMED_RECORD.get(
1016                      StaticUtils.getExceptionMessage(le)));
1017            if (resultCode == ResultCode.SUCCESS)
1018            {
1019              resultCode = ResultCode.PARAM_ERROR;
1020            }
1021            continue;
1022          }
1023          else
1024          {
1025            wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1026                 ERR_TRANSFORM_LDIF_UNRECOVERABLE_MALFORMED_RECORD.get(
1027                      StaticUtils.getExceptionMessage(le)));
1028            if (resultCode == ResultCode.SUCCESS)
1029            {
1030              resultCode = ResultCode.PARAM_ERROR;
1031            }
1032            break processingBlock;
1033          }
1034        }
1035        catch (final Exception e)
1036        {
1037          Debug.debugException(e);
1038          wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1039               ERR_TRANSFORM_LDIF_UNEXPECTED_READ_ERROR.get(
1040                    StaticUtils.getExceptionMessage(e)));
1041          resultCode = ResultCode.LOCAL_ERROR;
1042          break processingBlock;
1043        }
1044
1045
1046        // If the LDIF record is null, then we've run out of records so we're
1047        // done.
1048        if (ldifRecord == null)
1049        {
1050          break;
1051        }
1052
1053
1054        // Write the record to the output stream.
1055        try
1056        {
1057          if (ldifRecord instanceof PreEncodedLDIFEntry)
1058          {
1059            outputStream.write(
1060                 ((PreEncodedLDIFEntry) ldifRecord).getLDIFBytes());
1061          }
1062          else
1063          {
1064            final ByteStringBuffer buffer = getBuffer();
1065            if (wrapColumn.isPresent())
1066            {
1067              ldifRecord.toLDIF(buffer, wrapColumn.getValue());
1068            }
1069            else
1070            {
1071              ldifRecord.toLDIF(buffer, 0);
1072            }
1073            buffer.append(StaticUtils.EOL_BYTES);
1074            buffer.write(outputStream);
1075          }
1076        }
1077        catch (final Exception e)
1078        {
1079          Debug.debugException(e);
1080          wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1081               ERR_TRANSFORM_LDIF_WRITE_ERROR.get(targetFile.getAbsolutePath(),
1082                    StaticUtils.getExceptionMessage(e)));
1083          resultCode = ResultCode.LOCAL_ERROR;
1084          break processingBlock;
1085        }
1086
1087
1088        // If we've written a multiple of 1000 entries, print a progress
1089        // message.
1090        entriesWritten++;
1091        if ((! targetToStandardOutput.isPresent()) &&
1092            ((entriesWritten % 1000L) == 0))
1093        {
1094          final long numExcluded = excludedEntryCount.get();
1095          if (numExcluded > 0L)
1096          {
1097            wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1098                 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_WITH_EXCLUDED.get(
1099                      entriesWritten, numExcluded));
1100          }
1101          else
1102          {
1103            wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1104                 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_NONE_EXCLUDED.get(
1105                      entriesWritten));
1106          }
1107        }
1108      }
1109
1110
1111      if (! targetToStandardOutput.isPresent())
1112      {
1113        final long numExcluded = excludedEntryCount.get();
1114        if (numExcluded > 0L)
1115        {
1116          wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1117               INFO_TRANSFORM_LDIF_COMPLETE_WITH_EXCLUDED.get(entriesWritten,
1118                    numExcluded));
1119        }
1120        else
1121        {
1122          wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1123               INFO_TRANSFORM_LDIF_COMPLETE_NONE_EXCLUDED.get(entriesWritten));
1124        }
1125      }
1126    }
1127    finally
1128    {
1129      if (outputStream != null)
1130      {
1131        try
1132        {
1133          outputStream.close();
1134        }
1135        catch (final Exception e)
1136        {
1137          Debug.debugException(e);
1138          wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1139               ERR_TRANSFORM_LDIF_ERROR_CLOSING_OUTPUT_STREAM.get(
1140                    targetFile.getAbsolutePath(),
1141                    StaticUtils.getExceptionMessage(e)));
1142          if (resultCode == ResultCode.SUCCESS)
1143          {
1144            resultCode = ResultCode.LOCAL_ERROR;
1145          }
1146        }
1147      }
1148
1149      try
1150      {
1151        ldifReader.close();
1152      }
1153      catch (final Exception e)
1154      {
1155        Debug.debugException(e);
1156        // We can ignore this.
1157      }
1158    }
1159
1160
1161    return resultCode;
1162  }
1163
1164
1165
1166  /**
1167   * Retrieves the schema that should be used for processing.
1168   *
1169   * @return  The schema that was created.
1170   *
1171   * @throws  LDAPException  If a problem is encountered while retrieving the
1172   *                         schema.
1173   */
1174  private Schema getSchema()
1175          throws LDAPException
1176  {
1177    // If any schema paths were specified, then load the schema only from those
1178    // paths.
1179    if (schemaPath.isPresent())
1180    {
1181      final ArrayList<File> schemaFiles = new ArrayList<>(10);
1182      for (final File path : schemaPath.getValues())
1183      {
1184        if (path.isFile())
1185        {
1186          schemaFiles.add(path);
1187        }
1188        else
1189        {
1190          final TreeMap<String,File> fileMap = new TreeMap<>();
1191          for (final File schemaDirFile : path.listFiles())
1192          {
1193            final String name = schemaDirFile.getName();
1194            if (schemaDirFile.isFile() && name.toLowerCase().endsWith(".ldif"))
1195            {
1196              fileMap.put(name, schemaDirFile);
1197            }
1198          }
1199          schemaFiles.addAll(fileMap.values());
1200        }
1201      }
1202
1203      if (schemaFiles.isEmpty())
1204      {
1205        throw new LDAPException(ResultCode.PARAM_ERROR,
1206             ERR_TRANSFORM_LDIF_NO_SCHEMA_FILES.get(
1207                  schemaPath.getIdentifierString()));
1208      }
1209      else
1210      {
1211        try
1212        {
1213          return Schema.getSchema(schemaFiles);
1214        }
1215        catch (final Exception e)
1216        {
1217          Debug.debugException(e);
1218          throw new LDAPException(ResultCode.LOCAL_ERROR,
1219               ERR_TRANSFORM_LDIF_ERROR_LOADING_SCHEMA.get(
1220                    StaticUtils.getExceptionMessage(e)));
1221        }
1222      }
1223    }
1224    else
1225    {
1226      // If the INSTANCE_ROOT environment variable is set and it refers to a
1227      // directory that has a config/schema subdirectory that has one or more
1228      // schema files in it, then read the schema from that directory.
1229      try
1230      {
1231        final String instanceRootStr = System.getenv("INSTANCE_ROOT");
1232        if (instanceRootStr != null)
1233        {
1234          final File instanceRoot = new File(instanceRootStr);
1235          final File configDir = new File(instanceRoot, "config");
1236          final File schemaDir = new File(configDir, "schema");
1237          if (schemaDir.exists())
1238          {
1239            final TreeMap<String,File> fileMap = new TreeMap<>();
1240            for (final File schemaDirFile : schemaDir.listFiles())
1241            {
1242              final String name = schemaDirFile.getName();
1243              if (schemaDirFile.isFile() &&
1244                  name.toLowerCase().endsWith(".ldif"))
1245              {
1246                fileMap.put(name, schemaDirFile);
1247              }
1248            }
1249
1250            if (! fileMap.isEmpty())
1251            {
1252              return Schema.getSchema(new ArrayList<>(fileMap.values()));
1253            }
1254          }
1255        }
1256      }
1257      catch (final Exception e)
1258      {
1259        Debug.debugException(e);
1260      }
1261    }
1262
1263
1264    // If we've gotten here, then just return null and the tool will try to use
1265    // the default standard schema.
1266    return null;
1267  }
1268
1269
1270
1271  /**
1272   * Creates the entry and change record translators that will be used to
1273   * perform the transformations.
1274   *
1275   * @param  entryTranslators         A list to which all created entry
1276   *                                  translators should be written.
1277   * @param  changeRecordTranslators  A list to which all created change record
1278   *                                  translators should be written.
1279   * @param  schema                   The schema to use when processing.
1280   * @param  excludedEntryCount       A counter used to keep track of the number
1281   *                                  of entries that have been excluded from
1282   *                                  the result set.
1283   */
1284  private void createTranslators(
1285       final List<LDIFReaderEntryTranslator> entryTranslators,
1286       final List<LDIFReaderChangeRecordTranslator> changeRecordTranslators,
1287       final Schema schema, final AtomicLong excludedEntryCount)
1288  {
1289    if (scrambleAttribute.isPresent())
1290    {
1291      final Long seed;
1292      if (randomSeed.isPresent())
1293      {
1294        seed = randomSeed.getValue().longValue();
1295      }
1296      else
1297      {
1298        seed = null;
1299      }
1300
1301      final ScrambleAttributeTransformation t =
1302           new ScrambleAttributeTransformation(schema, seed,
1303                processDNs.isPresent(), scrambleAttribute.getValues(),
1304                scrambleJSONField.getValues());
1305      entryTranslators.add(t);
1306      changeRecordTranslators.add(t);
1307    }
1308
1309    if (sequentialAttribute.isPresent())
1310    {
1311      final long initialValue;
1312      if (initialSequentialValue.isPresent())
1313      {
1314        initialValue = initialSequentialValue.getValue().longValue();
1315      }
1316      else
1317      {
1318        initialValue = 0L;
1319      }
1320
1321      final long incrementAmount;
1322      if (sequentialValueIncrement.isPresent())
1323      {
1324        incrementAmount = sequentialValueIncrement.getValue().longValue();
1325      }
1326      else
1327      {
1328        incrementAmount = 1L;
1329      }
1330
1331      for (final String attrName : sequentialAttribute.getValues())
1332      {
1333
1334
1335        final ReplaceWithCounterTransformation t =
1336             new ReplaceWithCounterTransformation(schema, attrName,
1337                  initialValue, incrementAmount,
1338                  textBeforeSequentialValue.getValue(),
1339                  textAfterSequentialValue.getValue(), processDNs.isPresent());
1340        entryTranslators.add(t);
1341      }
1342    }
1343
1344    if (replaceValuesAttribute.isPresent())
1345    {
1346      final ReplaceAttributeTransformation t =
1347           new ReplaceAttributeTransformation(schema,
1348                replaceValuesAttribute.getValue(),
1349                replacementValue.getValues());
1350      entryTranslators.add(t);
1351    }
1352
1353    if (addAttributeName.isPresent())
1354    {
1355      final AddAttributeTransformation t = new AddAttributeTransformation(
1356           schema, addAttributeBaseDN.getValue(), addAttributeScope.getValue(),
1357           addAttributeFilter.getValue(),
1358           new Attribute(addAttributeName.getValue(), schema,
1359                addAttributeValue.getValues()),
1360           (! addToExistingValues.isPresent()));
1361      entryTranslators.add(t);
1362    }
1363
1364    if (renameAttributeFrom.isPresent())
1365    {
1366      final Iterator<String> renameFromIterator =
1367           renameAttributeFrom.getValues().iterator();
1368      final Iterator<String> renameToIterator =
1369           renameAttributeTo.getValues().iterator();
1370      while (renameFromIterator.hasNext())
1371      {
1372        final RenameAttributeTransformation t =
1373             new RenameAttributeTransformation(schema,
1374                  renameFromIterator.next(), renameToIterator.next(),
1375                  processDNs.isPresent());
1376        entryTranslators.add(t);
1377        changeRecordTranslators.add(t);
1378      }
1379    }
1380
1381    if (flattenBaseDN.isPresent())
1382    {
1383      final FlattenSubtreeTransformation t = new FlattenSubtreeTransformation(
1384           schema, flattenBaseDN.getValue(),
1385           flattenAddOmittedRDNAttributesToEntry.isPresent(),
1386           flattenAddOmittedRDNAttributesToRDN.isPresent(),
1387           flattenExcludeFilter.getValue());
1388      entryTranslators.add(t);
1389    }
1390
1391    if (moveSubtreeFrom.isPresent())
1392    {
1393      final Iterator<DN> moveFromIterator =
1394           moveSubtreeFrom.getValues().iterator();
1395      final Iterator<DN> moveToIterator = moveSubtreeTo.getValues().iterator();
1396      while (moveFromIterator.hasNext())
1397      {
1398        final MoveSubtreeTransformation t =
1399             new MoveSubtreeTransformation(moveFromIterator.next(),
1400                  moveToIterator.next());
1401        entryTranslators.add(t);
1402        changeRecordTranslators.add(t);
1403      }
1404    }
1405
1406    if (redactAttribute.isPresent())
1407    {
1408      final RedactAttributeTransformation t = new RedactAttributeTransformation(
1409           schema, processDNs.isPresent(),
1410           (! hideRedactedValueCount.isPresent()), redactAttribute.getValues());
1411      entryTranslators.add(t);
1412      changeRecordTranslators.add(t);
1413    }
1414
1415    if (excludeAttribute.isPresent())
1416    {
1417      final ExcludeAttributeTransformation t =
1418           new ExcludeAttributeTransformation(schema,
1419                excludeAttribute.getValues());
1420      entryTranslators.add(t);
1421      changeRecordTranslators.add(t);
1422    }
1423
1424    if (excludeEntryBaseDN.isPresent() || excludeEntryScope.isPresent() ||
1425        excludeEntryFilter.isPresent())
1426    {
1427      final ExcludeEntryTransformation t = new ExcludeEntryTransformation(
1428           schema, excludeEntryBaseDN.getValue(), excludeEntryScope.getValue(),
1429           excludeEntryFilter.getValue(),
1430           (! excludeNonMatchingEntries.isPresent()), excludedEntryCount);
1431      entryTranslators.add(t);
1432    }
1433
1434    entryTranslators.add(this);
1435  }
1436
1437
1438
1439  /**
1440   * {@inheritDoc}
1441   */
1442  @Override()
1443  public LinkedHashMap<String[],String> getExampleUsages()
1444  {
1445    final LinkedHashMap<String[],String> examples =
1446         new LinkedHashMap<>(StaticUtils.computeMapCapacity(4));
1447
1448    examples.put(
1449         new String[]
1450         {
1451           "--sourceLDIF", "input.ldif",
1452           "--targetLDIF", "scrambled.ldif",
1453           "--scrambleAttribute", "givenName",
1454           "--scrambleAttribute", "sn",
1455           "--scrambleAttribute", "cn",
1456           "--numThreads", "10",
1457           "--schemaPath", "/ds/config/schema",
1458           "--processDNs"
1459         },
1460         INFO_TRANSFORM_LDIF_EXAMPLE_SCRAMBLE.get());
1461
1462    examples.put(
1463         new String[]
1464         {
1465           "--sourceLDIF", "input.ldif",
1466           "--targetLDIF", "sequential.ldif",
1467           "--sequentialAttribute", "uid",
1468           "--initialSequentialValue", "1",
1469           "--sequentialValueIncrement", "1",
1470           "--textBeforeSequentialValue", "user.",
1471           "--numThreads", "10",
1472           "--schemaPath", "/ds/config/schema",
1473           "--processDNs"
1474         },
1475         INFO_TRANSFORM_LDIF_EXAMPLE_SEQUENTIAL.get());
1476
1477    examples.put(
1478         new String[]
1479         {
1480           "--sourceLDIF", "input.ldif",
1481           "--targetLDIF", "added-organization.ldif",
1482           "--addAttributeName", "o",
1483           "--addAttributeValue", "Example Corp.",
1484           "--addAttributeFilter", "(objectClass=person)",
1485           "--numThreads", "10",
1486           "--schemaPath", "/ds/config/schema"
1487         },
1488         INFO_TRANSFORM_LDIF_EXAMPLE_ADD.get());
1489
1490    examples.put(
1491         new String[]
1492         {
1493           "--sourceLDIF", "input.ldif",
1494           "--targetLDIF", "rebased.ldif",
1495           "--moveSubtreeFrom", "o=example.com",
1496           "--moveSubtreeTo", "dc=example,dc=com",
1497           "--numThreads", "10",
1498           "--schemaPath", "/ds/config/schema"
1499         },
1500         INFO_TRANSFORM_LDIF_EXAMPLE_REBASE.get());
1501
1502    return examples;
1503  }
1504
1505
1506
1507  /**
1508   * {@inheritDoc}
1509   */
1510  @Override()
1511  public Entry translate(final Entry original, final long firstLineNumber)
1512         throws LDIFException
1513  {
1514    final ByteStringBuffer buffer = getBuffer();
1515    if (wrapColumn.isPresent())
1516    {
1517      original.toLDIF(buffer, wrapColumn.getValue());
1518    }
1519    else
1520    {
1521      original.toLDIF(buffer, 0);
1522    }
1523    buffer.append(StaticUtils.EOL_BYTES);
1524
1525    return new PreEncodedLDIFEntry(original, buffer.toByteArray());
1526  }
1527
1528
1529
1530  /**
1531   * Retrieves a byte string buffer that can be used to perform LDIF encoding.
1532   *
1533   * @return  A byte string buffer that can be used to perform LDIF encoding.
1534   */
1535  private ByteStringBuffer getBuffer()
1536  {
1537    ByteStringBuffer buffer = byteStringBuffers.get();
1538    if (buffer == null)
1539    {
1540      buffer = new ByteStringBuffer();
1541      byteStringBuffers.set(buffer);
1542    }
1543    else
1544    {
1545      buffer.clear();
1546    }
1547
1548    return buffer;
1549  }
1550}