001/*
002 * Copyright 2008-2019 Ping Identity Corporation
003 * All Rights Reserved.
004 */
005/*
006 * Copyright (C) 2008-2019 Ping Identity Corporation
007 *
008 * This program is free software; you can redistribute it and/or modify
009 * it under the terms of the GNU General Public License (GPLv2 only)
010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011 * as published by the Free Software Foundation.
012 *
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016 * GNU General Public License for more details.
017 *
018 * You should have received a copy of the GNU General Public License
019 * along with this program; if not, see <http://www.gnu.org/licenses>.
020 */
021package com.unboundid.ldap.sdk.examples;
022
023
024
025import java.io.File;
026import java.io.FileInputStream;
027import java.io.InputStream;
028import java.io.IOException;
029import java.io.OutputStream;
030import java.util.ArrayList;
031import java.util.Iterator;
032import java.util.TreeMap;
033import java.util.LinkedHashMap;
034import java.util.List;
035import java.util.concurrent.atomic.AtomicLong;
036import java.util.zip.GZIPInputStream;
037
038import com.unboundid.ldap.sdk.Entry;
039import com.unboundid.ldap.sdk.LDAPConnection;
040import com.unboundid.ldap.sdk.LDAPException;
041import com.unboundid.ldap.sdk.ResultCode;
042import com.unboundid.ldap.sdk.Version;
043import com.unboundid.ldap.sdk.schema.Schema;
044import com.unboundid.ldap.sdk.schema.EntryValidator;
045import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils;
046import com.unboundid.ldif.DuplicateValueBehavior;
047import com.unboundid.ldif.LDIFException;
048import com.unboundid.ldif.LDIFReader;
049import com.unboundid.ldif.LDIFReaderEntryTranslator;
050import com.unboundid.ldif.LDIFWriter;
051import com.unboundid.util.Debug;
052import com.unboundid.util.LDAPCommandLineTool;
053import com.unboundid.util.StaticUtils;
054import com.unboundid.util.ThreadSafety;
055import com.unboundid.util.ThreadSafetyLevel;
056import com.unboundid.util.args.ArgumentException;
057import com.unboundid.util.args.ArgumentParser;
058import com.unboundid.util.args.BooleanArgument;
059import com.unboundid.util.args.FileArgument;
060import com.unboundid.util.args.IntegerArgument;
061import com.unboundid.util.args.StringArgument;
062
063
064
065/**
066 * This class provides a simple tool that can be used to validate that the
067 * contents of an LDIF file are valid.  This includes ensuring that the contents
068 * can be parsed as valid LDIF, and it can also ensure that the LDIF content
069 * conforms to the server schema.  It will obtain the schema by connecting to
070 * the server and retrieving the default schema (i.e., the schema which governs
071 * the root DSE).  By default, a thorough set of validation will be performed,
072 * but it is possible to disable certain types of validation.
073 * <BR><BR>
074 * Some of the APIs demonstrated by this example include:
075 * <UL>
076 *   <LI>Argument Parsing (from the {@code com.unboundid.util.args}
077 *       package)</LI>
078 *   <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util}
079 *       package)</LI>
080 *   <LI>LDIF Processing (from the {@code com.unboundid.ldif} package)</LI>
081 *   <LI>Schema Parsing (from the {@code com.unboundid.ldap.sdk.schema}
082 *       package)</LI>
083 * </UL>
084 * <BR><BR>
085 * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
086 * class (to obtain the information to use to connect to the server to read the
087 * schema), as well as the following additional arguments:
088 * <UL>
089 *   <LI>"--schemaDirectory {path}" -- specifies the path to a directory
090 *       containing files with schema definitions.  If this argument is
091 *       provided, then no attempt will be made to communicate with a directory
092 *       server.</LI>
093 *   <LI>"-f {path}" or "--ldifFile {path}" -- specifies the path to the LDIF
094 *       file to be validated.</LI>
095 *   <LI>"-c" or "--isCompressed" -- indicates that the LDIF file is
096 *       compressed.</LI>
097 *   <LI>"-R {path}" or "--rejectFile {path}" -- specifies the path to the file
098 *       to be written with information about all entries that failed
099 *       validation.</LI>
100 *   <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of
101 *       concurrent threads to use when processing the LDIF.  If this is not
102 *       provided, then a default of one thread will be used.</LI>
103 *   <LI>"--ignoreUndefinedObjectClasses" -- indicates that the validation
104 *       process should ignore validation failures due to entries that contain
105 *       object classes not defined in the server schema.</LI>
106 *   <LI>"--ignoreUndefinedAttributes" -- indicates that the validation process
107 *       should ignore validation failures due to entries that contain
108 *       attributes not defined in the server schema.</LI>
109 *   <LI>"--ignoreMalformedDNs" -- indicates that the validation process should
110 *       ignore validation failures due to entries with malformed DNs.</LI>
111 *   <LI>"--ignoreMissingRDNValues" -- indicates that the validation process
112 *       should ignore validation failures due to entries that contain an RDN
113 *       attribute value that is not present in the set of entry
114 *       attributes.</LI>
115 *   <LI>"--ignoreStructuralObjectClasses" -- indicates that the validation
116 *       process should ignore validation failures due to entries that either do
117 *       not have a structural object class or that have multiple structural
118 *       object classes.</LI>
119 *   <LI>"--ignoreProhibitedObjectClasses" -- indicates that the validation
120 *       process should ignore validation failures due to entries containing
121 *       auxiliary classes that are not allowed by a DIT content rule, or
122 *       abstract classes that are not subclassed by an auxiliary or structural
123 *       class contained in the entry.</LI>
124 *   <LI>"--ignoreProhibitedAttributes" -- indicates that the validation process
125 *       should ignore validation failures due to entries including attributes
126 *       that are not allowed or are explicitly prohibited by a DIT content
127 *       rule.</LI>
128 *   <LI>"--ignoreMissingAttributes" -- indicates that the validation process
129 *       should ignore validation failures due to entries missing required
130 *       attributes.</LI>
131 *   <LI>"--ignoreSingleValuedAttributes" -- indicates that the validation
132 *       process should ignore validation failures due to single-valued
133 *       attributes containing multiple values.</LI>
134 *   <LI>"--ignoreAttributeSyntax" -- indicates that the validation process
135 *       should ignore validation failures due to attribute values which violate
136 *       the associated attribute syntax.</LI>
137 *   <LI>"--ignoreSyntaxViolationsForAttribute" -- indicates that the validation
138 *       process should ignore validation failures due to attribute values which
139 *       violate the associated attribute syntax, but only for the specified
140 *       attribute types.</LI>
141 *   <LI>"--ignoreNameForms" -- indicates that the validation process should
142 *       ignore validation failures due to name form violations (in which the
143 *       entry's RDN does not comply with the associated name form).</LI>
144 * </UL>
145 */
146@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
147public final class ValidateLDIF
148       extends LDAPCommandLineTool
149       implements LDIFReaderEntryTranslator
150{
151  /**
152   * The end-of-line character for this platform.
153   */
154  private static final String EOL = System.getProperty("line.separator", "\n");
155
156
157
158  // The arguments used by this program.
159  private BooleanArgument ignoreDuplicateValues;
160  private BooleanArgument ignoreUndefinedObjectClasses;
161  private BooleanArgument ignoreUndefinedAttributes;
162  private BooleanArgument ignoreMalformedDNs;
163  private BooleanArgument ignoreMissingRDNValues;
164  private BooleanArgument ignoreMissingSuperiorObjectClasses;
165  private BooleanArgument ignoreStructuralObjectClasses;
166  private BooleanArgument ignoreProhibitedObjectClasses;
167  private BooleanArgument ignoreProhibitedAttributes;
168  private BooleanArgument ignoreMissingAttributes;
169  private BooleanArgument ignoreSingleValuedAttributes;
170  private BooleanArgument ignoreAttributeSyntax;
171  private BooleanArgument ignoreNameForms;
172  private BooleanArgument isCompressed;
173  private FileArgument    schemaDirectory;
174  private FileArgument    ldifFile;
175  private FileArgument    rejectFile;
176  private FileArgument    encryptionPassphraseFile;
177  private IntegerArgument numThreads;
178  private StringArgument  ignoreSyntaxViolationsForAttribute;
179
180  // The counter used to keep track of the number of entries processed.
181  private final AtomicLong entriesProcessed = new AtomicLong(0L);
182
183  // The counter used to keep track of the number of entries that could not be
184  // parsed as valid entries.
185  private final AtomicLong malformedEntries = new AtomicLong(0L);
186
187  // The entry validator that will be used to validate the entries.
188  private EntryValidator entryValidator;
189
190  // The LDIF writer that will be used to write rejected entries.
191  private LDIFWriter rejectWriter;
192
193
194
195  /**
196   * Parse the provided command line arguments and make the appropriate set of
197   * changes.
198   *
199   * @param  args  The command line arguments provided to this program.
200   */
201  public static void main(final String[] args)
202  {
203    final ResultCode resultCode = main(args, System.out, System.err);
204    if (resultCode != ResultCode.SUCCESS)
205    {
206      System.exit(resultCode.intValue());
207    }
208  }
209
210
211
212  /**
213   * Parse the provided command line arguments and make the appropriate set of
214   * changes.
215   *
216   * @param  args       The command line arguments provided to this program.
217   * @param  outStream  The output stream to which standard out should be
218   *                    written.  It may be {@code null} if output should be
219   *                    suppressed.
220   * @param  errStream  The output stream to which standard error should be
221   *                    written.  It may be {@code null} if error messages
222   *                    should be suppressed.
223   *
224   * @return  A result code indicating whether the processing was successful.
225   */
226  public static ResultCode main(final String[] args,
227                                final OutputStream outStream,
228                                final OutputStream errStream)
229  {
230    final ValidateLDIF validateLDIF = new ValidateLDIF(outStream, errStream);
231    return validateLDIF.runTool(args);
232  }
233
234
235
236  /**
237   * Creates a new instance of this tool.
238   *
239   * @param  outStream  The output stream to which standard out should be
240   *                    written.  It may be {@code null} if output should be
241   *                    suppressed.
242   * @param  errStream  The output stream to which standard error should be
243   *                    written.  It may be {@code null} if error messages
244   *                    should be suppressed.
245   */
246  public ValidateLDIF(final OutputStream outStream,
247                      final OutputStream errStream)
248  {
249    super(outStream, errStream);
250  }
251
252
253
254  /**
255   * Retrieves the name for this tool.
256   *
257   * @return  The name for this tool.
258   */
259  @Override()
260  public String getToolName()
261  {
262    return "validate-ldif";
263  }
264
265
266
267  /**
268   * Retrieves the description for this tool.
269   *
270   * @return  The description for this tool.
271   */
272  @Override()
273  public String getToolDescription()
274  {
275    return "Validate the contents of an LDIF file " +
276           "against the server schema.";
277  }
278
279
280
281  /**
282   * Retrieves the version string for this tool.
283   *
284   * @return  The version string for this tool.
285   */
286  @Override()
287  public String getToolVersion()
288  {
289    return Version.NUMERIC_VERSION_STRING;
290  }
291
292
293
294  /**
295   * Indicates whether this tool should provide support for an interactive mode,
296   * in which the tool offers a mode in which the arguments can be provided in
297   * a text-driven menu rather than requiring them to be given on the command
298   * line.  If interactive mode is supported, it may be invoked using the
299   * "--interactive" argument.  Alternately, if interactive mode is supported
300   * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
301   * interactive mode may be invoked by simply launching the tool without any
302   * arguments.
303   *
304   * @return  {@code true} if this tool supports interactive mode, or
305   *          {@code false} if not.
306   */
307  @Override()
308  public boolean supportsInteractiveMode()
309  {
310    return true;
311  }
312
313
314
315  /**
316   * Indicates whether this tool defaults to launching in interactive mode if
317   * the tool is invoked without any command-line arguments.  This will only be
318   * used if {@link #supportsInteractiveMode()} returns {@code true}.
319   *
320   * @return  {@code true} if this tool defaults to using interactive mode if
321   *          launched without any command-line arguments, or {@code false} if
322   *          not.
323   */
324  @Override()
325  public boolean defaultsToInteractiveMode()
326  {
327    return true;
328  }
329
330
331
332  /**
333   * Indicates whether this tool should provide arguments for redirecting output
334   * to a file.  If this method returns {@code true}, then the tool will offer
335   * an "--outputFile" argument that will specify the path to a file to which
336   * all standard output and standard error content will be written, and it will
337   * also offer a "--teeToStandardOut" argument that can only be used if the
338   * "--outputFile" argument is present and will cause all output to be written
339   * to both the specified output file and to standard output.
340   *
341   * @return  {@code true} if this tool should provide arguments for redirecting
342   *          output to a file, or {@code false} if not.
343   */
344  @Override()
345  protected boolean supportsOutputFile()
346  {
347    return true;
348  }
349
350
351
352  /**
353   * Indicates whether this tool should default to interactively prompting for
354   * the bind password if a password is required but no argument was provided
355   * to indicate how to get the password.
356   *
357   * @return  {@code true} if this tool should default to interactively
358   *          prompting for the bind password, or {@code false} if not.
359   */
360  @Override()
361  protected boolean defaultToPromptForBindPassword()
362  {
363    return true;
364  }
365
366
367
368  /**
369   * Indicates whether this tool supports the use of a properties file for
370   * specifying default values for arguments that aren't specified on the
371   * command line.
372   *
373   * @return  {@code true} if this tool supports the use of a properties file
374   *          for specifying default values for arguments that aren't specified
375   *          on the command line, or {@code false} if not.
376   */
377  @Override()
378  public boolean supportsPropertiesFile()
379  {
380    return true;
381  }
382
383
384
385  /**
386   * Indicates whether the LDAP-specific arguments should include alternate
387   * versions of all long identifiers that consist of multiple words so that
388   * they are available in both camelCase and dash-separated versions.
389   *
390   * @return  {@code true} if this tool should provide multiple versions of
391   *          long identifiers for LDAP-specific arguments, or {@code false} if
392   *          not.
393   */
394  @Override()
395  protected boolean includeAlternateLongIdentifiers()
396  {
397    return true;
398  }
399
400
401
402  /**
403   * Adds the arguments used by this program that aren't already provided by the
404   * generic {@code LDAPCommandLineTool} framework.
405   *
406   * @param  parser  The argument parser to which the arguments should be added.
407   *
408   * @throws  ArgumentException  If a problem occurs while adding the arguments.
409   */
410  @Override()
411  public void addNonLDAPArguments(final ArgumentParser parser)
412         throws ArgumentException
413  {
414    String description = "The path to the LDIF file to process.  The tool " +
415         "will automatically attempt to detect whether the file is " +
416         "encrypted or compressed.";
417    ldifFile = new FileArgument('f', "ldifFile", true, 1, "{path}", description,
418                                true, true, true, false);
419    ldifFile.addLongIdentifier("ldif-file", true);
420    parser.addArgument(ldifFile);
421
422
423    // Add an argument that makes it possible to read a compressed LDIF file.
424    // Note that this argument is no longer needed for dealing with compressed
425    // files, since the tool will automatically detect whether a file is
426    // compressed.  However, the argument is still provided for the purpose of
427    // backward compatibility.
428    description = "Indicates that the specified LDIF file is compressed " +
429                  "using gzip compression.";
430    isCompressed = new BooleanArgument('c', "isCompressed", description);
431    isCompressed.addLongIdentifier("is-compressed", true);
432    isCompressed.setHidden(true);
433    parser.addArgument(isCompressed);
434
435
436    // Add an argument that indicates that the tool should read the encryption
437    // passphrase from a file.
438    description = "Indicates that the specified LDIF file is encrypted and " +
439         "that the encryption passphrase is contained in the specified " +
440         "file.  If the LDIF data is encrypted and this argument is not " +
441         "provided, then the tool will interactively prompt for the " +
442         "encryption passphrase.";
443    encryptionPassphraseFile = new FileArgument(null,
444         "encryptionPassphraseFile", false, 1, null, description, true, true,
445         true, false);
446    encryptionPassphraseFile.addLongIdentifier("encryption-passphrase-file",
447         true);
448    encryptionPassphraseFile.addLongIdentifier("encryptionPasswordFile", true);
449    encryptionPassphraseFile.addLongIdentifier("encryption-password-file",
450         true);
451    parser.addArgument(encryptionPassphraseFile);
452
453
454    description = "The path to the file to which rejected entries should be " +
455                  "written.";
456    rejectFile = new FileArgument('R', "rejectFile", false, 1, "{path}",
457                                  description, false, true, true, false);
458    rejectFile.addLongIdentifier("reject-file", true);
459    parser.addArgument(rejectFile);
460
461    description = "The path to a directory containing one or more LDIF files " +
462                  "with the schema information to use.  If this is provided, " +
463                  "then no LDAP communication will be performed.";
464    schemaDirectory = new FileArgument(null, "schemaDirectory", false, 1,
465         "{path}", description, true, true, false, true);
466    schemaDirectory.addLongIdentifier("schema-directory", true);
467    parser.addArgument(schemaDirectory);
468
469    description = "The number of threads to use when processing the LDIF file.";
470    numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}",
471         description, 1, Integer.MAX_VALUE, 1);
472    numThreads.addLongIdentifier("num-threads", true);
473    parser.addArgument(numThreads);
474
475    description = "Ignore validation failures due to entries containing " +
476                  "duplicate values for the same attribute.";
477    ignoreDuplicateValues =
478         new BooleanArgument(null, "ignoreDuplicateValues", description);
479    ignoreDuplicateValues.setArgumentGroupName(
480         "Validation Strictness Arguments");
481    ignoreDuplicateValues.addLongIdentifier("ignore-duplicate-values", true);
482    parser.addArgument(ignoreDuplicateValues);
483
484    description = "Ignore validation failures due to object classes not " +
485                  "defined in the schema.";
486    ignoreUndefinedObjectClasses =
487         new BooleanArgument(null, "ignoreUndefinedObjectClasses", description);
488    ignoreUndefinedObjectClasses.setArgumentGroupName(
489         "Validation Strictness Arguments");
490    ignoreUndefinedObjectClasses.addLongIdentifier(
491         "ignore-undefined-object-classes", true);
492    parser.addArgument(ignoreUndefinedObjectClasses);
493
494    description = "Ignore validation failures due to attributes not defined " +
495                  "in the schema.";
496    ignoreUndefinedAttributes =
497         new BooleanArgument(null, "ignoreUndefinedAttributes", description);
498    ignoreUndefinedAttributes.setArgumentGroupName(
499         "Validation Strictness Arguments");
500    ignoreUndefinedAttributes.addLongIdentifier("ignore-undefined-attributes",
501         true);
502    parser.addArgument(ignoreUndefinedAttributes);
503
504    description = "Ignore validation failures due to entries with malformed " +
505                  "DNs.";
506    ignoreMalformedDNs =
507         new BooleanArgument(null, "ignoreMalformedDNs", description);
508    ignoreMalformedDNs.setArgumentGroupName("Validation Strictness Arguments");
509    ignoreMalformedDNs.addLongIdentifier("ignore-malformed-dns", true);
510    parser.addArgument(ignoreMalformedDNs);
511
512    description = "Ignore validation failures due to entries with RDN " +
513                  "attribute values that are missing from the set of entry " +
514                  "attributes.";
515    ignoreMissingRDNValues =
516         new BooleanArgument(null, "ignoreMissingRDNValues", description);
517    ignoreMissingRDNValues.setArgumentGroupName(
518         "Validation Strictness Arguments");
519    ignoreMissingRDNValues.addLongIdentifier("ignore-missing-rdn-values", true);
520    parser.addArgument(ignoreMissingRDNValues);
521
522    description = "Ignore validation failures due to entries without exactly " +
523                  "structural object class.";
524    ignoreStructuralObjectClasses =
525         new BooleanArgument(null, "ignoreStructuralObjectClasses",
526                             description);
527    ignoreStructuralObjectClasses.setArgumentGroupName(
528         "Validation Strictness Arguments");
529    ignoreStructuralObjectClasses.addLongIdentifier(
530         "ignore-structural-object-classes", true);
531    parser.addArgument(ignoreStructuralObjectClasses);
532
533    description = "Ignore validation failures due to entries with object " +
534                  "classes that are not allowed.";
535    ignoreProhibitedObjectClasses =
536         new BooleanArgument(null, "ignoreProhibitedObjectClasses",
537                             description);
538    ignoreProhibitedObjectClasses.setArgumentGroupName(
539         "Validation Strictness Arguments");
540    ignoreProhibitedObjectClasses.addLongIdentifier(
541         "ignore-prohibited-object-classes", true);
542    parser.addArgument(ignoreProhibitedObjectClasses);
543
544    description = "Ignore validation failures due to entries that are " +
545                  "one or more superior object classes.";
546    ignoreMissingSuperiorObjectClasses =
547         new BooleanArgument(null, "ignoreMissingSuperiorObjectClasses",
548              description);
549    ignoreMissingSuperiorObjectClasses.setArgumentGroupName(
550         "Validation Strictness Arguments");
551    ignoreMissingSuperiorObjectClasses.addLongIdentifier(
552         "ignore-missing-superior-object-classes", true);
553    parser.addArgument(ignoreMissingSuperiorObjectClasses);
554
555    description = "Ignore validation failures due to entries with attributes " +
556                  "that are not allowed.";
557    ignoreProhibitedAttributes =
558         new BooleanArgument(null, "ignoreProhibitedAttributes", description);
559    ignoreProhibitedAttributes.setArgumentGroupName(
560         "Validation Strictness Arguments");
561    ignoreProhibitedAttributes.addLongIdentifier(
562         "ignore-prohibited-attributes", true);
563    parser.addArgument(ignoreProhibitedAttributes);
564
565    description = "Ignore validation failures due to entries missing " +
566                  "required attributes.";
567    ignoreMissingAttributes =
568         new BooleanArgument(null, "ignoreMissingAttributes", description);
569    ignoreMissingAttributes.setArgumentGroupName(
570         "Validation Strictness Arguments");
571    ignoreMissingAttributes.addLongIdentifier("ignore-missing-attributes",
572         true);
573    parser.addArgument(ignoreMissingAttributes);
574
575    description = "Ignore validation failures due to entries with multiple " +
576                  "values for single-valued attributes.";
577    ignoreSingleValuedAttributes =
578         new BooleanArgument(null, "ignoreSingleValuedAttributes", description);
579    ignoreSingleValuedAttributes.setArgumentGroupName(
580         "Validation Strictness Arguments");
581    ignoreSingleValuedAttributes.addLongIdentifier(
582         "ignore-single-valued-attributes", true);
583    parser.addArgument(ignoreSingleValuedAttributes);
584
585    description = "Ignore validation failures due to entries with attribute " +
586                  "values that violate their associated syntax.  If this is " +
587                  "provided, then no attribute syntax violations will be " +
588                  "flagged.  If this is not provided, then all attribute " +
589                  "syntax violations will be flagged except for violations " +
590                  "in those attributes excluded by the " +
591                  "--ignoreSyntaxViolationsForAttribute argument.";
592    ignoreAttributeSyntax =
593         new BooleanArgument(null, "ignoreAttributeSyntax", description);
594    ignoreAttributeSyntax.setArgumentGroupName(
595         "Validation Strictness Arguments");
596    ignoreAttributeSyntax.addLongIdentifier("ignore-attribute-syntax", true);
597    parser.addArgument(ignoreAttributeSyntax);
598
599    description = "The name or OID of an attribute for which to ignore " +
600                  "validation failures due to violations of the associated " +
601                  "attribute syntax.  This argument can only be used if the " +
602                  "--ignoreAttributeSyntax argument is not provided.";
603    ignoreSyntaxViolationsForAttribute = new StringArgument(null,
604         "ignoreSyntaxViolationsForAttribute", false, 0, "{attr}", description);
605    ignoreSyntaxViolationsForAttribute.setArgumentGroupName(
606         "Validation Strictness Arguments");
607    ignoreSyntaxViolationsForAttribute.addLongIdentifier(
608         "ignore-syntax-violations-for-attribute", true);
609    parser.addArgument(ignoreSyntaxViolationsForAttribute);
610
611    description = "Ignore validation failures due to entries with RDNs " +
612                  "that violate the associated name form definition.";
613    ignoreNameForms = new BooleanArgument(null, "ignoreNameForms", description);
614    ignoreNameForms.setArgumentGroupName("Validation Strictness Arguments");
615    ignoreNameForms.addLongIdentifier("ignore-name-forms", true);
616    parser.addArgument(ignoreNameForms);
617
618
619    // The ignoreAttributeSyntax and ignoreAttributeSyntaxForAttribute arguments
620    // cannot be used together.
621    parser.addExclusiveArgumentSet(ignoreAttributeSyntax,
622         ignoreSyntaxViolationsForAttribute);
623  }
624
625
626
627  /**
628   * Performs the actual processing for this tool.  In this case, it gets a
629   * connection to the directory server and uses it to retrieve the server
630   * schema.  It then reads the LDIF file and validates each entry accordingly.
631   *
632   * @return  The result code for the processing that was performed.
633   */
634  @Override()
635  public ResultCode doToolProcessing()
636  {
637    // Get the connection to the directory server and use it to read the schema.
638    final Schema schema;
639    if (schemaDirectory.isPresent())
640    {
641      final File schemaDir = schemaDirectory.getValue();
642
643      try
644      {
645        final TreeMap<String,File> fileMap = new TreeMap<>();
646        for (final File f : schemaDir.listFiles())
647        {
648          final String name = f.getName();
649          if (f.isFile() && name.endsWith(".ldif"))
650          {
651            fileMap.put(name, f);
652          }
653        }
654
655        if (fileMap.isEmpty())
656        {
657          err("No LDIF files found in directory " +
658              schemaDir.getAbsolutePath());
659          return ResultCode.PARAM_ERROR;
660        }
661
662        final ArrayList<File> fileList = new ArrayList<>(fileMap.values());
663        schema = Schema.getSchema(fileList);
664      }
665      catch (final Exception e)
666      {
667        Debug.debugException(e);
668        err("Unable to read schema from files in directory " +
669            schemaDir.getAbsolutePath() + ":  " +
670             StaticUtils.getExceptionMessage(e));
671        return ResultCode.LOCAL_ERROR;
672      }
673    }
674    else
675    {
676      try
677      {
678        final LDAPConnection connection = getConnection();
679        schema = connection.getSchema();
680        connection.close();
681      }
682      catch (final LDAPException le)
683      {
684        Debug.debugException(le);
685        err("Unable to connect to the directory server and read the schema:  ",
686            le.getMessage());
687        return le.getResultCode();
688      }
689    }
690
691
692    // Get the encryption passphrase, if it was provided.
693    String encryptionPassphrase = null;
694    if (encryptionPassphraseFile.isPresent())
695    {
696      try
697      {
698        encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile(
699             encryptionPassphraseFile.getValue());
700      }
701      catch (final LDAPException e)
702      {
703        Debug.debugException(e);
704        err(e.getMessage());
705        return e.getResultCode();
706      }
707    }
708
709
710    // Create the entry validator and initialize its configuration.
711    entryValidator = new EntryValidator(schema);
712    entryValidator.setCheckAttributeSyntax(!ignoreAttributeSyntax.isPresent());
713    entryValidator.setCheckMalformedDNs(!ignoreMalformedDNs.isPresent());
714    entryValidator.setCheckEntryMissingRDNValues(
715         !ignoreMissingRDNValues.isPresent());
716    entryValidator.setCheckMissingAttributes(
717         !ignoreMissingAttributes.isPresent());
718    entryValidator.setCheckNameForms(!ignoreNameForms.isPresent());
719    entryValidator.setCheckProhibitedAttributes(
720         !ignoreProhibitedAttributes.isPresent());
721    entryValidator.setCheckProhibitedObjectClasses(
722         !ignoreProhibitedObjectClasses.isPresent());
723    entryValidator.setCheckMissingSuperiorObjectClasses(
724         !ignoreMissingSuperiorObjectClasses.isPresent());
725    entryValidator.setCheckSingleValuedAttributes(
726         !ignoreSingleValuedAttributes.isPresent());
727    entryValidator.setCheckStructuralObjectClasses(
728         !ignoreStructuralObjectClasses.isPresent());
729    entryValidator.setCheckUndefinedAttributes(
730         !ignoreUndefinedAttributes.isPresent());
731    entryValidator.setCheckUndefinedObjectClasses(
732         !ignoreUndefinedObjectClasses.isPresent());
733
734    if (ignoreSyntaxViolationsForAttribute.isPresent())
735    {
736      entryValidator.setIgnoreSyntaxViolationAttributeTypes(
737           ignoreSyntaxViolationsForAttribute.getValues());
738    }
739
740
741    // Create an LDIF reader that can be used to read through the LDIF file.
742    final LDIFReader ldifReader;
743    rejectWriter = null;
744    try
745    {
746      InputStream inputStream = new FileInputStream(ldifFile.getValue());
747
748      inputStream = ToolUtils.getPossiblyPassphraseEncryptedInputStream(
749           inputStream, encryptionPassphrase, false,
750           "LDIF file '" + ldifFile.getValue().getPath() +
751                "' is encrypted.  Please enter the encryption passphrase:",
752             "ERROR:  The provided passphrase was incorrect.",
753             getOut(), getErr()).getFirst();
754
755      if (isCompressed.isPresent())
756      {
757        inputStream = new GZIPInputStream(inputStream);
758      }
759      else
760      {
761        inputStream =
762             ToolUtils.getPossiblyGZIPCompressedInputStream(inputStream);
763      }
764
765      ldifReader = new LDIFReader(inputStream, numThreads.getValue(), this);
766    }
767    catch (final Exception e)
768    {
769      Debug.debugException(e);
770      err("Unable to open the LDIF reader:  ",
771           StaticUtils.getExceptionMessage(e));
772      return ResultCode.LOCAL_ERROR;
773    }
774
775    ldifReader.setSchema(schema);
776    if (ignoreDuplicateValues.isPresent())
777    {
778      ldifReader.setDuplicateValueBehavior(DuplicateValueBehavior.STRIP);
779    }
780    else
781    {
782      ldifReader.setDuplicateValueBehavior(DuplicateValueBehavior.REJECT);
783    }
784
785    try
786    {
787      // Create an LDIF writer that can be used to write information about
788      // rejected entries.
789      try
790      {
791        if (rejectFile.isPresent())
792        {
793          rejectWriter = new LDIFWriter(rejectFile.getValue());
794        }
795      }
796      catch (final Exception e)
797      {
798        Debug.debugException(e);
799        err("Unable to create the reject writer:  ",
800             StaticUtils.getExceptionMessage(e));
801        return ResultCode.LOCAL_ERROR;
802      }
803
804      ResultCode resultCode = ResultCode.SUCCESS;
805      while (true)
806      {
807        try
808        {
809          final Entry e = ldifReader.readEntry();
810          if (e == null)
811          {
812            // Because we're performing parallel processing and returning null
813            // from the translate method, LDIFReader.readEntry() should never
814            // return a non-null value.  However, it can throw an LDIFException
815            // if it encounters an invalid entry, or an IOException if there's
816            // a problem reading from the file, so we should still iterate
817            // through all of the entries to catch and report on those problems.
818            break;
819          }
820        }
821        catch (final LDIFException le)
822        {
823          Debug.debugException(le);
824          malformedEntries.incrementAndGet();
825
826          if (resultCode == ResultCode.SUCCESS)
827          {
828            resultCode = ResultCode.DECODING_ERROR;
829          }
830
831          if (rejectWriter != null)
832          {
833            try
834            {
835              rejectWriter.writeComment(
836                   "Unable to parse an entry read from LDIF:", false, false);
837              if (le.mayContinueReading())
838              {
839                rejectWriter.writeComment(
840                     StaticUtils.getExceptionMessage(le), false, true);
841              }
842              else
843              {
844                rejectWriter.writeComment(
845                     StaticUtils.getExceptionMessage(le), false,
846                     false);
847                rejectWriter.writeComment("Unable to continue LDIF processing.",
848                     false, true);
849                err("Aborting LDIF processing:  ",
850                     StaticUtils.getExceptionMessage(le));
851                return ResultCode.LOCAL_ERROR;
852              }
853            }
854            catch (final IOException ioe)
855            {
856              Debug.debugException(ioe);
857              err("Unable to write to the reject file:",
858                  StaticUtils.getExceptionMessage(ioe));
859              err("LDIF parse failure that triggered the rejection:  ",
860                  StaticUtils.getExceptionMessage(le));
861              return ResultCode.LOCAL_ERROR;
862            }
863          }
864        }
865        catch (final IOException ioe)
866        {
867          Debug.debugException(ioe);
868
869          if (rejectWriter != null)
870          {
871            try
872            {
873              rejectWriter.writeComment("I/O error reading from LDIF:", false,
874                   false);
875              rejectWriter.writeComment(StaticUtils.getExceptionMessage(ioe),
876                   false, true);
877              return ResultCode.LOCAL_ERROR;
878            }
879            catch (final Exception ex)
880            {
881              Debug.debugException(ex);
882              err("I/O error reading from LDIF:",
883                   StaticUtils.getExceptionMessage(ioe));
884              return ResultCode.LOCAL_ERROR;
885            }
886          }
887        }
888      }
889
890      if (malformedEntries.get() > 0)
891      {
892        out(malformedEntries.get() + " entries were malformed and could not " +
893            "be read from the LDIF file.");
894      }
895
896      if (entryValidator.getInvalidEntries() > 0)
897      {
898        if (resultCode == ResultCode.SUCCESS)
899        {
900          resultCode = ResultCode.OBJECT_CLASS_VIOLATION;
901        }
902
903        for (final String s : entryValidator.getInvalidEntrySummary(true))
904        {
905          out(s);
906        }
907      }
908      else
909      {
910        if (malformedEntries.get() == 0)
911        {
912          out("No errors were encountered.");
913        }
914      }
915
916      return resultCode;
917    }
918    finally
919    {
920      try
921      {
922        ldifReader.close();
923      }
924      catch (final Exception e)
925      {
926        Debug.debugException(e);
927      }
928
929      try
930      {
931        if (rejectWriter != null)
932        {
933          rejectWriter.close();
934        }
935      }
936      catch (final Exception e)
937      {
938        Debug.debugException(e);
939      }
940    }
941  }
942
943
944
945  /**
946   * Examines the provided entry to determine whether it conforms to the
947   * server schema.
948   *
949   * @param  entry           The entry to be examined.
950   * @param  firstLineNumber The line number of the LDIF source on which the
951   *                         provided entry begins.
952   *
953   * @return  The updated entry.  This method will always return {@code null}
954   *          because all of the real processing needed for the entry is
955   *          performed in this method and the entry isn't needed any more
956   *          after this method is done.
957   */
958  @Override()
959  public Entry translate(final Entry entry, final long firstLineNumber)
960  {
961    final ArrayList<String> invalidReasons = new ArrayList<>(5);
962    if (! entryValidator.entryIsValid(entry, invalidReasons))
963    {
964      if (rejectWriter != null)
965      {
966        synchronized (this)
967        {
968          try
969          {
970            rejectWriter.writeEntry(entry, listToString(invalidReasons));
971          }
972          catch (final IOException ioe)
973          {
974            Debug.debugException(ioe);
975          }
976        }
977      }
978    }
979
980    final long numEntries = entriesProcessed.incrementAndGet();
981    if ((numEntries % 1000L) == 0L)
982    {
983      out("Processed ", numEntries, " entries.");
984    }
985
986    return null;
987  }
988
989
990
991  /**
992   * Converts the provided list of strings into a single string.  It will
993   * contain line breaks after all but the last element.
994   *
995   * @param  l  The list of strings to convert to a single string.
996   *
997   * @return  The string from the provided list, or {@code null} if the provided
998   *          list is empty or {@code null}.
999   */
1000  private static String listToString(final List<String> l)
1001  {
1002    if ((l == null) || (l.isEmpty()))
1003    {
1004      return null;
1005    }
1006
1007    final StringBuilder buffer = new StringBuilder();
1008    final Iterator<String> iterator = l.iterator();
1009    while (iterator.hasNext())
1010    {
1011      buffer.append(iterator.next());
1012      if (iterator.hasNext())
1013      {
1014        buffer.append(EOL);
1015      }
1016    }
1017
1018    return buffer.toString();
1019  }
1020
1021
1022
1023  /**
1024   * {@inheritDoc}
1025   */
1026  @Override()
1027  public LinkedHashMap<String[],String> getExampleUsages()
1028  {
1029    final LinkedHashMap<String[],String> examples =
1030         new LinkedHashMap<>(StaticUtils.computeMapCapacity(2));
1031
1032    String[] args =
1033    {
1034      "--hostname", "server.example.com",
1035      "--port", "389",
1036      "--ldifFile", "data.ldif",
1037      "--rejectFile", "rejects.ldif",
1038      "--numThreads", "4"
1039    };
1040    String description =
1041         "Validate the contents of the 'data.ldif' file using the schema " +
1042         "defined in the specified directory server using four concurrent " +
1043         "threads.  All types of validation will be performed, and " +
1044         "information about any errors will be written to the 'rejects.ldif' " +
1045         "file.";
1046    examples.put(args, description);
1047
1048
1049    args = new String[]
1050    {
1051      "--schemaDirectory", "/ds/config/schema",
1052      "--ldifFile", "data.ldif",
1053      "--rejectFile", "rejects.ldif",
1054      "--ignoreStructuralObjectClasses",
1055      "--ignoreAttributeSyntax"
1056    };
1057    description =
1058         "Validate the contents of the 'data.ldif' file using the schema " +
1059         "defined in LDIF files contained in the /ds/config/schema directory " +
1060         "using a single thread.  Any errors resulting from entries that do " +
1061         "not have exactly one structural object class or from values which " +
1062         "violate the syntax for their associated attribute types will be " +
1063         "ignored.  Information about any other failures will be written to " +
1064         "the 'rejects.ldif' file.";
1065    examples.put(args, description);
1066
1067    return examples;
1068  }
1069
1070
1071
1072  /**
1073   * @return EntryValidator
1074   *
1075   * Returns the EntryValidator
1076   */
1077  public EntryValidator getEntryValidator()
1078  {
1079    return entryValidator;
1080  }
1081}