001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.validation.tests;
003
004import static org.openstreetmap.josm.tools.I18n.marktr;
005import static org.openstreetmap.josm.tools.I18n.tr;
006
007import java.awt.GridBagConstraints;
008import java.awt.event.ActionListener;
009import java.io.BufferedReader;
010import java.io.IOException;
011import java.util.ArrayList;
012import java.util.Arrays;
013import java.util.Collection;
014import java.util.HashMap;
015import java.util.HashSet;
016import java.util.List;
017import java.util.Locale;
018import java.util.Map;
019import java.util.Map.Entry;
020import java.util.Set;
021import java.util.regex.Matcher;
022import java.util.regex.Pattern;
023import java.util.regex.PatternSyntaxException;
024
025import javax.swing.JCheckBox;
026import javax.swing.JLabel;
027import javax.swing.JPanel;
028
029import org.openstreetmap.josm.Main;
030import org.openstreetmap.josm.command.ChangePropertyCommand;
031import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
032import org.openstreetmap.josm.command.Command;
033import org.openstreetmap.josm.command.SequenceCommand;
034import org.openstreetmap.josm.data.osm.OsmPrimitive;
035import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
036import org.openstreetmap.josm.data.osm.OsmUtils;
037import org.openstreetmap.josm.data.osm.Tag;
038import org.openstreetmap.josm.data.validation.Severity;
039import org.openstreetmap.josm.data.validation.Test.TagTest;
040import org.openstreetmap.josm.data.validation.TestError;
041import org.openstreetmap.josm.data.validation.util.Entities;
042import org.openstreetmap.josm.gui.preferences.validator.ValidatorPreference;
043import org.openstreetmap.josm.gui.progress.ProgressMonitor;
044import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset;
045import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem;
046import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets;
047import org.openstreetmap.josm.gui.tagging.presets.items.Check;
048import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup;
049import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem;
050import org.openstreetmap.josm.gui.widgets.EditableList;
051import org.openstreetmap.josm.io.CachedFile;
052import org.openstreetmap.josm.tools.GBC;
053import org.openstreetmap.josm.tools.MultiMap;
054import org.openstreetmap.josm.tools.Utils;
055
056/**
057 * Check for misspelled or wrong tags
058 *
059 * @author frsantos
060 * @since 3669
061 */
062public class TagChecker extends TagTest {
063
064    /** The config file of ignored tags */
065    public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg";
066    /** The config file of dictionary words */
067    public static final String SPELL_FILE = "resource://data/validator/words.cfg";
068
069    /** Normalized keys: the key should be substituted by the value if the key was not found in presets */
070    private static final Map<String, String> harmonizedKeys = new HashMap<>();
071    /** The spell check preset values */
072    private static volatile MultiMap<String, String> presetsValueData;
073    /** The TagChecker data */
074    private static final List<CheckerData> checkerData = new ArrayList<>();
075    private static final List<String> ignoreDataStartsWith = new ArrayList<>();
076    private static final List<String> ignoreDataEquals = new ArrayList<>();
077    private static final List<String> ignoreDataEndsWith = new ArrayList<>();
078    private static final List<Tag> ignoreDataTag = new ArrayList<>();
079
080    /** The preferences prefix */
081    protected static final String PREFIX = ValidatorPreference.PREFIX + "." + TagChecker.class.getSimpleName();
082
083    public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues";
084    public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys";
085    public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex";
086    public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes";
087
088    public static final String PREF_SOURCES = PREFIX + ".source";
089
090    public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + "BeforeUpload";
091    public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + "BeforeUpload";
092    public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + "BeforeUpload";
093    public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + "BeforeUpload";
094
095    protected boolean checkKeys;
096    protected boolean checkValues;
097    protected boolean checkComplex;
098    protected boolean checkFixmes;
099
100    protected JCheckBox prefCheckKeys;
101    protected JCheckBox prefCheckValues;
102    protected JCheckBox prefCheckComplex;
103    protected JCheckBox prefCheckFixmes;
104    protected JCheckBox prefCheckPaint;
105
106    protected JCheckBox prefCheckKeysBeforeUpload;
107    protected JCheckBox prefCheckValuesBeforeUpload;
108    protected JCheckBox prefCheckComplexBeforeUpload;
109    protected JCheckBox prefCheckFixmesBeforeUpload;
110    protected JCheckBox prefCheckPaintBeforeUpload;
111
112    // CHECKSTYLE.OFF: SingleSpaceSeparator
113    protected static final int EMPTY_VALUES      = 1200;
114    protected static final int INVALID_KEY       = 1201;
115    protected static final int INVALID_VALUE     = 1202;
116    protected static final int FIXME             = 1203;
117    protected static final int INVALID_SPACE     = 1204;
118    protected static final int INVALID_KEY_SPACE = 1205;
119    protected static final int INVALID_HTML      = 1206; /* 1207 was PAINT */
120    protected static final int LONG_VALUE        = 1208;
121    protected static final int LONG_KEY          = 1209;
122    protected static final int LOW_CHAR_VALUE    = 1210;
123    protected static final int LOW_CHAR_KEY      = 1211;
124    protected static final int MISSPELLED_VALUE  = 1212;
125    protected static final int MISSPELLED_KEY    = 1213;
126    protected static final int MULTIPLE_SPACES   = 1214;
127    // CHECKSTYLE.ON: SingleSpaceSeparator
128    // 1250 and up is used by tagcheck
129
130    protected EditableList sourcesList;
131
132    private static final Set<String> DEFAULT_SOURCES = new HashSet<>(Arrays.asList(/*DATA_FILE, */IGNORE_FILE, SPELL_FILE));
133
134    /**
135     * Constructor
136     */
137    public TagChecker() {
138        super(tr("Tag checker"), tr("This test checks for errors in tag keys and values."));
139    }
140
141    @Override
142    public void initialize() throws IOException {
143        initializeData();
144        initializePresets();
145    }
146
147    /**
148     * Reads the spellcheck file into a HashMap.
149     * The data file is a list of words, beginning with +/-. If it starts with +,
150     * the word is valid, but if it starts with -, the word should be replaced
151     * by the nearest + word before this.
152     *
153     * @throws IOException if any I/O error occurs
154     */
155    private static void initializeData() throws IOException {
156        checkerData.clear();
157        ignoreDataStartsWith.clear();
158        ignoreDataEquals.clear();
159        ignoreDataEndsWith.clear();
160        ignoreDataTag.clear();
161        harmonizedKeys.clear();
162
163        StringBuilder errorSources = new StringBuilder();
164        for (String source : Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES)) {
165            try (
166                CachedFile cf = new CachedFile(source);
167                BufferedReader reader = cf.getContentReader()
168            ) {
169                String okValue = null;
170                boolean tagcheckerfile = false;
171                boolean ignorefile = false;
172                boolean isFirstLine = true;
173                String line;
174                while ((line = reader.readLine()) != null && (tagcheckerfile || !line.isEmpty())) {
175                    if (line.startsWith("#")) {
176                        if (line.startsWith("# JOSM TagChecker")) {
177                            tagcheckerfile = true;
178                            if (!DEFAULT_SOURCES.contains(source)) {
179                                Main.info(tr("Adding {0} to tag checker", source));
180                            }
181                        } else
182                        if (line.startsWith("# JOSM IgnoreTags")) {
183                            ignorefile = true;
184                            if (!DEFAULT_SOURCES.contains(source)) {
185                                Main.info(tr("Adding {0} to ignore tags", source));
186                            }
187                        }
188                    } else if (ignorefile) {
189                        line = line.trim();
190                        if (line.length() < 4) {
191                            continue;
192                        }
193
194                        String key = line.substring(0, 2);
195                        line = line.substring(2);
196
197                        switch (key) {
198                        case "S:":
199                            ignoreDataStartsWith.add(line);
200                            break;
201                        case "E:":
202                            ignoreDataEquals.add(line);
203                            break;
204                        case "F:":
205                            ignoreDataEndsWith.add(line);
206                            break;
207                        case "K:":
208                            ignoreDataTag.add(Tag.ofString(line));
209                            break;
210                        default:
211                            if (!key.startsWith(";")) {
212                                Main.warn("Unsupported TagChecker key: " + key);
213                            }
214                        }
215                    } else if (tagcheckerfile) {
216                        if (!line.isEmpty()) {
217                            CheckerData d = new CheckerData();
218                            String err = d.getData(line);
219
220                            if (err == null) {
221                                checkerData.add(d);
222                            } else {
223                                Main.error(tr("Invalid tagchecker line - {0}: {1}", err, line));
224                            }
225                        }
226                    } else if (line.charAt(0) == '+') {
227                        okValue = line.substring(1);
228                    } else if (line.charAt(0) == '-' && okValue != null) {
229                        harmonizedKeys.put(harmonizeKey(line.substring(1)), okValue);
230                    } else {
231                        Main.error(tr("Invalid spellcheck line: {0}", line));
232                    }
233                    if (isFirstLine) {
234                        isFirstLine = false;
235                        if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) {
236                            Main.info(tr("Adding {0} to spellchecker", source));
237                        }
238                    }
239                }
240            } catch (IOException e) {
241                Main.error(e);
242                errorSources.append(source).append('\n');
243            }
244        }
245
246        if (errorSources.length() > 0)
247            throw new IOException(tr("Could not access data file(s):\n{0}", errorSources));
248    }
249
250    /**
251     * Reads the presets data.
252     *
253     */
254    public static void initializePresets() {
255
256        if (!Main.pref.getBoolean(PREF_CHECK_VALUES, true))
257            return;
258
259        Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets();
260        if (!presets.isEmpty()) {
261            presetsValueData = new MultiMap<>();
262            for (String a : OsmPrimitive.getUninterestingKeys()) {
263                presetsValueData.putVoid(a);
264            }
265            // TODO directionKeys are no longer in OsmPrimitive (search pattern is used instead)
266            for (String a : Main.pref.getCollection(ValidatorPreference.PREFIX + ".knownkeys",
267                    Arrays.asList(new String[]{"is_in", "int_ref", "fixme", "population"}))) {
268                presetsValueData.putVoid(a);
269            }
270            for (TaggingPreset p : presets) {
271                for (TaggingPresetItem i : p.data) {
272                    if (i instanceof KeyedItem) {
273                        addPresetValue(p, (KeyedItem) i);
274                    } else if (i instanceof CheckGroup) {
275                        for (Check c : ((CheckGroup) i).checks) {
276                            addPresetValue(p, c);
277                        }
278                    }
279                }
280            }
281        }
282    }
283
284    private static void addPresetValue(TaggingPreset p, KeyedItem ky) {
285        Collection<String> values = ky.getValues();
286        if (ky.key != null && values != null) {
287            try {
288                presetsValueData.putAll(ky.key, values);
289                harmonizedKeys.put(harmonizeKey(ky.key), ky.key);
290            } catch (NullPointerException e) {
291                Main.error(e, p+": Unable to initialize "+ky+'.');
292            }
293        }
294    }
295
296    /**
297     * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters)
298     * @param s string to check
299     * @return {@code true} if {@code s} contains characters with code below 0x20
300     */
301    private static boolean containsLow(String s) {
302        if (s == null)
303            return false;
304        for (int i = 0; i < s.length(); i++) {
305            if (s.charAt(i) < 0x20)
306                return true;
307        }
308        return false;
309    }
310
311    /**
312     * Determines if the given key is in internal presets.
313     * @param key key
314     * @return {@code true} if the given key is in internal presets
315     * @since 9023
316     */
317    public static boolean isKeyInPresets(String key) {
318        return presetsValueData.get(key) != null;
319    }
320
321    /**
322     * Determines if the given tag is in internal presets.
323     * @param key key
324     * @param value value
325     * @return {@code true} if the given tag is in internal presets
326     * @since 9023
327     */
328    public static boolean isTagInPresets(String key, String value) {
329        final Set<String> values = presetsValueData.get(key);
330        return values != null && (values.isEmpty() || values.contains(value));
331    }
332
333    /**
334     * Returns the list of ignored tags.
335     * @return the list of ignored tags
336     * @since 9023
337     */
338    public static List<Tag> getIgnoredTags() {
339        return new ArrayList<>(ignoreDataTag);
340    }
341
342    /**
343     * Determines if the given tag is ignored for checks "key/tag not in presets".
344     * @param key key
345     * @param value value
346     * @return {@code true} if the given tag is ignored
347     * @since 9023
348     */
349    public static boolean isTagIgnored(String key, String value) {
350        boolean tagInPresets = isTagInPresets(key, value);
351        boolean ignore = false;
352
353        for (String a : ignoreDataStartsWith) {
354            if (key.startsWith(a)) {
355                ignore = true;
356            }
357        }
358        for (String a : ignoreDataEquals) {
359            if (key.equals(a)) {
360                ignore = true;
361            }
362        }
363        for (String a : ignoreDataEndsWith) {
364            if (key.endsWith(a)) {
365                ignore = true;
366            }
367        }
368
369        if (!tagInPresets) {
370            for (Tag a : ignoreDataTag) {
371                if (key.equals(a.getKey()) && value.equals(a.getValue())) {
372                    ignore = true;
373                }
374            }
375        }
376        return ignore;
377    }
378
379    /**
380     * Checks the primitive tags
381     * @param p The primitive to check
382     */
383    @Override
384    public void check(OsmPrimitive p) {
385        // Just a collection to know if a primitive has been already marked with error
386        MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>();
387
388        if (checkComplex) {
389            Map<String, String> keys = p.getKeys();
390            for (CheckerData d : checkerData) {
391                if (d.match(p, keys)) {
392                    errors.add(TestError.builder(this, d.getSeverity(), d.getCode())
393                            .message(tr("Suspicious tag/value combinations"), d.getDescription())
394                            .primitives(p)
395                            .build());
396                    withErrors.put(p, "TC");
397                }
398            }
399        }
400
401        for (Entry<String, String> prop : p.getKeys().entrySet()) {
402            String s = marktr("Key ''{0}'' invalid.");
403            String key = prop.getKey();
404            String value = prop.getValue();
405            if (checkValues && (containsLow(value)) && !withErrors.contains(p, "ICV")) {
406                errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_VALUE)
407                        .message(tr("Tag value contains character with code less than 0x20"), s, key)
408                        .primitives(p)
409                        .build());
410                withErrors.put(p, "ICV");
411            }
412            if (checkKeys && (containsLow(key)) && !withErrors.contains(p, "ICK")) {
413                errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_KEY)
414                        .message(tr("Tag key contains character with code less than 0x20"), s, key)
415                        .primitives(p)
416                        .build());
417                withErrors.put(p, "ICK");
418            }
419            if (checkValues && (value != null && value.length() > 255) && !withErrors.contains(p, "LV")) {
420                errors.add(TestError.builder(this, Severity.ERROR, LONG_VALUE)
421                        .message(tr("Tag value longer than allowed"), s, key)
422                        .primitives(p)
423                        .build());
424                withErrors.put(p, "LV");
425            }
426            if (checkKeys && (key != null && key.length() > 255) && !withErrors.contains(p, "LK")) {
427                errors.add(TestError.builder(this, Severity.ERROR, LONG_KEY)
428                        .message(tr("Tag key longer than allowed"), s, key)
429                        .primitives(p)
430                        .build());
431                withErrors.put(p, "LK");
432            }
433            if (checkValues && (value == null || value.trim().isEmpty()) && !withErrors.contains(p, "EV")) {
434                errors.add(TestError.builder(this, Severity.WARNING, EMPTY_VALUES)
435                        .message(tr("Tags with empty values"), s, key)
436                        .primitives(p)
437                        .build());
438                withErrors.put(p, "EV");
439            }
440            if (checkKeys && key != null && key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) {
441                errors.add(TestError.builder(this, Severity.WARNING, INVALID_KEY_SPACE)
442                        .message(tr("Invalid white space in property key"), s, key)
443                        .primitives(p)
444                        .build());
445                withErrors.put(p, "IPK");
446            }
447            if (checkValues && value != null && (value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, "SPACE")) {
448                errors.add(TestError.builder(this, Severity.WARNING, INVALID_SPACE)
449                        .message(tr("Property values start or end with white space"), s, key)
450                        .primitives(p)
451                        .build());
452                withErrors.put(p, "SPACE");
453            }
454            if (checkValues && value != null && value.contains("  ") && !withErrors.contains(p, "SPACE")) {
455                errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_SPACES)
456                        .message(tr("Property values contain multiple white spaces"), s, key)
457                        .primitives(p)
458                        .build());
459                withErrors.put(p, "SPACE");
460            }
461            if (checkValues && value != null && !value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) {
462                errors.add(TestError.builder(this, Severity.OTHER, INVALID_HTML)
463                        .message(tr("Property values contain HTML entity"), s, key)
464                        .primitives(p)
465                        .build());
466                withErrors.put(p, "HTML");
467            }
468            if (checkValues && key != null && value != null && !value.isEmpty() && presetsValueData != null && !isTagIgnored(key, value)) {
469                if (!isKeyInPresets(key)) {
470                    String prettifiedKey = harmonizeKey(key);
471                    String fixedKey = harmonizedKeys.get(prettifiedKey);
472                    if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) {
473                        // misspelled preset key
474                        final TestError.Builder error = TestError.builder(this, Severity.WARNING, MISSPELLED_KEY)
475                                .message(tr("Misspelled property key"), marktr("Key ''{0}'' looks like ''{1}''."), key, fixedKey)
476                                .primitives(p);
477                        if (p.hasKey(fixedKey)) {
478                            errors.add(error.build());
479                        } else {
480                            errors.add(error.fix(() -> new ChangePropertyKeyCommand(p, key, fixedKey)).build());
481                        }
482                        withErrors.put(p, "WPK");
483                    } else {
484                        errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
485                                .message(tr("Presets do not contain property key"), marktr("Key ''{0}'' not in presets."), key)
486                                .primitives(p)
487                                .build());
488                        withErrors.put(p, "UPK");
489                    }
490                } else if (!isTagInPresets(key, value)) {
491                    // try to fix common typos and check again if value is still unknown
492                    String fixedValue = harmonizeValue(prop.getValue());
493                    Map<String, String> possibleValues = getPossibleValues(presetsValueData.get(key));
494                    if (possibleValues.containsKey(fixedValue)) {
495                        final String newKey = possibleValues.get(fixedValue);
496                        // misspelled preset value
497                        errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE)
498                                .message(tr("Misspelled property value"),
499                                        marktr("Value ''{0}'' for key ''{1}'' looks like ''{2}''."), prop.getValue(), key, fixedValue)
500                                .primitives(p)
501                                .fix(() -> new ChangePropertyCommand(p, key, newKey))
502                                .build());
503                        withErrors.put(p, "WPV");
504                    } else {
505                        // unknown preset value
506                        errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
507                                .message(tr("Presets do not contain property value"),
508                                        marktr("Value ''{0}'' for key ''{1}'' not in presets."), prop.getValue(), key)
509                                .primitives(p)
510                                .build());
511                        withErrors.put(p, "UPV");
512                    }
513                }
514            }
515            if (checkFixmes && key != null && value != null && !value.isEmpty() && isFixme(key, value) && !withErrors.contains(p, "FIXME")) {
516               errors.add(TestError.builder(this, Severity.OTHER, FIXME)
517                .message(tr("FIXMES"))
518                .primitives(p)
519                .build());
520               withErrors.put(p, "FIXME");
521            }
522        }
523    }
524
525    private static boolean isFixme(String key, String value) {
526        return key.toLowerCase(Locale.ENGLISH).contains("fixme") || key.contains("todo")
527          || value.toLowerCase(Locale.ENGLISH).contains("fixme") || value.contains("check and delete");
528    }
529
530    private static Map<String, String> getPossibleValues(Set<String> values) {
531        // generate a map with common typos
532        Map<String, String> map = new HashMap<>();
533        if (values != null) {
534            for (String value : values) {
535                map.put(value, value);
536                if (value.contains("_")) {
537                    map.put(value.replace("_", ""), value);
538                }
539            }
540        }
541        return map;
542    }
543
544    private static String harmonizeKey(String key) {
545        key = key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_');
546        return Utils.strip(key, "-_;:,");
547    }
548
549    private static String harmonizeValue(String value) {
550        value = value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_');
551        return Utils.strip(value, "-_;:,");
552    }
553
554    @Override
555    public void startTest(ProgressMonitor monitor) {
556        super.startTest(monitor);
557        checkKeys = Main.pref.getBoolean(PREF_CHECK_KEYS, true);
558        if (isBeforeUpload) {
559            checkKeys = checkKeys && Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true);
560        }
561
562        checkValues = Main.pref.getBoolean(PREF_CHECK_VALUES, true);
563        if (isBeforeUpload) {
564            checkValues = checkValues && Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true);
565        }
566
567        checkComplex = Main.pref.getBoolean(PREF_CHECK_COMPLEX, true);
568        if (isBeforeUpload) {
569            checkComplex = checkComplex && Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true);
570        }
571
572        checkFixmes = Main.pref.getBoolean(PREF_CHECK_FIXMES, true);
573        if (isBeforeUpload) {
574            checkFixmes = checkFixmes && Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true);
575        }
576    }
577
578    @Override
579    public void visit(Collection<OsmPrimitive> selection) {
580        if (checkKeys || checkValues || checkComplex || checkFixmes) {
581            super.visit(selection);
582        }
583    }
584
585    @Override
586    public void addGui(JPanel testPanel) {
587        GBC a = GBC.eol();
588        a.anchor = GridBagConstraints.EAST;
589
590        testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0));
591
592        prefCheckKeys = new JCheckBox(tr("Check property keys."), Main.pref.getBoolean(PREF_CHECK_KEYS, true));
593        prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words."));
594        testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0));
595
596        prefCheckKeysBeforeUpload = new JCheckBox();
597        prefCheckKeysBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true));
598        testPanel.add(prefCheckKeysBeforeUpload, a);
599
600        prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Main.pref.getBoolean(PREF_CHECK_COMPLEX, true));
601        prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules."));
602        testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0));
603
604        prefCheckComplexBeforeUpload = new JCheckBox();
605        prefCheckComplexBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true));
606        testPanel.add(prefCheckComplexBeforeUpload, a);
607
608        final Collection<String> sources = Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES);
609        sourcesList = new EditableList(tr("TagChecker source"));
610        sourcesList.setItems(sources);
611        testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0));
612        testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0));
613
614        ActionListener disableCheckActionListener = e -> handlePrefEnable();
615        prefCheckKeys.addActionListener(disableCheckActionListener);
616        prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener);
617        prefCheckComplex.addActionListener(disableCheckActionListener);
618        prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener);
619
620        handlePrefEnable();
621
622        prefCheckValues = new JCheckBox(tr("Check property values."), Main.pref.getBoolean(PREF_CHECK_VALUES, true));
623        prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets."));
624        testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0));
625
626        prefCheckValuesBeforeUpload = new JCheckBox();
627        prefCheckValuesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true));
628        testPanel.add(prefCheckValuesBeforeUpload, a);
629
630        prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Main.pref.getBoolean(PREF_CHECK_FIXMES, true));
631        prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value."));
632        testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0));
633
634        prefCheckFixmesBeforeUpload = new JCheckBox();
635        prefCheckFixmesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true));
636        testPanel.add(prefCheckFixmesBeforeUpload, a);
637    }
638
639    public void handlePrefEnable() {
640        boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected()
641                || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected();
642        sourcesList.setEnabled(selected);
643    }
644
645    @Override
646    public boolean ok() {
647        enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected();
648        testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected()
649                || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected();
650
651        Main.pref.put(PREF_CHECK_VALUES, prefCheckValues.isSelected());
652        Main.pref.put(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected());
653        Main.pref.put(PREF_CHECK_KEYS, prefCheckKeys.isSelected());
654        Main.pref.put(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected());
655        Main.pref.put(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected());
656        Main.pref.put(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected());
657        Main.pref.put(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected());
658        Main.pref.put(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected());
659        return Main.pref.putCollection(PREF_SOURCES, sourcesList.getItems());
660    }
661
662    @Override
663    public Command fixError(TestError testError) {
664        List<Command> commands = new ArrayList<>(50);
665
666        Collection<? extends OsmPrimitive> primitives = testError.getPrimitives();
667        for (OsmPrimitive p : primitives) {
668            Map<String, String> tags = p.getKeys();
669            if (tags.isEmpty()) {
670                continue;
671            }
672
673            for (Entry<String, String> prop: tags.entrySet()) {
674                String key = prop.getKey();
675                String value = prop.getValue();
676                if (value == null || value.trim().isEmpty()) {
677                    commands.add(new ChangePropertyCommand(p, key, null));
678                } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains("  ")) {
679                    commands.add(new ChangePropertyCommand(p, key, Tag.removeWhiteSpaces(value)));
680                } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains("  ")) {
681                    commands.add(new ChangePropertyKeyCommand(p, key, Tag.removeWhiteSpaces(key)));
682                } else {
683                    String evalue = Entities.unescape(value);
684                    if (!evalue.equals(value)) {
685                        commands.add(new ChangePropertyCommand(p, key, evalue));
686                    }
687                }
688            }
689        }
690
691        if (commands.isEmpty())
692            return null;
693        if (commands.size() == 1)
694            return commands.get(0);
695
696        return new SequenceCommand(tr("Fix tags"), commands);
697    }
698
699    @Override
700    public boolean isFixable(TestError testError) {
701        if (testError.getTester() instanceof TagChecker) {
702            int code = testError.getCode();
703            return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE ||
704                   code == INVALID_KEY_SPACE || code == INVALID_HTML || code == MISSPELLED_VALUE ||
705                   code == MULTIPLE_SPACES;
706        }
707
708        return false;
709    }
710
711    protected static class CheckerData {
712        private String description;
713        protected List<CheckerElement> data = new ArrayList<>();
714        private OsmPrimitiveType type;
715        private int code;
716        protected Severity severity;
717        // CHECKSTYLE.OFF: SingleSpaceSeparator
718        protected static final int TAG_CHECK_ERROR = 1250;
719        protected static final int TAG_CHECK_WARN  = 1260;
720        protected static final int TAG_CHECK_INFO  = 1270;
721        // CHECKSTYLE.ON: SingleSpaceSeparator
722
723        protected static class CheckerElement {
724            public Object tag;
725            public Object value;
726            public boolean noMatch;
727            public boolean tagAll;
728            public boolean valueAll;
729            public boolean valueBool;
730
731            private static Pattern getPattern(String str) {
732                if (str.endsWith("/i"))
733                    return Pattern.compile(str.substring(1, str.length()-2), Pattern.CASE_INSENSITIVE);
734                if (str.endsWith("/"))
735                    return Pattern.compile(str.substring(1, str.length()-1));
736
737                throw new IllegalStateException();
738            }
739
740            public CheckerElement(String exp) {
741                Matcher m = Pattern.compile("(.+)([!=]=)(.+)").matcher(exp);
742                m.matches();
743
744                String n = m.group(1).trim();
745
746                if ("*".equals(n)) {
747                    tagAll = true;
748                } else {
749                    tag = n.startsWith("/") ? getPattern(n) : n;
750                    noMatch = "!=".equals(m.group(2));
751                    n = m.group(3).trim();
752                    if ("*".equals(n)) {
753                        valueAll = true;
754                    } else if ("BOOLEAN_TRUE".equals(n)) {
755                        valueBool = true;
756                        value = OsmUtils.trueval;
757                    } else if ("BOOLEAN_FALSE".equals(n)) {
758                        valueBool = true;
759                        value = OsmUtils.falseval;
760                    } else {
761                        value = n.startsWith("/") ? getPattern(n) : n;
762                    }
763                }
764            }
765
766            public boolean match(Map<String, String> keys) {
767                for (Entry<String, String> prop: keys.entrySet()) {
768                    String key = prop.getKey();
769                    String val = valueBool ? OsmUtils.getNamedOsmBoolean(prop.getValue()) : prop.getValue();
770                    if ((tagAll || (tag instanceof Pattern ? ((Pattern) tag).matcher(key).matches() : key.equals(tag)))
771                            && (valueAll || (value instanceof Pattern ? ((Pattern) value).matcher(val).matches() : val.equals(value))))
772                        return !noMatch;
773                }
774                return noMatch;
775            }
776        }
777
778        private static final Pattern CLEAN_STR_PATTERN = Pattern.compile(" *# *([^#]+) *$");
779        private static final Pattern SPLIT_TRIMMED_PATTERN = Pattern.compile(" *: *");
780        private static final Pattern SPLIT_ELEMENTS_PATTERN = Pattern.compile(" *&& *");
781
782        public String getData(final String str) {
783            Matcher m = CLEAN_STR_PATTERN.matcher(str);
784            String trimmed = m.replaceFirst("").trim();
785            try {
786                description = m.group(1);
787                if (description != null && description.isEmpty()) {
788                    description = null;
789                }
790            } catch (IllegalStateException e) {
791                Main.error(e);
792                description = null;
793            }
794            String[] n = SPLIT_TRIMMED_PATTERN.split(trimmed, 3);
795            switch (n[0]) {
796            case "way":
797                type = OsmPrimitiveType.WAY;
798                break;
799            case "node":
800                type = OsmPrimitiveType.NODE;
801                break;
802            case "relation":
803                type = OsmPrimitiveType.RELATION;
804                break;
805            case "*":
806                type = null;
807                break;
808            default:
809                return tr("Could not find element type");
810            }
811            if (n.length != 3)
812                return tr("Incorrect number of parameters");
813
814            switch (n[1]) {
815            case "W":
816                severity = Severity.WARNING;
817                code = TAG_CHECK_WARN;
818                break;
819            case "E":
820                severity = Severity.ERROR;
821                code = TAG_CHECK_ERROR;
822                break;
823            case "I":
824                severity = Severity.OTHER;
825                code = TAG_CHECK_INFO;
826                break;
827            default:
828                return tr("Could not find warning level");
829            }
830            for (String exp: SPLIT_ELEMENTS_PATTERN.split(n[2])) {
831                try {
832                    data.add(new CheckerElement(exp));
833                } catch (IllegalStateException e) {
834                    Main.trace(e);
835                    return tr("Illegal expression ''{0}''", exp);
836                } catch (PatternSyntaxException e) {
837                    Main.trace(e);
838                    return tr("Illegal regular expression ''{0}''", exp);
839                }
840            }
841            return null;
842        }
843
844        public boolean match(OsmPrimitive osm, Map<String, String> keys) {
845            if (type != null && OsmPrimitiveType.from(osm) != type)
846                return false;
847
848            for (CheckerElement ce : data) {
849                if (!ce.match(keys))
850                    return false;
851            }
852            return true;
853        }
854
855        public String getDescription() {
856            return description;
857        }
858
859        public Severity getSeverity() {
860            return severity;
861        }
862
863        public int getCode() {
864            if (type == null)
865                return code;
866
867            return code + type.ordinal() + 1;
868        }
869    }
870}