001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.validation.tests; 003 004import static org.openstreetmap.josm.tools.I18n.marktr; 005import static org.openstreetmap.josm.tools.I18n.tr; 006 007import java.awt.GridBagConstraints; 008import java.awt.event.ActionListener; 009import java.io.BufferedReader; 010import java.io.IOException; 011import java.util.ArrayList; 012import java.util.Arrays; 013import java.util.Collection; 014import java.util.HashMap; 015import java.util.HashSet; 016import java.util.List; 017import java.util.Locale; 018import java.util.Map; 019import java.util.Map.Entry; 020import java.util.Set; 021import java.util.regex.Matcher; 022import java.util.regex.Pattern; 023import java.util.regex.PatternSyntaxException; 024 025import javax.swing.JCheckBox; 026import javax.swing.JLabel; 027import javax.swing.JPanel; 028 029import org.openstreetmap.josm.Main; 030import org.openstreetmap.josm.command.ChangePropertyCommand; 031import org.openstreetmap.josm.command.ChangePropertyKeyCommand; 032import org.openstreetmap.josm.command.Command; 033import org.openstreetmap.josm.command.SequenceCommand; 034import org.openstreetmap.josm.data.osm.OsmPrimitive; 035import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 036import org.openstreetmap.josm.data.osm.OsmUtils; 037import org.openstreetmap.josm.data.osm.Tag; 038import org.openstreetmap.josm.data.validation.Severity; 039import org.openstreetmap.josm.data.validation.Test.TagTest; 040import org.openstreetmap.josm.data.validation.TestError; 041import org.openstreetmap.josm.data.validation.util.Entities; 042import org.openstreetmap.josm.gui.preferences.validator.ValidatorPreference; 043import org.openstreetmap.josm.gui.progress.ProgressMonitor; 044import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset; 045import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem; 046import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets; 047import org.openstreetmap.josm.gui.tagging.presets.items.Check; 048import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup; 049import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem; 050import org.openstreetmap.josm.gui.widgets.EditableList; 051import org.openstreetmap.josm.io.CachedFile; 052import org.openstreetmap.josm.tools.GBC; 053import org.openstreetmap.josm.tools.MultiMap; 054import org.openstreetmap.josm.tools.Utils; 055 056/** 057 * Check for misspelled or wrong tags 058 * 059 * @author frsantos 060 * @since 3669 061 */ 062public class TagChecker extends TagTest { 063 064 /** The config file of ignored tags */ 065 public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg"; 066 /** The config file of dictionary words */ 067 public static final String SPELL_FILE = "resource://data/validator/words.cfg"; 068 069 /** Normalized keys: the key should be substituted by the value if the key was not found in presets */ 070 private static final Map<String, String> harmonizedKeys = new HashMap<>(); 071 /** The spell check preset values */ 072 private static volatile MultiMap<String, String> presetsValueData; 073 /** The TagChecker data */ 074 private static final List<CheckerData> checkerData = new ArrayList<>(); 075 private static final List<String> ignoreDataStartsWith = new ArrayList<>(); 076 private static final List<String> ignoreDataEquals = new ArrayList<>(); 077 private static final List<String> ignoreDataEndsWith = new ArrayList<>(); 078 private static final List<Tag> ignoreDataTag = new ArrayList<>(); 079 080 /** The preferences prefix */ 081 protected static final String PREFIX = ValidatorPreference.PREFIX + "." + TagChecker.class.getSimpleName(); 082 083 public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues"; 084 public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys"; 085 public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex"; 086 public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes"; 087 088 public static final String PREF_SOURCES = PREFIX + ".source"; 089 090 public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + "BeforeUpload"; 091 public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + "BeforeUpload"; 092 public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + "BeforeUpload"; 093 public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + "BeforeUpload"; 094 095 protected boolean checkKeys; 096 protected boolean checkValues; 097 protected boolean checkComplex; 098 protected boolean checkFixmes; 099 100 protected JCheckBox prefCheckKeys; 101 protected JCheckBox prefCheckValues; 102 protected JCheckBox prefCheckComplex; 103 protected JCheckBox prefCheckFixmes; 104 protected JCheckBox prefCheckPaint; 105 106 protected JCheckBox prefCheckKeysBeforeUpload; 107 protected JCheckBox prefCheckValuesBeforeUpload; 108 protected JCheckBox prefCheckComplexBeforeUpload; 109 protected JCheckBox prefCheckFixmesBeforeUpload; 110 protected JCheckBox prefCheckPaintBeforeUpload; 111 112 // CHECKSTYLE.OFF: SingleSpaceSeparator 113 protected static final int EMPTY_VALUES = 1200; 114 protected static final int INVALID_KEY = 1201; 115 protected static final int INVALID_VALUE = 1202; 116 protected static final int FIXME = 1203; 117 protected static final int INVALID_SPACE = 1204; 118 protected static final int INVALID_KEY_SPACE = 1205; 119 protected static final int INVALID_HTML = 1206; /* 1207 was PAINT */ 120 protected static final int LONG_VALUE = 1208; 121 protected static final int LONG_KEY = 1209; 122 protected static final int LOW_CHAR_VALUE = 1210; 123 protected static final int LOW_CHAR_KEY = 1211; 124 protected static final int MISSPELLED_VALUE = 1212; 125 protected static final int MISSPELLED_KEY = 1213; 126 protected static final int MULTIPLE_SPACES = 1214; 127 // CHECKSTYLE.ON: SingleSpaceSeparator 128 // 1250 and up is used by tagcheck 129 130 protected EditableList sourcesList; 131 132 private static final Set<String> DEFAULT_SOURCES = new HashSet<>(Arrays.asList(/*DATA_FILE, */IGNORE_FILE, SPELL_FILE)); 133 134 /** 135 * Constructor 136 */ 137 public TagChecker() { 138 super(tr("Tag checker"), tr("This test checks for errors in tag keys and values.")); 139 } 140 141 @Override 142 public void initialize() throws IOException { 143 initializeData(); 144 initializePresets(); 145 } 146 147 /** 148 * Reads the spellcheck file into a HashMap. 149 * The data file is a list of words, beginning with +/-. If it starts with +, 150 * the word is valid, but if it starts with -, the word should be replaced 151 * by the nearest + word before this. 152 * 153 * @throws IOException if any I/O error occurs 154 */ 155 private static void initializeData() throws IOException { 156 checkerData.clear(); 157 ignoreDataStartsWith.clear(); 158 ignoreDataEquals.clear(); 159 ignoreDataEndsWith.clear(); 160 ignoreDataTag.clear(); 161 harmonizedKeys.clear(); 162 163 StringBuilder errorSources = new StringBuilder(); 164 for (String source : Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES)) { 165 try ( 166 CachedFile cf = new CachedFile(source); 167 BufferedReader reader = cf.getContentReader() 168 ) { 169 String okValue = null; 170 boolean tagcheckerfile = false; 171 boolean ignorefile = false; 172 boolean isFirstLine = true; 173 String line; 174 while ((line = reader.readLine()) != null && (tagcheckerfile || !line.isEmpty())) { 175 if (line.startsWith("#")) { 176 if (line.startsWith("# JOSM TagChecker")) { 177 tagcheckerfile = true; 178 if (!DEFAULT_SOURCES.contains(source)) { 179 Main.info(tr("Adding {0} to tag checker", source)); 180 } 181 } else 182 if (line.startsWith("# JOSM IgnoreTags")) { 183 ignorefile = true; 184 if (!DEFAULT_SOURCES.contains(source)) { 185 Main.info(tr("Adding {0} to ignore tags", source)); 186 } 187 } 188 } else if (ignorefile) { 189 line = line.trim(); 190 if (line.length() < 4) { 191 continue; 192 } 193 194 String key = line.substring(0, 2); 195 line = line.substring(2); 196 197 switch (key) { 198 case "S:": 199 ignoreDataStartsWith.add(line); 200 break; 201 case "E:": 202 ignoreDataEquals.add(line); 203 break; 204 case "F:": 205 ignoreDataEndsWith.add(line); 206 break; 207 case "K:": 208 ignoreDataTag.add(Tag.ofString(line)); 209 break; 210 default: 211 if (!key.startsWith(";")) { 212 Main.warn("Unsupported TagChecker key: " + key); 213 } 214 } 215 } else if (tagcheckerfile) { 216 if (!line.isEmpty()) { 217 CheckerData d = new CheckerData(); 218 String err = d.getData(line); 219 220 if (err == null) { 221 checkerData.add(d); 222 } else { 223 Main.error(tr("Invalid tagchecker line - {0}: {1}", err, line)); 224 } 225 } 226 } else if (line.charAt(0) == '+') { 227 okValue = line.substring(1); 228 } else if (line.charAt(0) == '-' && okValue != null) { 229 harmonizedKeys.put(harmonizeKey(line.substring(1)), okValue); 230 } else { 231 Main.error(tr("Invalid spellcheck line: {0}", line)); 232 } 233 if (isFirstLine) { 234 isFirstLine = false; 235 if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) { 236 Main.info(tr("Adding {0} to spellchecker", source)); 237 } 238 } 239 } 240 } catch (IOException e) { 241 Main.error(e); 242 errorSources.append(source).append('\n'); 243 } 244 } 245 246 if (errorSources.length() > 0) 247 throw new IOException(tr("Could not access data file(s):\n{0}", errorSources)); 248 } 249 250 /** 251 * Reads the presets data. 252 * 253 */ 254 public static void initializePresets() { 255 256 if (!Main.pref.getBoolean(PREF_CHECK_VALUES, true)) 257 return; 258 259 Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets(); 260 if (!presets.isEmpty()) { 261 presetsValueData = new MultiMap<>(); 262 for (String a : OsmPrimitive.getUninterestingKeys()) { 263 presetsValueData.putVoid(a); 264 } 265 // TODO directionKeys are no longer in OsmPrimitive (search pattern is used instead) 266 for (String a : Main.pref.getCollection(ValidatorPreference.PREFIX + ".knownkeys", 267 Arrays.asList(new String[]{"is_in", "int_ref", "fixme", "population"}))) { 268 presetsValueData.putVoid(a); 269 } 270 for (TaggingPreset p : presets) { 271 for (TaggingPresetItem i : p.data) { 272 if (i instanceof KeyedItem) { 273 addPresetValue(p, (KeyedItem) i); 274 } else if (i instanceof CheckGroup) { 275 for (Check c : ((CheckGroup) i).checks) { 276 addPresetValue(p, c); 277 } 278 } 279 } 280 } 281 } 282 } 283 284 private static void addPresetValue(TaggingPreset p, KeyedItem ky) { 285 Collection<String> values = ky.getValues(); 286 if (ky.key != null && values != null) { 287 try { 288 presetsValueData.putAll(ky.key, values); 289 harmonizedKeys.put(harmonizeKey(ky.key), ky.key); 290 } catch (NullPointerException e) { 291 Main.error(e, p+": Unable to initialize "+ky+'.'); 292 } 293 } 294 } 295 296 /** 297 * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters) 298 * @param s string to check 299 * @return {@code true} if {@code s} contains characters with code below 0x20 300 */ 301 private static boolean containsLow(String s) { 302 if (s == null) 303 return false; 304 for (int i = 0; i < s.length(); i++) { 305 if (s.charAt(i) < 0x20) 306 return true; 307 } 308 return false; 309 } 310 311 /** 312 * Determines if the given key is in internal presets. 313 * @param key key 314 * @return {@code true} if the given key is in internal presets 315 * @since 9023 316 */ 317 public static boolean isKeyInPresets(String key) { 318 return presetsValueData.get(key) != null; 319 } 320 321 /** 322 * Determines if the given tag is in internal presets. 323 * @param key key 324 * @param value value 325 * @return {@code true} if the given tag is in internal presets 326 * @since 9023 327 */ 328 public static boolean isTagInPresets(String key, String value) { 329 final Set<String> values = presetsValueData.get(key); 330 return values != null && (values.isEmpty() || values.contains(value)); 331 } 332 333 /** 334 * Returns the list of ignored tags. 335 * @return the list of ignored tags 336 * @since 9023 337 */ 338 public static List<Tag> getIgnoredTags() { 339 return new ArrayList<>(ignoreDataTag); 340 } 341 342 /** 343 * Determines if the given tag is ignored for checks "key/tag not in presets". 344 * @param key key 345 * @param value value 346 * @return {@code true} if the given tag is ignored 347 * @since 9023 348 */ 349 public static boolean isTagIgnored(String key, String value) { 350 boolean tagInPresets = isTagInPresets(key, value); 351 boolean ignore = false; 352 353 for (String a : ignoreDataStartsWith) { 354 if (key.startsWith(a)) { 355 ignore = true; 356 } 357 } 358 for (String a : ignoreDataEquals) { 359 if (key.equals(a)) { 360 ignore = true; 361 } 362 } 363 for (String a : ignoreDataEndsWith) { 364 if (key.endsWith(a)) { 365 ignore = true; 366 } 367 } 368 369 if (!tagInPresets) { 370 for (Tag a : ignoreDataTag) { 371 if (key.equals(a.getKey()) && value.equals(a.getValue())) { 372 ignore = true; 373 } 374 } 375 } 376 return ignore; 377 } 378 379 /** 380 * Checks the primitive tags 381 * @param p The primitive to check 382 */ 383 @Override 384 public void check(OsmPrimitive p) { 385 // Just a collection to know if a primitive has been already marked with error 386 MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>(); 387 388 if (checkComplex) { 389 Map<String, String> keys = p.getKeys(); 390 for (CheckerData d : checkerData) { 391 if (d.match(p, keys)) { 392 errors.add(TestError.builder(this, d.getSeverity(), d.getCode()) 393 .message(tr("Suspicious tag/value combinations"), d.getDescription()) 394 .primitives(p) 395 .build()); 396 withErrors.put(p, "TC"); 397 } 398 } 399 } 400 401 for (Entry<String, String> prop : p.getKeys().entrySet()) { 402 String s = marktr("Key ''{0}'' invalid."); 403 String key = prop.getKey(); 404 String value = prop.getValue(); 405 if (checkValues && (containsLow(value)) && !withErrors.contains(p, "ICV")) { 406 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_VALUE) 407 .message(tr("Tag value contains character with code less than 0x20"), s, key) 408 .primitives(p) 409 .build()); 410 withErrors.put(p, "ICV"); 411 } 412 if (checkKeys && (containsLow(key)) && !withErrors.contains(p, "ICK")) { 413 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_KEY) 414 .message(tr("Tag key contains character with code less than 0x20"), s, key) 415 .primitives(p) 416 .build()); 417 withErrors.put(p, "ICK"); 418 } 419 if (checkValues && (value != null && value.length() > 255) && !withErrors.contains(p, "LV")) { 420 errors.add(TestError.builder(this, Severity.ERROR, LONG_VALUE) 421 .message(tr("Tag value longer than allowed"), s, key) 422 .primitives(p) 423 .build()); 424 withErrors.put(p, "LV"); 425 } 426 if (checkKeys && (key != null && key.length() > 255) && !withErrors.contains(p, "LK")) { 427 errors.add(TestError.builder(this, Severity.ERROR, LONG_KEY) 428 .message(tr("Tag key longer than allowed"), s, key) 429 .primitives(p) 430 .build()); 431 withErrors.put(p, "LK"); 432 } 433 if (checkValues && (value == null || value.trim().isEmpty()) && !withErrors.contains(p, "EV")) { 434 errors.add(TestError.builder(this, Severity.WARNING, EMPTY_VALUES) 435 .message(tr("Tags with empty values"), s, key) 436 .primitives(p) 437 .build()); 438 withErrors.put(p, "EV"); 439 } 440 if (checkKeys && key != null && key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) { 441 errors.add(TestError.builder(this, Severity.WARNING, INVALID_KEY_SPACE) 442 .message(tr("Invalid white space in property key"), s, key) 443 .primitives(p) 444 .build()); 445 withErrors.put(p, "IPK"); 446 } 447 if (checkValues && value != null && (value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, "SPACE")) { 448 errors.add(TestError.builder(this, Severity.WARNING, INVALID_SPACE) 449 .message(tr("Property values start or end with white space"), s, key) 450 .primitives(p) 451 .build()); 452 withErrors.put(p, "SPACE"); 453 } 454 if (checkValues && value != null && value.contains(" ") && !withErrors.contains(p, "SPACE")) { 455 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_SPACES) 456 .message(tr("Property values contain multiple white spaces"), s, key) 457 .primitives(p) 458 .build()); 459 withErrors.put(p, "SPACE"); 460 } 461 if (checkValues && value != null && !value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) { 462 errors.add(TestError.builder(this, Severity.OTHER, INVALID_HTML) 463 .message(tr("Property values contain HTML entity"), s, key) 464 .primitives(p) 465 .build()); 466 withErrors.put(p, "HTML"); 467 } 468 if (checkValues && key != null && value != null && !value.isEmpty() && presetsValueData != null && !isTagIgnored(key, value)) { 469 if (!isKeyInPresets(key)) { 470 String prettifiedKey = harmonizeKey(key); 471 String fixedKey = harmonizedKeys.get(prettifiedKey); 472 if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) { 473 // misspelled preset key 474 final TestError.Builder error = TestError.builder(this, Severity.WARNING, MISSPELLED_KEY) 475 .message(tr("Misspelled property key"), marktr("Key ''{0}'' looks like ''{1}''."), key, fixedKey) 476 .primitives(p); 477 if (p.hasKey(fixedKey)) { 478 errors.add(error.build()); 479 } else { 480 errors.add(error.fix(() -> new ChangePropertyKeyCommand(p, key, fixedKey)).build()); 481 } 482 withErrors.put(p, "WPK"); 483 } else { 484 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE) 485 .message(tr("Presets do not contain property key"), marktr("Key ''{0}'' not in presets."), key) 486 .primitives(p) 487 .build()); 488 withErrors.put(p, "UPK"); 489 } 490 } else if (!isTagInPresets(key, value)) { 491 // try to fix common typos and check again if value is still unknown 492 String fixedValue = harmonizeValue(prop.getValue()); 493 Map<String, String> possibleValues = getPossibleValues(presetsValueData.get(key)); 494 if (possibleValues.containsKey(fixedValue)) { 495 final String newKey = possibleValues.get(fixedValue); 496 // misspelled preset value 497 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE) 498 .message(tr("Misspelled property value"), 499 marktr("Value ''{0}'' for key ''{1}'' looks like ''{2}''."), prop.getValue(), key, fixedValue) 500 .primitives(p) 501 .fix(() -> new ChangePropertyCommand(p, key, newKey)) 502 .build()); 503 withErrors.put(p, "WPV"); 504 } else { 505 // unknown preset value 506 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE) 507 .message(tr("Presets do not contain property value"), 508 marktr("Value ''{0}'' for key ''{1}'' not in presets."), prop.getValue(), key) 509 .primitives(p) 510 .build()); 511 withErrors.put(p, "UPV"); 512 } 513 } 514 } 515 if (checkFixmes && key != null && value != null && !value.isEmpty() && isFixme(key, value) && !withErrors.contains(p, "FIXME")) { 516 errors.add(TestError.builder(this, Severity.OTHER, FIXME) 517 .message(tr("FIXMES")) 518 .primitives(p) 519 .build()); 520 withErrors.put(p, "FIXME"); 521 } 522 } 523 } 524 525 private static boolean isFixme(String key, String value) { 526 return key.toLowerCase(Locale.ENGLISH).contains("fixme") || key.contains("todo") 527 || value.toLowerCase(Locale.ENGLISH).contains("fixme") || value.contains("check and delete"); 528 } 529 530 private static Map<String, String> getPossibleValues(Set<String> values) { 531 // generate a map with common typos 532 Map<String, String> map = new HashMap<>(); 533 if (values != null) { 534 for (String value : values) { 535 map.put(value, value); 536 if (value.contains("_")) { 537 map.put(value.replace("_", ""), value); 538 } 539 } 540 } 541 return map; 542 } 543 544 private static String harmonizeKey(String key) { 545 key = key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_'); 546 return Utils.strip(key, "-_;:,"); 547 } 548 549 private static String harmonizeValue(String value) { 550 value = value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_'); 551 return Utils.strip(value, "-_;:,"); 552 } 553 554 @Override 555 public void startTest(ProgressMonitor monitor) { 556 super.startTest(monitor); 557 checkKeys = Main.pref.getBoolean(PREF_CHECK_KEYS, true); 558 if (isBeforeUpload) { 559 checkKeys = checkKeys && Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true); 560 } 561 562 checkValues = Main.pref.getBoolean(PREF_CHECK_VALUES, true); 563 if (isBeforeUpload) { 564 checkValues = checkValues && Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true); 565 } 566 567 checkComplex = Main.pref.getBoolean(PREF_CHECK_COMPLEX, true); 568 if (isBeforeUpload) { 569 checkComplex = checkComplex && Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true); 570 } 571 572 checkFixmes = Main.pref.getBoolean(PREF_CHECK_FIXMES, true); 573 if (isBeforeUpload) { 574 checkFixmes = checkFixmes && Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true); 575 } 576 } 577 578 @Override 579 public void visit(Collection<OsmPrimitive> selection) { 580 if (checkKeys || checkValues || checkComplex || checkFixmes) { 581 super.visit(selection); 582 } 583 } 584 585 @Override 586 public void addGui(JPanel testPanel) { 587 GBC a = GBC.eol(); 588 a.anchor = GridBagConstraints.EAST; 589 590 testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0)); 591 592 prefCheckKeys = new JCheckBox(tr("Check property keys."), Main.pref.getBoolean(PREF_CHECK_KEYS, true)); 593 prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words.")); 594 testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0)); 595 596 prefCheckKeysBeforeUpload = new JCheckBox(); 597 prefCheckKeysBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true)); 598 testPanel.add(prefCheckKeysBeforeUpload, a); 599 600 prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Main.pref.getBoolean(PREF_CHECK_COMPLEX, true)); 601 prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules.")); 602 testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0)); 603 604 prefCheckComplexBeforeUpload = new JCheckBox(); 605 prefCheckComplexBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true)); 606 testPanel.add(prefCheckComplexBeforeUpload, a); 607 608 final Collection<String> sources = Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES); 609 sourcesList = new EditableList(tr("TagChecker source")); 610 sourcesList.setItems(sources); 611 testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0)); 612 testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0)); 613 614 ActionListener disableCheckActionListener = e -> handlePrefEnable(); 615 prefCheckKeys.addActionListener(disableCheckActionListener); 616 prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener); 617 prefCheckComplex.addActionListener(disableCheckActionListener); 618 prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener); 619 620 handlePrefEnable(); 621 622 prefCheckValues = new JCheckBox(tr("Check property values."), Main.pref.getBoolean(PREF_CHECK_VALUES, true)); 623 prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets.")); 624 testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0)); 625 626 prefCheckValuesBeforeUpload = new JCheckBox(); 627 prefCheckValuesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true)); 628 testPanel.add(prefCheckValuesBeforeUpload, a); 629 630 prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Main.pref.getBoolean(PREF_CHECK_FIXMES, true)); 631 prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value.")); 632 testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0)); 633 634 prefCheckFixmesBeforeUpload = new JCheckBox(); 635 prefCheckFixmesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true)); 636 testPanel.add(prefCheckFixmesBeforeUpload, a); 637 } 638 639 public void handlePrefEnable() { 640 boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected() 641 || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected(); 642 sourcesList.setEnabled(selected); 643 } 644 645 @Override 646 public boolean ok() { 647 enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected(); 648 testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected() 649 || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected(); 650 651 Main.pref.put(PREF_CHECK_VALUES, prefCheckValues.isSelected()); 652 Main.pref.put(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected()); 653 Main.pref.put(PREF_CHECK_KEYS, prefCheckKeys.isSelected()); 654 Main.pref.put(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected()); 655 Main.pref.put(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected()); 656 Main.pref.put(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected()); 657 Main.pref.put(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected()); 658 Main.pref.put(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected()); 659 return Main.pref.putCollection(PREF_SOURCES, sourcesList.getItems()); 660 } 661 662 @Override 663 public Command fixError(TestError testError) { 664 List<Command> commands = new ArrayList<>(50); 665 666 Collection<? extends OsmPrimitive> primitives = testError.getPrimitives(); 667 for (OsmPrimitive p : primitives) { 668 Map<String, String> tags = p.getKeys(); 669 if (tags.isEmpty()) { 670 continue; 671 } 672 673 for (Entry<String, String> prop: tags.entrySet()) { 674 String key = prop.getKey(); 675 String value = prop.getValue(); 676 if (value == null || value.trim().isEmpty()) { 677 commands.add(new ChangePropertyCommand(p, key, null)); 678 } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains(" ")) { 679 commands.add(new ChangePropertyCommand(p, key, Tag.removeWhiteSpaces(value))); 680 } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains(" ")) { 681 commands.add(new ChangePropertyKeyCommand(p, key, Tag.removeWhiteSpaces(key))); 682 } else { 683 String evalue = Entities.unescape(value); 684 if (!evalue.equals(value)) { 685 commands.add(new ChangePropertyCommand(p, key, evalue)); 686 } 687 } 688 } 689 } 690 691 if (commands.isEmpty()) 692 return null; 693 if (commands.size() == 1) 694 return commands.get(0); 695 696 return new SequenceCommand(tr("Fix tags"), commands); 697 } 698 699 @Override 700 public boolean isFixable(TestError testError) { 701 if (testError.getTester() instanceof TagChecker) { 702 int code = testError.getCode(); 703 return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE || 704 code == INVALID_KEY_SPACE || code == INVALID_HTML || code == MISSPELLED_VALUE || 705 code == MULTIPLE_SPACES; 706 } 707 708 return false; 709 } 710 711 protected static class CheckerData { 712 private String description; 713 protected List<CheckerElement> data = new ArrayList<>(); 714 private OsmPrimitiveType type; 715 private int code; 716 protected Severity severity; 717 // CHECKSTYLE.OFF: SingleSpaceSeparator 718 protected static final int TAG_CHECK_ERROR = 1250; 719 protected static final int TAG_CHECK_WARN = 1260; 720 protected static final int TAG_CHECK_INFO = 1270; 721 // CHECKSTYLE.ON: SingleSpaceSeparator 722 723 protected static class CheckerElement { 724 public Object tag; 725 public Object value; 726 public boolean noMatch; 727 public boolean tagAll; 728 public boolean valueAll; 729 public boolean valueBool; 730 731 private static Pattern getPattern(String str) { 732 if (str.endsWith("/i")) 733 return Pattern.compile(str.substring(1, str.length()-2), Pattern.CASE_INSENSITIVE); 734 if (str.endsWith("/")) 735 return Pattern.compile(str.substring(1, str.length()-1)); 736 737 throw new IllegalStateException(); 738 } 739 740 public CheckerElement(String exp) { 741 Matcher m = Pattern.compile("(.+)([!=]=)(.+)").matcher(exp); 742 m.matches(); 743 744 String n = m.group(1).trim(); 745 746 if ("*".equals(n)) { 747 tagAll = true; 748 } else { 749 tag = n.startsWith("/") ? getPattern(n) : n; 750 noMatch = "!=".equals(m.group(2)); 751 n = m.group(3).trim(); 752 if ("*".equals(n)) { 753 valueAll = true; 754 } else if ("BOOLEAN_TRUE".equals(n)) { 755 valueBool = true; 756 value = OsmUtils.trueval; 757 } else if ("BOOLEAN_FALSE".equals(n)) { 758 valueBool = true; 759 value = OsmUtils.falseval; 760 } else { 761 value = n.startsWith("/") ? getPattern(n) : n; 762 } 763 } 764 } 765 766 public boolean match(Map<String, String> keys) { 767 for (Entry<String, String> prop: keys.entrySet()) { 768 String key = prop.getKey(); 769 String val = valueBool ? OsmUtils.getNamedOsmBoolean(prop.getValue()) : prop.getValue(); 770 if ((tagAll || (tag instanceof Pattern ? ((Pattern) tag).matcher(key).matches() : key.equals(tag))) 771 && (valueAll || (value instanceof Pattern ? ((Pattern) value).matcher(val).matches() : val.equals(value)))) 772 return !noMatch; 773 } 774 return noMatch; 775 } 776 } 777 778 private static final Pattern CLEAN_STR_PATTERN = Pattern.compile(" *# *([^#]+) *$"); 779 private static final Pattern SPLIT_TRIMMED_PATTERN = Pattern.compile(" *: *"); 780 private static final Pattern SPLIT_ELEMENTS_PATTERN = Pattern.compile(" *&& *"); 781 782 public String getData(final String str) { 783 Matcher m = CLEAN_STR_PATTERN.matcher(str); 784 String trimmed = m.replaceFirst("").trim(); 785 try { 786 description = m.group(1); 787 if (description != null && description.isEmpty()) { 788 description = null; 789 } 790 } catch (IllegalStateException e) { 791 Main.error(e); 792 description = null; 793 } 794 String[] n = SPLIT_TRIMMED_PATTERN.split(trimmed, 3); 795 switch (n[0]) { 796 case "way": 797 type = OsmPrimitiveType.WAY; 798 break; 799 case "node": 800 type = OsmPrimitiveType.NODE; 801 break; 802 case "relation": 803 type = OsmPrimitiveType.RELATION; 804 break; 805 case "*": 806 type = null; 807 break; 808 default: 809 return tr("Could not find element type"); 810 } 811 if (n.length != 3) 812 return tr("Incorrect number of parameters"); 813 814 switch (n[1]) { 815 case "W": 816 severity = Severity.WARNING; 817 code = TAG_CHECK_WARN; 818 break; 819 case "E": 820 severity = Severity.ERROR; 821 code = TAG_CHECK_ERROR; 822 break; 823 case "I": 824 severity = Severity.OTHER; 825 code = TAG_CHECK_INFO; 826 break; 827 default: 828 return tr("Could not find warning level"); 829 } 830 for (String exp: SPLIT_ELEMENTS_PATTERN.split(n[2])) { 831 try { 832 data.add(new CheckerElement(exp)); 833 } catch (IllegalStateException e) { 834 Main.trace(e); 835 return tr("Illegal expression ''{0}''", exp); 836 } catch (PatternSyntaxException e) { 837 Main.trace(e); 838 return tr("Illegal regular expression ''{0}''", exp); 839 } 840 } 841 return null; 842 } 843 844 public boolean match(OsmPrimitive osm, Map<String, String> keys) { 845 if (type != null && OsmPrimitiveType.from(osm) != type) 846 return false; 847 848 for (CheckerElement ce : data) { 849 if (!ce.match(keys)) 850 return false; 851 } 852 return true; 853 } 854 855 public String getDescription() { 856 return description; 857 } 858 859 public Severity getSeverity() { 860 return severity; 861 } 862 863 public int getCode() { 864 if (type == null) 865 return code; 866 867 return code + type.ordinal() + 1; 868 } 869 } 870}