001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.validation.tests; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.awt.Rectangle; 007import java.io.BufferedReader; 008import java.io.IOException; 009import java.io.InputStream; 010import java.io.Reader; 011import java.io.StringReader; 012import java.lang.reflect.Method; 013import java.text.MessageFormat; 014import java.util.ArrayList; 015import java.util.Arrays; 016import java.util.Collection; 017import java.util.Collections; 018import java.util.HashMap; 019import java.util.HashSet; 020import java.util.Iterator; 021import java.util.LinkedHashMap; 022import java.util.LinkedHashSet; 023import java.util.LinkedList; 024import java.util.List; 025import java.util.Locale; 026import java.util.Map; 027import java.util.Objects; 028import java.util.Optional; 029import java.util.Set; 030import java.util.function.Predicate; 031import java.util.regex.Matcher; 032import java.util.regex.Pattern; 033 034import org.openstreetmap.josm.command.ChangePropertyCommand; 035import org.openstreetmap.josm.command.ChangePropertyKeyCommand; 036import org.openstreetmap.josm.command.Command; 037import org.openstreetmap.josm.command.DeleteCommand; 038import org.openstreetmap.josm.command.SequenceCommand; 039import org.openstreetmap.josm.data.coor.LatLon; 040import org.openstreetmap.josm.data.osm.DataSet; 041import org.openstreetmap.josm.data.osm.INode; 042import org.openstreetmap.josm.data.osm.IRelation; 043import org.openstreetmap.josm.data.osm.IWay; 044import org.openstreetmap.josm.data.osm.OsmPrimitive; 045import org.openstreetmap.josm.data.osm.OsmUtils; 046import org.openstreetmap.josm.data.osm.Relation; 047import org.openstreetmap.josm.data.osm.Tag; 048import org.openstreetmap.josm.data.osm.Way; 049import org.openstreetmap.josm.data.preferences.sources.SourceEntry; 050import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper; 051import org.openstreetmap.josm.data.validation.OsmValidator; 052import org.openstreetmap.josm.data.validation.Severity; 053import org.openstreetmap.josm.data.validation.Test; 054import org.openstreetmap.josm.data.validation.TestError; 055import org.openstreetmap.josm.gui.mappaint.Environment; 056import org.openstreetmap.josm.gui.mappaint.Keyword; 057import org.openstreetmap.josm.gui.mappaint.MultiCascade; 058import org.openstreetmap.josm.gui.mappaint.mapcss.Condition; 059import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ClassCondition; 060import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ExpressionCondition; 061import org.openstreetmap.josm.gui.mappaint.mapcss.Expression; 062import org.openstreetmap.josm.gui.mappaint.mapcss.ExpressionFactory.Functions; 063import org.openstreetmap.josm.gui.mappaint.mapcss.ExpressionFactory.ParameterFunction; 064import org.openstreetmap.josm.gui.mappaint.mapcss.Instruction; 065import org.openstreetmap.josm.gui.mappaint.mapcss.LiteralExpression; 066import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule; 067import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule.Declaration; 068import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource; 069import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource.MapCSSRuleIndex; 070import org.openstreetmap.josm.gui.mappaint.mapcss.Selector; 071import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.AbstractSelector; 072import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector; 073import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.OptimizedGeneralSelector; 074import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.MapCSSParser; 075import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException; 076import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.TokenMgrError; 077import org.openstreetmap.josm.gui.progress.ProgressMonitor; 078import org.openstreetmap.josm.io.CachedFile; 079import org.openstreetmap.josm.io.FileWatcher; 080import org.openstreetmap.josm.io.IllegalDataException; 081import org.openstreetmap.josm.io.UTFInputStreamReader; 082import org.openstreetmap.josm.spi.preferences.Config; 083import org.openstreetmap.josm.tools.CheckParameterUtil; 084import org.openstreetmap.josm.tools.DefaultGeoProperty; 085import org.openstreetmap.josm.tools.GeoProperty; 086import org.openstreetmap.josm.tools.GeoPropertyIndex; 087import org.openstreetmap.josm.tools.I18n; 088import org.openstreetmap.josm.tools.JosmRuntimeException; 089import org.openstreetmap.josm.tools.Logging; 090import org.openstreetmap.josm.tools.MultiMap; 091import org.openstreetmap.josm.tools.Territories; 092import org.openstreetmap.josm.tools.Utils; 093 094/** 095 * MapCSS-based tag checker/fixer. 096 * @since 6506 097 */ 098public class MapCSSTagChecker extends Test.TagTest { 099 IndexData indexData; 100 101 /** 102 * Helper class to store indexes of rules. 103 * @author Gerd 104 * 105 */ 106 private static final class IndexData { 107 final Map<MapCSSRule, TagCheck> ruleToCheckMap = new HashMap<>(); 108 109 /** 110 * Rules for nodes 111 */ 112 final MapCSSRuleIndex nodeRules = new MapCSSRuleIndex(); 113 /** 114 * Rules for ways without tag area=no 115 */ 116 final MapCSSRuleIndex wayRules = new MapCSSRuleIndex(); 117 /** 118 * Rules for ways with tag area=no 119 */ 120 final MapCSSRuleIndex wayNoAreaRules = new MapCSSRuleIndex(); 121 /** 122 * Rules for relations that are not multipolygon relations 123 */ 124 final MapCSSRuleIndex relationRules = new MapCSSRuleIndex(); 125 /** 126 * Rules for multipolygon relations 127 */ 128 final MapCSSRuleIndex multipolygonRules = new MapCSSRuleIndex(); 129 130 private IndexData(MultiMap<String, TagCheck> checks, boolean includeOtherSeverity) { 131 buildIndex(checks, includeOtherSeverity); 132 } 133 134 private void buildIndex(MultiMap<String, TagCheck> checks, boolean includeOtherSeverity) { 135 List<TagCheck> allChecks = new ArrayList<>(); 136 for (Set<TagCheck> cs : checks.values()) { 137 allChecks.addAll(cs); 138 } 139 140 ruleToCheckMap.clear(); 141 nodeRules.clear(); 142 wayRules.clear(); 143 wayNoAreaRules.clear(); 144 relationRules.clear(); 145 multipolygonRules.clear(); 146 147 // optimization: filter rules for different primitive types 148 for (TagCheck c : allChecks) { 149 if (!includeOtherSeverity && Severity.OTHER == c.getSeverity() 150 && c.setClassExpressions.isEmpty()) { 151 // Ignore "information" level checks if not wanted, unless they also set a MapCSS class 152 continue; 153 } 154 155 for (Selector s : c.rule.selectors) { 156 // find the rightmost selector, this must be a GeneralSelector 157 Selector selRightmost = s; 158 while (selRightmost instanceof Selector.ChildOrParentSelector) { 159 selRightmost = ((Selector.ChildOrParentSelector) selRightmost).right; 160 } 161 MapCSSRule optRule = new MapCSSRule(s.optimizedBaseCheck(), c.rule.declaration); 162 163 ruleToCheckMap.put(optRule, c); 164 final String base = ((GeneralSelector) selRightmost).getBase(); 165 switch (base) { 166 case Selector.BASE_NODE: 167 nodeRules.add(optRule); 168 break; 169 case Selector.BASE_WAY: 170 wayNoAreaRules.add(optRule); 171 wayRules.add(optRule); 172 break; 173 case Selector.BASE_AREA: 174 wayRules.add(optRule); 175 multipolygonRules.add(optRule); 176 break; 177 case Selector.BASE_RELATION: 178 relationRules.add(optRule); 179 multipolygonRules.add(optRule); 180 break; 181 case Selector.BASE_ANY: 182 nodeRules.add(optRule); 183 wayRules.add(optRule); 184 wayNoAreaRules.add(optRule); 185 relationRules.add(optRule); 186 multipolygonRules.add(optRule); 187 break; 188 case Selector.BASE_CANVAS: 189 case Selector.BASE_META: 190 case Selector.BASE_SETTING: 191 break; 192 default: 193 final RuntimeException e = new JosmRuntimeException(MessageFormat.format("Unknown MapCSS base selector {0}", base)); 194 Logging.warn(tr("Failed to index validator rules. Error was: {0}", e.getMessage())); 195 Logging.error(e); 196 } 197 } 198 } 199 nodeRules.initIndex(); 200 wayRules.initIndex(); 201 wayNoAreaRules.initIndex(); 202 relationRules.initIndex(); 203 multipolygonRules.initIndex(); 204 } 205 206 /** 207 * Get the index of rules for the given primitive. 208 * @param p the primitve 209 * @return index of rules for the given primitive 210 */ 211 public MapCSSRuleIndex get(OsmPrimitive p) { 212 if (p instanceof INode) { 213 return nodeRules; 214 } else if (p instanceof IWay) { 215 if (OsmUtils.isFalse(p.get("area"))) { 216 return wayNoAreaRules; 217 } else { 218 return wayRules; 219 } 220 } else if (p instanceof IRelation) { 221 if (((IRelation<?>) p).isMultipolygon()) { 222 return multipolygonRules; 223 } else { 224 return relationRules; 225 } 226 } else { 227 throw new IllegalArgumentException("Unsupported type: " + p); 228 } 229 } 230 231 /** 232 * return the TagCheck for which the given indexed rule was created. 233 * @param rule an indexed rule 234 * @return the original TagCheck 235 */ 236 public TagCheck getCheck(MapCSSRule rule) { 237 return ruleToCheckMap.get(rule); 238 } 239 } 240 241 /** 242 * A grouped MapCSSRule with multiple selectors for a single declaration. 243 * @see MapCSSRule 244 */ 245 public static class GroupedMapCSSRule { 246 /** MapCSS selectors **/ 247 public final List<Selector> selectors; 248 /** MapCSS declaration **/ 249 public final Declaration declaration; 250 251 /** 252 * Constructs a new {@code GroupedMapCSSRule}. 253 * @param selectors MapCSS selectors 254 * @param declaration MapCSS declaration 255 */ 256 public GroupedMapCSSRule(List<Selector> selectors, Declaration declaration) { 257 this.selectors = selectors; 258 this.declaration = declaration; 259 } 260 261 @Override 262 public int hashCode() { 263 return Objects.hash(selectors, declaration); 264 } 265 266 @Override 267 public boolean equals(Object obj) { 268 if (this == obj) return true; 269 if (obj == null || getClass() != obj.getClass()) return false; 270 GroupedMapCSSRule that = (GroupedMapCSSRule) obj; 271 return Objects.equals(selectors, that.selectors) && 272 Objects.equals(declaration, that.declaration); 273 } 274 275 @Override 276 public String toString() { 277 return "GroupedMapCSSRule [selectors=" + selectors + ", declaration=" + declaration + ']'; 278 } 279 } 280 281 /** 282 * The preference key for tag checker source entries. 283 * @since 6670 284 */ 285 public static final String ENTRIES_PREF_KEY = "validator." + MapCSSTagChecker.class.getName() + ".entries"; 286 287 /** 288 * Constructs a new {@code MapCSSTagChecker}. 289 */ 290 public MapCSSTagChecker() { 291 super(tr("Tag checker (MapCSS based)"), tr("This test checks for errors in tag keys and values.")); 292 } 293 294 /** 295 * Represents a fix to a validation test. The fixing {@link Command} can be obtained by {@link #createCommand(OsmPrimitive, Selector)}. 296 */ 297 @FunctionalInterface 298 interface FixCommand { 299 /** 300 * Creates the fixing {@link Command} for the given primitive. The {@code matchingSelector} is used to evaluate placeholders 301 * (cf. {@link MapCSSTagChecker.TagCheck#insertArguments(Selector, String, OsmPrimitive)}). 302 * @param p OSM primitive 303 * @param matchingSelector matching selector 304 * @return fix command 305 */ 306 Command createCommand(OsmPrimitive p, Selector matchingSelector); 307 308 /** 309 * Checks that object is either an {@link Expression} or a {@link String}. 310 * @param obj object to check 311 * @throws IllegalArgumentException if object is not an {@code Expression} or a {@code String} 312 */ 313 static void checkObject(final Object obj) { 314 CheckParameterUtil.ensureThat(obj instanceof Expression || obj instanceof String, 315 () -> "instance of Exception or String expected, but got " + obj); 316 } 317 318 /** 319 * Evaluates given object as {@link Expression} or {@link String} on the matched {@link OsmPrimitive} and {@code matchingSelector}. 320 * @param obj object to evaluate ({@link Expression} or {@link String}) 321 * @param p OSM primitive 322 * @param matchingSelector matching selector 323 * @return result string 324 */ 325 static String evaluateObject(final Object obj, final OsmPrimitive p, final Selector matchingSelector) { 326 final String s; 327 if (obj instanceof Expression) { 328 s = (String) ((Expression) obj).evaluate(new Environment(p)); 329 } else if (obj instanceof String) { 330 s = (String) obj; 331 } else { 332 return null; 333 } 334 return TagCheck.insertArguments(matchingSelector, s, p); 335 } 336 337 /** 338 * Creates a fixing command which executes a {@link ChangePropertyCommand} on the specified tag. 339 * @param obj object to evaluate ({@link Expression} or {@link String}) 340 * @return created fix command 341 */ 342 static FixCommand fixAdd(final Object obj) { 343 checkObject(obj); 344 return new FixCommand() { 345 @Override 346 public Command createCommand(OsmPrimitive p, Selector matchingSelector) { 347 final Tag tag = Tag.ofString(evaluateObject(obj, p, matchingSelector)); 348 return new ChangePropertyCommand(p, tag.getKey(), tag.getValue()); 349 } 350 351 @Override 352 public String toString() { 353 return "fixAdd: " + obj; 354 } 355 }; 356 } 357 358 /** 359 * Creates a fixing command which executes a {@link ChangePropertyCommand} to delete the specified key. 360 * @param obj object to evaluate ({@link Expression} or {@link String}) 361 * @return created fix command 362 */ 363 static FixCommand fixRemove(final Object obj) { 364 checkObject(obj); 365 return new FixCommand() { 366 @Override 367 public Command createCommand(OsmPrimitive p, Selector matchingSelector) { 368 final String key = evaluateObject(obj, p, matchingSelector); 369 return new ChangePropertyCommand(p, key, ""); 370 } 371 372 @Override 373 public String toString() { 374 return "fixRemove: " + obj; 375 } 376 }; 377 } 378 379 /** 380 * Creates a fixing command which executes a {@link ChangePropertyKeyCommand} on the specified keys. 381 * @param oldKey old key 382 * @param newKey new key 383 * @return created fix command 384 */ 385 static FixCommand fixChangeKey(final String oldKey, final String newKey) { 386 return new FixCommand() { 387 @Override 388 public Command createCommand(OsmPrimitive p, Selector matchingSelector) { 389 return new ChangePropertyKeyCommand(p, 390 TagCheck.insertArguments(matchingSelector, oldKey, p), 391 TagCheck.insertArguments(matchingSelector, newKey, p)); 392 } 393 394 @Override 395 public String toString() { 396 return "fixChangeKey: " + oldKey + " => " + newKey; 397 } 398 }; 399 } 400 } 401 402 final MultiMap<String, TagCheck> checks = new MultiMap<>(); 403 404 /** 405 * Result of {@link TagCheck#readMapCSS} 406 * @since 8936 407 */ 408 public static class ParseResult { 409 /** Checks successfully parsed */ 410 public final List<TagCheck> parseChecks; 411 /** Errors that occurred during parsing */ 412 public final Collection<Throwable> parseErrors; 413 414 /** 415 * Constructs a new {@code ParseResult}. 416 * @param parseChecks Checks successfully parsed 417 * @param parseErrors Errors that occurred during parsing 418 */ 419 public ParseResult(List<TagCheck> parseChecks, Collection<Throwable> parseErrors) { 420 this.parseChecks = parseChecks; 421 this.parseErrors = parseErrors; 422 } 423 } 424 425 /** 426 * Tag check. 427 */ 428 public static class TagCheck implements Predicate<OsmPrimitive> { 429 /** The selector of this {@code TagCheck} */ 430 protected final GroupedMapCSSRule rule; 431 /** Commands to apply in order to fix a matching primitive */ 432 protected final List<FixCommand> fixCommands = new ArrayList<>(); 433 /** Tags (or arbitraty strings) of alternatives to be presented to the user */ 434 protected final List<String> alternatives = new ArrayList<>(); 435 /** An {@link org.openstreetmap.josm.gui.mappaint.mapcss.Instruction.AssignmentInstruction}-{@link Severity} pair. 436 * Is evaluated on the matching primitive to give the error message. Map is checked to contain exactly one element. */ 437 protected final Map<Instruction.AssignmentInstruction, Severity> errors = new HashMap<>(); 438 /** Unit tests */ 439 protected final Map<String, Boolean> assertions = new HashMap<>(); 440 /** MapCSS Classes to set on matching primitives */ 441 protected final Set<String> setClassExpressions = new HashSet<>(); 442 /** Denotes whether the object should be deleted for fixing it */ 443 protected boolean deletion; 444 /** A string used to group similar tests */ 445 protected String group; 446 447 TagCheck(GroupedMapCSSRule rule) { 448 this.rule = rule; 449 } 450 451 private static final String POSSIBLE_THROWS = possibleThrows(); 452 453 static final String possibleThrows() { 454 StringBuilder sb = new StringBuilder(); 455 for (Severity s : Severity.values()) { 456 if (sb.length() > 0) { 457 sb.append('/'); 458 } 459 sb.append("throw") 460 .append(s.name().charAt(0)) 461 .append(s.name().substring(1).toLowerCase(Locale.ENGLISH)); 462 } 463 return sb.toString(); 464 } 465 466 static TagCheck ofMapCSSRule(final GroupedMapCSSRule rule) throws IllegalDataException { 467 final TagCheck check = new TagCheck(rule); 468 for (Instruction i : rule.declaration.instructions) { 469 if (i instanceof Instruction.AssignmentInstruction) { 470 final Instruction.AssignmentInstruction ai = (Instruction.AssignmentInstruction) i; 471 if (ai.isSetInstruction) { 472 check.setClassExpressions.add(ai.key); 473 continue; 474 } 475 try { 476 final String val = ai.val instanceof Expression 477 ? Optional.ofNullable(((Expression) ai.val).evaluate(new Environment())).map(Object::toString).orElse(null) 478 : ai.val instanceof String 479 ? (String) ai.val 480 : ai.val instanceof Keyword 481 ? ((Keyword) ai.val).val 482 : null; 483 if (ai.key.startsWith("throw")) { 484 try { 485 check.errors.put(ai, Severity.valueOf(ai.key.substring("throw".length()).toUpperCase(Locale.ENGLISH))); 486 } catch (IllegalArgumentException e) { 487 Logging.log(Logging.LEVEL_WARN, 488 "Unsupported "+ai.key+" instruction. Allowed instructions are "+POSSIBLE_THROWS+'.', e); 489 } 490 } else if ("fixAdd".equals(ai.key)) { 491 check.fixCommands.add(FixCommand.fixAdd(ai.val)); 492 } else if ("fixRemove".equals(ai.key)) { 493 CheckParameterUtil.ensureThat(!(ai.val instanceof String) || !(val != null && val.contains("=")), 494 "Unexpected '='. Please only specify the key to remove in: " + ai); 495 check.fixCommands.add(FixCommand.fixRemove(ai.val)); 496 } else if (val != null && "fixChangeKey".equals(ai.key)) { 497 CheckParameterUtil.ensureThat(val.contains("=>"), "Separate old from new key by '=>'!"); 498 final String[] x = val.split("=>", 2); 499 check.fixCommands.add(FixCommand.fixChangeKey(Utils.removeWhiteSpaces(x[0]), Utils.removeWhiteSpaces(x[1]))); 500 } else if (val != null && "fixDeleteObject".equals(ai.key)) { 501 CheckParameterUtil.ensureThat("this".equals(val), "fixDeleteObject must be followed by 'this'"); 502 check.deletion = true; 503 } else if (val != null && "suggestAlternative".equals(ai.key)) { 504 check.alternatives.add(val); 505 } else if (val != null && "assertMatch".equals(ai.key)) { 506 check.assertions.put(val, Boolean.TRUE); 507 } else if (val != null && "assertNoMatch".equals(ai.key)) { 508 check.assertions.put(val, Boolean.FALSE); 509 } else if (val != null && "group".equals(ai.key)) { 510 check.group = val; 511 } else if (ai.key.startsWith("-")) { 512 Logging.debug("Ignoring extension instruction: " + ai.key + ": " + ai.val); 513 } else { 514 throw new IllegalDataException("Cannot add instruction " + ai.key + ": " + ai.val + '!'); 515 } 516 } catch (IllegalArgumentException e) { 517 throw new IllegalDataException(e); 518 } 519 } 520 } 521 if (check.errors.isEmpty() && check.setClassExpressions.isEmpty()) { 522 throw new IllegalDataException( 523 "No "+POSSIBLE_THROWS+" given! You should specify a validation error message for " + rule.selectors); 524 } else if (check.errors.size() > 1) { 525 throw new IllegalDataException( 526 "More than one "+POSSIBLE_THROWS+" given! You should specify a single validation error message for " 527 + rule.selectors); 528 } 529 return check; 530 } 531 532 static ParseResult readMapCSS(Reader css) throws ParseException { 533 CheckParameterUtil.ensureParameterNotNull(css, "css"); 534 535 final MapCSSStyleSource source = new MapCSSStyleSource(""); 536 final MapCSSParser preprocessor = new MapCSSParser(css, MapCSSParser.LexicalState.PREPROCESSOR); 537 final StringReader mapcss = new StringReader(preprocessor.pp_root(source)); 538 final MapCSSParser parser = new MapCSSParser(mapcss, MapCSSParser.LexicalState.DEFAULT); 539 parser.sheet(source); 540 // Ignore "meta" rule(s) from external rules of JOSM wiki 541 source.removeMetaRules(); 542 // group rules with common declaration block 543 Map<Declaration, List<Selector>> g = new LinkedHashMap<>(); 544 for (MapCSSRule rule : source.rules) { 545 if (!g.containsKey(rule.declaration)) { 546 List<Selector> sels = new ArrayList<>(); 547 sels.add(rule.selector); 548 g.put(rule.declaration, sels); 549 } else { 550 g.get(rule.declaration).add(rule.selector); 551 } 552 } 553 List<TagCheck> parseChecks = new ArrayList<>(); 554 for (Map.Entry<Declaration, List<Selector>> map : g.entrySet()) { 555 try { 556 parseChecks.add(TagCheck.ofMapCSSRule( 557 new GroupedMapCSSRule(map.getValue(), map.getKey()))); 558 } catch (IllegalDataException e) { 559 Logging.error("Cannot add MapCss rule: "+e.getMessage()); 560 source.logError(e); 561 } 562 } 563 return new ParseResult(parseChecks, source.getErrors()); 564 } 565 566 @Override 567 public boolean test(OsmPrimitive primitive) { 568 // Tests whether the primitive contains a deprecated tag which is represented by this MapCSSTagChecker. 569 return whichSelectorMatchesPrimitive(primitive) != null; 570 } 571 572 Selector whichSelectorMatchesPrimitive(OsmPrimitive primitive) { 573 return whichSelectorMatchesEnvironment(new Environment(primitive)); 574 } 575 576 Selector whichSelectorMatchesEnvironment(Environment env) { 577 for (Selector i : rule.selectors) { 578 env.clearSelectorMatchingInformation(); 579 if (i.matches(env)) { 580 return i; 581 } 582 } 583 return null; 584 } 585 586 /** 587 * Determines the {@code index}-th key/value/tag (depending on {@code type}) of the 588 * {@link org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector}. 589 * @param matchingSelector matching selector 590 * @param index index 591 * @param type selector type ("key", "value" or "tag") 592 * @param p OSM primitive 593 * @return argument value, can be {@code null} 594 */ 595 static String determineArgument(OptimizedGeneralSelector matchingSelector, int index, String type, OsmPrimitive p) { 596 try { 597 final Condition c = matchingSelector.getConditions().get(index); 598 final Tag tag = c instanceof Condition.ToTagConvertable 599 ? ((Condition.ToTagConvertable) c).asTag(p) 600 : null; 601 if (tag == null) { 602 return null; 603 } else if ("key".equals(type)) { 604 return tag.getKey(); 605 } else if ("value".equals(type)) { 606 return tag.getValue(); 607 } else if ("tag".equals(type)) { 608 return tag.toString(); 609 } 610 } catch (IndexOutOfBoundsException ignore) { 611 Logging.debug(ignore); 612 } 613 return null; 614 } 615 616 /** 617 * Replaces occurrences of <code>{i.key}</code>, <code>{i.value}</code>, <code>{i.tag}</code> in {@code s} by the corresponding 618 * key/value/tag of the {@code index}-th {@link Condition} of {@code matchingSelector}. 619 * @param matchingSelector matching selector 620 * @param s any string 621 * @param p OSM primitive 622 * @return string with arguments inserted 623 */ 624 static String insertArguments(Selector matchingSelector, String s, OsmPrimitive p) { 625 if (s != null && matchingSelector instanceof Selector.ChildOrParentSelector) { 626 return insertArguments(((Selector.ChildOrParentSelector) matchingSelector).right, s, p); 627 } else if (s == null || !(matchingSelector instanceof Selector.OptimizedGeneralSelector)) { 628 return s; 629 } 630 final Matcher m = Pattern.compile("\\{(\\d+)\\.(key|value|tag)\\}").matcher(s); 631 final StringBuffer sb = new StringBuffer(); 632 while (m.find()) { 633 final String argument = determineArgument((Selector.OptimizedGeneralSelector) matchingSelector, 634 Integer.parseInt(m.group(1)), m.group(2), p); 635 try { 636 // Perform replacement with null-safe + regex-safe handling 637 m.appendReplacement(sb, String.valueOf(argument).replace("^(", "").replace(")$", "")); 638 } catch (IndexOutOfBoundsException | IllegalArgumentException e) { 639 Logging.log(Logging.LEVEL_ERROR, tr("Unable to replace argument {0} in {1}: {2}", argument, sb, e.getMessage()), e); 640 } 641 } 642 m.appendTail(sb); 643 return sb.toString(); 644 } 645 646 /** 647 * Constructs a fix in terms of a {@link org.openstreetmap.josm.command.Command} for the {@link OsmPrimitive} 648 * if the error is fixable, or {@code null} otherwise. 649 * 650 * @param p the primitive to construct the fix for 651 * @return the fix or {@code null} 652 */ 653 Command fixPrimitive(OsmPrimitive p) { 654 if (fixCommands.isEmpty() && !deletion) { 655 return null; 656 } 657 try { 658 final Selector matchingSelector = whichSelectorMatchesPrimitive(p); 659 Collection<Command> cmds = new LinkedList<>(); 660 for (FixCommand fixCommand : fixCommands) { 661 cmds.add(fixCommand.createCommand(p, matchingSelector)); 662 } 663 if (deletion && !p.isDeleted()) { 664 cmds.add(new DeleteCommand(p)); 665 } 666 return new SequenceCommand(tr("Fix of {0}", getDescriptionForMatchingSelector(p, matchingSelector)), cmds); 667 } catch (IllegalArgumentException e) { 668 Logging.error(e); 669 return null; 670 } 671 } 672 673 /** 674 * Constructs a (localized) message for this deprecation check. 675 * @param p OSM primitive 676 * 677 * @return a message 678 */ 679 String getMessage(OsmPrimitive p) { 680 if (errors.isEmpty()) { 681 // Return something to avoid NPEs 682 return rule.declaration.toString(); 683 } else { 684 final Object val = errors.keySet().iterator().next().val; 685 return String.valueOf( 686 val instanceof Expression 687 ? ((Expression) val).evaluate(new Environment(p)) 688 : val 689 ); 690 } 691 } 692 693 /** 694 * Constructs a (localized) description for this deprecation check. 695 * @param p OSM primitive 696 * 697 * @return a description (possibly with alternative suggestions) 698 * @see #getDescriptionForMatchingSelector 699 */ 700 String getDescription(OsmPrimitive p) { 701 if (alternatives.isEmpty()) { 702 return getMessage(p); 703 } else { 704 /* I18N: {0} is the test error message and {1} is an alternative */ 705 return tr("{0}, use {1} instead", getMessage(p), Utils.join(tr(" or "), alternatives)); 706 } 707 } 708 709 /** 710 * Constructs a (localized) description for this deprecation check 711 * where any placeholders are replaced by values of the matched selector. 712 * 713 * @param matchingSelector matching selector 714 * @param p OSM primitive 715 * @return a description (possibly with alternative suggestions) 716 */ 717 String getDescriptionForMatchingSelector(OsmPrimitive p, Selector matchingSelector) { 718 return insertArguments(matchingSelector, getDescription(p), p); 719 } 720 721 Severity getSeverity() { 722 return errors.isEmpty() ? null : errors.values().iterator().next(); 723 } 724 725 @Override 726 public String toString() { 727 return getDescription(null); 728 } 729 730 /** 731 * Constructs a {@link TestError} for the given primitive, or returns null if the primitive does not give rise to an error. 732 * 733 * @param p the primitive to construct the error for 734 * @return an instance of {@link TestError}, or returns null if the primitive does not give rise to an error. 735 */ 736 TestError getErrorForPrimitive(OsmPrimitive p) { 737 final Environment env = new Environment(p); 738 return getErrorForPrimitive(p, whichSelectorMatchesEnvironment(env), env, null); 739 } 740 741 TestError getErrorForPrimitive(OsmPrimitive p, Selector matchingSelector, Environment env, Test tester) { 742 if (matchingSelector != null && !errors.isEmpty()) { 743 final Command fix = fixPrimitive(p); 744 final String description = getDescriptionForMatchingSelector(p, matchingSelector); 745 final String description1 = group == null ? description : group; 746 final String description2 = group == null ? null : description; 747 final List<OsmPrimitive> primitives; 748 if (env.child instanceof OsmPrimitive) { 749 primitives = Arrays.asList(p, (OsmPrimitive) env.child); 750 } else { 751 primitives = Collections.singletonList(p); 752 } 753 final TestError.Builder error = TestError.builder(tester, getSeverity(), 3000) 754 .messageWithManuallyTranslatedDescription(description1, description2, matchingSelector.toString()) 755 .primitives(primitives); 756 if (fix != null) { 757 return error.fix(() -> fix).build(); 758 } else { 759 return error.build(); 760 } 761 } else { 762 return null; 763 } 764 } 765 766 /** 767 * Returns the set of tagchecks on which this check depends on. 768 * @param schecks the collection of tagcheks to search in 769 * @return the set of tagchecks on which this check depends on 770 * @since 7881 771 */ 772 public Set<TagCheck> getTagCheckDependencies(Collection<TagCheck> schecks) { 773 Set<TagCheck> result = new HashSet<>(); 774 Set<String> classes = getClassesIds(); 775 if (schecks != null && !classes.isEmpty()) { 776 for (TagCheck tc : schecks) { 777 if (this.equals(tc)) { 778 continue; 779 } 780 for (String id : tc.setClassExpressions) { 781 if (classes.contains(id)) { 782 result.add(tc); 783 break; 784 } 785 } 786 } 787 } 788 return result; 789 } 790 791 /** 792 * Returns the list of ids of all MapCSS classes referenced in the rule selectors. 793 * @return the list of ids of all MapCSS classes referenced in the rule selectors 794 * @since 7881 795 */ 796 public Set<String> getClassesIds() { 797 Set<String> result = new HashSet<>(); 798 for (Selector s : rule.selectors) { 799 if (s instanceof AbstractSelector) { 800 for (Condition c : ((AbstractSelector) s).getConditions()) { 801 if (c instanceof ClassCondition) { 802 result.add(((ClassCondition) c).id); 803 } 804 } 805 } 806 } 807 return result; 808 } 809 } 810 811 static class MapCSSTagCheckerAndRule extends MapCSSTagChecker { 812 public final GroupedMapCSSRule rule; 813 814 MapCSSTagCheckerAndRule(GroupedMapCSSRule rule) { 815 this.rule = rule; 816 } 817 818 @Override 819 public synchronized boolean equals(Object obj) { 820 return super.equals(obj) 821 || (obj instanceof TagCheck && rule.equals(((TagCheck) obj).rule)) 822 || (obj instanceof GroupedMapCSSRule && rule.equals(obj)); 823 } 824 825 @Override 826 public synchronized int hashCode() { 827 return Objects.hash(super.hashCode(), rule); 828 } 829 830 @Override 831 public String toString() { 832 return "MapCSSTagCheckerAndRule [rule=" + rule + ']'; 833 } 834 } 835 836 /** 837 * Obtains all {@link TestError}s for the {@link OsmPrimitive} {@code p}. 838 * @param p The OSM primitive 839 * @param includeOtherSeverity if {@code true}, errors of severity {@link Severity#OTHER} (info) will also be returned 840 * @return all errors for the given primitive, with or without those of "info" severity 841 */ 842 public synchronized Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity) { 843 final List<TestError> res = new ArrayList<>(); 844 if (indexData == null) 845 indexData = new IndexData(checks, includeOtherSeverity); 846 847 MapCSSRuleIndex matchingRuleIndex = indexData.get(p); 848 849 Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null); 850 // the declaration indices are sorted, so it suffices to save the last used index 851 Declaration lastDeclUsed = null; 852 853 Iterator<MapCSSRule> candidates = matchingRuleIndex.getRuleCandidates(p); 854 while (candidates.hasNext()) { 855 MapCSSRule r = candidates.next(); 856 env.clearSelectorMatchingInformation(); 857 if (r.selector.matches(env)) { // as side effect env.parent will be set (if s is a child selector) 858 TagCheck check = indexData.getCheck(r); 859 if (check != null) { 860 if (r.declaration == lastDeclUsed) 861 continue; // don't apply one declaration more than once 862 lastDeclUsed = r.declaration; 863 864 r.declaration.execute(env); 865 if (!check.errors.isEmpty()) { 866 final TestError error = check.getErrorForPrimitive(p, r.selector, env, new MapCSSTagCheckerAndRule(check.rule)); 867 if (error != null) { 868 res.add(error); 869 } 870 } 871 872 } 873 } 874 } 875 return res; 876 } 877 878 private static Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity, 879 Collection<Set<TagCheck>> checksCol) { 880 final List<TestError> r = new ArrayList<>(); 881 final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null); 882 for (Set<TagCheck> schecks : checksCol) { 883 for (TagCheck check : schecks) { 884 boolean ignoreError = Severity.OTHER == check.getSeverity() && !includeOtherSeverity; 885 // Do not run "information" level checks if not wanted, unless they also set a MapCSS class 886 if (ignoreError && check.setClassExpressions.isEmpty()) { 887 continue; 888 } 889 final Selector selector = check.whichSelectorMatchesEnvironment(env); 890 if (selector != null) { 891 check.rule.declaration.execute(env); 892 if (!ignoreError && !check.errors.isEmpty()) { 893 final TestError error = check.getErrorForPrimitive(p, selector, env, new MapCSSTagCheckerAndRule(check.rule)); 894 if (error != null) { 895 r.add(error); 896 } 897 } 898 } 899 } 900 } 901 return r; 902 } 903 904 /** 905 * Visiting call for primitives. 906 * 907 * @param p The primitive to inspect. 908 */ 909 @Override 910 public void check(OsmPrimitive p) { 911 errors.addAll(getErrorsForPrimitive(p, ValidatorPrefHelper.PREF_OTHER.get())); 912 } 913 914 /** 915 * Adds a new MapCSS config file from the given URL. 916 * @param url The unique URL of the MapCSS config file 917 * @return List of tag checks and parsing errors, or null 918 * @throws ParseException if the config file does not match MapCSS syntax 919 * @throws IOException if any I/O error occurs 920 * @since 7275 921 */ 922 public synchronized ParseResult addMapCSS(String url) throws ParseException, IOException { 923 CheckParameterUtil.ensureParameterNotNull(url, "url"); 924 ParseResult result; 925 try (CachedFile cache = new CachedFile(url); 926 InputStream zip = cache.findZipEntryInputStream("validator.mapcss", ""); 927 InputStream s = zip != null ? zip : cache.getInputStream(); 928 Reader reader = new BufferedReader(UTFInputStreamReader.create(s))) { 929 if (zip != null) 930 I18n.addTexts(cache.getFile()); 931 result = TagCheck.readMapCSS(reader); 932 checks.remove(url); 933 checks.putAll(url, result.parseChecks); 934 indexData = null; 935 // Check assertions, useful for development of local files 936 if (Config.getPref().getBoolean("validator.check_assert_local_rules", false) && Utils.isLocalUrl(url)) { 937 for (String msg : checkAsserts(result.parseChecks)) { 938 Logging.warn(msg); 939 } 940 } 941 } 942 return result; 943 } 944 945 @Override 946 public synchronized void initialize() throws Exception { 947 checks.clear(); 948 indexData = null; 949 for (SourceEntry source : new ValidatorPrefHelper().get()) { 950 if (!source.active) { 951 continue; 952 } 953 String i = source.url; 954 try { 955 if (!i.startsWith("resource:")) { 956 Logging.info(tr("Adding {0} to tag checker", i)); 957 } else if (Logging.isDebugEnabled()) { 958 Logging.debug(tr("Adding {0} to tag checker", i)); 959 } 960 addMapCSS(i); 961 if (Config.getPref().getBoolean("validator.auto_reload_local_rules", true) && source.isLocal()) { 962 FileWatcher.getDefaultInstance().registerSource(source); 963 } 964 } catch (IOException | IllegalStateException | IllegalArgumentException ex) { 965 Logging.warn(tr("Failed to add {0} to tag checker", i)); 966 Logging.log(Logging.LEVEL_WARN, ex); 967 } catch (ParseException | TokenMgrError ex) { 968 Logging.warn(tr("Failed to add {0} to tag checker", i)); 969 Logging.warn(ex); 970 } 971 } 972 } 973 974 private static Method getFunctionMethod(String method) { 975 try { 976 return Functions.class.getDeclaredMethod(method, Environment.class, String.class); 977 } catch (NoSuchMethodException | SecurityException e) { 978 Logging.error(e); 979 return null; 980 } 981 } 982 983 private static Optional<String> getFirstInsideCountry(TagCheck check, Method insideMethod) { 984 return check.rule.selectors.stream() 985 .filter(s -> s instanceof GeneralSelector) 986 .flatMap(s -> ((GeneralSelector) s).getConditions().stream()) 987 .filter(c -> c instanceof ExpressionCondition) 988 .map(c -> ((ExpressionCondition) c).getExpression()) 989 .filter(c -> c instanceof ParameterFunction) 990 .map(c -> (ParameterFunction) c) 991 .filter(c -> c.getMethod().equals(insideMethod)) 992 .flatMap(c -> c.getArgs().stream()) 993 .filter(e -> e instanceof LiteralExpression) 994 .map(e -> ((LiteralExpression) e).getLiteral()) 995 .filter(l -> l instanceof String) 996 .map(l -> ((String) l).split(",")[0]) 997 .findFirst(); 998 } 999 1000 private static LatLon getLocation(TagCheck check, Method insideMethod) { 1001 Optional<String> inside = getFirstInsideCountry(check, insideMethod); 1002 if (inside.isPresent()) { 1003 GeoPropertyIndex<Boolean> index = Territories.getGeoPropertyIndex(inside.get()); 1004 if (index != null) { 1005 GeoProperty<Boolean> prop = index.getGeoProperty(); 1006 if (prop instanceof DefaultGeoProperty) { 1007 Rectangle bounds = ((DefaultGeoProperty) prop).getArea().getBounds(); 1008 return new LatLon(bounds.getCenterY(), bounds.getCenterX()); 1009 } 1010 } 1011 } 1012 return LatLon.ZERO; 1013 } 1014 1015 /** 1016 * Checks that rule assertions are met for the given set of TagChecks. 1017 * @param schecks The TagChecks for which assertions have to be checked 1018 * @return A set of error messages, empty if all assertions are met 1019 * @since 7356 1020 */ 1021 public Set<String> checkAsserts(final Collection<TagCheck> schecks) { 1022 Set<String> assertionErrors = new LinkedHashSet<>(); 1023 final Method insideMethod = getFunctionMethod("inside"); 1024 final DataSet ds = new DataSet(); 1025 for (final TagCheck check : schecks) { 1026 Logging.debug("Check: {0}", check); 1027 for (final Map.Entry<String, Boolean> i : check.assertions.entrySet()) { 1028 Logging.debug("- Assertion: {0}", i); 1029 final OsmPrimitive p = OsmUtils.createPrimitive(i.getKey(), getLocation(check, insideMethod), true); 1030 // Build minimal ordered list of checks to run to test the assertion 1031 List<Set<TagCheck>> checksToRun = new ArrayList<>(); 1032 Set<TagCheck> checkDependencies = check.getTagCheckDependencies(schecks); 1033 if (!checkDependencies.isEmpty()) { 1034 checksToRun.add(checkDependencies); 1035 } 1036 checksToRun.add(Collections.singleton(check)); 1037 // Add primitive to dataset to avoid DataIntegrityProblemException when evaluating selectors 1038 addPrimitive(ds, p); 1039 final Collection<TestError> pErrors = getErrorsForPrimitive(p, true, checksToRun); 1040 Logging.debug("- Errors: {0}", pErrors); 1041 @SuppressWarnings({"EqualsBetweenInconvertibleTypes", "EqualsIncompatibleType"}) 1042 final boolean isError = pErrors.stream().anyMatch(e -> e.getTester().equals(check.rule)); 1043 if (isError != i.getValue()) { 1044 final String error = MessageFormat.format("Expecting test ''{0}'' (i.e., {1}) to {2} {3} (i.e., {4})", 1045 check.getMessage(p), check.rule.selectors, i.getValue() ? "match" : "not match", i.getKey(), p.getKeys()); 1046 assertionErrors.add(error); 1047 } 1048 ds.removePrimitive(p); 1049 } 1050 } 1051 return assertionErrors; 1052 } 1053 1054 private static void addPrimitive(DataSet ds, OsmPrimitive p) { 1055 if (p instanceof Way) { 1056 ((Way) p).getNodes().forEach(n -> addPrimitive(ds, n)); 1057 } else if (p instanceof Relation) { 1058 ((Relation) p).getMembers().forEach(m -> addPrimitive(ds, m.getMember())); 1059 } 1060 ds.addPrimitive(p); 1061 } 1062 1063 @Override 1064 public synchronized int hashCode() { 1065 return Objects.hash(super.hashCode(), checks); 1066 } 1067 1068 @Override 1069 public synchronized boolean equals(Object obj) { 1070 if (this == obj) return true; 1071 if (obj == null || getClass() != obj.getClass()) return false; 1072 if (!super.equals(obj)) return false; 1073 MapCSSTagChecker that = (MapCSSTagChecker) obj; 1074 return Objects.equals(checks, that.checks); 1075 } 1076 1077 /** 1078 * Reload tagchecker rule. 1079 * @param rule tagchecker rule to reload 1080 * @since 12825 1081 */ 1082 public static void reloadRule(SourceEntry rule) { 1083 MapCSSTagChecker tagChecker = OsmValidator.getTest(MapCSSTagChecker.class); 1084 if (tagChecker != null) { 1085 try { 1086 tagChecker.addMapCSS(rule.url); 1087 } catch (IOException | ParseException | TokenMgrError e) { 1088 Logging.warn(e); 1089 } 1090 } 1091 } 1092 1093 @Override 1094 public void startTest(ProgressMonitor progressMonitor) { 1095 super.startTest(progressMonitor); 1096 super.setShowElements(true); 1097 if (indexData == null) { 1098 indexData = new IndexData(checks, ValidatorPrefHelper.PREF_OTHER.get()); 1099 } 1100 } 1101 1102 @Override 1103 public void endTest() { 1104 super.endTest(); 1105 // no need to keep the index, it is quickly build and doubles the memory needs 1106 indexData = null; 1107 } 1108 1109}