001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.validation.tests;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.awt.Rectangle;
007import java.io.BufferedReader;
008import java.io.IOException;
009import java.io.InputStream;
010import java.io.Reader;
011import java.io.StringReader;
012import java.lang.reflect.Method;
013import java.text.MessageFormat;
014import java.util.ArrayList;
015import java.util.Arrays;
016import java.util.Collection;
017import java.util.Collections;
018import java.util.HashMap;
019import java.util.HashSet;
020import java.util.Iterator;
021import java.util.LinkedHashMap;
022import java.util.LinkedHashSet;
023import java.util.LinkedList;
024import java.util.List;
025import java.util.Locale;
026import java.util.Map;
027import java.util.Objects;
028import java.util.Optional;
029import java.util.Set;
030import java.util.function.Predicate;
031import java.util.regex.Matcher;
032import java.util.regex.Pattern;
033
034import org.openstreetmap.josm.command.ChangePropertyCommand;
035import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
036import org.openstreetmap.josm.command.Command;
037import org.openstreetmap.josm.command.DeleteCommand;
038import org.openstreetmap.josm.command.SequenceCommand;
039import org.openstreetmap.josm.data.coor.LatLon;
040import org.openstreetmap.josm.data.osm.DataSet;
041import org.openstreetmap.josm.data.osm.INode;
042import org.openstreetmap.josm.data.osm.IRelation;
043import org.openstreetmap.josm.data.osm.IWay;
044import org.openstreetmap.josm.data.osm.OsmPrimitive;
045import org.openstreetmap.josm.data.osm.OsmUtils;
046import org.openstreetmap.josm.data.osm.Relation;
047import org.openstreetmap.josm.data.osm.Tag;
048import org.openstreetmap.josm.data.osm.Way;
049import org.openstreetmap.josm.data.preferences.sources.SourceEntry;
050import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
051import org.openstreetmap.josm.data.validation.OsmValidator;
052import org.openstreetmap.josm.data.validation.Severity;
053import org.openstreetmap.josm.data.validation.Test;
054import org.openstreetmap.josm.data.validation.TestError;
055import org.openstreetmap.josm.gui.mappaint.Environment;
056import org.openstreetmap.josm.gui.mappaint.Keyword;
057import org.openstreetmap.josm.gui.mappaint.MultiCascade;
058import org.openstreetmap.josm.gui.mappaint.mapcss.Condition;
059import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ClassCondition;
060import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ExpressionCondition;
061import org.openstreetmap.josm.gui.mappaint.mapcss.Expression;
062import org.openstreetmap.josm.gui.mappaint.mapcss.ExpressionFactory.Functions;
063import org.openstreetmap.josm.gui.mappaint.mapcss.ExpressionFactory.ParameterFunction;
064import org.openstreetmap.josm.gui.mappaint.mapcss.Instruction;
065import org.openstreetmap.josm.gui.mappaint.mapcss.LiteralExpression;
066import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule;
067import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule.Declaration;
068import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource;
069import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource.MapCSSRuleIndex;
070import org.openstreetmap.josm.gui.mappaint.mapcss.Selector;
071import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.AbstractSelector;
072import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector;
073import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.OptimizedGeneralSelector;
074import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.MapCSSParser;
075import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException;
076import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.TokenMgrError;
077import org.openstreetmap.josm.gui.progress.ProgressMonitor;
078import org.openstreetmap.josm.io.CachedFile;
079import org.openstreetmap.josm.io.FileWatcher;
080import org.openstreetmap.josm.io.IllegalDataException;
081import org.openstreetmap.josm.io.UTFInputStreamReader;
082import org.openstreetmap.josm.spi.preferences.Config;
083import org.openstreetmap.josm.tools.CheckParameterUtil;
084import org.openstreetmap.josm.tools.DefaultGeoProperty;
085import org.openstreetmap.josm.tools.GeoProperty;
086import org.openstreetmap.josm.tools.GeoPropertyIndex;
087import org.openstreetmap.josm.tools.I18n;
088import org.openstreetmap.josm.tools.JosmRuntimeException;
089import org.openstreetmap.josm.tools.Logging;
090import org.openstreetmap.josm.tools.MultiMap;
091import org.openstreetmap.josm.tools.Territories;
092import org.openstreetmap.josm.tools.Utils;
093
094/**
095 * MapCSS-based tag checker/fixer.
096 * @since 6506
097 */
098public class MapCSSTagChecker extends Test.TagTest {
099    IndexData indexData;
100
101    /**
102     * Helper class to store indexes of rules.
103     * @author Gerd
104     *
105     */
106    private static final class IndexData {
107        final Map<MapCSSRule, TagCheck> ruleToCheckMap = new HashMap<>();
108
109        /**
110         * Rules for nodes
111         */
112        final MapCSSRuleIndex nodeRules = new MapCSSRuleIndex();
113        /**
114         * Rules for ways without tag area=no
115         */
116        final MapCSSRuleIndex wayRules = new MapCSSRuleIndex();
117        /**
118         * Rules for ways with tag area=no
119         */
120        final MapCSSRuleIndex wayNoAreaRules = new MapCSSRuleIndex();
121        /**
122         * Rules for relations that are not multipolygon relations
123         */
124        final MapCSSRuleIndex relationRules = new MapCSSRuleIndex();
125        /**
126         * Rules for multipolygon relations
127         */
128        final MapCSSRuleIndex multipolygonRules = new MapCSSRuleIndex();
129
130        private IndexData(MultiMap<String, TagCheck> checks, boolean includeOtherSeverity) {
131            buildIndex(checks, includeOtherSeverity);
132        }
133
134        private void buildIndex(MultiMap<String, TagCheck> checks, boolean includeOtherSeverity) {
135            List<TagCheck> allChecks = new ArrayList<>();
136            for (Set<TagCheck> cs : checks.values()) {
137                allChecks.addAll(cs);
138            }
139
140            ruleToCheckMap.clear();
141            nodeRules.clear();
142            wayRules.clear();
143            wayNoAreaRules.clear();
144            relationRules.clear();
145            multipolygonRules.clear();
146
147            // optimization: filter rules for different primitive types
148            for (TagCheck c : allChecks) {
149                if (!includeOtherSeverity && Severity.OTHER == c.getSeverity()
150                        && c.setClassExpressions.isEmpty()) {
151                    // Ignore "information" level checks if not wanted, unless they also set a MapCSS class
152                    continue;
153                }
154
155                for (Selector s : c.rule.selectors) {
156                    // find the rightmost selector, this must be a GeneralSelector
157                    Selector selRightmost = s;
158                    while (selRightmost instanceof Selector.ChildOrParentSelector) {
159                        selRightmost = ((Selector.ChildOrParentSelector) selRightmost).right;
160                    }
161                    MapCSSRule optRule = new MapCSSRule(s.optimizedBaseCheck(), c.rule.declaration);
162
163                    ruleToCheckMap.put(optRule, c);
164                    final String base = ((GeneralSelector) selRightmost).getBase();
165                    switch (base) {
166                    case Selector.BASE_NODE:
167                        nodeRules.add(optRule);
168                        break;
169                    case Selector.BASE_WAY:
170                        wayNoAreaRules.add(optRule);
171                        wayRules.add(optRule);
172                        break;
173                    case Selector.BASE_AREA:
174                        wayRules.add(optRule);
175                        multipolygonRules.add(optRule);
176                        break;
177                    case Selector.BASE_RELATION:
178                        relationRules.add(optRule);
179                        multipolygonRules.add(optRule);
180                        break;
181                    case Selector.BASE_ANY:
182                        nodeRules.add(optRule);
183                        wayRules.add(optRule);
184                        wayNoAreaRules.add(optRule);
185                        relationRules.add(optRule);
186                        multipolygonRules.add(optRule);
187                        break;
188                    case Selector.BASE_CANVAS:
189                    case Selector.BASE_META:
190                    case Selector.BASE_SETTING:
191                        break;
192                    default:
193                        final RuntimeException e = new JosmRuntimeException(MessageFormat.format("Unknown MapCSS base selector {0}", base));
194                        Logging.warn(tr("Failed to index validator rules. Error was: {0}", e.getMessage()));
195                        Logging.error(e);
196                    }
197                }
198            }
199            nodeRules.initIndex();
200            wayRules.initIndex();
201            wayNoAreaRules.initIndex();
202            relationRules.initIndex();
203            multipolygonRules.initIndex();
204        }
205
206        /**
207         * Get the index of rules for the given primitive.
208         * @param p the primitve
209         * @return index of rules for the given primitive
210         */
211        public MapCSSRuleIndex get(OsmPrimitive p) {
212            if (p instanceof INode) {
213                return nodeRules;
214            } else if (p instanceof IWay) {
215                if (OsmUtils.isFalse(p.get("area"))) {
216                    return wayNoAreaRules;
217                } else {
218                    return wayRules;
219                }
220            } else if (p instanceof IRelation) {
221                if (((IRelation<?>) p).isMultipolygon()) {
222                    return multipolygonRules;
223                } else {
224                    return relationRules;
225                }
226            } else {
227                throw new IllegalArgumentException("Unsupported type: " + p);
228            }
229        }
230
231        /**
232         * return the TagCheck for which the given indexed rule was created.
233         * @param rule an indexed rule
234         * @return the original TagCheck
235         */
236        public TagCheck getCheck(MapCSSRule rule) {
237            return ruleToCheckMap.get(rule);
238        }
239    }
240
241    /**
242    * A grouped MapCSSRule with multiple selectors for a single declaration.
243    * @see MapCSSRule
244    */
245    public static class GroupedMapCSSRule {
246        /** MapCSS selectors **/
247        public final List<Selector> selectors;
248        /** MapCSS declaration **/
249        public final Declaration declaration;
250
251        /**
252         * Constructs a new {@code GroupedMapCSSRule}.
253         * @param selectors MapCSS selectors
254         * @param declaration MapCSS declaration
255         */
256        public GroupedMapCSSRule(List<Selector> selectors, Declaration declaration) {
257            this.selectors = selectors;
258            this.declaration = declaration;
259        }
260
261        @Override
262        public int hashCode() {
263            return Objects.hash(selectors, declaration);
264        }
265
266        @Override
267        public boolean equals(Object obj) {
268            if (this == obj) return true;
269            if (obj == null || getClass() != obj.getClass()) return false;
270            GroupedMapCSSRule that = (GroupedMapCSSRule) obj;
271            return Objects.equals(selectors, that.selectors) &&
272                    Objects.equals(declaration, that.declaration);
273        }
274
275        @Override
276        public String toString() {
277            return "GroupedMapCSSRule [selectors=" + selectors + ", declaration=" + declaration + ']';
278        }
279    }
280
281    /**
282     * The preference key for tag checker source entries.
283     * @since 6670
284     */
285    public static final String ENTRIES_PREF_KEY = "validator." + MapCSSTagChecker.class.getName() + ".entries";
286
287    /**
288     * Constructs a new {@code MapCSSTagChecker}.
289     */
290    public MapCSSTagChecker() {
291        super(tr("Tag checker (MapCSS based)"), tr("This test checks for errors in tag keys and values."));
292    }
293
294    /**
295     * Represents a fix to a validation test. The fixing {@link Command} can be obtained by {@link #createCommand(OsmPrimitive, Selector)}.
296     */
297    @FunctionalInterface
298    interface FixCommand {
299        /**
300         * Creates the fixing {@link Command} for the given primitive. The {@code matchingSelector} is used to evaluate placeholders
301         * (cf. {@link MapCSSTagChecker.TagCheck#insertArguments(Selector, String, OsmPrimitive)}).
302         * @param p OSM primitive
303         * @param matchingSelector  matching selector
304         * @return fix command
305         */
306        Command createCommand(OsmPrimitive p, Selector matchingSelector);
307
308        /**
309         * Checks that object is either an {@link Expression} or a {@link String}.
310         * @param obj object to check
311         * @throws IllegalArgumentException if object is not an {@code Expression} or a {@code String}
312         */
313        static void checkObject(final Object obj) {
314            CheckParameterUtil.ensureThat(obj instanceof Expression || obj instanceof String,
315                    () -> "instance of Exception or String expected, but got " + obj);
316        }
317
318        /**
319         * Evaluates given object as {@link Expression} or {@link String} on the matched {@link OsmPrimitive} and {@code matchingSelector}.
320         * @param obj object to evaluate ({@link Expression} or {@link String})
321         * @param p OSM primitive
322         * @param matchingSelector matching selector
323         * @return result string
324         */
325        static String evaluateObject(final Object obj, final OsmPrimitive p, final Selector matchingSelector) {
326            final String s;
327            if (obj instanceof Expression) {
328                s = (String) ((Expression) obj).evaluate(new Environment(p));
329            } else if (obj instanceof String) {
330                s = (String) obj;
331            } else {
332                return null;
333            }
334            return TagCheck.insertArguments(matchingSelector, s, p);
335        }
336
337        /**
338         * Creates a fixing command which executes a {@link ChangePropertyCommand} on the specified tag.
339         * @param obj object to evaluate ({@link Expression} or {@link String})
340         * @return created fix command
341         */
342        static FixCommand fixAdd(final Object obj) {
343            checkObject(obj);
344            return new FixCommand() {
345                @Override
346                public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
347                    final Tag tag = Tag.ofString(evaluateObject(obj, p, matchingSelector));
348                    return new ChangePropertyCommand(p, tag.getKey(), tag.getValue());
349                }
350
351                @Override
352                public String toString() {
353                    return "fixAdd: " + obj;
354                }
355            };
356        }
357
358        /**
359         * Creates a fixing command which executes a {@link ChangePropertyCommand} to delete the specified key.
360         * @param obj object to evaluate ({@link Expression} or {@link String})
361         * @return created fix command
362         */
363        static FixCommand fixRemove(final Object obj) {
364            checkObject(obj);
365            return new FixCommand() {
366                @Override
367                public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
368                    final String key = evaluateObject(obj, p, matchingSelector);
369                    return new ChangePropertyCommand(p, key, "");
370                }
371
372                @Override
373                public String toString() {
374                    return "fixRemove: " + obj;
375                }
376            };
377        }
378
379        /**
380         * Creates a fixing command which executes a {@link ChangePropertyKeyCommand} on the specified keys.
381         * @param oldKey old key
382         * @param newKey new key
383         * @return created fix command
384         */
385        static FixCommand fixChangeKey(final String oldKey, final String newKey) {
386            return new FixCommand() {
387                @Override
388                public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
389                    return new ChangePropertyKeyCommand(p,
390                            TagCheck.insertArguments(matchingSelector, oldKey, p),
391                            TagCheck.insertArguments(matchingSelector, newKey, p));
392                }
393
394                @Override
395                public String toString() {
396                    return "fixChangeKey: " + oldKey + " => " + newKey;
397                }
398            };
399        }
400    }
401
402    final MultiMap<String, TagCheck> checks = new MultiMap<>();
403
404    /**
405     * Result of {@link TagCheck#readMapCSS}
406     * @since 8936
407     */
408    public static class ParseResult {
409        /** Checks successfully parsed */
410        public final List<TagCheck> parseChecks;
411        /** Errors that occurred during parsing */
412        public final Collection<Throwable> parseErrors;
413
414        /**
415         * Constructs a new {@code ParseResult}.
416         * @param parseChecks Checks successfully parsed
417         * @param parseErrors Errors that occurred during parsing
418         */
419        public ParseResult(List<TagCheck> parseChecks, Collection<Throwable> parseErrors) {
420            this.parseChecks = parseChecks;
421            this.parseErrors = parseErrors;
422        }
423    }
424
425    /**
426     * Tag check.
427     */
428    public static class TagCheck implements Predicate<OsmPrimitive> {
429        /** The selector of this {@code TagCheck} */
430        protected final GroupedMapCSSRule rule;
431        /** Commands to apply in order to fix a matching primitive */
432        protected final List<FixCommand> fixCommands = new ArrayList<>();
433        /** Tags (or arbitraty strings) of alternatives to be presented to the user */
434        protected final List<String> alternatives = new ArrayList<>();
435        /** An {@link org.openstreetmap.josm.gui.mappaint.mapcss.Instruction.AssignmentInstruction}-{@link Severity} pair.
436         * Is evaluated on the matching primitive to give the error message. Map is checked to contain exactly one element. */
437        protected final Map<Instruction.AssignmentInstruction, Severity> errors = new HashMap<>();
438        /** Unit tests */
439        protected final Map<String, Boolean> assertions = new HashMap<>();
440        /** MapCSS Classes to set on matching primitives */
441        protected final Set<String> setClassExpressions = new HashSet<>();
442        /** Denotes whether the object should be deleted for fixing it */
443        protected boolean deletion;
444        /** A string used to group similar tests */
445        protected String group;
446
447        TagCheck(GroupedMapCSSRule rule) {
448            this.rule = rule;
449        }
450
451        private static final String POSSIBLE_THROWS = possibleThrows();
452
453        static final String possibleThrows() {
454            StringBuilder sb = new StringBuilder();
455            for (Severity s : Severity.values()) {
456                if (sb.length() > 0) {
457                    sb.append('/');
458                }
459                sb.append("throw")
460                .append(s.name().charAt(0))
461                .append(s.name().substring(1).toLowerCase(Locale.ENGLISH));
462            }
463            return sb.toString();
464        }
465
466        static TagCheck ofMapCSSRule(final GroupedMapCSSRule rule) throws IllegalDataException {
467            final TagCheck check = new TagCheck(rule);
468            for (Instruction i : rule.declaration.instructions) {
469                if (i instanceof Instruction.AssignmentInstruction) {
470                    final Instruction.AssignmentInstruction ai = (Instruction.AssignmentInstruction) i;
471                    if (ai.isSetInstruction) {
472                        check.setClassExpressions.add(ai.key);
473                        continue;
474                    }
475                    try {
476                        final String val = ai.val instanceof Expression
477                                ? Optional.ofNullable(((Expression) ai.val).evaluate(new Environment())).map(Object::toString).orElse(null)
478                                : ai.val instanceof String
479                                ? (String) ai.val
480                                : ai.val instanceof Keyword
481                                ? ((Keyword) ai.val).val
482                                : null;
483                        if (ai.key.startsWith("throw")) {
484                            try {
485                                check.errors.put(ai, Severity.valueOf(ai.key.substring("throw".length()).toUpperCase(Locale.ENGLISH)));
486                            } catch (IllegalArgumentException e) {
487                                Logging.log(Logging.LEVEL_WARN,
488                                        "Unsupported "+ai.key+" instruction. Allowed instructions are "+POSSIBLE_THROWS+'.', e);
489                            }
490                        } else if ("fixAdd".equals(ai.key)) {
491                            check.fixCommands.add(FixCommand.fixAdd(ai.val));
492                        } else if ("fixRemove".equals(ai.key)) {
493                            CheckParameterUtil.ensureThat(!(ai.val instanceof String) || !(val != null && val.contains("=")),
494                                    "Unexpected '='. Please only specify the key to remove in: " + ai);
495                            check.fixCommands.add(FixCommand.fixRemove(ai.val));
496                        } else if (val != null && "fixChangeKey".equals(ai.key)) {
497                            CheckParameterUtil.ensureThat(val.contains("=>"), "Separate old from new key by '=>'!");
498                            final String[] x = val.split("=>", 2);
499                            check.fixCommands.add(FixCommand.fixChangeKey(Utils.removeWhiteSpaces(x[0]), Utils.removeWhiteSpaces(x[1])));
500                        } else if (val != null && "fixDeleteObject".equals(ai.key)) {
501                            CheckParameterUtil.ensureThat("this".equals(val), "fixDeleteObject must be followed by 'this'");
502                            check.deletion = true;
503                        } else if (val != null && "suggestAlternative".equals(ai.key)) {
504                            check.alternatives.add(val);
505                        } else if (val != null && "assertMatch".equals(ai.key)) {
506                            check.assertions.put(val, Boolean.TRUE);
507                        } else if (val != null && "assertNoMatch".equals(ai.key)) {
508                            check.assertions.put(val, Boolean.FALSE);
509                        } else if (val != null && "group".equals(ai.key)) {
510                            check.group = val;
511                        } else if (ai.key.startsWith("-")) {
512                            Logging.debug("Ignoring extension instruction: " + ai.key + ": " + ai.val);
513                        } else {
514                            throw new IllegalDataException("Cannot add instruction " + ai.key + ": " + ai.val + '!');
515                        }
516                    } catch (IllegalArgumentException e) {
517                        throw new IllegalDataException(e);
518                    }
519                }
520            }
521            if (check.errors.isEmpty() && check.setClassExpressions.isEmpty()) {
522                throw new IllegalDataException(
523                        "No "+POSSIBLE_THROWS+" given! You should specify a validation error message for " + rule.selectors);
524            } else if (check.errors.size() > 1) {
525                throw new IllegalDataException(
526                        "More than one "+POSSIBLE_THROWS+" given! You should specify a single validation error message for "
527                                + rule.selectors);
528            }
529            return check;
530        }
531
532        static ParseResult readMapCSS(Reader css) throws ParseException {
533            CheckParameterUtil.ensureParameterNotNull(css, "css");
534
535            final MapCSSStyleSource source = new MapCSSStyleSource("");
536            final MapCSSParser preprocessor = new MapCSSParser(css, MapCSSParser.LexicalState.PREPROCESSOR);
537            final StringReader mapcss = new StringReader(preprocessor.pp_root(source));
538            final MapCSSParser parser = new MapCSSParser(mapcss, MapCSSParser.LexicalState.DEFAULT);
539            parser.sheet(source);
540            // Ignore "meta" rule(s) from external rules of JOSM wiki
541            source.removeMetaRules();
542            // group rules with common declaration block
543            Map<Declaration, List<Selector>> g = new LinkedHashMap<>();
544            for (MapCSSRule rule : source.rules) {
545                if (!g.containsKey(rule.declaration)) {
546                    List<Selector> sels = new ArrayList<>();
547                    sels.add(rule.selector);
548                    g.put(rule.declaration, sels);
549                } else {
550                    g.get(rule.declaration).add(rule.selector);
551                }
552            }
553            List<TagCheck> parseChecks = new ArrayList<>();
554            for (Map.Entry<Declaration, List<Selector>> map : g.entrySet()) {
555                try {
556                    parseChecks.add(TagCheck.ofMapCSSRule(
557                            new GroupedMapCSSRule(map.getValue(), map.getKey())));
558                } catch (IllegalDataException e) {
559                    Logging.error("Cannot add MapCss rule: "+e.getMessage());
560                    source.logError(e);
561                }
562            }
563            return new ParseResult(parseChecks, source.getErrors());
564        }
565
566        @Override
567        public boolean test(OsmPrimitive primitive) {
568            // Tests whether the primitive contains a deprecated tag which is represented by this MapCSSTagChecker.
569            return whichSelectorMatchesPrimitive(primitive) != null;
570        }
571
572        Selector whichSelectorMatchesPrimitive(OsmPrimitive primitive) {
573            return whichSelectorMatchesEnvironment(new Environment(primitive));
574        }
575
576        Selector whichSelectorMatchesEnvironment(Environment env) {
577            for (Selector i : rule.selectors) {
578                env.clearSelectorMatchingInformation();
579                if (i.matches(env)) {
580                    return i;
581                }
582            }
583            return null;
584        }
585
586        /**
587         * Determines the {@code index}-th key/value/tag (depending on {@code type}) of the
588         * {@link org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector}.
589         * @param matchingSelector matching selector
590         * @param index index
591         * @param type selector type ("key", "value" or "tag")
592         * @param p OSM primitive
593         * @return argument value, can be {@code null}
594         */
595        static String determineArgument(OptimizedGeneralSelector matchingSelector, int index, String type, OsmPrimitive p) {
596            try {
597                final Condition c = matchingSelector.getConditions().get(index);
598                final Tag tag = c instanceof Condition.ToTagConvertable
599                        ? ((Condition.ToTagConvertable) c).asTag(p)
600                        : null;
601                if (tag == null) {
602                    return null;
603                } else if ("key".equals(type)) {
604                    return tag.getKey();
605                } else if ("value".equals(type)) {
606                    return tag.getValue();
607                } else if ("tag".equals(type)) {
608                    return tag.toString();
609                }
610            } catch (IndexOutOfBoundsException ignore) {
611                Logging.debug(ignore);
612            }
613            return null;
614        }
615
616        /**
617         * Replaces occurrences of <code>{i.key}</code>, <code>{i.value}</code>, <code>{i.tag}</code> in {@code s} by the corresponding
618         * key/value/tag of the {@code index}-th {@link Condition} of {@code matchingSelector}.
619         * @param matchingSelector matching selector
620         * @param s any string
621         * @param p OSM primitive
622         * @return string with arguments inserted
623         */
624        static String insertArguments(Selector matchingSelector, String s, OsmPrimitive p) {
625            if (s != null && matchingSelector instanceof Selector.ChildOrParentSelector) {
626                return insertArguments(((Selector.ChildOrParentSelector) matchingSelector).right, s, p);
627            } else if (s == null || !(matchingSelector instanceof Selector.OptimizedGeneralSelector)) {
628                return s;
629            }
630            final Matcher m = Pattern.compile("\\{(\\d+)\\.(key|value|tag)\\}").matcher(s);
631            final StringBuffer sb = new StringBuffer();
632            while (m.find()) {
633                final String argument = determineArgument((Selector.OptimizedGeneralSelector) matchingSelector,
634                        Integer.parseInt(m.group(1)), m.group(2), p);
635                try {
636                    // Perform replacement with null-safe + regex-safe handling
637                    m.appendReplacement(sb, String.valueOf(argument).replace("^(", "").replace(")$", ""));
638                } catch (IndexOutOfBoundsException | IllegalArgumentException e) {
639                    Logging.log(Logging.LEVEL_ERROR, tr("Unable to replace argument {0} in {1}: {2}", argument, sb, e.getMessage()), e);
640                }
641            }
642            m.appendTail(sb);
643            return sb.toString();
644        }
645
646        /**
647         * Constructs a fix in terms of a {@link org.openstreetmap.josm.command.Command} for the {@link OsmPrimitive}
648         * if the error is fixable, or {@code null} otherwise.
649         *
650         * @param p the primitive to construct the fix for
651         * @return the fix or {@code null}
652         */
653        Command fixPrimitive(OsmPrimitive p) {
654            if (fixCommands.isEmpty() && !deletion) {
655                return null;
656            }
657            try {
658                final Selector matchingSelector = whichSelectorMatchesPrimitive(p);
659                Collection<Command> cmds = new LinkedList<>();
660                for (FixCommand fixCommand : fixCommands) {
661                    cmds.add(fixCommand.createCommand(p, matchingSelector));
662                }
663                if (deletion && !p.isDeleted()) {
664                    cmds.add(new DeleteCommand(p));
665                }
666                return new SequenceCommand(tr("Fix of {0}", getDescriptionForMatchingSelector(p, matchingSelector)), cmds);
667            } catch (IllegalArgumentException e) {
668                Logging.error(e);
669                return null;
670            }
671        }
672
673        /**
674         * Constructs a (localized) message for this deprecation check.
675         * @param p OSM primitive
676         *
677         * @return a message
678         */
679        String getMessage(OsmPrimitive p) {
680            if (errors.isEmpty()) {
681                // Return something to avoid NPEs
682                return rule.declaration.toString();
683            } else {
684                final Object val = errors.keySet().iterator().next().val;
685                return String.valueOf(
686                        val instanceof Expression
687                                ? ((Expression) val).evaluate(new Environment(p))
688                                : val
689                );
690            }
691        }
692
693        /**
694         * Constructs a (localized) description for this deprecation check.
695         * @param p OSM primitive
696         *
697         * @return a description (possibly with alternative suggestions)
698         * @see #getDescriptionForMatchingSelector
699         */
700        String getDescription(OsmPrimitive p) {
701            if (alternatives.isEmpty()) {
702                return getMessage(p);
703            } else {
704                /* I18N: {0} is the test error message and {1} is an alternative */
705                return tr("{0}, use {1} instead", getMessage(p), Utils.join(tr(" or "), alternatives));
706            }
707        }
708
709        /**
710         * Constructs a (localized) description for this deprecation check
711         * where any placeholders are replaced by values of the matched selector.
712         *
713         * @param matchingSelector matching selector
714         * @param p OSM primitive
715         * @return a description (possibly with alternative suggestions)
716         */
717        String getDescriptionForMatchingSelector(OsmPrimitive p, Selector matchingSelector) {
718            return insertArguments(matchingSelector, getDescription(p), p);
719        }
720
721        Severity getSeverity() {
722            return errors.isEmpty() ? null : errors.values().iterator().next();
723        }
724
725        @Override
726        public String toString() {
727            return getDescription(null);
728        }
729
730        /**
731         * Constructs a {@link TestError} for the given primitive, or returns null if the primitive does not give rise to an error.
732         *
733         * @param p the primitive to construct the error for
734         * @return an instance of {@link TestError}, or returns null if the primitive does not give rise to an error.
735         */
736        TestError getErrorForPrimitive(OsmPrimitive p) {
737            final Environment env = new Environment(p);
738            return getErrorForPrimitive(p, whichSelectorMatchesEnvironment(env), env, null);
739        }
740
741        TestError getErrorForPrimitive(OsmPrimitive p, Selector matchingSelector, Environment env, Test tester) {
742            if (matchingSelector != null && !errors.isEmpty()) {
743                final Command fix = fixPrimitive(p);
744                final String description = getDescriptionForMatchingSelector(p, matchingSelector);
745                final String description1 = group == null ? description : group;
746                final String description2 = group == null ? null : description;
747                final List<OsmPrimitive> primitives;
748                if (env.child instanceof OsmPrimitive) {
749                    primitives = Arrays.asList(p, (OsmPrimitive) env.child);
750                } else {
751                    primitives = Collections.singletonList(p);
752                }
753                final TestError.Builder error = TestError.builder(tester, getSeverity(), 3000)
754                        .messageWithManuallyTranslatedDescription(description1, description2, matchingSelector.toString())
755                        .primitives(primitives);
756                if (fix != null) {
757                    return error.fix(() -> fix).build();
758                } else {
759                    return error.build();
760                }
761            } else {
762                return null;
763            }
764        }
765
766        /**
767         * Returns the set of tagchecks on which this check depends on.
768         * @param schecks the collection of tagcheks to search in
769         * @return the set of tagchecks on which this check depends on
770         * @since 7881
771         */
772        public Set<TagCheck> getTagCheckDependencies(Collection<TagCheck> schecks) {
773            Set<TagCheck> result = new HashSet<>();
774            Set<String> classes = getClassesIds();
775            if (schecks != null && !classes.isEmpty()) {
776                for (TagCheck tc : schecks) {
777                    if (this.equals(tc)) {
778                        continue;
779                    }
780                    for (String id : tc.setClassExpressions) {
781                        if (classes.contains(id)) {
782                            result.add(tc);
783                            break;
784                        }
785                    }
786                }
787            }
788            return result;
789        }
790
791        /**
792         * Returns the list of ids of all MapCSS classes referenced in the rule selectors.
793         * @return the list of ids of all MapCSS classes referenced in the rule selectors
794         * @since 7881
795         */
796        public Set<String> getClassesIds() {
797            Set<String> result = new HashSet<>();
798            for (Selector s : rule.selectors) {
799                if (s instanceof AbstractSelector) {
800                    for (Condition c : ((AbstractSelector) s).getConditions()) {
801                        if (c instanceof ClassCondition) {
802                            result.add(((ClassCondition) c).id);
803                        }
804                    }
805                }
806            }
807            return result;
808        }
809    }
810
811    static class MapCSSTagCheckerAndRule extends MapCSSTagChecker {
812        public final GroupedMapCSSRule rule;
813
814        MapCSSTagCheckerAndRule(GroupedMapCSSRule rule) {
815            this.rule = rule;
816        }
817
818        @Override
819        public synchronized boolean equals(Object obj) {
820            return super.equals(obj)
821                    || (obj instanceof TagCheck && rule.equals(((TagCheck) obj).rule))
822                    || (obj instanceof GroupedMapCSSRule && rule.equals(obj));
823        }
824
825        @Override
826        public synchronized int hashCode() {
827            return Objects.hash(super.hashCode(), rule);
828        }
829
830        @Override
831        public String toString() {
832            return "MapCSSTagCheckerAndRule [rule=" + rule + ']';
833        }
834    }
835
836    /**
837     * Obtains all {@link TestError}s for the {@link OsmPrimitive} {@code p}.
838     * @param p The OSM primitive
839     * @param includeOtherSeverity if {@code true}, errors of severity {@link Severity#OTHER} (info) will also be returned
840     * @return all errors for the given primitive, with or without those of "info" severity
841     */
842    public synchronized Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity) {
843        final List<TestError> res = new ArrayList<>();
844        if (indexData == null)
845            indexData = new IndexData(checks, includeOtherSeverity);
846
847        MapCSSRuleIndex matchingRuleIndex = indexData.get(p);
848
849        Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
850        // the declaration indices are sorted, so it suffices to save the last used index
851        Declaration lastDeclUsed = null;
852
853        Iterator<MapCSSRule> candidates = matchingRuleIndex.getRuleCandidates(p);
854        while (candidates.hasNext()) {
855            MapCSSRule r = candidates.next();
856            env.clearSelectorMatchingInformation();
857            if (r.selector.matches(env)) { // as side effect env.parent will be set (if s is a child selector)
858                TagCheck check = indexData.getCheck(r);
859                if (check != null) {
860                    if (r.declaration == lastDeclUsed)
861                        continue; // don't apply one declaration more than once
862                    lastDeclUsed = r.declaration;
863
864                    r.declaration.execute(env);
865                    if (!check.errors.isEmpty()) {
866                        final TestError error = check.getErrorForPrimitive(p, r.selector, env, new MapCSSTagCheckerAndRule(check.rule));
867                        if (error != null) {
868                            res.add(error);
869                        }
870                    }
871
872                }
873            }
874        }
875        return res;
876    }
877
878    private static Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity,
879            Collection<Set<TagCheck>> checksCol) {
880        final List<TestError> r = new ArrayList<>();
881        final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
882        for (Set<TagCheck> schecks : checksCol) {
883            for (TagCheck check : schecks) {
884                boolean ignoreError = Severity.OTHER == check.getSeverity() && !includeOtherSeverity;
885                // Do not run "information" level checks if not wanted, unless they also set a MapCSS class
886                if (ignoreError && check.setClassExpressions.isEmpty()) {
887                    continue;
888                }
889                final Selector selector = check.whichSelectorMatchesEnvironment(env);
890                if (selector != null) {
891                    check.rule.declaration.execute(env);
892                    if (!ignoreError && !check.errors.isEmpty()) {
893                        final TestError error = check.getErrorForPrimitive(p, selector, env, new MapCSSTagCheckerAndRule(check.rule));
894                        if (error != null) {
895                            r.add(error);
896                        }
897                    }
898                }
899            }
900        }
901        return r;
902    }
903
904    /**
905     * Visiting call for primitives.
906     *
907     * @param p The primitive to inspect.
908     */
909    @Override
910    public void check(OsmPrimitive p) {
911        errors.addAll(getErrorsForPrimitive(p, ValidatorPrefHelper.PREF_OTHER.get()));
912    }
913
914    /**
915     * Adds a new MapCSS config file from the given URL.
916     * @param url The unique URL of the MapCSS config file
917     * @return List of tag checks and parsing errors, or null
918     * @throws ParseException if the config file does not match MapCSS syntax
919     * @throws IOException if any I/O error occurs
920     * @since 7275
921     */
922    public synchronized ParseResult addMapCSS(String url) throws ParseException, IOException {
923        CheckParameterUtil.ensureParameterNotNull(url, "url");
924        ParseResult result;
925        try (CachedFile cache = new CachedFile(url);
926             InputStream zip = cache.findZipEntryInputStream("validator.mapcss", "");
927             InputStream s = zip != null ? zip : cache.getInputStream();
928             Reader reader = new BufferedReader(UTFInputStreamReader.create(s))) {
929            if (zip != null)
930                I18n.addTexts(cache.getFile());
931            result = TagCheck.readMapCSS(reader);
932            checks.remove(url);
933            checks.putAll(url, result.parseChecks);
934            indexData = null;
935            // Check assertions, useful for development of local files
936            if (Config.getPref().getBoolean("validator.check_assert_local_rules", false) && Utils.isLocalUrl(url)) {
937                for (String msg : checkAsserts(result.parseChecks)) {
938                    Logging.warn(msg);
939                }
940            }
941        }
942        return result;
943    }
944
945    @Override
946    public synchronized void initialize() throws Exception {
947        checks.clear();
948        indexData = null;
949        for (SourceEntry source : new ValidatorPrefHelper().get()) {
950            if (!source.active) {
951                continue;
952            }
953            String i = source.url;
954            try {
955                if (!i.startsWith("resource:")) {
956                    Logging.info(tr("Adding {0} to tag checker", i));
957                } else if (Logging.isDebugEnabled()) {
958                    Logging.debug(tr("Adding {0} to tag checker", i));
959                }
960                addMapCSS(i);
961                if (Config.getPref().getBoolean("validator.auto_reload_local_rules", true) && source.isLocal()) {
962                    FileWatcher.getDefaultInstance().registerSource(source);
963                }
964            } catch (IOException | IllegalStateException | IllegalArgumentException ex) {
965                Logging.warn(tr("Failed to add {0} to tag checker", i));
966                Logging.log(Logging.LEVEL_WARN, ex);
967            } catch (ParseException | TokenMgrError ex) {
968                Logging.warn(tr("Failed to add {0} to tag checker", i));
969                Logging.warn(ex);
970            }
971        }
972    }
973
974    private static Method getFunctionMethod(String method) {
975        try {
976            return Functions.class.getDeclaredMethod(method, Environment.class, String.class);
977        } catch (NoSuchMethodException | SecurityException e) {
978            Logging.error(e);
979            return null;
980        }
981    }
982
983    private static Optional<String> getFirstInsideCountry(TagCheck check, Method insideMethod) {
984        return check.rule.selectors.stream()
985                .filter(s -> s instanceof GeneralSelector)
986                .flatMap(s -> ((GeneralSelector) s).getConditions().stream())
987                .filter(c -> c instanceof ExpressionCondition)
988                .map(c -> ((ExpressionCondition) c).getExpression())
989                .filter(c -> c instanceof ParameterFunction)
990                .map(c -> (ParameterFunction) c)
991                .filter(c -> c.getMethod().equals(insideMethod))
992                .flatMap(c -> c.getArgs().stream())
993                .filter(e -> e instanceof LiteralExpression)
994                .map(e -> ((LiteralExpression) e).getLiteral())
995                .filter(l -> l instanceof String)
996                .map(l -> ((String) l).split(",")[0])
997                .findFirst();
998    }
999
1000    private static LatLon getLocation(TagCheck check, Method insideMethod) {
1001        Optional<String> inside = getFirstInsideCountry(check, insideMethod);
1002        if (inside.isPresent()) {
1003            GeoPropertyIndex<Boolean> index = Territories.getGeoPropertyIndex(inside.get());
1004            if (index != null) {
1005                GeoProperty<Boolean> prop = index.getGeoProperty();
1006                if (prop instanceof DefaultGeoProperty) {
1007                    Rectangle bounds = ((DefaultGeoProperty) prop).getArea().getBounds();
1008                    return new LatLon(bounds.getCenterY(), bounds.getCenterX());
1009                }
1010            }
1011        }
1012        return LatLon.ZERO;
1013    }
1014
1015    /**
1016     * Checks that rule assertions are met for the given set of TagChecks.
1017     * @param schecks The TagChecks for which assertions have to be checked
1018     * @return A set of error messages, empty if all assertions are met
1019     * @since 7356
1020     */
1021    public Set<String> checkAsserts(final Collection<TagCheck> schecks) {
1022        Set<String> assertionErrors = new LinkedHashSet<>();
1023        final Method insideMethod = getFunctionMethod("inside");
1024        final DataSet ds = new DataSet();
1025        for (final TagCheck check : schecks) {
1026            Logging.debug("Check: {0}", check);
1027            for (final Map.Entry<String, Boolean> i : check.assertions.entrySet()) {
1028                Logging.debug("- Assertion: {0}", i);
1029                final OsmPrimitive p = OsmUtils.createPrimitive(i.getKey(), getLocation(check, insideMethod), true);
1030                // Build minimal ordered list of checks to run to test the assertion
1031                List<Set<TagCheck>> checksToRun = new ArrayList<>();
1032                Set<TagCheck> checkDependencies = check.getTagCheckDependencies(schecks);
1033                if (!checkDependencies.isEmpty()) {
1034                    checksToRun.add(checkDependencies);
1035                }
1036                checksToRun.add(Collections.singleton(check));
1037                // Add primitive to dataset to avoid DataIntegrityProblemException when evaluating selectors
1038                addPrimitive(ds, p);
1039                final Collection<TestError> pErrors = getErrorsForPrimitive(p, true, checksToRun);
1040                Logging.debug("- Errors: {0}", pErrors);
1041                @SuppressWarnings({"EqualsBetweenInconvertibleTypes", "EqualsIncompatibleType"})
1042                final boolean isError = pErrors.stream().anyMatch(e -> e.getTester().equals(check.rule));
1043                if (isError != i.getValue()) {
1044                    final String error = MessageFormat.format("Expecting test ''{0}'' (i.e., {1}) to {2} {3} (i.e., {4})",
1045                            check.getMessage(p), check.rule.selectors, i.getValue() ? "match" : "not match", i.getKey(), p.getKeys());
1046                    assertionErrors.add(error);
1047                }
1048                ds.removePrimitive(p);
1049            }
1050        }
1051        return assertionErrors;
1052    }
1053
1054    private static void addPrimitive(DataSet ds, OsmPrimitive p) {
1055        if (p instanceof Way) {
1056            ((Way) p).getNodes().forEach(n -> addPrimitive(ds, n));
1057        } else if (p instanceof Relation) {
1058            ((Relation) p).getMembers().forEach(m -> addPrimitive(ds, m.getMember()));
1059        }
1060        ds.addPrimitive(p);
1061    }
1062
1063    @Override
1064    public synchronized int hashCode() {
1065        return Objects.hash(super.hashCode(), checks);
1066    }
1067
1068    @Override
1069    public synchronized boolean equals(Object obj) {
1070        if (this == obj) return true;
1071        if (obj == null || getClass() != obj.getClass()) return false;
1072        if (!super.equals(obj)) return false;
1073        MapCSSTagChecker that = (MapCSSTagChecker) obj;
1074        return Objects.equals(checks, that.checks);
1075    }
1076
1077    /**
1078     * Reload tagchecker rule.
1079     * @param rule tagchecker rule to reload
1080     * @since 12825
1081     */
1082    public static void reloadRule(SourceEntry rule) {
1083        MapCSSTagChecker tagChecker = OsmValidator.getTest(MapCSSTagChecker.class);
1084        if (tagChecker != null) {
1085            try {
1086                tagChecker.addMapCSS(rule.url);
1087            } catch (IOException | ParseException | TokenMgrError e) {
1088                Logging.warn(e);
1089            }
1090        }
1091    }
1092
1093    @Override
1094    public void startTest(ProgressMonitor progressMonitor) {
1095        super.startTest(progressMonitor);
1096        super.setShowElements(true);
1097        if (indexData == null) {
1098            indexData = new IndexData(checks, ValidatorPrefHelper.PREF_OTHER.get());
1099        }
1100    }
1101
1102    @Override
1103    public void endTest() {
1104        super.endTest();
1105        // no need to keep the index, it is quickly build and doubles the memory needs
1106        indexData = null;
1107    }
1108
1109}