001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.validation.tests;
003
004import static org.openstreetmap.josm.tools.I18n.marktr;
005import static org.openstreetmap.josm.tools.I18n.tr;
006
007import java.util.ArrayList;
008import java.util.Arrays;
009import java.util.Collection;
010import java.util.HashMap;
011import java.util.HashSet;
012import java.util.List;
013import java.util.Locale;
014import java.util.Map;
015import java.util.Map.Entry;
016import java.util.Objects;
017import java.util.Set;
018import java.util.stream.Collectors;
019import java.util.stream.Stream;
020
021import org.openstreetmap.josm.command.Command;
022import org.openstreetmap.josm.command.DeleteCommand;
023import org.openstreetmap.josm.data.coor.EastNorth;
024import org.openstreetmap.josm.data.coor.LatLon;
025import org.openstreetmap.josm.data.osm.Node;
026import org.openstreetmap.josm.data.osm.OsmPrimitive;
027import org.openstreetmap.josm.data.osm.Relation;
028import org.openstreetmap.josm.data.osm.RelationMember;
029import org.openstreetmap.josm.data.osm.TagMap;
030import org.openstreetmap.josm.data.osm.Way;
031import org.openstreetmap.josm.data.preferences.DoubleProperty;
032import org.openstreetmap.josm.data.validation.Severity;
033import org.openstreetmap.josm.data.validation.Test;
034import org.openstreetmap.josm.data.validation.TestError;
035import org.openstreetmap.josm.tools.Geometry;
036import org.openstreetmap.josm.tools.Logging;
037import org.openstreetmap.josm.tools.Pair;
038import org.openstreetmap.josm.tools.SubclassFilteredCollection;
039import org.openstreetmap.josm.tools.Territories;
040import org.openstreetmap.josm.tools.Utils;
041
042/**
043 * Performs validation tests on addresses (addr:housenumber) and associatedStreet relations.
044 * @since 5644
045 */
046public class Addresses extends Test {
047
048    protected static final int HOUSE_NUMBER_WITHOUT_STREET = 2601;
049    protected static final int DUPLICATE_HOUSE_NUMBER = 2602;
050    protected static final int MULTIPLE_STREET_NAMES = 2603;
051    protected static final int MULTIPLE_STREET_RELATIONS = 2604;
052    protected static final int HOUSE_NUMBER_TOO_FAR = 2605;
053    protected static final int OBSOLETE_RELATION = 2606;
054
055    protected static final DoubleProperty MAX_DUPLICATE_DISTANCE = new DoubleProperty("validator.addresses.max_duplicate_distance", 200.0);
056    protected static final DoubleProperty MAX_STREET_DISTANCE = new DoubleProperty("validator.addresses.max_street_distance", 200.0);
057
058    // CHECKSTYLE.OFF: SingleSpaceSeparator
059    protected static final String ADDR_HOUSE_NUMBER  = "addr:housenumber";
060    protected static final String ADDR_INTERPOLATION = "addr:interpolation";
061    protected static final String ADDR_NEIGHBOURHOOD = "addr:neighbourhood";
062    protected static final String ADDR_PLACE         = "addr:place";
063    protected static final String ADDR_STREET        = "addr:street";
064    protected static final String ADDR_CITY          = "addr:city";
065    protected static final String ADDR_UNIT          = "addr:unit";
066    protected static final String ADDR_FLATS         = "addr:flats";
067    protected static final String ADDR_HOUSE_NAME    = "addr:housename";
068    protected static final String ADDR_POSTCODE      = "addr:postcode";
069    protected static final String ASSOCIATED_STREET  = "associatedStreet";
070    // CHECKSTYLE.ON: SingleSpaceSeparator
071
072    private Map<String, Collection<OsmPrimitive>> knownAddresses;
073    private Set<String> ignoredAddresses;
074
075
076    /**
077     * Constructor
078     */
079    public Addresses() {
080        super(tr("Addresses"), tr("Checks for errors in addresses and associatedStreet relations."));
081    }
082
083    protected List<Relation> getAndCheckAssociatedStreets(OsmPrimitive p) {
084        final List<Relation> list = p.referrers(Relation.class)
085                .filter(r -> r.hasTag("type", ASSOCIATED_STREET))
086                .collect(Collectors.toList());
087        if (list.size() > 1) {
088            Severity level;
089            // warning level only if several relations have different names, see #10945
090            final String name = list.get(0).get("name");
091            if (name == null || SubclassFilteredCollection.filter(list, r -> r.hasTag("name", name)).size() < list.size()) {
092                level = Severity.WARNING;
093            } else {
094                level = Severity.OTHER;
095            }
096            List<OsmPrimitive> errorList = new ArrayList<>(list);
097            errorList.add(0, p);
098            errors.add(TestError.builder(this, level, MULTIPLE_STREET_RELATIONS)
099                    .message(tr("Multiple associatedStreet relations"))
100                    .primitives(errorList)
101                    .build());
102        }
103        return list;
104    }
105
106    protected void checkHouseNumbersWithoutStreet(OsmPrimitive p) {
107        List<Relation> associatedStreets = getAndCheckAssociatedStreets(p);
108        // Find house number without proper location
109        // (neither addr:street, associatedStreet, addr:place, addr:neighbourhood or addr:interpolation)
110        if (p.hasKey(ADDR_HOUSE_NUMBER) && !p.hasKey(ADDR_STREET, ADDR_PLACE, ADDR_NEIGHBOURHOOD)) {
111            for (Relation r : associatedStreets) {
112                if (r.hasTag("type", ASSOCIATED_STREET)) {
113                    return;
114                }
115            }
116            if (p.referrers(Way.class).anyMatch(w -> w.hasKey(ADDR_INTERPOLATION) && w.hasKey(ADDR_STREET))) {
117                return;
118            }
119            // No street found
120            errors.add(TestError.builder(this, Severity.WARNING, HOUSE_NUMBER_WITHOUT_STREET)
121                    .message(tr("House number without street"))
122                    .primitives(p)
123                    .build());
124        }
125    }
126
127    static boolean isPOI(OsmPrimitive p) {
128        return p.hasKey("shop", "amenity", "tourism", "leisure", "emergency", "craft", "office", "name");
129    }
130
131    static boolean hasAddress(OsmPrimitive p) {
132        return p.hasKey(ADDR_HOUSE_NUMBER) && p.hasKey(ADDR_STREET, ADDR_PLACE);
133    }
134
135    /**
136     * adds the OsmPrimitive to the address map if it complies to the restrictions
137     * @param p OsmPrimitive that has an address
138     */
139    private void collectAddress(OsmPrimitive p) {
140        if (!isPOI(p)) {
141            String simplifiedAddress = getSimplifiedAddress(p);
142            if (!ignoredAddresses.contains(simplifiedAddress)) {
143                knownAddresses.computeIfAbsent(simplifiedAddress, x -> new ArrayList<>()).add(p);
144            }
145        }
146    }
147
148    protected void initAddressMap(OsmPrimitive primitive) {
149        knownAddresses = new HashMap<>();
150        ignoredAddresses = new HashSet<>();
151        for (OsmPrimitive p : primitive.getDataSet().allNonDeletedPrimitives()) {
152            if (p instanceof Node && p.hasKey(ADDR_UNIT, ADDR_FLATS)) {
153                for (OsmPrimitive r : p.getReferrers()) {
154                    if (hasAddress(r)) {
155                        // ignore addresses of buildings that are connected to addr:unit nodes
156                        // it's quite reasonable that there are more buildings with this address
157                        String simplifiedAddress = getSimplifiedAddress(r);
158                        if (!ignoredAddresses.contains(simplifiedAddress)) {
159                            ignoredAddresses.add(simplifiedAddress);
160                        } else if (knownAddresses.containsKey(simplifiedAddress)) {
161                            knownAddresses.remove(simplifiedAddress);
162                        }
163                    }
164                }
165            }
166            if (hasAddress(p)) {
167                collectAddress(p);
168            }
169        }
170    }
171
172    @Override
173    public void endTest() {
174        knownAddresses = null;
175        ignoredAddresses = null;
176        super.endTest();
177    }
178
179    protected void checkForDuplicate(OsmPrimitive p) {
180        if (knownAddresses == null) {
181            initAddressMap(p);
182        }
183        if (!isPOI(p) && hasAddress(p)) {
184            String simplifiedAddress = getSimplifiedAddress(p);
185            if (ignoredAddresses.contains(simplifiedAddress)) {
186                return;
187            }
188            if (knownAddresses.containsKey(simplifiedAddress)) {
189                double maxDistance = MAX_DUPLICATE_DISTANCE.get();
190                for (OsmPrimitive p2 : knownAddresses.get(simplifiedAddress)) {
191                    if (p == p2) {
192                        continue;
193                    }
194                    Severity severityLevel;
195                    String city1 = p.get(ADDR_CITY);
196                    String city2 = p2.get(ADDR_CITY);
197                    double distance = getDistance(p, p2);
198                    if (city1 != null && city2 != null) {
199                        if (city1.equals(city2)) {
200                            if (!p.hasKey(ADDR_POSTCODE) || !p2.hasKey(ADDR_POSTCODE) || p.get(ADDR_POSTCODE).equals(p2.get(ADDR_POSTCODE))) {
201                                severityLevel = Severity.WARNING;
202                            } else {
203                                // address including city identical but postcode differs
204                                // most likely perfectly fine
205                                severityLevel = Severity.OTHER;
206                            }
207                        } else {
208                            // address differs only by city - notify if very close, otherwise ignore
209                            if (distance < maxDistance) {
210                                severityLevel = Severity.OTHER;
211                            } else {
212                                continue;
213                            }
214                        }
215                    } else {
216                        // at least one address has no city specified
217                        if (p.hasKey(ADDR_POSTCODE) && p2.hasKey(ADDR_POSTCODE) && p.get(ADDR_POSTCODE).equals(p2.get(ADDR_POSTCODE))) {
218                            // address including postcode identical
219                            severityLevel = Severity.WARNING;
220                        } else {
221                            // city/postcode unclear - warn if very close, otherwise only notify
222                            // TODO: get city from surrounding boundaries?
223                            if (distance < maxDistance) {
224                                severityLevel = Severity.WARNING;
225                            } else {
226                                severityLevel = Severity.OTHER;
227                            }
228                        }
229                    }
230                    errors.add(TestError.builder(this, severityLevel, DUPLICATE_HOUSE_NUMBER)
231                            .message(tr("Duplicate house numbers"), marktr("''{0}'' ({1}m)"), simplifiedAddress, (int) distance)
232                            .primitives(Arrays.asList(p, p2)).build());
233                }
234                knownAddresses.get(simplifiedAddress).remove(p); // otherwise we would get every warning two times
235            }
236        }
237    }
238
239    static String getSimplifiedAddress(OsmPrimitive p) {
240        String simplifiedStreetName = p.hasKey(ADDR_STREET) ? p.get(ADDR_STREET) : p.get(ADDR_PLACE);
241        // ignore whitespaces and dashes in street name, so that "Mozart-Gasse", "Mozart Gasse" and "Mozartgasse" are all seen as equal
242        return Utils.strip(Stream.of(
243                simplifiedStreetName.replaceAll("[ -]", ""),
244                p.get(ADDR_HOUSE_NUMBER),
245                p.get(ADDR_HOUSE_NAME),
246                p.get(ADDR_UNIT),
247                p.get(ADDR_FLATS))
248            .filter(Objects::nonNull)
249            .collect(Collectors.joining(" ")))
250                .toUpperCase(Locale.ENGLISH);
251    }
252
253    @Override
254    public void visit(Node n) {
255        checkHouseNumbersWithoutStreet(n);
256        checkForDuplicate(n);
257    }
258
259    @Override
260    public void visit(Way w) {
261        checkHouseNumbersWithoutStreet(w);
262        checkForDuplicate(w);
263    }
264
265    @Override
266    public void visit(Relation r) {
267        checkHouseNumbersWithoutStreet(r);
268        checkForDuplicate(r);
269        if (r.hasTag("type", ASSOCIATED_STREET)) {
270            checkIfObsolete(r);
271            // Used to count occurrences of each house number in order to find duplicates
272            Map<String, List<OsmPrimitive>> map = new HashMap<>();
273            // Used to detect different street names
274            String relationName = r.get("name");
275            Set<OsmPrimitive> wrongStreetNames = new HashSet<>();
276            // Used to check distance
277            Set<OsmPrimitive> houses = new HashSet<>();
278            Set<Way> street = new HashSet<>();
279            for (RelationMember m : r.getMembers()) {
280                String role = m.getRole();
281                OsmPrimitive p = m.getMember();
282                if ("house".equals(role)) {
283                    houses.add(p);
284                    String number = p.get(ADDR_HOUSE_NUMBER);
285                    if (number != null) {
286                        number = number.trim().toUpperCase(Locale.ENGLISH);
287                        List<OsmPrimitive> list = map.get(number);
288                        if (list == null) {
289                            list = new ArrayList<>();
290                            map.put(number, list);
291                        }
292                        list.add(p);
293                    }
294                    if (relationName != null && p.hasKey(ADDR_STREET) && !relationName.equals(p.get(ADDR_STREET))) {
295                        if (wrongStreetNames.isEmpty()) {
296                            wrongStreetNames.add(r);
297                        }
298                        wrongStreetNames.add(p);
299                    }
300                } else if ("street".equals(role)) {
301                    if (p instanceof Way) {
302                        street.add((Way) p);
303                    }
304                    if (relationName != null && p.hasTagDifferent("name", relationName)) {
305                        if (wrongStreetNames.isEmpty()) {
306                            wrongStreetNames.add(r);
307                        }
308                        wrongStreetNames.add(p);
309                    }
310                }
311            }
312            // Report duplicate house numbers
313            for (Entry<String, List<OsmPrimitive>> entry : map.entrySet()) {
314                List<OsmPrimitive> list = entry.getValue();
315                if (list.size() > 1) {
316                    errors.add(TestError.builder(this, Severity.WARNING, DUPLICATE_HOUSE_NUMBER)
317                            .message(tr("Duplicate house numbers"), marktr("House number ''{0}'' duplicated"), entry.getKey())
318                            .primitives(list)
319                            .build());
320                }
321            }
322            // Report wrong street names
323            if (!wrongStreetNames.isEmpty()) {
324                errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_STREET_NAMES)
325                        .message(tr("Multiple street names in relation"))
326                        .primitives(wrongStreetNames)
327                        .build());
328            }
329            // Report addresses too far away
330            if (!street.isEmpty()) {
331                for (OsmPrimitive house : houses) {
332                    if (house.isUsable()) {
333                        checkDistance(house, street);
334                    }
335                }
336            }
337        }
338    }
339
340    /**
341     * returns rough distance between two OsmPrimitives
342     * @param a primitive a
343     * @param b primitive b
344     * @return distance of center of bounding boxes in meters
345     */
346    static double getDistance(OsmPrimitive a, OsmPrimitive b) {
347        LatLon centerA = a.getBBox().getCenter();
348        LatLon centerB = b.getBBox().getCenter();
349        return (centerA.greatCircleDistance(centerB));
350    }
351
352    protected void checkDistance(OsmPrimitive house, Collection<Way> street) {
353        EastNorth centroid;
354        if (house instanceof Node) {
355            centroid = ((Node) house).getEastNorth();
356        } else if (house instanceof Way) {
357            List<Node> nodes = ((Way) house).getNodes();
358            if (house.hasKey(ADDR_INTERPOLATION)) {
359                for (Node n : nodes) {
360                    if (n.hasKey(ADDR_HOUSE_NUMBER)) {
361                        checkDistance(n, street);
362                    }
363                }
364                return;
365            }
366            centroid = Geometry.getCentroid(nodes);
367        } else {
368            return; // TODO handle multipolygon houses ?
369        }
370        if (centroid == null) return; // fix #8305
371        double maxDistance = MAX_STREET_DISTANCE.get();
372        boolean hasIncompleteWays = false;
373        for (Way streetPart : street) {
374            for (Pair<Node, Node> chunk : streetPart.getNodePairs(false)) {
375                EastNorth p1 = chunk.a.getEastNorth();
376                EastNorth p2 = chunk.b.getEastNorth();
377                if (p1 != null && p2 != null) {
378                    EastNorth closest = Geometry.closestPointToSegment(p1, p2, centroid);
379                    if (closest.distance(centroid) <= maxDistance) {
380                        return;
381                    }
382                } else {
383                    Logging.warn("Addresses test skipped chunck "+chunk+" for street part "+streetPart+" because p1 or p2 is null");
384                }
385            }
386            if (!hasIncompleteWays && streetPart.isIncomplete()) {
387                hasIncompleteWays = true;
388            }
389        }
390        // No street segment found near this house, report error on if the relation does not contain incomplete street ways (fix #8314)
391        if (hasIncompleteWays) return;
392        List<OsmPrimitive> errorList = new ArrayList<>(street);
393        errorList.add(0, house);
394        errors.add(TestError.builder(this, Severity.WARNING, HOUSE_NUMBER_TOO_FAR)
395                .message(tr("House number too far from street"))
396                .primitives(errorList)
397                .build());
398    }
399
400    /**
401     * Check if an associatedStreet Relation is obsolete. This test marks only those relations which
402     * are complete and don't contain any information which isn't also tagged on the members.
403     * The strategy is to avoid any false positive.
404     * @param r the relation
405     */
406    private void checkIfObsolete(Relation r) {
407        if (r.isIncomplete())
408            return;
409        /** array of country codes for which the test should be performed. For now, only Germany */
410        String[] countryCodes = {"DE"};
411        TagMap neededtagsForHouse = new TagMap();
412        for (Entry<String, String> tag : r.getKeys().entrySet()) {
413            String key = tag.getKey();
414            if (key.startsWith("name:")) {
415                return; // maybe check if all members have corresponding tags?
416            } else if (key.startsWith("addr:")) {
417                neededtagsForHouse.put(key, tag.getValue());
418            } else {
419                switch (key) {
420                case "name":
421                case "type":
422                case "source":
423                    break;
424                default:
425                    // unexpected tag in relation
426                    return;
427                }
428            }
429        }
430
431        for (RelationMember m : r.getMembers()) {
432            if (m.getMember().isIncomplete() || !isInWarnCountry(m, countryCodes))
433                return;
434
435            String role = m.getRole();
436            if ("".equals(role)) {
437                if (m.isWay() && m.getMember().hasKey("highway")) {
438                    role = "street";
439                } else if (m.getMember().hasTag("building"))
440                    role = "house";
441            }
442            switch (role) {
443            case "house":
444            case "addr:houselink":
445            case "address":
446                if (!m.getMember().hasTag(ADDR_STREET) || !m.getMember().hasTag(ADDR_HOUSE_NUMBER))
447                    return;
448                for (Entry<String, String> tag : neededtagsForHouse.entrySet()) {
449                    if (!m.getMember().hasTag(tag.getKey(), tag.getValue()))
450                        return;
451                }
452                break;
453            case "street":
454                if (!m.getMember().hasTag("name") && r.hasTag("name"))
455                    return;
456                break;
457            default:
458                // unknown role: don't create auto-fix
459                return;
460            }
461        }
462        errors.add(TestError.builder(this, Severity.WARNING, OBSOLETE_RELATION)
463                .message(tr("Relation is obsolete"))
464                .primitives(r)
465                .build());
466    }
467
468    private static boolean isInWarnCountry(RelationMember m, String[] countryCodes) {
469        if (countryCodes.length == 0)
470            return true;
471        LatLon center = null;
472
473        if (m.isNode()) {
474            center = m.getNode().getCoor();
475        } else if (m.isWay()) {
476            center = m.getWay().getBBox().getCenter();
477        } else if (m.isRelation() && m.getRelation().isMultipolygon()) {
478            center = m.getRelation().getBBox().getCenter();
479        }
480        if (center == null)
481            return false;
482        for (String country : countryCodes) {
483            if (Territories.isIso3166Code(country, center))
484                return true;
485        }
486        return false;
487    }
488
489    /**
490     * remove obsolete relation.
491     */
492    @Override
493    public Command fixError(TestError testError) {
494        return new DeleteCommand(testError.getPrimitives());
495    }
496
497    @Override
498    public boolean isFixable(TestError testError) {
499        if (!(testError.getTester() instanceof Addresses))
500            return false;
501        return testError.getCode() == OBSOLETE_RELATION;
502    }
503
504}