001// License: GPL. See LICENSE file for details.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.InputStream;
007import java.io.InputStreamReader;
008import java.text.MessageFormat;
009import java.util.ArrayList;
010import java.util.Collection;
011import java.util.List;
012import java.util.regex.Matcher;
013import java.util.regex.Pattern;
014
015import javax.xml.stream.Location;
016import javax.xml.stream.XMLInputFactory;
017import javax.xml.stream.XMLStreamConstants;
018import javax.xml.stream.XMLStreamException;
019import javax.xml.stream.XMLStreamReader;
020
021import org.openstreetmap.josm.Main;
022import org.openstreetmap.josm.data.Bounds;
023import org.openstreetmap.josm.data.DataSource;
024import org.openstreetmap.josm.data.coor.LatLon;
025import org.openstreetmap.josm.data.osm.Changeset;
026import org.openstreetmap.josm.data.osm.DataSet;
027import org.openstreetmap.josm.data.osm.Node;
028import org.openstreetmap.josm.data.osm.NodeData;
029import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
030import org.openstreetmap.josm.data.osm.PrimitiveData;
031import org.openstreetmap.josm.data.osm.Relation;
032import org.openstreetmap.josm.data.osm.RelationData;
033import org.openstreetmap.josm.data.osm.RelationMemberData;
034import org.openstreetmap.josm.data.osm.Tagged;
035import org.openstreetmap.josm.data.osm.User;
036import org.openstreetmap.josm.data.osm.Way;
037import org.openstreetmap.josm.data.osm.WayData;
038import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
039import org.openstreetmap.josm.gui.progress.ProgressMonitor;
040import org.openstreetmap.josm.tools.CheckParameterUtil;
041import org.openstreetmap.josm.tools.date.DateUtils;
042
043/**
044 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it.
045 *
046 * For each xml element, there is a dedicated method.
047 * The XMLStreamReader cursor points to the start of the element, when the method is
048 * entered, and it must point to the end of the same element, when it is exited.
049 */
050public class OsmReader extends AbstractReader {
051
052    protected XMLStreamReader parser;
053
054    protected boolean cancel;
055
056    /** Used by plugins to register themselves as data postprocessors. */
057    private static List<OsmServerReadPostprocessor> postprocessors;
058
059    /** register a new postprocessor */
060    public static void registerPostprocessor(OsmServerReadPostprocessor pp) {
061        if (postprocessors == null) {
062            postprocessors = new ArrayList<>();
063        }
064        postprocessors.add(pp);
065    }
066
067    /** deregister a postprocessor previously registered with registerPostprocessor */
068    public static void deregisterPostprocessor(OsmServerReadPostprocessor pp) {
069        if (postprocessors != null) {
070            postprocessors.remove(pp);
071        }
072    }
073
074    /**
075     * constructor (for private and subclasses use only)
076     *
077     * @see #parseDataSet(InputStream, ProgressMonitor)
078     */
079    protected OsmReader() {
080    }
081
082    protected void setParser(XMLStreamReader parser) {
083        this.parser = parser;
084    }
085
086    protected void throwException(String msg, Throwable th) throws XMLStreamException {
087        throw new OsmParsingException(msg, parser.getLocation(), th);
088    }
089
090    protected void throwException(String msg) throws XMLStreamException {
091        throw new OsmParsingException(msg, parser.getLocation());
092    }
093
094    protected void parse() throws XMLStreamException {
095        int event = parser.getEventType();
096        while (true) {
097            if (event == XMLStreamConstants.START_ELEMENT) {
098                parseRoot();
099            } else if (event == XMLStreamConstants.END_ELEMENT)
100                return;
101            if (parser.hasNext()) {
102                event = parser.next();
103            } else {
104                break;
105            }
106        }
107        parser.close();
108    }
109
110    protected void parseRoot() throws XMLStreamException {
111        if ("osm".equals(parser.getLocalName())) {
112            parseOsm();
113        } else {
114            parseUnknown();
115        }
116    }
117
118    private void parseOsm() throws XMLStreamException {
119        String v = parser.getAttributeValue(null, "version");
120        if (v == null) {
121            throwException(tr("Missing mandatory attribute ''{0}''.", "version"));
122        }
123        if (!"0.6".equals(v)) {
124            throwException(tr("Unsupported version: {0}", v));
125        }
126        ds.setVersion(v);
127        String upload = parser.getAttributeValue(null, "upload");
128        if (upload != null) {
129            ds.setUploadDiscouraged(!Boolean.parseBoolean(upload));
130        }
131        String generator = parser.getAttributeValue(null, "generator");
132        Long uploadChangesetId = null;
133        if (parser.getAttributeValue(null, "upload-changeset") != null) {
134            uploadChangesetId = getLong("upload-changeset");
135        }
136        while (true) {
137            int event = parser.next();
138
139            if (cancel) {
140                cancel = false;
141                throw new OsmParsingCanceledException(tr("Reading was canceled"), parser.getLocation());
142            }
143
144            if (event == XMLStreamConstants.START_ELEMENT) {
145                switch (parser.getLocalName()) {
146                case "bounds":
147                    parseBounds(generator);
148                    break;
149                case "node":
150                    parseNode();
151                    break;
152                case "way":
153                    parseWay();
154                    break;
155                case "relation":
156                    parseRelation();
157                    break;
158                case "changeset":
159                    parseChangeset(uploadChangesetId);
160                    break;
161                default:
162                    parseUnknown();
163                }
164            } else if (event == XMLStreamConstants.END_ELEMENT)
165                return;
166        }
167    }
168
169    private void parseBounds(String generator) throws XMLStreamException {
170        String minlon = parser.getAttributeValue(null, "minlon");
171        String minlat = parser.getAttributeValue(null, "minlat");
172        String maxlon = parser.getAttributeValue(null, "maxlon");
173        String maxlat = parser.getAttributeValue(null, "maxlat");
174        String origin = parser.getAttributeValue(null, "origin");
175        if (minlon != null && maxlon != null && minlat != null && maxlat != null) {
176            if (origin == null) {
177                origin = generator;
178            }
179            Bounds bounds = new Bounds(
180                    Double.parseDouble(minlat), Double.parseDouble(minlon),
181                    Double.parseDouble(maxlat), Double.parseDouble(maxlon));
182            if (bounds.isOutOfTheWorld()) {
183                Bounds copy = new Bounds(bounds);
184                bounds.normalize();
185                Main.info("Bbox " + copy + " is out of the world, normalized to " + bounds);
186            }
187            DataSource src = new DataSource(bounds, origin);
188            ds.dataSources.add(src);
189        } else {
190            throwException(tr(
191                    "Missing mandatory attributes on element ''bounds''. Got minlon=''{0}'',minlat=''{1}'',maxlon=''{3}'',maxlat=''{4}'', origin=''{5}''.",
192                    minlon, minlat, maxlon, maxlat, origin
193            ));
194        }
195        jumpToEnd();
196    }
197
198    protected Node parseNode() throws XMLStreamException {
199        NodeData nd = new NodeData();
200        String lat = parser.getAttributeValue(null, "lat");
201        String lon = parser.getAttributeValue(null, "lon");
202        if (lat != null && lon != null) {
203            nd.setCoor(new LatLon(Double.parseDouble(lat), Double.parseDouble(lon)));
204        }
205        readCommon(nd);
206        Node n = new Node(nd.getId(), nd.getVersion());
207        n.setVisible(nd.isVisible());
208        n.load(nd);
209        externalIdMap.put(nd.getPrimitiveId(), n);
210        while (true) {
211            int event = parser.next();
212            if (event == XMLStreamConstants.START_ELEMENT) {
213                if ("tag".equals(parser.getLocalName())) {
214                    parseTag(n);
215                } else {
216                    parseUnknown();
217                }
218            } else if (event == XMLStreamConstants.END_ELEMENT)
219                return n;
220        }
221    }
222
223    protected Way parseWay() throws XMLStreamException {
224        WayData wd = new WayData();
225        readCommon(wd);
226        Way w = new Way(wd.getId(), wd.getVersion());
227        w.setVisible(wd.isVisible());
228        w.load(wd);
229        externalIdMap.put(wd.getPrimitiveId(), w);
230
231        Collection<Long> nodeIds = new ArrayList<>();
232        while (true) {
233            int event = parser.next();
234            if (event == XMLStreamConstants.START_ELEMENT) {
235                switch (parser.getLocalName()) {
236                case "nd":
237                    nodeIds.add(parseWayNode(w));
238                    break;
239                case "tag":
240                    parseTag(w);
241                    break;
242                default:
243                    parseUnknown();
244                }
245            } else if (event == XMLStreamConstants.END_ELEMENT) {
246                break;
247            }
248        }
249        if (w.isDeleted() && !nodeIds.isEmpty()) {
250            Main.info(tr("Deleted way {0} contains nodes", w.getUniqueId()));
251            nodeIds = new ArrayList<>();
252        }
253        ways.put(wd.getUniqueId(), nodeIds);
254        return w;
255    }
256
257    private long parseWayNode(Way w) throws XMLStreamException {
258        if (parser.getAttributeValue(null, "ref") == null) {
259            throwException(
260                    tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", w.getUniqueId())
261            );
262        }
263        long id = getLong("ref");
264        if (id == 0) {
265            throwException(
266                    tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", id)
267            );
268        }
269        jumpToEnd();
270        return id;
271    }
272
273    protected Relation parseRelation() throws XMLStreamException {
274        RelationData rd = new RelationData();
275        readCommon(rd);
276        Relation r = new Relation(rd.getId(), rd.getVersion());
277        r.setVisible(rd.isVisible());
278        r.load(rd);
279        externalIdMap.put(rd.getPrimitiveId(), r);
280
281        Collection<RelationMemberData> members = new ArrayList<>();
282        while (true) {
283            int event = parser.next();
284            if (event == XMLStreamConstants.START_ELEMENT) {
285                switch (parser.getLocalName()) {
286                case "member":
287                    members.add(parseRelationMember(r));
288                    break;
289                case "tag":
290                    parseTag(r);
291                    break;
292                default:
293                    parseUnknown();
294                }
295            } else if (event == XMLStreamConstants.END_ELEMENT) {
296                break;
297            }
298        }
299        if (r.isDeleted() && !members.isEmpty()) {
300            Main.info(tr("Deleted relation {0} contains members", r.getUniqueId()));
301            members = new ArrayList<>();
302        }
303        relations.put(rd.getUniqueId(), members);
304        return r;
305    }
306
307    private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException {
308        String role = null;
309        OsmPrimitiveType type = null;
310        long id = 0;
311        String value = parser.getAttributeValue(null, "ref");
312        if (value == null) {
313            throwException(tr("Missing attribute ''ref'' on member in relation {0}.",r.getUniqueId()));
314        }
315        try {
316            id = Long.parseLong(value);
317        } catch(NumberFormatException e) {
318            throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(r.getUniqueId()),value), e);
319        }
320        value = parser.getAttributeValue(null, "type");
321        if (value == null) {
322            throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(id), Long.toString(r.getUniqueId())));
323        }
324        try {
325            type = OsmPrimitiveType.fromApiTypeName(value);
326        } catch(IllegalArgumentException e) {
327            throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.", Long.toString(id), Long.toString(r.getUniqueId()), value), e);
328        }
329        value = parser.getAttributeValue(null, "role");
330        role = value;
331
332        if (id == 0) {
333            throwException(tr("Incomplete <member> specification with ref=0"));
334        }
335        jumpToEnd();
336        return new RelationMemberData(role, type, id);
337    }
338
339    private void parseChangeset(Long uploadChangesetId) throws XMLStreamException {
340
341        Long id = null;
342        if (parser.getAttributeValue(null, "id") != null) {
343            id = getLong("id");
344        }
345        // Read changeset info if neither upload-changeset nor id are set, or if they are both set to the same value
346        if (id == uploadChangesetId || (id != null && id.equals(uploadChangesetId))) {
347            uploadChangeset = new Changeset(id != null ? id.intValue() : 0);
348            while (true) {
349                int event = parser.next();
350                if (event == XMLStreamConstants.START_ELEMENT) {
351                    if ("tag".equals(parser.getLocalName())) {
352                        parseTag(uploadChangeset);
353                    } else {
354                        parseUnknown();
355                    }
356                } else if (event == XMLStreamConstants.END_ELEMENT)
357                    return;
358            }
359        } else {
360            jumpToEnd(false);
361        }
362    }
363
364    private void parseTag(Tagged t) throws XMLStreamException {
365        String key = parser.getAttributeValue(null, "k");
366        String value = parser.getAttributeValue(null, "v");
367        if (key == null || value == null) {
368            throwException(tr("Missing key or value attribute in tag."));
369        }
370        t.put(key.intern(), value.intern());
371        jumpToEnd();
372    }
373
374    protected void parseUnknown(boolean printWarning) throws XMLStreamException {
375        if (printWarning) {
376            Main.info(tr("Undefined element ''{0}'' found in input stream. Skipping.", parser.getLocalName()));
377        }
378        while (true) {
379            int event = parser.next();
380            if (event == XMLStreamConstants.START_ELEMENT) {
381                parseUnknown(false); /* no more warning for inner elements */
382            } else if (event == XMLStreamConstants.END_ELEMENT)
383                return;
384        }
385    }
386
387    protected void parseUnknown() throws XMLStreamException {
388        parseUnknown(true);
389    }
390
391    /**
392     * When cursor is at the start of an element, moves it to the end tag of that element.
393     * Nested content is skipped.
394     *
395     * This is basically the same code as parseUnknown(), except for the warnings, which
396     * are displayed for inner elements and not at top level.
397     */
398    private void jumpToEnd(boolean printWarning) throws XMLStreamException {
399        while (true) {
400            int event = parser.next();
401            if (event == XMLStreamConstants.START_ELEMENT) {
402                parseUnknown(printWarning);
403            } else if (event == XMLStreamConstants.END_ELEMENT)
404                return;
405        }
406    }
407
408    private void jumpToEnd() throws XMLStreamException {
409        jumpToEnd(true);
410    }
411
412    private User createUser(String uid, String name) throws XMLStreamException {
413        if (uid == null) {
414            if (name == null)
415                return null;
416            return User.createLocalUser(name);
417        }
418        try {
419            long id = Long.parseLong(uid);
420            return User.createOsmUser(id, name);
421        } catch(NumberFormatException e) {
422            throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid), e);
423        }
424        return null;
425    }
426
427    /**
428     * Read out the common attributes and put them into current OsmPrimitive.
429     */
430    private void readCommon(PrimitiveData current) throws XMLStreamException {
431        current.setId(getLong("id"));
432        if (current.getUniqueId() == 0) {
433            throwException(tr("Illegal object with ID=0."));
434        }
435
436        String time = parser.getAttributeValue(null, "timestamp");
437        if (time != null && time.length() != 0) {
438            current.setTimestamp(DateUtils.fromString(time));
439        }
440
441        String user = parser.getAttributeValue(null, "user");
442        String uid = parser.getAttributeValue(null, "uid");
443        current.setUser(createUser(uid, user));
444
445        String visible = parser.getAttributeValue(null, "visible");
446        if (visible != null) {
447            current.setVisible(Boolean.parseBoolean(visible));
448        }
449
450        String versionString = parser.getAttributeValue(null, "version");
451        int version = 0;
452        if (versionString != null) {
453            try {
454                version = Integer.parseInt(versionString);
455            } catch(NumberFormatException e) {
456                throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.",
457                        Long.toString(current.getUniqueId()), versionString), e);
458            }
459            switch (ds.getVersion()) {
460            case "0.6":
461                if (version <= 0 && !current.isNew()) {
462                    throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.",
463                            Long.toString(current.getUniqueId()), versionString));
464                } else if (version < 0 && current.isNew()) {
465                    Main.warn(tr("Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.",
466                            current.getUniqueId(), version, 0, "0.6"));
467                    version = 0;
468                }
469                break;
470            default:
471                // should not happen. API version has been checked before
472                throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion()));
473            }
474        } else {
475            // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6
476            if (!current.isNew() && ds.getVersion() != null && "0.6".equals(ds.getVersion())) {
477                throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId())));
478            }
479        }
480        current.setVersion(version);
481
482        String action = parser.getAttributeValue(null, "action");
483        if (action == null) {
484            // do nothing
485        } else if ("delete".equals(action)) {
486            current.setDeleted(true);
487            current.setModified(current.isVisible());
488        } else if ("modify".equals(action)) {
489            current.setModified(true);
490        }
491
492        String v = parser.getAttributeValue(null, "changeset");
493        if (v == null) {
494            current.setChangesetId(0);
495        } else {
496            try {
497                current.setChangesetId(Integer.parseInt(v));
498            } catch (IllegalArgumentException e) {
499                Main.debug(e.getMessage());
500                if (current.isNew()) {
501                    // for a new primitive we just log a warning
502                    Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId()));
503                    current.setChangesetId(0);
504                } else {
505                    // for an existing primitive this is a problem
506                    throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v), e);
507                }
508            } catch (IllegalStateException e) {
509                // thrown for positive changeset id on new primitives
510                Main.info(e.getMessage());
511                current.setChangesetId(0);
512            }
513            if (current.getChangesetId() <= 0) {
514                if (current.isNew()) {
515                    // for a new primitive we just log a warning
516                    Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId()));
517                    current.setChangesetId(0);
518                } else {
519                    // for an existing primitive this is a problem
520                    throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
521                }
522            }
523        }
524    }
525
526    private long getLong(String name) throws XMLStreamException {
527        String value = parser.getAttributeValue(null, name);
528        if (value == null) {
529            throwException(tr("Missing required attribute ''{0}''.",name));
530        }
531        try {
532            return Long.parseLong(value);
533        } catch(NumberFormatException e) {
534            throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.",name, value), e);
535        }
536        return 0; // should not happen
537    }
538
539    private static class OsmParsingException extends XMLStreamException {
540
541        public OsmParsingException(String msg, Location location) {
542            super(msg); /* cannot use super(msg, location) because it messes with the message preventing localization */
543            this.location = location;
544        }
545
546        public OsmParsingException(String msg, Location location, Throwable th) {
547            super(msg, th);
548            this.location = location;
549        }
550
551        @Override
552        public String getMessage() {
553            String msg = super.getMessage();
554            if (msg == null) {
555                msg = getClass().getName();
556            }
557            if (getLocation() == null)
558                return msg;
559            msg += " " + tr("(at line {0}, column {1})", getLocation().getLineNumber(), getLocation().getColumnNumber());
560            int offset = getLocation().getCharacterOffset();
561            if (offset > -1) {
562                msg += ". "+ tr("{0} bytes have been read", offset);
563            }
564            return msg;
565        }
566    }
567
568    /**
569     * Exception thrown after user cancelation.
570     */
571    private static final class OsmParsingCanceledException extends OsmParsingException implements ImportCancelException {
572        /**
573         * Constructs a new {@code OsmParsingCanceledException}.
574         * @param msg The error message
575         * @param location The parser location
576         */
577        public OsmParsingCanceledException(String msg, Location location) {
578            super(msg, location);
579        }
580    }
581
582    protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
583        if (progressMonitor == null) {
584            progressMonitor = NullProgressMonitor.INSTANCE;
585        }
586        ProgressMonitor.CancelListener cancelListener = new ProgressMonitor.CancelListener() {
587            @Override public void operationCanceled() {
588                cancel = true;
589            }
590        };
591        progressMonitor.addCancelListener(cancelListener);
592        CheckParameterUtil.ensureParameterNotNull(source, "source");
593        try {
594            progressMonitor.beginTask(tr("Prepare OSM data...", 2));
595            progressMonitor.indeterminateSubTask(tr("Parsing OSM data..."));
596
597            try (InputStreamReader ir = UTFInputStreamReader.create(source)) {
598                XMLStreamReader parser = XMLInputFactory.newInstance().createXMLStreamReader(ir);
599                setParser(parser);
600                parse();
601            }
602            progressMonitor.worked(1);
603
604            progressMonitor.indeterminateSubTask(tr("Preparing data set..."));
605            prepareDataSet();
606            progressMonitor.worked(1);
607
608            // iterate over registered postprocessors and give them each a chance
609            // to modify the dataset we have just loaded.
610            if (postprocessors != null) {
611                for (OsmServerReadPostprocessor pp : postprocessors) {
612                    pp.postprocessDataSet(getDataSet(), progressMonitor);
613                }
614            }
615            return getDataSet();
616        } catch(IllegalDataException e) {
617            throw e;
618        } catch(OsmParsingException e) {
619            throw new IllegalDataException(e.getMessage(), e);
620        } catch(XMLStreamException e) {
621            String msg = e.getMessage();
622            Pattern p = Pattern.compile("Message: (.+)");
623            Matcher m = p.matcher(msg);
624            if (m.find()) {
625                msg = m.group(1);
626            }
627            if (e.getLocation() != null)
628                throw new IllegalDataException(tr("Line {0} column {1}: ", e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e);
629            else
630                throw new IllegalDataException(msg, e);
631        } catch(Exception e) {
632            throw new IllegalDataException(e);
633        } finally {
634            progressMonitor.finishTask();
635            progressMonitor.removeCancelListener(cancelListener);
636        }
637    }
638
639    /**
640     * Parse the given input source and return the dataset.
641     *
642     * @param source the source input stream. Must not be null.
643     * @param progressMonitor  the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
644     *
645     * @return the dataset with the parsed data
646     * @throws IllegalDataException thrown if the an error was found while parsing the data from the source
647     * @throws IllegalArgumentException thrown if source is null
648     */
649    public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
650        return new OsmReader().doParseDataSet(source, progressMonitor);
651    }
652}