001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.IOException; 007import java.io.InputStream; 008import java.nio.charset.StandardCharsets; 009import java.time.Duration; 010import java.time.LocalDateTime; 011import java.time.Period; 012import java.time.ZoneOffset; 013import java.util.Arrays; 014import java.util.EnumMap; 015import java.util.List; 016import java.util.Locale; 017import java.util.Map; 018import java.util.NoSuchElementException; 019import java.util.Objects; 020import java.util.concurrent.ConcurrentHashMap; 021import java.util.concurrent.TimeUnit; 022import java.util.regex.Matcher; 023import java.util.regex.Pattern; 024 025import javax.xml.stream.XMLStreamConstants; 026import javax.xml.stream.XMLStreamException; 027 028import org.openstreetmap.josm.data.Bounds; 029import org.openstreetmap.josm.data.DataSource; 030import org.openstreetmap.josm.data.coor.LatLon; 031import org.openstreetmap.josm.data.osm.BBox; 032import org.openstreetmap.josm.data.osm.DataSet; 033import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 034import org.openstreetmap.josm.data.osm.PrimitiveId; 035import org.openstreetmap.josm.data.preferences.BooleanProperty; 036import org.openstreetmap.josm.data.preferences.ListProperty; 037import org.openstreetmap.josm.data.preferences.StringProperty; 038import org.openstreetmap.josm.gui.progress.ProgressMonitor; 039import org.openstreetmap.josm.io.NameFinder.SearchResult; 040import org.openstreetmap.josm.tools.HttpClient; 041import org.openstreetmap.josm.tools.Logging; 042import org.openstreetmap.josm.tools.UncheckedParseException; 043import org.openstreetmap.josm.tools.Utils; 044 045/** 046 * Read content from an Overpass server. 047 * 048 * @since 8744 049 */ 050public class OverpassDownloadReader extends BoundingBoxDownloader { 051 052 /** 053 * Property for current Overpass server. 054 * @since 12816 055 */ 056 public static final StringProperty OVERPASS_SERVER = new StringProperty("download.overpass.server", 057 "https://overpass-api.de/api/"); 058 /** 059 * Property for list of known Overpass servers. 060 * @since 12816 061 */ 062 public static final ListProperty OVERPASS_SERVER_HISTORY = new ListProperty("download.overpass.servers", 063 Arrays.asList("https://overpass-api.de/api/", "http://overpass.openstreetmap.ru/cgi/")); 064 /** 065 * Property to determine if Overpass API should be used for multi-fetch download. 066 * @since 12816 067 */ 068 public static final BooleanProperty FOR_MULTI_FETCH = new BooleanProperty("download.overpass.for-multi-fetch", false); 069 070 private static final String DATA_PREFIX = "?data="; 071 072 static final class OverpassOsmReader extends OsmReader { 073 @Override 074 protected void parseUnknown(boolean printWarning) throws XMLStreamException { 075 if ("remark".equals(parser.getLocalName()) && parser.getEventType() == XMLStreamConstants.START_ELEMENT) { 076 final String text = parser.getElementText(); 077 if (text.contains("runtime error")) { 078 throw new XMLStreamException(text); 079 } 080 } 081 super.parseUnknown(printWarning); 082 } 083 } 084 085 static final class OverpassOsmJsonReader extends OsmJsonReader { 086 087 } 088 089 /** 090 * Possible Overpass API output format, with the {@code [out:<directive>]} statement. 091 * @since 11916 092 */ 093 public enum OverpassOutpoutFormat { 094 /** Default output format: plain OSM XML */ 095 OSM_XML("xml"), 096 /** OSM JSON format (not GeoJson) */ 097 OSM_JSON("json"), 098 /** CSV, see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#Output_Format_.28out.29 */ 099 CSV("csv"), 100 /** Custom, see https://overpass-api.de/output_formats.html#custom */ 101 CUSTOM("custom"), 102 /** Popup, see https://overpass-api.de/output_formats.html#popup */ 103 POPUP("popup"), 104 /** PBF, see https://josm.openstreetmap.de/ticket/14653 */ 105 PBF("pbf"); 106 107 private final String directive; 108 109 OverpassOutpoutFormat(String directive) { 110 this.directive = directive; 111 } 112 113 /** 114 * Returns the directive used in {@code [out:<directive>]} statement. 115 * @return the directive used in {@code [out:<directive>]} statement 116 */ 117 public String getDirective() { 118 return directive; 119 } 120 121 /** 122 * Returns the {@code OverpassOutpoutFormat} matching the given directive. 123 * @param directive directive used in {@code [out:<directive>]} statement 124 * @return {@code OverpassOutpoutFormat} matching the given directive 125 * @throws IllegalArgumentException in case of invalid directive 126 */ 127 static OverpassOutpoutFormat from(String directive) { 128 for (OverpassOutpoutFormat oof : values()) { 129 if (oof.directive.equals(directive)) { 130 return oof; 131 } 132 } 133 throw new IllegalArgumentException(directive); 134 } 135 } 136 137 static final Pattern OUTPUT_FORMAT_STATEMENT = Pattern.compile(".*\\[out:([a-z]{3,})\\].*", Pattern.DOTALL); 138 139 static final Map<OverpassOutpoutFormat, Class<? extends AbstractReader>> outputFormatReaders = new ConcurrentHashMap<>(); 140 141 final String overpassServer; 142 final String overpassQuery; 143 144 /** 145 * Constructs a new {@code OverpassDownloadReader}. 146 * 147 * @param downloadArea The area to download 148 * @param overpassServer The Overpass server to use 149 * @param overpassQuery The Overpass query 150 */ 151 public OverpassDownloadReader(Bounds downloadArea, String overpassServer, String overpassQuery) { 152 super(downloadArea); 153 setDoAuthenticate(false); 154 this.overpassServer = overpassServer; 155 this.overpassQuery = overpassQuery.trim(); 156 } 157 158 /** 159 * Registers an OSM reader for the given Overpass output format. 160 * @param format Overpass output format 161 * @param readerClass OSM reader class 162 * @return the previous value associated with {@code format}, or {@code null} if there was no mapping 163 */ 164 public static final Class<? extends AbstractReader> registerOverpassOutpoutFormatReader( 165 OverpassOutpoutFormat format, Class<? extends AbstractReader> readerClass) { 166 return outputFormatReaders.put(Objects.requireNonNull(format), Objects.requireNonNull(readerClass)); 167 } 168 169 static { 170 registerOverpassOutpoutFormatReader(OverpassOutpoutFormat.OSM_XML, OverpassOsmReader.class); 171 registerOverpassOutpoutFormatReader(OverpassOutpoutFormat.OSM_JSON, OverpassOsmJsonReader.class); 172 } 173 174 @Override 175 protected String getBaseUrl() { 176 return overpassServer; 177 } 178 179 @Override 180 protected String getRequestForBbox(double lon1, double lat1, double lon2, double lat2) { 181 if (overpassQuery.isEmpty()) 182 return super.getRequestForBbox(lon1, lat1, lon2, lat2); 183 else { 184 final String query = this.overpassQuery 185 .replace("{{bbox}}", bbox(lon1, lat1, lon2, lat2)) 186 .replace("{{center}}", center(lon1, lat1, lon2, lat2)); 187 final String expandedOverpassQuery = expandExtendedQueries(query); 188 return "interpreter" + DATA_PREFIX + Utils.encodeUrl(expandedOverpassQuery); 189 } 190 } 191 192 /** 193 * Evaluates some features of overpass turbo extended query syntax. 194 * See https://wiki.openstreetmap.org/wiki/Overpass_turbo/Extended_Overpass_Turbo_Queries 195 * @param query unexpanded query 196 * @return expanded query 197 */ 198 static String expandExtendedQueries(String query) { 199 final StringBuffer sb = new StringBuffer(); 200 final Matcher matcher = Pattern.compile("\\{\\{(date|geocodeArea|geocodeBbox|geocodeCoords|geocodeId):([^}]+)\\}\\}").matcher(query); 201 while (matcher.find()) { 202 try { 203 switch (matcher.group(1)) { 204 case "date": 205 matcher.appendReplacement(sb, date(matcher.group(2), LocalDateTime.now())); 206 break; 207 case "geocodeArea": 208 matcher.appendReplacement(sb, geocodeArea(matcher.group(2))); 209 break; 210 case "geocodeBbox": 211 matcher.appendReplacement(sb, geocodeBbox(matcher.group(2))); 212 break; 213 case "geocodeCoords": 214 matcher.appendReplacement(sb, geocodeCoords(matcher.group(2))); 215 break; 216 case "geocodeId": 217 matcher.appendReplacement(sb, geocodeId(matcher.group(2))); 218 break; 219 default: 220 Logging.warn("Unsupported syntax: " + matcher.group(1)); 221 } 222 } catch (UncheckedParseException | IOException | NoSuchElementException | IndexOutOfBoundsException ex) { 223 final String msg = tr("Failed to evaluate {0}", matcher.group()); 224 Logging.log(Logging.LEVEL_WARN, msg, ex); 225 matcher.appendReplacement(sb, "// " + msg + "\n"); 226 } 227 } 228 matcher.appendTail(sb); 229 return sb.toString(); 230 } 231 232 static String bbox(double lon1, double lat1, double lon2, double lat2) { 233 return lat1 + "," + lon1 + "," + lat2 + "," + lon2; 234 } 235 236 static String center(double lon1, double lat1, double lon2, double lat2) { 237 LatLon c = new BBox(lon1, lat1, lon2, lat2).getCenter(); 238 return c.lat()+ "," + c.lon(); 239 } 240 241 static String date(String humanDuration, LocalDateTime from) { 242 // Convert to ISO 8601. Replace months by X temporarily to avoid conflict with minutes 243 String duration = humanDuration.toLowerCase(Locale.ENGLISH).replace(" ", "") 244 .replaceAll("years?", "Y").replaceAll("months?", "X").replaceAll("weeks?", "W") 245 .replaceAll("days?", "D").replaceAll("hours?", "H").replaceAll("minutes?", "M").replaceAll("seconds?", "S"); 246 Matcher matcher = Pattern.compile( 247 "((?:[0-9]+Y)?(?:[0-9]+X)?(?:[0-9]+W)?)"+ 248 "((?:[0-9]+D)?)" + 249 "((?:[0-9]+H)?(?:[0-9]+M)?(?:[0-9]+(?:[.,][0-9]{0,9})?S)?)?").matcher(duration); 250 boolean javaPer = false; 251 boolean javaDur = false; 252 if (matcher.matches()) { 253 javaPer = matcher.group(1) != null && !matcher.group(1).isEmpty(); 254 javaDur = matcher.group(3) != null && !matcher.group(3).isEmpty(); 255 duration = 'P' + matcher.group(1).replace('X', 'M') + matcher.group(2); 256 if (javaDur) { 257 duration += 'T' + matcher.group(3); 258 } 259 } 260 261 // Duration is now a full ISO 8601 duration string. Unfortunately Java does not allow to parse it entirely. 262 // We must split the "period" (years, months, weeks, days) from the "duration" (days, hours, minutes, seconds). 263 Period p = null; 264 Duration d = null; 265 int idx = duration.indexOf('T'); 266 if (javaPer) { 267 p = Period.parse(javaDur ? duration.substring(0, idx) : duration); 268 } 269 if (javaDur) { 270 d = Duration.parse(javaPer ? 'P' + duration.substring(idx, duration.length()) : duration); 271 } else if (!javaPer) { 272 d = Duration.parse(duration); 273 } 274 275 // Now that period and duration are known, compute the correct date/time 276 LocalDateTime dt = from; 277 if (p != null) { 278 dt = dt.minus(p); 279 } 280 if (d != null) { 281 dt = dt.minus(d); 282 } 283 284 // Returns the date/time formatted in ISO 8601 285 return dt.toInstant(ZoneOffset.UTC).toString(); 286 } 287 288 private static SearchResult searchName(String area) throws IOException { 289 return searchName(NameFinder.queryNominatim(area)); 290 } 291 292 static SearchResult searchName(List<SearchResult> results) { 293 return results.stream().filter( 294 x -> OsmPrimitiveType.NODE != x.getOsmId().getType()).iterator().next(); 295 } 296 297 static String geocodeArea(String area) throws IOException { 298 // Offsets defined in https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#By_element_id 299 final EnumMap<OsmPrimitiveType, Long> idOffset = new EnumMap<>(OsmPrimitiveType.class); 300 idOffset.put(OsmPrimitiveType.NODE, 0L); 301 idOffset.put(OsmPrimitiveType.WAY, 2_400_000_000L); 302 idOffset.put(OsmPrimitiveType.RELATION, 3_600_000_000L); 303 final PrimitiveId osmId = searchName(area).getOsmId(); 304 Logging.debug("Area ''{0}'' resolved to {1}", area, osmId); 305 return String.format("area(%d)", osmId.getUniqueId() + idOffset.get(osmId.getType())); 306 } 307 308 static String geocodeBbox(String area) throws IOException { 309 Bounds bounds = searchName(area).getBounds(); 310 return bounds.getMinLat() + "," + bounds.getMinLon() + "," + bounds.getMaxLat() + "," + bounds.getMaxLon(); 311 } 312 313 static String geocodeCoords(String area) throws IOException { 314 SearchResult result = searchName(area); 315 return result.getLat() + "," + result.getLon(); 316 } 317 318 static String geocodeId(String area) throws IOException { 319 PrimitiveId osmId = searchName(area).getOsmId(); 320 return String.format("%s(%d)", osmId.getType().getAPIName(), osmId.getUniqueId()); 321 } 322 323 @Override 324 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason, 325 boolean uncompressAccordingToContentDisposition) throws OsmTransferException { 326 try { 327 int index = urlStr.indexOf(DATA_PREFIX); 328 // Make an HTTP POST request instead of a simple GET, allows more complex queries 329 return super.getInputStreamRaw(urlStr.substring(0, index), 330 progressMonitor, reason, uncompressAccordingToContentDisposition, 331 "POST", Utils.decodeUrl(urlStr.substring(index + DATA_PREFIX.length())).getBytes(StandardCharsets.UTF_8)); 332 } catch (OsmApiException ex) { 333 final String errorIndicator = "Error</strong>: "; 334 if (ex.getMessage() != null && ex.getMessage().contains(errorIndicator)) { 335 final String errorPlusRest = ex.getMessage().split(errorIndicator)[1]; 336 if (errorPlusRest != null) { 337 ex.setErrorHeader(errorPlusRest.split("</")[0].replaceAll(".*::request_read_and_idx::", "")); 338 } 339 } 340 throw ex; 341 } 342 } 343 344 @Override 345 protected void adaptRequest(HttpClient request) { 346 // see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#timeout 347 final Matcher timeoutMatcher = Pattern.compile("\\[timeout:(\\d+)\\]").matcher(overpassQuery); 348 final int timeout; 349 if (timeoutMatcher.find()) { 350 timeout = (int) TimeUnit.SECONDS.toMillis(Integer.parseInt(timeoutMatcher.group(1))); 351 } else { 352 timeout = (int) TimeUnit.MINUTES.toMillis(3); 353 } 354 request.setConnectTimeout(timeout); 355 request.setReadTimeout(timeout); 356 } 357 358 @Override 359 protected String getTaskName() { 360 return tr("Contacting Server..."); 361 } 362 363 @Override 364 protected DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 365 AbstractReader reader = null; 366 Matcher m = OUTPUT_FORMAT_STATEMENT.matcher(overpassQuery); 367 if (m.matches()) { 368 Class<? extends AbstractReader> readerClass = outputFormatReaders.get(OverpassOutpoutFormat.from(m.group(1))); 369 if (readerClass != null) { 370 try { 371 reader = readerClass.getDeclaredConstructor().newInstance(); 372 } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException e) { 373 Logging.error(e); 374 } 375 } 376 } 377 if (reader == null) { 378 reader = new OverpassOsmReader(); 379 } 380 return reader.doParseDataSet(source, progressMonitor); 381 } 382 383 @Override 384 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException { 385 386 DataSet ds = super.parseOsm(progressMonitor); 387 388 // add bounds if necessary (note that Overpass API does not return bounds in the response XML) 389 if (ds != null && ds.getDataSources().isEmpty() && overpassQuery.contains("{{bbox}}")) { 390 if (crosses180th) { 391 Bounds bounds = new Bounds(lat1, lon1, lat2, 180.0); 392 DataSource src = new DataSource(bounds, getBaseUrl()); 393 ds.addDataSource(src); 394 395 bounds = new Bounds(lat1, -180.0, lat2, lon2); 396 src = new DataSource(bounds, getBaseUrl()); 397 ds.addDataSource(src); 398 } else { 399 Bounds bounds = new Bounds(lat1, lon1, lat2, lon2); 400 DataSource src = new DataSource(bounds, getBaseUrl()); 401 ds.addDataSource(src); 402 } 403 } 404 405 return ds; 406 } 407 408 /** 409 * Fixes Overpass API query to make sure it will be accepted by JOSM. 410 * @param query Overpass query to check 411 * @return fixed query 412 * @since 13335 413 */ 414 public static String fixQuery(String query) { 415 return query == null ? query : query 416 .replaceAll("out( body| skel| ids)?( id| qt)?;", "out meta$2;") 417 .replaceAll("(?s)\\[out:(csv)[^\\]]*\\]", "[out:xml]"); 418 } 419}