001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.tools; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.BufferedOutputStream; 007import java.io.BufferedReader; 008import java.io.ByteArrayInputStream; 009import java.io.IOException; 010import java.io.InputStream; 011import java.io.OutputStream; 012import java.net.CookieHandler; 013import java.net.CookieManager; 014import java.net.HttpURLConnection; 015import java.net.URL; 016import java.util.Collections; 017import java.util.List; 018import java.util.Locale; 019import java.util.Map; 020import java.util.Map.Entry; 021import java.util.Scanner; 022import java.util.TreeMap; 023import java.util.concurrent.TimeUnit; 024import java.util.regex.Matcher; 025import java.util.regex.Pattern; 026import java.util.zip.GZIPInputStream; 027 028import org.openstreetmap.josm.Main; 029import org.openstreetmap.josm.data.Version; 030import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 031import org.openstreetmap.josm.gui.progress.ProgressMonitor; 032import org.openstreetmap.josm.io.Compression; 033import org.openstreetmap.josm.io.ProgressInputStream; 034import org.openstreetmap.josm.io.ProgressOutputStream; 035import org.openstreetmap.josm.io.UTFInputStreamReader; 036import org.openstreetmap.josm.io.auth.DefaultAuthenticator; 037 038/** 039 * Provides a uniform access for a HTTP/HTTPS server. This class should be used in favour of {@link HttpURLConnection}. 040 * @since 9168 041 */ 042public final class HttpClient { 043 044 private URL url; 045 private final String requestMethod; 046 private int connectTimeout = (int) TimeUnit.SECONDS.toMillis(Main.pref.getInteger("socket.timeout.connect", 15)); 047 private int readTimeout = (int) TimeUnit.SECONDS.toMillis(Main.pref.getInteger("socket.timeout.read", 30)); 048 private byte[] requestBody; 049 private long ifModifiedSince; 050 private final Map<String, String> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); 051 private int maxRedirects = Main.pref.getInteger("socket.maxredirects", 5); 052 private boolean useCache; 053 private String reasonForRequest; 054 private HttpURLConnection connection; // to allow disconnecting before `response` is set 055 private Response response; 056 private boolean finishOnCloseOutput = true; 057 058 static { 059 CookieHandler.setDefault(new CookieManager()); 060 } 061 062 private HttpClient(URL url, String requestMethod) { 063 this.url = url; 064 this.requestMethod = requestMethod; 065 this.headers.put("Accept-Encoding", "gzip"); 066 } 067 068 /** 069 * Opens the HTTP connection. 070 * @return HTTP response 071 * @throws IOException if any I/O error occurs 072 */ 073 public Response connect() throws IOException { 074 return connect(null); 075 } 076 077 /** 078 * Opens the HTTP connection. 079 * @param progressMonitor progress monitor 080 * @return HTTP response 081 * @throws IOException if any I/O error occurs 082 * @since 9179 083 */ 084 public Response connect(ProgressMonitor progressMonitor) throws IOException { 085 if (progressMonitor == null) { 086 progressMonitor = NullProgressMonitor.INSTANCE; 087 } 088 final HttpURLConnection connection = (HttpURLConnection) url.openConnection(); 089 this.connection = connection; 090 connection.setRequestMethod(requestMethod); 091 connection.setRequestProperty("User-Agent", Version.getInstance().getFullAgentString()); 092 connection.setConnectTimeout(connectTimeout); 093 connection.setReadTimeout(readTimeout); 094 connection.setInstanceFollowRedirects(false); // we do that ourselves 095 if (ifModifiedSince > 0) { 096 connection.setIfModifiedSince(ifModifiedSince); 097 } 098 connection.setUseCaches(useCache); 099 if (!useCache) { 100 connection.setRequestProperty("Cache-Control", "no-cache"); 101 } 102 for (Map.Entry<String, String> header : headers.entrySet()) { 103 if (header.getValue() != null) { 104 connection.setRequestProperty(header.getKey(), header.getValue()); 105 } 106 } 107 108 progressMonitor.beginTask(tr("Contacting Server..."), 1); 109 progressMonitor.indeterminateSubTask(null); 110 111 if ("PUT".equals(requestMethod) || "POST".equals(requestMethod) || "DELETE".equals(requestMethod)) { 112 Main.info("{0} {1} ({2}) ...", requestMethod, url, Utils.getSizeString(requestBody.length, Locale.getDefault())); 113 connection.setFixedLengthStreamingMode(requestBody.length); 114 connection.setDoOutput(true); 115 try (OutputStream out = new BufferedOutputStream( 116 new ProgressOutputStream(connection.getOutputStream(), requestBody.length, progressMonitor, finishOnCloseOutput))) { 117 out.write(requestBody); 118 } 119 } 120 121 boolean successfulConnection = false; 122 try { 123 try { 124 connection.connect(); 125 final boolean hasReason = reasonForRequest != null && !reasonForRequest.isEmpty(); 126 Main.info("{0} {1}{2} -> {3}{4}", 127 requestMethod, url, hasReason ? (" (" + reasonForRequest + ')') : "", 128 connection.getResponseCode(), 129 connection.getContentLengthLong() > 0 130 ? (" (" + Utils.getSizeString(connection.getContentLengthLong(), Locale.getDefault()) + ')') 131 : "" 132 ); 133 if (Main.isDebugEnabled()) { 134 Main.debug("RESPONSE: " + connection.getHeaderFields()); 135 } 136 if (DefaultAuthenticator.getInstance().isEnabled() && connection.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) { 137 DefaultAuthenticator.getInstance().addFailedCredentialHost(url.getHost()); 138 } 139 } catch (IOException e) { 140 Main.info("{0} {1} -> !!!", requestMethod, url); 141 Main.warn(e); 142 //noinspection ThrowableResultOfMethodCallIgnored 143 Main.addNetworkError(url, Utils.getRootCause(e)); 144 throw e; 145 } 146 if (isRedirect(connection.getResponseCode())) { 147 final String redirectLocation = connection.getHeaderField("Location"); 148 if (redirectLocation == null) { 149 /* I18n: argument is HTTP response code */ 150 String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header." + 151 " Can''t redirect. Aborting.", connection.getResponseCode()); 152 throw new IOException(msg); 153 } else if (maxRedirects > 0) { 154 url = new URL(url, redirectLocation); 155 maxRedirects--; 156 Main.info(tr("Download redirected to ''{0}''", redirectLocation)); 157 return connect(); 158 } else if (maxRedirects == 0) { 159 String msg = tr("Too many redirects to the download URL detected. Aborting."); 160 throw new IOException(msg); 161 } 162 } 163 response = new Response(connection, progressMonitor); 164 successfulConnection = true; 165 return response; 166 } finally { 167 if (!successfulConnection) { 168 connection.disconnect(); 169 } 170 } 171 } 172 173 /** 174 * Returns the HTTP response which is set only after calling {@link #connect()}. 175 * Calling this method again, returns the identical object (unless another {@link #connect()} is performed). 176 * 177 * @return the HTTP response 178 * @since 9309 179 */ 180 public Response getResponse() { 181 return response; 182 } 183 184 /** 185 * A wrapper for the HTTP response. 186 */ 187 public static final class Response { 188 private final HttpURLConnection connection; 189 private final ProgressMonitor monitor; 190 private final int responseCode; 191 private final String responseMessage; 192 private boolean uncompress; 193 private boolean uncompressAccordingToContentDisposition; 194 private String responseData; 195 196 private Response(HttpURLConnection connection, ProgressMonitor monitor) throws IOException { 197 CheckParameterUtil.ensureParameterNotNull(connection, "connection"); 198 CheckParameterUtil.ensureParameterNotNull(monitor, "monitor"); 199 this.connection = connection; 200 this.monitor = monitor; 201 this.responseCode = connection.getResponseCode(); 202 this.responseMessage = connection.getResponseMessage(); 203 if (this.responseCode >= 300) { 204 String contentType = getContentType(); 205 if (contentType == null || ( 206 contentType.contains("text") || 207 contentType.contains("html") || 208 contentType.contains("xml")) 209 ) { 210 String content = this.fetchContent(); 211 if (content.isEmpty()) { 212 Main.debug("Server did not return any body"); 213 } else { 214 Main.debug("Response body: "); 215 Main.debug(this.fetchContent()); 216 } 217 } else { 218 Main.debug("Server returned content: {0} of length: {1}. Not printing.", contentType, this.getContentLength()); 219 } 220 } 221 } 222 223 /** 224 * Sets whether {@link #getContent()} should uncompress the input stream if necessary. 225 * 226 * @param uncompress whether the input stream should be uncompressed if necessary 227 * @return {@code this} 228 */ 229 public Response uncompress(boolean uncompress) { 230 this.uncompress = uncompress; 231 return this; 232 } 233 234 /** 235 * Sets whether {@link #getContent()} should uncompress the input stream according to {@code Content-Disposition} 236 * HTTP header. 237 * @param uncompressAccordingToContentDisposition whether the input stream should be uncompressed according to 238 * {@code Content-Disposition} 239 * @return {@code this} 240 * @since 9172 241 */ 242 public Response uncompressAccordingToContentDisposition(boolean uncompressAccordingToContentDisposition) { 243 this.uncompressAccordingToContentDisposition = uncompressAccordingToContentDisposition; 244 return this; 245 } 246 247 /** 248 * Returns the URL. 249 * @return the URL 250 * @see HttpURLConnection#getURL() 251 * @since 9172 252 */ 253 public URL getURL() { 254 return connection.getURL(); 255 } 256 257 /** 258 * Returns the request method. 259 * @return the HTTP request method 260 * @see HttpURLConnection#getRequestMethod() 261 * @since 9172 262 */ 263 public String getRequestMethod() { 264 return connection.getRequestMethod(); 265 } 266 267 /** 268 * Returns an input stream that reads from this HTTP connection, or, 269 * error stream if the connection failed but the server sent useful data. 270 * <p> 271 * Note: the return value can be null, if both the input and the error stream are null. 272 * Seems to be the case if the OSM server replies a 401 Unauthorized, see #3887 273 * @return input or error stream 274 * @throws IOException if any I/O error occurs 275 * 276 * @see HttpURLConnection#getInputStream() 277 * @see HttpURLConnection#getErrorStream() 278 */ 279 @SuppressWarnings("resource") 280 public InputStream getContent() throws IOException { 281 InputStream in; 282 try { 283 in = connection.getInputStream(); 284 } catch (IOException ioe) { 285 Main.debug(ioe); 286 in = connection.getErrorStream(); 287 if (in == null) { 288 in = new ByteArrayInputStream(new byte[]{}); 289 } 290 } 291 in = new ProgressInputStream(in, getContentLength(), monitor); 292 in = "gzip".equalsIgnoreCase(getContentEncoding()) ? new GZIPInputStream(in) : in; 293 Compression compression = Compression.NONE; 294 if (uncompress) { 295 final String contentType = getContentType(); 296 Main.debug("Uncompressing input stream according to Content-Type header: {0}", contentType); 297 compression = Compression.forContentType(contentType); 298 } 299 if (uncompressAccordingToContentDisposition && Compression.NONE.equals(compression)) { 300 final String contentDisposition = getHeaderField("Content-Disposition"); 301 final Matcher matcher = Pattern.compile("filename=\"([^\"]+)\"").matcher( 302 contentDisposition != null ? contentDisposition : ""); 303 if (matcher.find()) { 304 Main.debug("Uncompressing input stream according to Content-Disposition header: {0}", contentDisposition); 305 compression = Compression.byExtension(matcher.group(1)); 306 } 307 } 308 in = compression.getUncompressedInputStream(in); 309 return in; 310 } 311 312 /** 313 * Returns {@link #getContent()} wrapped in a buffered reader. 314 * 315 * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}. 316 * @return buffered reader 317 * @throws IOException if any I/O error occurs 318 */ 319 public BufferedReader getContentReader() throws IOException { 320 return new BufferedReader( 321 UTFInputStreamReader.create(getContent()) 322 ); 323 } 324 325 /** 326 * Fetches the HTTP response as String. 327 * @return the response 328 * @throws IOException if any I/O error occurs 329 */ 330 public synchronized String fetchContent() throws IOException { 331 if (responseData == null) { 332 try (Scanner scanner = new Scanner(getContentReader()).useDelimiter("\\A")) { // \A - beginning of input 333 responseData = scanner.hasNext() ? scanner.next() : ""; 334 } 335 } 336 return responseData; 337 } 338 339 /** 340 * Gets the response code from this HTTP connection. 341 * @return HTTP response code 342 * 343 * @see HttpURLConnection#getResponseCode() 344 */ 345 public int getResponseCode() { 346 return responseCode; 347 } 348 349 /** 350 * Gets the response message from this HTTP connection. 351 * @return HTTP response message 352 * 353 * @see HttpURLConnection#getResponseMessage() 354 * @since 9172 355 */ 356 public String getResponseMessage() { 357 return responseMessage; 358 } 359 360 /** 361 * Returns the {@code Content-Encoding} header. 362 * @return {@code Content-Encoding} HTTP header 363 * @see HttpURLConnection#getContentEncoding() 364 */ 365 public String getContentEncoding() { 366 return connection.getContentEncoding(); 367 } 368 369 /** 370 * Returns the {@code Content-Type} header. 371 * @return {@code Content-Type} HTTP header 372 */ 373 public String getContentType() { 374 return connection.getHeaderField("Content-Type"); 375 } 376 377 /** 378 * Returns the {@code Expire} header. 379 * @return {@code Expire} HTTP header 380 * @see HttpURLConnection#getExpiration() 381 * @since 9232 382 */ 383 public long getExpiration() { 384 return connection.getExpiration(); 385 } 386 387 /** 388 * Returns the {@code Last-Modified} header. 389 * @return {@code Last-Modified} HTTP header 390 * @see HttpURLConnection#getLastModified() 391 * @since 9232 392 */ 393 public long getLastModified() { 394 return connection.getLastModified(); 395 } 396 397 /** 398 * Returns the {@code Content-Length} header. 399 * @return {@code Content-Length} HTTP header 400 * @see HttpURLConnection#getContentLengthLong() 401 */ 402 public long getContentLength() { 403 return connection.getContentLengthLong(); 404 } 405 406 /** 407 * Returns the value of the named header field. 408 * @param name the name of a header field 409 * @return the value of the named header field, or {@code null} if there is no such field in the header 410 * @see HttpURLConnection#getHeaderField(String) 411 * @since 9172 412 */ 413 public String getHeaderField(String name) { 414 return connection.getHeaderField(name); 415 } 416 417 /** 418 * Returns an unmodifiable Map mapping header keys to a List of header values. 419 * As per RFC 2616, section 4.2 header names are case insensitive, so returned map is also case insensitive 420 * @return unmodifiable Map mapping header keys to a List of header values 421 * @see HttpURLConnection#getHeaderFields() 422 * @since 9232 423 */ 424 public Map<String, List<String>> getHeaderFields() { 425 // returned map from HttpUrlConnection is case sensitive, use case insensitive TreeMap to conform to RFC 2616 426 Map<String, List<String>> ret = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); 427 for (Entry<String, List<String>> e: connection.getHeaderFields().entrySet()) { 428 if (e.getKey() != null) { 429 ret.put(e.getKey(), e.getValue()); 430 } 431 } 432 return Collections.unmodifiableMap(ret); 433 } 434 435 /** 436 * @see HttpURLConnection#disconnect() 437 */ 438 public void disconnect() { 439 HttpClient.disconnect(connection); 440 } 441 } 442 443 /** 444 * Creates a new instance for the given URL and a {@code GET} request 445 * 446 * @param url the URL 447 * @return a new instance 448 */ 449 public static HttpClient create(URL url) { 450 return create(url, "GET"); 451 } 452 453 /** 454 * Creates a new instance for the given URL and a {@code GET} request 455 * 456 * @param url the URL 457 * @param requestMethod the HTTP request method to perform when calling 458 * @return a new instance 459 */ 460 public static HttpClient create(URL url, String requestMethod) { 461 return new HttpClient(url, requestMethod); 462 } 463 464 /** 465 * Returns the URL set for this connection. 466 * @return the URL 467 * @see #create(URL) 468 * @see #create(URL, String) 469 * @since 9172 470 */ 471 public URL getURL() { 472 return url; 473 } 474 475 /** 476 * Returns the request method set for this connection. 477 * @return the HTTP request method 478 * @see #create(URL, String) 479 * @since 9172 480 */ 481 public String getRequestMethod() { 482 return requestMethod; 483 } 484 485 /** 486 * Returns the set value for the given {@code header}. 487 * @param header HTTP header name 488 * @return HTTP header value 489 * @since 9172 490 */ 491 public String getRequestHeader(String header) { 492 return headers.get(header); 493 } 494 495 /** 496 * Sets whether not to set header {@code Cache-Control=no-cache} 497 * 498 * @param useCache whether not to set header {@code Cache-Control=no-cache} 499 * @return {@code this} 500 * @see HttpURLConnection#setUseCaches(boolean) 501 */ 502 public HttpClient useCache(boolean useCache) { 503 this.useCache = useCache; 504 return this; 505 } 506 507 /** 508 * Sets whether not to set header {@code Connection=close} 509 * <p> 510 * This might fix #7640, see 511 * <a href='https://web.archive.org/web/20140118201501/http://www.tikalk.com/java/forums/httpurlconnection-disable-keep-alive'>here</a>. 512 * 513 * @param keepAlive whether not to set header {@code Connection=close} 514 * @return {@code this} 515 */ 516 public HttpClient keepAlive(boolean keepAlive) { 517 return setHeader("Connection", keepAlive ? null : "close"); 518 } 519 520 /** 521 * Sets a specified timeout value, in milliseconds, to be used when opening a communications link to the resource referenced 522 * by this URLConnection. If the timeout expires before the connection can be established, a 523 * {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout. 524 * @param connectTimeout an {@code int} that specifies the connect timeout value in milliseconds 525 * @return {@code this} 526 * @see HttpURLConnection#setConnectTimeout(int) 527 */ 528 public HttpClient setConnectTimeout(int connectTimeout) { 529 this.connectTimeout = connectTimeout; 530 return this; 531 } 532 533 /** 534 * Sets the read timeout to a specified timeout, in milliseconds. A non-zero value specifies the timeout when reading from 535 * input stream when a connection is established to a resource. If the timeout expires before there is data available for 536 * read, a {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout. 537 * @param readTimeout an {@code int} that specifies the read timeout value in milliseconds 538 * @return {@code this} 539 * @see HttpURLConnection#setReadTimeout(int) 540 */ 541 public HttpClient setReadTimeout(int readTimeout) { 542 this.readTimeout = readTimeout; 543 return this; 544 } 545 546 /** 547 * Sets the {@code Accept} header. 548 * @param accept header value 549 * 550 * @return {@code this} 551 */ 552 public HttpClient setAccept(String accept) { 553 return setHeader("Accept", accept); 554 } 555 556 /** 557 * Sets the request body for {@code PUT}/{@code POST} requests. 558 * @param requestBody request body 559 * 560 * @return {@code this} 561 */ 562 public HttpClient setRequestBody(byte[] requestBody) { 563 this.requestBody = requestBody; 564 return this; 565 } 566 567 /** 568 * Sets the {@code If-Modified-Since} header. 569 * @param ifModifiedSince header value 570 * 571 * @return {@code this} 572 */ 573 public HttpClient setIfModifiedSince(long ifModifiedSince) { 574 this.ifModifiedSince = ifModifiedSince; 575 return this; 576 } 577 578 /** 579 * Sets the maximum number of redirections to follow. 580 * 581 * Set {@code maxRedirects} to {@code -1} in order to ignore redirects, i.e., 582 * to not throw an {@link IOException} in {@link #connect()}. 583 * @param maxRedirects header value 584 * 585 * @return {@code this} 586 */ 587 public HttpClient setMaxRedirects(int maxRedirects) { 588 this.maxRedirects = maxRedirects; 589 return this; 590 } 591 592 /** 593 * Sets an arbitrary HTTP header. 594 * @param key header name 595 * @param value header value 596 * 597 * @return {@code this} 598 */ 599 public HttpClient setHeader(String key, String value) { 600 this.headers.put(key, value); 601 return this; 602 } 603 604 /** 605 * Sets arbitrary HTTP headers. 606 * @param headers HTTP headers 607 * 608 * @return {@code this} 609 */ 610 public HttpClient setHeaders(Map<String, String> headers) { 611 this.headers.putAll(headers); 612 return this; 613 } 614 615 /** 616 * Sets a reason to show on console. Can be {@code null} if no reason is given. 617 * @param reasonForRequest Reason to show 618 * @return {@code this} 619 * @since 9172 620 */ 621 public HttpClient setReasonForRequest(String reasonForRequest) { 622 this.reasonForRequest = reasonForRequest; 623 return this; 624 } 625 626 /** 627 * Sets whether the progress monitor task will be finished when the output stream is closed. This is {@code true} by default. 628 * @param finishOnCloseOutput whether the progress monitor task will be finished when the output stream is closed 629 * @return {@code this} 630 * @since 10302 631 */ 632 public HttpClient setFinishOnCloseOutput(boolean finishOnCloseOutput) { 633 this.finishOnCloseOutput = finishOnCloseOutput; 634 return this; 635 } 636 637 private static boolean isRedirect(final int statusCode) { 638 switch (statusCode) { 639 case HttpURLConnection.HTTP_MOVED_PERM: // 301 640 case HttpURLConnection.HTTP_MOVED_TEMP: // 302 641 case HttpURLConnection.HTTP_SEE_OTHER: // 303 642 case 307: // TEMPORARY_REDIRECT: 643 case 308: // PERMANENT_REDIRECT: 644 return true; 645 default: 646 return false; 647 } 648 } 649 650 /** 651 * @see HttpURLConnection#disconnect() 652 * @since 9309 653 */ 654 public void disconnect() { 655 HttpClient.disconnect(connection); 656 } 657 658 private static void disconnect(final HttpURLConnection connection) { 659 // Fix upload aborts - see #263 660 connection.setConnectTimeout(100); 661 connection.setReadTimeout(100); 662 try { 663 Thread.sleep(100); 664 } catch (InterruptedException ex) { 665 Main.warn("InterruptedException in " + HttpClient.class + " during cancel"); 666 } 667 connection.disconnect(); 668 } 669}