001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.BufferedInputStream; 007import java.io.BufferedReader; 008import java.io.ByteArrayOutputStream; 009import java.io.Closeable; 010import java.io.File; 011import java.io.FileInputStream; 012import java.io.IOException; 013import java.io.InputStream; 014import java.net.HttpURLConnection; 015import java.net.MalformedURLException; 016import java.net.URL; 017import java.nio.charset.StandardCharsets; 018import java.nio.file.Files; 019import java.nio.file.StandardCopyOption; 020import java.util.ArrayList; 021import java.util.Arrays; 022import java.util.Enumeration; 023import java.util.List; 024import java.util.Map; 025import java.util.concurrent.ConcurrentHashMap; 026import java.util.concurrent.TimeUnit; 027import java.util.zip.ZipEntry; 028import java.util.zip.ZipFile; 029 030import org.openstreetmap.josm.Main; 031import org.openstreetmap.josm.tools.HttpClient; 032import org.openstreetmap.josm.tools.Pair; 033import org.openstreetmap.josm.tools.Utils; 034 035/** 036 * Downloads a file and caches it on disk in order to reduce network load. 037 * 038 * Supports URLs, local files, and a custom scheme (<code>resource:</code>) to get 039 * resources from the current *.jar file. (Local caching is only done for URLs.) 040 * <p> 041 * The mirrored file is only downloaded if it has been more than 7 days since 042 * last download. (Time can be configured.) 043 * <p> 044 * The file content is normally accessed with {@link #getInputStream()}, but 045 * you can also get the mirrored copy with {@link #getFile()}. 046 */ 047public class CachedFile implements Closeable { 048 049 /** 050 * Caching strategy. 051 */ 052 public enum CachingStrategy { 053 /** 054 * If cached file on disk is older than a certain time (7 days by default), 055 * consider the cache stale and try to download the file again. 056 */ 057 MaxAge, 058 /** 059 * Similar to MaxAge, considers the cache stale when a certain age is 060 * exceeded. In addition, a If-Modified-Since HTTP header is added. 061 * When the server replies "304 Not Modified", this is considered the same 062 * as a full download. 063 */ 064 IfModifiedSince 065 } 066 067 protected String name; 068 protected long maxAge; 069 protected String destDir; 070 protected String httpAccept; 071 protected CachingStrategy cachingStrategy; 072 073 private boolean fastFail; 074 private HttpClient activeConnection; 075 protected File cacheFile; 076 protected boolean initialized; 077 078 public static final long DEFAULT_MAXTIME = -1L; 079 public static final long DAYS = TimeUnit.DAYS.toSeconds(1); // factor to get caching time in days 080 081 private final Map<String, String> httpHeaders = new ConcurrentHashMap<>(); 082 083 /** 084 * Constructs a CachedFile object from a given filename, URL or internal resource. 085 * 086 * @param name can be:<ul> 087 * <li>relative or absolute file name</li> 088 * <li>{@code file:///SOME/FILE} the same as above</li> 089 * <li>{@code http://...} a URL. It will be cached on disk.</li> 090 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li> 091 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li> 092 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul> 093 */ 094 public CachedFile(String name) { 095 this.name = name; 096 } 097 098 /** 099 * Set the name of the resource. 100 * @param name can be:<ul> 101 * <li>relative or absolute file name</li> 102 * <li>{@code file:///SOME/FILE} the same as above</li> 103 * <li>{@code http://...} a URL. It will be cached on disk.</li> 104 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li> 105 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li> 106 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul> 107 * @return this object 108 */ 109 public CachedFile setName(String name) { 110 this.name = name; 111 return this; 112 } 113 114 /** 115 * Set maximum age of cache file. Only applies to URLs. 116 * When this time has passed after the last download of the file, the 117 * cache is considered stale and a new download will be attempted. 118 * @param maxAge the maximum cache age in seconds 119 * @return this object 120 */ 121 public CachedFile setMaxAge(long maxAge) { 122 this.maxAge = maxAge; 123 return this; 124 } 125 126 /** 127 * Set the destination directory for the cache file. Only applies to URLs. 128 * @param destDir the destination directory 129 * @return this object 130 */ 131 public CachedFile setDestDir(String destDir) { 132 this.destDir = destDir; 133 return this; 134 } 135 136 /** 137 * Set the accepted MIME types sent in the HTTP Accept header. Only applies to URLs. 138 * @param httpAccept the accepted MIME types 139 * @return this object 140 */ 141 public CachedFile setHttpAccept(String httpAccept) { 142 this.httpAccept = httpAccept; 143 return this; 144 } 145 146 /** 147 * Set the caching strategy. Only applies to URLs. 148 * @param cachingStrategy caching strategy 149 * @return this object 150 */ 151 public CachedFile setCachingStrategy(CachingStrategy cachingStrategy) { 152 this.cachingStrategy = cachingStrategy; 153 return this; 154 } 155 156 /** 157 * Sets the http headers. Only applies to URL pointing to http or https resources 158 * @param headers that should be sent together with request 159 * @return this object 160 */ 161 public CachedFile setHttpHeaders(Map<String, String> headers) { 162 this.httpHeaders.putAll(headers); 163 return this; 164 } 165 166 /** 167 * Sets whether opening HTTP connections should fail fast, i.e., whether a 168 * {@link HttpClient#setConnectTimeout(int) low connect timeout} should be used. 169 * @param fastFail whether opening HTTP connections should fail fast 170 */ 171 public void setFastFail(boolean fastFail) { 172 this.fastFail = fastFail; 173 } 174 175 public String getName() { 176 return name; 177 } 178 179 /** 180 * Returns maximum age of cache file. Only applies to URLs. 181 * When this time has passed after the last download of the file, the 182 * cache is considered stale and a new download will be attempted. 183 * @return the maximum cache age in seconds 184 */ 185 public long getMaxAge() { 186 return maxAge; 187 } 188 189 public String getDestDir() { 190 return destDir; 191 } 192 193 public String getHttpAccept() { 194 return httpAccept; 195 } 196 197 public CachingStrategy getCachingStrategy() { 198 return cachingStrategy; 199 } 200 201 /** 202 * Get InputStream to the requested resource. 203 * @return the InputStream 204 * @throws IOException when the resource with the given name could not be retrieved 205 */ 206 public InputStream getInputStream() throws IOException { 207 File file = getFile(); 208 if (file == null) { 209 if (name.startsWith("resource://")) { 210 InputStream is = getClass().getResourceAsStream( 211 name.substring("resource:/".length())); 212 if (is == null) 213 throw new IOException(tr("Failed to open input stream for resource ''{0}''", name)); 214 return is; 215 } else { 216 throw new IOException("No file found for: "+name); 217 } 218 } 219 return new FileInputStream(file); 220 } 221 222 /** 223 * Get the full content of the requested resource as a byte array. 224 * @return the full content of the requested resource as byte array 225 * @throws IOException in case of an I/O error 226 */ 227 public byte[] getByteContent() throws IOException { 228 try (InputStream is = getInputStream()) { 229 ByteArrayOutputStream buffer = new ByteArrayOutputStream(); 230 int nRead; 231 byte[] data = new byte[8192]; 232 while ((nRead = is.read(data, 0, data.length)) != -1) { 233 buffer.write(data, 0, nRead); 234 } 235 buffer.flush(); 236 return buffer.toByteArray(); 237 } 238 } 239 240 /** 241 * Returns {@link #getInputStream()} wrapped in a buffered reader. 242 * <p> 243 * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}. 244 * 245 * @return buffered reader 246 * @throws IOException if any I/O error occurs 247 * @since 9411 248 */ 249 public BufferedReader getContentReader() throws IOException { 250 return new BufferedReader(UTFInputStreamReader.create(getInputStream())); 251 } 252 253 /** 254 * Get local file for the requested resource. 255 * @return The local cache file for URLs. If the resource is a local file, 256 * returns just that file. 257 * @throws IOException when the resource with the given name could not be retrieved 258 */ 259 public synchronized File getFile() throws IOException { 260 if (initialized) 261 return cacheFile; 262 initialized = true; 263 URL url; 264 try { 265 url = new URL(name); 266 if ("file".equals(url.getProtocol())) { 267 cacheFile = new File(name.substring("file:/".length() - 1)); 268 if (!cacheFile.exists()) { 269 cacheFile = new File(name.substring("file://".length() - 1)); 270 } 271 } else { 272 cacheFile = checkLocal(url); 273 } 274 } catch (MalformedURLException e) { 275 if (name.startsWith("resource://")) { 276 return null; 277 } else if (name.startsWith("josmdir://")) { 278 cacheFile = new File(Main.pref.getUserDataDirectory(), name.substring("josmdir://".length())); 279 } else if (name.startsWith("josmplugindir://")) { 280 cacheFile = new File(Main.pref.getPluginsDirectory(), name.substring("josmplugindir://".length())); 281 } else { 282 cacheFile = new File(name); 283 } 284 } 285 if (cacheFile == null) 286 throw new IOException("Unable to get cache file for "+name); 287 return cacheFile; 288 } 289 290 /** 291 * Looks for a certain entry inside a zip file and returns the entry path. 292 * 293 * Replies a file in the top level directory of the ZIP file which has an 294 * extension <code>extension</code>. If more than one files have this 295 * extension, the last file whose name includes <code>namepart</code> 296 * is opened. 297 * 298 * @param extension the extension of the file we're looking for 299 * @param namepart the name part 300 * @return The zip entry path of the matching file. Null if this cached file 301 * doesn't represent a zip file or if there was no matching 302 * file in the ZIP file. 303 */ 304 public String findZipEntryPath(String extension, String namepart) { 305 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart); 306 if (ze == null) return null; 307 return ze.a; 308 } 309 310 /** 311 * Like {@link #findZipEntryPath}, but returns the corresponding InputStream. 312 * @param extension the extension of the file we're looking for 313 * @param namepart the name part 314 * @return InputStream to the matching file. Null if this cached file 315 * doesn't represent a zip file or if there was no matching 316 * file in the ZIP file. 317 * @since 6148 318 */ 319 public InputStream findZipEntryInputStream(String extension, String namepart) { 320 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart); 321 if (ze == null) return null; 322 return ze.b; 323 } 324 325 private Pair<String, InputStream> findZipEntryImpl(String extension, String namepart) { 326 File file = null; 327 try { 328 file = getFile(); 329 } catch (IOException ex) { 330 Main.warn(ex, false); 331 } 332 if (file == null) 333 return null; 334 Pair<String, InputStream> res = null; 335 try { 336 ZipFile zipFile = new ZipFile(file, StandardCharsets.UTF_8); 337 ZipEntry resentry = null; 338 Enumeration<? extends ZipEntry> entries = zipFile.entries(); 339 while (entries.hasMoreElements()) { 340 ZipEntry entry = entries.nextElement(); 341 // choose any file with correct extension. When more than one file, prefer the one which matches namepart 342 if (entry.getName().endsWith('.' + extension) && (resentry == null || entry.getName().indexOf(namepart) >= 0)) { 343 resentry = entry; 344 } 345 } 346 if (resentry != null) { 347 InputStream is = zipFile.getInputStream(resentry); 348 res = Pair.create(resentry.getName(), is); 349 } else { 350 Utils.close(zipFile); 351 } 352 } catch (IOException e) { 353 if (file.getName().endsWith(".zip")) { 354 Main.warn(e, tr("Failed to open file with extension ''{2}'' and namepart ''{3}'' in zip file ''{0}''. Exception was: {1}", 355 file.getName(), e.toString(), extension, namepart)); 356 } 357 } 358 return res; 359 } 360 361 /** 362 * Clear the cache for the given resource. 363 * This forces a fresh download. 364 * @param name the URL 365 */ 366 public static void cleanup(String name) { 367 cleanup(name, null); 368 } 369 370 /** 371 * Clear the cache for the given resource. 372 * This forces a fresh download. 373 * @param name the URL 374 * @param destDir the destination directory (see {@link #setDestDir(java.lang.String)}) 375 */ 376 public static void cleanup(String name, String destDir) { 377 URL url; 378 try { 379 url = new URL(name); 380 if (!"file".equals(url.getProtocol())) { 381 String prefKey = getPrefKey(url, destDir); 382 List<String> localPath = new ArrayList<>(Main.pref.getCollection(prefKey)); 383 if (localPath.size() == 2) { 384 File lfile = new File(localPath.get(1)); 385 if (lfile.exists()) { 386 Utils.deleteFile(lfile); 387 } 388 } 389 Main.pref.putCollection(prefKey, null); 390 } 391 } catch (MalformedURLException e) { 392 Main.warn(e); 393 } 394 } 395 396 /** 397 * Get preference key to store the location and age of the cached file. 398 * 2 resources that point to the same url, but that are to be stored in different 399 * directories will not share a cache file. 400 * @param url URL 401 * @param destDir destination directory 402 * @return Preference key 403 */ 404 private static String getPrefKey(URL url, String destDir) { 405 StringBuilder prefKey = new StringBuilder("mirror."); 406 if (destDir != null) { 407 prefKey.append(destDir).append('.'); 408 } 409 prefKey.append(url.toString()); 410 return prefKey.toString().replaceAll("=", "_"); 411 } 412 413 private File checkLocal(URL url) throws IOException { 414 String prefKey = getPrefKey(url, destDir); 415 String urlStr = url.toExternalForm(); 416 long age = 0L; 417 long maxAgeMillis = maxAge; 418 Long ifModifiedSince = null; 419 File localFile = null; 420 List<String> localPathEntry = new ArrayList<>(Main.pref.getCollection(prefKey)); 421 boolean offline = false; 422 try { 423 checkOfflineAccess(urlStr); 424 } catch (OfflineAccessException e) { 425 Main.trace(e); 426 offline = true; 427 } 428 if (localPathEntry.size() == 2) { 429 localFile = new File(localPathEntry.get(1)); 430 if (!localFile.exists()) { 431 localFile = null; 432 } else { 433 if (maxAge == DEFAULT_MAXTIME 434 || maxAge <= 0 // arbitrary value <= 0 is deprecated 435 ) { 436 maxAgeMillis = TimeUnit.SECONDS.toMillis(Main.pref.getLong("mirror.maxtime", TimeUnit.DAYS.toSeconds(7))); 437 } 438 age = System.currentTimeMillis() - Long.parseLong(localPathEntry.get(0)); 439 if (offline || age < maxAgeMillis) { 440 return localFile; 441 } 442 if (cachingStrategy == CachingStrategy.IfModifiedSince) { 443 ifModifiedSince = Long.valueOf(localPathEntry.get(0)); 444 } 445 } 446 } 447 if (destDir == null) { 448 destDir = Main.pref.getCacheDirectory().getPath(); 449 } 450 451 File destDirFile = new File(destDir); 452 if (!destDirFile.exists()) { 453 Utils.mkDirs(destDirFile); 454 } 455 456 // No local file + offline => nothing to do 457 if (offline) { 458 return null; 459 } 460 461 String a = urlStr.replaceAll("[^A-Za-z0-9_.-]", "_"); 462 String localPath = "mirror_" + a; 463 destDirFile = new File(destDir, localPath + ".tmp"); 464 try { 465 activeConnection = HttpClient.create(url) 466 .setAccept(httpAccept) 467 .setIfModifiedSince(ifModifiedSince == null ? 0L : ifModifiedSince) 468 .setHeaders(httpHeaders); 469 if (fastFail) { 470 activeConnection.setReadTimeout(1000); 471 } 472 final HttpClient.Response con = activeConnection.connect(); 473 if (ifModifiedSince != null && con.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) { 474 if (Main.isDebugEnabled()) { 475 Main.debug("304 Not Modified ("+urlStr+')'); 476 } 477 if (localFile == null) 478 throw new AssertionError(); 479 Main.pref.putCollection(prefKey, 480 Arrays.asList(Long.toString(System.currentTimeMillis()), localPathEntry.get(1))); 481 return localFile; 482 } else if (con.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { 483 throw new IOException(tr("The requested URL {0} was not found", urlStr)); 484 } 485 try (InputStream bis = new BufferedInputStream(con.getContent())) { 486 Files.copy(bis, destDirFile.toPath(), StandardCopyOption.REPLACE_EXISTING); 487 } 488 activeConnection = null; 489 localFile = new File(destDir, localPath); 490 if (Main.platform.rename(destDirFile, localFile)) { 491 Main.pref.putCollection(prefKey, 492 Arrays.asList(Long.toString(System.currentTimeMillis()), localFile.toString())); 493 } else { 494 Main.warn(tr("Failed to rename file {0} to {1}.", 495 destDirFile.getPath(), localFile.getPath())); 496 } 497 } catch (IOException e) { 498 if (age >= maxAgeMillis && age < maxAgeMillis*2) { 499 Main.warn(tr("Failed to load {0}, use cached file and retry next time: {1}", urlStr, e)); 500 return localFile; 501 } else { 502 throw e; 503 } 504 } 505 506 return localFile; 507 } 508 509 private static void checkOfflineAccess(String urlString) { 510 OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlString, Main.getJOSMWebsite()); 511 OnlineResource.OSM_API.checkOfflineAccess(urlString, OsmApi.getOsmApi().getServerUrl()); 512 } 513 514 /** 515 * Attempts to disconnect an URL connection. 516 * @see HttpClient#disconnect() 517 * @since 9411 518 */ 519 @Override 520 public void close() { 521 if (activeConnection != null) { 522 activeConnection.disconnect(); 523 } 524 } 525 526 /** 527 * Clears the cached file 528 * @throws IOException if any I/O error occurs 529 * @since 10993 530 */ 531 public void clear() throws IOException { 532 URL url; 533 try { 534 url = new URL(name); 535 if ("file".equals(url.getProtocol())) { 536 return; // this is local file - do not delete it 537 } 538 } catch (MalformedURLException e) { 539 return; // if it's not a URL, then it still might be a local file - better not to delete 540 } 541 File f = getFile(); 542 if (f != null && f.exists()) { 543 Utils.deleteFile(f); 544 } 545 } 546}