001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.BufferedInputStream;
007import java.io.BufferedOutputStream;
008import java.io.File;
009import java.io.FileInputStream;
010import java.io.FileOutputStream;
011import java.io.IOException;
012import java.io.InputStream;
013import java.io.OutputStream;
014import java.net.HttpURLConnection;
015import java.net.MalformedURLException;
016import java.net.URL;
017import java.nio.charset.StandardCharsets;
018import java.util.ArrayList;
019import java.util.Arrays;
020import java.util.Enumeration;
021import java.util.List;
022import java.util.zip.ZipEntry;
023import java.util.zip.ZipFile;
024
025import org.openstreetmap.josm.Main;
026import org.openstreetmap.josm.tools.CheckParameterUtil;
027import org.openstreetmap.josm.tools.Pair;
028import org.openstreetmap.josm.tools.Utils;
029
030/**
031 * Downloads a file and caches it on disk in order to reduce network load.
032 *
033 * Supports URLs, local files, and a custom scheme (<code>resource:</code>) to get
034 * resources from the current *.jar file. (Local caching is only done for URLs.)
035 * <p>
036 * The mirrored file is only downloaded if it has been more than 7 days since
037 * last download. (Time can be configured.)
038 * <p>
039 * The file content is normally accessed with {@link #getInputStream()}, but
040 * you can also get the mirrored copy with {@link #getFile()}.
041 */
042public class CachedFile {
043
044    /**
045     * Caching strategy.
046     */
047    public enum CachingStrategy {
048        /**
049         * If cached file on disk is older than a certain time (7 days by default),
050         * consider the cache stale and try to download the file again.
051         */
052        MaxAge,
053        /**
054         * Similar to MaxAge, considers the cache stale when a certain age is
055         * exceeded. In addition, a If-Modified-Since HTTP header is added.
056         * When the server replies "304 Not Modified", this is considered the same
057         * as a full download.
058         */
059        IfModifiedSince
060    }
061    protected String name;
062    protected long maxAge;
063    protected String destDir;
064    protected String httpAccept;
065    protected CachingStrategy cachingStrategy;
066
067    protected File cacheFile = null;
068    boolean initialized = false;
069
070    public static final long DEFAULT_MAXTIME = -1L;
071    public static final long DAYS = 24*60*60; // factor to get caching time in days
072
073    /**
074     * Constructs a CachedFile object from a given filename, URL or internal resource.
075     *
076     * @param name can be:<ul>
077     *  <li>relative or absolute file name</li>
078     *  <li>{@code file:///SOME/FILE} the same as above</li>
079     *  <li>{@code http://...} a URL. It will be cached on disk.</li></ul>
080     *  <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li>
081     *  <li>{@code josmdir://SOME/FILE} file inside josm config directory (since r7058)</li></ul>
082     */
083    public CachedFile(String name) {
084        this.name = name;
085    }
086
087    /**
088     * Set the name of the resource.
089     * @param name can be:<ul>
090     *  <li>relative or absolute file name</li>
091     *  <li>{@code file:///SOME/FILE} the same as above</li>
092     *  <li>{@code http://...} a URL. It will be cached on disk.</li></ul>
093     *  <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li>
094     *  <li>{@code josmdir://SOME/FILE} file inside josm config directory (since r7058)</li></ul>
095     * @return this object
096     */
097    public CachedFile setName(String name) {
098        this.name = name;
099        return this;
100    }
101
102    /**
103     * Set maximum age of cache file. Only applies to URLs.
104     * When this time has passed after the last download of the file, the
105     * cache is considered stale and a new download will be attempted.
106     * @param maxAge the maximum cache age in seconds
107     * @return this object
108     */
109    public CachedFile setMaxAge(long maxAge) {
110        this.maxAge = maxAge;
111        return this;
112    }
113
114    /**
115     * Set the destination directory for the cache file. Only applies to URLs.
116     * @param destDir the destination directory
117     * @return this object
118     */
119    public CachedFile setDestDir(String destDir) {
120        this.destDir = destDir;
121        return this;
122    }
123
124    /**
125     * Set the accepted MIME types sent in the HTTP Accept header. Only applies to URLs.
126     * @param httpAccept the accepted MIME types
127     * @return this object
128     */
129    public CachedFile setHttpAccept(String httpAccept) {
130        this.httpAccept = httpAccept;
131        return this;
132    }
133
134    /**
135     * Set the caching strategy. Only applies to URLs.
136     * @param cachingStrategy
137     * @return this object
138     */
139    public CachedFile setCachingStrategy(CachingStrategy cachingStrategy) {
140        this.cachingStrategy = cachingStrategy;
141        return this;
142    }
143
144    public String getName() {
145        return name;
146    }
147
148    public long getMaxAge() {
149        return maxAge;
150    }
151
152    public String getDestDir() {
153        return destDir;
154    }
155
156    public String getHttpAccept() {
157        return httpAccept;
158    }
159
160    public CachingStrategy getCachingStrategy() {
161        return cachingStrategy;
162    }
163
164    /**
165     * Get InputStream to the requested resource.
166     * @return the InputStream
167     * @throws IOException when the resource with the given name could not be retrieved
168     */
169    public InputStream getInputStream() throws IOException {
170        File file = getFile();
171        if (file == null) {
172            if (name.startsWith("resource://")) {
173                InputStream is = getClass().getResourceAsStream(
174                        name.substring("resource:/".length()));
175                if (is == null)
176                    throw new IOException(tr("Failed to open input stream for resource ''{0}''", name));
177                return is;
178            } else throw new IOException();
179        }
180        return new FileInputStream(file);
181    }
182
183    /**
184     * Get local file for the requested resource.
185     * @return The local cache file for URLs. If the resource is a local file,
186     * returns just that file.
187     * @throws IOException when the resource with the given name could not be retrieved
188     */
189    public File getFile() throws IOException {
190        if (initialized)
191            return cacheFile;
192        initialized = true;
193        URL url;
194        try {
195            url = new URL(name);
196            if ("file".equals(url.getProtocol())) {
197                cacheFile = new File(name.substring("file:/".length()));
198                if (!cacheFile.exists()) {
199                    cacheFile = new File(name.substring("file://".length()));
200                }
201            } else {
202                cacheFile = checkLocal(url);
203            }
204        } catch (MalformedURLException e) {
205            if (name.startsWith("resource://")) {
206                return null;
207            } else if (name.startsWith("josmdir://")) {
208                cacheFile = new File(Main.pref.getPreferencesDir(), name.substring("josmdir://".length()));
209            } else {
210                cacheFile = new File(name);
211            }
212        }
213        if (cacheFile == null)
214            throw new IOException("Unable to get cache file for "+name);
215        return cacheFile;
216    }
217
218    /**
219     * Looks for a certain entry inside a zip file and returns the entry path.
220     *
221     * Replies a file in the top level directory of the ZIP file which has an
222     * extension <code>extension</code>. If more than one files have this
223     * extension, the last file whose name includes <code>namepart</code>
224     * is opened.
225     *
226     * @param extension  the extension of the file we're looking for
227     * @param namepart the name part
228     * @return The zip entry path of the matching file. Null if this cached file
229     * doesn't represent a zip file or if there was no matching
230     * file in the ZIP file.
231     */
232    public String findZipEntryPath(String extension, String namepart) {
233        Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart);
234        if (ze == null) return null;
235        return ze.a;
236    }
237
238    /**
239     * Like {@link #findZipEntryPath}, but returns the corresponding InputStream.
240     * @param extension  the extension of the file we're looking for
241     * @param namepart the name part
242     * @return InputStream to the matching file. Null if this cached file
243     * doesn't represent a zip file or if there was no matching
244     * file in the ZIP file.
245     * @since 6148
246     */
247    public InputStream findZipEntryInputStream(String extension, String namepart) {
248        Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart);
249        if (ze == null) return null;
250        return ze.b;
251    }
252
253    @SuppressWarnings("resource")
254    private Pair<String, InputStream> findZipEntryImpl(String extension, String namepart) {
255        File file = null;
256        try {
257            file = getFile();
258        } catch (IOException ex) {
259        }
260        if (file == null)
261            return null;
262        Pair<String, InputStream> res = null;
263        try {
264            ZipFile zipFile = new ZipFile(file, StandardCharsets.UTF_8);
265            ZipEntry resentry = null;
266            Enumeration<? extends ZipEntry> entries = zipFile.entries();
267            while (entries.hasMoreElements()) {
268                ZipEntry entry = entries.nextElement();
269                if (entry.getName().endsWith("." + extension)) {
270                    /* choose any file with correct extension. When more than
271                        one file, prefer the one which matches namepart */
272                    if (resentry == null || entry.getName().indexOf(namepart) >= 0) {
273                        resentry = entry;
274                    }
275                }
276            }
277            if (resentry != null) {
278                InputStream is = zipFile.getInputStream(resentry);
279                res = Pair.create(resentry.getName(), is);
280            } else {
281                Utils.close(zipFile);
282            }
283        } catch (Exception e) {
284            if (file.getName().endsWith(".zip")) {
285                Main.warn(tr("Failed to open file with extension ''{2}'' and namepart ''{3}'' in zip file ''{0}''. Exception was: {1}",
286                        file.getName(), e.toString(), extension, namepart));
287            }
288        }
289        return res;
290    }
291
292    /**
293     * Clear the cache for the given resource.
294     * This forces a fresh download.
295     * @param name the URL
296     */
297    public static void cleanup(String name) {
298        cleanup(name, null);
299    }
300
301    /**
302     * Clear the cache for the given resource.
303     * This forces a fresh download.
304     * @param name the URL
305     * @param destDir the destination directory (see {@link #setDestDir(java.lang.String)})
306     */
307    public static void cleanup(String name, String destDir) {
308        URL url;
309        try {
310            url = new URL(name);
311            if (!"file".equals(url.getProtocol())) {
312                String prefKey = getPrefKey(url, destDir);
313                List<String> localPath = new ArrayList<>(Main.pref.getCollection(prefKey));
314                if (localPath.size() == 2) {
315                    File lfile = new File(localPath.get(1));
316                    if(lfile.exists()) {
317                        lfile.delete();
318                    }
319                }
320                Main.pref.putCollection(prefKey, null);
321            }
322        } catch (MalformedURLException e) {
323            Main.warn(e);
324        }
325    }
326
327    /**
328     * Get preference key to store the location and age of the cached file.
329     * 2 resources that point to the same url, but that are to be stored in different
330     * directories will not share a cache file.
331     */
332    private static String getPrefKey(URL url, String destDir) {
333        StringBuilder prefKey = new StringBuilder("mirror.");
334        if (destDir != null) {
335            prefKey.append(destDir);
336            prefKey.append(".");
337        }
338        prefKey.append(url.toString());
339        return prefKey.toString().replaceAll("=","_");
340    }
341
342    private File checkLocal(URL url) throws IOException {
343        String prefKey = getPrefKey(url, destDir);
344        String urlStr = url.toExternalForm();
345        long age = 0L;
346        long lMaxAge = maxAge;
347        Long ifModifiedSince = null;
348        File localFile = null;
349        List<String> localPathEntry = new ArrayList<>(Main.pref.getCollection(prefKey));
350        boolean offline = false;
351        try {
352            checkOfflineAccess(urlStr);
353        } catch (OfflineAccessException e) {
354            offline = true;
355        }
356        if (localPathEntry.size() == 2) {
357            localFile = new File(localPathEntry.get(1));
358            if (!localFile.exists()) {
359                localFile = null;
360            } else {
361                if ( maxAge == DEFAULT_MAXTIME
362                        || maxAge <= 0 // arbitrary value <= 0 is deprecated
363                ) {
364                    lMaxAge = Main.pref.getInteger("mirror.maxtime", 7*24*60*60); // one week
365                }
366                age = System.currentTimeMillis() - Long.parseLong(localPathEntry.get(0));
367                if (offline || age < lMaxAge*1000) {
368                    return localFile;
369                }
370                if (cachingStrategy == CachingStrategy.IfModifiedSince) {
371                    ifModifiedSince = Long.parseLong(localPathEntry.get(0));
372                }
373            }
374        }
375        if (destDir == null) {
376            destDir = Main.pref.getCacheDirectory().getPath();
377        }
378
379        File destDirFile = new File(destDir);
380        if (!destDirFile.exists()) {
381            destDirFile.mkdirs();
382        }
383
384        // No local file + offline => nothing to do
385        if (offline) {
386            return null;
387        }
388
389        String a = urlStr.replaceAll("[^A-Za-z0-9_.-]", "_");
390        String localPath = "mirror_" + a;
391        destDirFile = new File(destDir, localPath + ".tmp");
392        try {
393            HttpURLConnection con = connectFollowingRedirect(url, httpAccept, ifModifiedSince);
394            if (ifModifiedSince != null && con.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) {
395                if (Main.isDebugEnabled()) {
396                    Main.debug("304 Not Modified ("+urlStr+")");
397                }
398                if (localFile == null)
399                    throw new AssertionError();
400                Main.pref.putCollection(prefKey,
401                        Arrays.asList(Long.toString(System.currentTimeMillis()), localPathEntry.get(1)));
402                return localFile;
403            }
404            try (
405                InputStream bis = new BufferedInputStream(con.getInputStream());
406                OutputStream fos = new FileOutputStream(destDirFile);
407                OutputStream bos = new BufferedOutputStream(fos)
408            ) {
409                byte[] buffer = new byte[4096];
410                int length;
411                while ((length = bis.read(buffer)) > -1) {
412                    bos.write(buffer, 0, length);
413                }
414            }
415            localFile = new File(destDir, localPath);
416            if (Main.platform.rename(destDirFile, localFile)) {
417                Main.pref.putCollection(prefKey,
418                        Arrays.asList(Long.toString(System.currentTimeMillis()), localFile.toString()));
419            } else {
420                Main.warn(tr("Failed to rename file {0} to {1}.",
421                destDirFile.getPath(), localFile.getPath()));
422            }
423        } catch (IOException e) {
424            if (age >= lMaxAge*1000 && age < lMaxAge*1000*2) {
425                Main.warn(tr("Failed to load {0}, use cached file and retry next time: {1}", urlStr, e));
426                return localFile;
427            } else {
428                throw e;
429            }
430        }
431
432        return localFile;
433    }
434
435    private static void checkOfflineAccess(String urlString) {
436        OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlString, Main.getJOSMWebsite());
437        OnlineResource.OSM_API.checkOfflineAccess(urlString, Main.pref.get("osm-server.url", OsmApi.DEFAULT_API_URL));
438    }
439
440    /**
441     * Opens a connection for downloading a resource.
442     * <p>
443     * Manually follows redirects because
444     * {@link HttpURLConnection#setFollowRedirects(boolean)} fails if the redirect
445     * is going from a http to a https URL, see <a href="https://bugs.openjdk.java.net/browse/JDK-4620571">bug report</a>.
446     * <p>
447     * This can cause problems when downloading from certain GitHub URLs.
448     *
449     * @param downloadUrl The resource URL to download
450     * @param httpAccept The accepted MIME types sent in the HTTP Accept header. Can be {@code null}
451     * @param ifModifiedSince The download time of the cache file, optional
452     * @return The HTTP connection effectively linked to the resource, after all potential redirections
453     * @throws MalformedURLException If a redirected URL is wrong
454     * @throws IOException If any I/O operation goes wrong
455     * @throws OfflineAccessException if resource is accessed in offline mode, in any protocol
456     * @since 6867
457     */
458    public static HttpURLConnection connectFollowingRedirect(URL downloadUrl, String httpAccept, Long ifModifiedSince) throws MalformedURLException, IOException {
459        CheckParameterUtil.ensureParameterNotNull(downloadUrl, "downloadUrl");
460        String downloadString = downloadUrl.toExternalForm();
461
462        checkOfflineAccess(downloadString);
463
464        HttpURLConnection con = null;
465        int numRedirects = 0;
466        while(true) {
467            con = Utils.openHttpConnection(downloadUrl);
468            if (con == null) {
469                throw new IOException("Cannot open http connection to "+downloadString);
470            }
471            if (ifModifiedSince != null) {
472                con.setIfModifiedSince(ifModifiedSince);
473            }
474            con.setInstanceFollowRedirects(false);
475            con.setConnectTimeout(Main.pref.getInteger("socket.timeout.connect",15)*1000);
476            con.setReadTimeout(Main.pref.getInteger("socket.timeout.read",30)*1000);
477            if (Main.isDebugEnabled()) {
478                Main.debug("GET "+downloadString);
479            }
480            if (httpAccept != null) {
481                if (Main.isTraceEnabled()) {
482                    Main.trace("Accept: "+httpAccept);
483                }
484                con.setRequestProperty("Accept", httpAccept);
485            }
486            try {
487                con.connect();
488            } catch (IOException e) {
489                Main.addNetworkError(downloadUrl, Utils.getRootCause(e));
490                throw e;
491            }
492            switch(con.getResponseCode()) {
493            case HttpURLConnection.HTTP_OK:
494                return con;
495            case HttpURLConnection.HTTP_NOT_MODIFIED:
496                if (ifModifiedSince != null)
497                    return con;
498            case HttpURLConnection.HTTP_MOVED_PERM:
499            case HttpURLConnection.HTTP_MOVED_TEMP:
500            case HttpURLConnection.HTTP_SEE_OTHER:
501                String redirectLocation = con.getHeaderField("Location");
502                if (redirectLocation == null) {
503                    /* I18n: argument is HTTP response code */
504                    String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header."+
505                            " Can''t redirect. Aborting.", con.getResponseCode());
506                    throw new IOException(msg);
507                }
508                downloadUrl = new URL(redirectLocation);
509                downloadString = downloadUrl.toExternalForm();
510                // keep track of redirect attempts to break a redirect loops if it happens
511                // to occur for whatever reason
512                numRedirects++;
513                if (numRedirects >= Main.pref.getInteger("socket.maxredirects", 5)) {
514                    String msg = tr("Too many redirects to the download URL detected. Aborting.");
515                    throw new IOException(msg);
516                }
517                Main.info(tr("Download redirected to ''{0}''", downloadString));
518                break;
519            default:
520                String msg = tr("Failed to read from ''{0}''. Server responded with status code {1}.", downloadString, con.getResponseCode());
521                throw new IOException(msg);
522            }
523        }
524    }
525}