Source code

001package org.hl7.fhir.utilities.cache;
002
003/*-
004 * #%L
005 * org.hl7.fhir.utilities
006 * %%
007 * Copyright (C) 2014 - 2019 Health Level 7
008 * %%
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 * 
013 *      http://www.apache.org/licenses/LICENSE-2.0
014 * 
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 * #L%
021 */
022
023
024import java.io.BufferedOutputStream;
025import java.io.ByteArrayOutputStream;
026import java.io.File;
027import java.io.FileInputStream;
028import java.io.IOException;
029import java.io.InputStream;
030import java.net.HttpURLConnection;
031import java.net.URL;
032import java.net.URLConnection;
033import java.sql.Timestamp;
034import java.text.ParseException;
035import java.text.SimpleDateFormat;
036import java.time.Instant;
037import java.util.ArrayList;
038import java.util.Collections;
039import java.util.Comparator;
040import java.util.Date;
041import java.util.HashMap;
042import java.util.List;
043import java.util.Locale;
044import java.util.Map;
045import java.util.Map.Entry;
046
047import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
048import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
049import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
050import org.apache.commons.io.FileUtils;
051import org.hl7.fhir.exceptions.FHIRException;
052import org.hl7.fhir.utilities.IniFile;
053import org.hl7.fhir.utilities.TextFile;
054import org.hl7.fhir.utilities.Utilities;
055import org.hl7.fhir.utilities.cache.NpmPackage.NpmPackageFolder;
056import org.hl7.fhir.utilities.cache.PackageCacheManager.BuildRecord;
057import org.hl7.fhir.utilities.cache.PackageCacheManager.BuildRecordSorter;
058import org.hl7.fhir.utilities.json.JSONUtil;
059
060import com.google.gson.GsonBuilder;
061import com.google.gson.JsonArray;
062import com.google.gson.JsonElement;
063import com.google.gson.JsonObject;
064
065/**
066 * Package cache manager
067 * 
068 * API:
069 * 
070 * constructor
071 *  getPackageUrl
072 *  getPackageId
073 *  findPackageCache 
074 *  addPackageToCache  
075 * 
076 * @author Grahame Grieve
077 *
078 */
079public class PackageCacheManager {
080
081  public class BuildRecordSorter implements Comparator<BuildRecord> {
082
083    @Override
084    public int compare(BuildRecord arg0, BuildRecord arg1) {
085      return arg1.date.compareTo(arg0.date);
086    }
087  }
088
089  public class BuildRecord {
090
091    private String url;
092    private String packageId;
093    private String repo;
094    private Date date;
095    public BuildRecord(String url, String packageId, String repo, Date date) {
096      super();
097      this.url = url;
098      this.packageId = packageId;
099      this.repo = repo;
100      this.date = date;
101    }
102    public String getUrl() {
103      return url;
104    }
105    public String getPackageId() {
106      return packageId;
107    }
108    public String getRepo() {
109      return repo;
110    }
111    public Date getDate() {
112      return date;
113    }
114    
115
116  }
117
118  /** if you don't provide and implementation of this interface, the PackageCacheManager will use the web directly. 
119   * 
120   * You can use this interface to 
121   * @author graha
122   *
123   */
124  public interface INetworkServices {
125    
126    InputStream resolvePackage(String packageId, String version);
127  }
128  
129  public class VersionHistory {
130    private String id;
131    private String canonical;
132    private String current;
133    private Map<String, String> versions = new HashMap<>();
134    public String getCanonical() {
135      return canonical;
136    }
137    public String getCurrent() {
138      return current;
139    }
140    public Map<String, String> getVersions() {
141      return versions;
142    }
143    public String getId() {
144      return id;
145    } 
146  }
147
148
149  public class PackageEntry {
150
151    private byte[] bytes;
152    private String name;
153
154    public PackageEntry(String name) {
155      this.name = name;
156    }
157
158    public PackageEntry(String name, byte[] bytes) {
159      this.name = name;
160      this.bytes = bytes;
161    }
162  }
163
164  public static final String PACKAGE_REGEX = "^[a-z][a-z0-9\\_\\-]*(\\.[a-z0-9\\_\\-]+)+$";
165  public static final String PACKAGE_VERSION_REGEX = "^[a-z][a-z0-9\\_\\-]*(\\.[a-z0-9\\_\\-]+)+\\#[a-z0-9\\-\\_]+(\\.[a-z0-9\\-\\_]+)*$";
166
167  private static final int BUFFER_SIZE = 1024;
168  private static final String CACHE_VERSION = "3"; // second version - see wiki page
169  private static final int ANALYSIS_VERSION = 2;
170
171  private String cacheFolder;
172  private boolean buildLoaded;
173  private JsonArray buildInfo;
174  private boolean progress = true;
175  private List<NpmPackage> temporaryPackages = new ArrayList<NpmPackage>();
176  private Map<String, String> ciList = new HashMap<String, String>();
177  private List<String> allUrls;
178  private Map<String, VersionHistory> historyCache = new HashMap<>();
179  
180  public PackageCacheManager(boolean userMode, int toolsVersion) throws IOException {
181    if (userMode)
182      cacheFolder = Utilities.path(System.getProperty("user.home"), ".fhir", "packages");
183    else
184      cacheFolder = Utilities.path("var", "lib", ".fhir", "packages");
185    if (!(new File(cacheFolder).exists()))
186      Utilities.createDirectory(cacheFolder);
187    if (!(new File(Utilities.path(cacheFolder, "packages.ini")).exists()))
188      TextFile.stringToFile("[cache]\r\nversion="+CACHE_VERSION+"\r\n\r\n[urls]\r\n\r\n[local]\r\n\r\n", Utilities.path(cacheFolder, "packages.ini"), false);  
189    IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
190    boolean save = false;
191    String v = ini.getStringProperty("cache", "version");    
192    if (!CACHE_VERSION.equals(v)) {
193      clearCache();
194      ini.setStringProperty("cache", "version", CACHE_VERSION, null);
195      save = true;
196    }
197    save = initUrlMaps(ini, save);
198    if (save) {
199      if (!CACHE_VERSION.equals(ini.getStringProperty("cache", "version"))) {
200        throw new Error("what?");
201      }
202      ini.save();
203    }
204  }
205
206  public static String userDir() throws IOException {
207    return Utilities.path(System.getProperty("user.home"), ".fhir", "packages");
208  }
209
210  // ========================= Utilities ============================================================================
211  
212  private List<String> sorted(String[] keys) {
213    List<String> names = new ArrayList<String>();
214    for (String s : keys)
215      names.add(s);
216    Collections.sort(names);
217    return names;
218  }
219
220  private NpmPackage loadPackageInfo(String path) throws IOException {
221    NpmPackage pi = NpmPackage.fromFolder(path);
222    return pi;
223  }
224
225  public String getFolder() {
226    return cacheFolder;
227  }
228
229
230
231
232
233  private NpmPackage checkCurrency(String id, NpmPackage p) {
234    // special case: current versions roll over, and we have to check their currency
235    try {
236      String url = ciList.get(id);
237      JsonObject json = fetchJson(Utilities.pathURL(url, "package.manifest.json"));
238      String currDate = JSONUtil.str(json, "date");
239      String packDate = p.date();
240      if (!currDate.equals(packDate))
241        return null; // nup, we need a new copy 
242      return p;
243    } catch (Exception e) {
244      return p;
245    }
246  }
247  
248  private JsonObject fetchJson(String source) throws IOException {
249    URL url = new URL(source);
250    URLConnection c = url.openConnection();
251    return (JsonObject) new com.google.gson.JsonParser().parse(TextFile.streamToString(c.getInputStream()));
252  }
253  
254  private InputStream fetchFromUrlSpecific(String source, boolean optional) throws FHIRException {
255    try {
256      URL url = new URL(source);
257      URLConnection c = url.openConnection();
258      return c.getInputStream();
259    } catch (Exception e) {
260      if (optional)
261        return null;
262      else
263        throw new FHIRException(e.getMessage(), e);
264    }
265  }
266
267  // ========================= Full Cache Management ============================================================================
268
269  private void clearCache() throws IOException {
270    for (File f : new File(cacheFolder).listFiles()) {
271      if (f.isDirectory())
272        FileUtils.deleteDirectory(f);
273      else if (!f.getName().equals("packages.ini"))
274        FileUtils.forceDelete(f);
275    }    
276    IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
277    ini.removeSection("packages");
278    ini.save();
279  }
280
281//  private void checkDeleteVersion(String id, String ver, int minVer) {
282//    if (hasPackage(id, ver)) {
283//      boolean del = true;
284//      NpmPackage pck;
285//      try {
286//        pck = loadPackageFromCacheOnly(id, ver);
287//        if (pck.getNpm().has("tools-version")) {
288//          del = pck.getNpm().get("tools-version").getAsInt() < minVer;
289//        }
290//      } catch (Exception e) {
291//      }
292//      if (del)
293//        try {
294//          removePackage(id, ver);
295//        } catch (IOException e) {
296//        }
297//    }
298//  }
299  
300//private void convertPackageCacheFrom1To2() throws IOException {
301//for (File f : new File(cacheFolder).listFiles()) {
302//  if (f.isDirectory() && f.getName().contains("-")) {
303//    String s = f.getName();
304//    int i = s.lastIndexOf("-");
305//    s = s.substring(0, i)+"#"+s.substring(i+1);
306//    File nf = new File(Utilities.path(cacheFolder, s));
307//    if (!f.renameTo(nf))
308//      throw new IOException("Unable to rename "+f.getAbsolutePath()+" to "+nf.getAbsolutePath());
309//  }
310//}
311//}
312  
313  // ========================= URL maps (while waiting for package registry) ============================================================================
314  
315  public boolean initUrlMaps(IniFile ini, boolean save) {
316    save = checkIniHasMapping("hl7.fhir.core", "http://hl7.org/fhir", ini) || save;
317    save = checkIniHasMapping("hl7.fhir.pubpack", "http://fhir.org/packages/hl7.fhir.pubpack", ini) || save;
318    save = checkIniHasMapping("hl7.fhir.xver-extensions", "http://fhir.org/packages/hl7.fhir.xver-extensions", ini) || save;
319    
320    save = checkIniHasMapping("hl7.fhir.r2.core", "http://hl7.org/fhir/DSTU2/hl7.fhir.r2.core.tgz", ini) || save;
321    save = checkIniHasMapping("hl7.fhir.r2.examples", "http://hl7.org/fhir/DSTU2/hl7.fhir.r2.examples.tgz", ini) || save;
322    save = checkIniHasMapping("hl7.fhir.r2.elements", "http://hl7.org/fhir/DSTU2/hl7.fhir.r2.elements.tgz", ini) || save;
323    save = checkIniHasMapping("hl7.fhir.r2.expansions", "http://hl7.org/fhir/DSTU2/hl7.fhir.r2.expansions.tgz", ini) || save;
324    save = checkIniHasMapping("hl7.fhir.r2b.core", "http://hl7.org/fhir/2016May/hl7.fhir.r2b.core.tgz", ini) || save;
325    save = checkIniHasMapping("hl7.fhir.r2b.examples", "http://hl7.org/fhir/2016May/hl7.fhir.r2b.examples.tgz", ini) || save;
326    save = checkIniHasMapping("hl7.fhir.r2b.elements", "http://hl7.org/fhir/2016May/hl7.fhir.r2b.elements.tgz", ini) || save;
327    save = checkIniHasMapping("hl7.fhir.r2b.expansions", "http://hl7.org/fhir/2016May/hl7.fhir.r2b.expansions.tgz", ini) || save;
328    save = checkIniHasMapping("hl7.fhir.r3.core", "http://hl7.org/fhir/STU3/hl7.fhir.r3.core.tgz", ini) || save;
329    save = checkIniHasMapping("hl7.fhir.r3.examples", "http://hl7.org/fhir/STU3/hl7.fhir.r3.examples.tgz", ini) || save;
330    save = checkIniHasMapping("hl7.fhir.r3.elements", "http://hl7.org/fhir/STU3/hl7.fhir.r3.elements.tgz", ini) || save;
331    save = checkIniHasMapping("hl7.fhir.r3.expansions", "http://hl7.org/fhir/STU3/hl7.fhir.r3.expansions.tgz", ini) || save;
332    save = checkIniHasMapping("hl7.fhir.r4.core", "http://hl7.org/fhir/R4/hl7.fhir.r4.core.tgz", ini) || save;
333    save = checkIniHasMapping("hl7.fhir.r4.examples", "http://hl7.org/fhir/R4/hl7.fhir.r4.examples.tgz", ini) || save;
334    save = checkIniHasMapping("hl7.fhir.r4.elements", "http://hl7.org/fhir/R4/hl7.fhir.r4.elements.tgz", ini) || save;
335    save = checkIniHasMapping("hl7.fhir.r4.expansions", "http://hl7.org/fhir/R4/hl7.fhir.r4.expansions.tgz", ini) || save;
336
337    save = checkIniHasMapping("hl7.fhir.r5.core", "http://hl7.org/fhir/2020Feb/hl7.fhir.r5.core.tgz", ini) || save;
338    save = checkIniHasMapping("hl7.fhir.r5.expansions", "http://hl7.org/fhir/2020Feb/hl7.fhir.r5.expansions.tgz", ini) || save;
339
340    save = checkIniHasMapping("fhir.argonaut.ehr", "http://fhir.org/guides/argonaut", ini) || save;
341    save = checkIniHasMapping("fhir.argonaut.pd", "http://fhir.org/guides/argonaut-pd", ini) || save;
342    save = checkIniHasMapping("fhir.argonaut.scheduling", "http://fhir.org/guides/argonaut-scheduling", ini) || save;
343    save = checkIniHasMapping("fhir.hspc.acog", "http://hl7.org/fhir/fpar", ini) || save;
344    save = checkIniHasMapping("hl7.fhir.au.argonaut", "http://hl7.org.au/fhir/argonaut", ini) || save;
345    save = checkIniHasMapping("hl7.fhir.au.base", "http://hl7.org.au/fhir/base", ini) || save;
346    save = checkIniHasMapping("hl7.fhir.au.pd", "http://hl7.org.au/fhir/pd", ini) || save;
347    save = checkIniHasMapping("hl7.fhir.smart", "http://hl7.org/fhir/smart-app-launch", ini) || save;
348    save = checkIniHasMapping("hl7.fhir.snomed", "http://hl7.org/fhir/ig/snomed", ini) || save;
349    save = checkIniHasMapping("hl7.fhir.test.v10", "http://hl7.org/fhir/test-ig-10", ini) || save;
350    save = checkIniHasMapping("hl7.fhir.test.v30", "http://hl7.org/fhir/test", ini) || save;
351    save = checkIniHasMapping("hl7.fhir.us.breastcancer", "http://hl7.org/fhir/us/breastcancer", ini) || save;
352    save = checkIniHasMapping("hl7.fhir.us.ccda", "http://hl7.org/fhir/us/ccda", ini) || save;
353    save = checkIniHasMapping("hl7.fhir.us.cds.opioids", "http://hl7.org/fhir/ig/opioid-cds", ini) || save;
354    save = checkIniHasMapping("hl7.fhir.us.core", "http://hl7.org/fhir/us/core", ini) || save;
355    save = checkIniHasMapping("hl7.fhir.us.dafresearch", "http://hl7.org/fhir/us/daf-research", ini) || save;
356    save = checkIniHasMapping("hl7.fhir.us.ecr", "http://fhir.hl7.org/us/ecr", ini) || save;
357    save = checkIniHasMapping("hl7.fhir.us.hai", "http://hl7.org/fhir/us/hai", ini) || save;
358    save = checkIniHasMapping("hl7.fhir.us.hedis", "http://hl7.org/fhir/us/hedis", ini) || save;
359    save = checkIniHasMapping("hl7.fhir.us.meds", "http://hl7.org/fhir/us/meds", ini) || save;
360    save = checkIniHasMapping("hl7.fhir.us.qicore", "http://hl7.org/fhir/us/qicore", ini) || save;
361    save = checkIniHasMapping("hl7.fhir.us.sdc", "http://hl7.org/fhir/us/sdc", ini) || save;
362    save = checkIniHasMapping("hl7.fhir.us.sdcde", "http://hl7.org/fhir/us/sdcde", ini) || save;
363    save = checkIniHasMapping("hl7.fhir.uv.genomicsreporting", "http://hl7.org/fhir/uv/genomics-reporting", ini) || save;
364    save = checkIniHasMapping("hl7.fhir.uv.immds", "http://hl7.org/fhir/uv/cdsi", ini) || save;
365    save = checkIniHasMapping("hl7.fhir.uv.ips", "http://hl7.org/fhir/uv/ips", ini) || save;
366    save = checkIniHasMapping("hl7.fhir.uv.phd", "http://hl7.org/fhir/devices", ini) || save;
367    save = checkIniHasMapping("hl7.fhir.uv.vhdir", "http://hl7.org/fhir/ig/vhdir", ini) || save;
368    save = checkIniHasMapping("hl7.fhir.vn.base", "http://hl7.org/fhir/ig/vietnam", ini) || save;
369    save = checkIniHasMapping("hl7.fhir.uv.bulkdata", "http://hl7.org/fhir/uv/bulkdata", ini) || save;
370    save = checkIniHasMapping("hl7.fhir.us.bulkdata", "http://hl7.org/fhir/uv/bulkdata", ini) || save;
371    save = checkIniHasMapping("hl7.fhir.vocabpoc", "http://hl7.org/fhir/ig/vocab-poc", ini) || save;
372    return save;
373  }
374
375  private boolean checkIniHasMapping(String pid, String curl, IniFile ini) {
376    if (curl.equals(ini.getStringProperty("urls", pid)))
377      return false;
378    ini.setStringProperty("urls", pid, curl, null);
379    return true;
380  }
381
382  public void recordMap(String url, String id) throws IOException {
383    if (url == null)
384      return;
385    
386    if (!(new File(Utilities.path(cacheFolder, "packages.ini")).exists()))
387        throw new Error("File "+Utilities.path(cacheFolder, "packages.ini")+" not found #1");
388    IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
389    ini.setStringProperty("urls", id, url, null);
390    if (!CACHE_VERSION.equals(ini.getStringProperty("cache", "version"))) {
391      throw new Error("File "+Utilities.path(cacheFolder, "packages.ini")+" cache version mismatch: expected '"+CACHE_VERSION+"', found '"+ini.getStringProperty("cache", "version")+"'");
392    }
393    if (!(new File(Utilities.path(cacheFolder, "packages.ini")).exists()))
394      throw new Error("File "+Utilities.path(cacheFolder, "packages.ini")+" not found #2");
395
396    ini.save();
397    if (!(new File(Utilities.path(cacheFolder, "packages.ini")).exists()))
398      throw new Error("File "+Utilities.path(cacheFolder, "packages.ini")+" not found #3");
399  }
400
401  public String getPackageUrl(String id) throws IOException {
402    IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
403    return ini.getStringProperty("urls", id); 
404  }
405  
406  public String getPackageId(String url) throws IOException {
407    IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
408    String[] ids = ini.getPropertyNames("urls");
409    if (ids != null) {
410      for (String id : ids) {
411        if (url.equals(ini.getStringProperty("urls", id)))
412          return id;
413      }
414    }
415    return null;
416  }
417  
418  public void loadFromBuildServer() throws IOException, ParseException {
419    buildLoaded = true; // whether it succeeds or not
420    URL url = new URL("https://build.fhir.org/ig/qas.json?nocache=" + System.currentTimeMillis());
421    HttpURLConnection connection = (HttpURLConnection) url.openConnection();
422    connection.setRequestMethod("GET");
423    InputStream json = connection.getInputStream();
424    buildInfo = (JsonArray) new com.google.gson.JsonParser().parse(TextFile.streamToString(json));
425  
426    List<BuildRecord> builds = new ArrayList<>();
427    
428    for (JsonElement n : buildInfo) {
429      JsonObject o = (JsonObject) n;
430      if (o.has("url") && o.has("package-id") && o.get("package-id").getAsString().contains(".")) {
431        String u = o.get("url").getAsString();
432        if (u.contains("/ImplementationGuide/"))
433          u = u.substring(0, u.indexOf("/ImplementationGuide/"));
434        builds.add(new BuildRecord(u, o.get("package-id").getAsString(), o.get("repo").getAsString(), readDate(o.get("date").getAsString())));
435      }
436    }
437    Collections.sort(builds, new BuildRecordSorter());
438    for (BuildRecord bld : builds) {
439      if (!ciList.containsKey(bld.getPackageId())) {
440        recordMap(bld.getUrl(), bld.getPackageId());
441        ciList.put(bld.getPackageId(), "https://build.fhir.org/ig/"+bld.getRepo());        
442      }
443    }
444  }
445
446  private Date readDate(String s) throws ParseException {
447    SimpleDateFormat sdf = new SimpleDateFormat("EEE, dd MMM, yyyy HH:mm:ss Z", new Locale("en", "US"));
448    return sdf.parse(s);
449  }
450
451  public boolean isBuildLoaded() {
452    return buildLoaded;
453  }
454
455
456  public String buildPath(String url) {
457    for (JsonElement e : buildInfo) {
458      JsonObject j = (JsonObject) e;
459      if (j.has("url") && (url.equals(j.get("url").getAsString()) || j.get("url").getAsString().startsWith(url+"/ImplementationGuide"))) {
460        return "https://build.fhir.org/ig/"+j.get("repo").getAsString();
461      }
462    }
463    return null;
464  }
465 
466  public boolean checkBuildLoaded() throws IOException, ParseException {
467    if (isBuildLoaded())
468      return true;
469    loadFromBuildServer();
470    return false;
471  }
472  
473  public Map<String, String> getCiList() {
474    return ciList;
475  }
476
477  public List<String> getUrls() throws IOException {
478    if (allUrls == null)
479    {
480      IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
481      allUrls = new ArrayList<>();
482      for (String s : ini.getPropertyNames("urls"))
483        allUrls.add(ini.getStringProperty("urls", s));
484      try {
485        URL url = new URL("https://raw.githubusercontent.com/FHIR/ig-registry/master/fhir-ig-list.json?nocache=" + System.currentTimeMillis());
486        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
487        connection.setRequestMethod("GET");
488        InputStream json = connection.getInputStream();
489        JsonObject packages = (JsonObject) new com.google.gson.JsonParser().parse(TextFile.streamToString(json));
490        JsonArray guides = packages.getAsJsonArray("guides");
491        for (JsonElement g : guides) {
492          JsonObject gi = (JsonObject) g;
493          if (gi.has("canonical"))
494            if (!allUrls.contains(gi.get("canonical").getAsString()))
495              allUrls.add(gi.get("canonical").getAsString());
496        }
497      } catch (Exception e) {
498        System.out.println("Listing known Implementation Guides failed: "+e.getMessage());
499      }
500    }
501    return allUrls;    
502  }
503
504  // ========================= Package API ============================================================================
505
506  public void removePackage(String id, String ver) throws IOException  {
507    String f = Utilities.path(cacheFolder, id+"#"+ver);
508    Utilities.clearDirectory(f);    
509    IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
510    ini.removeProperty("packages", id+"#"+ver);
511    ini.save();
512    new File(f).delete();
513  }
514
515  /**
516   * get the latest version of the package from what is in the cache
517   * @param id
518   * @return
519   * @throws IOException
520   */
521  public NpmPackage loadPackageFromCacheOnly(String id) throws IOException {
522    List<String> l = sorted(new File(cacheFolder).list());
523    for (int i = l.size()-1; i >= 0; i--) {
524      String f = l.get(i);
525      if (f.startsWith(id+"#")) {
526        return loadPackageInfo(Utilities.path(cacheFolder, f)); 
527      }
528    }
529    return null;    
530  }
531  
532  /** 
533   * Load the identified package from the cache - it it exists 
534   * 
535   * This is for special purpose only. Generally, use the loadPackage method
536   *  
537   * @param id
538   * @param version
539   * @return
540   * @throws IOException
541   */
542  public NpmPackage loadPackageFromCacheOnly(String id, String version) throws IOException {
543    for (NpmPackage p : temporaryPackages) {
544      if (p.name().equals(id) && ("current".equals(version) || "dev".equals(version) || p.version().equals(version)))
545        return p;
546    }
547    for (String f : sorted(new File(cacheFolder).list())) {
548      if (f.equals(id+"#"+version)) {
549        return loadPackageInfo(Utilities.path(cacheFolder, f)); 
550      }
551    }
552    if ("dev".equals(version))
553      return loadPackageFromCacheOnly(id, "current");
554    else
555      return null;
556  }
557
558  /**
559   * Add an already fetched package to the cache
560   */
561  public NpmPackage addPackageToCache(String id, String version, InputStream tgz, String sourceDesc) throws IOException {
562    if (progress ) {
563      System.out.println("Installing "+id+"#"+(version == null ? "?" : version)+" to the package cache");
564      System.out.print("  Fetching:");
565    }
566    
567    NpmPackage npm = NpmPackage.fromPackage(tgz, sourceDesc, true);
568    
569    recordMap(npm.canonical(), npm.name());
570    
571    if (progress ) {
572      System.out.println();
573      System.out.print("  Installing: ");
574    }
575    if (npm.name() == null || id == null || !id.equals(npm.name())) {
576      if (!id.equals("hl7.fhir.r5.core")) {// temporary work around
577        throw new IOException("Attempt to import a mis-identified package. Expected "+id+", got "+npm.name());
578      }
579    }
580    if (version == null)
581      version = npm.version();
582    
583    String packRoot = Utilities.path(cacheFolder, id+"#"+version);
584    Utilities.createDirectory(packRoot);
585    Utilities.clearDirectory(packRoot);
586      
587    int i = 0;
588    int c = 0;
589    int size = 0;
590    for (Entry<String, NpmPackageFolder> e : npm.getFolders().entrySet()) {
591      String dir = e.getKey().equals("package") ? Utilities.path(packRoot, "package") : Utilities.path(packRoot, "package",  e.getKey());;
592      if (!(new File(dir).exists()))
593        Utilities.createDirectory(dir);
594      for (Entry<String, byte[]> fe : e.getValue().getContent().entrySet()) {
595        String fn = Utilities.path(dir, fe.getKey());
596        byte[] cnt = fe.getValue();
597        TextFile.bytesToFile(cnt, fn);
598        size = size + cnt.length;
599        i++;
600        if (progress && i % 50 == 0) {
601          c++;
602          System.out.print(".");
603          if (c == 120) {
604            System.out.println("");
605            System.out.print("  ");
606            c = 2;
607          }
608        }    
609      }
610    }
611 
612    
613    IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
614    ini.setTimeStampFormat("yyyyMMddhhmmss");
615    ini.setTimestampProperty("packages", id+"#"+version, Timestamp.from(Instant.now()), null);
616    ini.setIntegerProperty("package-sizes", id+"#"+version, size, null);
617    ini.save();
618    if (progress)
619      System.out.println(" done.");
620
621    NpmPackage pck = loadPackageInfo(packRoot);
622    if (!id.equals(JSONUtil.str(npm.getNpm(), "name")) || !version.equals(JSONUtil.str(npm.getNpm(), "version"))) {
623      if (!id.equals(JSONUtil.str(npm.getNpm(), "name"))) {
624        npm.getNpm().addProperty("original-name", JSONUtil.str(npm.getNpm(), "name"));
625        npm.getNpm().remove("name");
626        npm.getNpm().addProperty("name", id);
627      }
628      if (!version.equals(JSONUtil.str(npm.getNpm(), "version"))) {
629        npm.getNpm().addProperty("original-version", JSONUtil.str(npm.getNpm(), "version"));
630        npm.getNpm().remove("version");
631        npm.getNpm().addProperty("version", version);
632      }
633      TextFile.stringToFile(new GsonBuilder().setPrettyPrinting().create().toJson(npm.getNpm()), Utilities.path(cacheFolder, id+"#"+version, "package", "package.json"), false);
634    }
635
636    return pck;
637  }
638
639  public NpmPackage loadPackage(String id) throws FHIRException, IOException {
640    throw new Error("Not done yet");
641  }
642  
643  public NpmPackage loadPackage(String id, String v) throws FHIRException, IOException {
644    NpmPackage p = loadPackageFromCacheOnly(id, v);
645    if (p != null) {
646      if ("current".equals(v)) {
647        p = checkCurrency(id, p);
648      }
649      if (p != null)
650        return p;
651    }
652
653    if ("dev".equals(v)) {
654      p = loadPackageFromCacheOnly(id, "current");
655      p = checkCurrency(id, p);
656      if (p != null)
657        return p;
658      v = "current";
659    }
660
661    String url = getPackageUrl(id);
662    if (url == null)
663      throw new FHIRException("Unable to resolve the package '"+id+"'");
664    if (url.contains(".tgz")) {
665      InputStream stream = fetchFromUrlSpecific(url, true);
666      if (stream != null)
667        return addPackageToCache(id, v, stream, url);
668      throw new FHIRException("Unable to find the package source for '"+id+"' at "+url);      
669    }
670    if (v == null) {
671      InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(url, "package.tgz"), true);
672      if (stream == null && isBuildLoaded()) { 
673        stream = fetchFromUrlSpecific(Utilities.pathURL(buildPath(url), "package.tgz"), true);
674      }
675      if (stream != null)
676        return addPackageToCache(id, null, stream, url);
677      throw new FHIRException("Unable to find the package source for '"+id+"' at "+url);
678    } else if ("current".equals(v) && ciList.containsKey(id)){
679      InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "package.tgz"), true);
680      return addPackageToCache(id, v, stream, Utilities.pathURL(ciList.get(id), "package.tgz"));
681    } else {
682      String pu = Utilities.pathURL(url, "package-list.json");
683      JsonObject json;
684      try {
685        json = fetchJson(pu);
686      } catch (Exception e) {
687        String pv = Utilities.pathURL(url, v, "package.tgz");
688        try {
689          InputStream stream = fetchFromUrlSpecific(pv, true);
690          return addPackageToCache(id, v, stream, pv);
691        } catch (Exception e1) {
692          throw new FHIRException("Error fetching package directly ("+pv+"), or fetching package list for "+id+" from "+pu+": "+e1.getMessage(), e1);
693        }  
694      }
695      if (!id.equals(JSONUtil.str(json, "package-id")))
696        throw new FHIRException("Package ids do not match in "+pu+": "+id+" vs "+JSONUtil.str(json, "package-id"));
697      for (JsonElement e : json.getAsJsonArray("list")) {
698        JsonObject vo = (JsonObject) e;
699        if (v.equals(JSONUtil.str(vo, "version"))) {
700          InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(JSONUtil.str(vo, "path"), "package.tgz"), true);
701          if (stream == null)
702            throw new FHIRException("Unable to find the package source for '"+id+"#"+v+"' at "+Utilities.pathURL(JSONUtil.str(vo, "path"), "package.tgz"));
703          return addPackageToCache(id, v, stream, Utilities.pathURL(JSONUtil.str(vo, "path"), "package.tgz"));
704        }
705      }
706//      // special case: current version
707//      if (id.equals("hl7.fhir.core") && v.equals(currentVersion)) {
708//        InputStream stream = fetchFromUrlSpecific(Utilities.pathURL("http://build.fhir.org", "package.tgz"), true);
709//        if (stream == null)
710//          throw new FHIRException("Unable to find the package source for '"+id+"#"+v+"' at "+Utilities.pathURL("http://build.fhir.org", "package.tgz"));
711//        return addPackageToCache(id, v, stream);
712//      }
713      throw new FHIRException("Unable to resolve version "+v+" for package "+id);
714    }
715  }
716
717
718//  public void loadFromFolder(String packagesFolder) throws IOException {
719//    for (File f : new File(packagesFolder).listFiles()) {
720//      if (f.getName().endsWith(".tgz")) {
721//        temporaryPackages.add(extractLocally(new FileInputStream(f), f.getName()));
722//      }
723//    }
724//  }
725//
726
727  /**
728   * turn true if a package exists
729   * @param id
730   * @param version
731   * @return
732   */
733  public boolean hasPackage(String id, String version) {
734    for (NpmPackage p : temporaryPackages) {
735      if (p.name().equals(id) && ("current".equals(version) || "dev".equals(version) || p.version().equals(version)))
736        return true;
737    }
738    for (String f : sorted(new File(cacheFolder).list())) {
739      if (f.equals(id+"#"+version)) {
740        return true; 
741      }
742    }
743    if ("dev".equals(version))
744      return hasPackage(id, "current");
745    else
746      return false;
747  }
748
749
750  /**
751   * List which versions of a package are available
752   * 
753   * @param url
754   * @return
755   * @throws IOException
756   */
757
758  public VersionHistory listVersions(String url) throws IOException {
759    if (historyCache.containsKey(url))
760      return historyCache.get(url);
761    
762    URL url1 = new URL(Utilities.pathURL(url, "package-list.json")+"?nocache=" + System.currentTimeMillis());
763    HttpURLConnection connection = (HttpURLConnection) url1.openConnection();
764    connection.setRequestMethod("GET");
765    InputStream json = connection.getInputStream();
766    JsonObject packageList = (JsonObject) new com.google.gson.JsonParser().parse(TextFile.streamToString(json));
767    VersionHistory res = new VersionHistory();
768    res.id = JSONUtil.str(packageList, "package-id");
769    res.canonical = JSONUtil.str(packageList, "canonical");
770    for (JsonElement j : packageList.getAsJsonArray("list")) {
771      JsonObject jo = (JsonObject) j;
772      if ("current".equals(JSONUtil.str(jo, "version")))
773        res.current = JSONUtil.str(jo, "path");
774      else
775        res.versions.put(JSONUtil.str(jo, "version"), JSONUtil.str(jo, "path"));
776    }
777    historyCache.put(url, res);
778    return res;
779  }
780
781
782  /**
783   * Clear the cache
784   * 
785   * @throws IOException
786   */
787  public void clear() throws IOException {
788    clearCache();
789  }
790    
791  public void loadFromFolder(String packagesFolder) throws IOException {
792    for (File f : new File(packagesFolder).listFiles()) {
793      if (f.getName().endsWith(".tgz")) {
794        temporaryPackages.add(NpmPackage.fromPackage(new FileInputStream(f)));
795      }
796    }
797  }
798}