001package org.hl7.fhir.dstu3.utils;
002
003/*-
004 * #%L
005 * org.hl7.fhir.dstu3
006 * %%
007 * Copyright (C) 2014 - 2019 Health Level 7
008 * %%
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 * 
013 *      http://www.apache.org/licenses/LICENSE-2.0
014 * 
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 * #L%
021 */
022
023
024import java.io.File;
025import java.io.FileInputStream;
026import java.io.FileNotFoundException;
027import java.io.IOException;
028import java.net.URISyntaxException;
029import java.text.SimpleDateFormat;
030import java.util.Date;
031import java.util.UUID;
032import java.util.zip.ZipEntry;
033import java.util.zip.ZipInputStream;
034
035import org.hl7.fhir.dstu3.formats.IParser;
036import org.hl7.fhir.dstu3.formats.JsonParser;
037import org.hl7.fhir.dstu3.formats.XmlParser;
038import org.hl7.fhir.dstu3.model.Bundle;
039import org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent;
040import org.hl7.fhir.dstu3.model.Bundle.BundleType;
041import org.hl7.fhir.dstu3.model.Bundle.HTTPVerb;
042import org.hl7.fhir.dstu3.model.Resource;
043import org.hl7.fhir.dstu3.utils.client.FHIRToolingClient;
044import org.hl7.fhir.exceptions.FHIRException;
045import org.hl7.fhir.exceptions.FHIRFormatError;
046import org.hl7.fhir.utilities.IniFile;
047import org.hl7.fhir.utilities.Utilities;
048
049public class BatchLoader {
050
051  public static void main(String[] args) throws IOException, Exception {
052    if (args.length < 3) {
053      System.out.println("Batch uploader takes 3 parameters in order: server base url, file/folder to upload, and batch size");
054    } else {
055      String server = args[0];
056      String file = args[1];
057      int size = Integer.parseInt(args[2]);
058      if (file.endsWith(".xml")) {
059        throw new FHIRException("Unimplemented file type "+file);
060      } else if (file.endsWith(".json")) {
061        throw new FHIRException("Unimplemented file type "+file);
062//      } else if (file.endsWith(".zip")) {
063//        LoadZipFile(server, file, p, size, 0, -1);
064      } else if (new File(file).isDirectory()) {
065        LoadDirectory(server, file, size);
066      } else 
067        throw new FHIRException("Unknown file type "+file);
068    }
069  }
070
071  private static void LoadDirectory(String server, String folder, int size) throws IOException, Exception {
072    System.out.print("Connecting to "+server+".. ");
073    FHIRToolingClient client = new FHIRToolingClient(server);
074    System.out.println("Done");
075    
076    IniFile ini = new IniFile(Utilities.path(folder, "batch-load-progress.ini"));
077    for (File f : new File(folder).listFiles()) {
078      if (f.getName().endsWith(".json") || f.getName().endsWith(".xml")) {
079        if (!ini.getBooleanProperty("finished", f.getName())) {
080          sendFile(client, f, size, ini);
081        }
082      }
083    }
084  }
085
086  
087  private static void sendFile(FHIRToolingClient client, File f, int size, IniFile ini) throws FHIRFormatError, FileNotFoundException, IOException {
088    long ms = System.currentTimeMillis();
089    System.out.print("Loading "+f.getName()+".. ");
090    IParser parser = f.getName().endsWith(".json") ? new JsonParser() : new XmlParser();
091    Resource res = parser.parse(new FileInputStream(f));
092    System.out.println("  done: ("+Long.toString(System.currentTimeMillis()-ms)+" ms)");
093    
094    if (res instanceof Bundle) {
095      Bundle bnd = (Bundle) res;
096      int cursor = ini.hasProperty("progress", f.getName()) ? ini.getIntegerProperty("progress", f.getName()) : 0;
097      while (cursor < bnd.getEntry().size()) {
098        Bundle bt = new Bundle();
099        bt.setType(BundleType.BATCH);     
100        bt.setId(UUID.randomUUID().toString().toLowerCase());
101        for (int i = cursor; i < Math.min(bnd.getEntry().size(), cursor+size); i++) {
102          if (i >=0 && i < bnd.getEntry().size()) {
103            BundleEntryComponent be = bt.addEntry();
104            be.setResource(bnd.getEntry().get(i).getResource());
105            be.getRequest().setMethod(HTTPVerb.PUT);
106            be.getRequest().setUrl(be.getResource().getResourceType().toString()+"/"+be.getResource().getId());
107          }
108        }
109        System.out.print(f.getName()+" ("+cursor+"/"+bnd.getEntry().size()+"): ");
110        ms = System.currentTimeMillis();
111        Bundle resp = client.transaction(bt);
112
113        int ncursor = cursor+size;
114        for (int i = 0; i < resp.getEntry().size(); i++) {
115          BundleEntryComponent t = resp.getEntry().get(i);
116          if (!t.getResponse().getStatus().startsWith("2")) { 
117            System.out.println("failed status at "+Integer.toString(i)+": "+t.getResponse().getStatus());
118            ncursor = cursor+i-1;
119            break;
120          }
121        }
122        cursor = ncursor;
123        System.out.println("  .. done: ("+Long.toString(System.currentTimeMillis()-ms)+" ms) "+SimpleDateFormat.getInstance().format(new Date()));
124        ini.setIntegerProperty("progress", f.getName(), cursor, null);
125        ini.save();
126      }
127      ini.setBooleanProperty("finished", f.getName(), true, null);
128      ini.save();
129    } else {
130      client.update(res);
131      ini.setBooleanProperty("finished", f.getName(), true, null);
132      ini.save();
133    }    
134  }
135//
136//  private static void LoadZipFile(String server, String file, IParser p, int size, int start, int end) throws IOException, Exception {
137//    System.out.println("Load Zip file "+file);
138//    Bundle b = new Bundle();
139//    b.setType(BundleType.COLLECTION);
140//    b.setId(UUID.randomUUID().toString().toLowerCase());
141//    ZipInputStream zip = new ZipInputStream(new FileInputStream(file));
142//    ZipEntry entry;
143//    while((entry = zip.getNextEntry())!=null)
144//    {
145//      try {
146//        Resource r = p.parse(zip);
147//        b.addEntry().setResource(r);
148//      } catch (Exception e) {
149//        throw new Exception("Error parsing "+entry.getName()+": "+e.getMessage(), e);
150//      }
151//    }
152//    loadBundle(server, b, size, start, end);
153//  }
154//
155//  
156//  private static int loadBundle(String server, Bundle b, int size, int start, int end) throws URISyntaxException {
157//    System.out.println("Post to "+server+". size = "+Integer.toString(size)+", start = "+Integer.toString(start)+", total = "+Integer.toString(b.getEntry().size()));
158//    FHIRToolingClient client = new FHIRToolingClient(server);
159//    int c = start;
160//    if (end == -1)
161//      end = b.getEntry().size();
162//    while (c < end) {
163//      Bundle bt = new Bundle();
164//      bt.setType(BundleType.BATCH);     
165//      bt.setId(UUID.randomUUID().toString().toLowerCase());
166//      for (int i = c; i < Math.min(b.getEntry().size(), c+size); i++) {
167//        BundleEntryComponent be = bt.addEntry();
168//        be.setResource(b.getEntry().get(i).getResource());
169//        be.getRequest().setMethod(HTTPVerb.PUT);
170//        be.getRequest().setUrl(be.getResource().getResourceType().toString()+"/"+be.getResource().getId());
171//      }
172//      System.out.print("  posting..");
173//      long ms = System.currentTimeMillis();
174//      Bundle resp = client.transaction(bt);
175//      
176//      for (int i = 0; i < resp.getEntry().size(); i++) {
177//        BundleEntryComponent t = resp.getEntry().get(i);
178//        if (!t.getResponse().getStatus().startsWith("2")) { 
179//          System.out.println("failed status at "+Integer.toString(i)+": "+t.getResponse().getStatus());
180//          return c+i;
181//        }
182//      }
183//      c = c + size;
184//      System.out.println("  ..done: "+Integer.toString(c)+". ("+Long.toString(System.currentTimeMillis()-ms)+" ms)");
185//    }
186//    System.out.println(" done");
187//    return c;
188//  }
189
190}