001package org.hl7.fhir.r4.utils;
002
003/*-
004 * #%L
005 * org.hl7.fhir.r4
006 * %%
007 * Copyright (C) 2014 - 2019 Health Level 7
008 * %%
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 * 
013 *      http://www.apache.org/licenses/LICENSE-2.0
014 * 
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 * #L%
021 */
022
023
024import java.io.File;
025import java.io.FileInputStream;
026import java.io.FileNotFoundException;
027import java.io.IOException;
028import java.net.URISyntaxException;
029import java.text.SimpleDateFormat;
030import java.util.Date;
031import java.util.UUID;
032import java.util.zip.ZipEntry;
033import java.util.zip.ZipInputStream;
034
035import org.hl7.fhir.exceptions.FHIRException;
036import org.hl7.fhir.exceptions.FHIRFormatError;
037import org.hl7.fhir.r4.formats.IParser;
038import org.hl7.fhir.r4.formats.JsonParser;
039import org.hl7.fhir.r4.formats.XmlParser;
040import org.hl7.fhir.r4.model.Bundle;
041import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent;
042import org.hl7.fhir.r4.model.Bundle.BundleType;
043import org.hl7.fhir.r4.model.Bundle.HTTPVerb;
044import org.hl7.fhir.r4.model.Resource;
045import org.hl7.fhir.r4.utils.client.FHIRToolingClient;
046import org.hl7.fhir.r4.utils.client.ToolingClientLogger;
047import org.hl7.fhir.utilities.IniFile;
048import org.hl7.fhir.utilities.Utilities;
049
050public class BatchLoader {
051
052        public static void main(String[] args) throws IOException, Exception {
053          if (args.length < 3) {
054                System.out.println("Batch uploader takes 3 parameters in order: server base url, file/folder to upload, and batch size");
055          } else {
056                String server = args[0];
057                String file = args[1];
058                int size = Integer.parseInt(args[2]);
059                if (file.endsWith(".xml")) {
060                        throw new FHIRException("Unimplemented file type "+file);
061                } else if (file.endsWith(".json")) {
062                        throw new FHIRException("Unimplemented file type "+file);
063//              } else if (file.endsWith(".zip")) {
064//                      LoadZipFile(server, file, p, size, 0, -1);
065                } else if (new File(file).isDirectory()) {
066                  LoadDirectory(server, file, size);
067                } else 
068                        throw new FHIRException("Unknown file type "+file);
069          }
070        }
071
072        private static void LoadDirectory(String server, String folder, int size) throws IOException, Exception {
073          System.out.print("Connecting to "+server+".. ");
074          FHIRToolingClient client = new FHIRToolingClient(server);
075    System.out.println("Done");
076          
077          IniFile ini = new IniFile(Utilities.path(folder, "batch-load-progress.ini"));
078          for (File f : new File(folder).listFiles()) {
079            if (f.getName().endsWith(".json") || f.getName().endsWith(".xml")) {
080              if (!ini.getBooleanProperty("finished", f.getName())) {
081                sendFile(client, f, size, ini);
082              }
083            }
084          }
085  }
086
087        
088  private static void sendFile(FHIRToolingClient client, File f, int size, IniFile ini) throws FHIRFormatError, FileNotFoundException, IOException {
089    long ms = System.currentTimeMillis();
090    System.out.print("Loading "+f.getName()+".. ");
091    IParser parser = f.getName().endsWith(".json") ? new JsonParser() : new XmlParser();
092    Resource res = parser.parse(new FileInputStream(f));
093    System.out.println("  done: ("+Long.toString(System.currentTimeMillis()-ms)+" ms)");
094    
095    if (res instanceof Bundle) {
096      Bundle bnd = (Bundle) res;
097      int cursor = ini.hasProperty("progress", f.getName()) ? ini.getIntegerProperty("progress", f.getName()) : 0;
098      while (cursor < bnd.getEntry().size()) {
099        Bundle bt = new Bundle();
100        bt.setType(BundleType.BATCH);     
101        bt.setId(UUID.randomUUID().toString().toLowerCase());
102        for (int i = cursor; i < Math.min(bnd.getEntry().size(), cursor+size); i++) {
103          BundleEntryComponent be = bt.addEntry();
104          be.setResource(bnd.getEntry().get(i).getResource());
105          be.getRequest().setMethod(HTTPVerb.PUT);
106          be.getRequest().setUrl(be.getResource().getResourceType().toString()+"/"+be.getResource().getId());
107        }
108        System.out.print(f.getName()+" ("+cursor+"/"+bnd.getEntry().size()+"): ");
109        ms = System.currentTimeMillis();
110        Bundle resp = client.transaction(bt);
111
112        int ncursor = cursor+size;
113        for (int i = 0; i < resp.getEntry().size(); i++) {
114          BundleEntryComponent t = resp.getEntry().get(i);
115          if (!t.getResponse().getStatus().startsWith("2")) { 
116            System.out.println("failed status at "+Integer.toString(i)+": "+t.getResponse().getStatus());
117            ncursor = cursor+i-1;
118            break;
119          }
120        }
121        cursor = ncursor;
122        System.out.println("  .. done: ("+Long.toString(System.currentTimeMillis()-ms)+" ms) "+SimpleDateFormat.getInstance().format(new Date()));
123        ini.setIntegerProperty("progress", f.getName(), cursor, null);
124        ini.save();
125      }
126      ini.setBooleanProperty("finished", f.getName(), true, null);
127      ini.save();
128    } else {
129      client.update(res);
130      ini.setBooleanProperty("finished", f.getName(), true, null);
131      ini.save();
132    }    
133  }
134//
135//  private static void LoadZipFile(String server, String file, IParser p, int size, int start, int end) throws IOException, Exception {
136//              System.out.println("Load Zip file "+file);
137//              Bundle b = new Bundle();
138//              b.setType(BundleType.COLLECTION);
139//              b.setId(UUID.randomUUID().toString().toLowerCase());
140//              ZipInputStream zip = new ZipInputStream(new FileInputStream(file));
141//              ZipEntry entry;
142//    while((entry = zip.getNextEntry())!=null)
143//    {
144//      try {
145//        Resource r = p.parse(zip);
146//        b.addEntry().setResource(r);
147//      } catch (Exception e) {
148//              throw new Exception("Error parsing "+entry.getName()+": "+e.getMessage(), e);
149//      }
150//    }
151//              loadBundle(server, b, size, start, end);
152//      }
153//
154//  
155//      private static int loadBundle(String server, Bundle b, int size, int start, int end) throws URISyntaxException {
156//              System.out.println("Post to "+server+". size = "+Integer.toString(size)+", start = "+Integer.toString(start)+", total = "+Integer.toString(b.getEntry().size()));
157//              FHIRToolingClient client = new FHIRToolingClient(server);
158//        int c = start;
159//        if (end == -1)
160//          end = b.getEntry().size();
161//        while (c < end) {
162//                      Bundle bt = new Bundle();
163//                      bt.setType(BundleType.BATCH);                   
164//                      bt.setId(UUID.randomUUID().toString().toLowerCase());
165//                      for (int i = c; i < Math.min(b.getEntry().size(), c+size); i++) {
166//                              BundleEntryComponent be = bt.addEntry();
167//                              be.setResource(b.getEntry().get(i).getResource());
168//                              be.getRequest().setMethod(HTTPVerb.PUT);
169//                              be.getRequest().setUrl(be.getResource().getResourceType().toString()+"/"+be.getResource().getId());
170//                      }
171//                      System.out.print("  posting..");
172//                      long ms = System.currentTimeMillis();
173//                      Bundle resp = client.transaction(bt);
174//                      
175//                      for (int i = 0; i < resp.getEntry().size(); i++) {
176//                        BundleEntryComponent t = resp.getEntry().get(i);
177//                        if (!t.getResponse().getStatus().startsWith("2")) { 
178//                          System.out.println("failed status at "+Integer.toString(i)+": "+t.getResponse().getStatus());
179//                          return c+i;
180//                        }
181//                      }
182//                      c = c + size;
183//      System.out.println("  ..done: "+Integer.toString(c)+". ("+Long.toString(System.currentTimeMillis()-ms)+" ms)");
184//        }
185//              System.out.println(" done");
186//              return c;
187//      }
188
189}