Source code

001package org.hl7.fhir.dstu3.context;
002
003/*-
004 * #%L
005 * org.hl7.fhir.dstu3
006 * %%
007 * Copyright (C) 2014 - 2019 Health Level 7
008 * %%
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 * 
013 *      http://www.apache.org/licenses/LICENSE-2.0
014 * 
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 * #L%
021 */
022
023
024import java.io.ByteArrayOutputStream;
025import java.io.File;
026import java.io.FileInputStream;
027import java.io.FileNotFoundException;
028import java.io.FileOutputStream;
029import java.io.IOException;
030import java.util.ArrayList;
031import java.util.HashMap;
032import java.util.HashSet;
033import java.util.List;
034import java.util.Map;
035import java.util.Set;
036
037import ca.uhn.fhir.rest.api.Constants;
038import org.hl7.fhir.dstu3.formats.IParser.OutputStyle;
039import org.hl7.fhir.dstu3.formats.JsonParser;
040import org.hl7.fhir.dstu3.model.BooleanType;
041import org.hl7.fhir.dstu3.model.Bundle;
042import org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent;
043import org.hl7.fhir.dstu3.model.CodeSystem;
044import org.hl7.fhir.dstu3.model.CodeSystem.CodeSystemHierarchyMeaning;
045import org.hl7.fhir.dstu3.model.CodeSystem.ConceptDefinitionComponent;
046import org.hl7.fhir.dstu3.model.CodeSystem.ConceptDefinitionDesignationComponent;
047import org.hl7.fhir.dstu3.model.CodeableConcept;
048import org.hl7.fhir.dstu3.model.Coding;
049import org.hl7.fhir.dstu3.model.ConceptMap;
050import org.hl7.fhir.dstu3.model.DataElement;
051import org.hl7.fhir.dstu3.model.ExpansionProfile;
052import org.hl7.fhir.dstu3.model.OperationDefinition;
053import org.hl7.fhir.dstu3.model.OperationOutcome;
054import org.hl7.fhir.dstu3.model.Parameters;
055import org.hl7.fhir.dstu3.model.Parameters.ParametersParameterComponent;
056import org.hl7.fhir.dstu3.model.PrimitiveType;
057import org.hl7.fhir.dstu3.model.Questionnaire;
058import org.hl7.fhir.dstu3.model.Reference;
059import org.hl7.fhir.dstu3.model.Resource;
060import org.hl7.fhir.dstu3.model.SearchParameter;
061import org.hl7.fhir.dstu3.model.StringType;
062import org.hl7.fhir.dstu3.model.StructureDefinition;
063import org.hl7.fhir.dstu3.model.StructureDefinition.TypeDerivationRule;
064import org.hl7.fhir.dstu3.model.StructureMap;
065import org.hl7.fhir.dstu3.model.UriType;
066import org.hl7.fhir.dstu3.model.ValueSet;
067import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetComponent;
068import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetFilterComponent;
069import org.hl7.fhir.dstu3.model.ValueSet.ValueSetComposeComponent;
070import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionComponent;
071import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionContainsComponent;
072import org.hl7.fhir.dstu3.terminologies.ValueSetExpander.ETooCostly;
073import org.hl7.fhir.dstu3.terminologies.ValueSetExpander.TerminologyServiceErrorClass;
074import org.hl7.fhir.dstu3.terminologies.ValueSetExpander.ValueSetExpansionOutcome;
075import org.hl7.fhir.dstu3.terminologies.ValueSetExpanderFactory;
076import org.hl7.fhir.dstu3.terminologies.ValueSetExpansionCache;
077import org.hl7.fhir.dstu3.utils.ToolingExtensions;
078import org.hl7.fhir.dstu3.utils.client.FHIRToolingClient;
079import org.hl7.fhir.exceptions.FHIRException;
080import org.hl7.fhir.exceptions.NoTerminologyServiceException;
081import org.hl7.fhir.exceptions.TerminologyServiceException;
082import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
083import org.hl7.fhir.utilities.TextFile;
084import org.hl7.fhir.utilities.Utilities;
085import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
086import org.hl7.fhir.utilities.validation.ValidationMessage.IssueType;
087
088import com.google.gson.JsonObject;
089import com.google.gson.JsonSyntaxException;
090
091public abstract class BaseWorkerContext implements IWorkerContext {
092
093  // all maps are to the full URI
094  protected Map<String, CodeSystem> codeSystems = new HashMap<String, CodeSystem>();
095  protected Set<String> nonSupportedCodeSystems = new HashSet<String>();
096  protected Map<String, ValueSet> valueSets = new HashMap<String, ValueSet>();
097  protected Map<String, ConceptMap> maps = new HashMap<String, ConceptMap>();
098  protected Map<String, StructureMap> transforms = new HashMap<String, StructureMap>();
099  protected Map<String, DataElement> dataElements = new HashMap<String, DataElement>();
100  protected Map<String, StructureDefinition> profiles = new HashMap<String, StructureDefinition>();
101  protected Map<String, SearchParameter> searchParameters = new HashMap<String, SearchParameter>();
102  protected Map<String, StructureDefinition> extensionDefinitions = new HashMap<String, StructureDefinition>();
103  protected Map<String, Questionnaire> questionnaires = new HashMap<String, Questionnaire>();
104  protected Map<String, OperationDefinition> operations = new HashMap<String, OperationDefinition>();
105
106  protected ValueSetExpanderFactory expansionCache = new ValueSetExpansionCache(this);
107  protected boolean cacheValidation; // if true, do an expansion and cache the expansion
108  private Set<String> failed = new HashSet<String>(); // value sets for which we don't try to do expansion, since the first attempt to get a comprehensive expansion was not successful
109  protected Map<String, Map<String, ValidationResult>> validationCache = new HashMap<String, Map<String,ValidationResult>>();
110  protected String tsServer;
111  protected String validationCachePath;
112  protected String name;
113
114  // private ValueSetExpansionCache expansionCache; //   
115
116  protected FHIRToolingClient txServer;
117  private Bundle bndCodeSystems;
118  private boolean canRunWithoutTerminology;
119  protected boolean allowLoadingDuplicates;
120  protected boolean noTerminologyServer;
121  protected String cache;
122  private int expandCodesLimit = 1000;
123  protected ILoggingService logger;
124  protected ExpansionProfile expProfile;
125
126  public Map<String, CodeSystem> getCodeSystems() {
127    return codeSystems;
128  }
129
130  public Map<String, DataElement> getDataElements() {
131    return dataElements;
132  }
133
134  public Map<String, ValueSet> getValueSets() {
135    return valueSets;
136  }
137
138  public Map<String, ConceptMap> getMaps() {
139    return maps;
140  }
141
142  public Map<String, StructureDefinition> getProfiles() {
143    return profiles;
144  }
145
146  public Map<String, StructureDefinition> getExtensionDefinitions() {
147    return extensionDefinitions;
148  }
149
150  public Map<String, Questionnaire> getQuestionnaires() {
151    return questionnaires;
152  }
153
154  public Map<String, OperationDefinition> getOperations() {
155    return operations;
156  }
157
158  public void seeExtensionDefinition(String url, StructureDefinition ed) throws Exception {
159    if (extensionDefinitions.get(ed.getUrl()) != null)
160      throw new Exception("duplicate extension definition: " + ed.getUrl());
161    extensionDefinitions.put(ed.getId(), ed);
162    extensionDefinitions.put(url, ed);
163    extensionDefinitions.put(ed.getUrl(), ed);
164  }
165
166  public void dropExtensionDefinition(String id) {
167    StructureDefinition sd = extensionDefinitions.get(id);
168    extensionDefinitions.remove(id);
169    if (sd!= null)
170      extensionDefinitions.remove(sd.getUrl());
171  }
172
173  public void seeQuestionnaire(String url, Questionnaire theQuestionnaire) throws Exception {
174    if (questionnaires.get(theQuestionnaire.getId()) != null)
175      throw new Exception("duplicate extension definition: "+theQuestionnaire.getId());
176    questionnaires.put(theQuestionnaire.getId(), theQuestionnaire);
177    questionnaires.put(url, theQuestionnaire);
178  }
179
180  public void seeOperation(OperationDefinition opd) throws Exception {
181    if (operations.get(opd.getUrl()) != null)
182      throw new Exception("duplicate extension definition: "+opd.getUrl());
183    operations.put(opd.getUrl(), opd);
184    operations.put(opd.getId(), opd);
185  }
186
187  public void seeValueSet(String url, ValueSet vs) throws Exception {
188    if (valueSets.containsKey(vs.getUrl()) && !allowLoadingDuplicates)
189      throw new Exception("Duplicate value set "+vs.getUrl());
190    valueSets.put(vs.getId(), vs);
191    valueSets.put(url, vs);
192    valueSets.put(vs.getUrl(), vs);
193  }
194
195  public void dropValueSet(String id) {
196    ValueSet vs = valueSets.get(id);
197    valueSets.remove(id);
198    if (vs != null)
199      valueSets.remove(vs.getUrl());
200  }
201
202  public void seeCodeSystem(String url, CodeSystem cs) throws FHIRException {
203    if (codeSystems.containsKey(cs.getUrl()) && !allowLoadingDuplicates)
204      throw new FHIRException("Duplicate code system "+cs.getUrl());
205    codeSystems.put(cs.getId(), cs);
206    codeSystems.put(url, cs);
207    codeSystems.put(cs.getUrl(), cs);
208  }
209
210  public void dropCodeSystem(String id) {
211    CodeSystem cs = codeSystems.get(id);
212    codeSystems.remove(id);
213    if (cs != null)
214      codeSystems.remove(cs.getUrl());
215  }
216
217  public void seeProfile(String url, StructureDefinition p) throws Exception {
218    if (profiles.containsKey(p.getUrl()))
219      throw new Exception("Duplicate Profile "+p.getUrl());
220    profiles.put(p.getId(), p);
221    profiles.put(url, p);
222    profiles.put(p.getUrl(), p);
223  }
224
225  public void dropProfile(String id) {
226    StructureDefinition sd = profiles.get(id);
227    profiles.remove(id);
228    if (sd!= null)
229      profiles.remove(sd.getUrl());
230  }
231
232  @Override
233  public CodeSystem fetchCodeSystem(String system) {
234    return codeSystems.get(system);
235  } 
236
237  @Override
238  public boolean supportsSystem(String system) throws TerminologyServiceException {
239    if (codeSystems.containsKey(system))
240      return true;
241    else if (nonSupportedCodeSystems.contains(system))
242      return false;
243    else if (system.startsWith("http://example.org") || system.startsWith("http://acme.com") || system.startsWith("http://hl7.org/fhir/valueset-") || system.startsWith("urn:oid:"))
244      return false;
245    else {
246      if (noTerminologyServer)
247        return false;
248      if (bndCodeSystems == null) {
249        try {
250          tlog("Terminology server: Check for supported code systems for "+system);
251          bndCodeSystems = txServer.fetchFeed(txServer.getAddress()+"/CodeSystem?content-mode=not-present&_summary=true&_count=1000");
252        } catch (Exception e) {
253          if (canRunWithoutTerminology) {
254            noTerminologyServer = true;
255            log("==============!! Running without terminology server !!============== ("+e.getMessage()+")");
256            return false;
257          } else
258            throw new TerminologyServiceException(e);
259        }
260      }
261      if (bndCodeSystems != null) {
262        for (BundleEntryComponent be : bndCodeSystems.getEntry()) {
263          CodeSystem cs = (CodeSystem) be.getResource();
264          if (!codeSystems.containsKey(cs.getUrl())) {
265            codeSystems.put(cs.getUrl(), null);
266          }
267        }
268      }
269      if (codeSystems.containsKey(system))
270        return true;
271    }
272    nonSupportedCodeSystems.add(system);
273    return false;
274  }
275
276  private void log(String message) {
277    if (logger != null)
278      logger.logMessage(message);
279    else
280      System.out.println(message);
281  }
282
283  @Override
284  public ValueSetExpansionOutcome expandVS(ValueSet vs, boolean cacheOk, boolean heirarchical) {
285    try {
286      if (vs.hasExpansion()) {
287        return new ValueSetExpansionOutcome(vs.copy());
288      }
289      String cacheFn = null;
290      if (cache != null) {
291        cacheFn = Utilities.path(cache, determineCacheId(vs, heirarchical)+".json");
292        if (new File(cacheFn).exists())
293          return loadFromCache(vs.copy(), cacheFn);
294      }
295      if (cacheOk && vs.hasUrl()) {
296        if (expProfile == null)
297          throw new Exception("No ExpansionProfile provided");
298        ValueSetExpansionOutcome vse = expansionCache.getExpander().expand(vs, expProfile.setExcludeNested(!heirarchical));
299        if (vse.getValueset() != null) {
300          if (cache != null) {
301            FileOutputStream s = new FileOutputStream(cacheFn);
302            newJsonParser().compose(new FileOutputStream(cacheFn), vse.getValueset());
303            s.close();
304          }
305        }
306        return vse;
307      } else {
308        ValueSetExpansionOutcome res = expandOnServer(vs, cacheFn);
309        if (cacheFn != null) {
310          if (res.getValueset() != null) {
311            saveToCache(res.getValueset(), cacheFn);
312          } else { 
313            OperationOutcome oo = new OperationOutcome();
314            oo.addIssue().getDetails().setText(res.getError());
315            saveToCache(oo, cacheFn);
316          }
317        }
318        return res;
319      }
320    } catch (NoTerminologyServiceException e) {
321      return new ValueSetExpansionOutcome(e.getMessage() == null ? e.getClass().getName() : e.getMessage(), TerminologyServiceErrorClass.NOSERVICE);
322    } catch (Exception e) {
323      return new ValueSetExpansionOutcome(e.getMessage() == null ? e.getClass().getName() : e.getMessage(), TerminologyServiceErrorClass.UNKNOWN);
324    }
325  }
326
327  private ValueSetExpansionOutcome loadFromCache(ValueSet vs, String cacheFn) throws FileNotFoundException, Exception {
328    JsonParser parser = new JsonParser();
329    Resource r = parser.parse(new FileInputStream(cacheFn));
330    if (r instanceof OperationOutcome)
331      return new ValueSetExpansionOutcome(((OperationOutcome) r).getIssue().get(0).getDetails().getText(), TerminologyServiceErrorClass.UNKNOWN);
332    else {
333      vs.setExpansion(((ValueSet) r).getExpansion()); // because what is cached might be from a different value set
334      return new ValueSetExpansionOutcome(vs);
335    }
336  }
337
338  private void saveToCache(Resource res, String cacheFn) throws FileNotFoundException, Exception {
339    JsonParser parser = new JsonParser();
340    parser.compose(new FileOutputStream(cacheFn), res);
341  }
342
343  private String determineCacheId(ValueSet vs, boolean heirarchical) throws Exception {
344    // just the content logical definition is hashed
345    ValueSet vsid = new ValueSet();
346    vsid.setCompose(vs.getCompose());
347    JsonParser parser = new JsonParser();
348    parser.setOutputStyle(OutputStyle.NORMAL);
349    ByteArrayOutputStream b = new  ByteArrayOutputStream();
350    parser.compose(b, vsid);
351    b.close();
352    String s = new String(b.toByteArray(), Constants.CHARSET_UTF8);
353    // any code systems we can find, we add these too. 
354    for (ConceptSetComponent inc : vs.getCompose().getInclude()) {
355      CodeSystem cs = fetchCodeSystem(inc.getSystem());
356      if (cs != null) {
357        String css = cacheValue(cs);
358        s = s + css;
359      }
360    }
361    s = s + "-"+Boolean.toString(heirarchical);
362    String r = Integer.toString(s.hashCode());
363    //    TextFile.stringToFile(s, Utilities.path(cache, r+".id.json"));
364    return r;
365  }
366
367
368  private String cacheValue(CodeSystem cs) throws IOException {
369    CodeSystem csid = new CodeSystem();
370    csid.setId(cs.getId());
371    csid.setVersion(cs.getVersion());
372    csid.setContent(cs.getContent());
373    csid.setHierarchyMeaning(CodeSystemHierarchyMeaning.GROUPEDBY);
374    for (ConceptDefinitionComponent cc : cs.getConcept()) 
375      csid.getConcept().add(processCSConcept(cc));
376    JsonParser parser = new JsonParser();
377    parser.setOutputStyle(OutputStyle.NORMAL);
378    ByteArrayOutputStream b = new  ByteArrayOutputStream();
379    parser.compose(b, csid);
380    b.close();
381    return new String(b.toByteArray(), Constants.CHARSET_UTF8);
382  }
383
384
385  private ConceptDefinitionComponent processCSConcept(ConceptDefinitionComponent cc) {
386    ConceptDefinitionComponent ccid = new ConceptDefinitionComponent();
387    ccid.setCode(cc.getCode());
388    ccid.setDisplay(cc.getDisplay());
389    for (ConceptDefinitionComponent cci : cc.getConcept()) 
390      ccid.getConcept().add(processCSConcept(cci));
391    return ccid;
392  }
393
394  public ValueSetExpansionOutcome expandOnServer(ValueSet vs, String fn) throws Exception {
395    if (noTerminologyServer)
396      return new ValueSetExpansionOutcome("Error expanding ValueSet: running without terminology services", TerminologyServiceErrorClass.NOSERVICE);
397    if (expProfile == null)
398      throw new Exception("No ExpansionProfile provided");
399
400    try {
401      Map<String, String> params = new HashMap<String, String>();
402      params.put("_limit", Integer.toString(expandCodesLimit ));
403      params.put("_incomplete", "true");
404      tlog("Terminology Server: $expand on "+getVSSummary(vs));
405      ValueSet result = txServer.expandValueset(vs, expProfile.setIncludeDefinition(false), params);
406      return new ValueSetExpansionOutcome(result);  
407    } catch (Exception e) {
408      return new ValueSetExpansionOutcome("Error expanding ValueSet \""+vs.getUrl()+": "+e.getMessage(), TerminologyServiceErrorClass.UNKNOWN);
409    }
410  }
411
412  private String getVSSummary(ValueSet vs) {
413    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
414    for (ConceptSetComponent cc : vs.getCompose().getInclude())
415      b.append("Include "+getIncSummary(cc));
416    for (ConceptSetComponent cc : vs.getCompose().getExclude())
417      b.append("Exclude "+getIncSummary(cc));
418    return b.toString();
419  }
420
421  private String getIncSummary(ConceptSetComponent cc) {
422    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
423    for (UriType vs : cc.getValueSet())
424      b.append(vs.asStringValue());
425    String vsd = b.length() > 0 ? " where the codes are in the value sets ("+b.toString()+")" : "";
426    String system = cc.getSystem();
427    if (cc.hasConcept())
428      return Integer.toString(cc.getConcept().size())+" codes from "+system+vsd;
429    if (cc.hasFilter()) {
430      String s = "";
431      for (ConceptSetFilterComponent f : cc.getFilter()) {
432        if (!Utilities.noString(s))
433          s = s + " & ";
434        s = s + f.getProperty()+" "+f.getOp().toCode()+" "+f.getValue();
435      }
436      return "from "+system+" where "+s+vsd;
437    }
438    return "All codes from "+system+vsd;
439  }
440
441  private ValidationResult handleByCache(ValueSet vs, Coding coding, boolean tryCache) {
442    String cacheId = cacheId(coding);
443    Map<String, ValidationResult> cache = validationCache.get(vs.getUrl());
444    if (cache == null) {
445      cache = new HashMap<String, IWorkerContext.ValidationResult>();
446      validationCache.put(vs.getUrl(), cache);
447    }
448    if (cache.containsKey(cacheId))
449      return cache.get(cacheId);
450    if (!tryCache)
451      return null;
452    if (!cacheValidation)
453      return null;
454    if (failed.contains(vs.getUrl()))
455      return null;
456    ValueSetExpansionOutcome vse = expandVS(vs, true, false);
457    if (vse.getValueset() == null || notcomplete(vse.getValueset())) {
458      failed.add(vs.getUrl());
459      return null;
460    }
461
462    ValidationResult res = validateCode(coding, vse.getValueset());
463    cache.put(cacheId, res);
464    return res;
465  }
466
467  private boolean notcomplete(ValueSet vs) {
468    if (!vs.hasExpansion())
469      return true;
470    if (!vs.getExpansion().getExtensionsByUrl("http://hl7.org/fhir/StructureDefinition/valueset-unclosed").isEmpty())
471      return true;
472    if (!vs.getExpansion().getExtensionsByUrl("http://hl7.org/fhir/StructureDefinition/valueset-toocostly").isEmpty())
473      return true;
474    return false;
475  }
476
477  private ValidationResult handleByCache(ValueSet vs, CodeableConcept concept, boolean tryCache) {
478    String cacheId = cacheId(concept);
479    Map<String, ValidationResult> cache = validationCache.get(vs.getUrl());
480    if (cache == null) {
481      cache = new HashMap<String, IWorkerContext.ValidationResult>();
482      validationCache.put(vs.getUrl(), cache);
483    }
484    if (cache.containsKey(cacheId))
485      return cache.get(cacheId);
486
487    if (validationCache.containsKey(vs.getUrl()) && validationCache.get(vs.getUrl()).containsKey(cacheId))
488      return validationCache.get(vs.getUrl()).get(cacheId);
489    if (!tryCache)
490      return null;
491    if (!cacheValidation)
492      return null;
493    if (failed.contains(vs.getUrl()))
494      return null;
495    ValueSetExpansionOutcome vse = expandVS(vs, true, false);
496    if (vse.getValueset() == null || notcomplete(vse.getValueset())) {
497      failed.add(vs.getUrl());
498      return null;
499    }
500    ValidationResult res = validateCode(concept, vse.getValueset());
501    cache.put(cacheId, res);
502    return res;
503  }
504
505  private String cacheId(Coding coding) {
506    return "|"+coding.getSystem()+"|"+coding.getVersion()+"|"+coding.getCode()+"|"+coding.getDisplay();
507  }
508
509  private String cacheId(CodeableConcept cc) {
510    StringBuilder b = new StringBuilder();
511    for (Coding c : cc.getCoding()) {
512      b.append("#");
513      b.append(cacheId(c));
514    }    
515    return b.toString();
516  }
517
518  private ValidationResult verifyCodeExternal(ValueSet vs, Coding coding, boolean tryCache) throws Exception {
519    ValidationResult res = vs == null ? null : handleByCache(vs, coding, tryCache);
520    if (res != null)
521      return res;
522    Parameters pin = new Parameters();
523    pin.addParameter().setName("coding").setValue(coding);
524    if (vs != null)
525      pin.addParameter().setName("valueSet").setResource(vs);
526    res = serverValidateCode(pin, vs == null);
527    if (vs != null) {
528      Map<String, ValidationResult> cache = validationCache.get(vs.getUrl());
529      cache.put(cacheId(coding), res);
530    }
531    return res;
532  }
533
534  private ValidationResult verifyCodeExternal(ValueSet vs, CodeableConcept cc, boolean tryCache) throws Exception {
535    ValidationResult res = handleByCache(vs, cc, tryCache);
536    if (res != null)
537      return res;
538    Parameters pin = new Parameters();
539    pin.addParameter().setName("codeableConcept").setValue(cc);
540    pin.addParameter().setName("valueSet").setResource(vs);
541    res = serverValidateCode(pin, false);
542    Map<String, ValidationResult> cache = validationCache.get(vs.getUrl());
543    cache.put(cacheId(cc), res);
544    return res;
545  }
546
547  private ValidationResult serverValidateCode(Parameters pin, boolean doCache) throws Exception {
548    if (noTerminologyServer)
549      return new ValidationResult(null, null, TerminologyServiceErrorClass.NOSERVICE);
550    String cacheName = doCache ? generateCacheName(pin) : null;
551    ValidationResult res = loadFromCache(cacheName);
552    if (res != null)
553      return res;
554    tlog("Terminology Server: $validate-code "+describeValidationParameters(pin));
555    for (ParametersParameterComponent pp : pin.getParameter())
556      if (pp.getName().equals("profile"))
557        throw new Error("Can only specify profile in the context");
558    if (expProfile == null)
559      throw new Exception("No ExpansionProfile provided");
560    pin.addParameter().setName("profile").setResource(expProfile);
561
562    Parameters pout = txServer.operateType(ValueSet.class, "validate-code", pin);
563    boolean ok = false;
564    String message = "No Message returned";
565    String display = null;
566    TerminologyServiceErrorClass err = TerminologyServiceErrorClass.UNKNOWN;
567    for (ParametersParameterComponent p : pout.getParameter()) {
568      if (p.getName().equals("result"))
569        ok = ((BooleanType) p.getValue()).getValue().booleanValue();
570      else if (p.getName().equals("message"))
571        message = ((StringType) p.getValue()).getValue();
572      else if (p.getName().equals("display"))
573        display = ((StringType) p.getValue()).getValue();
574      else if (p.getName().equals("cause")) {
575        try {
576          IssueType it = IssueType.fromCode(((StringType) p.getValue()).getValue());
577          if (it == IssueType.UNKNOWN)
578            err = TerminologyServiceErrorClass.UNKNOWN;
579          else if (it == IssueType.NOTSUPPORTED)
580            err = TerminologyServiceErrorClass.VALUESET_UNSUPPORTED;
581        } catch (FHIRException e) {
582        }
583      }
584    }
585    if (!ok)
586      res = new ValidationResult(IssueSeverity.ERROR, message, err);
587    else if (display != null)
588      res = new ValidationResult(new ConceptDefinitionComponent().setDisplay(display));
589    else
590      res = new ValidationResult(new ConceptDefinitionComponent());
591    saveToCache(res, cacheName);
592    return res;
593  }
594
595
596  private void tlog(String msg) {
597    //    log(msg);
598  }
599
600  @SuppressWarnings("rawtypes")
601  private String describeValidationParameters(Parameters pin) {
602    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
603    for (ParametersParameterComponent p : pin.getParameter()) {
604      if (p.hasValue() && p.getValue() instanceof PrimitiveType) {
605        b.append(p.getName()+"="+((PrimitiveType) p.getValue()).asStringValue());
606      } else if (p.hasValue() && p.getValue() instanceof Coding) {
607        b.append("system="+((Coding) p.getValue()).getSystem());
608        b.append("code="+((Coding) p.getValue()).getCode());
609        b.append("display="+((Coding) p.getValue()).getDisplay());
610      } else if (p.hasValue() && p.getValue() instanceof CodeableConcept) {
611        if (((CodeableConcept) p.getValue()).hasCoding()) {
612          Coding c = ((CodeableConcept) p.getValue()).getCodingFirstRep();
613          b.append("system="+c.getSystem());
614          b.append("code="+c.getCode());
615          b.append("display="+c.getDisplay());
616        } else if (((CodeableConcept) p.getValue()).hasText()) {
617          b.append("text="+((CodeableConcept) p.getValue()).getText());
618        }
619      } else if (p.hasResource() && (p.getResource() instanceof ValueSet)) {
620        b.append("valueset="+getVSSummary((ValueSet) p.getResource()));
621      } 
622    }
623    return b.toString();
624  }
625
626  private ValidationResult loadFromCache(String fn) throws FileNotFoundException, IOException {
627    if (fn == null)
628      return null;
629    if (!(new File(fn).exists()))
630      return null;
631    String cnt = TextFile.fileToString(fn);
632    if (cnt.startsWith("!error: "))
633      return new ValidationResult(IssueSeverity.ERROR, cnt.substring(8));
634    else if (cnt.startsWith("!warning: "))
635      return new ValidationResult(IssueSeverity.ERROR, cnt.substring(10));
636    else
637      return new ValidationResult(new ConceptDefinitionComponent().setDisplay(cnt));
638  }
639
640  private void saveToCache(ValidationResult res, String cacheName) throws IOException {
641    if (cacheName == null)
642      return;
643    if (res.getDisplay() != null)
644      TextFile.stringToFile(res.getDisplay(), cacheName);
645    else if (res.getMessage() != null) {
646      if (res.getSeverity() == IssueSeverity.WARNING)
647        TextFile.stringToFile("!warning: "+res.getMessage(), cacheName);
648      else 
649        TextFile.stringToFile("!error: "+res.getMessage(), cacheName);
650    }
651  }
652
653  private String generateCacheName(Parameters pin) throws IOException {
654    if (cache == null)
655      return null;
656    String json = new JsonParser().composeString(pin);
657    return Utilities.path(cache, "vc"+Integer.toString(json.hashCode())+".json");
658  }
659
660  @Override
661  public ValueSetExpansionComponent expandVS(ConceptSetComponent inc, boolean heirachical) throws TerminologyServiceException {
662    ValueSet vs = new ValueSet();
663    vs.setCompose(new ValueSetComposeComponent());
664    vs.getCompose().getInclude().add(inc);
665    ValueSetExpansionOutcome vse = expandVS(vs, true, heirachical);
666    ValueSet valueset = vse.getValueset();
667    if (valueset == null)
668      throw new TerminologyServiceException("Error Expanding ValueSet: "+vse.getError());
669    return valueset.getExpansion();
670  }
671
672  @Override
673  public ValidationResult validateCode(String system, String code, String display) {
674    try {
675      if (codeSystems.containsKey(system) && codeSystems.get(system) != null)
676        return verifyCodeInCodeSystem(codeSystems.get(system), system, code, display);
677      else 
678        return verifyCodeExternal(null, new Coding().setSystem(system).setCode(code).setDisplay(display), false);
679    } catch (Exception e) {
680      return new ValidationResult(IssueSeverity.FATAL, "Error validating code \""+code+"\" in system \""+system+"\": "+e.getMessage());
681    }
682  }
683
684
685  @Override
686  public ValidationResult validateCode(Coding code, ValueSet vs) {
687    if (codeSystems.containsKey(code.getSystem()) && codeSystems.get(code.getSystem()) != null) 
688      try {
689        return verifyCodeInCodeSystem(codeSystems.get(code.getSystem()), code.getSystem(), code.getCode(), code.getDisplay());
690      } catch (Exception e) {
691        return new ValidationResult(IssueSeverity.FATAL, "Error validating code \""+code+"\" in system \""+code.getSystem()+"\": "+e.getMessage());
692      }
693    else if (vs.hasExpansion()) 
694      try {
695        return verifyCodeInternal(vs, code.getSystem(), code.getCode(), code.getDisplay());
696      } catch (Exception e) {
697        return new ValidationResult(IssueSeverity.FATAL, "Error validating code \""+code+"\" in system \""+code.getSystem()+"\": "+e.getMessage());
698      }
699    else 
700      try {
701        return verifyCodeExternal(vs, code, true);
702      } catch (Exception e) {
703        return new ValidationResult(IssueSeverity.WARNING, "Error validating code \""+code+"\" in system \""+code.getSystem()+"\": "+e.getMessage());
704      }
705  }
706
707  @Override
708  public ValidationResult validateCode(CodeableConcept code, ValueSet vs) {
709    try {
710      if (vs.hasExpansion()) 
711        return verifyCodeInternal(vs, code);
712      else {
713        // we'll try expanding first; if that doesn't work, then we'll just pass it to the server to validate 
714        // ... could be a problem if the server doesn't have the code systems we have locally, so we try not to depend on the server
715        try {
716          ValueSetExpansionOutcome vse = expandVS(vs, true, false);
717          if (vse.getValueset() != null && !hasTooCostlyExpansion(vse.getValueset()))
718            return verifyCodeInternal(vse.getValueset(), code);
719        } catch (Exception e) {
720          // failed? we'll just try the server
721        }        
722        return verifyCodeExternal(vs, code, true);
723      }
724    } catch (Exception e) {
725      return new ValidationResult(IssueSeverity.FATAL, "Error validating code \""+code.toString()+"\": "+e.getMessage(), TerminologyServiceErrorClass.SERVER_ERROR);
726    }
727  }
728
729
730  private boolean hasTooCostlyExpansion(ValueSet valueset) {
731    return valueset != null && valueset.hasExpansion() && ToolingExtensions.hasExtension(valueset.getExpansion(), "http://hl7.org/fhir/StructureDefinition/valueset-toocostly");
732  }
733
734  @Override
735  public ValidationResult validateCode(String system, String code, String display, ValueSet vs) {
736    try {
737      if (system == null && display == null)
738        return verifyCodeInternal(vs, code);
739      if ((codeSystems.containsKey(system)  && codeSystems.get(system) != null) || vs.hasExpansion()) 
740        return verifyCodeInternal(vs, system, code, display);
741      else 
742        return verifyCodeExternal(vs, new Coding().setSystem(system).setCode(code).setDisplay(display), true);
743    } catch (Exception e) {
744      return new ValidationResult(IssueSeverity.FATAL, "Error validating code \""+code+"\" in system \""+system+"\": "+e.getMessage(), TerminologyServiceErrorClass.SERVER_ERROR);
745    }
746  }
747
748  @Override
749  public ValidationResult validateCode(String system, String code, String display, ConceptSetComponent vsi) {
750    try {
751      ValueSet vs = new ValueSet();
752      vs.setUrl(Utilities.makeUuidUrn());
753      vs.getCompose().addInclude(vsi);
754      return verifyCodeExternal(vs, new Coding().setSystem(system).setCode(code).setDisplay(display), true);
755    } catch (Exception e) {
756      return new ValidationResult(IssueSeverity.FATAL, "Error validating code \""+code+"\" in system \""+system+"\": "+e.getMessage());
757    }
758  }
759
760  public void initTS(String cachePath, String tsServer) throws Exception {
761    cache = cachePath;
762    this.tsServer = tsServer;
763    expansionCache = new ValueSetExpansionCache(this, null);
764    validationCachePath = Utilities.path(cachePath, "validation.cache");
765    try {
766      loadValidationCache();
767    } catch (Exception e) {
768      e.printStackTrace();
769    }
770  }
771
772  protected void loadValidationCache() throws JsonSyntaxException, Exception {
773  }
774
775  @Override
776  public List<ConceptMap> findMapsForSource(String url) {
777    List<ConceptMap> res = new ArrayList<ConceptMap>();
778    for (ConceptMap map : maps.values())
779      if (((Reference) map.getSource()).getReference().equals(url)) 
780        res.add(map);
781    return res;
782  }
783
784  private ValidationResult verifyCodeInternal(ValueSet vs, CodeableConcept code) throws Exception {
785    for (Coding c : code.getCoding()) {
786      ValidationResult res = verifyCodeInternal(vs, c.getSystem(), c.getCode(), c.getDisplay());
787      if (res.isOk())
788        return res;
789    }
790    if (code.getCoding().isEmpty())
791      return new ValidationResult(IssueSeverity.ERROR, "None code provided");
792    else
793      return new ValidationResult(IssueSeverity.ERROR, "None of the codes are in the specified value set");
794  }
795
796  private ValidationResult verifyCodeInternal(ValueSet vs, String system, String code, String display) throws Exception {
797    if (vs.hasExpansion())
798      return verifyCodeInExpansion(vs, system, code, display);
799    else {
800      ValueSetExpansionOutcome vse = expansionCache.getExpander().expand(vs, null);
801      if (vse.getValueset() != null) 
802        return verifyCodeExternal(vs, new Coding().setSystem(system).setCode(code).setDisplay(display), false);
803      else
804        return verifyCodeInExpansion(vse.getValueset(), system, code, display);
805    }
806  }
807
808  private ValidationResult verifyCodeInternal(ValueSet vs, String code) throws FileNotFoundException, ETooCostly, IOException, FHIRException {
809    if (vs.hasExpansion())
810      return verifyCodeInExpansion(vs, code);
811    else {
812      ValueSetExpansionOutcome vse = expansionCache.getExpander().expand(vs, null);
813      if (vse.getValueset() == null)
814        return new ValidationResult(IssueSeverity.ERROR, vse.getError(), vse.getErrorClass());
815      else
816        return verifyCodeInExpansion(vse.getValueset(), code);
817    }
818  }
819
820  private ValidationResult verifyCodeInCodeSystem(CodeSystem cs, String system, String code, String display) throws Exception {
821    ConceptDefinitionComponent cc = findCodeInConcept(cs.getConcept(), code);
822    if (cc == null)
823      if (cs.getContent().equals(CodeSystem.CodeSystemContentMode.COMPLETE))
824        return new ValidationResult(IssueSeverity.ERROR, "Unknown Code "+code+" in "+cs.getUrl());
825      else if (!cs.getContent().equals(CodeSystem.CodeSystemContentMode.NOTPRESENT))
826        return new ValidationResult(IssueSeverity.WARNING, "Unknown Code "+code+" in partial code list of "+cs.getUrl());
827      else 
828        return verifyCodeExternal(null, new Coding().setSystem(system).setCode(code).setDisplay(display), false);
829    //
830    //        return new ValidationResult(IssueSeverity.WARNING, "A definition was found for "+cs.getUrl()+", but it has no codes in the definition");
831    //      return new ValidationResult(IssueSeverity.ERROR, "Unknown Code "+code+" in "+cs.getUrl());
832    if (display == null)
833      return new ValidationResult(cc);
834    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
835    if (cc.hasDisplay()) {
836      b.append(cc.getDisplay());
837      if (display.equalsIgnoreCase(cc.getDisplay()))
838        return new ValidationResult(cc);
839    }
840    for (ConceptDefinitionDesignationComponent ds : cc.getDesignation()) {
841      b.append(ds.getValue());
842      if (display.equalsIgnoreCase(ds.getValue()))
843        return new ValidationResult(cc);
844    }
845    return new ValidationResult(IssueSeverity.WARNING, "Display Name for "+code+" must be one of '"+b.toString()+"'", cc);
846  }
847
848
849  private ValidationResult verifyCodeInExpansion(ValueSet vs, String system,String code, String display) {
850    ValueSetExpansionContainsComponent cc = findCode(vs.getExpansion().getContains(), code);
851    if (cc == null)
852      return new ValidationResult(IssueSeverity.ERROR, "Unknown Code "+code+" in "+vs.getUrl());
853    if (display == null)
854      return new ValidationResult(new ConceptDefinitionComponent().setCode(code).setDisplay(cc.getDisplay()));
855    if (cc.hasDisplay()) {
856      if (display.equalsIgnoreCase(cc.getDisplay()))
857        return new ValidationResult(new ConceptDefinitionComponent().setCode(code).setDisplay(cc.getDisplay()));
858      return new ValidationResult(IssueSeverity.WARNING, "Display Name for "+code+" must be '"+cc.getDisplay()+"'", new ConceptDefinitionComponent().setCode(code).setDisplay(cc.getDisplay()));
859    }
860    return null;
861  }
862
863  private ValidationResult verifyCodeInExpansion(ValueSet vs, String code) throws FHIRException {
864    if (vs.getExpansion().hasExtension("http://hl7.org/fhir/StructureDefinition/valueset-toocostly")) {
865      throw new FHIRException("Unable to validate core - value set is too costly to expand"); 
866    } else {
867      ValueSetExpansionContainsComponent cc = findCode(vs.getExpansion().getContains(), code);
868      if (cc == null)
869        return new ValidationResult(IssueSeverity.ERROR, "Unknown Code "+code+" in "+vs.getUrl());
870      return null;
871    }
872  }
873
874  private ValueSetExpansionContainsComponent findCode(List<ValueSetExpansionContainsComponent> contains, String code) {
875    for (ValueSetExpansionContainsComponent cc : contains) {
876      if (code.equals(cc.getCode()))
877        return cc;
878      ValueSetExpansionContainsComponent c = findCode(cc.getContains(), code);
879      if (c != null)
880        return c;
881    }
882    return null;
883  }
884
885  private ConceptDefinitionComponent findCodeInConcept(List<ConceptDefinitionComponent> concept, String code) {
886    for (ConceptDefinitionComponent cc : concept) {
887      if (code.equals(cc.getCode()))
888        return cc;
889      ConceptDefinitionComponent c = findCodeInConcept(cc.getConcept(), code);
890      if (c != null)
891        return c;
892    }
893    return null;
894  }
895
896  public Set<String> getNonSupportedCodeSystems() {
897    return nonSupportedCodeSystems;
898  }
899
900  public boolean isCanRunWithoutTerminology() {
901    return canRunWithoutTerminology;
902  }
903
904  public void setCanRunWithoutTerminology(boolean canRunWithoutTerminology) {
905    this.canRunWithoutTerminology = canRunWithoutTerminology;
906  }
907
908  public int getExpandCodesLimit() {
909    return expandCodesLimit;
910  }
911
912  public void setExpandCodesLimit(int expandCodesLimit) {
913    this.expandCodesLimit = expandCodesLimit;
914  }
915
916  public void setLogger(ILoggingService logger) {
917    this.logger = logger;
918  }
919
920  public ExpansionProfile getExpansionProfile() {
921    return expProfile;
922  }
923
924  public void setExpansionProfile(ExpansionProfile expProfile) {
925    this.expProfile = expProfile;
926  }
927
928  @Override
929  public boolean isNoTerminologyServer() {
930    return noTerminologyServer;
931  }
932
933  public String getName() {
934    return name;
935  }
936
937  public void setName(String name) {
938    this.name = name;
939  }
940
941  @Override
942  public Set<String> getResourceNamesAsSet() {
943    Set<String> res = new HashSet<String>();
944    res.addAll(getResourceNames());
945    return res;
946  }
947
948  public void reportStatus(JsonObject json) {
949    json.addProperty("codeystem-count", codeSystems.size());
950    json.addProperty("valueset-count", valueSets.size());
951    json.addProperty("conceptmap-count", maps.size());
952    json.addProperty("transforms-count", transforms.size());
953    json.addProperty("structures-count", profiles.size());
954  }
955
956  public void cacheResource(Resource r) throws Exception {
957    if (r instanceof ValueSet)
958      seeValueSet(((ValueSet) r).getUrl(), (ValueSet) r);
959    else if (r instanceof CodeSystem)
960      seeCodeSystem(((CodeSystem) r).getUrl(), (CodeSystem) r);
961    else if (r instanceof StructureDefinition) {
962      StructureDefinition sd = (StructureDefinition) r;
963      if ("http://hl7.org/fhir/StructureDefinition/Extension".equals(sd.getBaseDefinition()))
964        seeExtensionDefinition(sd.getUrl(), sd);
965      else if (sd.getDerivation() == TypeDerivationRule.CONSTRAINT) 
966        seeProfile(sd.getUrl(), sd);
967    }
968  }
969
970  public void dropResource(String type, String id) throws FHIRException {
971    if (type.equals("ValueSet"))
972      dropValueSet(id);   
973    if (type.equals("CodeSystem"))
974      dropCodeSystem(id);   
975    if (type.equals("StructureDefinition")) {
976      dropProfile(id);   
977      dropExtensionDefinition(id);
978    }
979  }
980
981  public boolean isAllowLoadingDuplicates() {
982    return allowLoadingDuplicates;
983  }
984
985  public void setAllowLoadingDuplicates(boolean allowLoadingDuplicates) {
986    this.allowLoadingDuplicates = allowLoadingDuplicates;
987  }
988
989  @Override
990  public StructureDefinition fetchTypeDefinition(String typeName) {
991    return fetchResource(StructureDefinition.class, "http://hl7.org/fhir/StructureDefinition/"+typeName);
992  }
993
994
995}