001package org.hl7.fhir.dstu3.terminologies;
002
003/*-
004 * #%L
005 * org.hl7.fhir.dstu3
006 * %%
007 * Copyright (C) 2014 - 2019 Health Level 7
008 * %%
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 * 
013 *      http://www.apache.org/licenses/LICENSE-2.0
014 * 
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 * #L%
021 */
022
023
024import org.apache.commons.lang3.NotImplementedException;
025import org.hl7.fhir.dstu3.context.IWorkerContext;
026import org.hl7.fhir.dstu3.model.*;
027import org.hl7.fhir.dstu3.model.CodeSystem.CodeSystemContentMode;
028import org.hl7.fhir.dstu3.model.CodeSystem.ConceptDefinitionComponent;
029import org.hl7.fhir.dstu3.model.CodeSystem.ConceptDefinitionDesignationComponent;
030import org.hl7.fhir.dstu3.model.ValueSet.*;
031import org.hl7.fhir.dstu3.utils.ToolingExtensions;
032import org.hl7.fhir.exceptions.FHIRException;
033import org.hl7.fhir.exceptions.FHIRFormatError;
034import org.hl7.fhir.exceptions.NoTerminologyServiceException;
035import org.hl7.fhir.exceptions.TerminologyServiceException;
036import org.hl7.fhir.utilities.Utilities;
037
038import java.io.FileNotFoundException;
039import java.io.IOException;
040import java.util.*;
041
042import static org.apache.commons.lang3.StringUtils.isNotBlank;
043
044/*
045 * Copyright (c) 2011+, HL7, Inc
046 * All rights reserved.
047 *
048 * Redistribution and use in source and binary forms, with or without modification,
049 * are permitted provided that the following conditions are met:
050 *
051 * Redistributions of source code must retain the above copyright notice, this
052 * list of conditions and the following disclaimer.
053 * Redistributions in binary form must reproduce the above copyright notice,
054 * this list of conditions and the following disclaimer in the documentation
055 * and/or other materials provided with the distribution.
056 * Neither the name of HL7 nor the names of its contributors may be used to
057 * endorse or promote products derived from this software without specific
058 * prior written permission.
059 *
060 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
061 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
062 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
063 * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
064 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
065 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
066 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
067 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
068 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
069 * POSSIBILITY OF SUCH DAMAGE.
070 *
071 */
072
073public class ValueSetExpanderSimple implements ValueSetExpander {
074
075  private List<ValueSetExpansionContainsComponent> codes = new ArrayList<ValueSet.ValueSetExpansionContainsComponent>();
076  private List<ValueSetExpansionContainsComponent> roots = new ArrayList<ValueSet.ValueSetExpansionContainsComponent>();
077  private Map<String, ValueSetExpansionContainsComponent> map = new HashMap<String, ValueSet.ValueSetExpansionContainsComponent>();
078  private IWorkerContext context;
079  private boolean canBeHeirarchy = true;
080  private Set<String> excludeKeys = new HashSet<String>();
081  private Set<String> excludeSystems = new HashSet<String>();
082  private ValueSetExpanderFactory factory;
083  private ValueSet focus;
084  private int maxExpansionSize = 500;
085
086  private int total;
087
088  public ValueSetExpanderSimple(IWorkerContext context, ValueSetExpanderFactory factory) {
089    super();
090    this.context = context;
091    this.factory = factory;
092  }
093
094  public void setMaxExpansionSize(int theMaxExpansionSize) {
095    maxExpansionSize = theMaxExpansionSize;
096  }
097
098  private ValueSetExpansionContainsComponent addCode(String system, String code, String display, ValueSetExpansionContainsComponent parent, List<ConceptDefinitionDesignationComponent> designations,
099                                                     ExpansionProfile profile, boolean isAbstract, boolean inactive, List<ValueSet> filters) {
100    if (filters != null && !filters.isEmpty() && !filterContainsCode(filters, system, code))
101      return null;
102    ValueSetExpansionContainsComponent n = new ValueSet.ValueSetExpansionContainsComponent();
103    n.setSystem(system);
104    n.setCode(code);
105    if (isAbstract)
106      n.setAbstract(true);
107    if (inactive)
108      n.setInactive(true);
109
110    if (profile.getIncludeDesignations() && designations != null) {
111      for (ConceptDefinitionDesignationComponent t : designations) {
112        ToolingExtensions.addLanguageTranslation(n, t.getLanguage(), t.getValue());
113      }
114    }
115    ConceptDefinitionDesignationComponent t = profile.hasLanguage() ? getMatchingLang(designations, profile.getLanguage()) : null;
116    if (t == null)
117      n.setDisplay(display);
118    else
119      n.setDisplay(t.getValue());
120
121    String s = key(n);
122    if (map.containsKey(s) || excludeKeys.contains(s)) {
123      canBeHeirarchy = false;
124    } else {
125      codes.add(n);
126      map.put(s, n);
127      total++;
128    }
129    if (canBeHeirarchy && parent != null) {
130      parent.getContains().add(n);
131    } else {
132      roots.add(n);
133    }
134    return n;
135  }
136
137  private boolean filterContainsCode(List<ValueSet> filters, String system, String code) {
138    for (ValueSet vse : filters)
139      if (expansionContainsCode(vse.getExpansion().getContains(), system, code))
140        return true;
141    return false;
142  }
143
144  private boolean expansionContainsCode(List<ValueSetExpansionContainsComponent> contains, String system, String code) {
145    for (ValueSetExpansionContainsComponent cc : contains) {
146      if (system.equals(cc.getSystem()) && code.equals(cc.getCode()))
147        return true;
148      if (expansionContainsCode(cc.getContains(), system, code))
149        return true;
150    }
151    return false;
152  }
153
154  private ConceptDefinitionDesignationComponent getMatchingLang(List<ConceptDefinitionDesignationComponent> list, String lang) {
155    for (ConceptDefinitionDesignationComponent t : list)
156      if (t.getLanguage().equals(lang))
157        return t;
158    for (ConceptDefinitionDesignationComponent t : list)
159      if (t.getLanguage().startsWith(lang))
160        return t;
161    return null;
162  }
163
164  private void addCodeAndDescendents(CodeSystem cs, String system, ConceptDefinitionComponent def, ValueSetExpansionContainsComponent parent, ExpansionProfile profile, List<ValueSet> filters)
165    throws FHIRException {
166    if (!CodeSystemUtilities.isDeprecated(cs, def)) {
167      ValueSetExpansionContainsComponent np = null;
168      boolean abs = CodeSystemUtilities.isNotSelectable(cs, def);
169      boolean inc = CodeSystemUtilities.isInactive(cs, def);
170      if (canBeHeirarchy || !abs)
171        np = addCode(system, def.getCode(), def.getDisplay(), parent, def.getDesignation(), profile, abs, inc, filters);
172      for (ConceptDefinitionComponent c : def.getConcept())
173        addCodeAndDescendents(cs, system, c, np, profile, filters);
174    } else
175      for (ConceptDefinitionComponent c : def.getConcept())
176        addCodeAndDescendents(cs, system, c, null, profile, filters);
177
178  }
179
180  private void addCodes(ValueSetExpansionComponent expand, List<ValueSetExpansionParameterComponent> params, ExpansionProfile profile, List<ValueSet> filters) throws ETooCostly {
181    if (expand.getContains().size() > maxExpansionSize)
182      throw new ETooCostly("Too many codes to display (>" + Integer.toString(expand.getContains().size()) + ")");
183    for (ValueSetExpansionParameterComponent p : expand.getParameter()) {
184      if (!existsInParams(params, p.getName(), p.getValue()))
185        params.add(p);
186    }
187
188    copyImportContains(expand.getContains(), null, profile, filters);
189  }
190
191  private void excludeCode(String theSystem, String theCode) {
192    ValueSetExpansionContainsComponent n = new ValueSet.ValueSetExpansionContainsComponent();
193    n.setSystem(theSystem);
194    n.setCode(theCode);
195    String s = key(n);
196    excludeKeys.add(s);
197  }
198
199  private void excludeCodes(ConceptSetComponent exc, List<ValueSetExpansionParameterComponent> params) throws TerminologyServiceException {
200    if (exc.hasSystem() && exc.getConcept().size() == 0 && exc.getFilter().size() == 0) {
201      excludeSystems.add(exc.getSystem());
202    }
203
204    if (exc.hasValueSet())
205      throw new Error("Processing Value set references in exclude is not yet done");
206    // importValueSet(imp.getValue(), params, profile);
207
208    CodeSystem cs = context.fetchCodeSystem(exc.getSystem());
209    if ((cs == null || cs.getContent() != CodeSystemContentMode.COMPLETE) && context.supportsSystem(exc.getSystem())) {
210      excludeCodes(context.expandVS(exc, false), params);
211      return;
212    }
213
214    for (ConceptReferenceComponent c : exc.getConcept()) {
215      excludeCode(exc.getSystem(), c.getCode());
216    }
217
218    if (exc.getFilter().size() > 0)
219      throw new NotImplementedException("not done yet");
220  }
221
222  private void excludeCodes(ValueSetExpansionComponent expand, List<ValueSetExpansionParameterComponent> params) {
223    for (ValueSetExpansionContainsComponent c : expand.getContains()) {
224      excludeCode(c.getSystem(), c.getCode());
225    }
226  }
227
228  private boolean existsInParams(List<ValueSetExpansionParameterComponent> params, String name, Type value) {
229    for (ValueSetExpansionParameterComponent p : params) {
230      if (p.getName().equals(name) && PrimitiveType.compareDeep(p.getValue(), value, false))
231        return true;
232    }
233    return false;
234  }
235
236  @Override
237  public ValueSetExpansionOutcome expand(ValueSet source, ExpansionProfile profile) {
238
239    if (profile == null)
240      profile = makeDefaultExpansion();
241    try {
242      focus = source.copy();
243      focus.setExpansion(new ValueSet.ValueSetExpansionComponent());
244      focus.getExpansion().setTimestampElement(DateTimeType.now());
245      focus.getExpansion().setIdentifier(Factory.createUUID());
246      if (!profile.getUrl().startsWith("urn:uuid:"))
247        focus.getExpansion().addParameter().setName("profile").setValue(new UriType(profile.getUrl()));
248
249      if (source.hasCompose())
250        handleCompose(source.getCompose(), focus.getExpansion().getParameter(), profile);
251
252      if (canBeHeirarchy) {
253        for (ValueSetExpansionContainsComponent c : roots) {
254          focus.getExpansion().getContains().add(c);
255        }
256      } else {
257        for (ValueSetExpansionContainsComponent c : codes) {
258          if (map.containsKey(key(c)) && !c.getAbstract()) { // we may have added abstract codes earlier while we still thought it might be heirarchical, but later we gave up, so now ignore them
259            focus.getExpansion().getContains().add(c);
260            c.getContains().clear(); // make sure any heirarchy is wiped
261          }
262        }
263      }
264
265      if (total > 0) {
266        focus.getExpansion().setTotal(total);
267      }
268
269      return new ValueSetExpansionOutcome(focus);
270    } catch (NoTerminologyServiceException e) {
271      // well, we couldn't expand, so we'll return an interface to a checker that can check membership of the set
272      // that might fail too, but it might not, later.
273      return new ValueSetExpansionOutcome(new ValueSetCheckerSimple(source, factory, context), e.getMessage(), TerminologyServiceErrorClass.NOSERVICE);
274    } catch (RuntimeException e) {
275      // TODO: we should put something more specific instead of just Exception below, since
276      // it swallows bugs.. what would be expected to be caught there?
277      throw e;
278    } catch (Exception e) {
279      // well, we couldn't expand, so we'll return an interface to a checker that can check membership of the set
280      // that might fail too, but it might not, later.
281      return new ValueSetExpansionOutcome(new ValueSetCheckerSimple(source, factory, context), e.getMessage(), TerminologyServiceErrorClass.UNKNOWN);
282    }
283  }
284
285  private ExpansionProfile makeDefaultExpansion() {
286    ExpansionProfile res = new ExpansionProfile();
287    res.setUrl("urn:uuid:" + UUID.randomUUID().toString().toLowerCase());
288    res.setExcludeNested(true);
289    res.setIncludeDesignations(false);
290    return res;
291  }
292
293  private void addToHeirarchy(List<ValueSetExpansionContainsComponent> target, List<ValueSetExpansionContainsComponent> source) {
294    for (ValueSetExpansionContainsComponent s : source) {
295      target.add(s);
296    }
297  }
298
299  private String getCodeDisplay(CodeSystem cs, String code) throws TerminologyServiceException {
300    ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), code);
301    if (def == null)
302      throw new TerminologyServiceException("Unable to find code '" + code + "' in code system " + cs.getUrl());
303    return def.getDisplay();
304  }
305
306  private ConceptDefinitionComponent getConceptForCode(List<ConceptDefinitionComponent> clist, String code) {
307    for (ConceptDefinitionComponent c : clist) {
308      if (code.equals(c.getCode()))
309        return c;
310      ConceptDefinitionComponent v = getConceptForCode(c.getConcept(), code);
311      if (v != null)
312        return v;
313    }
314    return null;
315  }
316
317  private void handleCompose(ValueSetComposeComponent compose, List<ValueSetExpansionParameterComponent> params, ExpansionProfile profile)
318    throws ETooCostly, FileNotFoundException, IOException, FHIRException {
319    // Exclude comes first because we build up a map of things to exclude
320    for (ConceptSetComponent inc : compose.getExclude())
321      excludeCodes(inc, params);
322    canBeHeirarchy = !profile.getExcludeNested() && excludeKeys.isEmpty() && excludeSystems.isEmpty();
323    boolean first = true;
324    for (ConceptSetComponent inc : compose.getInclude()) {
325      if (first == true)
326        first = false;
327      else
328        canBeHeirarchy = false;
329      includeCodes(inc, params, profile);
330    }
331
332  }
333
334  private ValueSet importValueSet(String value, List<ValueSetExpansionParameterComponent> params, ExpansionProfile profile)
335    throws ETooCostly, TerminologyServiceException, FileNotFoundException, IOException, FHIRFormatError {
336    if (value == null)
337      throw new TerminologyServiceException("unable to find value set with no identity");
338    ValueSet vs = context.fetchResource(ValueSet.class, value);
339    if (vs == null)
340      throw new TerminologyServiceException("Unable to find imported value set " + value);
341    ValueSetExpansionOutcome vso = factory.getExpander().expand(vs, profile);
342    if (vso.getError() != null)
343      throw new TerminologyServiceException("Unable to expand imported value set: " + vso.getError());
344    if (vso.getService() != null)
345      throw new TerminologyServiceException("Unable to expand imported value set " + value);
346    if (vs.hasVersion())
347      if (!existsInParams(params, "version", new UriType(vs.getUrl() + "|" + vs.getVersion())))
348        params.add(new ValueSetExpansionParameterComponent().setName("version").setValue(new UriType(vs.getUrl() + "|" + vs.getVersion())));
349    for (ValueSetExpansionParameterComponent p : vso.getValueset().getExpansion().getParameter()) {
350      if (!existsInParams(params, p.getName(), p.getValue()))
351        params.add(p);
352    }
353    canBeHeirarchy = false; // if we're importing a value set, we have to be combining, so we won't try for a heirarchy
354    return vso.getValueset();
355  }
356
357  private void copyImportContains(List<ValueSetExpansionContainsComponent> list, ValueSetExpansionContainsComponent parent, ExpansionProfile profile, List<ValueSet> filter) {
358    for (ValueSetExpansionContainsComponent c : list) {
359      ValueSetExpansionContainsComponent np = addCode(c.getSystem(), c.getCode(), c.getDisplay(), parent, null, profile, c.getAbstract(), c.getInactive(), filter);
360      copyImportContains(c.getContains(), np, profile, filter);
361    }
362  }
363
364  private void includeCodes(ConceptSetComponent inc, List<ValueSetExpansionParameterComponent> params, ExpansionProfile profile) throws ETooCostly, FileNotFoundException, IOException, FHIRException {
365    List<ValueSet> imports = new ArrayList<ValueSet>();
366    for (UriType imp : inc.getValueSet())
367      imports.add(importValueSet(imp.getValue(), params, profile));
368
369    if (!inc.hasSystem()) {
370      if (imports.isEmpty()) // though this is not supposed to be the case
371        return;
372      ValueSet base = imports.get(0);
373      imports.remove(0);
374      copyImportContains(base.getExpansion().getContains(), null, profile, imports);
375    } else {
376      CodeSystem cs = context.fetchCodeSystem(inc.getSystem());
377      if ((cs == null || cs.getContent() != CodeSystemContentMode.COMPLETE) && context.supportsSystem(inc.getSystem())) {
378        addCodes(context.expandVS(inc, canBeHeirarchy), params, profile, imports);
379        return;
380      }
381
382      if (cs == null) {
383        if (context.isNoTerminologyServer())
384          throw new NoTerminologyServiceException("unable to find code system " + inc.getSystem().toString());
385        else
386          throw new TerminologyServiceException("unable to find code system " + inc.getSystem().toString());
387      }
388      if (cs.getContent() != CodeSystemContentMode.COMPLETE)
389        throw new TerminologyServiceException("Code system " + inc.getSystem().toString() + " is incomplete");
390      if (cs.hasVersion())
391        if (!existsInParams(params, "version", new UriType(cs.getUrl() + "|" + cs.getVersion())))
392          params.add(new ValueSetExpansionParameterComponent().setName("version").setValue(new UriType(cs.getUrl() + "|" + cs.getVersion())));
393
394      if (inc.getConcept().size() == 0 && inc.getFilter().size() == 0) {
395        // special case - add all the code system
396        for (ConceptDefinitionComponent def : cs.getConcept()) {
397          addCodeAndDescendents(cs, inc.getSystem(), def, null, profile, imports);
398        }
399      }
400
401      if (!inc.getConcept().isEmpty()) {
402        canBeHeirarchy = false;
403        for (ConceptReferenceComponent c : inc.getConcept()) {
404          addCode(inc.getSystem(), c.getCode(), Utilities.noString(c.getDisplay()) ? getCodeDisplay(cs, c.getCode()) : c.getDisplay(), null, convertDesignations(c.getDesignation()), profile, false,
405            CodeSystemUtilities.isInactive(cs, c.getCode()), imports);
406        }
407      }
408      if (inc.getFilter().size() > 1) {
409        canBeHeirarchy = false; // which will bt the case if we get around to supporting this
410        throw new TerminologyServiceException("Multiple filters not handled yet"); // need to and them, and this isn't done yet. But this shouldn't arise in non loinc and snomed value sets
411      }
412      if (inc.getFilter().size() == 1) {
413        ConceptSetFilterComponent fc = inc.getFilter().get(0);
414        if ("concept".equals(fc.getProperty()) && fc.getOp() == FilterOperator.ISA) {
415          // special: all codes in the target code system under the value
416          ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue());
417          if (def == null)
418            throw new TerminologyServiceException("Code '" + fc.getValue() + "' not found in system '" + inc.getSystem() + "'");
419          addCodeAndDescendents(cs, inc.getSystem(), def, null, profile, imports);
420        } else if ("concept".equals(fc.getProperty()) && fc.getOp() == FilterOperator.DESCENDENTOF) {
421          // special: all codes in the target code system under the value
422          ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue());
423          if (def == null)
424            throw new TerminologyServiceException("Code '" + fc.getValue() + "' not found in system '" + inc.getSystem() + "'");
425          for (ConceptDefinitionComponent c : def.getConcept())
426            addCodeAndDescendents(cs, inc.getSystem(), c, null, profile, imports);
427        } else if ("display".equals(fc.getProperty()) && fc.getOp() == FilterOperator.EQUAL) {
428          // gg; note: wtf is this: if the filter is display=v, look up the code 'v', and see if it's diplsay is 'v'?
429          canBeHeirarchy = false;
430          ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue());
431          if (def != null) {
432            if (isNotBlank(def.getDisplay()) && isNotBlank(fc.getValue())) {
433              if (def.getDisplay().contains(fc.getValue())) {
434                addCode(inc.getSystem(), def.getCode(), def.getDisplay(), null, def.getDesignation(), profile, CodeSystemUtilities.isNotSelectable(cs, def), CodeSystemUtilities.isInactive(cs, def),
435                  imports);
436              }
437            }
438          }
439        } else
440          throw new NotImplementedException("Search by property[" + fc.getProperty() + "] and op[" + fc.getOp() + "] is not supported yet");
441      }
442    }
443  }
444
445  private List<ConceptDefinitionDesignationComponent> convertDesignations(List<ConceptReferenceDesignationComponent> list) {
446    List<ConceptDefinitionDesignationComponent> res = new ArrayList<CodeSystem.ConceptDefinitionDesignationComponent>();
447    for (ConceptReferenceDesignationComponent t : list) {
448      ConceptDefinitionDesignationComponent c = new ConceptDefinitionDesignationComponent();
449      c.setLanguage(t.getLanguage());
450      c.setUse(t.getUse());
451      c.setValue(t.getValue());
452    }
453    return res;
454  }
455
456  private String key(String uri, String code) {
457    return "{" + uri + "}" + code;
458  }
459
460  private String key(ValueSetExpansionContainsComponent c) {
461    return key(c.getSystem(), c.getCode());
462  }
463
464}