RepositoryFinderSupport.java

  1. /*
  2.  * *********************************************************************************************************************
  3.  *
  4.  * blueMarine II: Semantic Media Centre
  5.  * http://tidalwave.it/projects/bluemarine2
  6.  *
  7.  * Copyright (C) 2015 - 2021 by Tidalwave s.a.s. (http://tidalwave.it)
  8.  *
  9.  * *********************************************************************************************************************
  10.  *
  11.  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
  12.  * the License. You may obtain a copy of the License at
  13.  *
  14.  *     http://www.apache.org/licenses/LICENSE-2.0
  15.  *
  16.  * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
  17.  * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the License for the
  18.  * specific language governing permissions and limitations under the License.
  19.  *
  20.  * *********************************************************************************************************************
  21.  *
  22.  * git clone https://bitbucket.org/tidalwave/bluemarine2-src
  23.  * git clone https://github.com/tidalwave-it/bluemarine2-src
  24.  *
  25.  * *********************************************************************************************************************
  26.  */
  27. package it.tidalwave.bluemarine2.model.impl.catalog.finder;

  28. import javax.annotation.Nonnegative;
  29. import javax.annotation.Nonnull;
  30. import javax.inject.Inject;
  31. import java.util.Arrays;
  32. import java.util.ArrayList;
  33. import java.util.List;
  34. import java.util.Optional;
  35. import java.util.concurrent.atomic.AtomicInteger;
  36. import java.util.function.Function;
  37. import java.util.regex.Matcher;
  38. import java.util.regex.Pattern;
  39. import java.util.stream.Stream;
  40. import java.io.IOException;
  41. import java.io.InputStream;
  42. import org.eclipse.rdf4j.model.IRI;
  43. import org.eclipse.rdf4j.model.Value;
  44. import org.eclipse.rdf4j.model.impl.SimpleValueFactory;
  45. import org.eclipse.rdf4j.query.QueryLanguage;
  46. import org.eclipse.rdf4j.query.TupleQuery;
  47. import org.eclipse.rdf4j.query.TupleQueryResult;
  48. import org.eclipse.rdf4j.repository.Repository;
  49. import org.eclipse.rdf4j.repository.RepositoryConnection;
  50. import org.springframework.util.StreamUtils;
  51. import org.springframework.beans.factory.annotation.Configurable;
  52. import it.tidalwave.util.Id;
  53. import it.tidalwave.util.Finder;
  54. import it.tidalwave.util.LoggingUtilities;
  55. import it.tidalwave.util.ReflectionUtils;
  56. import it.tidalwave.util.Task;
  57. import it.tidalwave.util.spi.FinderSupport;
  58. import it.tidalwave.role.ContextManager;
  59. import it.tidalwave.bluemarine2.util.ImmutableTupleQueryResult;
  60. import it.tidalwave.bluemarine2.model.spi.CacheManager;
  61. import it.tidalwave.bluemarine2.model.spi.CacheManager.Cache;
  62. import it.tidalwave.bluemarine2.model.spi.SourceAwareFinder;
  63. import it.tidalwave.bluemarine2.model.impl.catalog.factory.RepositoryEntityFactory;
  64. import lombok.extern.slf4j.Slf4j;
  65. import lombok.EqualsAndHashCode;
  66. import lombok.Getter;
  67. import lombok.RequiredArgsConstructor;
  68. import lombok.Setter;
  69. import lombok.ToString;
  70. import static java.util.stream.Collectors.*;
  71. import static java.nio.charset.StandardCharsets.UTF_8;
  72. import static it.tidalwave.bluemarine2.util.RdfUtilities.streamOf;
  73. import static it.tidalwave.bluemarine2.model.vocabulary.BMMO.*;

  74. /***********************************************************************************************************************
  75.  *
  76.  * A base class for creating {@link Finder}s.
  77.  *
  78.  * @param <ENTITY>  the entity the {@code Finder} should find
  79.  * @param <FINDER>  the subclass
  80.  *
  81.  * @stereotype      Finder
  82.  *
  83.  * @author  Fabrizio Giudici
  84.  *
  85.  **********************************************************************************************************************/
  86. @Configurable @Slf4j
  87. public class RepositoryFinderSupport<ENTITY, FINDER extends Finder<ENTITY>>
  88.         extends FinderSupport<ENTITY, FINDER>
  89.         implements SourceAwareFinder<ENTITY, FINDER>
  90.   {
  91.     private static final String REGEX_BINDING_TAG = "^@([A-Za-z0-9]*)@";

  92.     private static final String REGEX_BINDING_TAG_LINE = REGEX_BINDING_TAG + ".*$";

  93.     private static final String REGEX_COMMENT = "^ *#.*";

  94.     private static final String PREFIXES = "PREFIX foaf:  <http://xmlns.com/foaf/0.1/>\n"
  95.                                          + "PREFIX rdf:   <http://www.w3.org/1999/02/22-rdf-syntax-ns#>\n"
  96.                                          + "PREFIX rel:   <http://purl.org/vocab/relationship/>\n"
  97.                                          + "PREFIX bmmo:  <http://bluemarine.tidalwave.it/2015/04/mo/>\n"
  98.                                          + "PREFIX mo:    <http://purl.org/ontology/mo/>\n"
  99.                                          + "PREFIX vocab: <http://dbtune.org/musicbrainz/resource/vocab/>\n"
  100.                                          + "PREFIX xs:    <http://www.w3.org/2001/XMLSchema#>\n";

  101.     private static final String QUERY_COUNT_HOLDER = "queryCount";

  102.     private static final long serialVersionUID = 1896412264314804227L;

  103.     private static final SimpleValueFactory FACTORY = SimpleValueFactory.getInstance();

  104.     @Nonnull
  105.     protected final transient Repository repository;

  106.     @Nonnull
  107.     private final Class<ENTITY> entityClass;

  108.     @Nonnull
  109.     private final String idName;

  110.     @Nonnull
  111.     private final transient Optional<Id> id;

  112.     @Nonnull
  113.     private final transient Optional<Value> source;

  114.     @Nonnull
  115.     private final transient Optional<Value> sourceFallback;

  116.     @Inject
  117.     private transient ContextManager contextManager;

  118.     @Inject
  119.     private transient RepositoryEntityFactory entityFactory;

  120.     @Inject
  121.     private transient CacheManager cacheManager;

  122.     // FIXME: move to a stats bean
  123.     private static final AtomicInteger queryCount = new AtomicInteger();

  124.     @Getter @Setter
  125.     private static boolean dumpThreadOnQuery = false;

  126.     /*******************************************************************************************************************
  127.      *
  128.      *
  129.      *
  130.      ******************************************************************************************************************/
  131.     @RequiredArgsConstructor(staticName = "withSparql") @EqualsAndHashCode @ToString
  132.     protected static class QueryAndParameters
  133.       {
  134.         @Getter @Nonnull
  135.         private final String sparql;

  136.         @Nonnull
  137.         private final List<Object> parameters = new ArrayList<>();

  138.         @Nonnull
  139.         public QueryAndParameters withParameter (@Nonnull final String name, @Nonnull final Optional<? extends Value> value)
  140.           {
  141.             return value.map(v -> withParameter(name, v)).orElse(this);
  142.           }

  143.         @Nonnull
  144.         public QueryAndParameters withParameter (@Nonnull final String name, @Nonnull final Value value)
  145.           {
  146.             parameters.addAll(List.of(name, value));
  147.             return this;
  148.           }

  149.         @Nonnull
  150.         public Object[] getParameters()
  151.           {
  152.             return parameters.toArray();
  153.           }

  154.         @Nonnull
  155.         private String getCountSparql()
  156.           {
  157.             return String.format("SELECT (COUNT(*) AS ?%s)%n  {%n%s%n  }",
  158.                                  QUERY_COUNT_HOLDER,
  159.                                  sparql.replaceAll("ORDER BY[\\s\\S]*", ""));
  160.           }
  161.       }

  162.     /*******************************************************************************************************************
  163.      *
  164.      *
  165.      *
  166.      ******************************************************************************************************************/
  167.     protected RepositoryFinderSupport (@Nonnull final Repository repository, @Nonnull final String idName)
  168.       {
  169.         this.repository = repository;
  170.         this.entityClass = (Class<ENTITY>)ReflectionUtils.getTypeArguments(RepositoryFinderSupport.class, getClass()).get(0);
  171.         this.idName = idName;
  172.         this.id = Optional.empty();
  173.         this.source = Optional.of(O_SOURCE_EMBEDDED); // FIXME: resets
  174.         this.sourceFallback = Optional.empty(); // FIXME: resets
  175.       }

  176.     /*******************************************************************************************************************
  177.      *
  178.      *
  179.      *
  180.      ******************************************************************************************************************/
  181.     private RepositoryFinderSupport (@Nonnull final Repository repository,
  182.                                      @Nonnull final Class<ENTITY> entityClass,
  183.                                      @Nonnull final String idName,
  184.                                      @Nonnull final Optional<Id> id,
  185.                                      @Nonnull final Optional<Value> source,
  186.                                      @Nonnull final Optional<Value> sourceFallback)
  187.       {
  188.         this.repository = repository;
  189.         this.entityClass = entityClass;
  190.         this.idName = idName;
  191.         this.id = id;
  192.         this.source = source;
  193.         this.sourceFallback = sourceFallback;
  194.       }

  195.     /*******************************************************************************************************************
  196.      *
  197.      * Clone constructor.
  198.      *
  199.      ******************************************************************************************************************/
  200.     public RepositoryFinderSupport (@Nonnull final RepositoryFinderSupport<ENTITY, FINDER> other,
  201.                                     @Nonnull final Object override)
  202.       {
  203.         super(other, override);
  204.         final RepositoryFinderSupport<ENTITY, FINDER> source = getSource(RepositoryFinderSupport.class, other, override);
  205.         this.repository = source.repository;
  206.         this.entityClass = source.entityClass;
  207.         this.idName = source.idName;
  208.         this.id = source.id;
  209.         this.source = source.source;
  210.         this.sourceFallback = source.sourceFallback;
  211.      }

  212.     /*******************************************************************************************************************
  213.      *
  214.      * {@inheritDoc}
  215.      *
  216.      ******************************************************************************************************************/
  217.     @Override @Nonnull
  218.     protected final List<? extends ENTITY> computeNeededResults()
  219.       {
  220.         return query(QueryAndParameters::getSparql,
  221.                      result -> createEntities(repository, entityClass, result),
  222.                      result -> String.format("%d entities", result.size()));
  223.       }

  224.     /*******************************************************************************************************************
  225.      *
  226.      * {@inheritDoc}
  227.      *
  228.      ******************************************************************************************************************/
  229.     @Override @Nonnegative
  230.     public int count()
  231.       {
  232.         return query(QueryAndParameters::getCountSparql,
  233.                      result -> Integer.parseInt(result.next().getValue(QUERY_COUNT_HOLDER).stringValue()),
  234.                      result -> String.format("%d", result));
  235.       }

  236.     /*******************************************************************************************************************
  237.      *
  238.      * {@inheritDoc}
  239.      *
  240.      ******************************************************************************************************************/
  241.     @Override @Nonnull
  242.     public FINDER withId (@Nonnull final Id id)
  243.       {
  244.         return clonedWith(new RepositoryFinderSupport(repository,
  245.                                                  entityClass,
  246.                                                  idName,
  247.                                                  Optional.of(id),
  248.                                                  source,
  249.                                                  sourceFallback));
  250.       }

  251.     /*******************************************************************************************************************
  252.      *
  253.      * {@inheritDoc}
  254.      *
  255.      ******************************************************************************************************************/
  256.     @Override @Nonnull
  257.     public FINDER importedFrom (@Nonnull final Optional<Id> optionalSource)
  258.       {
  259.         return optionalSource.map(this::importedFrom).orElse((FINDER)this);
  260.       }

  261.     /*******************************************************************************************************************
  262.      *
  263.      * {@inheritDoc}
  264.      *
  265.      ******************************************************************************************************************/
  266.     @Override @Nonnull
  267.     public FINDER importedFrom (@Nonnull final Id source)
  268.       {
  269.         return clonedWith(new RepositoryFinderSupport(repository,
  270.                                                  entityClass,
  271.                                                  idName,
  272.                                                  id,
  273.                                                  Optional.of(FACTORY.createLiteral(source.toString())),
  274.                                                  sourceFallback));
  275.       }

  276.     /*******************************************************************************************************************
  277.      *
  278.      * {@inheritDoc}
  279.      *
  280.      ******************************************************************************************************************/
  281.     @Override @Nonnull
  282.     public FINDER withFallback (@Nonnull final Optional<Id> sourceFallback)
  283.       {
  284.         return sourceFallback.map(this::withFallback).orElse((FINDER)this);
  285.       }

  286.     /*******************************************************************************************************************
  287.      *
  288.      * {@inheritDoc}
  289.      *
  290.      ******************************************************************************************************************/
  291.     @Override @Nonnull
  292.     public FINDER withFallback (@Nonnull final Id sourceFallback)
  293.       {
  294.         return clonedWith(new RepositoryFinderSupport(repository,
  295.                                                  entityClass,
  296.                                                  idName,
  297.                                                  id,
  298.                                                  source,
  299.                                                  Optional.of(FACTORY.createLiteral(sourceFallback.toString()))));
  300.       }

  301.     /*******************************************************************************************************************
  302.      *
  303.      * Returns the count of queries performed so far.
  304.      *
  305.      * @return      the count of queries
  306.      *
  307.      ******************************************************************************************************************/
  308.     @Nonnegative
  309.     public static int getQueryCount()
  310.       {
  311.         return queryCount.intValue();
  312.       }

  313.     /*******************************************************************************************************************
  314.      *
  315.      * Resets the count of queries performed so far.
  316.      *
  317.      ******************************************************************************************************************/
  318.     public static void resetQueryCount()
  319.       {
  320.         queryCount.set(0);
  321.       }

  322.     /*******************************************************************************************************************
  323.      *
  324.      * Prepares the SPARQL query and its parameters.
  325.      *
  326.      * @return      the SPARQL query and its parameters
  327.      *
  328.      ******************************************************************************************************************/
  329.     @Nonnull
  330.     protected /* abstract */ QueryAndParameters prepareQuery()
  331.       {
  332.         throw new UnsupportedOperationException("Must be implemented by subclasses");
  333.       }

  334.     /*******************************************************************************************************************
  335.      *
  336.      * Performs a query, eventually using the cache.
  337.      *
  338.      * @param   sparqlSelector  a function that select the SPARQL statement to use
  339.      * @param   finalizer       a function to transform the query raw result into the final result
  340.      * @param   resultToString  a function that provide the logging string for the result
  341.      * @return                  the found entities
  342.      *
  343.      ******************************************************************************************************************/
  344.     @Nonnull
  345.     private <E> E query (@Nonnull final Function<QueryAndParameters, String> sparqlSelector,
  346.                          @Nonnull final Function<TupleQueryResult, E> finalizer,
  347.                          @Nonnull final Function<E, String> resultToString)
  348.       {
  349.         log.info("query() - {}", entityClass);
  350.         final long baseTime = System.nanoTime();
  351.         final QueryAndParameters queryAndParameters = prepareQuery()
  352.                 .withParameter(idName, id.map(this::iriFor))
  353.                 .withParameter("source", source)
  354.                 .withParameter("fallback", sourceFallback.equals(source) ? Optional.empty() : sourceFallback);
  355.         final Object[] parameters = queryAndParameters.getParameters();
  356.         final String originalSparql = sparqlSelector.apply(queryAndParameters);
  357.         final String sparql = PREFIXES + Stream.of(originalSparql.split("\n"))
  358.                                                .filter(s -> matchesTag(s, parameters))
  359.                                                .map(s -> s.replaceAll(REGEX_BINDING_TAG, ""))
  360.                                                .collect(joining("\n"));
  361.         log(originalSparql, sparql, parameters);
  362.         final E result = query(sparql, finalizer, parameters);
  363.         queryCount.incrementAndGet();
  364.         final long elapsedTime = System.nanoTime() - baseTime;
  365.         log.info(">>>> query returned {} in {} msec", resultToString.apply(result), elapsedTime / 1E6);
  366.         LoggingUtilities.dumpStack(this, dumpThreadOnQuery);

  367.         return result;
  368.       }

  369.     /*******************************************************************************************************************
  370.      *
  371.      * Performs a query.
  372.      *
  373.      * @param   sparql          the SPARQL of the query
  374.      * @param   finalizer       a function to transform the query raw result into the final result
  375.      * @param   parameters      an optional set of parameters of the query ("name", value, "name", value ,,,)
  376.      * @return                  the found entities
  377.      *
  378.      ******************************************************************************************************************/
  379.     @Nonnull
  380.     private <R> R query (@Nonnull final String sparql,
  381.                          @Nonnull final Function<TupleQueryResult, R> finalizer,
  382.                          @Nonnull final Object ... parameters)
  383.       {
  384.         try (final RepositoryConnection connection = repository.getConnection())
  385.           {
  386.             final TupleQuery query = connection.prepareTupleQuery(QueryLanguage.SPARQL, sparql);

  387.             for (int i = 0; i < parameters.length; i += 2)
  388.               {
  389.                 query.setBinding((String)parameters[i], (Value)parameters[i + 1]);
  390.               }
  391.             //
  392.             // Don't cache entities because they are injected with DCI roles in function of the context.
  393.             // Caching tuples is safe.
  394.             final Cache cache = cacheManager.getCache(RepositoryFinderSupport.class);
  395.             final String key = String.format("%s # %s", compacted(sparql), Arrays.toString(parameters));

  396.             try (final ImmutableTupleQueryResult result = cache.getCachedObject(key,
  397.                                                                 () -> new ImmutableTupleQueryResult(query.evaluate())))
  398.               {
  399.                 // ImmutableTupleQueryResult is not thread safe, so clone the cached instance
  400.                 return finalizer.apply(new ImmutableTupleQueryResult(result));
  401.               }
  402.           }
  403.       }

  404.     /*******************************************************************************************************************
  405.      *
  406.      * Facility method that creates an {@link IRI} given an {@link Id}.
  407.      *
  408.      * @param   id  the {@code Id}
  409.      * @return      the {@code IRI}
  410.      *
  411.      ******************************************************************************************************************/
  412.     @Nonnull
  413.     protected Value iriFor (@Nonnull final Id id)
  414.       {
  415.         return FACTORY.createIRI(id.stringValue());
  416.       }

  417.     /*******************************************************************************************************************
  418.      *
  419.      *
  420.      ******************************************************************************************************************/
  421.     @Nonnull
  422.     protected Value literalFor (final boolean b)
  423.       {
  424.         return FACTORY.createLiteral(b);
  425.       }

  426.     /*******************************************************************************************************************
  427.      *
  428.      * Reads a SPARQL statement from a named resource
  429.      *
  430.      * @param   clazz   the reference class
  431.      * @param   name    the resource name
  432.      * @return          the SPARQL statement
  433.      *
  434.      ******************************************************************************************************************/
  435.     @Nonnull
  436.     protected static String readSparql (@Nonnull final Class<?> clazz, @Nonnull final String name)
  437.       {
  438.         try (final InputStream is = clazz.getResourceAsStream(name))
  439.           {
  440.             return Stream.of(StreamUtils.copyToString(is, UTF_8)
  441.                                         .split("\n"))
  442.                                         .filter(s -> !s.matches(REGEX_COMMENT))
  443.                                         .collect(joining("\n"));
  444.           }
  445.         catch (IOException e)
  446.           {
  447.             throw new RuntimeException(e);
  448.           }
  449.       }

  450.     /*******************************************************************************************************************
  451.      *
  452.      * Instantiates an entity for each given {@link TupleQueryResult}. Entities are instantiated in the DCI contents
  453.      * associated to this {@link Finder} - see {@link #getContexts()}.
  454.      *
  455.      * @param   <E>             the static type of the entities to instantiate
  456.      * @param   repository      the repository we're querying
  457.      * @param   entityClass     the dynamic type of the entities to instantiate
  458.      * #param   queryResult     the {@code TupleQueryResult}
  459.      * @return                  the instantiated entities
  460.      *
  461.      ******************************************************************************************************************/
  462.     @Nonnull
  463.     private <E> List<E> createEntities (@Nonnull final Repository repository,
  464.                                         @Nonnull final Class<E> entityClass,
  465.                                         @Nonnull final TupleQueryResult queryResult)
  466.       {
  467.         return contextManager.runWithContexts(getContexts(), new Task<>()
  468.           {
  469.             @Override @Nonnull
  470.             public List<E> run()
  471.               {
  472.                 return streamOf(queryResult)
  473.                         .map(bindingSet -> entityFactory.createEntity(repository, entityClass, bindingSet))
  474.                         .collect(toList());
  475.               }
  476.           });
  477.         // TODO: requires TheseFoolishThings 3.1-ALPHA-3
  478. //        return contextManager.runWithContexts(getContexts(), () -> streamOf(queryResult)
  479. //                .map(bindingSet -> entityFactory.createEntity(repository, entityClass, bindingSet))
  480. //                .collect(toList()));
  481.       }

  482.     /*******************************************************************************************************************
  483.      *
  484.      * Returns {@code true} if the given string contains a binding tag (in the form {@code @TAG@}) that matches one
  485.      * of the bindings; or if there are no binding tags. This is used as a filter to eliminate portions of SPARQL
  486.      * queries that don't match any binding.
  487.      *
  488.      * @param   string      the string
  489.      * @param   bindings    the bindings
  490.      * @return              {@code true} if there is a match
  491.      *
  492.      ******************************************************************************************************************/
  493.     private static boolean matchesTag (@Nonnull final String string, @Nonnull final Object[] bindings)
  494.       {
  495.         final Pattern patternBindingTagLine = Pattern.compile(REGEX_BINDING_TAG_LINE);
  496.         final Matcher matcher = patternBindingTagLine.matcher(string);

  497.         if (!matcher.matches())
  498.           {
  499.             return true;
  500.           }

  501.         final String tag = matcher.group(1);

  502.         for (int i = 0; i < bindings.length; i+= 2)
  503.           {
  504.             if (tag.equals(bindings[i]))
  505.               {
  506.                 return true;
  507.               }
  508.           }

  509.         return false;
  510.       }

  511.     /*******************************************************************************************************************
  512.      *
  513.      * Logs the query at various detail levels.
  514.      *
  515.      * @param   originalSparql  the original SPARQL statement
  516.      * @param   sparql          the SPARQL statement after binding tag filtering
  517.      * @param   bindings        the bindings
  518.      *
  519.      ******************************************************************************************************************/
  520.     private void log (@Nonnull final String originalSparql,
  521.                       @Nonnull final String sparql,
  522.                       @Nonnull final Object ... bindings)
  523.       {
  524.         if (log.isTraceEnabled())
  525.           {
  526.             Stream.of(originalSparql.split("\n")).forEach(s -> log.trace(">>>> original query: {}", s));
  527.           }

  528.         if (log.isDebugEnabled())
  529.           {
  530.             Stream.of(sparql.split("\n")).forEach(s -> log.debug(">>>> query: {}", s));
  531.           }

  532.         if (!log.isDebugEnabled() && log.isInfoEnabled())
  533.           {
  534.             log.info(">>>> query: {}", compacted(sparql));
  535.           }

  536.         if (log.isInfoEnabled())
  537.           {
  538.             log.info(">>>> query parameters: {}", Arrays.toString(bindings));
  539.           }
  540.      }

  541.     /*******************************************************************************************************************
  542.      *
  543.      *
  544.      ******************************************************************************************************************/
  545.     @Nonnull
  546.     private static String compacted (@Nonnull final String sparql)
  547.       {
  548.         return sparql.replace("\n", " ").replaceAll("\\s+", " ").trim();
  549.       }
  550.   }