Documentation updates for 4.2 (#205)

* Updating docs for 4.2.

* Migrating StripProvenance over to use the ProvenanceSerialization interface.

* Updating the configuration tutorial.

* Initial readme updates for 4.2

* Adding the start of the onnx export tutorial.

* Adding first draft of release notes for 4.2.

* Fixing the JEP 290 filter.

* Changing Model.castModel so it's not static.

* Adding a reproducibility tutorial and updating the irises and onnx export tutorials.

* Updating gitignore file.

* Updating reproducibility tutorial with a bigger diff example.

* Updating the v4.2 release notes.

* Adding TF-Java PR number.

* Updating docs for the HDBSCAN implementation.

* Updating 4.2 release notes.

* Adding Tribuo v4.1.1 release notes.

* Finishing ONNX export tutorial.

* Updating ONNX export tutorial.

* Docs updates after rebase.

* CastModel fix in OCIModelCLI.

* Javadoc updates after rebase.

* Fixing the circular PR reference in the 4.2 release notes.

* Fixing an accidental deletion in the external models tutorial.

Author: Craigacp

.gitignore

@@ -16,16 +16,32 @@ bin/
 .*.swp
 
 # Other files
-*.jar
-*.class
 *.er
 *.log
 *.bck
 *.so
+*.patch
+
+# Binaries
+*.jar
+*.class
+
+# Archives
+*.gz
+*.zip
 
 # Serialised models
 *.ser
 
 # Temporary stuff
 junk/*
 .DS_Store
+.ipynb_checkpoints
+
+# Profiling files
+*.jfr
+*.iprof
+*.jfc
+
+# Tutorial files
+tutorials/*.svm

Common/LibSVM/src/main/java/org/tribuo/common/libsvm/LibSVMTrainer.java

@@ -131,6 +131,7 @@ protected LibSVMTrainer() {}
     /**
      * Constructs a LibSVMTrainer from the parameters.
      * @param parameters The SVM parameters.
+     * @param seed The RNG seed.
      */
     protected LibSVMTrainer(SVMParameters<T> parameters, long seed) {
         this.parameters = parameters.getParameters();

Core/src/main/java/org/tribuo/Model.java

@@ -300,20 +300,21 @@ public String toString() {
 
     /**
      * Casts the model to the specified output type, assuming it is valid.
-     * <p>
      * If it's not valid, throws {@link ClassCastException}.
-     * @param inputModel The model to cast.
+     * <p>
+     * This method is intended for use on a deserialized model to restore it's
+     * generic type in a safe way.
      * @param outputType The output type to cast to.
-     * @param <T> The output type.
+     * @param <U> The output type.
      * @return The model cast to the correct value.
      */
-    public static <T extends Output<T>> Model<T> castModel(Model<?> inputModel, Class<T> outputType) {
-        if (inputModel.validate(outputType)) {
+    public <U extends Output<U>> Model<U> castModel(Class<U> outputType) {
+        if (validate(outputType)) {
             @SuppressWarnings("unchecked") // guarded by validate
-            Model<T> castedModel = (Model<T>) inputModel;
+            Model<U> castedModel = (Model<U>) this;
             return castedModel;
         } else {
-            throw new ClassCastException("Attempted to cast model to " + outputType.getName() + " which is not valid for model " + inputModel.toString());
+            throw new ClassCastException("Attempted to cast model to " + outputType.getName() + " which is not valid for model " + this.toString());
         }
     }
 

Core/src/main/java/org/tribuo/ensemble/BaggingTrainer.java

@@ -49,6 +49,7 @@
  * "The Elements of Statistical Learning"
  * Springer 2001. <a href="http://web.stanford.edu/~hastie/ElemStatLearn/">PDF</a>
  * </pre>
+ * @param <T> The prediction type.
  */
 public class BaggingTrainer<T extends Output<T>> implements Trainer<T> {
 
@@ -177,6 +178,7 @@ public EnsembleModel<T> train(Dataset<T> examples, Map<String, Provenance> runPr
      * @param labelIDs The output domain.
      * @param randInt A random int from an rng instance
      * @param runProvenance Provenance for this instance.
+     * @param invocationCount The invocation count for the inner trainer.
      * @return The trained ensemble member.
      */
     protected Model<T> trainSingleModel(Dataset<T> examples, ImmutableFeatureMap featureIDs, ImmutableOutputInfo<T> labelIDs, int randInt, Map<String,Provenance> runProvenance, int invocationCount) {

Core/src/main/java/org/tribuo/ensemble/EnsembleCombiner.java

@@ -79,9 +79,10 @@ default ONNXNode exportCombiner(ONNXNode input) {
      * will be required to provide ONNX support.
      * @param input the node to be ensembled according to this implementation.
      * @param weight The node of weights for ensembling.
+     * @param <U> The type of the weights input reference.
      * @return The leaf node of the graph of operations added to ensemble input.
      */
-    default <T extends ONNXRef<?>> ONNXNode exportCombiner(ONNXNode input, T weight) {
+    default <U extends ONNXRef<?>> ONNXNode exportCombiner(ONNXNode input, U weight) {
         Logger.getLogger(this.getClass().getName()).severe("Tried to export an ensemble combiner to ONNX format, but this is not implemented.");
         throw new IllegalStateException("This ensemble cannot be exported as the combiner '" + this.getClass() + "' uses the default implementation of EnsembleCombiner.exportCombiner.");
     }

Data/src/main/java/org/tribuo/data/sql/SQLDBConfig.java

@@ -72,7 +72,7 @@ private SQLDBConfig() {}
     /**
      * Constructs a SQL database configuration.
      * <p>
-     * Note it is recommended that wallet based connections are used rather than this constructor using {@link SQLDBConfig(String,Map)}.
+     * Note it is recommended that wallet based connections are used rather than this constructor using {@link #SQLDBConfig(String,Map)}.
      * @param connectionString The connection string.
      * @param username The username.
      * @param password The password.
@@ -87,7 +87,7 @@ public SQLDBConfig(String connectionString, String username, String password, Ma
     /**
      * Constructs a SQL database configuration.
      * <p>
-     * Note it is recommended that wallet based connections are used rather than this constructor using {@link SQLDBConfig(String,Map)}.
+     * Note it is recommended that wallet based connections are used rather than this constructor using {@link #SQLDBConfig(String,Map)}.
      * @param host The host to connect to.
      * @param port The port to connect on.
      * @param db The db name.

Interop/OCI/src/main/java/org/tribuo/interop/oci/OCIModel.java

@@ -309,6 +309,7 @@ public static ConfigFileAuthenticationDetailsProvider makeAuthProvider(Path conf
      * @param configFile      The OCI configuration file, if null use the default file.
      * @param endpointURL     The endpoint URL.
      * @param outputConverter The converter for the specified output type.
+     * @param <T> The output type.
      * @return An OCIModel ready to score new inputs.
      */
     public static <T extends Output<T>> OCIModel<T> createOCIModel(OutputFactory<T> factory,
@@ -332,6 +333,7 @@ public static <T extends Output<T>> OCIModel<T> createOCIModel(OutputFactory<T>
      * @param profileName     The profile name in the OCI configuration file, if null uses the default profile.
      * @param endpointURL     The endpoint URL.
      * @param outputConverter The converter for the specified output type.
+     * @param <T> The output type.
      * @return An OCIModel ready to score new inputs.
      */
     public static <T extends Output<T>> OCIModel<T> createOCIModel(OutputFactory<T> factory,

Interop/OCI/src/main/java/org/tribuo/interop/oci/OCIModelCLI.java

@@ -64,7 +64,7 @@ private static void createModelAndDeploy(OCIModelOptions options) throws IOExcep
         // Load the Tribuo model
         Model<Label> model;
         try (ObjectInputStream ois = new ObjectInputStream(Files.newInputStream(options.modelPath))) {
-            model = Model.castModel((Model<?>) ois.readObject(),Label.class);
+            model = ((Model<?>)ois.readObject()).castModel(Label.class);
         }
         if (!(model instanceof ONNXExportable)) {
             throw new IllegalArgumentException("Model not ONNXExportable, received " + model.toString());

Interop/OCI/src/main/java/org/tribuo/interop/oci/OCIUtil.java

@@ -373,6 +373,7 @@ public static <T extends Output<T>, U extends Model<T> & ONNXExportable> String
     /**
      * Creates the OCI DS model artifact zip file.
      * @param onnxFile The ONNX file to create.
+     * @param config The model artifact configuration.
      * @return The path referring to the zip file.
      * @throws IOException If the file could not be created or the ONNX file could not be read.
      */

Json/src/main/java/org/tribuo/json/StripProvenance.java

@@ -16,18 +16,15 @@
 
 package org.tribuo.json;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
 import com.oracle.labs.mlrg.olcut.config.ConfigurationManager;
 import com.oracle.labs.mlrg.olcut.config.Option;
 import com.oracle.labs.mlrg.olcut.config.Options;
 import com.oracle.labs.mlrg.olcut.config.UsageException;
-import com.oracle.labs.mlrg.olcut.config.json.JsonProvenanceModule;
+import com.oracle.labs.mlrg.olcut.config.json.JsonProvenanceSerialization;
 import com.oracle.labs.mlrg.olcut.provenance.ListProvenance;
 import com.oracle.labs.mlrg.olcut.provenance.ObjectProvenance;
 import com.oracle.labs.mlrg.olcut.provenance.Provenance;
 import com.oracle.labs.mlrg.olcut.provenance.ProvenanceUtil;
-import com.oracle.labs.mlrg.olcut.provenance.io.ObjectMarshalledProvenance;
 import com.oracle.labs.mlrg.olcut.provenance.primitives.HashProvenance;
 import com.oracle.labs.mlrg.olcut.util.IOUtil;
 import com.oracle.labs.mlrg.olcut.util.LabsLogFormatter;
@@ -315,11 +312,8 @@ public static <T extends Output<T>> void main(String[] args) {
             ModelProvenance oldProvenance = input.getProvenance();
 
             logger.info("Marshalling provenance and creating JSON.");
-            List<ObjectMarshalledProvenance> list = ProvenanceUtil.marshalProvenance(oldProvenance);
-            ObjectMapper mapper = new ObjectMapper();
-            mapper.registerModule(new JsonProvenanceModule());
-            mapper.enable(SerializationFeature.INDENT_OUTPUT);
-            String jsonResult = mapper.writeValueAsString(list);
+            JsonProvenanceSerialization jsonProvenanceSerialization = new JsonProvenanceSerialization(true);
+            String jsonResult = jsonProvenanceSerialization.marshalAndSerialize(oldProvenance);
 
             logger.info("Hashing JSON file");
             MessageDigest digest = o.hashType.getDigest();
@@ -340,8 +334,7 @@ public static <T extends Output<T>> void main(String[] args) {
 
             ModelProvenance newProvenance = tuple.provenance;
             logger.info("Marshalling provenance and creating JSON.");
-            List<ObjectMarshalledProvenance> newList = ProvenanceUtil.marshalProvenance(newProvenance);
-            String newJsonResult = mapper.writeValueAsString(newList);
+            String newJsonResult = jsonProvenanceSerialization.marshalAndSerialize(newProvenance);
 
             logger.info("Old provenance = \n" + jsonResult);
             logger.info("New provenance = \n" + newJsonResult);