From 306a5d58caaa3ec4505abc2802701a11d27bf808 Mon Sep 17 00:00:00 2001 From: Heitor Date: Wed, 17 Apr 2024 14:46:51 +1200 Subject: [PATCH 1/9] Update adding SSL tab and fast evaluation (capymoa support) --- .../moa/classifiers/AbstractClassifier.java | 20 + .../main/java/moa/classifiers/Classifier.java | 10 +- .../classifiers/SemiSupervisedLearner.java | 16 +- .../ClusterAndLabelClassifier.java | 330 ++++++++++++ .../semisupervised/SSLTaskTester.java | 81 +++ .../SelfTrainingClassifier.java | 349 +++++++++++++ .../SelfTrainingIncrementalClassifier.java | 222 ++++++++ .../SelfTrainingWeightingClassifier.java | 115 +++++ .../AttributeSimilarityCalculator.java | 235 +++++++++ ...EuclideanDistanceSimilarityCalculator.java | 23 + .../GoodAll3SimilarityCalculator.java | 23 + .../IgnoreSimilarityCalculator.java | 20 + ...currenceFrequencySimilarityCalculator.java | 24 + .../LinSimilarityCalculator.java | 32 ++ ...currenceFrequencySimilarityCalculator.java | 26 + .../moa/clusterers/clustream/Clustream.java | 30 +- .../clusterers/clustream/ClustreamKernel.java | 113 ++-- .../evaluation/EfficientEvaluationLoops.java | 484 ++++++++++++------ .../LearningPerformanceEvaluator.java | 46 +- .../java/moa/gui/SemiSupervisedTabPanel.java | 29 ++ .../gui/SemiSupervisedTaskManagerPanel.java | 468 +++++++++++++++++ moa/src/main/java/moa/learners/Learner.java | 14 +- ...ateInterleavedTestThenTrainSSLDelayed.java | 351 +++++++++++++ .../moa/tasks/SemiSupervisedMainTask.java | 24 + moa/src/main/resources/moa/gui/GUI.props | 1 + 25 files changed, 2865 insertions(+), 221 deletions(-) create mode 100644 moa/src/main/java/moa/classifiers/semisupervised/ClusterAndLabelClassifier.java create mode 100644 moa/src/main/java/moa/classifiers/semisupervised/SSLTaskTester.java create mode 100644 moa/src/main/java/moa/classifiers/semisupervised/SelfTrainingClassifier.java create mode 100644 moa/src/main/java/moa/classifiers/semisupervised/SelfTrainingIncrementalClassifier.java create mode 100644 moa/src/main/java/moa/classifiers/semisupervised/SelfTrainingWeightingClassifier.java create mode 100644 moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/AttributeSimilarityCalculator.java create mode 100644 moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/EuclideanDistanceSimilarityCalculator.java create mode 100644 moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/GoodAll3SimilarityCalculator.java create mode 100644 moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/IgnoreSimilarityCalculator.java create mode 100644 moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/InverseOccurrenceFrequencySimilarityCalculator.java create mode 100644 moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/LinSimilarityCalculator.java create mode 100644 moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/OccurrenceFrequencySimilarityCalculator.java create mode 100644 moa/src/main/java/moa/gui/SemiSupervisedTabPanel.java create mode 100644 moa/src/main/java/moa/gui/SemiSupervisedTaskManagerPanel.java create mode 100644 moa/src/main/java/moa/tasks/EvaluateInterleavedTestThenTrainSSLDelayed.java create mode 100644 moa/src/main/java/moa/tasks/SemiSupervisedMainTask.java diff --git a/moa/src/main/java/moa/classifiers/AbstractClassifier.java b/moa/src/main/java/moa/classifiers/AbstractClassifier.java index f60467d0d..29350a48e 100644 --- a/moa/src/main/java/moa/classifiers/AbstractClassifier.java +++ b/moa/src/main/java/moa/classifiers/AbstractClassifier.java @@ -105,6 +105,26 @@ public double[] getVotesForInstance(Example example){ @Override public abstract double[] getVotesForInstance(Instance inst); + @Override + public double getConfidenceForPrediction(Instance inst, double prediction) { + double[] votes = this.getVotesForInstance(inst); + double predictionValue = votes[(int) prediction]; + + double sum = 0.0; + for (double vote : votes) + sum += vote; + + // Check if the sum is zero + if (sum == 0.0) + return 0.0; // Return 0 if sum is zero to avoid division by zero + return predictionValue / sum; + } + + @Override + public double getConfidenceForPrediction(Example example, double prediction) { + return getConfidenceForPrediction(example.getData(), prediction); + } + @Override public Prediction getPredictionForInstance(Example example){ return getPredictionForInstance(example.getData()); diff --git a/moa/src/main/java/moa/classifiers/Classifier.java b/moa/src/main/java/moa/classifiers/Classifier.java index 101d7fe3d..7a5acaa4e 100644 --- a/moa/src/main/java/moa/classifiers/Classifier.java +++ b/moa/src/main/java/moa/classifiers/Classifier.java @@ -15,7 +15,7 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . - * + * */ package moa.classifiers; @@ -76,7 +76,7 @@ public interface Classifier extends Learner> { * test instance in each class */ public double[] getVotesForInstance(Instance inst); - + /** * Sets the reference to the header of the data stream. The header of the * data stream is extended from WEKA @@ -86,7 +86,7 @@ public interface Classifier extends Learner> { * @param ih the reference to the data stream header */ //public void setModelContext(InstancesHeader ih); - + /** * Gets the reference to the header of the data stream. The header of the * data stream is extended from WEKA @@ -96,6 +96,8 @@ public interface Classifier extends Learner> { * @return the reference to the data stream header */ //public InstancesHeader getModelContext(); - + public Prediction getPredictionForInstance(Instance inst); + + public double getConfidenceForPrediction(Instance inst, double prediction); } diff --git a/moa/src/main/java/moa/classifiers/SemiSupervisedLearner.java b/moa/src/main/java/moa/classifiers/SemiSupervisedLearner.java index 05caec5a0..ba6954de0 100644 --- a/moa/src/main/java/moa/classifiers/SemiSupervisedLearner.java +++ b/moa/src/main/java/moa/classifiers/SemiSupervisedLearner.java @@ -19,12 +19,16 @@ */ package moa.classifiers; +import com.yahoo.labs.samoa.instances.Instance; +import moa.core.Example; +import moa.learners.Learner; + /** - * Learner interface for incremental semi supervised models. It is used only in the GUI Regression Tab. - * - * @author Richard Kirkby (rkirkby@cs.waikato.ac.nz) - * @version $Revision: 7 $ + * Updated learner interface for semi-supervised methods. */ -public interface SemiSupervisedLearner { - +public interface SemiSupervisedLearner extends Learner> { + // Returns the pseudo-label used. If no pseudo-label was used, then return -1. + int trainOnUnlabeledInstance(Instance instance); + + void addInitialWarmupTrainingInstances(); } diff --git a/moa/src/main/java/moa/classifiers/semisupervised/ClusterAndLabelClassifier.java b/moa/src/main/java/moa/classifiers/semisupervised/ClusterAndLabelClassifier.java new file mode 100644 index 000000000..95ad2bf54 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/semisupervised/ClusterAndLabelClassifier.java @@ -0,0 +1,330 @@ +package moa.classifiers.semisupervised; + +import com.github.javacliparser.FlagOption; +import com.github.javacliparser.IntOption; +import com.yahoo.labs.samoa.instances.Instance; +import moa.classifiers.AbstractClassifier; +import moa.classifiers.SemiSupervisedLearner; +import moa.cluster.Cluster; +import moa.cluster.Clustering; +import moa.clusterers.clustream.Clustream; +import moa.clusterers.clustream.ClustreamKernel; +import moa.core.*; +import moa.options.ClassOption; +import moa.tasks.TaskMonitor; + +import java.util.*; + +/** + * A simple semi-supervised classifier that serves as a baseline. + * The idea is to group the incoming data into micro-clusters, each of which + * is assigned a label. The micro-clusters will then be used for classification of unlabeled data. + */ +public class ClusterAndLabelClassifier extends AbstractClassifier + implements SemiSupervisedLearner { + + private static final long serialVersionUID = 1L; + + public ClassOption clustererOption = new ClassOption("clustream", 'c', + "Used to configure clustream", + Clustream.class, "Clustream"); + + /** Lets user decide if they want to use pseudo-labels */ + public FlagOption usePseudoLabelOption = new FlagOption("pseudoLabel", 'p', + "Using pseudo-label while training"); + + public FlagOption debugModeOption = new FlagOption("debugMode", 'e', + "Print information about the clusters on stdout"); + + /** Decides the labels based on k-nearest cluster, k defaults to 1 */ + public IntOption kNearestClusterOption = new IntOption("kNearestCluster", 'k', + "Issue predictions based on the majority vote from k-nearest cluster", 1); + + /** Number of nearest clusters used to issue prediction */ + private int k; + + private Clustream clustream; + + /** To train using pseudo-label or not */ + private boolean usePseudoLabel; + + /** Number of nearest clusters used to issue prediction */ +// private int k; + + // Statistics + protected long instancesSeen; + protected long instancesPseudoLabeled; + protected long instancesCorrectPseudoLabeled; + + @Override + public String getPurposeString() { + return "A basic semi-supervised learner"; + } + + @Override + public void prepareForUseImpl(TaskMonitor monitor, ObjectRepository repository) { + this.clustream = (Clustream) getPreparedClassOption(this.clustererOption); + this.clustream.prepareForUse(); + this.usePseudoLabel = usePseudoLabelOption.isSet(); + this.k = kNearestClusterOption.getValue(); + super.prepareForUseImpl(monitor, repository); + } + + @Override + public void resetLearningImpl() { + this.clustream.resetLearning(); + this.instancesSeen = 0; + this.instancesCorrectPseudoLabeled = 0; + this.instancesPseudoLabeled = 0; + } + + + @Override + public void trainOnInstanceImpl(Instance instance) { + ++this.instancesSeen; + Objects.requireNonNull(this.clustream, "Cluster must not be null!"); + if(this.clustream.getModelContext() == null) + this.clustream.setModelContext(this.getModelContext()); + this.clustream.trainOnInstance(instance); + } + + @Override + public int trainOnUnlabeledInstance(Instance instance) { + // Creates a copy of the instance to be pseudoLabeled + Instance unlabeledInstance = instance.copy(); + // In case the label is available for debugging purposes (i.e. checking the pseudoLabel accuracy), + // we want to save it, but then immediately remove the label to avoid it being used + int groundTruthClassLabel = -999; + if(! unlabeledInstance.classIsMissing()) { + groundTruthClassLabel = (int) unlabeledInstance.classValue(); + unlabeledInstance.setMissing(unlabeledInstance.classIndex()); + } + + int pseudoLabel = -1; + if (this.usePseudoLabel) { + ClustreamKernel closestCluster = getNearestClustreamKernel(this.clustream, unlabeledInstance, false); + pseudoLabel = (closestCluster != null ? Utils.maxIndex(closestCluster.classObserver) : -1); + + unlabeledInstance.setClassValue(pseudoLabel); + this.clustream.trainOnInstance(unlabeledInstance); + + if (pseudoLabel == groundTruthClassLabel) { + ++this.instancesCorrectPseudoLabeled; + } + ++this.instancesPseudoLabeled; + } + else { // Update the cluster without using the pseudoLabel + this.clustream.trainOnInstance(unlabeledInstance); + } + return pseudoLabel; + } + + @Override + public void addInitialWarmupTrainingInstances() { + } + + @Override + public double[] getVotesForInstance(Instance instance) { + Objects.requireNonNull(this.clustream, "Cluster must not be null!"); + // Creates a copy of the instance to be used in here (avoid changing the instance passed to this method) + Instance unlabeledInstance = instance.copy(); + + if(! unlabeledInstance.classIsMissing()) + unlabeledInstance.setMissing(unlabeledInstance.classIndex()); + + Clustering clustering = clustream.getMicroClusteringResult(); + + double[] votes = new double[unlabeledInstance.numClasses()]; + + if(clustering != null) { + if (k == 1) { + ClustreamKernel closestKernel = getNearestClustreamKernel(clustream, unlabeledInstance, false); + if (closestKernel != null) + votes = closestKernel.classObserver; + } + else { + votes = getVotesFromKClusters(this.findKNearestClusters(unlabeledInstance, this.k)); + } + } + return votes; + } + + /** + * Gets the predictions from K nearest clusters + * @param kClusters array of k nearest clusters + * @return the final predictions + */ + private double[] getVotesFromKClusters(ClustreamKernel[] kClusters) { + DoubleVector result = new DoubleVector(); + + for(ClustreamKernel microCluster : kClusters) { + if(microCluster == null) + continue; + + int maxIndex = Utils.maxIndex(microCluster.classObserver); + result.setValue(maxIndex, 1.0); + } + if(result.numValues() > 0) { + result.normalize(); + } + return result.getArrayRef(); + } + + /** + * Finds K nearest cluster from an instance + * @param instance the instance X + * @param k K closest clusters + * @return set of K closest clusters + */ + private ClustreamKernel[] findKNearestClusters(Instance instance, int k) { + Set sortedClusters = new TreeSet<>(new DistanceKernelComparator(instance)); + Clustering clustering = clustream.getMicroClusteringResult(); + + if (clustering == null || clustering.size() == 0) + return new ClustreamKernel[0]; + + // There should be a better way of doing this instead of creating a separate array list + ArrayList clusteringArray = new ArrayList<>(); + for(int i = 0 ; i < clustering.getClustering().size() ; ++i) + clusteringArray.add((ClustreamKernel) clustering.getClustering().get(i)); + + // Sort the clusters according to their distance to instance + sortedClusters.addAll(clusteringArray); + ClustreamKernel[] topK = new ClustreamKernel[k]; + // Keep only the topK clusters, i.e. the closest clusters to instance + Iterator it = sortedClusters.iterator(); + int i = 0; + while (it.hasNext() && i < k) + topK[i++] = it.next(); + + ////////////////////////////////// + if(this.debugModeOption.isSet()) + debugVotingScheme(clustering, instance, topK, true); + ////////////////////////////////// + + return topK; + } + + class DistanceKernelComparator implements Comparator { + + private Instance instance; + + public DistanceKernelComparator(Instance instance) { + this.instance = instance; + } + + @Override + public int compare(ClustreamKernel C1, ClustreamKernel C2) { + double distanceC1 = Clustream.distanceIgnoreNaN(C1.getCenter(), instance.toDoubleArray()); + double distanceC2 = Clustream.distanceIgnoreNaN(C2.getCenter(), instance.toDoubleArray()); + return Double.compare(distanceC1, distanceC2); + } + } + + private ClustreamKernel getNearestClustreamKernel(Clustream clustream, Instance instance, boolean includeClass) { + double minDistance = Double.MAX_VALUE; + ClustreamKernel closestCluster = null; + + List excluded = new ArrayList<>(); + if (!includeClass) + excluded.add(instance.classIndex()); + + Clustering clustering = clustream.getMicroClusteringResult(); + AutoExpandVector kernels = clustering.getClustering(); + + double[] arrayInstance = instance.toDoubleArray(); + + + for(int i = 0 ; i < kernels.size() ; ++i) { + double[] clusterCenter = kernels.get(i).getCenter(); + double distance = Clustream.distanceIgnoreNaN(arrayInstance, clusterCenter); + ////////////////////////////// + if(this.debugModeOption.isSet()) + debugClustreamMicroCluster((ClustreamKernel) kernels.get(i), clusterCenter, distance, true); + ////////////////////////////// + if(distance < minDistance) { + minDistance = distance; + closestCluster = (ClustreamKernel) kernels.get(i); + } + } + /////////////////////////// + if(this.debugModeOption.isSet()) + debugShowInstance(instance); + /////////////////////////// + + return closestCluster; + } + + @Override + protected Measurement[] getModelMeasurementsImpl() { + // instances seen * the number of ensemble members + return new Measurement[]{ + new Measurement("#pseudo-labeled", this.instancesPseudoLabeled), + new Measurement("#correct pseudo-labeled", this.instancesCorrectPseudoLabeled), + new Measurement("accuracy pseudo-labeled", this.instancesCorrectPseudoLabeled / (double) this.instancesPseudoLabeled * 100) + }; + } + + @Override + public void getModelDescription(StringBuilder out, int indent) { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + public boolean isRandomizable() { + return false; + } + + //////////////////////////////////////////////////////////////////////////////////////////////// + /////////////////////////////////// DEBUG METHODS ////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////////////////////// + + private void debugShowInstance(Instance instance) { + System.out.print("Instance: ["); + for(int i = 0 ; i < instance.numAttributes() ; ++i) { + System.out.print(instance.value(i) + " "); + } + System.out.println("]"); + } + + private void debugClustreamMicroCluster(ClustreamKernel cluster, double[] clusterCenter, double distance, boolean showMicroClusterValues) { + System.out.print(" MicroCluster: " + cluster.getId()); + if(showMicroClusterValues) { + System.out.print(" ["); + for (int j = 0; j < clusterCenter.length; ++j) { + System.out.print(String.format("%.4f ", clusterCenter[j]) + " "); + } + System.out.print("]"); + } + System.out.print(" distance to instance: " + String.format("%.4f ",distance) + " classObserver: [ "); + + for(int g = 0 ; g < cluster.classObserver.length ; ++g) { + System.out.print(cluster.classObserver[g] + " "); + } + System.out.print("] maxIndex (vote): " + Utils.maxIndex(cluster.classObserver)); + System.out.println(); + } + + private void debugVotingScheme(Clustering clustering, Instance instance, ClustreamKernel[] topK, boolean showAllClusters) { + System.out.println("[DEBUG] Voting Scheme: "); + AutoExpandVector kernels = clustering.getClustering(); + + double[] arrayInstance = instance.toDoubleArray(); + + System.out.println(" TopK: "); + for(int z = 0 ; z < topK.length ; ++z) { + double[] clusterCenter = topK[z].getCenter(); + double distance = Clustream.distanceIgnoreNaN(arrayInstance, clusterCenter); + debugClustreamMicroCluster(topK[z], clusterCenter, distance, true); + } + + if(showAllClusters) { + System.out.println(" All microclusters: "); + for (int x = 0; x < kernels.size(); ++x) { + double[] clusterCenter = kernels.get(x).getCenter(); + double distance = Clustream.distanceIgnoreNaN(arrayInstance, clusterCenter); + debugClustreamMicroCluster((ClustreamKernel) kernels.get(x), clusterCenter, distance, true); + } + } + } +} diff --git a/moa/src/main/java/moa/classifiers/semisupervised/SSLTaskTester.java b/moa/src/main/java/moa/classifiers/semisupervised/SSLTaskTester.java new file mode 100644 index 000000000..b878f31f2 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/semisupervised/SSLTaskTester.java @@ -0,0 +1,81 @@ +package moa.classifiers.semisupervised; + +import com.yahoo.labs.samoa.instances.Instance; +import moa.classifiers.AbstractClassifier; +import moa.classifiers.SemiSupervisedLearner; +import moa.core.Measurement; +import moa.core.ObjectRepository; +import moa.tasks.TaskMonitor; + +/*** + * This class shall be removed later. Just used to verify the EvaluateInterleavedTestThenTrainSSLDelayed + * works as expected. + */ +public class SSLTaskTester extends AbstractClassifier implements SemiSupervisedLearner{ + + protected long instancesWarmupCounter; + protected long instancesLabeledCounter; + protected long instancesUnlabeledCounter; + protected long instancesTestCounter; + + @Override + public void prepareForUseImpl(TaskMonitor monitor, ObjectRepository repository) { + super.prepareForUseImpl(monitor, repository); + + this.instancesTestCounter = 0; + this.instancesUnlabeledCounter = 0; + this.instancesLabeledCounter = 0; + } + + @Override + public boolean isRandomizable() { + return false; + } + + @Override + public double[] getVotesForInstance(Instance inst) { + // TODO Auto-generated method stub + ++this.instancesTestCounter; + double[] dummy = new double[inst.numClasses()]; + return dummy; + } + + @Override + public void resetLearningImpl() { + // TODO Auto-generated method stub + } + + @Override + public void addInitialWarmupTrainingInstances() { + ++this.instancesWarmupCounter; + } + + @Override + public void trainOnInstanceImpl(Instance inst) { + // TODO Auto-generated method stub + ++this.instancesLabeledCounter; + } + + @Override + public int trainOnUnlabeledInstance(Instance instance) { + ++this.instancesUnlabeledCounter; + return -1; + } + + @Override + protected Measurement[] getModelMeasurementsImpl() { + return new Measurement[]{ + new Measurement("#labeled", this.instancesLabeledCounter), + new Measurement("#unlabeled", this.instancesUnlabeledCounter), + new Measurement("#warmup", this.instancesWarmupCounter) +// new Measurement("accuracy supervised learner", this.evaluatorSupervisedDebug.getPerformanceMeasurements()[1].getValue()) + }; + } + + @Override + public void getModelDescription(StringBuilder out, int indent) { + // TODO Auto-generated method stub + + } + +} diff --git a/moa/src/main/java/moa/classifiers/semisupervised/SelfTrainingClassifier.java b/moa/src/main/java/moa/classifiers/semisupervised/SelfTrainingClassifier.java new file mode 100644 index 000000000..851aace57 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/semisupervised/SelfTrainingClassifier.java @@ -0,0 +1,349 @@ +package moa.classifiers.semisupervised; + +import com.github.javacliparser.FloatOption; +import com.github.javacliparser.IntOption; +import com.github.javacliparser.MultiChoiceOption; +import com.yahoo.labs.samoa.instances.Instance; +import moa.classifiers.AbstractClassifier; +import moa.classifiers.Classifier; +import moa.classifiers.SemiSupervisedLearner; +import moa.core.Measurement; +import moa.core.ObjectRepository; +import moa.core.Utils; +import moa.options.ClassOption; +import moa.tasks.TaskMonitor; + +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.List; + +/** + * Self-training classifier: it is trained with a limited number of labeled data at first, + * then it predicts the labels of unlabeled data, the most confident predictions are used + * for training in the next iteration. + */ +public class SelfTrainingClassifier extends AbstractClassifier implements SemiSupervisedLearner { + + private static final long serialVersionUID = 1L; + + /* ------------------- + * GUI options + * -------------------*/ + public ClassOption learnerOption = new ClassOption("learner", 'l', + "Any learner to be self-trained", AbstractClassifier.class, + "moa.classifiers.trees.HoeffdingTree"); + + public IntOption batchSizeOption = new IntOption("batchSize", 'b', + "Size of one batch to self-train", + 1000, 1, Integer.MAX_VALUE); + + public MultiChoiceOption thresholdChoiceOption = new MultiChoiceOption("thresholdValue", 't', + "Ways to define the confidence threshold", + new String[] { "Fixed", "AdaptiveWindowing", "AdaptiveVariance" }, + new String[] { + "The threshold is input once and remains unchanged", + "The threshold is updated every h-interval of time", + "The threshold is updated if the confidence score drifts off from the average" + }, 0); + + public FloatOption thresholdOption = new FloatOption("confidenceThreshold", 'c', + "Threshold to evaluate the confidence of a prediction", + 0.7, 0.0, Double.MAX_VALUE); + + public IntOption horizonOption = new IntOption("horizon", 'h', + "The interval of time to update the threshold", 1000); + + public FloatOption ratioThresholdOption = new FloatOption("ratioThreshold", 'r', + "How large should the threshold be wrt to the average confidence score", + 0.8, 0.0, Double.MAX_VALUE); + + public MultiChoiceOption confidenceOption = new MultiChoiceOption("confidenceComputation", + 's', "Choose the method to estimate the prediction uncertainty", + new String[]{ "DistanceMeasure", "FromLearner" }, + new String[]{ "Confidence score from pair-wise distance with the ground truth", + "Confidence score estimated by the learner itself" }, 1); + + /* ------------------- + * Attributes + * -------------------*/ + + /** A learner to be self-trained */ + private Classifier learner; + + /** The size of one batch */ + private int batchSize; + + /** The confidence threshold to decide which predictions to include in the next training batch */ + private double threshold; + + /** Contains the unlabeled instances */ + private List U; + + /** Contains the labeled instances */ + private List L; + + /** Contains the predictions of one batch's training */ +// private List Uhat; + + /** Contains the most confident prediction */ +// private List mostConfident; + + private int horizon; + private int t; + private double ratio; + private double LS; + private double SS; + private double N; + private double lastConfidenceScore; + + // Statistics + protected long instancesSeen; + protected long instancesPseudoLabeled; + protected long instancesCorrectPseudoLabeled; + + + @Override + public String getPurposeString() { return "A self-training classifier"; } + + @Override + public void prepareForUseImpl(TaskMonitor monitor, ObjectRepository repository) { + this.learner = (Classifier) getPreparedClassOption(learnerOption); + this.batchSize = batchSizeOption.getValue(); + this.threshold = thresholdOption.getValue(); + this.ratio = ratioThresholdOption.getValue(); + this.horizon = horizonOption.getValue(); + LS = SS = N = t = 0; + allocateBatch(); + super.prepareForUseImpl(monitor, repository); + } + + @Override + public double[] getVotesForInstance(Instance inst) { + return learner.getVotesForInstance(inst); + } + + @Override + public void resetLearningImpl() { + this.learner.resetLearning(); + lastConfidenceScore = LS = SS = N = t = 0; + allocateBatch(); + + this.instancesSeen = 0; + this.instancesCorrectPseudoLabeled = 0; + this.instancesPseudoLabeled = 0; + } + + @Override + public void trainOnInstanceImpl(Instance inst) { + this.instancesSeen++; + updateThreshold(); + t++; + + L.add(inst); + learner.trainOnInstance(inst); + + + /* if batch B is full, launch the self-training process */ + if (isBatchFull()) { + trainOnUnlabeledBatch(); + } + } + + private void trainOnUnlabeledBatch() { + List> Uhat = predictOnBatch(U); + List> mostConfident = null; + + // chose the method to estimate prediction uncertainty +// if (confidenceOption.getChosenIndex() == 0) +// mostConfident = getMostConfidentDistanceBased(Uhat); +// else + mostConfident = getMostConfidentFromLearner(Uhat); + // train from the most confident examples + for(AbstractMap.SimpleEntry x : mostConfident) { + learner.trainOnInstance(x.getKey()); + if(x.getKey().classValue() == x.getValue()) + ++this.instancesCorrectPseudoLabeled; + ++this.instancesPseudoLabeled; + } + cleanBatch(); + } + + @Override + public void addInitialWarmupTrainingInstances() { + // TODO: add counter, but this may not be necessary for this class + } + + // TODO: Verify if we need to do something else. + @Override + public int trainOnUnlabeledInstance(Instance instance) { + this.instancesSeen++; + U.add(instance); + + if (isBatchFull()) { + trainOnUnlabeledBatch(); + } +// this.trainOnInstanceImpl(instance); + return -1; + } + + private void updateThreshold() { + if (thresholdChoiceOption.getChosenIndex() == 1) updateThresholdWindowing(); + if (thresholdChoiceOption.getChosenIndex() == 2) updateThresholdVariance(); + } + + /** + * Dynamically updates the confidence threshold at the end of each labeledInstancesBuffer horizon + */ + private void updateThresholdWindowing() { + if (t % horizon == 0) { + if (N == 0 || LS == 0 || SS == 0) return; + threshold = (LS / N) * ratio; + t = 0; + } + } + + /** + * Dynamically updates the confidence threshold: + * adapt the threshold if the last confidence score falls out of z-index = 1 zone + */ + private void updateThresholdVariance() { + // TODO update right when it detects a drift, or to wait until H drifts have happened? + if (N == 0 || LS == 0 || SS == 0) return; + double variance = (SS - LS * LS / N) / (N - 1); + double mean = LS / N; + double zscore = (lastConfidenceScore - mean) / variance; + if (Math.abs(zscore) > 1.0) { + threshold = mean * ratio; + } + } + + /** + * Gives prediction for each instance in a given batch. + * @param batch the batch containing unlabeled instances + * @return result the result to save the prediction in + */ + private List> predictOnBatch(List batch) { + List> batchWithPredictions = new ArrayList<>(); + + + for (Instance instance : batch) { + Instance copy = instance.copy(); // use copy because we do not want to modify the original data + double classValue = -1.0; + if(!instance.classIsMissing()) // if it is not missing, assume this is a debug execution and store it for checking pseudo-labelling accuracy. + classValue = instance.classValue(); + + copy.setClassValue(Utils.maxIndex(learner.getVotesForInstance(copy))); + batchWithPredictions.add(new AbstractMap.SimpleEntry (copy, classValue)); + } + + return batchWithPredictions; + } + + /** + * Gets the most confident predictions + * @param batch batch of instances to give prediction to + * @return mostConfident instances that are more confidence than a threshold + */ + private List> getMostConfidentFromLearner(List> batch) { + List> mostConfident = new ArrayList<>(); + for (AbstractMap.SimpleEntry x : batch) { + double[] votes = learner.getVotesForInstance(x.getKey()); + if (votes[Utils.maxIndex(votes)] >= threshold) { + mostConfident.add(x); + } + } + return mostConfident; + } + + /** + * Gets the most confident predictions that exceed the indicated threshold + * @param batch the batch containing the predictions + * @return mostConfident the result containing the most confident prediction from the given batch + */ +// private List> getMostConfidentDistanceBased(List> batch) { +// /* +// * Use distance measure to estimate the confidence of a prediction +// * +// * for each instance X in the batch: +// * for each instance XL in the labeled data: (ground-truth) +// * if X.label == XL.label: (only consider instances sharing the same label) +// * confidence[X] += distance(X, XL) +// * confidence[X] = confidence[X] / |L| (taking the average) +// */ +// List> mostConfident = new ArrayList<>(); +// +// double[] confidences = new double[batch.size()]; +// double conf; +// int i = 0; +// for (AbstractMap.SimpleEntry X : batch) { +// conf = 0; +// for (Instance XL : this.L) { +// if (XL.classValue() == X.getKey().classValue()) { +// conf += Clusterer.distance(XL.toDoubleArray(), X.getKey().toDoubleArray()) / this.L.size(); +// } +// } +// conf = (1.0 / conf > 1.0 ? 1.0 : 1 / conf); // reverse so the distance becomes the confidence +// confidences[i++] = conf; +// // accumulate the statistics +// LS += conf; +// SS += conf * conf; +// N++; +// } +// +// for (double confidence : confidences) lastConfidenceScore += confidence / confidences.length; +// +// /* The confidences are computed using the distance measures, +// * so naturally, the lower the score, the more certain the prediction is. +// * Here we simply retrieve the instances whose confidence score are below a threshold */ +// for (int j = 0; j < confidences.length; j++) { +// if (confidences[j] >= threshold) { +// mostConfident.add(batch.get(j)); +// } +// } +// +// return mostConfident; +// } + + /** + * Checks whether the batch is full + * @return true if the batch is full, false otherwise + */ + private boolean isBatchFull() { + return U.size() + L.size() >= batchSize; + } + + /** Cleans the batch (and its associated variables) */ + private void cleanBatch() { + L.clear(); + U.clear(); +// mostConfident.clear(); + } + + /** Allocates memory to the batch */ + private void allocateBatch() { + this.U = new ArrayList<>(); + this.L = new ArrayList<>(); +// this.mostConfident = new ArrayList<>(); + } + + + @Override + protected Measurement[] getModelMeasurementsImpl() { + // instances seen * the number of ensemble members + return new Measurement[]{ + new Measurement("#pseudo-labeled", this.instancesPseudoLabeled), + new Measurement("#correct pseudo-labeled", this.instancesCorrectPseudoLabeled), + new Measurement("accuracy pseudo-labeled", this.instancesCorrectPseudoLabeled / (double) this.instancesPseudoLabeled * 100) + }; + } + + @Override + public void getModelDescription(StringBuilder out, int indent) { + + } + + @Override + public boolean isRandomizable() { + return false; + } +} diff --git a/moa/src/main/java/moa/classifiers/semisupervised/SelfTrainingIncrementalClassifier.java b/moa/src/main/java/moa/classifiers/semisupervised/SelfTrainingIncrementalClassifier.java new file mode 100644 index 000000000..d9f60a977 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/semisupervised/SelfTrainingIncrementalClassifier.java @@ -0,0 +1,222 @@ +package moa.classifiers.semisupervised; + +import com.github.javacliparser.FloatOption; +import com.github.javacliparser.IntOption; +import com.github.javacliparser.MultiChoiceOption; +import com.yahoo.labs.samoa.instances.Instance; +import moa.classifiers.AbstractClassifier; +import moa.classifiers.Classifier; +import moa.classifiers.SemiSupervisedLearner; +import moa.core.Measurement; +import moa.core.ObjectRepository; +import moa.core.Utils; +import moa.options.ClassOption; +import moa.tasks.TaskMonitor; + +/** + * Self-training classifier: Incremental version. + * Instead of using a batch, the model will be update with every instance that arrives. + */ +public class SelfTrainingIncrementalClassifier extends AbstractClassifier implements SemiSupervisedLearner { + + private static final long serialVersionUID = 1L; + + public ClassOption learnerOption = new ClassOption("learner", 'l', + "Any learner to be self-trained", AbstractClassifier.class, + "moa.classifiers.trees.HoeffdingTree"); + + public MultiChoiceOption thresholdChoiceOption = new MultiChoiceOption("thresholdValue", 't', + "Ways to define the confidence threshold", + new String[] { "Fixed", "AdaptiveWindowing", "AdaptiveVariance" }, + new String[] { + "The threshold is input once and remains unchanged", + "The threshold is updated every h-interval of time", + "The threshold is updated if the confidence score drifts off from the average" + }, 0); + + public FloatOption thresholdOption = new FloatOption("confidenceThreshold", 'c', + "Threshold to evaluate the confidence of a prediction", 0.9, 0.0, 1.0); + + public IntOption horizonOption = new IntOption("horizon", 'h', + "The interval of time to update the threshold", 1000); + + public FloatOption ratioThresholdOption = new FloatOption("ratioThreshold", 'r', + "How large should the threshold be wrt to the average confidence score", + 0.95, 0.0, Double.MAX_VALUE); + + /* ------------------- + * Attributes + * -------------------*/ + /** A learner to be self-trained */ + private Classifier learner; + + /** The confidence threshold to decide which predictions to include in the next training batch */ + private double threshold; + + /** Whether the threshold is to be adaptive or fixed*/ + private boolean adaptiveThreshold; + + /** Interval of time to update the threshold */ + private int horizon; + + /** Keep track of time */ + private int t; + + /** Ratio of the threshold wrt the average confidence score*/ + private double ratio; + + // statistics needed to update the confidence threshold + private double LS; + private double SS; + private double N; + private double lastConfidenceScore; + + // Statistics + protected long instancesSeen; + protected long instancesPseudoLabeled; + protected long instancesCorrectPseudoLabeled; + + @Override + public void prepareForUseImpl(TaskMonitor monitor, ObjectRepository repository) { + this.learner = (Classifier) getPreparedClassOption(learnerOption); + this.threshold = thresholdOption.getValue(); + this.horizon = horizonOption.getValue(); + this.ratio = ratioThresholdOption.getValue(); + super.prepareForUseImpl(monitor, repository); + } + + @Override + public String getPurposeString() { + return "A self-training classifier that trains at every instance (not using a batch)"; + } + + @Override + public double[] getVotesForInstance(Instance inst) { + return learner.getVotesForInstance(inst); + } + + @Override + public void resetLearningImpl() { + LS = SS = N = lastConfidenceScore = 0; + this.instancesSeen = 0; + this.instancesCorrectPseudoLabeled = 0; + this.instancesPseudoLabeled = 0; + } + + @Override + public void trainOnInstanceImpl(Instance inst) { + /* + * update the threshold + * + * if X is labeled: + * L.train(X) + * else: + * X_hat <- L.predict_probab(X) + * if X_hat.highest_proba > threshold: + * L.train(X_hat) + */ + + updateThreshold(); + + ++this.instancesSeen; + + if (/*!inst.classIsMasked() &&*/ !inst.classIsMissing()) { + learner.trainOnInstance(inst); + } else { + double pseudoLabel = getPrediction(inst); + double confidenceScore = learner.getConfidenceForPrediction(inst, pseudoLabel); + if (confidenceScore >= threshold) { + Instance instCopy = inst.copy(); + instCopy.setClassValue(pseudoLabel); + learner.trainOnInstance(instCopy); + } + // accumulate the statistics to update the adaptive threshold + LS += confidenceScore; + SS += confidenceScore * confidenceScore; + N++; + lastConfidenceScore = confidenceScore; + +// if(pseudoLabel == inst.maskedClassValue()) { +// ++this.instancesCorrectPseudoLabeled; +// } + ++this.instancesPseudoLabeled; + } + + t++; + } + + private void updateThreshold() { + if (thresholdChoiceOption.getChosenIndex() == 1) updateThresholdWindowing(); + if (thresholdChoiceOption.getChosenIndex() == 2) updateThresholdVariance(); + } + + @Override + public void addInitialWarmupTrainingInstances() { + // TODO: add counter, but this may not be necessary for this class + } + + // TODO: Verify if we need to do something else. + @Override + public int trainOnUnlabeledInstance(Instance instance) { + this.trainOnInstanceImpl(instance); + return -1; + } + + /** + * Updates the threshold after each labeledInstancesBuffer horizon + */ + private void updateThresholdWindowing() { + if (t % horizon == 0) { + if (N == 0 || LS == 0 || SS == 0) return; + threshold = (LS / N) * ratio; + threshold = (Math.min(threshold, 1.0)); + // N = LS = SS = 0; // to reset or not? + t = 0; + } + } + + /** + * Update the thresholds based on the variance: + * if the z-score of the last confidence score wrt the mean is more than 1.0, + * update the confidence threshold + */ + private void updateThresholdVariance() { + if (N == 0 || LS == 0 || SS == 0) return; + double variance = (SS - LS * LS / N) / (N - 1); + double mean = LS / N; + double zscore = (lastConfidenceScore - mean) / variance; + if (Math.abs(zscore) > 1.0) { + threshold = mean * ratio; + threshold = (Math.min(threshold, 1.0)); + } + } + + /** + * Gets the prediction from an instance (a shortcut to pass getVotesForInstance) + * @param inst the instance + * @return the most likely prediction (the label with the highest probability in getVotesForInstance) + */ + private double getPrediction(Instance inst) { + return Utils.maxIndex(this.getVotesForInstance(inst)); + } + + @Override + protected Measurement[] getModelMeasurementsImpl() { + // instances seen * the number of ensemble members + return new Measurement[]{ + new Measurement("#pseudo-labeled", -1), // this.instancesPseudoLabeled), + new Measurement("#correct pseudo-labeled", -1), //this.instancesCorrectPseudoLabeled), + new Measurement("accuracy pseudo-labeled", -1) //this.instancesCorrectPseudoLabeled / (double) this.instancesPseudoLabeled * 100) + }; + } + + @Override + public void getModelDescription(StringBuilder out, int indent) { + + } + + @Override + public boolean isRandomizable() { + return false; + } +} diff --git a/moa/src/main/java/moa/classifiers/semisupervised/SelfTrainingWeightingClassifier.java b/moa/src/main/java/moa/classifiers/semisupervised/SelfTrainingWeightingClassifier.java new file mode 100644 index 000000000..39520ea03 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/semisupervised/SelfTrainingWeightingClassifier.java @@ -0,0 +1,115 @@ +package moa.classifiers.semisupervised; + +import com.github.javacliparser.FlagOption; +import com.yahoo.labs.samoa.instances.Instance; +import moa.classifiers.AbstractClassifier; +import moa.classifiers.Classifier; +import moa.classifiers.SemiSupervisedLearner; +import moa.core.Measurement; +import moa.core.ObjectRepository; +import moa.core.Utils; +import moa.options.ClassOption; +import moa.tasks.TaskMonitor; + +/** + * Variance of Self-training: all instances are used to self-train the learner, but each has a weight, depending + * on the confidence of their prediction + */ +public class SelfTrainingWeightingClassifier extends AbstractClassifier implements SemiSupervisedLearner { + + + @Override + public String getPurposeString() { + return "Self-training classifier that weights instances by confidence score (threshold not used)"; + } + + public ClassOption learnerOption = new ClassOption("learner", 'l', + "Any learner to be self-trained", AbstractClassifier.class, + "moa.classifiers.trees.HoeffdingTree"); + + public FlagOption equalWeightOption = new FlagOption("equalWeight", 'w', + "Assigns to all instances a weight equal to 1"); + + /** If set to True, all instances have weight 1; otherwise, the weights are based on the confidence score */ + private boolean equalWeight; + + /** The learner to be self-trained */ + private Classifier learner; + + // Statistics + protected long instancesSeen; + protected long instancesPseudoLabeled; + protected long instancesCorrectPseudoLabeled; + + @Override + public void prepareForUseImpl(TaskMonitor monitor, ObjectRepository repository) { + this.learner = (Classifier) getPreparedClassOption(learnerOption); + this.equalWeight = equalWeightOption.isSet(); + super.prepareForUseImpl(monitor, repository); + } + + @Override + public double[] getVotesForInstance(Instance inst) { + return learner.getVotesForInstance(inst); + } + + @Override + public void resetLearningImpl() { + this.learner.resetLearning(); + this.instancesSeen = 0; + this.instancesCorrectPseudoLabeled = 0; + this.instancesPseudoLabeled = 0; + } + + @Override + public void trainOnInstanceImpl(Instance inst) { + ++this.instancesSeen; + + if (/*!inst.classIsMasked() &&*/ !inst.classIsMissing()) { + learner.trainOnInstance(inst); + } else { + Instance instCopy = inst.copy(); + int pseudoLabel = Utils.maxIndex(learner.getVotesForInstance(instCopy)); + instCopy.setClassValue(pseudoLabel); + if (!equalWeight) instCopy.setWeight(learner.getConfidenceForPrediction(instCopy, pseudoLabel)); + learner.trainOnInstance(instCopy); + +// if(pseudoLabel == inst.maskedClassValue()) { +// ++this.instancesCorrectPseudoLabeled; +// } + ++this.instancesPseudoLabeled; + } + } + + @Override + public void addInitialWarmupTrainingInstances() { + // TODO: add counter, but this may not be necessary for this class + } + + // TODO: Verify if we need to do something else. + @Override + public int trainOnUnlabeledInstance(Instance instance) { + this.trainOnInstanceImpl(instance); + return -1; + } + + @Override + protected Measurement[] getModelMeasurementsImpl() { + // instances seen * the number of ensemble members + return new Measurement[]{ + new Measurement("#pseudo-labeled", -1), // this.instancesPseudoLabeled), + new Measurement("#correct pseudo-labeled", -1), //this.instancesCorrectPseudoLabeled), + new Measurement("accuracy pseudo-labeled", -1) //this.instancesCorrectPseudoLabeled / (double) this.instancesPseudoLabeled * 100) + }; + } + + + + @Override + public void getModelDescription(StringBuilder out, int indent) {} + + @Override + public boolean isRandomizable() { + return false; + } +} diff --git a/moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/AttributeSimilarityCalculator.java b/moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/AttributeSimilarityCalculator.java new file mode 100644 index 000000000..43be4f75f --- /dev/null +++ b/moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/AttributeSimilarityCalculator.java @@ -0,0 +1,235 @@ +package moa.classifiers.semisupervised.attributeSimilarity; + +import com.yahoo.labs.samoa.instances.Attribute; +import com.yahoo.labs.samoa.instances.Instance; +import moa.core.DoubleVector; + +import java.util.HashMap; +import java.util.Map; + +/** + * An observer that collects statistics for similarity computation of categorical attributes. + * This observer observes the categorical attributes of one dataset. + */ +public abstract class AttributeSimilarityCalculator { + + /** + *

Collection of statistics of one attribute, including:

+ *
    + *
  • ID: index of the attribute
    • + *
  • f_k: frequency of a value of an attribute
    • + *
+ */ + class AttributeStatistics extends Attribute { + /** ID of the attribute */ + private int id; + + /** Frequency of the values of the attribute */ + private DoubleVector fk; + + /** The decorated attribute */ + private Attribute attribute; + + /** + * Creates a new collection of statistics of an attribute + * @param id ID of the attribute + */ + AttributeStatistics(int id) { + this.id = id; + this.fk = new DoubleVector(); + } + + AttributeStatistics(Attribute attr, int id) { + this.attribute = attr; + this.id = id; + this.fk = new DoubleVector(); + } + + /** Gets the ID of the attribute */ + int getId() { return this.id; } + + /** Gets the decorated attribute */ + Attribute getAttribute() { return this.attribute; } + + /** + * Gets f_k(x) i.e. the number of times x is a value of the attribute of ID k + * @param value the attribute value + * @return the number of times x is a value of the attribute of ID k + */ + int getFrequencyOfValue(int value) { + return (int) this.fk.getValue(value); + } + + /** + * Updates the frequency of a value + * @param value the value X_k + * @param frequency the frequency + */ + void updateFrequencyOfValue(int value, int frequency) { + this.fk.addToValue((int)value, frequency); + } + } + + /** Size of the dataset (number of instances) */ + protected int N; + + /** Dimension of the dataset (number of attributes) */ + protected int d; + + /** Storing the statistics of each attribute */ + //private AttributeStatistics[] attrStats; + protected Map attributeStats; + + /** A small value to avoid division by 0 */ + protected static double SMALL_VALUE = 1e-5; + + /** Creates a new observer */ + public AttributeSimilarityCalculator() { + this.N = this.d = 0; + this.attributeStats = new HashMap<>(); + } + + /** + * Creates a new observer with a predefined number of attributes + * @param d number of attributes + */ + public AttributeSimilarityCalculator(int d) { + this.d = d; + this.attributeStats = new HashMap<>(); + } + + /** + * Returns the size of the dataset + * @return the size of the dataset (number of instances) + */ + public int getSize() { return this.N; } + + /** + * Increases the number of instances seen so far + * @param amount the amount to increase + */ + public void increaseSize(int amount) { this.N += amount; } + + /** + * Returns the dimension size + * @return the dimension size (number of attributes) + */ + public int getDimension() { return this.d; } + + /** + * Specifies the dimension of the dataset + * @param d the dimension + */ + public void setDimension(int d) { this.d = d; } + + /** + * Returns the number of values taken by A_k collected online i.e. n_k + * @param attr the attribute A_k + * @return number of values taken by A_k (n_k) + */ + public int getNumberOfAttributes(Attribute attr) { + if (attributeStats.containsKey(attr)) return attributeStats.get(attr).numValues(); + return 0; + } + + /** + * Gets the frequency of value x of attribute A_k i.e. f_k(x) + * @param attr the attribute + * @param value the value + * @return the number of times x occurs as value of attribute A_k; 0 if attribute k has not been observed so far + */ + public double getFrequencyOfValueByAttribute(Attribute attr, int value) { + if (attributeStats.containsKey(attr)) return attributeStats.get(attr).getFrequencyOfValue(value); + return 0; + } + + /** + * Gets the sample probability of attribute A_k to take the value x in the dataset + * i.e. p_k(x) = f_k(x) / N + * @param attr the attribute A_k + * @param value the value x + * @return the sample probability p_k(x) + */ + public double getSampleProbabilityOfAttributeByValue(Attribute attr, int value) { + return this.getFrequencyOfValueByAttribute(attr, value) / this.N; + } + + /** + * Gets another probability estimate of attribute A_k to take the value x in the dataset + * i.e. p_k^2 = f_k(x) * [ f_k(x) - 1 ] / [ N * (N - 1) ] + * @param attr the attribute A_k + * @param value the value x + * @return the sample probability p_k^2(x) + */ + public double getProbabilityEstimateOfAttributeByValue(Attribute attr, int value) { + double fX = getFrequencyOfValueByAttribute(attr, value); + if (N == 1) return 0; + return (fX * (fX - 1)) / (N * (N - 1)); + } + + /** + * Updates the statistics of an attribute A_k, e.g. frequency of the value (f_k) + * @param id ID of the attribute A_k + * @param attr the attribute A_k + * @param value the value of A_k + */ + public void updateAttributeStatistics(int id, Attribute attr, int value) { + if (!attributeStats.containsKey(attr)) { + AttributeStatistics stat = new AttributeStatistics(attr, id); + stat.updateFrequencyOfValue(value, 1); + attributeStats.put(attr, stat); + } else { +// System.out.println("attributeStats.get(attr).updateFrequencyOfValue(value, 1);" + attr + " " + value); + if(value >= 0) + attributeStats.get(attr).updateFrequencyOfValue(value, 1); + else + System.out.println("if(value < 0)"); + } + } + + /** + * Computes the similarity of categorical attributes of two instances X and Y, denoted S(X, Y). + * S(X, Y) = Sum of [w_k * S_k(X_k, Y_k)] for k from 1 to d, + * X_k and Y_k are from A_k (attribute k of the dataset). + * + * Note that X and Y must come from the same dataset, contain the same set of attributes, + * and numeric attributes will not be taken into account. + * @param X instance X + * @param Y instance Y + * @return the similarity of categorical attributes of X and Y + */ + public double computeSimilarityOfInstance(Instance X, Instance Y) { + // for k from 1 to d + double S = 0; + for (int i = 0; i < X.numAttributes(); i++) { + // sanity check + if (!X.attribute(i).equals(Y.attribute(i))) continue; // if X and Y's attributes are not aligned + Attribute Ak = X.attribute(i); + if (Ak.isNumeric() || !attributeStats.containsKey(Ak) || i == X.classIndex()) continue; + // computation + double wk = computeWeightOfAttribute(Ak, X, Y); + double Sk = computePerAttributeSimilarity(Ak, (int)X.value(Ak), (int)Y.value(Ak)); + S += (wk * Sk); + } + return S; + } + + /** + * Computes the per-attribute similarity S_k(X_k, Y_k) between two value X_k and Y_k + * of the attribute A_k. X_k and Y_k must be from A_k. + * + * To be overriden by subclasses. + * @param attr the attribute A_k + * @param X_k the value of X_k + * @param Y_k the value of Y_k + * @return the per-attribute similarity S_k(X_k, Y_k) + */ + public abstract double computePerAttributeSimilarity(Attribute attr, double X_k, double Y_k); + + /** + * Computes the weight w_k of an attribute A_k. To be overriden by subclasses. + * @param attr the attribute A_k + * @return the weight w_k of A_k + */ + public abstract double computeWeightOfAttribute(Attribute attr, Instance X, Instance Y); +} diff --git a/moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/EuclideanDistanceSimilarityCalculator.java b/moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/EuclideanDistanceSimilarityCalculator.java new file mode 100644 index 000000000..3d08cf4d6 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/EuclideanDistanceSimilarityCalculator.java @@ -0,0 +1,23 @@ +package moa.classifiers.semisupervised.attributeSimilarity; + +import com.yahoo.labs.samoa.instances.Attribute; +import com.yahoo.labs.samoa.instances.Instance; + +/** + * Computes the per-attribute similarity of categorical attributes with Euclidean distance, + * i.e. to consider them as numeric attributes + */ +public class EuclideanDistanceSimilarityCalculator extends AttributeSimilarityCalculator { + + @Override + public double computePerAttributeSimilarity(Attribute attr, double X_k, double Y_k) { + // TODO NOT CORRECT !!! To fix!!! + return Math.sqrt((X_k - Y_k) * (X_k - Y_k)); + } + + @Override + public double computeWeightOfAttribute(Attribute attr, Instance X, Instance Y) { + return 1; + } + +} diff --git a/moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/GoodAll3SimilarityCalculator.java b/moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/GoodAll3SimilarityCalculator.java new file mode 100644 index 000000000..0c0119b9a --- /dev/null +++ b/moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/GoodAll3SimilarityCalculator.java @@ -0,0 +1,23 @@ +package moa.classifiers.semisupervised.attributeSimilarity; + +import com.yahoo.labs.samoa.instances.Attribute; +import com.yahoo.labs.samoa.instances.Instance; + +/** + * Computes the similarity of categorical attributes using GoodAll3: + * if X_k == Y_k: 1 - p_k^2(x) + * else: 0 + */ +public class GoodAll3SimilarityCalculator extends AttributeSimilarityCalculator { + + @Override + public double computePerAttributeSimilarity(Attribute attr, double X_k, double Y_k) { + if (X_k == Y_k) return 1 - getProbabilityEstimateOfAttributeByValue(attr, (int)X_k); + return 0; + } + + @Override + public double computeWeightOfAttribute(Attribute attr, Instance X, Instance Y) { + return 1.0 / (float) d; + } +} diff --git a/moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/IgnoreSimilarityCalculator.java b/moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/IgnoreSimilarityCalculator.java new file mode 100644 index 000000000..2c92e6037 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/IgnoreSimilarityCalculator.java @@ -0,0 +1,20 @@ +package moa.classifiers.semisupervised.attributeSimilarity; + +import com.yahoo.labs.samoa.instances.Attribute; +import com.yahoo.labs.samoa.instances.Instance; + +/** + * Does nothing, just ignores the categorical attributes + */ +public class IgnoreSimilarityCalculator extends AttributeSimilarityCalculator { + + @Override + public double computePerAttributeSimilarity(Attribute attr, double X_k, double Y_k) { + return 0; + } + + @Override + public double computeWeightOfAttribute(Attribute attr, Instance X, Instance Y) { + return 0; + } +} diff --git a/moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/InverseOccurrenceFrequencySimilarityCalculator.java b/moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/InverseOccurrenceFrequencySimilarityCalculator.java new file mode 100644 index 000000000..a45a5c4d2 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/InverseOccurrenceFrequencySimilarityCalculator.java @@ -0,0 +1,24 @@ +package moa.classifiers.semisupervised.attributeSimilarity; + +import com.yahoo.labs.samoa.instances.Attribute; +import com.yahoo.labs.samoa.instances.Instance; + +/** + * Computes the similarity between categorical attributes using Inverse Occurrence Frequency (IOF) + */ +public class InverseOccurrenceFrequencySimilarityCalculator extends AttributeSimilarityCalculator { + @Override + public double computePerAttributeSimilarity(Attribute attr, double X_k, double Y_k) { + if (X_k == Y_k) return 1.0; + double fX = Math.max(attributeStats.get(attr).getFrequencyOfValue((int)X_k), SMALL_VALUE); + double fY = Math.max(attributeStats.get(attr).getFrequencyOfValue((int)Y_k), SMALL_VALUE); + double logX = fX > 0 ? Math.log(fX) : 0.0; + double logY = fY > 0 ? Math.log(fY) : 0.0; + return 1 / (1 + logX * logY); + } + + @Override + public double computeWeightOfAttribute(Attribute attr, Instance X, Instance Y) { + return 0; + } +} diff --git a/moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/LinSimilarityCalculator.java b/moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/LinSimilarityCalculator.java new file mode 100644 index 000000000..35bd68b66 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/LinSimilarityCalculator.java @@ -0,0 +1,32 @@ +package moa.classifiers.semisupervised.attributeSimilarity; + +import com.yahoo.labs.samoa.instances.Attribute; +import com.yahoo.labs.samoa.instances.Instance; + +/** + * Computes the similarity of categorical attributes using Lin formula: + * if X_k == Y_k: S_k = 2 * log[p_k(X_k)] + * else: S_k = 2 * log[p_k(X_k) + p_k(Y_k)] + */ +public class LinSimilarityCalculator extends AttributeSimilarityCalculator { + + @Override + public double computePerAttributeSimilarity(Attribute attr, double X_k, double Y_k) { + double pX = getSampleProbabilityOfAttributeByValue(attr, (int)X_k); + double pY = getSampleProbabilityOfAttributeByValue(attr, (int)Y_k); + if (X_k == Y_k) return 2.0 * Math.log(pX); + return 2.0 * Math.log(pX + pY); + } + + @Override + public double computeWeightOfAttribute(Attribute attr, Instance X, Instance Y) { + double deno = 0; + for (int i = 0; i < d; i++) { + double pX = getSampleProbabilityOfAttributeByValue(attr, (int)X.value(i)); + double pY = getSampleProbabilityOfAttributeByValue(attr, (int)Y.value(i)); + deno += Math.log(pX) + Math.log(pY); + } + if (deno == 0) return 1.0; + return 1.0 / deno; + } +} diff --git a/moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/OccurrenceFrequencySimilarityCalculator.java b/moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/OccurrenceFrequencySimilarityCalculator.java new file mode 100644 index 000000000..b23b53859 --- /dev/null +++ b/moa/src/main/java/moa/classifiers/semisupervised/attributeSimilarity/OccurrenceFrequencySimilarityCalculator.java @@ -0,0 +1,26 @@ +package moa.classifiers.semisupervised.attributeSimilarity; + +import com.yahoo.labs.samoa.instances.Attribute; +import com.yahoo.labs.samoa.instances.Instance; + +/** + * Computes the attribute similarity using Occurrence Frequency (OF): + * if X_k == Y_k: S_k(X_k, Y_k) = 1 + * else: S_k(X_k, Y_k) = 1 / (1 + log(N / f_k(X_k)) * log(N / f_k(Y_k))) + */ +public class OccurrenceFrequencySimilarityCalculator extends AttributeSimilarityCalculator { + + @Override + public double computePerAttributeSimilarity(Attribute attr, double X_k, double Y_k) { + if (X_k == Y_k) return 1; + if (attributeStats.get(attr) == null) return SMALL_VALUE; + double fX = Math.max(attributeStats.get(attr).getFrequencyOfValue((int)X_k), SMALL_VALUE); + double fY = Math.max(attributeStats.get(attr).getFrequencyOfValue((int)Y_k), SMALL_VALUE); + return 1.0 / (1.0 + Math.log(N / fX) * Math.log(N / fY)); + } + + @Override + public double computeWeightOfAttribute(Attribute attr, Instance X, Instance Y) { + return 1.0 / (double) d; + } +} diff --git a/moa/src/main/java/moa/clusterers/clustream/Clustream.java b/moa/src/main/java/moa/clusterers/clustream/Clustream.java index 58e0428bd..93e221c7a 100644 --- a/moa/src/main/java/moa/clusterers/clustream/Clustream.java +++ b/moa/src/main/java/moa/clusterers/clustream/Clustream.java @@ -14,8 +14,8 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * - * + * + * */ package moa.clusterers.clustream; @@ -98,7 +98,9 @@ public void trainOnInstanceImpl(Instance instance) { // Clustering kmeans_clustering = kMeans(k, buffer); for ( int i = 0; i < kmeans_clustering.size(); i++ ) { - kernels[i] = new ClustreamKernel( new DenseInstance(1.0,centers[i].getCenter()), dim, timestamp, t, m ); + Instance newInstance = new DenseInstance(1.0,centers[i].getCenter()); + newInstance.setDataset(instance.dataset()); + kernels[i] = new ClustreamKernel(newInstance, dim, timestamp, t, m ); } buffer.clear(); @@ -111,7 +113,7 @@ public void trainOnInstanceImpl(Instance instance) { double minDistance = Double.MAX_VALUE; for ( int i = 0; i < kernels.length; i++ ) { //System.out.println(i+" "+kernels[i].getWeight()+" "+kernels[i].getDeviation()); - double distance = distance(instance.toDoubleArray(), kernels[i].getCenter() ); + double distance = distanceIgnoreNaN(instance.toDoubleArray(), kernels[i].getCenter() ); if ( distance < minDistance ) { closestKernel = kernels[i]; minDistance = distance; @@ -213,6 +215,26 @@ private static double distance(double[] pointA, double [] pointB){ return Math.sqrt(distance); } + /*** + * This function avoids the undesirable situation where the whole distance becomes NaN if one of the attributes + * is NaN. + * (SSL) This was observed when calculating the distance between an instance without the class label and a center + * which was updated using the class label. + * @param pointA + * @param pointB + * @return + */ + public static double distanceIgnoreNaN(double[] pointA, double [] pointB){ + double distance = 0.0; + for (int i = 0; i < pointA.length; i++) { + if(!(Double.isNaN(pointA[i]) || Double.isNaN(pointB[i]))) { + double d = pointA[i] - pointB[i]; + distance += d * d; + } + } + return Math.sqrt(distance); + } + //wrapper... we need to rewrite kmeans to points, not clusters, doesnt make sense anymore // public static Clustering kMeans( int k, ArrayList points, int dim ) { // ArrayList cl = new ArrayList(); diff --git a/moa/src/main/java/moa/clusterers/clustream/ClustreamKernel.java b/moa/src/main/java/moa/clusterers/clustream/ClustreamKernel.java index d4f901ba4..609ad8fdb 100644 --- a/moa/src/main/java/moa/clusterers/clustream/ClustreamKernel.java +++ b/moa/src/main/java/moa/clusterers/clustream/ClustreamKernel.java @@ -14,8 +14,8 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * - * + * + * */ package moa.clusterers.clustream; @@ -25,9 +25,9 @@ import com.yahoo.labs.samoa.instances.Instance; public class ClustreamKernel extends CFCluster { - private static final long serialVersionUID = 1L; + private static final long serialVersionUID = 1L; - private final static double EPSILON = 0.00005; + private final static double EPSILON = 0.00005; public static final double MIN_VARIANCE = 1e-50; protected double LST; @@ -36,66 +36,111 @@ public class ClustreamKernel extends CFCluster { int m; double t; + public double[] classObserver; + + public static int ID_GENERATOR = 0; - public ClustreamKernel( Instance instance, int dimensions, long timestamp , double t, int m) { + public ClustreamKernel(Instance instance, int dimensions, long timestamp , double t, int m) { super(instance, dimensions); + +// Avoid situations where the instance header hasn't been defined and runtime errors. + if(instance.dataset() != null) { + this.classObserver = new double[instance.numClasses()]; +// instance.numAttributes() <= instance.classIndex() -> edge case where the class index is equal the +// number of attributes (i.e. there is no class value in the attributes array). + if (instance.numAttributes() > instance.classIndex() && + !instance.classIsMissing() && + instance.classValue() >= 0 && + instance.classValue() < instance.numClasses()) { + this.classObserver[(int) instance.classValue()]++; + } + } + this.setId(ID_GENERATOR++); this.t = t; this.m = m; this.LST = timestamp; - this.SST = timestamp*timestamp; + this.SST = timestamp*timestamp; } public ClustreamKernel( ClustreamKernel cluster, double t, int m ) { super(cluster); + this.setId(ID_GENERATOR++); this.t = t; this.m = m; this.LST = cluster.LST; this.SST = cluster.SST; + this.classObserver = cluster.classObserver; } public void insert( Instance instance, long timestamp ) { - N++; - LST += timestamp; - SST += timestamp*timestamp; - - for ( int i = 0; i < instance.numValues(); i++ ) { - LS[i] += instance.value(i); - SS[i] += instance.value(i)*instance.value(i); - } + if(this.classObserver == null) + this.classObserver = new double[instance.numClasses()]; + if(!instance.classIsMissing() && + instance.classValue() >= 0 && + instance.classValue() < instance.numClasses()) { + this.classObserver[(int)instance.classValue()]++; + } + N++; + LST += timestamp; + SST += timestamp*timestamp; + + for ( int i = 0; i < instance.numValues(); i++ ) { + LS[i] += instance.value(i); + SS[i] += instance.value(i)*instance.value(i); + } } @Override public void add( CFCluster other2 ) { ClustreamKernel other = (ClustreamKernel) other2; - assert( other.LS.length == this.LS.length ); - this.N += other.N; - this.LST += other.LST; - this.SST += other.SST; - - for ( int i = 0; i < LS.length; i++ ) { - this.LS[i] += other.LS[i]; - this.SS[i] += other.SS[i]; - } + assert( other.LS.length == this.LS.length ); + this.N += other.N; + this.LST += other.LST; + this.SST += other.SST; + this.classObserver = sumClassObservers(other.classObserver, this.classObserver); + + for ( int i = 0; i < LS.length; i++ ) { + this.LS[i] += other.LS[i]; + this.SS[i] += other.SS[i]; + } } + private double[] sumClassObservers(double[] A, double[] B) { + double[] result = null; + if (A != null && B != null) { + result = new double[A.length]; + if(A.length == B.length) + for(int i = 0 ; i < A.length ; ++i) + result[i] += A[i] + B[i]; + } + return result; + } + +// @Override +// public void add( CFCluster other2, long timestamp) { +// this.add(other2); +// // accumulate the count +// this.accumulateWeight(other2, timestamp); +// } + public double getRelevanceStamp() { - if ( N < 2*m ) - return getMuTime(); - - return getMuTime() + getSigmaTime() * getQuantile( ((double)m)/(2*N) ); + if ( N < 2*m ) + return getMuTime(); + + return getMuTime() + getSigmaTime() * getQuantile( ((double)m)/(2*N) ); } private double getMuTime() { - return LST / N; + return LST / N; } private double getSigmaTime() { - return Math.sqrt(SST/N - (LST/N)*(LST/N)); + return Math.sqrt(SST/N - (LST/N)*(LST/N)); } private double getQuantile( double z ) { - assert( z >= 0 && z <= 1 ); - return Math.sqrt( 2 ) * inverseError( 2*z - 1 ); + assert( z >= 0 && z <= 1 ); + return Math.sqrt( 2 ) * inverseError( 2*z - 1 ); } @Override @@ -187,7 +232,7 @@ private double[] getVarianceVector() { } } else{ - + } } return res; @@ -223,7 +268,7 @@ private double calcNormalizedDistance(double[] point) { return Math.sqrt(res); } - /** + /** * Approximates the inverse error function. Clustream needs this. * @param x */ @@ -266,7 +311,7 @@ protected void getClusterSpecificInfo(ArrayList infoTitle, ArrayList windowedResults; public double[] cumulativeResults; - public ArrayList targets; - public ArrayList predictions; public HashMap otherMeasurements; public PrequentialResult(ArrayList windowedResults, double[] cumulativeResults) { this.windowedResults = windowedResults; this.cumulativeResults = cumulativeResults; - this.targets = null; - this.predictions = null; - } - - public PrequentialResult(ArrayList windowedResults, double[] cumulativeResults, - ArrayList targets, ArrayList predictions) { - this.windowedResults = windowedResults; - this.cumulativeResults = cumulativeResults; - this.targets = targets; - this.predictions = predictions; } - /*** - * This constructor is useful to store metrics beyond the evaluation metrics available through the evaluators. - * @param windowedResults - * @param cumulativeResults - * @param otherMeasurements - */ public PrequentialResult(ArrayList windowedResults, double[] cumulativeResults, HashMap otherMeasurements) { this(windowedResults, cumulativeResults); @@ -93,29 +52,23 @@ public PrequentialResult(ArrayList windowedResults, double[] cumulativ * @param windowedEvaluator * @param maxInstances * @param windowSize - * @return PrequentialResult is a custom class that holds the respective results from the execution + * @return the return has to be an ArrayList because we don't know ahead of time how many windows will be produced */ public static PrequentialResult PrequentialEvaluation(ExampleStream stream, Learner learner, LearningPerformanceEvaluator basicEvaluator, LearningPerformanceEvaluator windowedEvaluator, - long maxInstances, long windowSize, - boolean storeY, boolean storePredictions) { + long maxInstances, long windowSize) { int instancesProcessed = 0; if (!stream.hasMoreInstances()) stream.restart(); ArrayList windowed_results = new ArrayList<>(); - ArrayList targetValues = new ArrayList<>(); - ArrayList predictions = new ArrayList<>(); - while (stream.hasMoreInstances() && (maxInstances == -1 || instancesProcessed < maxInstances)) { - Example instance = stream.nextInstance(); - if (storeY) - targetValues.add(instance.getData().classValue()); + Example instance = stream.nextInstance(); double[] prediction = learner.getVotesForInstance(instance); if (basicEvaluator != null) @@ -123,9 +76,6 @@ public static PrequentialResult PrequentialEvaluation(ExampleStream stream, Lear if (windowedEvaluator != null) windowedEvaluator.addResult(instance, prediction); - if (storePredictions) - predictions.add(prediction.length == 0? 0 : prediction[0]); - learner.trainOnInstance(instance); instancesProcessed++; @@ -156,62 +106,280 @@ public static PrequentialResult PrequentialEvaluation(ExampleStream stream, Lear for (int i = 0; i < cumulative_results.length; ++i) cumulative_results[i] = measurements[i].getValue(); } - if (!storePredictions && !storeY) - return new PrequentialResult(windowed_results, cumulative_results); - else - return new PrequentialResult(windowed_results, cumulative_results, targetValues, predictions); + + return new PrequentialResult(windowed_results, cumulative_results); } + public static PrequentialResult PrequentialSSLEvaluation(ExampleStream stream, Learner learner, + LearningPerformanceEvaluator basicEvaluator, + LearningPerformanceEvaluator windowedEvaluator, + long maxInstances, + long windowSize, + long initialWindowSize, + long delayLength, + double labelProbability, + int randomSeed, + boolean debugPseudoLabels) { +// int delayLength = this.delayLengthOption.getValue(); +// double labelProbability = this.labelProbabilityOption.getValue(); + + RandomGenerator taskRandom = new MersenneTwister(randomSeed); +// ExampleStream stream = (ExampleStream) getPreparedClassOption(this.streamOption); +// Learner learner = getLearner(stream); - /*** - * The following code can be used to provide examples of how to use the class. - * In the future, some of these examples can be turned into tests. - * @param args - */ - public static void main(String[] args) { - examplePrequentialEvaluation_edge_cases1(); - examplePrequentialEvaluation_edge_cases2(); - examplePrequentialEvaluation_edge_cases3(); - examplePrequentialEvaluation_edge_cases4(); - examplePrequentialEvaluation_SampleFrequency_TestThenTrain(); - examplePrequentialRegressionEvaluation(); - examplePrequentialEvaluation(); - exampleTestThenTrainEvaluation(); - exampleWindowedEvaluation(); - - // Run time efficiency evaluation examples - StreamingRandomPatches srp10 = new StreamingRandomPatches(); - srp10.getOptions().setViaCLIString("-s 10"); // 10 learners - srp10.setRandomSeed(5); - srp10.prepareForUse(); - - StreamingRandomPatches srp100 = new StreamingRandomPatches(); - srp100.getOptions().setViaCLIString("-s 100"); // 100 learners - srp100.setRandomSeed(5); - srp100.prepareForUse(); - - int maxInstances = 100000; - examplePrequentialEfficiency(srp10, maxInstances); - examplePrequentialEfficiency(srp100, maxInstances); + int instancesProcessed = 0; + int numCorrectPseudoLabeled = 0; + int numUnlabeledData = 0; + int numInstancesTested = 0; + + if (!stream.hasMoreInstances()) + stream.restart(); + + ArrayList windowed_results = new ArrayList<>(); + + HashMap other_measures = new HashMap<>(); + + // The buffer is a list of tuples. The first element is the index when + // it should be emitted. The second element is the instance itself. + List> delayBuffer = new ArrayList>(); + + while (stream.hasMoreInstances() && + (maxInstances == -1 || instancesProcessed < maxInstances)) { + + // TRAIN on delayed instances + while (delayBuffer.size() > 0 + && delayBuffer.get(0).getKey() == instancesProcessed) { + Example delayedExample = delayBuffer.remove(0).getValue(); +// System.out.println("[TRAIN][DELAY] "+delayedExample.getData().toString()); + learner.trainOnInstance(delayedExample); + } + + Example instance = stream.nextInstance(); + Example unlabeledExample = instance.copy(); + int trueClass = (int) ((Instance) instance.getData()).classValue(); + + // In case it is set, then the label is not removed. We want to pass the + // labelled data to the learner even in trainOnUnlabeled data to generate statistics such as number + // of correctly pseudo-labeled instances. + if (!debugPseudoLabels) { + // Remove the label of the unlabeledExample indirectly through + // unlabeledInstanceData. + Instance __instance = (Instance) unlabeledExample.getData(); + __instance.setMissing(__instance.classIndex()); + } + + // WARMUP + // Train on the initial instances. These are not used for testing! + if (instancesProcessed < initialWindowSize) { +// if (learner instanceof SemiSupervisedLearner) +// ((SemiSupervisedLearner) learner).addInitialWarmupTrainingInstances(); +// System.out.println("[TRAIN][INITIAL_WINDOW] "+instance.getData().toString()); + learner.trainOnInstance(instance); + instancesProcessed++; + continue; + } + + Boolean is_labeled = labelProbability > taskRandom.nextDouble(); + if (!is_labeled) { + numUnlabeledData++; + } + + // TEST + // Obtain the prediction for the testInst (i.e. no label) +// System.out.println("[TEST] " + unlabeledExample.getData().toString()); + double[] prediction = learner.getVotesForInstance(unlabeledExample); + numInstancesTested++; + + if (basicEvaluator != null) + basicEvaluator.addResult(instance, prediction); + if (windowedEvaluator != null) + windowedEvaluator.addResult(instance, prediction); + + int pseudoLabel = -1; + // TRAIN + if (is_labeled && delayLength >= 0) { + // The instance will be labeled but has been delayed + if (learner instanceof SemiSupervisedLearner) { +// System.out.println("[TRAIN_UNLABELED][DELAYED] " + unlabeledExample.getData().toString()); + pseudoLabel = ((SemiSupervisedLearner) learner).trainOnUnlabeledInstance((Instance) unlabeledExample.getData()); + } + delayBuffer.add(new MutablePair(1 + instancesProcessed + delayLength, instance)); + } else if (is_labeled) { +// System.out.println("[TRAIN] " + instance.getData().toString()); + // The instance will be labeled and is not delayed e.g delayLength = -1 + learner.trainOnInstance(instance); + } else { + // The instance will never be labeled + if (learner instanceof SemiSupervisedLearner) { +// System.out.println("[TRAIN_UNLABELED][IMMEDIATE] " + unlabeledExample.getData().toString()); + pseudoLabel = ((SemiSupervisedLearner) learner).trainOnUnlabeledInstance((Instance) unlabeledExample.getData()); + } + } + if(trueClass == pseudoLabel) + numCorrectPseudoLabeled++; + + instancesProcessed++; + + if (windowedEvaluator != null) + if (instancesProcessed % windowSize == 0) { + Measurement[] measurements = windowedEvaluator.getPerformanceMeasurements(); + double[] values = new double[measurements.length]; + for (int i = 0; i < values.length; ++i) + values[i] = measurements[i].getValue(); + windowed_results.add(values); + } + } + if (windowedEvaluator != null) + if (instancesProcessed % windowSize != 0) { + Measurement[] measurements = windowedEvaluator.getPerformanceMeasurements(); + double[] values = new double[measurements.length]; + for (int i = 0; i < values.length; ++i) + values[i] = measurements[i].getValue(); + windowed_results.add(values); + } + + double[] cumulative_results = null; + + if (basicEvaluator != null) { + Measurement[] measurements = basicEvaluator.getPerformanceMeasurements(); + cumulative_results = new double[measurements.length]; + for (int i = 0; i < cumulative_results.length; ++i) + cumulative_results[i] = measurements[i].getValue(); + } + + // TODO: Add this measures in a windowed way. + other_measures.put("num_unlabeled_instances", (double) numUnlabeledData); + other_measures.put("num_correct_pseudo_labeled", (double) numCorrectPseudoLabeled); + other_measures.put("num_instances_tested", (double) numInstancesTested); + other_measures.put("pseudo_label_accuracy", (double) numCorrectPseudoLabeled/numInstancesTested); + return new PrequentialResult(windowed_results, cumulative_results, other_measures); + } + + /******************************************************************************************************************/ + /******************************************************************************************************************/ + /***************************************** TESTS ******************************************************************/ + /******************************************************************************************************************/ + /******************************************************************************************************************/ + + private static void testPrequentialSSL(String file_path, Learner learner, + long maxInstances, + long windowSize, + long initialWindowSize, + long delayLength, + double labelProbability) { + System.out.println( + "maxInstances: " + maxInstances + ", " + + "windowSize: " + windowSize + ", " + + "initialWindowSize: " + initialWindowSize + ", " + + "delayLength: " + delayLength + ", " + + "labelProbability: " + labelProbability + ); + + // Record the start time + long startTime = System.currentTimeMillis(); + + ArffFileStream stream = new ArffFileStream(file_path, -1); + stream.prepareForUse(); + + BasicClassificationPerformanceEvaluator basic_evaluator = new BasicClassificationPerformanceEvaluator(); + basic_evaluator.recallPerClassOption.setValue(true); + basic_evaluator.prepareForUse(); + + WindowClassificationPerformanceEvaluator windowed_evaluator = new WindowClassificationPerformanceEvaluator(); + windowed_evaluator.widthOption.setValue((int) windowSize); + windowed_evaluator.prepareForUse(); + + PrequentialResult result = PrequentialSSLEvaluation(stream, learner, + basic_evaluator, + windowed_evaluator, + maxInstances, + windowSize, + initialWindowSize, + delayLength, + labelProbability, 1, true); + + // Record the end time + long endTime = System.currentTimeMillis(); + + // Calculate the elapsed time in milliseconds + long elapsedTime = endTime - startTime; + + // Print the elapsed time + System.out.println("Elapsed Time: " + elapsedTime / 1000 + " seconds"); + System.out.println("Number of unlabeled instances: " + result.otherMeasurements.get("num_unlabeled_instances")); + + System.out.println("\tBasic performance"); + for (int i = 0; i < result.cumulativeResults.length; ++i) + System.out.println(basic_evaluator.getPerformanceMeasurements()[i].getName() + ": " + result.cumulativeResults[i]); + + System.out.println("\tWindowed performance"); + for (int j = 0; j < result.windowedResults.size(); ++j) { + System.out.print("Window: " + j + ", "); + for (int i = 0; i < 2; ++i) // results.get(results.size()-1).length; ++i) + System.out.println(windowed_evaluator.getPerformanceMeasurements()[i].getName() + ": " + result.windowedResults.get(j)[i]); + } } + public static void main(String[] args) { + String hyper_arff = "/Users/gomeshe/Desktop/data/Hyper100k.arff"; + String debug_arff = "/Users/gomeshe/Desktop/data/debug_prequential_SSL.arff"; + String ELEC_arff = "/Users/gomeshe/Dropbox/ciencia_computacao/lecturer/research/ssl_disagreement/datasets/ELEC/elecNormNew.arff"; + + NaiveBayes learner = new NaiveBayes(); + learner.prepareForUse(); + +// testPrequentialSSL(debug_arff, learner, 100, 10, 0, 0, 1.0); // OK +// testPrequentialSSL(debug_arff, learner, 100, 10, 1, 0, 1.0); //OK +// testPrequentialSSL(debug_arff, learner, 10, 10, 5, 0, 1.0); // OK +// testPrequentialSSL(debug_arff, learner, 10, 10, -1, 1, 1.0); // OK +// testPrequentialSSL(debug_arff, learner, 20, 10, -1, 10, 1.0); // OK +// testPrequentialSSL(debug_arff, learner, 20, 10, -1, 2, 0.5); // OK +// testPrequentialSSL(debug_arff, learner, 100, 10, 50, 2, 0.0); // OK +// testPrequentialSSL(debug_arff, learner, 100, 10, 0, 90, 1.0); // OK +// testPrequentialSSL(debug_arff, learner, 100, 10, 0, -1, 0.5); // OK + +// testPrequentialSSL(hyper_arff, learner, -1, 1000, -1, -1, 1.0); +// testPrequentialSSL(hyper_arff, learner, -1, 1000, -1, -1, 0.5); // OK + +// testPrequentialSSL(hyper_arff, learner, -1, 1000, 1000, -1, 0.5); + + ClusterAndLabelClassifier ssl_learner = new ClusterAndLabelClassifier(); + ssl_learner.prepareForUse(); + + testPrequentialSSL(ELEC_arff, ssl_learner, 10000, 1000, -1, -1, 0.01); + +// testWindowedEvaluation(); +// testTestThenTrainEvaluation(); +// testPrequentialEvaluation(); +// +// StreamingRandomPatches learner = new StreamingRandomPatches(); +// learner.getOptions().setViaCLIString("-s 100"); // 10 learners +//// learner.setRandomSeed(5); +// learner.prepareForUse(); +// testPrequentialEfficiency1(learner); + +// testPrequentialEvaluation_edge_cases1(); +// testPrequentialEvaluation_edge_cases2(); +// testPrequentialEvaluation_edge_cases3(); +// testPrequentialEvaluation_edge_cases4(); +// testPrequentialEvaluation_SampleFrequency_TestThenTrain(); + +// testPrequentialRegressionEvaluation(); + } - private static void examplePrequentialEfficiency(Learner learner, int maxInstances) { - System.out.println("Assessing efficiency for " + learner.getCLICreationString(learner.getClass()) + - " maxInstances: " + maxInstances); + private static void testPrequentialEfficiency1(Learner learner) { // Record the start time long startTime = System.currentTimeMillis(); - AgrawalGenerator stream = new AgrawalGenerator(); + ArffFileStream stream = new ArffFileStream("/Users/gomeshe/Desktop/data/Hyper100k.arff", -1); stream.prepareForUse(); BasicClassificationPerformanceEvaluator basic_evaluator = new BasicClassificationPerformanceEvaluator(); basic_evaluator.recallPerClassOption.setValue(true); basic_evaluator.prepareForUse(); - PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, null, - maxInstances, 1, false, false); + PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, null, 100000, 1); // Record the end time long endTime = System.currentTimeMillis(); @@ -227,18 +395,25 @@ private static void examplePrequentialEfficiency(Learner learner, int maxInstanc System.out.println(basic_evaluator.getPerformanceMeasurements()[i].getName() + ": " + results.cumulativeResults[i]); } - private static void examplePrequentialEvaluation_edge_cases1() { + private static void testPrequentialEvaluation_edge_cases1() { // Record the start time long startTime = System.currentTimeMillis(); NaiveBayes learner = new NaiveBayes(); learner.prepareForUse(); - AgrawalGenerator stream = new AgrawalGenerator(); + ArffFileStream stream = new ArffFileStream("/Users/gomeshe/Desktop/data/Hyper100k.arff", -1); stream.prepareForUse(); - PrequentialResult results = PrequentialEvaluation(stream, learner, null, null, - 100000, 1000, false, false); +// BasicClassificationPerformanceEvaluator basic_evaluator = new BasicClassificationPerformanceEvaluator(); +// basic_evaluator.recallPerClassOption.setValue(true); +// basic_evaluator.prepareForUse(); +// +// WindowClassificationPerformanceEvaluator windowed_evaluator = new WindowClassificationPerformanceEvaluator(); +// windowed_evaluator.widthOption.setValue(1000); +// windowed_evaluator.prepareForUse(); + + PrequentialResult results = PrequentialEvaluation(stream, learner, null, null, 100000, 1000); // Record the end time long endTime = System.currentTimeMillis(); @@ -248,16 +423,28 @@ private static void examplePrequentialEvaluation_edge_cases1() { // Print the elapsed time System.out.println("Elapsed Time: " + elapsedTime / 1000 + " seconds"); + +// System.out.println("\tBasic performance"); +// for (int i = 0; i < results.basicResults.length; ++i) +// System.out.println(basic_evaluator.getPerformanceMeasurements()[i].getName() + ": " + results.basicResults[i]); + +// System.out.println("\tWindowed performance"); +// for (int j = 0; j < results.windowedResults.size(); ++j) { +// System.out.println("\t" + j); +// for (int i = 0; i < 2; ++i) // results.get(results.size()-1).length; ++i) +// System.out.println(windowed_evaluator.getPerformanceMeasurements()[i].getName() + ": " + results.windowedResults.get(j)[i]); +// } } - private static void examplePrequentialEvaluation_edge_cases2() { + + private static void testPrequentialEvaluation_edge_cases2() { // Record the start time long startTime = System.currentTimeMillis(); NaiveBayes learner = new NaiveBayes(); learner.prepareForUse(); - AgrawalGenerator stream = new AgrawalGenerator(); + ArffFileStream stream = new ArffFileStream("/Users/gomeshe/Desktop/data/Hyper100k.arff", -1); stream.prepareForUse(); BasicClassificationPerformanceEvaluator basic_evaluator = new BasicClassificationPerformanceEvaluator(); @@ -268,8 +455,7 @@ private static void examplePrequentialEvaluation_edge_cases2() { windowed_evaluator.widthOption.setValue(1000); windowed_evaluator.prepareForUse(); - PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, windowed_evaluator, - 1000, 10000, false, false); + PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, windowed_evaluator, 1000, 10000); // Record the end time long endTime = System.currentTimeMillis(); @@ -292,14 +478,14 @@ private static void examplePrequentialEvaluation_edge_cases2() { } } - private static void examplePrequentialEvaluation_edge_cases3() { + private static void testPrequentialEvaluation_edge_cases3() { // Record the start time long startTime = System.currentTimeMillis(); NaiveBayes learner = new NaiveBayes(); learner.prepareForUse(); - AgrawalGenerator stream = new AgrawalGenerator(); + ArffFileStream stream = new ArffFileStream("/Users/gomeshe/Desktop/data/Hyper100k.arff", -1); stream.prepareForUse(); BasicClassificationPerformanceEvaluator basic_evaluator = new BasicClassificationPerformanceEvaluator(); @@ -310,8 +496,7 @@ private static void examplePrequentialEvaluation_edge_cases3() { windowed_evaluator.widthOption.setValue(1000); windowed_evaluator.prepareForUse(); - PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, windowed_evaluator, - 10, 1, false, false); + PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, windowed_evaluator, 10, 1); // Record the end time long endTime = System.currentTimeMillis(); @@ -324,26 +509,24 @@ private static void examplePrequentialEvaluation_edge_cases3() { System.out.println("\tBasic performance"); for (int i = 0; i < results.cumulativeResults.length; ++i) - System.out.println(basic_evaluator.getPerformanceMeasurements()[i].getName() + ": " + - results.cumulativeResults[i]); + System.out.println(basic_evaluator.getPerformanceMeasurements()[i].getName() + ": " + results.cumulativeResults[i]); System.out.println("\tWindowed performance"); for (int j = 0; j < results.windowedResults.size(); ++j) { System.out.println("\t" + j); for (int i = 0; i < 2; ++i) // results.get(results.size()-1).length; ++i) - System.out.println(windowed_evaluator.getPerformanceMeasurements()[i].getName() + ": " + - results.windowedResults.get(j)[i]); + System.out.println(windowed_evaluator.getPerformanceMeasurements()[i].getName() + ": " + results.windowedResults.get(j)[i]); } } - private static void examplePrequentialEvaluation_edge_cases4() { + private static void testPrequentialEvaluation_edge_cases4() { // Record the start time long startTime = System.currentTimeMillis(); NaiveBayes learner = new NaiveBayes(); learner.prepareForUse(); - AgrawalGenerator stream = new AgrawalGenerator(); + ArffFileStream stream = new ArffFileStream("/Users/gomeshe/Desktop/data/Hyper100k.arff", -1); stream.prepareForUse(); BasicClassificationPerformanceEvaluator basic_evaluator = new BasicClassificationPerformanceEvaluator(); @@ -354,8 +537,7 @@ private static void examplePrequentialEvaluation_edge_cases4() { windowed_evaluator.widthOption.setValue(10000); windowed_evaluator.prepareForUse(); - PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, windowed_evaluator, - 100000, 10000, false, false); + PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, windowed_evaluator, -1, 10000); // Record the end time long endTime = System.currentTimeMillis(); @@ -378,22 +560,26 @@ private static void examplePrequentialEvaluation_edge_cases4() { } } - private static void examplePrequentialEvaluation_SampleFrequency_TestThenTrain() { + + private static void testPrequentialEvaluation_SampleFrequency_TestThenTrain() { // Record the start time long startTime = System.currentTimeMillis(); NaiveBayes learner = new NaiveBayes(); learner.prepareForUse(); - AgrawalGenerator stream = new AgrawalGenerator(); + ArffFileStream stream = new ArffFileStream("/Users/gomeshe/Desktop/data/Hyper100k.arff", -1); stream.prepareForUse(); BasicClassificationPerformanceEvaluator basic_evaluator = new BasicClassificationPerformanceEvaluator(); basic_evaluator.recallPerClassOption.setValue(true); basic_evaluator.prepareForUse(); - PrequentialResult results = PrequentialEvaluation(stream, learner, null, basic_evaluator, - 100000, 10000, false, false); +// WindowClassificationPerformanceEvaluator windowed_evaluator = new WindowClassificationPerformanceEvaluator(); +// windowed_evaluator.widthOption.setValue(10000); +// windowed_evaluator.prepareForUse(); + + PrequentialResult results = PrequentialEvaluation(stream, learner, null, basic_evaluator, -1, 10000); // Record the end time long endTime = System.currentTimeMillis(); @@ -404,6 +590,10 @@ private static void examplePrequentialEvaluation_SampleFrequency_TestThenTrain() // Print the elapsed time System.out.println("Elapsed Time: " + elapsedTime / 1000 + " seconds"); +// System.out.println("\tBasic performance"); +// for (int i = 0; i < results.basicResults.length; ++i) +// System.out.println(basic_evaluator.getPerformanceMeasurements()[i].getName() + ": " + results.basicResults[i]); + System.out.println("\tWindowed performance"); for (int j = 0; j < results.windowedResults.size(); ++j) { System.out.println("\t" + j); @@ -412,23 +602,26 @@ private static void examplePrequentialEvaluation_SampleFrequency_TestThenTrain() } } - private static void examplePrequentialRegressionEvaluation() { + + private static void testPrequentialRegressionEvaluation() { // Record the start time long startTime = System.currentTimeMillis(); FIMTDD learner = new FIMTDD(); +// learner.getOptions().setViaCLIString("-s 10"); // 10 learners +// learner.setRandomSeed(5); learner.prepareForUse(); - HyperplaneGenerator stream = new HyperplaneGenerator(); + ArffFileStream stream = new ArffFileStream("/Users/gomeshe/Desktop/data/metrotraffic_with_nominals.arff", -1); stream.prepareForUse(); BasicRegressionPerformanceEvaluator basic_evaluator = new BasicRegressionPerformanceEvaluator(); WindowRegressionPerformanceEvaluator windowed_evaluator = new WindowRegressionPerformanceEvaluator(); windowed_evaluator.widthOption.setValue(1000); +// windowed_evaluator.prepareForUse(); - PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, windowed_evaluator, - 10000, 1000, false, false); + PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, windowed_evaluator, 100000, 1000); // Record the end time long endTime = System.currentTimeMillis(); @@ -451,7 +644,7 @@ private static void examplePrequentialRegressionEvaluation() { } } - private static void examplePrequentialEvaluation() { + private static void testPrequentialEvaluation() { // Record the start time long startTime = System.currentTimeMillis(); @@ -471,7 +664,7 @@ private static void examplePrequentialEvaluation() { windowed_evaluator.widthOption.setValue(1000); windowed_evaluator.prepareForUse(); - PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, windowed_evaluator, 100000, 1000, false, false); + PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, windowed_evaluator, 100000, 1000); // Record the end time long endTime = System.currentTimeMillis(); @@ -494,22 +687,23 @@ private static void examplePrequentialEvaluation() { } } - private static void exampleTestThenTrainEvaluation() { + private static void testTestThenTrainEvaluation() { // Record the start time long startTime = System.currentTimeMillis(); NaiveBayes learner = new NaiveBayes(); +// learner.getOptions().setViaCLIString("-s 10"); // 10 learners +// learner.setRandomSeed(5); learner.prepareForUse(); - HyperplaneGenerator stream = new HyperplaneGenerator(); + ArffFileStream stream = new ArffFileStream("/Users/gomeshe/Desktop/data/Hyper100k.arff", -1); stream.prepareForUse(); BasicClassificationPerformanceEvaluator evaluator = new BasicClassificationPerformanceEvaluator(); evaluator.recallPerClassOption.setValue(true); evaluator.prepareForUse(); - PrequentialResult results = PrequentialEvaluation(stream, learner, evaluator, null, - 100000, 100000, false, false); + PrequentialResult results = PrequentialEvaluation(stream, learner, evaluator, null, 100000, 100000); // Record the end time long endTime = System.currentTimeMillis(); @@ -521,18 +715,19 @@ private static void exampleTestThenTrainEvaluation() { System.out.println("Elapsed Time: " + elapsedTime / 1000 + " seconds"); for (int i = 0; i < results.cumulativeResults.length; ++i) - System.out.println(evaluator.getPerformanceMeasurements()[i].getName() + ": " + - results.cumulativeResults[i]); + System.out.println(evaluator.getPerformanceMeasurements()[i].getName() + ": " + results.cumulativeResults[i]); } - private static void exampleWindowedEvaluation() { + private static void testWindowedEvaluation() { // Record the start time long startTime = System.currentTimeMillis(); NaiveBayes learner = new NaiveBayes(); +// learner.getOptions().setViaCLIString("-s 10"); // 10 learners +// learner.setRandomSeed(5); learner.prepareForUse(); - HyperplaneGenerator stream = new HyperplaneGenerator(); + ArffFileStream stream = new ArffFileStream("/Users/gomeshe/Desktop/data/Hyper100k.arff", -1); stream.prepareForUse(); WindowClassificationPerformanceEvaluator evaluator = new WindowClassificationPerformanceEvaluator(); @@ -540,8 +735,7 @@ private static void exampleWindowedEvaluation() { evaluator.recallPerClassOption.setValue(true); evaluator.prepareForUse(); - PrequentialResult results = PrequentialEvaluation(stream, learner, null, evaluator, - 100000, 10000, false, false); + PrequentialResult results = PrequentialEvaluation(stream, learner, null, evaluator, 100000, 10000); // Record the end time long endTime = System.currentTimeMillis(); @@ -555,9 +749,7 @@ private static void exampleWindowedEvaluation() { for (int j = 0; j < results.windowedResults.size(); ++j) { System.out.println("\t" + j); for (int i = 0; i < 2; ++i) // results.get(results.size()-1).length; ++i) - System.out.println(evaluator.getPerformanceMeasurements()[i].getName() + ": " + - results.windowedResults.get(j)[i]); + System.out.println(evaluator.getPerformanceMeasurements()[i].getName() + ": " + results.windowedResults.get(j)[i]); } } - } \ No newline at end of file diff --git a/moa/src/main/java/moa/evaluation/LearningPerformanceEvaluator.java b/moa/src/main/java/moa/evaluation/LearningPerformanceEvaluator.java index a7c655be8..911ac4c50 100644 --- a/moa/src/main/java/moa/evaluation/LearningPerformanceEvaluator.java +++ b/moa/src/main/java/moa/evaluation/LearningPerformanceEvaluator.java @@ -15,7 +15,7 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . - * + * */ package moa.evaluation; @@ -35,35 +35,37 @@ * @author Richard Kirkby (rkirkby@cs.waikato.ac.nz) * @version $Revision: 7 $ */ -public interface LearningPerformanceEvaluator extends MOAObject, CapabilitiesHandler { +public interface LearningPerformanceEvaluator extends MOAObject, CapabilitiesHandler, AutoCloseable { - /** - * Resets this evaluator. It must be similar to - * starting a new evaluator from scratch. - * - */ + /** + * Resets this evaluator. It must be similar to + * starting a new evaluator from scratch. + * + */ public void reset(); - /** - * Adds a learning result to this evaluator. - * - * @param example the example to be classified - * @param classVotes an array containing the estimated membership - * probabilities of the test instance in each class - */ - public void addResult(E example, double[] classVotes); - public void addResult(E testInst, Prediction prediction); + /** + * Adds a learning result to this evaluator. + * + * @param example the example to be classified + * @param classVotes an array containing the estimated membership + * probabilities of the test instance in each class + */ + public void addResult(E example, double[] classVotes); + public void addResult(E testInst, Prediction prediction); - /** - * Gets the current measurements monitored by this evaluator. - * - * @return an array of measurements monitored by this evaluator - */ + /** + * Gets the current measurements monitored by this evaluator. + * + * @return an array of measurements monitored by this evaluator + */ public Measurement[] getPerformanceMeasurements(); @Override default ImmutableCapabilities defineImmutableCapabilities() { - return new ImmutableCapabilities(Capability.VIEW_STANDARD); + return new ImmutableCapabilities(Capability.VIEW_STANDARD); } + default void close() throws Exception { + } } diff --git a/moa/src/main/java/moa/gui/SemiSupervisedTabPanel.java b/moa/src/main/java/moa/gui/SemiSupervisedTabPanel.java new file mode 100644 index 000000000..f9dc784e1 --- /dev/null +++ b/moa/src/main/java/moa/gui/SemiSupervisedTabPanel.java @@ -0,0 +1,29 @@ +package moa.gui; + +import java.awt.*; + +public class SemiSupervisedTabPanel extends AbstractTabPanel { + + protected SemiSupervisedTaskManagerPanel taskManagerPanel; + + protected PreviewPanel previewPanel; + + public SemiSupervisedTabPanel() { + this.taskManagerPanel = new SemiSupervisedTaskManagerPanel(); + this.previewPanel = new PreviewPanel(); + this.taskManagerPanel.setPreviewPanel(this.previewPanel); + setLayout(new BorderLayout()); + add(this.taskManagerPanel, BorderLayout.NORTH); + add(this.previewPanel, BorderLayout.CENTER); + } + + @Override + public String getTabTitle() { + return "Semi-Supervised Learning"; + } + + @Override + public String getDescription() { + return "MOA Semi-Supervised Learning"; + } +} diff --git a/moa/src/main/java/moa/gui/SemiSupervisedTaskManagerPanel.java b/moa/src/main/java/moa/gui/SemiSupervisedTaskManagerPanel.java new file mode 100644 index 000000000..0e2f251c6 --- /dev/null +++ b/moa/src/main/java/moa/gui/SemiSupervisedTaskManagerPanel.java @@ -0,0 +1,468 @@ +package moa.gui; + +import moa.core.StringUtils; +import moa.options.ClassOption; +import moa.options.OptionHandler; +import moa.tasks.EvaluateInterleavedTestThenTrainSSLDelayed; +import moa.tasks.SemiSupervisedMainTask; +import moa.tasks.Task; +import moa.tasks.TaskThread; +import nz.ac.waikato.cms.gui.core.BaseFileChooser; + +import javax.swing.*; +import javax.swing.event.ListSelectionEvent; +import javax.swing.event.ListSelectionListener; +import javax.swing.table.AbstractTableModel; +import javax.swing.table.DefaultTableCellRenderer; +import javax.swing.table.TableCellRenderer; +import java.awt.*; +import java.awt.datatransfer.Clipboard; +import java.awt.datatransfer.StringSelection; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.awt.event.MouseAdapter; +import java.awt.event.MouseEvent; +import java.io.*; +import java.util.ArrayList; +import java.util.prefs.Preferences; + +public class SemiSupervisedTaskManagerPanel extends JPanel { + + private static final long serialVersionUID = 1L; + + public static final int MILLISECS_BETWEEN_REFRESH = 600; + + public static String exportFileExtension = "log"; + + public class ProgressCellRenderer extends JProgressBar implements + TableCellRenderer { + + private static final long serialVersionUID = 1L; + + public ProgressCellRenderer() { + super(SwingConstants.HORIZONTAL, 0, 10000); + setBorderPainted(false); + setStringPainted(true); + } + + @Override + public Component getTableCellRendererComponent(JTable table, + Object value, boolean isSelected, boolean hasFocus, int row, + int column) { + double frac = -1.0; + if (value instanceof Double) { + frac = ((Double) value).doubleValue(); + } + if (frac >= 0.0) { + setIndeterminate(false); + setValue((int) (frac * 10000.0)); + setString(StringUtils.doubleToString(frac * 100.0, 2, 2)); + } else { + setValue(0); + } + return this; + } + + @Override + public void validate() { } + + @Override + public void revalidate() { } + + @Override + protected void firePropertyChange(String propertyName, Object oldValue, + Object newValue) { } + + @Override + public void firePropertyChange(String propertyName, boolean oldValue, + boolean newValue) { } + } + + protected class TaskTableModel extends AbstractTableModel { + + private static final long serialVersionUID = 1L; + + @Override + public String getColumnName(int col) { + switch (col) { + case 0: + return "command"; + case 1: + return "status"; + case 2: + return "time elapsed"; + case 3: + return "current activity"; + case 4: + return "% complete"; + } + return null; + } + + @Override + public int getColumnCount() { + return 5; + } + + @Override + public int getRowCount() { + return SemiSupervisedTaskManagerPanel.this.taskList.size(); + } + + @Override + public Object getValueAt(int row, int col) { + TaskThread thread = SemiSupervisedTaskManagerPanel.this.taskList.get(row); + switch (col) { + case 0: + return ((OptionHandler) thread.getTask()).getCLICreationString(SemiSupervisedMainTask.class); + case 1: + return thread.getCurrentStatusString(); + case 2: + return StringUtils.secondsToDHMSString(thread.getCPUSecondsElapsed()); + case 3: + return thread.getCurrentActivityString(); + case 4: + return Double.valueOf(thread.getCurrentActivityFracComplete()); + } + return null; + } + + @Override + public boolean isCellEditable(int row, int col) { + return false; + } + } + + protected SemiSupervisedMainTask currentTask; + + protected java.util.List taskList = new ArrayList<>(); + + protected JButton configureTaskButton = new JButton("Configure"); + + protected JTextField taskDescField = new JTextField(); + + protected JButton runTaskButton = new JButton("Run"); + + protected TaskTableModel taskTableModel; + + protected JTable taskTable; + + protected JButton pauseTaskButton = new JButton("Pause"); + + protected JButton resumeTaskButton = new JButton("Resume"); + + protected JButton cancelTaskButton = new JButton("Cancel"); + + protected JButton deleteTaskButton = new JButton("Delete"); + + protected PreviewPanel previewPanel; + + private Preferences prefs; + + private final String PREF_NAME = "currentTask"; + + public SemiSupervisedTaskManagerPanel() { + // Read current task preference + prefs = Preferences.userRoot().node(this.getClass().getName()); + currentTask = new EvaluateInterleavedTestThenTrainSSLDelayed(); + String taskText = this.currentTask.getCLICreationString(SemiSupervisedMainTask.class); + String propertyValue = prefs.get(PREF_NAME, taskText); + //this.taskDescField.setText(propertyValue); + setTaskString(propertyValue, false); //Not store preference + this.taskDescField.setEditable(false); + + final Component comp = this.taskDescField; + this.taskDescField.addMouseListener(new MouseAdapter() { + + @Override + public void mouseClicked(MouseEvent evt) { + if (evt.getClickCount() == 1) { + if ((evt.getButton() == MouseEvent.BUTTON3) + || ((evt.getButton() == MouseEvent.BUTTON1) && evt.isAltDown() && evt.isShiftDown())) { + JPopupMenu menu = new JPopupMenu(); + JMenuItem item; + + item = new JMenuItem("Copy configuration to clipboard"); + item.addActionListener(new ActionListener() { + + @Override + public void actionPerformed(ActionEvent e) { + copyClipBoardConfiguration(); + } + }); + menu.add(item); + + item = new JMenuItem("Save selected tasks to file"); + item.addActionListener(new ActionListener() { + + @Override + public void actionPerformed(ActionEvent arg0) { + saveLogSelectedTasks(); + } + }); + menu.add(item); + + + item = new JMenuItem("Enter configuration..."); + item.addActionListener(new ActionListener() { + + @Override + public void actionPerformed(ActionEvent arg0) { + String newTaskString = JOptionPane.showInputDialog("Insert command line"); + if (newTaskString != null) { + setTaskString(newTaskString); + } + } + }); + menu.add(item); + + menu.show(comp, evt.getX(), evt.getY()); + } + } + } + }); + + JPanel configPanel = new JPanel(); + configPanel.setLayout(new BorderLayout()); + configPanel.add(this.configureTaskButton, BorderLayout.WEST); + configPanel.add(this.taskDescField, BorderLayout.CENTER); + configPanel.add(this.runTaskButton, BorderLayout.EAST); + this.taskTableModel = new TaskTableModel(); + this.taskTable = new JTable(this.taskTableModel); + DefaultTableCellRenderer centerRenderer = new DefaultTableCellRenderer(); + centerRenderer.setHorizontalAlignment(SwingConstants.CENTER); + this.taskTable.getColumnModel().getColumn(1).setCellRenderer( + centerRenderer); + this.taskTable.getColumnModel().getColumn(2).setCellRenderer( + centerRenderer); + this.taskTable.getColumnModel().getColumn(4).setCellRenderer( + new ProgressCellRenderer()); + JPanel controlPanel = new JPanel(); + controlPanel.add(this.pauseTaskButton); + controlPanel.add(this.resumeTaskButton); + controlPanel.add(this.cancelTaskButton); + controlPanel.add(this.deleteTaskButton); + setLayout(new BorderLayout()); + add(configPanel, BorderLayout.NORTH); + add(new JScrollPane(this.taskTable), BorderLayout.CENTER); + add(controlPanel, BorderLayout.SOUTH); + this.taskTable.getSelectionModel().addListSelectionListener( + new ListSelectionListener() { + + @Override + public void valueChanged(ListSelectionEvent arg0) { + taskSelectionChanged(); + } + }); + this.configureTaskButton.addActionListener(new ActionListener() { + + @Override + public void actionPerformed(ActionEvent arg0) { + String newTaskString = ClassOptionSelectionPanel.showSelectClassDialog( + SemiSupervisedTaskManagerPanel.this, + "Configure task", SemiSupervisedMainTask.class, + SemiSupervisedTaskManagerPanel.this.currentTask.getCLICreationString(SemiSupervisedMainTask.class), + null); + setTaskString(newTaskString); + } + }); + this.runTaskButton.addActionListener(new ActionListener() { + + @Override + public void actionPerformed(ActionEvent arg0) { + runTask((Task) SemiSupervisedTaskManagerPanel.this.currentTask.copy()); + } + }); + this.pauseTaskButton.addActionListener(new ActionListener() { + + @Override + public void actionPerformed(ActionEvent arg0) { + pauseSelectedTasks(); + } + }); + this.resumeTaskButton.addActionListener(new ActionListener() { + + @Override + public void actionPerformed(ActionEvent arg0) { + resumeSelectedTasks(); + } + }); + this.cancelTaskButton.addActionListener(new ActionListener() { + + @Override + public void actionPerformed(ActionEvent arg0) { + cancelSelectedTasks(); + } + }); + this.deleteTaskButton.addActionListener(new ActionListener() { + + @Override + public void actionPerformed(ActionEvent arg0) { + deleteSelectedTasks(); + } + }); + + Timer updateListTimer = new Timer( + MILLISECS_BETWEEN_REFRESH, new ActionListener() { + + @Override + public void actionPerformed(ActionEvent e) { + SemiSupervisedTaskManagerPanel.this.taskTable.repaint(); + } + }); + updateListTimer.start(); + setPreferredSize(new Dimension(0, 200)); + } + + public void setPreviewPanel(PreviewPanel previewPanel) { + this.previewPanel = previewPanel; + } + + public void setTaskString(String cliString) { + setTaskString(cliString, true); + } + + public void setTaskString(String cliString, boolean storePreference) { + try { + this.currentTask = (SemiSupervisedMainTask) ClassOption.cliStringToObject( + cliString, SemiSupervisedMainTask.class, null); + String taskText = this.currentTask.getCLICreationString(SemiSupervisedMainTask.class); + this.taskDescField.setText(taskText); + if (storePreference) { + //Save task text as a preference + prefs.put(PREF_NAME, taskText); + } + } catch (Exception ex) { + GUIUtils.showExceptionDialog(this, "Problem with task", ex); + } + } + + public void runTask(Task task) { + TaskThread thread = new TaskThread(task); + this.taskList.add(0, thread); + this.taskTableModel.fireTableDataChanged(); + this.taskTable.setRowSelectionInterval(0, 0); + thread.start(); + } + + public void taskSelectionChanged() { + TaskThread[] selectedTasks = getSelectedTasks(); + if (selectedTasks.length == 1) { + setTaskString(((OptionHandler) selectedTasks[0].getTask()).getCLICreationString(SemiSupervisedMainTask.class)); + if (this.previewPanel != null) { + this.previewPanel.setTaskThreadToPreview(selectedTasks[0]); + } + } else { + this.previewPanel.setTaskThreadToPreview(null); + } + } + + public TaskThread[] getSelectedTasks() { + int[] selectedRows = this.taskTable.getSelectedRows(); + TaskThread[] selectedTasks = new TaskThread[selectedRows.length]; + for (int i = 0; i < selectedRows.length; i++) { + selectedTasks[i] = this.taskList.get(selectedRows[i]); + } + return selectedTasks; + } + + public void pauseSelectedTasks() { + TaskThread[] selectedTasks = getSelectedTasks(); + for (TaskThread thread : selectedTasks) { + thread.pauseTask(); + } + } + + public void resumeSelectedTasks() { + TaskThread[] selectedTasks = getSelectedTasks(); + for (TaskThread thread : selectedTasks) { + thread.resumeTask(); + } + } + + public void cancelSelectedTasks() { + TaskThread[] selectedTasks = getSelectedTasks(); + for (TaskThread thread : selectedTasks) { + thread.cancelTask(); + } + } + + public void deleteSelectedTasks() { + TaskThread[] selectedTasks = getSelectedTasks(); + for (TaskThread thread : selectedTasks) { + thread.cancelTask(); + this.taskList.remove(thread); + } + this.taskTableModel.fireTableDataChanged(); + } + + public void copyClipBoardConfiguration() { + + StringSelection selection = new StringSelection(this.taskDescField.getText().trim()); + Clipboard clipboard = Toolkit.getDefaultToolkit().getSystemClipboard(); + clipboard.setContents(selection, selection); + + } + + public void saveLogSelectedTasks() { + String tasksLog = ""; + TaskThread[] selectedTasks = getSelectedTasks(); + for (TaskThread thread : selectedTasks) { + tasksLog += ((OptionHandler) thread.getTask()).getCLICreationString(SemiSupervisedMainTask.class) + "\n"; + } + + BaseFileChooser fileChooser = new BaseFileChooser(); + fileChooser.setAcceptAllFileFilterUsed(true); + fileChooser.addChoosableFileFilter(new FileExtensionFilter( + exportFileExtension)); + if (fileChooser.showSaveDialog(this) == BaseFileChooser.APPROVE_OPTION) { + File chosenFile = fileChooser.getSelectedFile(); + String fileName = chosenFile.getPath(); + if (!chosenFile.exists() + && !fileName.endsWith(exportFileExtension)) { + fileName = fileName + "." + exportFileExtension; + } + try { + PrintWriter out = new PrintWriter(new BufferedWriter( + new FileWriter(fileName))); + out.write(tasksLog); + out.close(); + } catch (IOException ioe) { + GUIUtils.showExceptionDialog( + this, + "Problem saving file " + fileName, ioe); + } + } + } + + private static void createAndShowGUI() { + + // Create and set up the labeledInstancesBuffer. + JFrame frame = new JFrame("Test"); + frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); + + // Create and set up the content pane. + JPanel panel = new SemiSupervisedTabPanel(); + panel.setOpaque(true); // content panes must be opaque + frame.setContentPane(panel); + + // Display the labeledInstancesBuffer. + frame.pack(); + // frame.setSize(400, 400); + frame.setVisible(true); + } + + public static void main(String[] args) { + try { + UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName()); + SwingUtilities.invokeLater(new Runnable() { + @Override + public void run() { + createAndShowGUI(); + } + }); + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/moa/src/main/java/moa/learners/Learner.java b/moa/src/main/java/moa/learners/Learner.java index be959a8d0..a806ad587 100644 --- a/moa/src/main/java/moa/learners/Learner.java +++ b/moa/src/main/java/moa/learners/Learner.java @@ -19,14 +19,10 @@ */ package moa.learners; +import com.yahoo.labs.samoa.instances.*; import moa.MOAObject; import moa.core.Example; -import com.yahoo.labs.samoa.instances.InstanceData; -import com.yahoo.labs.samoa.instances.InstancesHeader; -import com.yahoo.labs.samoa.instances.MultiLabelInstance; -import com.yahoo.labs.samoa.instances.Prediction; - import moa.core.Measurement; import moa.gui.AWTRenderable; import moa.options.OptionHandler; @@ -95,6 +91,14 @@ public interface Learner extends MOAObject, OptionHandler, AW */ public double[] getVotesForInstance(E example); + /** + * + * @param example the instance whose confidence we are observing + * @param label + * @return + */ + public double getConfidenceForPrediction(E example, double label); + /** * Gets the current measurements of this learner. * diff --git a/moa/src/main/java/moa/tasks/EvaluateInterleavedTestThenTrainSSLDelayed.java b/moa/src/main/java/moa/tasks/EvaluateInterleavedTestThenTrainSSLDelayed.java new file mode 100644 index 000000000..b5b02904a --- /dev/null +++ b/moa/src/main/java/moa/tasks/EvaluateInterleavedTestThenTrainSSLDelayed.java @@ -0,0 +1,351 @@ +package moa.tasks; + +import com.github.javacliparser.FileOption; +import com.github.javacliparser.FlagOption; +import com.github.javacliparser.FloatOption; +import com.github.javacliparser.IntOption; +import com.yahoo.labs.samoa.instances.Instance; +import moa.classifiers.MultiClassClassifier; +import moa.classifiers.SemiSupervisedLearner; +import moa.core.*; +import moa.evaluation.LearningEvaluation; +import moa.evaluation.LearningPerformanceEvaluator; +import moa.evaluation.preview.LearningCurve; +import moa.learners.Learner; +import moa.options.ClassOption; +import moa.streams.ExampleStream; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.commons.math3.random.MersenneTwister; +import org.apache.commons.math3.random.RandomGenerator; + +/** + * An evaluation task that relies on the mechanism of Interleaved Test Then + * Train, + * applied on semi-supervised data streams + */ +public class EvaluateInterleavedTestThenTrainSSLDelayed extends SemiSupervisedMainTask { + + @Override + public String getPurposeString() { + return "Evaluates a classifier on a semi-supervised stream by testing only the labeled data, " + + "then training with each example in sequence."; + } + + private static final long serialVersionUID = 1L; + + public IntOption randomSeedOption = new IntOption( + "instanceRandomSeed", 'r', + "Seed for random generation of instances.", 1); + + public FlagOption onlyLabeledDataOption = new FlagOption("labeledDataOnly", 'a', + "Learner only trained on labeled data"); + + public ClassOption standardLearnerOption = new ClassOption("standardLearner", 'b', + "A standard learner to train. This will be ignored if labeledDataOnly flag is not set.", + MultiClassClassifier.class, "moa.classifiers.trees.HoeffdingTree"); + + public ClassOption sslLearnerOption = new ClassOption("sslLearner", 'l', + "A semi-supervised learner to train.", SemiSupervisedLearner.class, + "moa.classifiers.semisupervised.ClusterAndLabelClassifier"); + + public ClassOption streamOption = new ClassOption("stream", 's', + "Stream to learn from.", ExampleStream.class, + "moa.streams.ArffFileStream"); + + public ClassOption evaluatorOption = new ClassOption("evaluator", 'e', + "Classification performance evaluation method.", + LearningPerformanceEvaluator.class, + "BasicClassificationPerformanceEvaluator"); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + /** Option: Probability of instance being unlabeled */ + public FloatOption labelProbabilityOption = new FloatOption("labelProbability", 'j', + "The ratio of labeled data", + 0.01); + + public IntOption delayLengthOption = new IntOption("delay", 'k', + "Number of instances before test instance is used for training. -1 = no delayed labeling.", + -1, -1, Integer.MAX_VALUE); + + public IntOption initialWindowSizeOption = new IntOption("initialTrainingWindow", 'p', + "Number of instances used for training in the beginning of the stream (-1 = no initialWindow).", + -1, -1, Integer.MAX_VALUE); + + public FlagOption debugPseudoLabelsOption = new FlagOption("debugPseudoLabels", 'w', + "Learner also receives the labeled data, but it is not used for training (just for statistics)"); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + public IntOption instanceLimitOption = new IntOption("instanceLimit", 'i', + "Maximum number of instances to test/train on (-1 = no limit).", + 100000000, -1, Integer.MAX_VALUE); + + public IntOption timeLimitOption = new IntOption("timeLimit", 't', + "Maximum number of seconds to test/train for (-1 = no limit).", -1, + -1, Integer.MAX_VALUE); + + public IntOption sampleFrequencyOption = new IntOption("sampleFrequency", + 'f', + "How many instances between samples of the learning performance.", + 100000, 0, Integer.MAX_VALUE); + + public IntOption memCheckFrequencyOption = new IntOption( + "memCheckFrequency", 'q', + "How many instances between memory bound checks.", 100000, 0, + Integer.MAX_VALUE); + + public FileOption dumpFileOption = new FileOption("dumpFile", 'd', + "File to append intermediate csv results to.", null, "csv", true); + + public FileOption outputPredictionFileOption = new FileOption("outputPredictionFile", 'o', + "File to append output predictions to.", null, "pred", true); + + public FileOption debugOutputUnlabeledClassInformation = new FileOption("debugOutputUnlabeledClassInformation", 'h', + "Single column containing the class label or -999 indicating missing labels.", null, "csv", true); + + private int numUnlabeledData = 0; + + private Learner getLearner(ExampleStream stream) { + Learner learner; + if (this.onlyLabeledDataOption.isSet()) { + learner = (Learner) getPreparedClassOption(this.standardLearnerOption); + } else { + learner = (SemiSupervisedLearner) getPreparedClassOption(this.sslLearnerOption); + } + + learner.setModelContext(stream.getHeader()); + if (learner.isRandomizable()) { + learner.setRandomSeed(this.randomSeedOption.getValue()); + learner.resetLearning(); + } + return learner; + } + + private String getLearnerString() { + if (this.onlyLabeledDataOption.isSet()) { + return this.standardLearnerOption.getValueAsCLIString(); + } else { + return this.sslLearnerOption.getValueAsCLIString(); + } + } + + private PrintStream newPrintStream(File f, String err_msg) { + if (f == null) + return null; + try { + return new PrintStream(new FileOutputStream(f, f.exists()), true); + } catch (FileNotFoundException e) { + throw new RuntimeException(err_msg, e); + } + } + + private Object internalDoMainTask(TaskMonitor monitor, ObjectRepository repository, LearningPerformanceEvaluator evaluator) + { + int maxInstances = this.instanceLimitOption.getValue(); + int maxSeconds = this.timeLimitOption.getValue(); + int delayLength = this.delayLengthOption.getValue(); + double labelProbability = this.labelProbabilityOption.getValue(); + String streamString = this.streamOption.getValueAsCLIString(); + RandomGenerator taskRandom = new MersenneTwister(this.randomSeedOption.getValue()); + ExampleStream stream = (ExampleStream) getPreparedClassOption(this.streamOption); + Learner learner = getLearner(stream); + String learnerString = getLearnerString(); + + // A number of output files used for debugging and manual evaluation + PrintStream dumpStream = newPrintStream(this.dumpFileOption.getFile(), "Failed to create dump file"); + PrintStream predStream = newPrintStream(this.outputPredictionFileOption.getFile(), + "Failed to create prediction file"); + PrintStream labelStream = newPrintStream(this.debugOutputUnlabeledClassInformation.getFile(), + "Failed to create unlabeled class information file"); + if (labelStream != null) + labelStream.println("class"); + + // Setup evaluation + monitor.setCurrentActivity("Evaluating learner...", -1.0); + LearningCurve learningCurve = new LearningCurve("learning evaluation instances"); + + boolean firstDump = true; + boolean preciseCPUTiming = TimingUtils.enablePreciseTiming(); + long evaluateStartTime = TimingUtils.getNanoCPUTimeOfCurrentThread(); + long lastEvaluateStartTime = evaluateStartTime; + long instancesProcessed = 0; + int secondsElapsed = 0; + double RAMHours = 0.0; + + // The buffer is a list of tuples. The first element is the index when + // it should be emitted. The second element is the instance itself. + List> delayBuffer = new ArrayList>(); + + while (stream.hasMoreInstances() + && ((maxInstances < 0) || (instancesProcessed < maxInstances)) + && ((maxSeconds < 0) || (secondsElapsed < maxSeconds))) { + instancesProcessed++; + + // TRAIN on delayed instances + while (delayBuffer.size() > 0 + && delayBuffer.get(0).getKey() == instancesProcessed) { + Example delayedExample = delayBuffer.remove(0).getValue(); + learner.trainOnInstance(delayedExample); + } + + // Obtain the next Example from the stream. + // The instance is expected to be labeled. + Example originalExample = stream.nextInstance(); + Example unlabeledExample = originalExample.copy(); + int trueClass = (int) ((Instance) originalExample.getData()).classValue(); + + // In case it is set, then the label is not removed. We want to pass the + // labelled data to the learner even in trainOnUnlabeled data to generate statistics such as number + // of correctly pseudo-labeled instances. + if (!debugPseudoLabelsOption.isSet()) { + // Remove the label of the unlabeledExample indirectly through + // unlabeledInstanceData. + Instance instance = (Instance) unlabeledExample.getData(); + instance.setMissing(instance.classIndex()); + } + + // WARMUP + // Train on the initial instances. These are not used for testing! + if (instancesProcessed <= this.initialWindowSizeOption.getValue()) { + if (learner instanceof SemiSupervisedLearner) + ((SemiSupervisedLearner) learner).addInitialWarmupTrainingInstances(); + learner.trainOnInstance(originalExample); + continue; + } + + Boolean is_labeled = labelProbability > taskRandom.nextDouble(); + if (!is_labeled) { + this.numUnlabeledData++; + if (labelStream != null) + labelStream.println(-999); + } else { + if (labelStream != null) + labelStream.println((int) trueClass); + } + + // TEST + // Obtain the prediction for the testInst (i.e. no label) + double[] prediction = learner.getVotesForInstance(unlabeledExample); + + // Output prediction + if (predStream != null) { + // Assuming that the class label is not missing for the originalInstanceData + predStream.println(Utils.maxIndex(prediction) + "," + trueClass); + } + evaluator.addResult(originalExample, prediction); + + // TRAIN + if (is_labeled && delayLength >= 0) { + // The instance will be labeled but has been delayed + if (learner instanceof SemiSupervisedLearner) + { + ((SemiSupervisedLearner) learner).trainOnUnlabeledInstance((Instance) unlabeledExample.getData()); + } + delayBuffer.add( + new MutablePair(1 + instancesProcessed + delayLength, originalExample)); + } else if (is_labeled) { + // The instance will be labeled and is not delayed e.g delayLength = -1 + learner.trainOnInstance(originalExample); + } else { + // The instance will never be labeled + if (learner instanceof SemiSupervisedLearner) + ((SemiSupervisedLearner) learner).trainOnUnlabeledInstance((Instance) unlabeledExample.getData()); + } + + if (instancesProcessed % this.sampleFrequencyOption.getValue() == 0 || !stream.hasMoreInstances()) { + long evaluateTime = TimingUtils.getNanoCPUTimeOfCurrentThread(); + double time = TimingUtils.nanoTimeToSeconds(evaluateTime - evaluateStartTime); + double timeIncrement = TimingUtils.nanoTimeToSeconds(evaluateTime - lastEvaluateStartTime); + double RAMHoursIncrement = learner.measureByteSize() / (1024.0 * 1024.0 * 1024.0); // GBs + RAMHoursIncrement *= (timeIncrement / 3600.0); // Hours + RAMHours += RAMHoursIncrement; + lastEvaluateStartTime = evaluateTime; + learningCurve.insertEntry(new LearningEvaluation( + new Measurement[] { + new Measurement( + "learning evaluation instances", + instancesProcessed), + new Measurement( + "evaluation time (" + + (preciseCPUTiming ? "cpu " + : "") + + "seconds)", + time), + new Measurement( + "model cost (RAM-Hours)", + RAMHours), + new Measurement( + "Unlabeled instances", + this.numUnlabeledData) + }, + evaluator, learner)); + if (dumpStream != null) { + if (firstDump) { + dumpStream.print("Learner,stream,randomSeed,"); + dumpStream.println(learningCurve.headerToString()); + firstDump = false; + } + dumpStream.print(learnerString + "," + streamString + "," + + this.randomSeedOption.getValueAsCLIString() + ","); + dumpStream.println(learningCurve.entryToString(learningCurve.numEntries() - 1)); + dumpStream.flush(); + } + } + if (instancesProcessed % INSTANCES_BETWEEN_MONITOR_UPDATES == 0) { + if (monitor.taskShouldAbort()) { + return null; + } + long estimatedRemainingInstances = stream.estimatedRemainingInstances(); + if (maxInstances > 0) { + long maxRemaining = maxInstances - instancesProcessed; + if ((estimatedRemainingInstances < 0) + || (maxRemaining < estimatedRemainingInstances)) { + estimatedRemainingInstances = maxRemaining; + } + } + monitor.setCurrentActivityFractionComplete(estimatedRemainingInstances < 0 ? -1.0 + : (double) instancesProcessed / (double) (instancesProcessed + estimatedRemainingInstances)); + if (monitor.resultPreviewRequested()) { + monitor.setLatestResultPreview(learningCurve.copy()); + } + secondsElapsed = (int) TimingUtils.nanoTimeToSeconds(TimingUtils.getNanoCPUTimeOfCurrentThread() + - evaluateStartTime); + } + } + if (dumpStream != null) { + dumpStream.close(); + } + if (predStream != null) { + predStream.close(); + } + return learningCurve; + } + + @Override + protected Object doMainTask(TaskMonitor monitor, ObjectRepository repository) { + // Some resource must be closed at the end of the task + try ( + LearningPerformanceEvaluator evaluator = (LearningPerformanceEvaluator) getPreparedClassOption(this.evaluatorOption) + ) { + return internalDoMainTask(monitor, repository, evaluator); + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + + @Override + public Class getTaskResultType() { + return LearningCurve.class; + } +} diff --git a/moa/src/main/java/moa/tasks/SemiSupervisedMainTask.java b/moa/src/main/java/moa/tasks/SemiSupervisedMainTask.java new file mode 100644 index 000000000..fecf7feae --- /dev/null +++ b/moa/src/main/java/moa/tasks/SemiSupervisedMainTask.java @@ -0,0 +1,24 @@ +package moa.tasks; + +import moa.streams.clustering.ClusterEvent; + +import java.util.ArrayList; + +/** + * + */ +public abstract class SemiSupervisedMainTask extends MainTask { + + private static final long serialVersionUID = 1L; + + protected ArrayList events; + + protected void setEventsList(ArrayList events) { + this.events = events; + } + + public ArrayList getEventsList() { + return this.events; + } + +} diff --git a/moa/src/main/resources/moa/gui/GUI.props b/moa/src/main/resources/moa/gui/GUI.props index d1deabd85..3bb990469 100644 --- a/moa/src/main/resources/moa/gui/GUI.props +++ b/moa/src/main/resources/moa/gui/GUI.props @@ -8,6 +8,7 @@ Tabs=\ moa.gui.ClassificationTabPanel,\ moa.gui.RegressionTabPanel,\ + moa.gui.SemiSupervisedTabPanel,\ moa.gui.MultiLabelTabPanel,\ moa.gui.MultiTargetTabPanel,\ moa.gui.clustertab.ClusteringTabPanel,\ From cb9dc58ae9c27231feefab1481ddc877d51a378e Mon Sep 17 00:00:00 2001 From: Spencer Sun Date: Mon, 20 May 2024 14:42:04 +1200 Subject: [PATCH 2/9] fix: add storing functionality in EfficientEvaluationLoops --- .../evaluation/EfficientEvaluationLoops.java | 52 ++++++++++++++----- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/moa/src/main/java/moa/evaluation/EfficientEvaluationLoops.java b/moa/src/main/java/moa/evaluation/EfficientEvaluationLoops.java index 250aa3642..719c18ead 100644 --- a/moa/src/main/java/moa/evaluation/EfficientEvaluationLoops.java +++ b/moa/src/main/java/moa/evaluation/EfficientEvaluationLoops.java @@ -4,6 +4,7 @@ import moa.classifiers.SemiSupervisedLearner; import moa.classifiers.semisupervised.ClusterAndLabelClassifier; import moa.core.Example; +import moa.core.InstanceExample; import moa.core.Measurement; import moa.learners.Learner; import moa.streams.ArffFileStream; @@ -25,12 +26,24 @@ public class EfficientEvaluationLoops { public static class PrequentialResult { public ArrayList windowedResults; public double[] cumulativeResults; + public ArrayList targets; + public ArrayList predictions; public HashMap otherMeasurements; public PrequentialResult(ArrayList windowedResults, double[] cumulativeResults) { this.windowedResults = windowedResults; this.cumulativeResults = cumulativeResults; + this.targets = null; + this.predictions = null; + } + + public PrequentialResult(ArrayList windowedResults, double[] cumulativeResults, + ArrayList targets, ArrayList predictions) { + this.windowedResults = windowedResults; + this.cumulativeResults = cumulativeResults; + this.targets = targets; + this.predictions = predictions; } public PrequentialResult(ArrayList windowedResults, double[] cumulativeResults, @@ -57,18 +70,24 @@ public PrequentialResult(ArrayList windowedResults, double[] cumulativ public static PrequentialResult PrequentialEvaluation(ExampleStream stream, Learner learner, LearningPerformanceEvaluator basicEvaluator, LearningPerformanceEvaluator windowedEvaluator, - long maxInstances, long windowSize) { + long maxInstances, long windowSize, + boolean storeY, boolean storePredictions) { int instancesProcessed = 0; if (!stream.hasMoreInstances()) stream.restart(); ArrayList windowed_results = new ArrayList<>(); + ArrayList targetValues = new ArrayList<>(); + ArrayList predictions = new ArrayList<>(); + while (stream.hasMoreInstances() && (maxInstances == -1 || instancesProcessed < maxInstances)) { - Example instance = stream.nextInstance(); + Example instance = stream.nextInstance(); + if (storeY) + targetValues.add(instance.getData().classValue()); double[] prediction = learner.getVotesForInstance(instance); if (basicEvaluator != null) @@ -76,6 +95,9 @@ public static PrequentialResult PrequentialEvaluation(ExampleStream stream, Lear if (windowedEvaluator != null) windowedEvaluator.addResult(instance, prediction); + if (storePredictions) + predictions.add(prediction.length == 0? 0 : prediction[0]); + learner.trainOnInstance(instance); instancesProcessed++; @@ -106,8 +128,10 @@ public static PrequentialResult PrequentialEvaluation(ExampleStream stream, Lear for (int i = 0; i < cumulative_results.length; ++i) cumulative_results[i] = measurements[i].getValue(); } - - return new PrequentialResult(windowed_results, cumulative_results); + if (!storePredictions && !storeY) + return new PrequentialResult(windowed_results, cumulative_results); + else + return new PrequentialResult(windowed_results, cumulative_results, targetValues, predictions); } public static PrequentialResult PrequentialSSLEvaluation(ExampleStream stream, Learner learner, @@ -379,7 +403,7 @@ private static void testPrequentialEfficiency1(Learner learner) { basic_evaluator.recallPerClassOption.setValue(true); basic_evaluator.prepareForUse(); - PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, null, 100000, 1); + PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, null, 100000, 1, false, false); // Record the end time long endTime = System.currentTimeMillis(); @@ -413,7 +437,7 @@ private static void testPrequentialEvaluation_edge_cases1() { // windowed_evaluator.widthOption.setValue(1000); // windowed_evaluator.prepareForUse(); - PrequentialResult results = PrequentialEvaluation(stream, learner, null, null, 100000, 1000); + PrequentialResult results = PrequentialEvaluation(stream, learner, null, null, 100000, 1000, false, false); // Record the end time long endTime = System.currentTimeMillis(); @@ -455,7 +479,7 @@ private static void testPrequentialEvaluation_edge_cases2() { windowed_evaluator.widthOption.setValue(1000); windowed_evaluator.prepareForUse(); - PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, windowed_evaluator, 1000, 10000); + PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, windowed_evaluator, 1000, 10000, false, false); // Record the end time long endTime = System.currentTimeMillis(); @@ -496,7 +520,7 @@ private static void testPrequentialEvaluation_edge_cases3() { windowed_evaluator.widthOption.setValue(1000); windowed_evaluator.prepareForUse(); - PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, windowed_evaluator, 10, 1); + PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, windowed_evaluator, 10, 1, false, false); // Record the end time long endTime = System.currentTimeMillis(); @@ -537,7 +561,7 @@ private static void testPrequentialEvaluation_edge_cases4() { windowed_evaluator.widthOption.setValue(10000); windowed_evaluator.prepareForUse(); - PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, windowed_evaluator, -1, 10000); + PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, windowed_evaluator, -1, 10000, false, false); // Record the end time long endTime = System.currentTimeMillis(); @@ -579,7 +603,7 @@ private static void testPrequentialEvaluation_SampleFrequency_TestThenTrain() { // windowed_evaluator.widthOption.setValue(10000); // windowed_evaluator.prepareForUse(); - PrequentialResult results = PrequentialEvaluation(stream, learner, null, basic_evaluator, -1, 10000); + PrequentialResult results = PrequentialEvaluation(stream, learner, null, basic_evaluator, -1, 10000, false, false); // Record the end time long endTime = System.currentTimeMillis(); @@ -621,7 +645,7 @@ private static void testPrequentialRegressionEvaluation() { windowed_evaluator.widthOption.setValue(1000); // windowed_evaluator.prepareForUse(); - PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, windowed_evaluator, 100000, 1000); + PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, windowed_evaluator, 100000, 1000, false, false); // Record the end time long endTime = System.currentTimeMillis(); @@ -664,7 +688,7 @@ private static void testPrequentialEvaluation() { windowed_evaluator.widthOption.setValue(1000); windowed_evaluator.prepareForUse(); - PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, windowed_evaluator, 100000, 1000); + PrequentialResult results = PrequentialEvaluation(stream, learner, basic_evaluator, windowed_evaluator, 100000, 1000, false, false); // Record the end time long endTime = System.currentTimeMillis(); @@ -703,7 +727,7 @@ private static void testTestThenTrainEvaluation() { evaluator.recallPerClassOption.setValue(true); evaluator.prepareForUse(); - PrequentialResult results = PrequentialEvaluation(stream, learner, evaluator, null, 100000, 100000); + PrequentialResult results = PrequentialEvaluation(stream, learner, evaluator, null, 100000, 100000, false, false); // Record the end time long endTime = System.currentTimeMillis(); @@ -735,7 +759,7 @@ private static void testWindowedEvaluation() { evaluator.recallPerClassOption.setValue(true); evaluator.prepareForUse(); - PrequentialResult results = PrequentialEvaluation(stream, learner, null, evaluator, 100000, 10000); + PrequentialResult results = PrequentialEvaluation(stream, learner, null, evaluator, 100000, 10000, false, false); // Record the end time long endTime = System.currentTimeMillis(); From 876585af74e2befe5b0e16d974a79baaa2fde8d4 Mon Sep 17 00:00:00 2001 From: Spencer Sun Date: Tue, 11 Jun 2024 14:23:11 +1200 Subject: [PATCH 3/9] fix: fix the instance index for window regression evaluation --- .../moa/evaluation/WindowRegressionPerformanceEvaluator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/moa/src/main/java/moa/evaluation/WindowRegressionPerformanceEvaluator.java b/moa/src/main/java/moa/evaluation/WindowRegressionPerformanceEvaluator.java index eb9745a08..23e977598 100644 --- a/moa/src/main/java/moa/evaluation/WindowRegressionPerformanceEvaluator.java +++ b/moa/src/main/java/moa/evaluation/WindowRegressionPerformanceEvaluator.java @@ -175,6 +175,7 @@ public double getCoefficientOfDetermination() { return 0.0; } + public double getAdjustedCoefficientOfDetermination() { return 1 - ((1-getCoefficientOfDetermination())*(getTotalWeightObserved() - 1)) / (getTotalWeightObserved() - numAttributes - 1); @@ -197,6 +198,7 @@ private double getRelativeSquareError() { } public double getTotalWeightObserved() { +// return this.weightObserved.total(); return this.TotalweightObserved; } From 360d8d497b9b78848bece925c2233b4797614b8a Mon Sep 17 00:00:00 2001 From: Anton Lee Date: Fri, 3 Oct 2025 13:05:35 +1300 Subject: [PATCH 4/9] fix storePredictions and storeY --- .github/workflows/capymoa.yml | 35 +++++ .gitignore | 27 ++++ README.md | 2 - .../evaluation/EfficientEvaluationLoops.java | 131 ++++++++++++------ 4 files changed, 147 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/capymoa.yml diff --git a/.github/workflows/capymoa.yml b/.github/workflows/capymoa.yml new file mode 100644 index 000000000..d71a88f91 --- /dev/null +++ b/.github/workflows/capymoa.yml @@ -0,0 +1,35 @@ +name: Package Jar for CapyMOA + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + cache: maven + + - name: Package Jar + working-directory: ./moa + # Package as JAR while skipping tests, javadoc, and latex build + run: mvn package -DskipTests -Dmaven.javadoc.skip=true -Dlatex.skipBuild=true + + # Upload jar file as artifact + - name: Upload Jar + uses: actions/upload-artifact@v4 + with: + name: moa-jar + path: ./moa/target/moa-*-jar-with-dependencies.jar + if-no-files-found: error + retention-days: 7 diff --git a/.gitignore b/.gitignore index 3bd1f62db..47c1c1e7c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,31 @@ *.iml *~ *.bak +.DS_Store +.settings +.project +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.nar +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* +replay_pid* diff --git a/README.md b/README.md index ca33590d2..6bcb48a19 100755 --- a/README.md +++ b/README.md @@ -30,5 +30,3 @@ If you want to refer to MOA in a publication, please cite the following JMLR pap > Albert Bifet, Geoff Holmes, Richard Kirkby, Bernhard Pfahringer (2010); > MOA: Massive Online Analysis; Journal of Machine Learning Research 11: 1601-1604 - - diff --git a/moa/src/main/java/moa/evaluation/EfficientEvaluationLoops.java b/moa/src/main/java/moa/evaluation/EfficientEvaluationLoops.java index 719c18ead..73a1b405c 100644 --- a/moa/src/main/java/moa/evaluation/EfficientEvaluationLoops.java +++ b/moa/src/main/java/moa/evaluation/EfficientEvaluationLoops.java @@ -4,8 +4,8 @@ import moa.classifiers.SemiSupervisedLearner; import moa.classifiers.semisupervised.ClusterAndLabelClassifier; import moa.core.Example; -import moa.core.InstanceExample; import moa.core.Measurement; +import moa.core.Utils; import moa.learners.Learner; import moa.streams.ArffFileStream; import moa.streams.ExampleStream; @@ -26,30 +26,39 @@ public class EfficientEvaluationLoops { public static class PrequentialResult { public ArrayList windowedResults; public double[] cumulativeResults; - public ArrayList targets; - public ArrayList predictions; - + public ArrayList targets; + public ArrayList predictions; public HashMap otherMeasurements; - public PrequentialResult(ArrayList windowedResults, double[] cumulativeResults) { - this.windowedResults = windowedResults; - this.cumulativeResults = cumulativeResults; - this.targets = null; - this.predictions = null; - } - - public PrequentialResult(ArrayList windowedResults, double[] cumulativeResults, - ArrayList targets, ArrayList predictions) { + public PrequentialResult( + ArrayList windowedResults, + double[] cumulativeResults, + ArrayList targets, + ArrayList predictions, + HashMap otherMeasurements + ) { this.windowedResults = windowedResults; this.cumulativeResults = cumulativeResults; this.targets = targets; this.predictions = predictions; + this.otherMeasurements = otherMeasurements; } - public PrequentialResult(ArrayList windowedResults, double[] cumulativeResults, - HashMap otherMeasurements) { - this(windowedResults, cumulativeResults); - this.otherMeasurements = otherMeasurements; + public PrequentialResult( + ArrayList windowedResults, + double[] cumulativeResults, + ArrayList targets, + ArrayList predictions + ) { + this(windowedResults, cumulativeResults, targets, predictions, null); + } + + public PrequentialResult( + ArrayList windowedResults, + double[] cumulativeResults, + HashMap otherMeasurements + ) { + this(windowedResults, cumulativeResults, null, null, otherMeasurements); } } @@ -65,11 +74,13 @@ public PrequentialResult(ArrayList windowedResults, double[] cumulativ * @param windowedEvaluator * @param maxInstances * @param windowSize + * @param storeY + * @param storePredictions * @return the return has to be an ArrayList because we don't know ahead of time how many windows will be produced */ public static PrequentialResult PrequentialEvaluation(ExampleStream stream, Learner learner, - LearningPerformanceEvaluator basicEvaluator, - LearningPerformanceEvaluator windowedEvaluator, + LearningPerformanceEvaluator> basicEvaluator, + LearningPerformanceEvaluator> windowedEvaluator, long maxInstances, long windowSize, boolean storeY, boolean storePredictions) { int instancesProcessed = 0; @@ -78,30 +89,31 @@ public static PrequentialResult PrequentialEvaluation(ExampleStream stream, Lear stream.restart(); ArrayList windowed_results = new ArrayList<>(); - ArrayList targetValues = new ArrayList<>(); - ArrayList predictions = new ArrayList<>(); + ArrayList targetValues = new ArrayList<>(); + ArrayList predictions = new ArrayList<>(); while (stream.hasMoreInstances() && (maxInstances == -1 || instancesProcessed < maxInstances)) { Example instance = stream.nextInstance(); - if (storeY) - targetValues.add(instance.getData().classValue()); double[] prediction = learner.getVotesForInstance(instance); + + // Update evaluators and store predictions if requested if (basicEvaluator != null) basicEvaluator.addResult(instance, prediction); if (windowedEvaluator != null) windowedEvaluator.addResult(instance, prediction); - if (storePredictions) - predictions.add(prediction.length == 0? 0 : prediction[0]); + predictions.add(Utils.maxIndex(prediction)); + if (storeY) + targetValues.add((int)Math.round(instance.getData().classValue())); learner.trainOnInstance(instance); - instancesProcessed++; + // Store windowed results if requested if (windowedEvaluator != null) if (instancesProcessed % windowSize == 0) { Measurement[] measurements = windowedEvaluator.getPerformanceMeasurements(); @@ -128,22 +140,30 @@ public static PrequentialResult PrequentialEvaluation(ExampleStream stream, Lear for (int i = 0; i < cumulative_results.length; ++i) cumulative_results[i] = measurements[i].getValue(); } - if (!storePredictions && !storeY) - return new PrequentialResult(windowed_results, cumulative_results); - else - return new PrequentialResult(windowed_results, cumulative_results, targetValues, predictions); + + return new PrequentialResult( + windowed_results, + cumulative_results, + targetValues, + predictions + ); } - public static PrequentialResult PrequentialSSLEvaluation(ExampleStream stream, Learner learner, - LearningPerformanceEvaluator basicEvaluator, - LearningPerformanceEvaluator windowedEvaluator, - long maxInstances, - long windowSize, - long initialWindowSize, - long delayLength, - double labelProbability, - int randomSeed, - boolean debugPseudoLabels) { + public static PrequentialResult PrequentialSSLEvaluation( + ExampleStream> stream, + Learner learner, + LearningPerformanceEvaluator basicEvaluator, + LearningPerformanceEvaluator windowedEvaluator, + long maxInstances, + long windowSize, + long initialWindowSize, + long delayLength, + double labelProbability, + int randomSeed, + boolean debugPseudoLabels, + boolean storeY, + boolean storePredictions + ) { // int delayLength = this.delayLengthOption.getValue(); // double labelProbability = this.labelProbabilityOption.getValue(); @@ -161,11 +181,13 @@ public static PrequentialResult PrequentialSSLEvaluation(ExampleStream stream, L ArrayList windowed_results = new ArrayList<>(); + ArrayList targetValues = new ArrayList<>(); + ArrayList predictions = new ArrayList<>(); HashMap other_measures = new HashMap<>(); // The buffer is a list of tuples. The first element is the index when // it should be emitted. The second element is the instance itself. - List> delayBuffer = new ArrayList>(); + List>> delayBuffer = new ArrayList>>(); while (stream.hasMoreInstances() && (maxInstances == -1 || instancesProcessed < maxInstances)) { @@ -178,8 +200,8 @@ public static PrequentialResult PrequentialSSLEvaluation(ExampleStream stream, L learner.trainOnInstance(delayedExample); } - Example instance = stream.nextInstance(); - Example unlabeledExample = instance.copy(); + Example instance = stream.nextInstance(); + Example unlabeledExample = instance.copy(); int trueClass = (int) ((Instance) instance.getData()).classValue(); // In case it is set, then the label is not removed. We want to pass the @@ -218,6 +240,10 @@ public static PrequentialResult PrequentialSSLEvaluation(ExampleStream stream, L basicEvaluator.addResult(instance, prediction); if (windowedEvaluator != null) windowedEvaluator.addResult(instance, prediction); + if (storeY) + targetValues.add((int)Math.round(instance.getData().classValue())); + if (storePredictions) + predictions.add(Utils.maxIndex(prediction)); int pseudoLabel = -1; // TRAIN @@ -227,7 +253,7 @@ public static PrequentialResult PrequentialSSLEvaluation(ExampleStream stream, L // System.out.println("[TRAIN_UNLABELED][DELAYED] " + unlabeledExample.getData().toString()); pseudoLabel = ((SemiSupervisedLearner) learner).trainOnUnlabeledInstance((Instance) unlabeledExample.getData()); } - delayBuffer.add(new MutablePair(1 + instancesProcessed + delayLength, instance)); + delayBuffer.add(new MutablePair<>(1 + instancesProcessed + delayLength, instance)); } else if (is_labeled) { // System.out.println("[TRAIN] " + instance.getData().toString()); // The instance will be labeled and is not delayed e.g delayLength = -1 @@ -276,7 +302,15 @@ public static PrequentialResult PrequentialSSLEvaluation(ExampleStream stream, L other_measures.put("num_correct_pseudo_labeled", (double) numCorrectPseudoLabeled); other_measures.put("num_instances_tested", (double) numInstancesTested); other_measures.put("pseudo_label_accuracy", (double) numCorrectPseudoLabeled/numInstancesTested); - return new PrequentialResult(windowed_results, cumulative_results, other_measures); + + + return new PrequentialResult( + windowed_results, + cumulative_results, + targetValues, + predictions, + other_measures + ); } /******************************************************************************************************************/ @@ -320,7 +354,12 @@ private static void testPrequentialSSL(String file_path, Learner learner, windowSize, initialWindowSize, delayLength, - labelProbability, 1, true); + labelProbability, + 1, + true, + false, + false + ); // Record the end time long endTime = System.currentTimeMillis(); From a38153452b14f4c5809b63361be0f9843c5a0c78 Mon Sep 17 00:00:00 2001 From: Anton Lee Date: Fri, 17 Oct 2025 11:37:39 +1300 Subject: [PATCH 5/9] fix capymoa packaging --- .github/workflows/capymoa.yml | 2 +- CapyMOA.md | 29 +++++++++++++++++++++++++++++ moa/pom.xml | 10 ++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 CapyMOA.md diff --git a/.github/workflows/capymoa.yml b/.github/workflows/capymoa.yml index d71a88f91..290898ff2 100644 --- a/.github/workflows/capymoa.yml +++ b/.github/workflows/capymoa.yml @@ -23,7 +23,7 @@ jobs: - name: Package Jar working-directory: ./moa # Package as JAR while skipping tests, javadoc, and latex build - run: mvn package -DskipTests -Dmaven.javadoc.skip=true -Dlatex.skipBuild=true + run: mvn -B package -DskipTests -Dmaven.javadoc.skip=true -Dlatex.skipBuild=true # Upload jar file as artifact - name: Upload Jar diff --git a/CapyMOA.md b/CapyMOA.md new file mode 100644 index 000000000..23f33a639 --- /dev/null +++ b/CapyMOA.md @@ -0,0 +1,29 @@ +# CapyMOA + +CapyMOA is a datastream learning framework that integrates the [Massive Online Analysis +(MOA)](https://moa.cms.waikato.ac.nz/) library with the python ecosystem. + +To build MOA for use with CapyMOA run: +```bash +cd moa +mvn package -DskipTests -Dmaven.javadoc.skip=true -Dlatex.skipBuild=true +``` +This will create a `target/moa-*-jar-with-dependencies.jar` file that can be used by +CapyMOA. To let CapyMOA know where this file is, set the `CAPYMOA_MOA_JAR` environment +variable to the path of this file. + +You can do this temporarily in your terminal session with: +```bash +export CAPYMOA_MOA_JAR=/path/to/moa/target/moa-*-jar-with-dependencies.jar +``` +To check that CapyMOA can find MOA, run: +```bash +python -c "import capymoa; capymoa.about()" +# CapyMOA 0.10.0 +# CAPYMOA_DATASETS_DIR: .../datasets +# CAPYMOA_MOA_JAR: .../moa/moa/target/moa-2024.07.2-SNAPSHOT-jar-with-dependencies.jar +# CAPYMOA_JVM_ARGS: ['-Xmx8g', '-Xss10M'] +# JAVA_HOME: /usr/lib/jvm/java-21-openjdk +# MOA version: aa955ebbcbd99e9e1d19ab16582e3e5a6fca5801ba250e4d164c16a89cf798ea +# JAVA version: 21.0.7 +``` diff --git a/moa/pom.xml b/moa/pom.xml index b1f03c533..450155fae 100644 --- a/moa/pom.xml +++ b/moa/pom.xml @@ -259,6 +259,16 @@ org.apache.maven.plugins maven-assembly-plugin + + + + moa.gui.GUI + + + + jar-with-dependencies + + From 4d8063bdb464f0e6fc4878d1e20f7b9b786e8c66 Mon Sep 17 00:00:00 2001 From: Anton Lee Date: Fri, 17 Oct 2025 12:42:28 +1300 Subject: [PATCH 6/9] remove non-deterministic 'HerosTest.java' The heros method uses system time to calculate a dynamic resource cost this is non-deterministic. --- .../java/moa/classifiers/meta/HerosTest.java | 30 ------------------- moa/src/test/java/moa/test/MoaTestCase.java | 25 ++++------------ 2 files changed, 5 insertions(+), 50 deletions(-) delete mode 100644 moa/src/test/java/moa/classifiers/meta/HerosTest.java diff --git a/moa/src/test/java/moa/classifiers/meta/HerosTest.java b/moa/src/test/java/moa/classifiers/meta/HerosTest.java deleted file mode 100644 index f48d840b1..000000000 --- a/moa/src/test/java/moa/classifiers/meta/HerosTest.java +++ /dev/null @@ -1,30 +0,0 @@ -package moa.classifiers.meta; - -import junit.framework.Test; -import junit.framework.TestSuite; -import moa.classifiers.AbstractMultipleClassifierTestCase; -import moa.classifiers.Classifier; -import moa.classifiers.meta.heros.Heros; - -/** - * Tests the Heros classifier. - */ -public class HerosTest extends AbstractMultipleClassifierTestCase { - public HerosTest(String name) { - super(name); - this.setNumberTests(1); - } - - @Override - protected Classifier[] getRegressionClassifierSetups() { - return new Classifier[] { new Heros(), }; - } - - public static Test suite() { - return new TestSuite(HerosTest.class); - } - - public static void main(String[] args) { - runTest(suite()); - } -} diff --git a/moa/src/test/java/moa/test/MoaTestCase.java b/moa/src/test/java/moa/test/MoaTestCase.java index 061ec665b..5ff59639d 100644 --- a/moa/src/test/java/moa/test/MoaTestCase.java +++ b/moa/src/test/java/moa/test/MoaTestCase.java @@ -71,21 +71,11 @@ public MoaTestCase(String name) { * @return the class that is being tested or null if none could * be determined */ - protected Class getTestedClass() { - Class result; - - result = null; - - if (getClass().getName().endsWith("Test")) { - try { - result = Class.forName(getClass().getName().replaceAll("Test$", "")); - } - catch (Exception e) { - result = null; - } + protected Class getTestedClass() throws ClassNotFoundException { + if (!getClass().getName().endsWith("Test")) { + throw new IllegalStateException("Class name must end with 'Test': " + getClass().getName()); } - - return result; + return Class.forName(getClass().getName().replaceAll("Test$", "")); } /** @@ -105,14 +95,9 @@ protected boolean canHandleHeadless() { */ @Override protected void setUp() throws Exception { - Class cls; - super.setUp(); - - cls = getTestedClass(); - if (cls != null) - m_Regression = new Regression(cls); + m_Regression = new Regression(getTestedClass()); m_TestHelper = newTestHelper(); m_Headless = Boolean.getBoolean(PROPERTY_HEADLESS); m_NoRegressionTest = Boolean.getBoolean(PROPERTY_NOREGRESSION); From 8d083bf225545d51c2f500c9d7529db2fe96191b Mon Sep 17 00:00:00 2001 From: Anton Lee Date: Fri, 17 Oct 2025 13:28:02 +1300 Subject: [PATCH 7/9] fix tests --- .github/workflows/capymoa.yml | 4 ++ .../clusterers/clustream/ClustreamKernel.java | 24 ++++++---- .../moa/integration/SimpleClusterTest.java | 44 +++++++++++-------- .../trees/AdaHoeffdingOptionTree.ref | 2 +- 4 files changed, 46 insertions(+), 28 deletions(-) diff --git a/.github/workflows/capymoa.yml b/.github/workflows/capymoa.yml index 290898ff2..ff8155eb6 100644 --- a/.github/workflows/capymoa.yml +++ b/.github/workflows/capymoa.yml @@ -20,6 +20,10 @@ jobs: distribution: 'temurin' cache: maven + - name: Unit Tests + working-directory: ./moa + run: mvn -B test + - name: Package Jar working-directory: ./moa # Package as JAR while skipping tests, javadoc, and latex build diff --git a/moa/src/main/java/moa/clusterers/clustream/ClustreamKernel.java b/moa/src/main/java/moa/clusterers/clustream/ClustreamKernel.java index 609ad8fdb..980597e9c 100644 --- a/moa/src/main/java/moa/clusterers/clustream/ClustreamKernel.java +++ b/moa/src/main/java/moa/clusterers/clustream/ClustreamKernel.java @@ -46,12 +46,8 @@ public ClustreamKernel(Instance instance, int dimensions, long timestamp , doubl // Avoid situations where the instance header hasn't been defined and runtime errors. if(instance.dataset() != null) { this.classObserver = new double[instance.numClasses()]; -// instance.numAttributes() <= instance.classIndex() -> edge case where the class index is equal the -// number of attributes (i.e. there is no class value in the attributes array). - if (instance.numAttributes() > instance.classIndex() && - !instance.classIsMissing() && - instance.classValue() >= 0 && - instance.classValue() < instance.numClasses()) { +// + if (this.instanceHasClass(instance)) { this.classObserver[(int) instance.classValue()]++; } } @@ -72,12 +68,22 @@ public ClustreamKernel( ClustreamKernel cluster, double t, int m ) { this.classObserver = cluster.classObserver; } + private boolean instanceHasClass(Instance instance) { + // TODO: Why is this check necessary? Shouldn't classIsMissing() be enough? + // Edge case where the class index is out of bounds. number of attributes + // (i.e. there is no class value in the attributes array). + return instance.numAttributes() > instance.classIndex() && + !instance.classIsMissing() && // Also check for missing class. + instance.classValue() >= 0 && // Or invalid class values. + instance.classValue() < instance.numClasses(); + } + public void insert( Instance instance, long timestamp ) { if(this.classObserver == null) this.classObserver = new double[instance.numClasses()]; - if(!instance.classIsMissing() && - instance.classValue() >= 0 && - instance.classValue() < instance.numClasses()) { + System.out.println(instance.classIndex()); + + if(this.instanceHasClass(instance)) { this.classObserver[(int)instance.classValue()]++; } N++; diff --git a/moa/src/test/java/moa/integration/SimpleClusterTest.java b/moa/src/test/java/moa/integration/SimpleClusterTest.java index 678627b38..89f910a0f 100644 --- a/moa/src/test/java/moa/integration/SimpleClusterTest.java +++ b/moa/src/test/java/moa/integration/SimpleClusterTest.java @@ -22,26 +22,34 @@ public class SimpleClusterTest extends TestCase { final static String [] Clusterers = new String[]{"ClusterGenerator", "CobWeb", "KMeans", "clustream.Clustream", "clustree.ClusTree", "denstream.WithDBSCAN -i 1000", "streamkm.StreamKM"}; - @Test - public void testClusterGenerator(){testClusterer(Clusterers[0]);} -// @Test -// public void testCobWeb(){testClusterer(Clusterers[1]);} - @Test - public void testClustream(){testClusterer(Clusterers[3]);} - @Test - public void testClusTree(){testClusterer(Clusterers[4]);} - @Test - public void testDenStream(){testClusterer(Clusterers[5]);} - @Test - public void testStreamKM(){testClusterer(Clusterers[6]);} + @Test + public void testClusterGenerator() throws Exception { + testClusterer(Clusterers[0]); + } + + @Test + public void testClustream() throws Exception { + testClusterer(Clusterers[3]); + } + + @Test + public void testClusTree() throws Exception { + testClusterer(Clusterers[4]); + } + + @Test + public void testDenStream() throws Exception { + testClusterer(Clusterers[5]); + } + + @Test + public void testStreamKM() throws Exception { + testClusterer(Clusterers[6]); + } - void testClusterer(String clusterer) { + void testClusterer(String clusterer) throws Exception { System.out.println("Processing: " + clusterer); - try { - doTask(new String[]{"EvaluateClustering -l " + clusterer}); - } catch (Exception e) { - assertTrue("Failed on clusterer " + clusterer + ": " + e.getMessage(), false); - } + doTask(new String[]{"EvaluateClustering -l " + clusterer}); } // code copied from moa.DoTask.main, to allow exceptions to be thrown in case of failure diff --git a/moa/src/test/resources/moa/classifiers/trees/AdaHoeffdingOptionTree.ref b/moa/src/test/resources/moa/classifiers/trees/AdaHoeffdingOptionTree.ref index 37c4259c5..d5ff74078 100644 --- a/moa/src/test/resources/moa/classifiers/trees/AdaHoeffdingOptionTree.ref +++ b/moa/src/test/resources/moa/classifiers/trees/AdaHoeffdingOptionTree.ref @@ -49,7 +49,7 @@ Index 30000 Votes 0: 3.612203649887567E14 - 1: 1.24768970176524544E17 + 1: 1.2476897017652454E17 Measurements classified instances: 29999 classifications correct (percent): 85.57618587 From 31c94d5e3341073bb3aa8838e1fb2632b82cb12b Mon Sep 17 00:00:00 2001 From: Anton Lee Date: Mon, 20 Oct 2025 16:50:15 +1300 Subject: [PATCH 8/9] change java version in workflow --- .github/workflows/capymoa.yml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/capymoa.yml b/.github/workflows/capymoa.yml index ff8155eb6..9ea01b903 100644 --- a/.github/workflows/capymoa.yml +++ b/.github/workflows/capymoa.yml @@ -1,4 +1,4 @@ -name: Package Jar for CapyMOA +name: CapyMOA Test and Package on: push: @@ -13,16 +13,26 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up JDK 17 + + # TODO: moa tests are sensitive to java versions https://github.com/Waikato/moa/issues/273 + - name: Set up JDK 21 uses: actions/setup-java@v4 with: - java-version: '17' - distribution: 'temurin' + java-version: '21' + distribution: 'zulu' cache: maven + - name: Version + working-directory: ./moa + run: mvn -v + + - name: lscpu + run: lscpu + - name: Unit Tests working-directory: ./moa - run: mvn -B test + # Skip flakey tests + run: mvn -B -q test -Dtest=\!MLPTest,\!SecondTestClass - name: Package Jar working-directory: ./moa From b2262f58056269876552067367a87b73f3a672ae Mon Sep 17 00:00:00 2001 From: Anton Lee Date: Mon, 20 Oct 2025 17:58:00 +1300 Subject: [PATCH 9/9] disable tests that fail on GH actions CI --- .github/workflows/capymoa.yml | 13 +++++++------ .../java/moa/classifiers/deeplearning/CANDTest.java | 5 +++++ .../java/moa/classifiers/deeplearning/MLPTest.java | 5 +++++ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/workflows/capymoa.yml b/.github/workflows/capymoa.yml index 9ea01b903..4e8971bcc 100644 --- a/.github/workflows/capymoa.yml +++ b/.github/workflows/capymoa.yml @@ -26,17 +26,18 @@ jobs: working-directory: ./moa run: mvn -v - - name: lscpu - run: lscpu - - name: Unit Tests working-directory: ./moa - # Skip flakey tests - run: mvn -B -q test -Dtest=\!MLPTest,\!SecondTestClass + # -B: non-interactive (batch) mode + # -q: quiet output + run: mvn -B -q test - name: Package Jar working-directory: ./moa - # Package as JAR while skipping tests, javadoc, and latex build + # -B: non-interactive (batch) mode + # -DskipTests: skip tests (they were run earlier) + # -Dmaven.javadoc.skip=true: skip javadoc generation + # -Dlatex.skipBuild=true: skip latex documentation build this needs extra dependencies run: mvn -B package -DskipTests -Dmaven.javadoc.skip=true -Dlatex.skipBuild=true # Upload jar file as artifact diff --git a/moa/src/test/java/moa/classifiers/deeplearning/CANDTest.java b/moa/src/test/java/moa/classifiers/deeplearning/CANDTest.java index cc954704a..a9d60d023 100644 --- a/moa/src/test/java/moa/classifiers/deeplearning/CANDTest.java +++ b/moa/src/test/java/moa/classifiers/deeplearning/CANDTest.java @@ -20,6 +20,8 @@ */ package moa.classifiers.deeplearning; +import org.junit.Ignore; + import junit.framework.Test; import junit.framework.TestSuite; import moa.classifiers.AbstractMultipleClassifierTestCase; @@ -31,6 +33,9 @@ * @author Nuwan Gunasekara (ng98 at students dot waikato dot ac dot nz) * @version $Revision$ */ +// TODO: test fails on GitHub runner but not locally (https://github.com/Waikato/moa/issues/322) +// potentially hardware related +@Ignore public class CANDTest extends AbstractMultipleClassifierTestCase { diff --git a/moa/src/test/java/moa/classifiers/deeplearning/MLPTest.java b/moa/src/test/java/moa/classifiers/deeplearning/MLPTest.java index 2f7a3fb6d..4039c33c3 100644 --- a/moa/src/test/java/moa/classifiers/deeplearning/MLPTest.java +++ b/moa/src/test/java/moa/classifiers/deeplearning/MLPTest.java @@ -20,6 +20,8 @@ */ package moa.classifiers.deeplearning; +import org.junit.Ignore; + import junit.framework.Test; import junit.framework.TestSuite; import moa.classifiers.AbstractMultipleClassifierTestCase; @@ -31,6 +33,9 @@ * @author Nuwan Gunasekara (ng98 at students dot waikato dot ac dot nz) * @version $Revision$ */ +// TODO: test fails on GitHub runner but not locally (https://github.com/Waikato/moa/issues/322) +// potentially hardware related +@Ignore public class MLPTest extends AbstractMultipleClassifierTestCase {