diff options
Diffstat (limited to 'src/de/lmu/ifi/dbs/elki/application')
28 files changed, 1151 insertions, 904 deletions
diff --git a/src/de/lmu/ifi/dbs/elki/application/AbstractApplication.java b/src/de/lmu/ifi/dbs/elki/application/AbstractApplication.java index dabf5224..3770bbc4 100644 --- a/src/de/lmu/ifi/dbs/elki/application/AbstractApplication.java +++ b/src/de/lmu/ifi/dbs/elki/application/AbstractApplication.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.application; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2013 + Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -38,17 +38,15 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionUtil; import de.lmu.ifi.dbs.elki.utilities.optionhandling.ParameterException; -import de.lmu.ifi.dbs.elki.utilities.optionhandling.Parameterizable; import de.lmu.ifi.dbs.elki.utilities.optionhandling.UnspecifiedParameterException; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.SerializedParameterization; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.TrackParameters; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.TrackedParameter; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ClassParameter; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.FileParameter; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.Flag; -import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.Parameter; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.StringParameter; -import de.lmu.ifi.dbs.elki.utilities.pairs.Pair; /** * AbstractApplication sets the values for flags verbose and help. @@ -63,7 +61,7 @@ import de.lmu.ifi.dbs.elki.utilities.pairs.Pair; * @apiviz.uses LoggingConfiguration oneway * @apiviz.excludeSubtypes */ -public abstract class AbstractApplication implements Parameterizable { +public abstract class AbstractApplication { /** * We need a static logger in this class, for code used in "main" methods. */ @@ -177,7 +175,7 @@ public abstract class AbstractApplication implements Parameterizable { * @param options Options to show in usage. * @return a usage message explaining all known options */ - public static String usage(Collection<Pair<Object, Parameter<?>>> options) { + public static String usage(Collection<TrackedParameter> options) { StringBuilder usage = new StringBuilder(); usage.append(INFORMATION); diff --git a/src/de/lmu/ifi/dbs/elki/application/ClassifierHoldoutEvaluationTask.java b/src/de/lmu/ifi/dbs/elki/application/ClassifierHoldoutEvaluationTask.java new file mode 100644 index 00000000..c4e364e6 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/application/ClassifierHoldoutEvaluationTask.java @@ -0,0 +1,253 @@ +package de.lmu.ifi.dbs.elki.application; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2014 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; + +import de.lmu.ifi.dbs.elki.algorithm.classification.Classifier; +import de.lmu.ifi.dbs.elki.data.ClassLabel; +import de.lmu.ifi.dbs.elki.data.type.TypeUtil; +import de.lmu.ifi.dbs.elki.database.AbstractDatabase; +import de.lmu.ifi.dbs.elki.database.Database; +import de.lmu.ifi.dbs.elki.database.StaticArrayDatabase; +import de.lmu.ifi.dbs.elki.database.relation.Relation; +import de.lmu.ifi.dbs.elki.datasource.DatabaseConnection; +import de.lmu.ifi.dbs.elki.datasource.FileBasedDatabaseConnection; +import de.lmu.ifi.dbs.elki.datasource.MultipleObjectsBundleDatabaseConnection; +import de.lmu.ifi.dbs.elki.datasource.bundle.MultipleObjectsBundle; +import de.lmu.ifi.dbs.elki.evaluation.classification.ConfusionMatrix; +import de.lmu.ifi.dbs.elki.evaluation.classification.holdout.AbstractHoldout; +import de.lmu.ifi.dbs.elki.evaluation.classification.holdout.Holdout; +import de.lmu.ifi.dbs.elki.evaluation.classification.holdout.StratifiedCrossValidation; +import de.lmu.ifi.dbs.elki.evaluation.classification.holdout.TrainingAndTestSet; +import de.lmu.ifi.dbs.elki.index.IndexFactory; +import de.lmu.ifi.dbs.elki.logging.Logging; +import de.lmu.ifi.dbs.elki.logging.statistics.Duration; +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2014 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +import de.lmu.ifi.dbs.elki.utilities.exceptions.UnableToComplyException; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectListParameter; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter; +import de.lmu.ifi.dbs.elki.workflow.AlgorithmStep; + +/** + * Evaluate a classifier. + * + * TODO: split into application and task. + * + * TODO: add support for predefined test and training pairs! + * + * @author Erich Schubert + * + * @param <O> Object type + */ +public class ClassifierHoldoutEvaluationTask<O> extends AbstractApplication { + /** + * Class logger. + */ + private static final Logging LOG = Logging.getLogger(ClassifierHoldoutEvaluationTask.class); + + /** + * Holds the database connection to get the initial data from. + */ + protected DatabaseConnection databaseConnection = null; + + /** + * Indexes to add. + */ + protected Collection<IndexFactory<?, ?>> indexFactories; + + /** + * Classifier to evaluate. + */ + protected Classifier<O> algorithm; + + /** + * Holds the holdout. + */ + protected Holdout holdout; + + /** + * Constructor. + * + * @param databaseConnection Data source + * @param indexFactories Data indexes + * @param algorithm Classification algorithm + * @param holdout Evaluation holdout + */ + public ClassifierHoldoutEvaluationTask(DatabaseConnection databaseConnection, Collection<IndexFactory<?, ?>> indexFactories, Classifier<O> algorithm, Holdout holdout) { + this.databaseConnection = databaseConnection; + this.indexFactories = indexFactories; + this.algorithm = algorithm; + this.holdout = holdout; + } + + @Override + public void run() throws UnableToComplyException { + Duration ptime = LOG.newDuration("evaluation.time.load").begin(); + MultipleObjectsBundle allData = databaseConnection.loadData(); + holdout.initialize(allData); + LOG.statistics(ptime.end()); + + Duration time = LOG.newDuration("evaluation.time.total").begin(); + ArrayList<ClassLabel> labels = holdout.getLabels(); + int[][] confusion = new int[labels.size()][labels.size()]; + for(int p = 0; p < holdout.numberOfPartitions(); p++) { + TrainingAndTestSet partition = holdout.nextPartitioning(); + // Load the data set into a database structure (for indexing) + Duration dur = LOG.newDuration(this.getClass().getName() + ".fold-" + (p + 1) + ".init.time").begin(); + Database db = new StaticArrayDatabase(new MultipleObjectsBundleDatabaseConnection(partition.getTraining()), indexFactories); + db.initialize(); + LOG.statistics(dur.end()); + // Train the classifier + dur = LOG.newDuration(this.getClass().getName() + ".fold-" + (p + 1) + ".train.time").begin(); + Relation<ClassLabel> lrel = db.getRelation(TypeUtil.CLASSLABEL); + algorithm.buildClassifier(db, lrel); + LOG.statistics(dur.end()); + // Evaluate the test set + dur = LOG.newDuration(this.getClass().getName() + ".fold-" + (p + 1) + ".evaluation.time").begin(); + // FIXME: this part is still a big hack, unfortunately! + MultipleObjectsBundle test = partition.getTest(); + int lcol = AbstractHoldout.findClassLabelColumn(test); + int tcol = (lcol == 0) ? 1 : 0; + for(int i = 0, l = test.dataLength(); i < l; ++i) { + @SuppressWarnings("unchecked") + O obj = (O) test.data(i, tcol); + ClassLabel truelbl = (ClassLabel) test.data(i, lcol); + ClassLabel predlbl = algorithm.classify(obj); + int pred = Collections.binarySearch(labels, predlbl); + int real = Collections.binarySearch(labels, truelbl); + confusion[pred][real]++; + } + LOG.statistics(dur.end()); + } + LOG.statistics(time.end()); + ConfusionMatrix m = new ConfusionMatrix(labels, confusion); + LOG.statistics(m.toString()); + } + + /** + * Parameterization class. + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer<O> extends AbstractApplication.Parameterizer { + /** + * Parameter to specify the holdout for evaluation, must extend + * {@link de.lmu.ifi.dbs.elki.evaluation.classification.holdout.Holdout}. + * <p> + * Key: {@code -classifier.holdout} + * </p> + * <p> + * Default value: {@link StratifiedCrossValidation} + * </p> + */ + public static final OptionID HOLDOUT_ID = new OptionID("evaluation.holdout", "Holdout class used in evaluation."); + + /** + * Holds the database connection to get the initial data from. + */ + protected DatabaseConnection databaseConnection = null; + + /** + * Indexes to add. + */ + protected Collection<IndexFactory<?, ?>> indexFactories; + + /** + * Classifier to evaluate. + */ + protected Classifier<O> algorithm; + + /** + * Holds the holdout. + */ + protected Holdout holdout; + + @Override + protected void makeOptions(Parameterization config) { + super.makeOptions(config); + // Get database connection. + final ObjectParameter<DatabaseConnection> dbcP = new ObjectParameter<>(AbstractDatabase.Parameterizer.DATABASE_CONNECTION_ID, DatabaseConnection.class, FileBasedDatabaseConnection.class); + if(config.grab(dbcP)) { + databaseConnection = dbcP.instantiateClass(config); + } + // Get indexes. + final ObjectListParameter<IndexFactory<?, ?>> indexFactoryP = new ObjectListParameter<>(AbstractDatabase.Parameterizer.INDEX_ID, IndexFactory.class, true); + if(config.grab(indexFactoryP)) { + indexFactories = indexFactoryP.instantiateClasses(config); + } + ObjectParameter<Classifier<O>> algorithmP = new ObjectParameter<>(AlgorithmStep.Parameterizer.ALGORITHM_ID, Classifier.class); + if(config.grab(algorithmP)) { + algorithm = algorithmP.instantiateClass(config); + } + + ObjectParameter<Holdout> holdoutP = new ObjectParameter<>(HOLDOUT_ID, Holdout.class, StratifiedCrossValidation.class); + if(config.grab(holdoutP)) { + holdout = holdoutP.instantiateClass(config); + } + } + + @Override + protected ClassifierHoldoutEvaluationTask<O> makeInstance() { + return new ClassifierHoldoutEvaluationTask<O>(databaseConnection, indexFactories, algorithm, holdout); + } + } + + /** + * Runs the classifier evaluation task accordingly to the specified + * parameters. + * + * @param args parameter list according to description + */ + public static void main(String[] args) { + runCLIApplication(ClassifierHoldoutEvaluationTask.class, args); + } +} diff --git a/src/de/lmu/ifi/dbs/elki/application/ComputeSingleColorHistogram.java b/src/de/lmu/ifi/dbs/elki/application/ComputeSingleColorHistogram.java deleted file mode 100644 index b2c8afff..00000000 --- a/src/de/lmu/ifi/dbs/elki/application/ComputeSingleColorHistogram.java +++ /dev/null @@ -1,166 +0,0 @@ -package de.lmu.ifi.dbs.elki.application; - -/* - This file is part of ELKI: - Environment for Developing KDD-Applications Supported by Index-Structures - - Copyright (C) 2013 - Ludwig-Maximilians-Universität München - Lehr- und Forschungseinheit für Datenbanksysteme - ELKI Development Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -import java.io.File; -import java.io.IOException; - -import de.lmu.ifi.dbs.elki.data.images.ComputeColorHistogram; -import de.lmu.ifi.dbs.elki.data.images.ComputeNaiveRGBColorHistogram; -import de.lmu.ifi.dbs.elki.utilities.FormatUtil; -import de.lmu.ifi.dbs.elki.utilities.exceptions.UnableToComplyException; -import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; -import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; -import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.FileParameter; -import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter; - -/** - * Application that computes the color histogram vector for a single image. - * - * @author Erich Schubert - * - * @apiviz.composedOf ComputeColorHistogram - * @apiviz.has File - */ -public class ComputeSingleColorHistogram extends AbstractApplication { - /** - * Class parameter for computing the color histogram. - * <p> - * Key: {@code -colorhist.generator} - * </p> - */ - public static OptionID COLORHIST_ID = new OptionID("colorhist.generator", "Class that is used to generate a color histogram."); - - /** - * Parameter that specifies the name of the input file. - * <p> - * Key: {@code -colorhist.in} - * </p> - */ - public static final OptionID INPUT_ID = new OptionID("colorhist.in", "Input image file for color histogram."); - - /** - * Parameter that specifies the name of the mask input file. - * <p> - * Key: {@code -colorhist.mask} - * </p> - */ - public static final OptionID MASK_ID = new OptionID("colorhist.mask", "Input mask image file."); - - /** - * Class that will compute the actual histogram - */ - private ComputeColorHistogram histogrammaker; - - /** - * Input file. - */ - private File inputFile; - - /** - * Mask file. - */ - private File maskFile; - - /** - * Constructor. - * - * @param histogrammaker Class to compute histograms with - * @param inputFile Input file - * @param maskFile Mask file - */ - public ComputeSingleColorHistogram(ComputeColorHistogram histogrammaker, File inputFile, File maskFile) { - super(); - this.histogrammaker = histogrammaker; - this.inputFile = inputFile; - this.maskFile = maskFile; - } - - @Override - public void run() throws UnableToComplyException { - double[] hist; - try { - hist = histogrammaker.computeColorHistogram(inputFile, maskFile); - } - catch(IOException e) { - throw new UnableToComplyException(e); - } - System.out.println(FormatUtil.format(hist, " ")); - } - - /** - * Parameterization class. - * - * @author Erich Schubert - * - * @apiviz.exclude - */ - public static class Parameterizer extends AbstractApplication.Parameterizer { - /** - * Class that will compute the actual histogram - */ - private ComputeColorHistogram histogrammaker; - - /** - * Input file. - */ - private File inputFile; - - /** - * Mask file. - */ - private File maskFile; - - @Override - protected void makeOptions(Parameterization config) { - super.makeOptions(config); - final ObjectParameter<ComputeColorHistogram> colorhistP = new ObjectParameter<>(COLORHIST_ID, ComputeColorHistogram.class, ComputeNaiveRGBColorHistogram.class); - if(config.grab(colorhistP)) { - histogrammaker = colorhistP.instantiateClass(config); - } - final FileParameter inputP = new FileParameter(INPUT_ID, FileParameter.FileType.INPUT_FILE); - if(config.grab(inputP)) { - inputFile = inputP.getValue(); - } - final FileParameter maskP = new FileParameter(MASK_ID, FileParameter.FileType.INPUT_FILE, true); - if(config.grab(maskP)) { - maskFile = maskP.getValue(); - } - } - - @Override - protected ComputeSingleColorHistogram makeInstance() { - return new ComputeSingleColorHistogram(histogrammaker, inputFile, maskFile); - } - } - - /** - * Main method to run this application. - * - * @param args the arguments to run this application - */ - public static void main(String[] args) { - runCLIApplication(ComputeSingleColorHistogram.class, args); - } -}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/application/ConvertToBundleApplication.java b/src/de/lmu/ifi/dbs/elki/application/ConvertToBundleApplication.java index 8b177691..ac140abb 100644 --- a/src/de/lmu/ifi/dbs/elki/application/ConvertToBundleApplication.java +++ b/src/de/lmu/ifi/dbs/elki/application/ConvertToBundleApplication.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.application; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2013 + Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -33,7 +33,6 @@ import de.lmu.ifi.dbs.elki.datasource.DatabaseConnection; import de.lmu.ifi.dbs.elki.datasource.FileBasedDatabaseConnection; import de.lmu.ifi.dbs.elki.datasource.bundle.BundleWriter; import de.lmu.ifi.dbs.elki.datasource.bundle.MultipleObjectsBundle; -import de.lmu.ifi.dbs.elki.datasource.bundle.StreamFromBundle; import de.lmu.ifi.dbs.elki.logging.Logging; import de.lmu.ifi.dbs.elki.utilities.exceptions.UnableToComplyException; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; @@ -86,7 +85,7 @@ public class ConvertToBundleApplication extends AbstractApplication { try { FileOutputStream fos = new FileOutputStream(outfile); FileChannel channel = fos.getChannel(); - writer.writeBundleStream(new StreamFromBundle(bundle), channel); + writer.writeBundleStream(bundle.asStream(), channel); channel.close(); fos.close(); } diff --git a/src/de/lmu/ifi/dbs/elki/application/ELKILauncher.java b/src/de/lmu/ifi/dbs/elki/application/ELKILauncher.java index 5c969157..5a3eac39 100644 --- a/src/de/lmu/ifi/dbs/elki/application/ELKILauncher.java +++ b/src/de/lmu/ifi/dbs/elki/application/ELKILauncher.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.application; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2013 + Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -23,6 +23,7 @@ package de.lmu.ifi.dbs.elki.application; along with this program. If not, see <http://www.gnu.org/licenses/>. */ +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Arrays; @@ -50,25 +51,31 @@ public class ELKILauncher { * @param args Command line arguments. */ public static void main(String[] args) { - if (args.length > 0 && args[0].charAt(0) != '-') { + if(args.length > 0 && args[0].charAt(0) != '-') { try { Class<?> cls = findMainClass(args[0]); Method m = cls.getMethod("main", String[].class); Object a = Arrays.copyOfRange(args, 1, args.length); try { m.invoke(null, a); - } catch (Exception e) { + } + catch(InvocationTargetException e) { + LoggingUtil.exception(e.getCause()); + } + catch(Exception e) { LoggingUtil.exception(e); } return; - } catch (Exception e) { + } + catch(Exception e) { // Ignore } } try { Method m = DEFAULT_APPLICATION.getMethod("main", String[].class); m.invoke(null, (Object) args); - } catch (Exception e) { + } + catch(Exception e) { LoggingUtil.exception(e); } } @@ -80,23 +87,26 @@ public class ELKILauncher { * @return Class * @throws ClassNotFoundException */ - private static Class<?> findMainClass(String name) throws ClassNotFoundException { + public static Class<? extends AbstractApplication> findMainClass(String name) throws ClassNotFoundException { try { - return Class.forName(name); - } catch (ClassNotFoundException e) { + return Class.forName(name).asSubclass(AbstractApplication.class); + } + catch(ClassNotFoundException | ClassCastException e) { // pass } try { - return Class.forName(AbstractApplication.class.getPackage().getName() + '.' + name); - } catch (ClassNotFoundException e) { + return Class.forName(AbstractApplication.class.getPackage().getName() + '.' + name)// + .asSubclass(AbstractApplication.class); + } + catch(ClassNotFoundException | ClassCastException e) { // pass } - for (Class<?> c : InspectionUtil.cachedFindAllImplementations(AbstractApplication.class)) { - if (c.isAnnotationPresent(Alias.class)) { + for(Class<?> c : InspectionUtil.cachedFindAllImplementations(AbstractApplication.class)) { + if(c.isAnnotationPresent(Alias.class)) { Alias aliases = c.getAnnotation(Alias.class); - for (String alias : aliases.value()) { - if (alias.equalsIgnoreCase(name)) { - return c; + for(String alias : aliases.value()) { + if(alias.equalsIgnoreCase(name)) { + return c.asSubclass(AbstractApplication.class); } } } diff --git a/src/de/lmu/ifi/dbs/elki/application/GeneratorXMLSpec.java b/src/de/lmu/ifi/dbs/elki/application/GeneratorXMLSpec.java index 50f97d87..da86a150 100644 --- a/src/de/lmu/ifi/dbs/elki/application/GeneratorXMLSpec.java +++ b/src/de/lmu/ifi/dbs/elki/application/GeneratorXMLSpec.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.application; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2013 + Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team diff --git a/src/de/lmu/ifi/dbs/elki/application/KDDCLIApplication.java b/src/de/lmu/ifi/dbs/elki/application/KDDCLIApplication.java index 7e072989..7d9cd016 100644 --- a/src/de/lmu/ifi/dbs/elki/application/KDDCLIApplication.java +++ b/src/de/lmu/ifi/dbs/elki/application/KDDCLIApplication.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.application; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2013 + Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -30,10 +30,9 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameteriz import de.lmu.ifi.dbs.elki.workflow.OutputStep; /** - * Provides a KDDCLIApplication that can be used to perform any algorithm - * implementing {@link Algorithm Algorithm} using any DatabaseConnection - * implementing {@link de.lmu.ifi.dbs.elki.datasource.DatabaseConnection - * DatabaseConnection}. + * Basic command line application for Knowledge Discovery in Databases use + * cases. It allows running unsupervised {@link Algorithm}s to run on any + * {@link de.lmu.ifi.dbs.elki.datasource.DatabaseConnection DatabaseConnection}. * * @author Arthur Zimek * diff --git a/src/de/lmu/ifi/dbs/elki/application/cache/CacheDoubleDistanceInOnDiskMatrix.java b/src/de/lmu/ifi/dbs/elki/application/cache/CacheDoubleDistanceInOnDiskMatrix.java index 6a28282c..02290ced 100644 --- a/src/de/lmu/ifi/dbs/elki/application/cache/CacheDoubleDistanceInOnDiskMatrix.java +++ b/src/de/lmu/ifi/dbs/elki/application/cache/CacheDoubleDistanceInOnDiskMatrix.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.application.cache; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2013 + Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -28,17 +28,17 @@ import java.io.IOException; import de.lmu.ifi.dbs.elki.application.AbstractApplication; import de.lmu.ifi.dbs.elki.database.Database; -import de.lmu.ifi.dbs.elki.database.ids.DBIDFactory; -import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDArrayIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRange; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction; import de.lmu.ifi.dbs.elki.distance.distancefunction.external.DiskCacheBasedDoubleDistanceFunction; -import de.lmu.ifi.dbs.elki.distance.distancevalue.NumberDistance; import de.lmu.ifi.dbs.elki.logging.Logging; import de.lmu.ifi.dbs.elki.persistent.OnDiskUpperTriangleMatrix; import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; +import de.lmu.ifi.dbs.elki.utilities.io.ByteArrayUtil; import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.FileParameter; @@ -54,9 +54,8 @@ import de.lmu.ifi.dbs.elki.workflow.InputStep; * @apiviz.has DistanceFunction * * @param <O> Object type - * @param <D> Distance type */ -public class CacheDoubleDistanceInOnDiskMatrix<O, D extends NumberDistance<D, ?>> extends AbstractApplication { +public class CacheDoubleDistanceInOnDiskMatrix<O> extends AbstractApplication { /** * The logger for this class. */ @@ -75,7 +74,7 @@ public class CacheDoubleDistanceInOnDiskMatrix<O, D extends NumberDistance<D, ?> /** * Distance function that is to be cached. */ - private DistanceFunction<O, D> distance; + private DistanceFunction<O> distance; /** * Output file. @@ -89,7 +88,7 @@ public class CacheDoubleDistanceInOnDiskMatrix<O, D extends NumberDistance<D, ?> * @param distance Distance function * @param out Matrix output file */ - public CacheDoubleDistanceInOnDiskMatrix(InputStep input, DistanceFunction<O, D> distance, File out) { + public CacheDoubleDistanceInOnDiskMatrix(InputStep input, DistanceFunction<O> distance, File out) { super(); this.input = input; this.distance = distance; @@ -100,40 +99,35 @@ public class CacheDoubleDistanceInOnDiskMatrix<O, D extends NumberDistance<D, ?> public void run() { Database database = input.getDatabase(); Relation<O> relation = database.getRelation(distance.getInputTypeRestriction()); - DistanceQuery<O, D> distanceQuery = database.getDistanceQuery(relation, distance); - - int matrixsize = 0; - for (DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { - int intid = DBIDUtil.asInteger(iditer); - matrixsize = Math.max(matrixsize, intid + 1); - if (intid < 0) { - throw new AbortException("OnDiskMatrixCache does not allow negative DBIDs."); - } - } + DistanceQuery<O> distanceQuery = database.getDistanceQuery(relation, distance); + + DBIDRange ids = DBIDUtil.assertRange(relation.getDBIDs()); + int matrixsize = ids.size(); OnDiskUpperTriangleMatrix matrix; try { - matrix = new OnDiskUpperTriangleMatrix(out, DiskCacheBasedDoubleDistanceFunction.DOUBLE_CACHE_MAGIC, 0, 8, matrixsize); - } catch (IOException e) { + matrix = new OnDiskUpperTriangleMatrix(out, DiskCacheBasedDoubleDistanceFunction.DOUBLE_CACHE_MAGIC, 0, ByteArrayUtil.SIZE_DOUBLE, matrixsize); + } + catch(IOException e) { throw new AbortException("Error creating output matrix.", e); } - for (DBIDIter id1 = relation.iterDBIDs(); id1.valid(); id1.advance()) { - for (DBIDIter id2 = relation.iterDBIDs(); id2.valid(); id2.advance()) { - if (DBIDUtil.asInteger(id2) >= DBIDUtil.asInteger(id1)) { - double d = distanceQuery.distance(id1, id2).doubleValue(); - if (debugExtraCheckSymmetry) { - double d2 = distanceQuery.distance(id2, id1).doubleValue(); - if (Math.abs(d - d2) > 0.0000001) { - LOG.warning("Distance function doesn't appear to be symmetric!"); - } - } - try { - matrix.getRecordBuffer(DBIDUtil.asInteger(id1), DBIDUtil.asInteger(id2)).putDouble(d); - } catch (IOException e) { - throw new AbortException("Error writing distance record " + DBIDFactory.FACTORY.toString(id1) + "," + DBIDFactory.FACTORY.toString(id2) + " to matrix.", e); + DBIDArrayIter id1 = ids.iter(), id2 = ids.iter(); + for(; id1.valid(); id1.advance()) { + for(id2.seek(id1.getOffset()); id2.valid(); id2.advance()) { + double d = distanceQuery.distance(id1, id2); + if(debugExtraCheckSymmetry) { + double d2 = distanceQuery.distance(id2, id1); + if(Math.abs(d - d2) > 0.0000001) { + LOG.warning("Distance function doesn't appear to be symmetric!"); } } + try { + matrix.getRecordBuffer(id1.getOffset(), id2.getOffset()).putDouble(d); + } + catch(IOException e) { + throw new AbortException("Error writing distance record " + DBIDUtil.toString(id1) + "," + DBIDUtil.toString(id2) + " to matrix.", e); + } } } } @@ -145,7 +139,7 @@ public class CacheDoubleDistanceInOnDiskMatrix<O, D extends NumberDistance<D, ?> * * @apiviz.exclude */ - public static class Parameterizer<O, D extends NumberDistance<D, ?>> extends AbstractApplication.Parameterizer { + public static class Parameterizer<O> extends AbstractApplication.Parameterizer { /** * Parameter that specifies the name of the directory to be re-parsed. * <p> @@ -170,7 +164,7 @@ public class CacheDoubleDistanceInOnDiskMatrix<O, D extends NumberDistance<D, ?> /** * Distance function that is to be cached. */ - private DistanceFunction<O, D> distance = null; + private DistanceFunction<O> distance = null; /** * Output file. @@ -182,19 +176,19 @@ public class CacheDoubleDistanceInOnDiskMatrix<O, D extends NumberDistance<D, ?> super.makeOptions(config); input = config.tryInstantiate(InputStep.class); // Distance function parameter - final ObjectParameter<DistanceFunction<O, D>> dpar = new ObjectParameter<>(DISTANCE_ID, DistanceFunction.class); - if (config.grab(dpar)) { + final ObjectParameter<DistanceFunction<O>> dpar = new ObjectParameter<>(DISTANCE_ID, DistanceFunction.class); + if(config.grab(dpar)) { distance = dpar.instantiateClass(config); } // Output file parameter final FileParameter cpar = new FileParameter(CACHE_ID, FileParameter.FileType.OUTPUT_FILE); - if (config.grab(cpar)) { + if(config.grab(cpar)) { out = cpar.getValue(); } } @Override - protected CacheDoubleDistanceInOnDiskMatrix<O, D> makeInstance() { + protected CacheDoubleDistanceInOnDiskMatrix<O> makeInstance() { return new CacheDoubleDistanceInOnDiskMatrix<>(input, distance, out); } } diff --git a/src/de/lmu/ifi/dbs/elki/application/cache/CacheDoubleDistanceKNNLists.java b/src/de/lmu/ifi/dbs/elki/application/cache/CacheDoubleDistanceKNNLists.java index d8ddbf17..1361620f 100644 --- a/src/de/lmu/ifi/dbs/elki/application/cache/CacheDoubleDistanceKNNLists.java +++ b/src/de/lmu/ifi/dbs/elki/application/cache/CacheDoubleDistanceKNNLists.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.application.cache; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2013 + Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -33,20 +33,17 @@ import java.nio.channels.FileLock; import de.lmu.ifi.dbs.elki.application.AbstractApplication; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; -import de.lmu.ifi.dbs.elki.database.ids.distance.DistanceDBIDListIter; -import de.lmu.ifi.dbs.elki.database.ids.distance.DoubleDistanceDBIDList; -import de.lmu.ifi.dbs.elki.database.ids.distance.DoubleDistanceDBIDListIter; -import de.lmu.ifi.dbs.elki.database.ids.distance.KNNList; +import de.lmu.ifi.dbs.elki.database.ids.DoubleDBIDListIter; +import de.lmu.ifi.dbs.elki.database.ids.KNNList; import de.lmu.ifi.dbs.elki.database.query.DatabaseQuery; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction; -import de.lmu.ifi.dbs.elki.distance.distancevalue.NumberDistance; import de.lmu.ifi.dbs.elki.logging.Logging; import de.lmu.ifi.dbs.elki.logging.progress.FiniteProgress; -import de.lmu.ifi.dbs.elki.persistent.ByteArrayUtil; import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; +import de.lmu.ifi.dbs.elki.utilities.io.ByteArrayUtil; import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.CommonConstraints; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; @@ -63,9 +60,8 @@ import de.lmu.ifi.dbs.elki.workflow.InputStep; * @apiviz.has DistanceFunction * * @param <O> Object type - * @param <D> Distance type */ -public class CacheDoubleDistanceKNNLists<O, D extends NumberDistance<D, ?>> extends AbstractApplication { +public class CacheDoubleDistanceKNNLists<O> extends AbstractApplication { /** * The logger for this class. */ @@ -79,7 +75,7 @@ public class CacheDoubleDistanceKNNLists<O, D extends NumberDistance<D, ?>> exte /** * Distance function that is to be cached. */ - private DistanceFunction<O, D> distance; + private DistanceFunction<O> distance; /** * Number of neighbors to precompute. @@ -107,7 +103,7 @@ public class CacheDoubleDistanceKNNLists<O, D extends NumberDistance<D, ?>> exte * @param k Number of nearest neighbors * @param out Matrix output file */ - public CacheDoubleDistanceKNNLists(InputStep input, DistanceFunction<O, D> distance, int k, File out) { + public CacheDoubleDistanceKNNLists(InputStep input, DistanceFunction<O> distance, int k, File out) { super(); this.input = input; this.distance = distance; @@ -119,8 +115,8 @@ public class CacheDoubleDistanceKNNLists<O, D extends NumberDistance<D, ?>> exte public void run() { Database database = input.getDatabase(); Relation<O> relation = database.getRelation(distance.getInputTypeRestriction()); - DistanceQuery<O, D> distanceQuery = database.getDistanceQuery(relation, distance); - KNNQuery<O, D> knnQ = database.getKNNQuery(distanceQuery, DatabaseQuery.HINT_HEAVY_USE); + DistanceQuery<O> distanceQuery = database.getDistanceQuery(relation, distance); + KNNQuery<O> knnQ = database.getKNNQuery(distanceQuery, DatabaseQuery.HINT_HEAVY_USE); // open file. try (RandomAccessFile file = new RandomAccessFile(out, "rw"); @@ -136,7 +132,7 @@ public class CacheDoubleDistanceKNNLists<O, D extends NumberDistance<D, ?>> exte FiniteProgress prog = LOG.isVerbose() ? new FiniteProgress("Computing kNN", relation.size(), LOG) : null; for(DBIDIter it = relation.iterDBIDs(); it.valid(); it.advance()) { - final KNNList<D> nn = knnQ.getKNNForDBID(it, k); + final KNNList nn = knnQ.getKNNForDBID(it, k); final int nnsize = nn.size(); // Grow the buffer when needed: @@ -151,17 +147,9 @@ public class CacheDoubleDistanceKNNLists<O, D extends NumberDistance<D, ?>> exte ByteArrayUtil.writeUnsignedVarint(buffer, it.internalGetIndex()); ByteArrayUtil.writeUnsignedVarint(buffer, nnsize); int c = 0; - if(nn instanceof DoubleDistanceDBIDList) { - for(DoubleDistanceDBIDListIter ni = ((DoubleDistanceDBIDList) nn).iter(); ni.valid(); ni.advance(), c++) { - ByteArrayUtil.writeUnsignedVarint(buffer, ni.internalGetIndex()); - buffer.putDouble(ni.doubleDistance()); - } - } - else { - for(DistanceDBIDListIter<D> ni = nn.iter(); ni.valid(); ni.advance(), c++) { - ByteArrayUtil.writeUnsignedVarint(buffer, ni.internalGetIndex()); - buffer.putDouble(ni.getDistance().doubleValue()); - } + for(DoubleDBIDListIter ni = nn.iter(); ni.valid(); ni.advance(), c++) { + ByteArrayUtil.writeUnsignedVarint(buffer, ni.internalGetIndex()); + buffer.putDouble(ni.doubleValue()); } if(c != nn.size()) { throw new AbortException("Sizes did not agree. Cache is invalid."); @@ -169,13 +157,9 @@ public class CacheDoubleDistanceKNNLists<O, D extends NumberDistance<D, ?>> exte buffer.flip(); channel.write(buffer); - if(prog != null) { - prog.incrementProcessed(LOG); - } - } - if(prog != null) { - prog.ensureCompleted(LOG); + LOG.incrementProcessed(prog); } + LOG.ensureCompleted(prog); lock.release(); } catch(IOException e) { @@ -191,7 +175,7 @@ public class CacheDoubleDistanceKNNLists<O, D extends NumberDistance<D, ?>> exte * * @apiviz.exclude */ - public static class Parameterizer<O, D extends NumberDistance<D, ?>> extends AbstractApplication.Parameterizer { + public static class Parameterizer<O> extends AbstractApplication.Parameterizer { /** * Parameter that specifies the name of the directory to be re-parsed. * <p> @@ -224,7 +208,7 @@ public class CacheDoubleDistanceKNNLists<O, D extends NumberDistance<D, ?>> exte /** * Distance function that is to be cached. */ - private DistanceFunction<O, D> distance = null; + private DistanceFunction<O> distance = null; /** * Number of neighbors to precompute. @@ -241,7 +225,7 @@ public class CacheDoubleDistanceKNNLists<O, D extends NumberDistance<D, ?>> exte super.makeOptions(config); input = config.tryInstantiate(InputStep.class); // Distance function parameter - final ObjectParameter<DistanceFunction<O, D>> dpar = new ObjectParameter<>(DISTANCE_ID, DistanceFunction.class); + final ObjectParameter<DistanceFunction<O>> dpar = new ObjectParameter<>(DISTANCE_ID, DistanceFunction.class); if(config.grab(dpar)) { distance = dpar.instantiateClass(config); } @@ -258,7 +242,7 @@ public class CacheDoubleDistanceKNNLists<O, D extends NumberDistance<D, ?>> exte } @Override - protected CacheDoubleDistanceKNNLists<O, D> makeInstance() { + protected CacheDoubleDistanceKNNLists<O> makeInstance() { return new CacheDoubleDistanceKNNLists<>(input, distance, k, out); } } diff --git a/src/de/lmu/ifi/dbs/elki/application/cache/CacheDoubleDistanceRangeQueries.java b/src/de/lmu/ifi/dbs/elki/application/cache/CacheDoubleDistanceRangeQueries.java index f61874dd..bf9ecdb0 100644 --- a/src/de/lmu/ifi/dbs/elki/application/cache/CacheDoubleDistanceRangeQueries.java +++ b/src/de/lmu/ifi/dbs/elki/application/cache/CacheDoubleDistanceRangeQueries.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.application.cache; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2013 + Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -33,20 +33,17 @@ import java.nio.channels.FileLock; import de.lmu.ifi.dbs.elki.application.AbstractApplication; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; -import de.lmu.ifi.dbs.elki.database.ids.distance.DistanceDBIDList; -import de.lmu.ifi.dbs.elki.database.ids.distance.DistanceDBIDListIter; -import de.lmu.ifi.dbs.elki.database.ids.distance.DoubleDistanceDBIDList; -import de.lmu.ifi.dbs.elki.database.ids.distance.DoubleDistanceDBIDListIter; +import de.lmu.ifi.dbs.elki.database.ids.DoubleDBIDList; +import de.lmu.ifi.dbs.elki.database.ids.DoubleDBIDListIter; import de.lmu.ifi.dbs.elki.database.query.DatabaseQuery; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.query.range.RangeQuery; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction; -import de.lmu.ifi.dbs.elki.distance.distancevalue.DoubleDistance; import de.lmu.ifi.dbs.elki.logging.Logging; import de.lmu.ifi.dbs.elki.logging.progress.FiniteProgress; -import de.lmu.ifi.dbs.elki.persistent.ByteArrayUtil; import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; +import de.lmu.ifi.dbs.elki.utilities.io.ByteArrayUtil; import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.CommonConstraints; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; @@ -78,7 +75,7 @@ public class CacheDoubleDistanceRangeQueries<O> extends AbstractApplication { /** * Distance function that is to be cached. */ - private DistanceFunction<O, DoubleDistance> distance; + private DistanceFunction<O> distance; /** * Query radius. @@ -106,7 +103,7 @@ public class CacheDoubleDistanceRangeQueries<O> extends AbstractApplication { * @param radius Query radius * @param out Matrix output file */ - public CacheDoubleDistanceRangeQueries(InputStep input, DistanceFunction<O, DoubleDistance> distance, double radius, File out) { + public CacheDoubleDistanceRangeQueries(InputStep input, DistanceFunction<O> distance, double radius, File out) { super(); this.input = input; this.distance = distance; @@ -118,11 +115,10 @@ public class CacheDoubleDistanceRangeQueries<O> extends AbstractApplication { public void run() { Database database = input.getDatabase(); Relation<O> relation = database.getRelation(distance.getInputTypeRestriction()); - DistanceQuery<O, DoubleDistance> distanceQuery = database.getDistanceQuery(relation, distance); - DoubleDistance rad = new DoubleDistance(radius); - RangeQuery<O, DoubleDistance> rangeQ = database.getRangeQuery(distanceQuery, rad, DatabaseQuery.HINT_HEAVY_USE); + DistanceQuery<O> distanceQuery = database.getDistanceQuery(relation, distance); + RangeQuery<O> rangeQ = database.getRangeQuery(distanceQuery, radius, DatabaseQuery.HINT_HEAVY_USE); - LOG.verbose("Performing range queries with radius " + rad); + LOG.verbose("Performing range queries with radius " + radius); // open file. try (RandomAccessFile file = new RandomAccessFile(out, "rw"); @@ -139,16 +135,16 @@ public class CacheDoubleDistanceRangeQueries<O> extends AbstractApplication { FiniteProgress prog = LOG.isVerbose() ? new FiniteProgress("Computing range queries", relation.size(), LOG) : null; - for (DBIDIter it = relation.iterDBIDs(); it.valid(); it.advance()) { - final DistanceDBIDList<DoubleDistance> nn = rangeQ.getRangeForDBID(it, rad); + for(DBIDIter it = relation.iterDBIDs(); it.valid(); it.advance()) { + final DoubleDBIDList nn = rangeQ.getRangeForDBID(it, radius); final int nnsize = nn.size(); // Grow the buffer when needed: - if (nnsize * 12 + 10 > bufsize) { - while (nnsize * 12 + 10 > bufsize) { + if(nnsize * 12 + 10 > bufsize) { + while(nnsize * 12 + 10 > bufsize) { bufsize <<= 1; } - LOG.verbose("Resizing buffer to "+bufsize+" to store "+nnsize+" results:"); + LOG.verbose("Resizing buffer to " + bufsize + " to store " + nnsize + " results:"); buffer = ByteBuffer.allocateDirect(bufsize); } @@ -156,32 +152,22 @@ public class CacheDoubleDistanceRangeQueries<O> extends AbstractApplication { ByteArrayUtil.writeUnsignedVarint(buffer, it.internalGetIndex()); ByteArrayUtil.writeUnsignedVarint(buffer, nnsize); int c = 0; - if (nn instanceof DoubleDistanceDBIDList) { - for (DoubleDistanceDBIDListIter ni = ((DoubleDistanceDBIDList) nn).iter(); ni.valid(); ni.advance(), c++) { - ByteArrayUtil.writeUnsignedVarint(buffer, ni.internalGetIndex()); - buffer.putDouble(ni.doubleDistance()); - } - } else { - for (DistanceDBIDListIter<DoubleDistance> ni = nn.iter(); ni.valid(); ni.advance(), c++) { - ByteArrayUtil.writeUnsignedVarint(buffer, ni.internalGetIndex()); - buffer.putDouble(ni.getDistance().doubleValue()); - } + for(DoubleDBIDListIter ni = nn.iter(); ni.valid(); ni.advance(), c++) { + ByteArrayUtil.writeUnsignedVarint(buffer, ni.internalGetIndex()); + buffer.putDouble(ni.doubleValue()); } - if (c != nn.size()) { + if(c != nn.size()) { throw new AbortException("Sizes did not agree. Cache is invalid."); } buffer.flip(); channel.write(buffer); - if (prog != null) { - prog.incrementProcessed(LOG); - } - } - if (prog != null) { - prog.ensureCompleted(LOG); + LOG.incrementProcessed(prog); } + LOG.ensureCompleted(prog); lock.release(); - } catch (IOException e) { + } + catch(IOException e) { LOG.exception(e); } // FIXME: close! @@ -227,7 +213,7 @@ public class CacheDoubleDistanceRangeQueries<O> extends AbstractApplication { /** * Distance function that is to be cached. */ - private DistanceFunction<O, DoubleDistance> distance = null; + private DistanceFunction<O> distance = null; /** * Number of neighbors to precompute. @@ -244,18 +230,18 @@ public class CacheDoubleDistanceRangeQueries<O> extends AbstractApplication { super.makeOptions(config); input = config.tryInstantiate(InputStep.class); // Distance function parameter - final ObjectParameter<DistanceFunction<O, DoubleDistance>> dpar = new ObjectParameter<>(DISTANCE_ID, DistanceFunction.class); - if (config.grab(dpar)) { + final ObjectParameter<DistanceFunction<O>> dpar = new ObjectParameter<>(DISTANCE_ID, DistanceFunction.class); + if(config.grab(dpar)) { distance = dpar.instantiateClass(config); } final DoubleParameter kpar = new DoubleParameter(RADIUS_ID); kpar.addConstraint(CommonConstraints.GREATER_EQUAL_ZERO_DOUBLE); - if (config.grab(kpar)) { + if(config.grab(kpar)) { radius = kpar.doubleValue(); } // Output file parameter final FileParameter cpar = new FileParameter(CACHE_ID, FileParameter.FileType.OUTPUT_FILE); - if (config.grab(cpar)) { + if(config.grab(cpar)) { out = cpar.getValue(); } } diff --git a/src/de/lmu/ifi/dbs/elki/application/cache/CacheFloatDistanceInOnDiskMatrix.java b/src/de/lmu/ifi/dbs/elki/application/cache/CacheFloatDistanceInOnDiskMatrix.java index 5499415b..e4865a6b 100644 --- a/src/de/lmu/ifi/dbs/elki/application/cache/CacheFloatDistanceInOnDiskMatrix.java +++ b/src/de/lmu/ifi/dbs/elki/application/cache/CacheFloatDistanceInOnDiskMatrix.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.application.cache; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2013 + Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -28,23 +28,24 @@ import java.io.IOException; import de.lmu.ifi.dbs.elki.application.AbstractApplication; import de.lmu.ifi.dbs.elki.database.Database; -import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDArrayIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRange; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction; import de.lmu.ifi.dbs.elki.distance.distancefunction.external.DiskCacheBasedFloatDistanceFunction; -import de.lmu.ifi.dbs.elki.distance.distancevalue.NumberDistance; import de.lmu.ifi.dbs.elki.logging.Logging; import de.lmu.ifi.dbs.elki.persistent.OnDiskUpperTriangleMatrix; import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; +import de.lmu.ifi.dbs.elki.utilities.io.ByteArrayUtil; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.FileParameter; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter; import de.lmu.ifi.dbs.elki.workflow.InputStep; /** - * Precompute an on-disk distance matrix, using double precision. + * Precompute an on-disk distance matrix, using float precision. * * @author Erich Schubert * @@ -52,9 +53,8 @@ import de.lmu.ifi.dbs.elki.workflow.InputStep; * @apiviz.has DistanceFunction * * @param <O> Object type - * @param <D> Distance type */ -public class CacheFloatDistanceInOnDiskMatrix<O, D extends NumberDistance<D, ?>> extends AbstractApplication { +public class CacheFloatDistanceInOnDiskMatrix<O> extends AbstractApplication { /** * The logger for this class. */ @@ -66,11 +66,6 @@ public class CacheFloatDistanceInOnDiskMatrix<O, D extends NumberDistance<D, ?>> private static final boolean debugExtraCheckSymmetry = false; /** - * Storage size: 4 bytes floats - */ - private static final int FLOAT_SIZE = 4; - - /** * Data source to process. */ private InputStep input; @@ -78,7 +73,7 @@ public class CacheFloatDistanceInOnDiskMatrix<O, D extends NumberDistance<D, ?>> /** * Distance function that is to be cached. */ - private DistanceFunction<O, D> distance; + private DistanceFunction<O> distance; /** * Output file. @@ -92,7 +87,7 @@ public class CacheFloatDistanceInOnDiskMatrix<O, D extends NumberDistance<D, ?>> * @param distance Distance function * @param out Matrix output file */ - public CacheFloatDistanceInOnDiskMatrix(InputStep input, DistanceFunction<O, D> distance, File out) { + public CacheFloatDistanceInOnDiskMatrix(InputStep input, DistanceFunction<O> distance, File out) { super(); this.input = input; this.distance = distance; @@ -103,40 +98,35 @@ public class CacheFloatDistanceInOnDiskMatrix<O, D extends NumberDistance<D, ?>> public void run() { Database database = input.getDatabase(); Relation<O> relation = database.getRelation(distance.getInputTypeRestriction()); - DistanceQuery<O, D> distanceQuery = database.getDistanceQuery(relation, distance); - - int matrixsize = 0; - for (DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { - final int intid = DBIDUtil.asInteger(iditer); - matrixsize = Math.max(matrixsize, intid + 1); - if (intid < 0) { - throw new AbortException("OnDiskMatrixCache does not allow negative DBIDs."); - } - } + DistanceQuery<O> distanceQuery = database.getDistanceQuery(relation, distance); + + DBIDRange ids = DBIDUtil.assertRange(relation.getDBIDs()); + int matrixsize = ids.size(); OnDiskUpperTriangleMatrix matrix; try { - matrix = new OnDiskUpperTriangleMatrix(out, DiskCacheBasedFloatDistanceFunction.FLOAT_CACHE_MAGIC, 0, FLOAT_SIZE, matrixsize); - } catch (IOException e) { + matrix = new OnDiskUpperTriangleMatrix(out, DiskCacheBasedFloatDistanceFunction.FLOAT_CACHE_MAGIC, 0, ByteArrayUtil.SIZE_FLOAT, matrixsize); + } + catch(IOException e) { throw new AbortException("Error creating output matrix.", e); } - for (DBIDIter id1 = relation.iterDBIDs(); id1.valid(); id1.advance()) { - for (DBIDIter id2 = relation.iterDBIDs(); id2.valid(); id2.advance()) { - if (DBIDUtil.asInteger(id2) >= DBIDUtil.asInteger(id1)) { - float d = distanceQuery.distance(id1, id2).floatValue(); - if (debugExtraCheckSymmetry) { - float d2 = distanceQuery.distance(id2, id1).floatValue(); - if (Math.abs(d - d2) > 0.0000001) { - LOG.warning("Distance function doesn't appear to be symmetric!"); - } - } - try { - matrix.getRecordBuffer(DBIDUtil.asInteger(id1), DBIDUtil.asInteger(id2)).putFloat(d); - } catch (IOException e) { - throw new AbortException("Error writing distance record " + id1 + "," + id2 + " to matrix.", e); + DBIDArrayIter id1 = ids.iter(), id2 = ids.iter(); + for(; id1.valid(); id1.advance()) { + for(id2.seek(id1.getOffset()); id2.valid(); id2.advance()) { + float d = (float) distanceQuery.distance(id1, id2); + if(debugExtraCheckSymmetry) { + float d2 = (float) distanceQuery.distance(id2, id1); + if(Math.abs(d - d2) > 0.0000001) { + LOG.warning("Distance function doesn't appear to be symmetric!"); } } + try { + matrix.getRecordBuffer(id1.getOffset(), id2.getOffset()).putFloat(d); + } + catch(IOException e) { + throw new AbortException("Error writing distance record " + DBIDUtil.toString(id1) + "," + DBIDUtil.toString(id2) + " to matrix.", e); + } } } } @@ -148,7 +138,7 @@ public class CacheFloatDistanceInOnDiskMatrix<O, D extends NumberDistance<D, ?>> * * @apiviz.exclude */ - public static class Parameterizer<O, D extends NumberDistance<D, ?>> extends AbstractApplication.Parameterizer { + public static class Parameterizer<O> extends AbstractApplication.Parameterizer { /** * Data source to process. */ @@ -157,7 +147,7 @@ public class CacheFloatDistanceInOnDiskMatrix<O, D extends NumberDistance<D, ?>> /** * Distance function that is to be cached. */ - private DistanceFunction<O, D> distance = null; + private DistanceFunction<O> distance = null; /** * Output file. @@ -169,19 +159,19 @@ public class CacheFloatDistanceInOnDiskMatrix<O, D extends NumberDistance<D, ?>> super.makeOptions(config); input = config.tryInstantiate(InputStep.class); // Distance function parameter - final ObjectParameter<DistanceFunction<O, D>> dpar = new ObjectParameter<>(CacheDoubleDistanceInOnDiskMatrix.Parameterizer.DISTANCE_ID, DistanceFunction.class); - if (config.grab(dpar)) { + final ObjectParameter<DistanceFunction<O>> dpar = new ObjectParameter<>(CacheDoubleDistanceInOnDiskMatrix.Parameterizer.DISTANCE_ID, DistanceFunction.class); + if(config.grab(dpar)) { distance = dpar.instantiateClass(config); } // Output file parameter final FileParameter cpar = new FileParameter(CacheDoubleDistanceInOnDiskMatrix.Parameterizer.CACHE_ID, FileParameter.FileType.OUTPUT_FILE); - if (config.grab(cpar)) { + if(config.grab(cpar)) { out = cpar.getValue(); } } @Override - protected CacheFloatDistanceInOnDiskMatrix<O, D> makeInstance() { + protected CacheFloatDistanceInOnDiskMatrix<O> makeInstance() { return new CacheFloatDistanceInOnDiskMatrix<>(input, distance, out); } } diff --git a/src/de/lmu/ifi/dbs/elki/application/cache/package-info.java b/src/de/lmu/ifi/dbs/elki/application/cache/package-info.java index 78a93a07..3327947b 100644 --- a/src/de/lmu/ifi/dbs/elki/application/cache/package-info.java +++ b/src/de/lmu/ifi/dbs/elki/application/cache/package-info.java @@ -11,7 +11,7 @@ This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures -Copyright (C) 2013 +Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team diff --git a/src/de/lmu/ifi/dbs/elki/application/geo/VisualizeGeodesicDistances.java b/src/de/lmu/ifi/dbs/elki/application/geo/VisualizeGeodesicDistances.java index 75770ba7..dfeb0349 100644 --- a/src/de/lmu/ifi/dbs/elki/application/geo/VisualizeGeodesicDistances.java +++ b/src/de/lmu/ifi/dbs/elki/application/geo/VisualizeGeodesicDistances.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.application.geo; This file is part of ELKI:
Environment for Developing KDD-Applications Supported by Index-Structures
- Copyright (C) 2012
+ Copyright (C) 2014
Ludwig-Maximilians-Universität München
Lehr- und Forschungseinheit für Datenbanksysteme
ELKI Development Team
@@ -175,13 +175,9 @@ public class VisualizeGeodesicDistances extends AbstractApplication { }
}
}
- if (prog != null) {
- prog.incrementProcessed(LOG);
- }
- }
- if (prog != null) {
- prog.ensureCompleted(LOG);
+ LOG.incrementProcessed(prog);
}
+ LOG.ensureCompleted(prog);
try {
ImageIO.write(img, "png", out);
diff --git a/src/de/lmu/ifi/dbs/elki/application/geo/package-info.java b/src/de/lmu/ifi/dbs/elki/application/geo/package-info.java index 3c60474d..3e067d67 100644 --- a/src/de/lmu/ifi/dbs/elki/application/geo/package-info.java +++ b/src/de/lmu/ifi/dbs/elki/application/geo/package-info.java @@ -5,7 +5,7 @@ This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures -Copyright (C) 2013 +Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team diff --git a/src/de/lmu/ifi/dbs/elki/application/greedyensemble/ComputeKNNOutlierScores.java b/src/de/lmu/ifi/dbs/elki/application/greedyensemble/ComputeKNNOutlierScores.java index a890172c..6ddcfa84 100644 --- a/src/de/lmu/ifi/dbs/elki/application/greedyensemble/ComputeKNNOutlierScores.java +++ b/src/de/lmu/ifi/dbs/elki/application/greedyensemble/ComputeKNNOutlierScores.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.application.greedyensemble; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2013 + Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -29,9 +29,11 @@ import java.io.PrintStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import javax.xml.bind.DatatypeConverter; + import de.lmu.ifi.dbs.elki.algorithm.AbstractAlgorithm; -import de.lmu.ifi.dbs.elki.algorithm.outlier.KNNOutlier; -import de.lmu.ifi.dbs.elki.algorithm.outlier.KNNWeightOutlier; +import de.lmu.ifi.dbs.elki.algorithm.outlier.distance.KNNOutlier; +import de.lmu.ifi.dbs.elki.algorithm.outlier.distance.KNNWeightOutlier; import de.lmu.ifi.dbs.elki.algorithm.outlier.lof.LDF; import de.lmu.ifi.dbs.elki.algorithm.outlier.lof.LDOF; import de.lmu.ifi.dbs.elki.algorithm.outlier.lof.LOF; @@ -49,15 +51,14 @@ import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery; import de.lmu.ifi.dbs.elki.database.query.knn.PreprocessorKNNQuery; +import de.lmu.ifi.dbs.elki.database.relation.DoubleRelation; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction; import de.lmu.ifi.dbs.elki.distance.distancefunction.minkowski.EuclideanDistanceFunction; -import de.lmu.ifi.dbs.elki.distance.distancevalue.NumberDistance; -import de.lmu.ifi.dbs.elki.index.preprocessed.knn.MaterializeKNNPreprocessor; import de.lmu.ifi.dbs.elki.logging.Logging; import de.lmu.ifi.dbs.elki.math.statistics.kernelfunctions.GaussianKernelDensityFunction; import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; -import de.lmu.ifi.dbs.elki.utilities.Base64; +import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; import de.lmu.ifi.dbs.elki.utilities.FormatUtil; import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; @@ -85,9 +86,11 @@ import de.lmu.ifi.dbs.elki.workflow.InputStep; * </p> * * @author Erich Schubert + * + * @param <O> Vector type */ @Reference(authors = "E. Schubert, R. Wojdanowski, A. Zimek, H.-P. Kriegel", title = "On Evaluation of Outlier Rankings and Outlier Scores", booktitle = "Proc. 12th SIAM International Conference on Data Mining (SDM), Anaheim, CA, 2012.") -public class ComputeKNNOutlierScores<O extends NumberVector<?>, D extends NumberDistance<D, ?>> extends AbstractApplication { +public class ComputeKNNOutlierScores<O extends NumberVector> extends AbstractApplication { /** * Our logger class. */ @@ -101,7 +104,7 @@ public class ComputeKNNOutlierScores<O extends NumberVector<?>, D extends Number /** * Distance function to use */ - final DistanceFunction<? super O, D> distf; + final DistanceFunction<? super O> distf; /** * Starting value of k. @@ -145,7 +148,7 @@ public class ComputeKNNOutlierScores<O extends NumberVector<?>, D extends Number * @param outfile Output file * @param scaling Scaling function */ - public ComputeKNNOutlierScores(InputStep inputstep, DistanceFunction<? super O, D> distf, int startk, int stepk, int maxk, ByLabelOutlier bylabel, File outfile, ScalingFunction scaling) { + public ComputeKNNOutlierScores(InputStep inputstep, DistanceFunction<? super O> distf, int startk, int stepk, int maxk, ByLabelOutlier bylabel, File outfile, ScalingFunction scaling) { super(); this.distf = distf; this.startk = startk; @@ -163,12 +166,7 @@ public class ComputeKNNOutlierScores<O extends NumberVector<?>, D extends Number final Relation<O> relation = database.getRelation(distf.getInputTypeRestriction()); // If there is no kNN preprocessor already, then precompute. - KNNQuery<O, D> knnq = QueryUtil.getKNNQuery(relation, distf, maxk + 2); - if(!(knnq instanceof PreprocessorKNNQuery)) { - LOG.verbose("Running preprocessor ..."); - MaterializeKNNPreprocessor<O, D> preproc = new MaterializeKNNPreprocessor<>(relation, distf, maxk + 2); - database.addIndex(preproc); - } + KNNQuery<O> knnq = DatabaseUtil.precomputedKNNQuery(database, relation, distf, maxk + 2); // Test that we now get a proper index query knnq = QueryUtil.getKNNQuery(relation, distf, maxk + 2); @@ -194,7 +192,7 @@ public class ComputeKNNOutlierScores<O extends NumberVector<?>, D extends Number md.update(DBIDUtil.toString(iter).getBytes()); } fout.append("# DBID-series MD5:"); - fout.append(Base64.encodeBase64(md.digest())); + fout.append(DatatypeConverter.printBase64Binary(md.digest())); fout.append(FormatUtil.NEWLINE); } catch(NoSuchAlgorithmException e) { @@ -221,7 +219,7 @@ public class ComputeKNNOutlierScores<O extends NumberVector<?>, D extends Number runForEachK(new AlgRunner() { @Override public void run(int k, String kstr) { - KNNOutlier<O, D> knn = new KNNOutlier<>(distf, k); + KNNOutlier<O> knn = new KNNOutlier<>(distf, k); OutlierResult knnresult = knn.run(database, relation); writeResult(fout, ids, knnresult, scaling, "KNN-" + kstr); database.getHierarchy().removeSubtree(knnresult); @@ -232,7 +230,7 @@ public class ComputeKNNOutlierScores<O extends NumberVector<?>, D extends Number runForEachK(new AlgRunner() { @Override public void run(int k, String kstr) { - KNNWeightOutlier<O, D> knnw = new KNNWeightOutlier<>(distf, k); + KNNWeightOutlier<O> knnw = new KNNWeightOutlier<>(distf, k); OutlierResult knnresult = knnw.run(database, relation); writeResult(fout, ids, knnresult, scaling, "KNNW-" + kstr); database.getHierarchy().removeSubtree(knnresult); @@ -243,7 +241,7 @@ public class ComputeKNNOutlierScores<O extends NumberVector<?>, D extends Number runForEachK(new AlgRunner() { @Override public void run(int k, String kstr) { - LOF<O, D> lof = new LOF<>(k, distf); + LOF<O> lof = new LOF<>(k, distf); OutlierResult lofresult = lof.run(database, relation); writeResult(fout, ids, lofresult, scaling, "LOF-" + kstr); database.getHierarchy().removeSubtree(lofresult); @@ -254,7 +252,7 @@ public class ComputeKNNOutlierScores<O extends NumberVector<?>, D extends Number runForEachK(new AlgRunner() { @Override public void run(int k, String kstr) { - SimplifiedLOF<O, D> lof = new SimplifiedLOF<>(k, distf); + SimplifiedLOF<O> lof = new SimplifiedLOF<>(k, distf); OutlierResult lofresult = lof.run(database, relation); writeResult(fout, ids, lofresult, scaling, "Simplified-LOF-" + kstr); database.getHierarchy().removeSubtree(lofresult); @@ -265,7 +263,7 @@ public class ComputeKNNOutlierScores<O extends NumberVector<?>, D extends Number runForEachK(new AlgRunner() { @Override public void run(int k, String kstr) { - LoOP<O, D> loop = new LoOP<>(k, k, distf, distf, 1.0); + LoOP<O> loop = new LoOP<>(k, k, distf, distf, 1.0); OutlierResult loopresult = loop.run(database, relation); writeResult(fout, ids, loopresult, scaling, "LOOP-" + kstr); database.getHierarchy().removeSubtree(loopresult); @@ -278,7 +276,7 @@ public class ComputeKNNOutlierScores<O extends NumberVector<?>, D extends Number runForEachK(new AlgRunner() { @Override public void run(int k, String kstr) { - LDOF<O, D> ldof = new LDOF<>(distf, k + 1); + LDOF<O> ldof = new LDOF<>(distf, k); OutlierResult ldofresult = ldof.run(database, relation); writeResult(fout, ids, ldofresult, scaling, "LDOF-" + kstr); database.getHierarchy().removeSubtree(ldofresult); @@ -290,7 +288,7 @@ public class ComputeKNNOutlierScores<O extends NumberVector<?>, D extends Number runForEachK(new AlgRunner() { @Override public void run(int k, String kstr) { - LDF<O, D> ldf = new LDF<>(k, distf, GaussianKernelDensityFunction.KERNEL, 2, .1); + LDF<O> ldf = new LDF<>(k, distf, GaussianKernelDensityFunction.KERNEL, 2, .1); OutlierResult ldfresult = ldf.run(database, relation); writeResult(fout, ids, ldfresult, scaling, "LDF-" + kstr); database.getHierarchy().removeSubtree(ldfresult); @@ -312,13 +310,13 @@ public class ComputeKNNOutlierScores<O extends NumberVector<?>, D extends Number ((OutlierScalingFunction) scaling).prepare(result); } out.append(label); - Relation<Double> scores = result.getScores(); + DoubleRelation scores = result.getScores(); for(DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { - double value = scores.get(iter); + double value = scores.doubleValue(iter); if(scaling != null) { value = scaling.getScaled(value); } - out.append(' ').append(FormatUtil.NF.format(value)); + out.append(' ').append(Double.toString(value)); } out.append(FormatUtil.NEWLINE); } @@ -355,7 +353,7 @@ public class ComputeKNNOutlierScores<O extends NumberVector<?>, D extends Number * * @apiviz.exclude */ - public static class Parameterizer<O extends NumberVector<?>, D extends NumberDistance<D, ?>> extends AbstractApplication.Parameterizer { + public static class Parameterizer<O extends NumberVector> extends AbstractApplication.Parameterizer { /** * Option ID for k step size. */ @@ -399,7 +397,7 @@ public class ComputeKNNOutlierScores<O extends NumberVector<?>, D extends Number /** * Distance function to use */ - DistanceFunction<? super O, D> distf; + DistanceFunction<? super O> distf; /** * By label outlier -- reference @@ -422,7 +420,7 @@ public class ComputeKNNOutlierScores<O extends NumberVector<?>, D extends Number // Data input inputstep = config.tryInstantiate(InputStep.class); // Distance function - ObjectParameter<DistanceFunction<? super O, D>> distP = AbstractAlgorithm.makeParameterDistanceFunction(EuclideanDistanceFunction.class, DistanceFunction.class); + ObjectParameter<DistanceFunction<? super O>> distP = AbstractAlgorithm.makeParameterDistanceFunction(EuclideanDistanceFunction.class, DistanceFunction.class); if(config.grab(distP)) { distf = distP.instantiateClass(config); } @@ -457,7 +455,7 @@ public class ComputeKNNOutlierScores<O extends NumberVector<?>, D extends Number } @Override - protected ComputeKNNOutlierScores<O, D> makeInstance() { + protected ComputeKNNOutlierScores<O> makeInstance() { return new ComputeKNNOutlierScores<>(inputstep, distf, startk, stepk, maxk, bylabel, outfile, scaling); } } diff --git a/src/de/lmu/ifi/dbs/elki/application/greedyensemble/GreedyEnsembleExperiment.java b/src/de/lmu/ifi/dbs/elki/application/greedyensemble/GreedyEnsembleExperiment.java index ab4bf001..147b4d28 100644 --- a/src/de/lmu/ifi/dbs/elki/application/greedyensemble/GreedyEnsembleExperiment.java +++ b/src/de/lmu/ifi/dbs/elki/application/greedyensemble/GreedyEnsembleExperiment.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.application.greedyensemble; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2013 + Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -45,15 +45,16 @@ import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.database.relation.RelationUtil; -import de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDoubleDistanceFunction; +import de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDistanceFunction; import de.lmu.ifi.dbs.elki.distance.distancefunction.correlation.WeightedPearsonCorrelationDistanceFunction; import de.lmu.ifi.dbs.elki.distance.distancefunction.minkowski.WeightedEuclideanDistanceFunction; import de.lmu.ifi.dbs.elki.distance.distancefunction.minkowski.WeightedManhattanDistanceFunction; import de.lmu.ifi.dbs.elki.distance.distancefunction.minkowski.WeightedSquaredEuclideanDistanceFunction; -import de.lmu.ifi.dbs.elki.evaluation.roc.ROC; +import de.lmu.ifi.dbs.elki.evaluation.scores.ROCEvaluation; +import de.lmu.ifi.dbs.elki.evaluation.scores.adapter.DecreasingVectorIter; +import de.lmu.ifi.dbs.elki.evaluation.scores.adapter.VectorNonZero; import de.lmu.ifi.dbs.elki.logging.Logging; import de.lmu.ifi.dbs.elki.math.MeanVariance; -import de.lmu.ifi.dbs.elki.math.geometry.XYCurve; import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; import de.lmu.ifi.dbs.elki.utilities.FormatUtil; @@ -180,12 +181,12 @@ public class GreedyEnsembleExperiment extends AbstractApplication { public void run() { // Note: the database contains the *result vectors*, not the original data. final Database database = inputstep.getDatabase(); - Relation<NumberVector<?>> relation = database.getRelation(TypeUtil.NUMBER_VECTOR_FIELD); - final NumberVector.Factory<NumberVector<?>, ?> factory = RelationUtil.getNumberVectorFactory(relation); + Relation<NumberVector> relation = database.getRelation(TypeUtil.NUMBER_VECTOR_FIELD); + final NumberVector.Factory<NumberVector> factory = RelationUtil.getNumberVectorFactory(relation); final Relation<String> labels = DatabaseUtil.guessLabelRepresentation(database); final DBID firstid = DBIDUtil.deref(labels.iterDBIDs()); final String firstlabel = labels.get(firstid); - if (!firstlabel.matches("bylabel")) { + if(!firstlabel.matches("bylabel")) { throw new AbortException("No 'by label' reference outlier found, which is needed for weighting!"); } relation = applyPrescaling(prescaling, relation, firstid); @@ -193,10 +194,10 @@ public class GreedyEnsembleExperiment extends AbstractApplication { // Dimensionality and reference vector final int dim = RelationUtil.dimensionality(relation); - final NumberVector<?> refvec = relation.get(firstid); + final NumberVector refvec = relation.get(firstid); // Build the positive index set for ROC AUC. - ROC.Predicate<ROC.DecreasingVectorIter> positive = new ROC.VectorNonZero(refvec); + VectorNonZero positive = new VectorNonZero(refvec); final int desired_outliers = (int) (rate * dim); int union_outliers = 0; @@ -205,26 +206,26 @@ public class GreedyEnsembleExperiment extends AbstractApplication { // candidates. { int k = 0; - ArrayList<ROC.DecreasingVectorIter> iters = new ArrayList<>(numcand); - if (minvote >= numcand) { + ArrayList<DecreasingVectorIter> iters = new ArrayList<>(numcand); + if(minvote >= numcand) { minvote = Math.max(1, numcand - 1); } - for (DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { // Skip "by label", obviously - if (DBIDUtil.equal(firstid, iditer)) { + if(DBIDUtil.equal(firstid, iditer)) { continue; } - iters.add(new ROC.DecreasingVectorIter(relation.get(iditer))); + iters.add(new DecreasingVectorIter(relation.get(iditer))); } - loop: while (union_outliers < desired_outliers) { - for (ROC.DecreasingVectorIter iter : iters) { - if (!iter.valid()) { + loop: while(union_outliers < desired_outliers) { + for(DecreasingVectorIter iter : iters) { + if(!iter.valid()) { LOG.warning("Union_outliers=" + union_outliers + " < desired_outliers=" + desired_outliers + " minvote=" + minvote); break loop; } int cur = iter.dim(); outliers_seen[cur] += 1; - if (outliers_seen[cur] == minvote) { + if(outliers_seen[cur] == minvote) { union_outliers += 1; } iter.advance(); @@ -237,32 +238,32 @@ public class GreedyEnsembleExperiment extends AbstractApplication { final double[] estimated_weights = new double[dim]; final double[] estimated_truth = new double[dim]; updateEstimations(outliers_seen, union_outliers, estimated_weights, estimated_truth); - NumberVector<?> estimated_truth_vec = factory.newNumberVector(estimated_truth); + NumberVector estimated_truth_vec = factory.newNumberVector(estimated_truth); - PrimitiveDoubleDistanceFunction<NumberVector<?>> wdist = getDistanceFunction(estimated_weights); - PrimitiveDoubleDistanceFunction<NumberVector<?>> tdist = wdist; + PrimitiveDistanceFunction<NumberVector> wdist = getDistanceFunction(estimated_weights); + PrimitiveDistanceFunction<NumberVector> tdist = wdist; // Build the naive ensemble: final double[] naiveensemble = new double[dim]; { double[] buf = new double[numcand]; - for (int d = 0; d < dim; d++) { + for(int d = 0; d < dim; d++) { int i = 0; - for (DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { - if (DBIDUtil.equal(firstid, iditer)) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + if(DBIDUtil.equal(firstid, iditer)) { continue; } - final NumberVector<?> vec = relation.get(iditer); + final NumberVector vec = relation.get(iditer); buf[i] = vec.doubleValue(d); i++; } naiveensemble[d] = voting.combine(buf, i); - if (Double.isNaN(naiveensemble[d])) { + if(Double.isNaN(naiveensemble[d])) { LOG.warning("NaN after combining: " + FormatUtil.format(buf) + " i=" + i + " " + voting.toString()); } } } - NumberVector<?> naivevec = factory.newNumberVector(naiveensemble); + NumberVector naivevec = factory.newNumberVector(naiveensemble); // Compute single AUC scores and estimations. // Remember the method most similar to the estimation @@ -275,27 +276,27 @@ public class GreedyEnsembleExperiment extends AbstractApplication { { final double[] greedyensemble = new double[dim]; // Compute individual scores - for (DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { - if (DBIDUtil.equal(firstid, iditer)) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + if(DBIDUtil.equal(firstid, iditer)) { continue; } // fout.append(labels.get(id)); - final NumberVector<?> vec = relation.get(iditer); + final NumberVector vec = relation.get(iditer); singleEnsemble(greedyensemble, vec); final Vector v2 = new Vector(greedyensemble); - double auc = XYCurve.areaUnderCurve(ROC.materializeROC(positive, new ROC.DecreasingVectorIter(v2))); - double estimated = wdist.doubleDistance(v2, estimated_truth_vec); - double cost = tdist.doubleDistance(v2, refvec); + double auc = ROCEvaluation.computeROCAUC(positive, new DecreasingVectorIter(v2)); + double estimated = wdist.distance(v2, estimated_truth_vec); + double cost = tdist.distance(v2, refvec); LOG.verbose("ROC AUC: " + auc + " estimated " + estimated + " cost " + cost + " " + labels.get(iditer)); - if (auc > bestauc) { + if(auc > bestauc) { bestauc = auc; bestaucstr = labels.get(iditer); } - if (cost < bestcost) { + if(cost < bestcost) { bestcost = cost; bestcoststr = labels.get(iditer); } - if (estimated < bestest || bestid == null) { + if(estimated < bestest || bestid == null) { bestest = estimated; bestid = DBIDUtil.deref(iditer); } @@ -303,12 +304,12 @@ public class GreedyEnsembleExperiment extends AbstractApplication { } // Initialize ensemble with "best" method - if (prescaling != null) { + if(prescaling != null) { LOG.verbose("Input prescaling: " + prescaling); } LOG.verbose("Distance function: " + wdist); LOG.verbose("Ensemble voting: " + voting); - if (scaling != null) { + if(scaling != null) { LOG.verbose("Ensemble rescaling: " + scaling); } LOG.verbose("Initial estimation of outliers: " + union_outliers); @@ -323,31 +324,31 @@ public class GreedyEnsembleExperiment extends AbstractApplication { singleEnsemble(greedyensemble, relation.get(bestid)); // Greedily grow the ensemble final double[] testensemble = new double[dim]; - while (enscands.size() > 0) { - NumberVector<?> greedyvec = factory.newNumberVector(greedyensemble); - final double oldd = wdist.doubleDistance(estimated_truth_vec, greedyvec); + while(enscands.size() > 0) { + NumberVector greedyvec = factory.newNumberVector(greedyensemble); + final double oldd = wdist.distance(estimated_truth_vec, greedyvec); final int heapsize = enscands.size(); ArrayList<DoubleDBIDPair> heap = new ArrayList<>(heapsize); double[] tmp = new double[dim]; - for (DBIDIter iter = enscands.iter(); iter.valid(); iter.advance()) { - final NumberVector<?> vec = relation.get(iter); + for(DBIDIter iter = enscands.iter(); iter.valid(); iter.advance()) { + final NumberVector vec = relation.get(iter); singleEnsemble(tmp, vec); final Vector v2 = new Vector(greedyensemble); - double diversity = wdist.doubleDistance(v2, greedyvec); + double diversity = wdist.distance(v2, greedyvec); heap.add(DBIDUtil.newPair(diversity, iter)); } Collections.sort(heap); // , Collections.reverseOrder()); - while (heap.size() > 0) { + while(heap.size() > 0) { DBIDRef bestadd = heap.remove(heap.size() - 1); enscands.remove(bestadd); - final NumberVector<?> vec = relation.get(bestadd); + final NumberVector vec = relation.get(bestadd); // Build combined ensemble. { double buf[] = new double[ensemble.size() + 1]; - for (int i = 0; i < dim; i++) { + for(int i = 0; i < dim; i++) { int j = 0; - for (DBIDIter iter = ensemble.iter(); iter.valid(); iter.advance()) { + for(DBIDIter iter = ensemble.iter(); iter.valid(); iter.advance()) { buf[j] = relation.get(iter).doubleValue(i); j++; } @@ -356,46 +357,48 @@ public class GreedyEnsembleExperiment extends AbstractApplication { } } applyScaling(testensemble, scaling); - NumberVector<?> testvec = factory.newNumberVector(testensemble); - double newd = wdist.doubleDistance(estimated_truth_vec, testvec); + NumberVector testvec = factory.newNumberVector(testensemble); + double newd = wdist.distance(estimated_truth_vec, testvec); // LOG.verbose("Distances: " + oldd + " vs. " + newd + " " + // labels.get(bestadd)); - if (newd < oldd) { + if(newd < oldd) { System.arraycopy(testensemble, 0, greedyensemble, 0, dim); ensemble.add(bestadd); // logger.verbose("Growing ensemble with: " + labels.get(bestadd)); break; // Recompute heap - } else { + } + else { dropped.add(bestadd); // logger.verbose("Discarding: " + labels.get(bestadd)); - if (refine_truth) { + if(refine_truth) { // Update target vectors and weights - ArrayList<ROC.DecreasingVectorIter> iters = new ArrayList<>(numcand); - for (DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + ArrayList<DecreasingVectorIter> iters = new ArrayList<>(numcand); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { // Skip "by label", obviously - if (DBIDUtil.equal(firstid, iditer) || dropped.contains(iditer)) { + if(DBIDUtil.equal(firstid, iditer) || dropped.contains(iditer)) { continue; } - iters.add(new ROC.DecreasingVectorIter(relation.get(iditer))); + iters.add(new DecreasingVectorIter(relation.get(iditer))); } - if (minvote >= iters.size()) { + if(minvote >= iters.size()) { minvote = iters.size() - 1; } union_outliers = 0; Arrays.fill(outliers_seen, 0); - while (union_outliers < desired_outliers) { - for (ROC.DecreasingVectorIter iter : iters) { - if (!iter.valid()) { + while(union_outliers < desired_outliers) { + for(DecreasingVectorIter iter : iters) { + if(!iter.valid()) { break; } int cur = iter.dim(); - if (outliers_seen[cur] == 0) { + if(outliers_seen[cur] == 0) { outliers_seen[cur] = 1; - } else { + } + else { outliers_seen[cur] += 1; } - if (outliers_seen[cur] == minvote) { + if(outliers_seen[cur] == minvote) { union_outliers += 1; } iter.advance(); @@ -411,15 +414,15 @@ public class GreedyEnsembleExperiment extends AbstractApplication { // Build the improved ensemble: StringBuilder greedylbl = new StringBuilder(); { - for (DBIDIter iter = ensemble.iter(); iter.valid(); iter.advance()) { - if (greedylbl.length() > 0) { + for(DBIDIter iter = ensemble.iter(); iter.valid(); iter.advance()) { + if(greedylbl.length() > 0) { greedylbl.append(' '); } greedylbl.append(labels.get(iter)); } } - NumberVector<?> greedyvec = factory.newNumberVector(greedyensemble); - if (refine_truth) { + NumberVector greedyvec = factory.newNumberVector(greedyensemble); + if(refine_truth) { LOG.verbose("Estimated outliers remaining: " + union_outliers); } LOG.verbose("Greedy ensemble (" + ensemble.size() + "): " + greedylbl.toString()); @@ -429,15 +432,15 @@ public class GreedyEnsembleExperiment extends AbstractApplication { // Evaluate the naive ensemble and the "shrunk" ensemble double naiveauc, naivecost; { - naiveauc = XYCurve.areaUnderCurve(ROC.materializeROC(positive, new ROC.DecreasingVectorIter(naivevec))); - naivecost = tdist.doubleDistance(naivevec, refvec); + naiveauc = ROCEvaluation.computeROCAUC(positive, new DecreasingVectorIter(naivevec)); + naivecost = tdist.distance(naivevec, refvec); LOG.verbose("Naive ensemble AUC: " + naiveauc + " cost: " + naivecost); LOG.verbose("Naive ensemble Gain: " + gain(naiveauc, bestauc, 1) + " cost gain: " + gain(naivecost, bestcost, 0)); } double greedyauc, greedycost; { - greedyauc = XYCurve.areaUnderCurve(ROC.materializeROC(positive, new ROC.DecreasingVectorIter(greedyvec))); - greedycost = tdist.doubleDistance(greedyvec, refvec); + greedyauc = ROCEvaluation.computeROCAUC(positive, new DecreasingVectorIter(greedyvec)); + greedycost = tdist.distance(greedyvec, refvec); LOG.verbose("Greedy ensemble AUC: " + greedyauc + " cost: " + greedycost); LOG.verbose("Greedy ensemble Gain to best: " + gain(greedyauc, bestauc, 1) + " cost gain: " + gain(greedycost, bestcost, 0)); LOG.verbose("Greedy ensemble Gain to naive: " + gain(greedyauc, naiveauc, 1) + " cost gain: " + gain(greedycost, naivecost, 0)); @@ -447,17 +450,17 @@ public class GreedyEnsembleExperiment extends AbstractApplication { MeanVariance meancost = new MeanVariance(); HashSetModifiableDBIDs candidates = DBIDUtil.newHashSet(relation.getDBIDs()); candidates.remove(firstid); - for (int i = 0; i < 1000; i++) { + for(int i = 0; i < 1000; i++) { // Build the improved ensemble: final double[] randomensemble = new double[dim]; { DBIDs random = DBIDUtil.randomSample(candidates, ensemble.size(), (long) i); double[] buf = new double[random.size()]; - for (int d = 0; d < dim; d++) { + for(int d = 0; d < dim; d++) { int j = 0; - for (DBIDIter iter = random.iter(); iter.valid(); iter.advance()) { + for(DBIDIter iter = random.iter(); iter.valid(); iter.advance()) { assert (!DBIDUtil.equal(firstid, iter)); - final NumberVector<?> vec = relation.get(iter); + final NumberVector vec = relation.get(iter); buf[j] = vec.doubleValue(d); j++; } @@ -465,10 +468,10 @@ public class GreedyEnsembleExperiment extends AbstractApplication { } } applyScaling(randomensemble, scaling); - NumberVector<?> randomvec = factory.newNumberVector(randomensemble); - double auc = XYCurve.areaUnderCurve(ROC.materializeROC(positive, new ROC.DecreasingVectorIter(randomvec))); + NumberVector randomvec = factory.newNumberVector(randomensemble); + double auc = ROCEvaluation.computeROCAUC(positive, new DecreasingVectorIter(randomvec)); meanauc.put(auc); - double cost = tdist.doubleDistance(randomvec, refvec); + double cost = tdist.distance(randomvec, refvec); meancost.put(cost); } LOG.verbose("Random ensemble AUC: " + meanauc.getMean() + " + stddev: " + meanauc.getSampleStddev() + " = " + (meanauc.getMean() + meanauc.getSampleStddev())); @@ -489,12 +492,12 @@ public class GreedyEnsembleExperiment extends AbstractApplication { * @param ensemble * @param vec */ - protected void singleEnsemble(final double[] ensemble, final NumberVector<?> vec) { + protected void singleEnsemble(final double[] ensemble, final NumberVector vec) { double buf[] = new double[1]; - for (int i = 0; i < ensemble.length; i++) { + for(int i = 0; i < ensemble.length; i++) { buf[0] = vec.doubleValue(i); ensemble[i] = voting.combine(buf, 1); - if (Double.isNaN(ensemble[i])) { + if(Double.isNaN(ensemble[i])) { LOG.warning("NaN after combining: " + FormatUtil.format(buf) + " " + voting.toString()); } } @@ -510,17 +513,17 @@ public class GreedyEnsembleExperiment extends AbstractApplication { * @param skip DBIDs to pass unmodified * @return New relation */ - public static Relation<NumberVector<?>> applyPrescaling(ScalingFunction scaling, Relation<NumberVector<?>> relation, DBIDs skip) { - if (scaling == null) { + public static Relation<NumberVector> applyPrescaling(ScalingFunction scaling, Relation<NumberVector> relation, DBIDs skip) { + if(scaling == null) { return relation; } - NumberVector.Factory<NumberVector<?>, ?> factory = (NumberVector.Factory<NumberVector<?>, ?>) RelationUtil.assumeVectorField(relation).getFactory(); + NumberVector.Factory<NumberVector> factory = RelationUtil.getNumberVectorFactory(relation); DBIDs ids = relation.getDBIDs(); - WritableDataStore<NumberVector<?>> contents = DataStoreUtil.makeStorage(ids, DataStoreFactory.HINT_HOT, NumberVector.class); - for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { - NumberVector<?> v = relation.get(iter); + WritableDataStore<NumberVector> contents = DataStoreUtil.makeStorage(ids, DataStoreFactory.HINT_HOT, NumberVector.class); + for(DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + NumberVector v = relation.get(iter); double[] raw = v.getColumnVector().getArrayRef(); - if (!skip.contains(iter)) { + if(!skip.contains(iter)) { applyScaling(raw, scaling); } contents.put(iter, factory.newNumberVector(raw, ArrayLikeUtil.DOUBLEARRAYADAPTER)); @@ -529,15 +532,15 @@ public class GreedyEnsembleExperiment extends AbstractApplication { } private static void applyScaling(double[] raw, ScalingFunction scaling) { - if (scaling == null) { + if(scaling == null) { return; } - if (scaling instanceof OutlierScalingFunction) { + if(scaling instanceof OutlierScalingFunction) { ((OutlierScalingFunction) scaling).prepare(raw, ArrayLikeUtil.DOUBLEARRAYADAPTER); } - for (int i = 0; i < raw.length; i++) { + for(int i = 0; i < raw.length; i++) { final double newval = scaling.getScaled(raw[i]); - if (Double.isNaN(newval)) { + if(Double.isNaN(newval)) { LOG.warning("NaN after prescaling: " + raw[i] + " " + scaling.toString() + " -> " + newval); } raw[i] = newval; @@ -550,19 +553,20 @@ public class GreedyEnsembleExperiment extends AbstractApplication { // final double orate = union_outliers * 1.0 / (outliers_seen.length); final double oscore = 1.; // .5 - .5 * orate; final double iscore = 0.; // 1 - .5 * orate; - for (int i = 0; i < outliers.length; i++) { - if (outliers[i] >= minvote) { + for(int i = 0; i < outliers.length; i++) { + if(outliers[i] >= minvote) { weights[i] = oweight; truth[i] = oscore; - } else { + } + else { weights[i] = iweight; truth[i] = iscore; } } } - private PrimitiveDoubleDistanceFunction<NumberVector<?>> getDistanceFunction(double[] estimated_weights) { - switch(distance) { + private PrimitiveDistanceFunction<NumberVector> getDistanceFunction(double[] estimated_weights) { + switch(distance){ case SQEUCLIDEAN: return new WeightedSquaredEuclideanDistanceFunction(estimated_weights); case EUCLIDEAN: @@ -658,29 +662,29 @@ public class GreedyEnsembleExperiment extends AbstractApplication { inputstep = config.tryInstantiate(InputStep.class); // Voting method ObjectParameter<EnsembleVoting> votingP = new ObjectParameter<>(VOTING_ID, EnsembleVoting.class, EnsembleVotingMean.class); - if (config.grab(votingP)) { + if(config.grab(votingP)) { voting = votingP.instantiateClass(config); } // Similarity measure EnumParameter<Distance> distanceP = new EnumParameter<>(DISTANCE_ID, Distance.class); - if (config.grab(distanceP)) { + if(config.grab(distanceP)) { distance = distanceP.getValue(); } // Prescaling ObjectParameter<ScalingFunction> prescalingP = new ObjectParameter<>(PRESCALING_ID, ScalingFunction.class); prescalingP.setOptional(true); - if (config.grab(prescalingP)) { + if(config.grab(prescalingP)) { prescaling = prescalingP.instantiateClass(config); } // Ensemble scaling ObjectParameter<ScalingFunction> scalingP = new ObjectParameter<>(SCALING_ID, ScalingFunction.class); scalingP.setOptional(true); - if (config.grab(scalingP)) { + if(config.grab(scalingP)) { scaling = scalingP.instantiateClass(config); } // Expected rate of outliers DoubleParameter rateP = new DoubleParameter(RATE_ID, 0.01); - if (config.grab(rateP)) { + if(config.grab(rateP)) { rate = rateP.doubleValue(); } } diff --git a/src/de/lmu/ifi/dbs/elki/application/greedyensemble/VisualizePairwiseGainMatrix.java b/src/de/lmu/ifi/dbs/elki/application/greedyensemble/VisualizePairwiseGainMatrix.java index 3ef30c10..60678bea 100644 --- a/src/de/lmu/ifi/dbs/elki/application/greedyensemble/VisualizePairwiseGainMatrix.java +++ b/src/de/lmu/ifi/dbs/elki/application/greedyensemble/VisualizePairwiseGainMatrix.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.application.greedyensemble; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2013 + Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -39,13 +39,14 @@ import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.database.relation.RelationUtil; -import de.lmu.ifi.dbs.elki.evaluation.roc.ROC; +import de.lmu.ifi.dbs.elki.evaluation.scores.ROCEvaluation; +import de.lmu.ifi.dbs.elki.evaluation.scores.adapter.DecreasingVectorIter; +import de.lmu.ifi.dbs.elki.evaluation.scores.adapter.VectorNonZero; import de.lmu.ifi.dbs.elki.evaluation.similaritymatrix.ComputeSimilarityMatrixImage; import de.lmu.ifi.dbs.elki.evaluation.similaritymatrix.ComputeSimilarityMatrixImage.SimilarityMatrix; import de.lmu.ifi.dbs.elki.logging.Logging; import de.lmu.ifi.dbs.elki.logging.progress.FiniteProgress; import de.lmu.ifi.dbs.elki.math.DoubleMinMax; -import de.lmu.ifi.dbs.elki.math.geometry.XYCurve; import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; import de.lmu.ifi.dbs.elki.result.ResultUtil; import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; @@ -134,21 +135,21 @@ public class VisualizePairwiseGainMatrix extends AbstractApplication { @Override public void run() { final Database database = inputstep.getDatabase(); - Relation<NumberVector<?>> relation = database.getRelation(TypeUtil.NUMBER_VECTOR_FIELD); + Relation<NumberVector> relation = database.getRelation(TypeUtil.NUMBER_VECTOR_FIELD); final Relation<String> labels = DatabaseUtil.guessLabelRepresentation(database); final DBID firstid = DBIDUtil.deref(labels.iterDBIDs()); final String firstlabel = labels.get(firstid); - if (!firstlabel.matches(".*by.?label.*")) { + if(!firstlabel.matches(".*by.?label.*")) { throw new AbortException("No 'by label' reference outlier found, which is needed for weighting!"); } relation = GreedyEnsembleExperiment.applyPrescaling(prescaling, relation, firstid); // Dimensionality and reference vector final int dim = RelationUtil.dimensionality(relation); - final NumberVector<?> refvec = relation.get(firstid); + final NumberVector refvec = relation.get(firstid); // Build the truth vector - ROC.VectorNonZero pos = new ROC.VectorNonZero(refvec); + VectorNonZero pos = new VectorNonZero(refvec); ArrayModifiableDBIDs ids = DBIDUtil.newArray(relation.getDBIDs()); ids.remove(firstid); @@ -162,46 +163,40 @@ public class VisualizePairwiseGainMatrix extends AbstractApplication { FiniteProgress prog = LOG.isVerbose() ? new FiniteProgress("Computing ensemble gain.", size * (size + 1) >> 1, LOG) : null; double[] buf = new double[2]; // Vote combination buffer. int a = 0; - for (DBIDIter id = ids.iter(); id.valid(); id.advance(), a++) { - final NumberVector<?> veca = relation.get(id); + for(DBIDIter id = ids.iter(); id.valid(); id.advance(), a++) { + final NumberVector veca = relation.get(id); // Direct AUC score: { - double auc = XYCurve.areaUnderCurve(ROC.materializeROC(pos, new ROC.DecreasingVectorIter(veca))); + double auc = ROCEvaluation.computeROCAUC(pos, new DecreasingVectorIter(veca)); data[a][a] = auc; // minmax.put(auc); - if (prog != null) { - prog.incrementProcessed(LOG); - } + LOG.incrementProcessed(prog); } // Compare to others, exploiting symmetry DBIDArrayIter id2 = ids.iter(); id2.seek(a + 1); - for (int b = a + 1; b < size; b++, id2.advance()) { - final NumberVector<?> vecb = relation.get(id2); + for(int b = a + 1; b < size; b++, id2.advance()) { + final NumberVector vecb = relation.get(id2); double[] combined = new double[dim]; - for (int d = 0; d < dim; d++) { + for(int d = 0; d < dim; d++) { buf[0] = veca.doubleValue(d); buf[1] = vecb.doubleValue(d); combined[d] = voting.combine(buf); } - double auc = XYCurve.areaUnderCurve(ROC.materializeROC(pos, new ROC.DecreasingVectorIter(new Vector(combined)))); + double auc = ROCEvaluation.computeROCAUC(pos, new DecreasingVectorIter(new Vector(combined))); // logger.verbose(auc + " " + labels.get(ids.get(a)) + " " + // labels.get(ids.get(b))); data[a][b] = auc; data[b][a] = auc; commax.put(data[a][b]); // minmax.put(auc); - if (prog != null) { - prog.incrementProcessed(LOG); - } + LOG.incrementProcessed(prog); } } - if (prog != null) { - prog.ensureCompleted(LOG); - } + LOG.ensureCompleted(prog); } - for (int a = 0; a < size; a++) { - for (int b = a + 1; b < size; b++) { + for(int a = 0; a < size; a++) { + for(int b = a + 1; b < size; b++) { double ref = Math.max(data[a][a], data[b][b]); data[a][b] = (data[a][b] - ref) / (1 - ref); data[b][a] = (data[b][a] - ref) / (1 - ref); @@ -210,7 +205,7 @@ public class VisualizePairwiseGainMatrix extends AbstractApplication { minmax.put(data[a][b]); } } - for (int a = 0; a < size; a++) { + for(int a = 0; a < size; a++) { data[a][a] = 0; } @@ -218,25 +213,27 @@ public class VisualizePairwiseGainMatrix extends AbstractApplication { boolean hasneg = (minmax.getMin() < -1E-3); LinearScaling scale; - if (!hasneg) { + if(!hasneg) { scale = LinearScaling.fromMinMax(0., minmax.getMax()); - } else { + } + else { scale = LinearScaling.fromMinMax(0.0, Math.max(minmax.getMax(), -minmax.getMin())); } scale = LinearScaling.fromMinMax(0., .5); BufferedImage img = new BufferedImage(size, size, BufferedImage.TYPE_INT_RGB); - for (int x = 0; x < size; x++) { - for (int y = x; y < size; y++) { + for(int x = 0; x < size; x++) { + for(int y = x; y < size; y++) { double val = data[x][y]; val = Math.max(-1, Math.min(1., scale.getScaled(val))); // Compute color: final int col; { - if (val >= 0) { + if(val >= 0) { int ival = 0xFF & (int) (255 * val); col = 0xff000000 | (ival << 8); - } else { + } + else { int ival = 0xFF & (int) (255 * -val); col = 0xff000000 | (ival << 16); } @@ -255,8 +252,8 @@ public class VisualizePairwiseGainMatrix extends AbstractApplication { factory.processNewResult(database, database); List<VisualizationTask> tasks = ResultUtil.filterResults(database, VisualizationTask.class); - for (VisualizationTask task : tasks) { - if (task.getFactory() == factory) { + for(VisualizationTask task : tasks) { + if(task.getFactory() == factory) { showVisualization(context, factory, task); } } @@ -320,12 +317,12 @@ public class VisualizePairwiseGainMatrix extends AbstractApplication { // Prescaling ObjectParameter<ScalingFunction> prescalingP = new ObjectParameter<>(GreedyEnsembleExperiment.Parameterizer.PRESCALING_ID, ScalingFunction.class); prescalingP.setOptional(true); - if (config.grab(prescalingP)) { + if(config.grab(prescalingP)) { prescaling = prescalingP.instantiateClass(config); } ObjectParameter<EnsembleVoting> votingP = new ObjectParameter<>(GreedyEnsembleExperiment.Parameterizer.VOTING_ID, EnsembleVoting.class, EnsembleVotingMean.class); - if (config.grab(votingP)) { + if(config.grab(votingP)) { voting = votingP.instantiateClass(config); } } diff --git a/src/de/lmu/ifi/dbs/elki/application/greedyensemble/package-info.java b/src/de/lmu/ifi/dbs/elki/application/greedyensemble/package-info.java index f8e8f705..05aad761 100644 --- a/src/de/lmu/ifi/dbs/elki/application/greedyensemble/package-info.java +++ b/src/de/lmu/ifi/dbs/elki/application/greedyensemble/package-info.java @@ -11,7 +11,7 @@ This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2013 + Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team diff --git a/src/de/lmu/ifi/dbs/elki/application/internal/CheckELKIServices.java b/src/de/lmu/ifi/dbs/elki/application/internal/CheckELKIServices.java index 7dca7f37..f5940e7d 100644 --- a/src/de/lmu/ifi/dbs/elki/application/internal/CheckELKIServices.java +++ b/src/de/lmu/ifi/dbs/elki/application/internal/CheckELKIServices.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.application.internal; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2013 + Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -27,15 +27,22 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; import java.io.InputStreamReader; import java.io.PrintStream; +import java.net.JarURLConnection; import java.net.URISyntaxException; import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.Enumeration; import java.util.HashSet; import java.util.List; +import java.util.TreeSet; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -65,50 +72,66 @@ public class CheckELKIServices { private Pattern strip = Pattern.compile("^[\\s#]*(?:deprecated:\\s*)?(.*?)[\\s]*$"); /** - * Package to skip matches in - unreleased code. - */ - private String[] skippackages = { "experimentalcode.", "project." }; - - /** * Main method. * * @param argv Command line arguments */ public static void main(String[] argv) { - boolean update = false, noskip = false; - for(String arg : argv) { - if("-update".equals(arg)) { - update = true; - } - if("-all".equals(arg)) { - noskip = true; - } + String update = null; + if(argv.length == 2 && "-update".equals(argv[0])) { + update = argv[1]; + LOG.info("Updating service files in folder: " + update); + } + else if(argv.length != 0) { + throw new AbortException("Incorrect command line parameters."); } - new CheckELKIServices().checkServices(update, noskip); + new CheckELKIServices().checkServices(update); } /** * Retrieve all properties and check them. * - * @param update Flag to enable automatic updating - * @param noskip Override filters, include all (in particular, - * experimentalcode) + * @param update Folder to update service files in */ - public void checkServices(boolean update, boolean noskip) { - URL u = getClass().getClassLoader().getResource(ELKIServiceLoader.PREFIX); + public void checkServices(String update) { + TreeSet<String> props = new TreeSet<>(); + Enumeration<URL> us; try { - for(String prop : new File(u.toURI()).list()) { - if(".svn".equals(prop)) { + us = getClass().getClassLoader().getResources(ELKIServiceLoader.PREFIX); + } + catch(IOException e) { + throw new AbortException("Error enumerating service folders.", e); + } + while(us.hasMoreElements()) { + URL u = us.nextElement(); + try { + if(("jar".equals(u.getProtocol()))) { + JarURLConnection con = (JarURLConnection) u.openConnection(); + try (JarFile jar = con.getJarFile()) { + Enumeration<JarEntry> entries = jar.entries(); + while(entries.hasMoreElements()) { + String prop = entries.nextElement().getName(); + if(prop.length() > ELKIServiceLoader.PREFIX.length() && prop.startsWith(ELKIServiceLoader.PREFIX)) { + props.add(prop.substring(ELKIServiceLoader.PREFIX.length())); + } + } + } continue; } - if(LOG.isVerbose()) { - LOG.verbose("Checking property: " + prop); + if("file".equals(u.getProtocol())) { + props.addAll(Arrays.asList(new File(u.toURI()).list())); } - checkService(prop, update, noskip); } + catch(IOException | URISyntaxException e) { + throw new AbortException("Error enumerating service folders.", e); + } + } - catch(URISyntaxException e) { - throw new AbortException("Cannot check all properties, as some are not in a file: URL."); + for(String prop : props) { + if(LOG.isVerbose()) { + LOG.verbose("Checking property: " + prop); + } + checkService(prop, update); } } @@ -116,11 +139,9 @@ public class CheckELKIServices { * Check a single service class * * @param prop Class name. - * @param update Flag to enable automatic updating - * @param noskip Override filters, include all (in particular, - * experimentalcode) + * @param update Folder to update service files in */ - private void checkService(String prop, boolean update, boolean noskip) { + private void checkService(String prop, String update) { Class<?> cls; try { cls = Class.forName(prop); @@ -129,46 +150,37 @@ public class CheckELKIServices { LOG.warning("Service file name is not a class name: " + prop); return; } - List<Class<?>> impls = InspectionUtil.findAllImplementations(cls, false); + List<Class<?>> impls = InspectionUtil.findAllImplementations(cls, false, false); HashSet<String> names = new HashSet<>(); for(Class<?> c2 : impls) { - boolean skip = false; - for(String pkg : skippackages) { - if(c2.getName().startsWith(pkg)) { - skip = true; - break; - } - } - if(!noskip && skip) { - continue; - } names.add(c2.getName()); } + Matcher m = strip.matcher(""); try { - InputStream is = getClass().getClassLoader().getResource(ELKIServiceLoader.PREFIX + cls.getName()).openStream(); - BufferedReader r = new BufferedReader(new InputStreamReader(is, "utf-8")); - for(String line;;) { - line = r.readLine(); - // End of stream: - if(line == null) { - break; - } - Matcher m = strip.matcher(line); - if(m.matches()) { + Enumeration<URL> us = getClass().getClassLoader().getResources(ELKIServiceLoader.PREFIX + cls.getName()); + while(us.hasMoreElements()) { + URL u = us.nextElement(); + boolean injar = "jar".equals(u.getProtocol()); + BufferedReader r = new BufferedReader(new InputStreamReader(u.openStream(), "utf-8")); + for(String line;;) { + line = r.readLine(); + // End of stream: + if(line == null) { + break; + } + m.reset(line); + if(!m.matches()) { + LOG.warning("Line: " + line + " didn't match regexp."); + continue; + } String stripped = m.group(1); if(stripped.length() > 0) { - if(names.contains(stripped)) { - names.remove(stripped); - } - else { - LOG.warning("Name " + stripped + " found for property " + prop + " but no class discovered (or referenced twice?)."); + if(!names.remove(stripped) && !injar) { + LOG.warning("Name " + stripped + " found for property " + prop + " but no class discovered (or listed twice)."); } } } - else { - LOG.warning("Line: " + line + " didn't match regexp."); - } } } catch(IOException e) { @@ -179,40 +191,30 @@ public class CheckELKIServices { ArrayList<String> sorted = new ArrayList<>(names); // TODO: sort by package, then classname Collections.sort(sorted); - if(!update) { + if(update == null) { StringBuilder message = new StringBuilder(); message.append("Class ").append(prop).append(" lacks suggestions:").append(FormatUtil.NEWLINE); for(String remaining : sorted) { message.append("# ").append(remaining).append(FormatUtil.NEWLINE); } LOG.warning(message.toString()); + return; } - else { - // Try to automatically update: - URL f = getClass().getClassLoader().getResource(ELKIServiceLoader.PREFIX + cls.getName()); - String fnam = f.getFile(); - if(fnam == null) { - LOG.warning("Cannot update: " + f + " seems to be in a jar file."); - } - else { - try { - FileOutputStream out = new FileOutputStream(fnam, true); - PrintStream pr = new PrintStream(out); - pr.println(); - pr.println("### Automatically appended entries:"); - for(String remaining : sorted) { - pr.println(remaining); - } - pr.flush(); - pr.close(); - out.flush(); - out.close(); - LOG.warning("Updated: " + fnam); - } - catch(IOException e) { - LOG.exception(e); - } + // Try to automatically update: + try { + Files.createDirectories(Paths.get(update + File.separator + ELKIServiceLoader.PREFIX)); + String fname = update + File.separator + ELKIServiceLoader.PREFIX + prop; + PrintStream pr = new PrintStream(new FileOutputStream(fname, true)); + pr.println(); // In case there was no linefeed at the end. + pr.println("### Automatically appended entries:"); + for(String remaining : sorted) { + pr.println(remaining); } + pr.close(); + LOG.warning("Updated service file: " + fname); + } + catch(IOException e) { + LOG.exception(e); } } } diff --git a/src/de/lmu/ifi/dbs/elki/application/internal/CheckParameterizables.java b/src/de/lmu/ifi/dbs/elki/application/internal/CheckParameterizables.java index 68a4b127..b0970b8f 100644 --- a/src/de/lmu/ifi/dbs/elki/application/internal/CheckParameterizables.java +++ b/src/de/lmu/ifi/dbs/elki/application/internal/CheckParameterizables.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.application.internal; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2013 + Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -23,17 +23,28 @@ package de.lmu.ifi.dbs.elki.application.internal; along with this program. If not, see <http://www.gnu.org/licenses/>. */ +import java.io.File; +import java.io.IOException; import java.lang.reflect.Constructor; import java.lang.reflect.Method; import java.lang.reflect.Modifier; +import java.net.JarURLConnection; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; import de.lmu.ifi.dbs.elki.logging.Logging; -import de.lmu.ifi.dbs.elki.logging.LoggingConfiguration; import de.lmu.ifi.dbs.elki.logging.Logging.Level; +import de.lmu.ifi.dbs.elki.logging.LoggingConfiguration; import de.lmu.ifi.dbs.elki.utilities.ClassGenericsUtil; +import de.lmu.ifi.dbs.elki.utilities.ELKIServiceLoader; import de.lmu.ifi.dbs.elki.utilities.InspectionUtil; +import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; -import de.lmu.ifi.dbs.elki.utilities.optionhandling.Parameterizable; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; /** @@ -43,7 +54,7 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameteriz * @author Erich Schubert * * @apiviz.landmark - * @apiviz.uses Parameterizable + * @apiviz.uses AbstractParameterizer */ public class CheckParameterizables { /** @@ -52,96 +63,208 @@ public class CheckParameterizables { private static final Logging LOG = Logging.getLogger(CheckParameterizables.class); /** + * Known parameterizable classes/interfaces. + */ + private List<Class<?>> knownParameterizables; + + /** * Validate all "Parameterizable" objects for parts of the API contract that * cannot be specified in Java interfaces (such as constructors, static * methods) */ public void checkParameterizables() { LoggingConfiguration.setVerbose(Level.VERBOSE); - for(final Class<?> cls : InspectionUtil.findAllImplementations(Object.class, false)) { - final Constructor<?> constructor; - try { - constructor = cls.getDeclaredConstructor(Parameterization.class); - } - catch(NoClassDefFoundError e) { - LOG.verbose("Class discovered but not found?!? " + cls.getName()); - // Not found ?!? - continue; - } - catch(Exception e) { - // Not parameterizable. - continue; - } - checkParameterizable(cls, constructor); - } - for(final Class<?> cls : InspectionUtil.findAllImplementations(Parameterizable.class, false)) { - boolean hasConstructor = false; - // check for a V3 Parameterizer class - for(Class<?> inner : cls.getDeclaredClasses()) { - if(AbstractParameterizer.class.isAssignableFrom(inner)) { - try { - Class<? extends AbstractParameterizer> pcls = inner.asSubclass(AbstractParameterizer.class); - pcls.newInstance(); - if(checkParameterizer(cls, pcls)) { - hasConstructor = true; + knownParameterizables = new ArrayList<>(); + try { + Enumeration<URL> us = getClass().getClassLoader().getResources(ELKIServiceLoader.PREFIX); + while(us.hasMoreElements()) { + URL u = us.nextElement(); + if("file".equals(u.getProtocol())) { + for(String prop : new File(u.toURI()).list()) { + try { + knownParameterizables.add(Class.forName(prop)); + } + catch(ClassNotFoundException e) { + LOG.warning("Service file name is not a class name: " + prop); continue; } } - catch(Exception e) { - LOG.verbose("Could not run Parameterizer: " + inner.getName() + ": " + e); - // continue. Probably non-public + } + else if(("jar".equals(u.getProtocol()))) { + JarURLConnection con = (JarURLConnection) u.openConnection(); + try (JarFile jar = con.getJarFile()) { + Enumeration<JarEntry> entries = jar.entries(); + while(entries.hasMoreElements()) { + String prop = entries.nextElement().getName(); + if(prop.length() > ELKIServiceLoader.PREFIX.length() && prop.startsWith(ELKIServiceLoader.PREFIX)) { + prop = prop.substring(ELKIServiceLoader.PREFIX.length()); + try { + knownParameterizables.add(Class.forName(prop)); + } + catch(ClassNotFoundException e) { + LOG.warning("Service file name is not a class name: " + prop); + continue; + } + } + } } } } + } + catch(IOException | URISyntaxException e) { + throw new AbortException("Error enumerating service folders.", e); + } - // check for a V2 factory method. - try { - ClassGenericsUtil.getParameterizationFactoryMethod(cls, Object.class); - hasConstructor = true; - // logger.debugFine("Found factory method for class: "+ cls.getName()); - } - catch(NoClassDefFoundError e) { - LOG.verbose("Class discovered but not found?!? " + cls.getName()); - // Not found ?!? + final String internal = de.lmu.ifi.dbs.elki.utilities.optionhandling.Parameterizer.class.getPackage().getName(); + for(final Class<?> cls : InspectionUtil.findAllImplementations(Object.class, false, false)) { + // Classes in the same package are special and don't cause warnings. + if(cls.getName().startsWith(internal)) { continue; } - catch(Exception e) { - // do nothing. - } try { - cls.getConstructor(Parameterization.class); - hasConstructor = true; + State state = State.NO_CONSTRUCTOR; + state = checkV3Parameterization(cls, state); + if(state == State.ERROR) { + continue; + } + state = checkV2Parameterization(cls, state); + if(state == State.ERROR) { + continue; + } + state = checkV1Parameterization(cls, state); + if(state == State.ERROR) { + continue; + } + state = checkDefaultConstructor(cls, state); + if(state == State.ERROR) { + continue; + } + boolean expectedParameterizer = checkSupertypes(cls); + if(state == State.NO_CONSTRUCTOR && expectedParameterizer) { + LOG.verbose("Class " + cls.getName() + // + " implements a parameterizable interface, but doesn't have a public and parameterless constructor!"); + } + if(state == State.INSTANTIABLE && !expectedParameterizer) { + LOG.verbose("Class " + cls.getName() + // + " has a parameterizer, but there is no service file for any of its interfaces."); + } } catch(NoClassDefFoundError e) { - LOG.verbose("Class discovered but not found?!? " + cls.getName()); - // Not found ?!? - continue; + LOG.verbose("Class discovered but not found: " + cls.getName() + " (missing: " + e.getMessage() + ")"); } - catch(Exception e) { - // do nothing. + } + } + + /** + * Check all supertypes of a class. + * + * @param cls Class to check. + * @return {@code true} when at least one supertype is a known parameterizable + * type. + */ + private boolean checkSupertypes(Class<?> cls) { + for(Class<?> c : knownParameterizables) { + if(c.isAssignableFrom(cls)) { + return true; } - try { - cls.getConstructor(); - hasConstructor = true; + } + return false; + } + + /** + * Current verification state. + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + enum State { + NO_CONSTRUCTOR, // + INSTANTIABLE, // + DEFAULT_INSTANTIABLE, // + ERROR, // + } + + /** Check for a V1 constructor. */ + private State checkV1Parameterization(Class<?> cls, State state) throws NoClassDefFoundError { + final Constructor<?> constructor; + try { + constructor = cls.getDeclaredConstructor(Parameterization.class); + if(state == State.INSTANTIABLE) { + LOG.warning("More than one parameterization method in class " + cls.getName()); } - catch(NoClassDefFoundError e) { - LOG.verbose("Class discovered but not found?!? " + cls.getName()); - // Not found ?!? - continue; + LOG.warning("V1 constructor found in class: " + cls.getName()); + if(!Modifier.isPublic(constructor.getModifiers())) { + LOG.verbose("Constructor for class " + cls.getName() + " is not public!"); } - catch(Exception e) { - // do nothing. + return State.INSTANTIABLE; + } + catch(NoSuchMethodException e) { + // Not parameterizable? + return state; + } + } + + /** Check for a V2 constructor. */ + private State checkV2Parameterization(Class<?> cls, State state) throws NoClassDefFoundError { + try { + ClassGenericsUtil.getParameterizationFactoryMethod(cls, Object.class); + if(state == State.INSTANTIABLE) { + LOG.warning("More than one parameterization method in class " + cls.getName()); } - if(!hasConstructor) { - LOG.verbose("Class " + cls.getName() + " is Parameterizable but doesn't have a constructor with the appropriate signature!"); + LOG.warning("V2 factory method found in class: " + cls.getName()); + return State.INSTANTIABLE; + } + catch(NoSuchMethodException e) { + // do nothing. + return state; + } + } + + /** Check for a V3 constructor. */ + private State checkV3Parameterization(Class<?> cls, State state) throws NoClassDefFoundError { + // check for a V3 Parameterizer class + for(Class<?> inner : cls.getDeclaredClasses()) { + if(AbstractParameterizer.class.isAssignableFrom(inner)) { + try { + Class<? extends AbstractParameterizer> pcls = inner.asSubclass(AbstractParameterizer.class); + pcls.newInstance(); + if(checkParameterizer(cls, pcls)) { + if(state == State.INSTANTIABLE) { + LOG.warning("More than one parameterization method in class " + cls.getName()); + } + state = State.INSTANTIABLE; + } + } + catch(Exception e) { + LOG.verbose("Could not run Parameterizer: " + inner.getName() + ": " + e.getMessage()); + // continue. Probably non-public + } + catch(Error e) { + LOG.verbose("Could not run Parameterizer: " + inner.getName() + ": " + e.getMessage()); + // continue. Probably non-public + } } } + return state; + } + + /** Check for a default constructor. */ + private State checkDefaultConstructor(Class<?> cls, State state) throws NoClassDefFoundError { + try { + cls.getConstructor(); + return State.DEFAULT_INSTANTIABLE; + } + catch(Exception e) { + // do nothing. + } + return state; } private boolean checkParameterizer(Class<?> cls, Class<? extends AbstractParameterizer> par) { + int checkResult = 0; try { par.getConstructor(); - boolean hasMakeInstance = false; final Method methods[] = par.getDeclaredMethods(); for(int i = 0; i < methods.length; ++i) { final Method meth = methods[i]; @@ -150,33 +273,26 @@ public class CheckParameterizables { if(meth.getParameterTypes().length == 0) { // And check for proper return type. if(cls.isAssignableFrom(meth.getReturnType())) { - hasMakeInstance = true; + checkResult = 1; + } + else if(checkResult == 0) { + checkResult = 2; // Nothing better } } + else if(checkResult == 0) { + checkResult += 3; + } } } - if(hasMakeInstance) { - return true; - } } catch(Exception e) { LOG.warning("No proper Parameterizer.makeInstance for " + cls.getName() + ": " + e); return false; } - LOG.warning("No proper Parameterizer.makeInstance for " + cls.getName() + " found!"); - return false; - } - - private void checkParameterizable(Class<?> cls, Constructor<?> constructor) { - // Classes in the same package are special and don't cause warnings. - if(!cls.getName().startsWith(Parameterizable.class.getPackage().getName())) { - if(!Modifier.isPublic(constructor.getModifiers())) { - LOG.verbose("Constructor for class " + cls.getName() + " is not public!"); - } - if(!Parameterizable.class.isAssignableFrom(cls)) { - LOG.verbose("Class " + cls.getName() + " should implement Parameterizable!"); - } + if(checkResult > 1) { + LOG.warning("No proper Parameterizer.makeInstance for " + cls.getName() + " found!"); } + return checkResult == 1; } /** diff --git a/src/de/lmu/ifi/dbs/elki/application/internal/DocumentParameters.java b/src/de/lmu/ifi/dbs/elki/application/internal/DocumentParameters.java index b9a8ae43..fbeb72d8 100644 --- a/src/de/lmu/ifi/dbs/elki/application/internal/DocumentParameters.java +++ b/src/de/lmu/ifi/dbs/elki/application/internal/DocumentParameters.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.application.internal; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2013 + Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -56,17 +56,17 @@ import org.w3c.dom.Element; import de.lmu.ifi.dbs.elki.KDDTask; import de.lmu.ifi.dbs.elki.application.AbstractApplication; import de.lmu.ifi.dbs.elki.logging.Logging; -import de.lmu.ifi.dbs.elki.logging.LoggingConfiguration; import de.lmu.ifi.dbs.elki.logging.Logging.Level; +import de.lmu.ifi.dbs.elki.logging.LoggingConfiguration; import de.lmu.ifi.dbs.elki.utilities.ClassGenericsUtil; import de.lmu.ifi.dbs.elki.utilities.ELKIServiceLoader; import de.lmu.ifi.dbs.elki.utilities.InspectionUtil; import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; -import de.lmu.ifi.dbs.elki.utilities.optionhandling.Parameterizable; import de.lmu.ifi.dbs.elki.utilities.optionhandling.Parameterizer; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.SerializedParameterization; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.TrackParameters; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.TrackedParameter; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.UnParameterization; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ClassParameter; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.Parameter; @@ -74,12 +74,11 @@ import de.lmu.ifi.dbs.elki.utilities.pairs.Pair; import de.lmu.ifi.dbs.elki.utilities.xml.HTMLUtil; /** - * Class to generate HTML parameter descriptions for all classes implementing - * the {@link Parameterizable} interface. Used in documentation generation only. + * Class to generate HTML parameter descriptions for all classes that have ELKI + * {@link Parameter}s. Used in documentation generation only. * * @author Erich Schubert - * - * @apiviz.uses Parameterizable + * * @apiviz.uses Parameter */ public class DocumentParameters { @@ -116,23 +115,23 @@ public class DocumentParameters { */ public static void main(String[] args) { LoggingConfiguration.setVerbose(Level.VERBOSE); - if (args.length != 2 && args.length != 4) { + if(args.length != 2 && args.length != 4) { LOG.warning("I need exactly two or four file names to operate!"); System.exit(1); } - if (!args[0].endsWith(".html")) { + if(!args[0].endsWith(".html")) { LOG.warning("First file name doesn't end with .html!"); System.exit(1); } - if (!args[1].endsWith(".html")) { + if(!args[1].endsWith(".html")) { LOG.warning("Second file name doesn't end with .html!"); System.exit(1); } - if (args.length > 2 && !args[2].endsWith(".wiki")) { + if(args.length > 2 && !args[2].endsWith(".wiki")) { LOG.warning("Third file name doesn't end with .wiki!"); System.exit(1); } - if (args.length > 3 && !args[3].endsWith(".wiki")) { + if(args.length > 3 && !args[3].endsWith(".wiki")) { LOG.warning("Fourth file name doesn't end with .wiki!"); System.exit(1); } @@ -145,7 +144,8 @@ public class DocumentParameters { Map<OptionID, List<Pair<Parameter<?>, Class<?>>>> byopt = new HashMap<>(); try { buildParameterIndex(byclass, byopt); - } catch (Exception e) { + } + catch(Exception e) { LOG.exception(e); System.exit(1); } @@ -154,7 +154,8 @@ public class DocumentParameters { FileOutputStream byclassfo; try { byclassfo = new FileOutputStream(byclsname); - } catch (FileNotFoundException e) { + } + catch(FileNotFoundException e) { LOG.exception("Can't create output stream!", e); throw new RuntimeException(e); } @@ -165,16 +166,18 @@ public class DocumentParameters { byclassstream.flush(); byclassstream.close(); byclassfo.close(); - } catch (IOException e) { + } + catch(IOException e) { LOG.exception("IO Exception writing output.", e); throw new RuntimeException(e); } } - if (byclsnamew != null) { + if(byclsnamew != null) { FileOutputStream byclassfo; try { byclassfo = new FileOutputStream(byclsnamew); - } catch (FileNotFoundException e) { + } + catch(FileNotFoundException e) { LOG.exception("Can't create output stream!", e); throw new RuntimeException(e); } @@ -184,7 +187,8 @@ public class DocumentParameters { byclassstream.flush(); byclassstream.close(); byclassfo.close(); - } catch (IOException e) { + } + catch(IOException e) { LOG.exception("IO Exception writing output.", e); throw new RuntimeException(e); } @@ -194,7 +198,8 @@ public class DocumentParameters { FileOutputStream byoptfo; try { byoptfo = new FileOutputStream(byoptname); - } catch (FileNotFoundException e) { + } + catch(FileNotFoundException e) { LOG.exception("Can't create output stream!", e); throw new RuntimeException(e); } @@ -205,17 +210,19 @@ public class DocumentParameters { byoptstream.flush(); byoptstream.close(); byoptfo.close(); - } catch (IOException e) { + } + catch(IOException e) { LOG.exception("IO Exception writing output.", e); throw new RuntimeException(e); } } - if (byoptnamew != null) { + if(byoptnamew != null) { FileOutputStream byoptfo; try { byoptfo = new FileOutputStream(byoptnamew); - } catch (FileNotFoundException e) { + } + catch(FileNotFoundException e) { LOG.exception("Can't create output stream!", e); throw new RuntimeException(e); } @@ -225,7 +232,8 @@ public class DocumentParameters { byoptstream.flush(); byoptstream.close(); byoptfo.close(); - } catch (IOException e) { + } + catch(IOException e) { LOG.exception("IO Exception writing output.", e); throw new RuntimeException(e); } @@ -235,53 +243,58 @@ public class DocumentParameters { } private static void buildParameterIndex(Map<Class<?>, List<Parameter<?>>> byclass, Map<OptionID, List<Pair<Parameter<?>, Class<?>>>> byopt) { - final ArrayList<Pair<Object, Parameter<?>>> options = new ArrayList<>(); + final ArrayList<TrackedParameter> options = new ArrayList<>(); ExecutorService es = Executors.newSingleThreadExecutor(); - for (final Class<?> cls : InspectionUtil.findAllImplementations(Parameterizable.class, false)) { + for(final Class<?> cls : InspectionUtil.findAllImplementations(Object.class, false, true)) { // Doesn't have a proper name? - if (cls.getCanonicalName() == null) { + if(cls.getCanonicalName() == null) { continue; } // Some of the "applications" do currently not have appropriate // constructors / parameterizers and may start AWT threads - skip them. - if (AbstractApplication.class.isAssignableFrom(cls)) { + if(AbstractApplication.class.isAssignableFrom(cls)) { continue; } UnParameterization config = new UnParameterization(); - final TrackParameters track = new TrackParameters(config); + final TrackParameters track = new TrackParameters(config, cls); // LoggingUtil.warning("Instantiating " + cls.getName()); FutureTask<?> instantiator = new FutureTask<>(new Runnable() { @Override public void run() { // Try a V3 style parameterizer first. Parameterizer par = ClassGenericsUtil.getParameterizer(cls); - if (par != null) { + if(par != null) { par.configure(track); - } else { + } + else { try { ClassGenericsUtil.tryInstantiate(Object.class, cls, track); - } catch (java.lang.NoSuchMethodException e) { - LOG.warning("Could not instantiate class " + cls.getName() + " - no appropriate constructor or parameterizer found."); - } catch (java.lang.reflect.InvocationTargetException e) { - if (e.getCause() instanceof RuntimeException) { + } + catch(java.lang.NoSuchMethodException | java.lang.IllegalAccessException e) { + // LOG.warning("Could not instantiate class " + cls.getName() + + // " - no appropriate constructor or parameterizer found."); + } + catch(java.lang.reflect.InvocationTargetException e) { + if(e.getCause() instanceof RuntimeException) { throw (RuntimeException) e.getCause(); } - if (e.getCause() instanceof Error) { + if(e.getCause() instanceof Error) { throw (Error) e.getCause(); } throw new RuntimeException(e.getCause()); - } catch (RuntimeException e) { + } + catch(RuntimeException e) { throw e; - } catch (Exception e) { - throw new RuntimeException(e); - } catch (java.lang.Error e) { + } + catch(Exception | java.lang.Error e) { throw new RuntimeException(e); } } - for (Pair<Object, Parameter<?>> pair : track.getAllParameters()) { - if (pair.first == null) { - pair.first = cls; + for(TrackedParameter pair : track.getAllParameters()) { + if(pair.getOwner() == null) { + LOG.warning("No owner for parameter " + pair.getParameter() + " expected a " + cls.getName()); + continue; } options.add(pair); } @@ -291,15 +304,18 @@ public class DocumentParameters { try { // Wait up to one second. instantiator.get(1L, TimeUnit.SECONDS); - } catch (TimeoutException e) { + } + catch(TimeoutException e) { LOG.warning("Timeout on instantiating " + cls.getName()); es.shutdownNow(); throw new RuntimeException(e); - } catch (java.util.concurrent.ExecutionException e) { + } + catch(java.util.concurrent.ExecutionException e) { // Do full reporting only on release branch. - if (cls.getName().startsWith("de.lmu.ifi.dbs.elki")) { + if(cls.getName().startsWith("de.lmu.ifi.dbs.elki")) { LOG.warning("Error instantiating " + cls.getName(), e.getCause()); - } else { + } + else { LOG.warning("Error instantiating " + cls.getName()); } // es.shutdownNow(); @@ -308,11 +324,13 @@ public class DocumentParameters { // } // throw new RuntimeException(e.getCause()); continue; - } catch (Exception e) { + } + catch(Exception e) { // Do full reporting only on release branch. - if (cls.getName().startsWith("de.lmu.ifi.dbs.elki")) { + if(cls.getName().startsWith("de.lmu.ifi.dbs.elki")) { LOG.warning("Error instantiating " + cls.getName(), e.getCause()); - } else { + } + else { LOG.warning("Error instantiating " + cls.getName()); } // es.shutdownNow(); @@ -321,34 +339,35 @@ public class DocumentParameters { } } LOG.debug("Documenting " + options.size() + " parameter instances."); - for (Pair<Object, Parameter<?>> pp : options) { - if (pp.first == null || pp.second == null) { - LOG.debugFiner("Null: " + pp.first + " " + pp.second); + for(TrackedParameter pp : options) { + if(pp.getOwner() == null || pp.getParameter() == null) { + LOG.debugFiner("Null: " + pp.getOwner() + " " + pp.getParameter()); continue; } Class<?> c; - if (pp.first instanceof Class) { - c = (Class<?>) pp.first; - } else { - c = pp.first.getClass(); + if(pp.getOwner() instanceof Class) { + c = (Class<?>) pp.getOwner(); } - Parameter<?> o = pp.second; + else { + c = pp.getOwner().getClass(); + } + Parameter<?> o = pp.getParameter(); // just collect unique occurrences { List<Parameter<?>> byc = byclass.get(c); boolean inlist = false; - if (byc != null) { - for (Parameter<?> par : byc) { - if (par.getOptionID() == o.getOptionID()) { + if(byc != null) { + for(Parameter<?> par : byc) { + if(par.getOptionID() == o.getOptionID()) { inlist = true; break; } } } - if (!inlist) { + if(!inlist) { List<Parameter<?>> ex = byclass.get(c); - if (ex == null) { + if(ex == null) { ex = new ArrayList<>(); byclass.put(c, ex); } @@ -358,17 +377,17 @@ public class DocumentParameters { { List<Pair<Parameter<?>, Class<?>>> byo = byopt.get(o.getOptionID()); boolean inlist = false; - if (byo != null) { - for (Pair<Parameter<?>, Class<?>> pair : byo) { - if (pair.second.equals(c)) { + if(byo != null) { + for(Pair<Parameter<?>, Class<?>> pair : byo) { + if(pair.second.equals(c)) { inlist = true; break; } } } - if (!inlist) { + if(!inlist) { List<Pair<Parameter<?>, Class<?>>> ex = byopt.get(o.getOptionID()); - if (ex == null) { + if(ex == null) { ex = new ArrayList<>(); byopt.put(o.getOptionID(), ex); } @@ -382,14 +401,18 @@ public class DocumentParameters { protected static Constructor<?> getConstructor(final Class<?> cls) { try { return cls.getConstructor(Parameterization.class); - } catch (java.lang.NoClassDefFoundError e) { + } + catch(java.lang.NoClassDefFoundError e) { // Class not actually found - } catch (RuntimeException e) { + } + catch(RuntimeException e) { // Not parameterizable, usually not even found ... LOG.warning("RuntimeException: ", e); - } catch (Exception e) { + } + catch(Exception e) { // Not parameterizable. - } catch (java.lang.Error e) { + } + catch(java.lang.Error e) { // Not parameterizable. LOG.warning("Error: ", e); } @@ -401,7 +424,8 @@ public class DocumentParameters { DocumentBuilder builder; try { builder = factory.newDocumentBuilder(); - } catch (ParserConfigurationException e1) { + } + catch(ParserConfigurationException e1) { throw new RuntimeException(e1); } DOMImplementation impl = builder.getDOMImplementation(); @@ -454,7 +478,7 @@ public class DocumentParameters { List<Class<?>> classes = new ArrayList<>(byclass.keySet()); Collections.sort(classes, new InspectionUtil.ClassSorter()); - for (Class<?> cls : classes) { + for(Class<?> cls : classes) { // DT = definition term Element classdt = htmldoc.createElement(HTMLUtil.HTML_DT_TAG); // Anchor for references @@ -477,7 +501,7 @@ public class DocumentParameters { // nested definition list for options Element classdl = htmldoc.createElement(HTMLUtil.HTML_DL_TAG); classdd.appendChild(classdl); - for (Parameter<?> opt : byclass.get(cls)) { + for(Parameter<?> opt : byclass.get(cls)) { // DT definition term: option name, in TT for typewriter optics Element elemdt = htmldoc.createElement(HTMLUtil.HTML_DT_TAG); { @@ -489,18 +513,18 @@ public class DocumentParameters { // DD definition description - put the option description here. Element elemdd = htmldoc.createElement(HTMLUtil.HTML_DD_TAG); Element elemp = htmldoc.createElement(HTMLUtil.HTML_P_TAG); - if (opt.getShortDescription() != null) { + if(opt.getShortDescription() != null) { HTMLUtil.appendMultilineText(htmldoc, elemp, opt.getShortDescription()); } elemdd.appendChild(elemp); // class restriction? - if (opt instanceof ClassParameter<?>) { + if(opt instanceof ClassParameter<?>) { appendClassRestriction(htmldoc, ((ClassParameter<?>) opt).getRestrictionClass(), elemdd); } // default value? completions? appendDefaultValueIfSet(htmldoc, opt, elemdd); // known values? - if (opt instanceof ClassParameter<?>) { + if(opt instanceof ClassParameter<?>) { appendKnownImplementationsIfNonempty(htmldoc, (ClassParameter<?>) opt, elemdd); } classdl.appendChild(elemdd); @@ -513,7 +537,7 @@ public class DocumentParameters { * Write to a Wiki format. * * @author Erich Schubert - * + * * @apiviz.exclude */ private static class WikiStream { @@ -535,20 +559,20 @@ public class DocumentParameters { } private void insertNewline() { - if (newline == 2) { + if(newline == 2) { out.print("[[br]]"); } - if (newline != 0) { + if(newline != 0) { printIndent(); newline = 0; } } private void printIndent() { - if (newline > 0) { + if(newline > 0) { out.println(); } - for (int i = indent; i > 0; i--) { + for(int i = indent; i > 0; i--) { out.print(' '); } } @@ -573,7 +597,7 @@ public class DocumentParameters { insertNewline(); out.print("[[javadoc("); out.print(cls.getCanonicalName()); - if (base != null) { + if(base != null) { out.print(","); out.print(ClassParameter.canonicalClassName(cls, base)); } @@ -585,14 +609,14 @@ public class DocumentParameters { List<Class<?>> classes = new ArrayList<>(byclass.keySet()); Collections.sort(classes, new InspectionUtil.ClassSorter()); - for (Class<?> cls : classes) { + for(Class<?> cls : classes) { out.indent = 0; out.printitem("'''"); out.javadocLink(cls, KDDTask.class); out.println("''':"); out.indent = 1; out.newline = 1; // No BR needed, we increase the indent. - for (Parameter<?> opt : byclass.get(cls)) { + for(Parameter<?> opt : byclass.get(cls)) { out.printitem("* "); out.print("{{{"); // typewriter out.print(SerializedParameterization.OPTION_PREFIX); @@ -600,21 +624,21 @@ public class DocumentParameters { out.print(" "); out.print(opt.getSyntax()); out.println("}}}"); - if (opt.getShortDescription() != null) { + if(opt.getShortDescription() != null) { appendMultilineTextWiki(out, opt.getShortDescription()); } // class restriction? - if (opt instanceof ClassParameter<?>) { + if(opt instanceof ClassParameter<?>) { appendClassRestrictionWiki(out, ((ClassParameter<?>) opt).getRestrictionClass()); } // default value? - if (opt.hasDefaultValue()) { + if(opt.hasDefaultValue()) { appendDefaultValueWiki(out, opt); } // known values? - if (FULL_WIKI_OUTPUT) { - if (opt instanceof ClassParameter<?>) { - if (((ClassParameter<?>) opt).getKnownImplementations().size() > 0) { + if(FULL_WIKI_OUTPUT) { + if(opt instanceof ClassParameter<?>) { + if(((ClassParameter<?>) opt).getKnownImplementations().size() > 0) { appendKnownImplementationsWiki(out, (ClassParameter<?>) opt); } } @@ -625,7 +649,7 @@ public class DocumentParameters { private static int appendMultilineTextWiki(WikiStream out, String text) { final String[] lines = text.split("\n"); - for (int i = 0; i < lines.length; i++) { + for(int i = 0; i < lines.length; i++) { out.println(lines[i]); } return lines.length; @@ -636,7 +660,8 @@ public class DocumentParameters { DocumentBuilder builder; try { builder = factory.newDocumentBuilder(); - } catch (ParserConfigurationException e1) { + } + catch(ParserConfigurationException e1) { throw new RuntimeException(e1); } DOMImplementation impl = builder.getDOMImplementation(); @@ -689,7 +714,7 @@ public class DocumentParameters { List<OptionID> opts = new ArrayList<>(byopt.keySet()); Collections.sort(opts, new SortByOption()); - for (OptionID oid : opts) { + for(OptionID oid : opts) { final Parameter<?> firstopt = byopt.get(oid).get(0).getFirst(); // DT = definition term Element optdt = htmldoc.createElement(HTMLUtil.HTML_DT_TAG); @@ -715,12 +740,12 @@ public class DocumentParameters { } // class restriction? Class<?> superclass = null; - if (firstopt instanceof ClassParameter<?>) { + if(firstopt instanceof ClassParameter<?>) { // Find superclass heuristically superclass = ((ClassParameter<?>) firstopt).getRestrictionClass(); - for (Pair<Parameter<?>, Class<?>> clinst : byopt.get(oid)) { + for(Pair<Parameter<?>, Class<?>> clinst : byopt.get(oid)) { ClassParameter<?> cls = (ClassParameter<?>) clinst.getFirst(); - if (!cls.getRestrictionClass().equals(superclass) && cls.getRestrictionClass().isAssignableFrom(superclass)) { + if(!cls.getRestrictionClass().equals(superclass) && cls.getRestrictionClass().isAssignableFrom(superclass)) { superclass = cls.getRestrictionClass(); } } @@ -729,7 +754,7 @@ public class DocumentParameters { // default value? appendDefaultValueIfSet(htmldoc, firstopt, optdd); // known values? - if (firstopt instanceof ClassParameter<?>) { + if(firstopt instanceof ClassParameter<?>) { appendKnownImplementationsIfNonempty(htmldoc, (ClassParameter<?>) firstopt, optdd); } maindl.appendChild(optdd); @@ -741,7 +766,7 @@ public class DocumentParameters { optdd.appendChild(p); } optdd.appendChild(classesul); - for (Pair<Parameter<?>, Class<?>> clinst : byopt.get(oid)) { + for(Pair<Parameter<?>, Class<?>> clinst : byopt.get(oid)) { // DT definition term: option name, in TT for typewriter optics Element classli = htmldoc.createElement(HTMLUtil.HTML_LI_TAG); @@ -752,24 +777,26 @@ public class DocumentParameters { classa.setTextContent(clinst.getSecond().getName()); classli.appendChild(classa); } - if (clinst.getFirst() instanceof ClassParameter<?> && firstopt instanceof ClassParameter<?>) { + if(clinst.getFirst() instanceof ClassParameter<?> && firstopt instanceof ClassParameter<?>) { ClassParameter<?> cls = (ClassParameter<?>) clinst.getFirst(); - if (cls.getRestrictionClass() != null) { + if(cls.getRestrictionClass() != null) { // TODO: if it is null, it could still be different! - if (!cls.getRestrictionClass().equals(superclass)) { + if(!cls.getRestrictionClass().equals(superclass)) { appendClassRestriction(htmldoc, cls.getRestrictionClass(), classli); } - } else { + } + else { appendNoClassRestriction(htmldoc, classli); } } Parameter<?> param = clinst.getFirst(); - if (param.getDefaultValue() != null) { - if (!param.getDefaultValue().equals(firstopt.getDefaultValue())) { + if(param.getDefaultValue() != null) { + if(!param.getDefaultValue().equals(firstopt.getDefaultValue())) { appendDefaultValueIfSet(htmldoc, param, classli); } - } else { - if (firstopt.getDefaultValue() != null) { + } + else { + if(firstopt.getDefaultValue() != null) { appendNoDefaultValue(htmldoc, classli); } } @@ -784,7 +811,7 @@ public class DocumentParameters { List<OptionID> opts = new ArrayList<>(byopt.keySet()); Collections.sort(opts, new SortByOption()); - for (OptionID oid : opts) { + for(OptionID oid : opts) { final Parameter<?> firstopt = byopt.get(oid).get(0).getFirst(); out.indent = 1; out.printitem(""); @@ -800,53 +827,55 @@ public class DocumentParameters { appendMultilineTextWiki(out, firstopt.getShortDescription()); // class restriction? Class<?> superclass = null; - if (firstopt instanceof ClassParameter<?>) { + if(firstopt instanceof ClassParameter<?>) { // Find superclass heuristically superclass = ((ClassParameter<?>) firstopt).getRestrictionClass(); - for (Pair<Parameter<?>, Class<?>> clinst : byopt.get(oid)) { + for(Pair<Parameter<?>, Class<?>> clinst : byopt.get(oid)) { ClassParameter<?> cls = (ClassParameter<?>) clinst.getFirst(); - if (!cls.getRestrictionClass().equals(superclass) && cls.getRestrictionClass().isAssignableFrom(superclass)) { + if(!cls.getRestrictionClass().equals(superclass) && cls.getRestrictionClass().isAssignableFrom(superclass)) { superclass = cls.getRestrictionClass(); } } appendClassRestrictionWiki(out, superclass); } // default value? - if (firstopt.hasDefaultValue()) { + if(firstopt.hasDefaultValue()) { appendDefaultValueWiki(out, firstopt); } - if (FULL_WIKI_OUTPUT) { + if(FULL_WIKI_OUTPUT) { // known values? - if (firstopt instanceof ClassParameter<?>) { - if (((ClassParameter<?>) firstopt).getKnownImplementations().size() > 0) { + if(firstopt instanceof ClassParameter<?>) { + if(((ClassParameter<?>) firstopt).getKnownImplementations().size() > 0) { appendKnownImplementationsWiki(out, (ClassParameter<?>) firstopt); } } // List of classes that use this parameter out.println("Used by:"); - for (Pair<Parameter<?>, Class<?>> clinst : byopt.get(oid)) { + for(Pair<Parameter<?>, Class<?>> clinst : byopt.get(oid)) { out.indent = 3; out.printitem("* "); out.javadocLink(clinst.getSecond(), null); out.println(); - if (clinst.getFirst() instanceof ClassParameter<?> && firstopt instanceof ClassParameter<?>) { + if(clinst.getFirst() instanceof ClassParameter<?> && firstopt instanceof ClassParameter<?>) { ClassParameter<?> cls = (ClassParameter<?>) clinst.getFirst(); - if (cls.getRestrictionClass() != null) { + if(cls.getRestrictionClass() != null) { // TODO: if it is null, it could still be different! - if (!cls.getRestrictionClass().equals(superclass)) { + if(!cls.getRestrictionClass().equals(superclass)) { appendClassRestrictionWiki(out, cls.getRestrictionClass()); } - } else { + } + else { appendNoClassRestrictionWiki(out); } } Parameter<?> param = clinst.getFirst(); - if (param.getDefaultValue() != null) { - if (!param.getDefaultValue().equals(firstopt.getDefaultValue())) { + if(param.getDefaultValue() != null) { + if(!param.getDefaultValue().equals(firstopt.getDefaultValue())) { appendDefaultValueWiki(out, param); } - } else { - if (firstopt.getDefaultValue() != null) { + } + else { + if(firstopt.getDefaultValue() != null) { appendNoDefaultValueWiki(out); } } @@ -864,15 +893,16 @@ public class DocumentParameters { } private static void appendClassRestriction(Document htmldoc, Class<?> restriction, Element elemdd) { - if (restriction == null) { + if(restriction == null) { LOG.warning("No restriction class!"); return; } Element p = htmldoc.createElement(HTMLUtil.HTML_P_TAG); p.appendChild(htmldoc.createTextNode(HEADER_CLASS_RESTRICTION)); - if (restriction.isInterface()) { + if(restriction.isInterface()) { p.appendChild(htmldoc.createTextNode(HEADER_CLASS_RESTRICTION_IMPLEMENTING)); - } else { + } + else { p.appendChild(htmldoc.createTextNode(HEADER_CLASS_RESTRICTION_EXTENDING)); } Element defa = htmldoc.createElement(HTMLUtil.HTML_A_TAG); @@ -890,14 +920,15 @@ public class DocumentParameters { } private static void appendClassRestrictionWiki(WikiStream out, Class<?> restriction) { - if (restriction == null) { + if(restriction == null) { LOG.warning("No restriction class!"); return; } out.print(HEADER_CLASS_RESTRICTION); - if (restriction.isInterface()) { + if(restriction.isInterface()) { out.print(HEADER_CLASS_RESTRICTION_IMPLEMENTING); - } else { + } + else { out.print(HEADER_CLASS_RESTRICTION_EXTENDING); } out.javadocLink(restriction, null); @@ -911,14 +942,14 @@ public class DocumentParameters { } private static void appendKnownImplementationsIfNonempty(Document htmldoc, ClassParameter<?> opt, Element elemdd) { - if (opt.getRestrictionClass() != Object.class) { + if(opt.getRestrictionClass() != Object.class) { List<Class<?>> iter = opt.getKnownImplementations(); - if (!iter.isEmpty()) { + if(!iter.isEmpty()) { Element p = htmldoc.createElement(HTMLUtil.HTML_P_TAG); p.appendChild(htmldoc.createTextNode(HEADER_KNOWN_IMPLEMENTATIONS)); elemdd.appendChild(p); Element ul = htmldoc.createElement(HTMLUtil.HTML_UL_TAG); - for (Class<?> c : iter) { + for(Class<?> c : iter) { Element li = htmldoc.createElement(HTMLUtil.HTML_LI_TAG); Element defa = htmldoc.createElement(HTMLUtil.HTML_A_TAG); defa.setAttribute(HTMLUtil.HTML_HREF_ATTRIBUTE, linkForClassName(c.getName())); @@ -930,7 +961,7 @@ public class DocumentParameters { } // Report when not in properties file. Iterator<Class<?>> clss = new ELKIServiceLoader(opt.getRestrictionClass()); - if (!clss.hasNext()) { + if(!clss.hasNext() && !opt.getRestrictionClass().getName().startsWith("experimentalcode.")) { LOG.warning(opt.getRestrictionClass().getName() + " not in properties. No autocompletion available in release GUI."); } } @@ -940,7 +971,7 @@ public class DocumentParameters { out.println(HEADER_KNOWN_IMPLEMENTATIONS); List<Class<?>> implementations = opt.getKnownImplementations(); out.indent++; - for (Class<?> c : implementations) { + for(Class<?> c : implementations) { out.printitem("* "); out.javadocLink(c, opt.getRestrictionClass()); out.println(); @@ -956,12 +987,13 @@ public class DocumentParameters { * @param optdd HTML Element */ private static void appendDefaultValueIfSet(Document htmldoc, Parameter<?> par, Element optdd) { - if (par.hasDefaultValue()) { + if(par.hasDefaultValue()) { Element p = htmldoc.createElement(HTMLUtil.HTML_P_TAG); p.appendChild(htmldoc.createTextNode(HEADER_DEFAULT_VALUE)); - if (par instanceof ClassParameter<?>) { + if(par instanceof ClassParameter<?>) { appendDefaultClassLink(htmldoc, par, p); - } else { + } + else { Object def = par.getDefaultValue(); p.appendChild(htmldoc.createTextNode(def.toString())); } @@ -984,10 +1016,11 @@ public class DocumentParameters { private static void appendDefaultValueWiki(WikiStream out, Parameter<?> par) { out.print(HEADER_DEFAULT_VALUE); - if (par instanceof ClassParameter<?>) { + if(par instanceof ClassParameter<?>) { final Class<?> name = ((ClassParameter<?>) par).getDefaultValue(); out.javadocLink(name, null); - } else { + } + else { Object def = par.getDefaultValue(); out.print(def.toString()); } diff --git a/src/de/lmu/ifi/dbs/elki/application/internal/DocumentReferences.java b/src/de/lmu/ifi/dbs/elki/application/internal/DocumentReferences.java index 7878a759..34816b6f 100644 --- a/src/de/lmu/ifi/dbs/elki/application/internal/DocumentReferences.java +++ b/src/de/lmu/ifi/dbs/elki/application/internal/DocumentReferences.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.application.internal; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2013 + Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -29,9 +29,11 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.io.PrintStream; +import java.lang.reflect.Field; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; @@ -76,12 +78,12 @@ public class DocumentReferences { LoggingUtil.warning("I need exactly one or two file names to operate!"); System.exit(1); } - if(!args[0].endsWith(".html") || (args.length > 1 && !args[1].endsWith(".wiki"))) { + if(!args[0].endsWith(".html") || (args.length > 1 && !args[1].endsWith(".trac"))) { LoggingUtil.warning("File name doesn't end in expected extension!"); System.exit(1); } - List<Pair<Reference, List<Class<?>>>> refs = sortedReferences(); + List<Pair<Reference, List<Object>>> refs = sortedReferences(); try { File references = new File(args[0]); FileOutputStream reffo = new FileOutputStream(references); @@ -113,7 +115,7 @@ public class DocumentReferences { } } - private static Document documentReferences(List<Pair<Reference, List<Class<?>>>> refs) { + private static Document documentReferences(List<Pair<Reference, List<Object>>> refs) { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); DocumentBuilder builder; try { @@ -168,25 +170,40 @@ public class DocumentReferences { // Main definition list Element maindl = htmldoc.createElement(HTMLUtil.HTML_DL_TAG); body.appendChild(maindl); - for(Pair<Reference, List<Class<?>>> pair : refs) { + for(Pair<Reference, List<Object>> pair : refs) { // DT = definition term Element classdt = htmldoc.createElement(HTMLUtil.HTML_DT_TAG); // Anchor for references { boolean first = true; - for(Class<?> cls : pair.second) { + for(Object o : pair.second) { if(!first) { classdt.appendChild(htmldoc.createTextNode(", ")); } - Element classan = htmldoc.createElement(HTMLUtil.HTML_A_TAG); - classan.setAttribute(HTMLUtil.HTML_NAME_ATTRIBUTE, cls.getName()); - classdt.appendChild(classan); + if(o instanceof Class<?>) { + Class<?> cls = (Class<?>) o; + Element classan = htmldoc.createElement(HTMLUtil.HTML_A_TAG); + classan.setAttribute(HTMLUtil.HTML_NAME_ATTRIBUTE, cls.getName()); + classdt.appendChild(classan); - // Link back to original class - Element classa = htmldoc.createElement(HTMLUtil.HTML_A_TAG); - classa.setAttribute(HTMLUtil.HTML_HREF_ATTRIBUTE, linkForClassName(cls.getName())); - classa.setTextContent(cls.getName()); - classdt.appendChild(classa); + // Link back to original class + Element classa = htmldoc.createElement(HTMLUtil.HTML_A_TAG); + classa.setAttribute(HTMLUtil.HTML_HREF_ATTRIBUTE, linkForClassName(cls.getName())); + classa.setTextContent(cls.getName()); + classdt.appendChild(classa); + } + else if(o instanceof Package) { + Package pkg = (Package) o; + Element classan = htmldoc.createElement(HTMLUtil.HTML_A_TAG); + classan.setAttribute(HTMLUtil.HTML_NAME_ATTRIBUTE, pkg.getName()); + classdt.appendChild(classan); + + // Link back to original class + Element classa = htmldoc.createElement(HTMLUtil.HTML_A_TAG); + classa.setAttribute(HTMLUtil.HTML_HREF_ATTRIBUTE, linkForPackageName(pkg.getName())); + classa.setTextContent(pkg.getName()); + classdt.appendChild(classa); + } first = false; } @@ -215,9 +232,11 @@ public class DocumentReferences { titlediv.appendChild(titleb); classdd.appendChild(titlediv); // Booktitle - Element booktitlediv = htmldoc.createElement(HTMLUtil.HTML_DIV_TAG); - booktitlediv.setTextContent("In: " + ref.booktitle()); - classdd.appendChild(booktitlediv); + if(ref.booktitle().length() > 0) { + Element booktitlediv = htmldoc.createElement(HTMLUtil.HTML_DIV_TAG); + booktitlediv.setTextContent("In: " + ref.booktitle()); + classdd.appendChild(booktitlediv); + } // URL if(ref.url().length() > 0) { Element urldiv = htmldoc.createElement(HTMLUtil.HTML_DIV_TAG); @@ -232,20 +251,31 @@ public class DocumentReferences { return htmldoc; } - private static void documentReferencesWiki(List<Pair<Reference, List<Class<?>>>> refs, PrintStream refstreamW) { - for(Pair<Reference, List<Class<?>>> pair : refs) { - // JavaDoc links for relevant classes. + private static void documentReferencesWiki(List<Pair<Reference, List<Object>>> refs, PrintStream refstreamW) { + for(Pair<Reference, List<Object>> pair : refs) { + // JavaDoc links for relevant classes and packages. { boolean first = true; - for(Class<?> cls : pair.second) { + for(Object o : pair.second) { if(!first) { refstreamW.println(",[[br]]"); } - refstreamW.print("[[javadoc("); - refstreamW.print(cls.getName()); - refstreamW.print(","); - refstreamW.print(cls.getName()); - refstreamW.print(")]]"); + if(o instanceof Class<?>) { + Class<?> cls = (Class<?>) o; + refstreamW.print("[[javadoc("); + refstreamW.print(cls.getName()); + refstreamW.print(","); + refstreamW.print(cls.getName()); + refstreamW.print(")]]"); + } + else if(o instanceof Package) { + Package pkg = (Package) o; + refstreamW.print("[[javadoc("); + refstreamW.print(pkg.getName()); + refstreamW.print(","); + refstreamW.print(pkg.getName()); + refstreamW.print(")]]"); + } first = false; } @@ -264,7 +294,9 @@ public class DocumentReferences { // Title refstreamW.println(indent + "'''" + ref.title() + "'''" + " [[br]]"); // Booktitle - refstreamW.println(indent + "In: " + ref.booktitle() + " [[br]]"); + if(ref.booktitle().length() > 0) { + refstreamW.println(indent + "In: " + ref.booktitle() + " [[br]]"); + } // URL if(ref.url().length() > 0) { refstreamW.println(indent + "Online: [" + ref.url() + "][[br]]"); @@ -275,27 +307,28 @@ public class DocumentReferences { } } - private static List<Pair<Reference, List<Class<?>>>> sortedReferences() { - List<Pair<Reference, List<Class<?>>>> refs = new ArrayList<>(); - Map<Reference, List<Class<?>>> map = new HashMap<>(); + private static List<Pair<Reference, List<Object>>> sortedReferences() { + List<Pair<Reference, List<Object>>> refs = new ArrayList<>(); + Map<Reference, List<Object>> map = new HashMap<>(); - for(final Class<?> cls : InspectionUtil.findAllImplementations(Object.class, true)) { + HashSet<Package> packages = new HashSet<>(); + for(Class<?> cls : InspectionUtil.findAllImplementations(Object.class, true, false)) { inspectClass(cls, refs, map); + if(packages.add(cls.getPackage())) { + inspectPackage(cls.getPackage(), refs, map); + } } return refs; } - private static void inspectClass(final Class<?> cls, List<Pair<Reference, List<Class<?>>>> refs, Map<Reference, List<Class<?>>> map) { + private static void inspectClass(final Class<?> cls, List<Pair<Reference, List<Object>>> refs, Map<Reference, List<Object>> map) { + if(cls.getSimpleName().equals("package-info")) { + return; + } try { if(cls.isAnnotationPresent(Reference.class)) { Reference ref = cls.getAnnotation(Reference.class); - List<Class<?>> list = map.get(ref); - if(list == null) { - list = new ArrayList<>(5); - map.put(ref, list); - refs.add(new Pair<>(ref, list)); - } - list.add(cls); + addReference(cls, ref, refs, map); } // Inner classes for(Class<?> c2 : cls.getDeclaredClasses()) { @@ -304,13 +337,13 @@ public class DocumentReferences { for(Method m : cls.getDeclaredMethods()) { if(m.isAnnotationPresent(Reference.class)) { Reference ref = m.getAnnotation(Reference.class); - List<Class<?>> list = map.get(ref); - if(list == null) { - list = new ArrayList<>(5); - map.put(ref, list); - refs.add(new Pair<>(ref, list)); - } - list.add(cls); + addReference(cls, ref, refs, map); + } + } + for(Field f : cls.getDeclaredFields()) { + if(f.isAnnotationPresent(Reference.class)) { + Reference ref = f.getAnnotation(Reference.class); + addReference(cls, ref, refs, map); } } } @@ -324,19 +357,39 @@ public class DocumentReferences { } } + private static void addReference(Object cls, Reference ref, List<Pair<Reference, List<Object>>> refs, Map<Reference, List<Object>> map) { + List<Object> list = map.get(ref); + if(list == null) { + list = new ArrayList<>(3); + map.put(ref, list); + refs.add(new Pair<>(ref, list)); + } + list.add(cls); + } + + private static void inspectPackage(Package p, List<Pair<Reference, List<Object>>> refs, Map<Reference, List<Object>> map) { + if(p.isAnnotationPresent(Reference.class)) { + Reference ref = p.getAnnotation(Reference.class); + addReference(p, ref, refs, map); + } + } + private static String linkForClassName(String name) { - String link = name.replace(".", "/") + ".html"; - return link; + return name.replace(".", "/") + ".html"; + } + + private static String linkForPackageName(String name) { + return name.replace(".", "/") + "/package-summary.html"; } /** - * Fin all classes that have the reference annotation + * Find all classes that have the reference annotation * * @return All classes with the reference annotation. */ public static ArrayList<Class<?>> findAllClassesWithReferences() { ArrayList<Class<?>> references = new ArrayList<>(); - for(final Class<?> cls : InspectionUtil.findAllImplementations(Object.class, true)) { + for(final Class<?> cls : InspectionUtil.findAllImplementations(Object.class, true, false)) { if(cls.isAnnotationPresent(Reference.class)) { references.add(cls); } diff --git a/src/de/lmu/ifi/dbs/elki/application/internal/package-info.java b/src/de/lmu/ifi/dbs/elki/application/internal/package-info.java index ecd1724d..19c3a795 100644 --- a/src/de/lmu/ifi/dbs/elki/application/internal/package-info.java +++ b/src/de/lmu/ifi/dbs/elki/application/internal/package-info.java @@ -5,7 +5,7 @@ This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures -Copyright (C) 2013 +Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team diff --git a/src/de/lmu/ifi/dbs/elki/application/jsmap/JSONBuffer.java b/src/de/lmu/ifi/dbs/elki/application/jsmap/JSONBuffer.java index e1ecc2cb..b831d633 100644 --- a/src/de/lmu/ifi/dbs/elki/application/jsmap/JSONBuffer.java +++ b/src/de/lmu/ifi/dbs/elki/application/jsmap/JSONBuffer.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.application.jsmap; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2013 + Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team diff --git a/src/de/lmu/ifi/dbs/elki/application/jsmap/JSONResultHandler.java b/src/de/lmu/ifi/dbs/elki/application/jsmap/JSONResultHandler.java index 16331b02..3ce22e2a 100644 --- a/src/de/lmu/ifi/dbs/elki/application/jsmap/JSONResultHandler.java +++ b/src/de/lmu/ifi/dbs/elki/application/jsmap/JSONResultHandler.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.application.jsmap; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2013 + Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team diff --git a/src/de/lmu/ifi/dbs/elki/application/jsmap/JSONWebServer.java b/src/de/lmu/ifi/dbs/elki/application/jsmap/JSONWebServer.java index 6f8c8be7..4c861bef 100644 --- a/src/de/lmu/ifi/dbs/elki/application/jsmap/JSONWebServer.java +++ b/src/de/lmu/ifi/dbs/elki/application/jsmap/JSONWebServer.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.application.jsmap; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2013 + Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -44,6 +44,7 @@ import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; +import de.lmu.ifi.dbs.elki.database.relation.DoubleRelation; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.datasource.bundle.SingleObjectBundle; import de.lmu.ifi.dbs.elki.logging.Logging; @@ -147,7 +148,7 @@ public class JSONWebServer implements HttpHandler { // TODO: refactor to JSONFormatters! // Format a NumberVector if (data instanceof NumberVector) { - NumberVector<?> v = (NumberVector<?>) data; + NumberVector v = (NumberVector) data; re.appendKeyArray(bundle.meta(j)); for (int i = 0; i < v.getDimensionality(); i++) { re.append(v.doubleValue(i)); @@ -312,7 +313,7 @@ public class JSONWebServer implements HttpHandler { outlierMetaToJSON(re, meta); re.appendKeyArray("scores"); - Relation<Double> scores = or.getScores(); + DoubleRelation scores = or.getScores(); DBIDIter iter = or.getOrdering().iter(scores.getDBIDs()).iter(); for (int i = 0; i < offset && iter.valid(); i++) { iter.advance(); @@ -320,8 +321,8 @@ public class JSONWebServer implements HttpHandler { for (int i = 0; i < pagesize && iter.valid(); i++, iter.advance()) { re.startHash(); bundleToJSON(re, iter); - final Double val = scores.get(iter); - if (val != null) { + final double val = scores.doubleValue(iter); + if (!Double.isNaN(val)) { re.appendKeyValue("score", val); } re.closeHash(); diff --git a/src/de/lmu/ifi/dbs/elki/application/jsmap/package-info.java b/src/de/lmu/ifi/dbs/elki/application/jsmap/package-info.java index 458bbd93..c410b2a7 100644 --- a/src/de/lmu/ifi/dbs/elki/application/jsmap/package-info.java +++ b/src/de/lmu/ifi/dbs/elki/application/jsmap/package-info.java @@ -5,7 +5,7 @@ This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures -Copyright (C) 2013 +Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team diff --git a/src/de/lmu/ifi/dbs/elki/application/package-info.java b/src/de/lmu/ifi/dbs/elki/application/package-info.java index b2e79f5d..0e050e5b 100644 --- a/src/de/lmu/ifi/dbs/elki/application/package-info.java +++ b/src/de/lmu/ifi/dbs/elki/application/package-info.java @@ -5,7 +5,7 @@ This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures -Copyright (C) 2013 +Copyright (C) 2014 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team |