diff options
| author | Andrej Shadura <andrewsh@debian.org> | 2019-03-09 22:30:33 +0000 |
|---|---|---|
| committer | Andrej Shadura <andrewsh@debian.org> | 2019-03-09 22:30:33 +0000 |
| commit | ace5fa7f57d49756c0e1b111a30f3b6a9436c1cb (patch) | |
| tree | 041e034bddeeaf574c02ca8f040b1359cef00133 | |
| parent | c36aa2a8fd31ca5e225ff30278e910070cd2c8c1 (diff) | |
Import Upstream version 0.5.0
578 files changed, 24193 insertions, 4017 deletions
diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.AbstractDistanceBasedAlgorithm b/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.AbstractDistanceBasedAlgorithm index dd8d4482..69a3aaed 100644 --- a/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.AbstractDistanceBasedAlgorithm +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.AbstractDistanceBasedAlgorithm @@ -5,11 +5,14 @@ de.lmu.ifi.dbs.elki.algorithm.clustering.DBSCAN de.lmu.ifi.dbs.elki.algorithm.clustering.OPTICS de.lmu.ifi.dbs.elki.algorithm.clustering.DeLiClu de.lmu.ifi.dbs.elki.algorithm.clustering.SLINK +de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMedoidsPAM +de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMedoidsEM de.lmu.ifi.dbs.elki.algorithm.clustering.correlation.HiCO de.lmu.ifi.dbs.elki.algorithm.clustering.subspace.HiSC de.lmu.ifi.dbs.elki.algorithm.outlier.ABOD de.lmu.ifi.dbs.elki.algorithm.outlier.DBOutlierDetection de.lmu.ifi.dbs.elki.algorithm.outlier.DBOutlierScore +de.lmu.ifi.dbs.elki.algorithm.outlier.HilOut de.lmu.ifi.dbs.elki.algorithm.outlier.INFLO de.lmu.ifi.dbs.elki.algorithm.outlier.KNNOutlier de.lmu.ifi.dbs.elki.algorithm.outlier.KNNWeightOutlier @@ -22,3 +25,4 @@ de.lmu.ifi.dbs.elki.algorithm.statistics.AveragePrecisionAtK de.lmu.ifi.dbs.elki.algorithm.statistics.RankingQualityHistogram de.lmu.ifi.dbs.elki.algorithm.statistics.DistanceStatisticsWithClasses de.lmu.ifi.dbs.elki.algorithm.statistics.EvaluateRankingQuality +tutorial.outlier.DistanceStddevOutlier
\ No newline at end of file diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.AbstractPrimitiveDistanceBasedAlgorithm b/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.AbstractPrimitiveDistanceBasedAlgorithm index 375277a4..7ee58e22 100644 --- a/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.AbstractPrimitiveDistanceBasedAlgorithm +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.AbstractPrimitiveDistanceBasedAlgorithm @@ -1,3 +1,4 @@ de.lmu.ifi.dbs.elki.algorithm.DependencyDerivator de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMeansLloyd de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMeansMacQueen +de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMediansLloyd
\ No newline at end of file diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.Algorithm b/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.Algorithm index 466a746a..37621d8b 100644 --- a/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.Algorithm +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.Algorithm @@ -2,8 +2,12 @@ de.lmu.ifi.dbs.elki.algorithm.NullAlgorithm de.lmu.ifi.dbs.elki.algorithm.clustering.DBSCAN de.lmu.ifi.dbs.elki.algorithm.clustering.DeLiClu de.lmu.ifi.dbs.elki.algorithm.clustering.EM +de.lmu.ifi.dbs.elki.algorithm.clustering.gdbscan.GeneralizedDBSCAN de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMeansLloyd de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMeansMacQueen +de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMediansLloyd +de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMedoidsPAM +de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMedoidsEM de.lmu.ifi.dbs.elki.algorithm.clustering.OPTICSXi de.lmu.ifi.dbs.elki.algorithm.clustering.OPTICS de.lmu.ifi.dbs.elki.algorithm.clustering.SLINK @@ -26,14 +30,17 @@ de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.ByLabelHierarchicalClustering de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.ByModelClustering de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.TrivialAllInOne de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.TrivialAllNoise +de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.ByLabelOrAllInOneClustering de.lmu.ifi.dbs.elki.algorithm.outlier.ABOD de.lmu.ifi.dbs.elki.algorithm.outlier.AggarwalYuEvolutionary de.lmu.ifi.dbs.elki.algorithm.outlier.AggarwalYuNaive +de.lmu.ifi.dbs.elki.algorithm.outlier.ALOCI de.lmu.ifi.dbs.elki.algorithm.outlier.DBOutlierDetection de.lmu.ifi.dbs.elki.algorithm.outlier.DBOutlierScore de.lmu.ifi.dbs.elki.algorithm.outlier.EMOutlier de.lmu.ifi.dbs.elki.algorithm.outlier.GaussianModel de.lmu.ifi.dbs.elki.algorithm.outlier.GaussianUniformMixture +de.lmu.ifi.dbs.elki.algorithm.outlier.HilOut de.lmu.ifi.dbs.elki.algorithm.outlier.INFLO de.lmu.ifi.dbs.elki.algorithm.outlier.KNNOutlier de.lmu.ifi.dbs.elki.algorithm.outlier.KNNWeightOutlier @@ -42,10 +49,11 @@ de.lmu.ifi.dbs.elki.algorithm.outlier.LOCI de.lmu.ifi.dbs.elki.algorithm.outlier.LOF de.lmu.ifi.dbs.elki.algorithm.outlier.LoOP de.lmu.ifi.dbs.elki.algorithm.outlier.OPTICSOF -de.lmu.ifi.dbs.elki.algorithm.outlier.OUTRES de.lmu.ifi.dbs.elki.algorithm.outlier.ReferenceBasedOutlierDetection -de.lmu.ifi.dbs.elki.algorithm.outlier.SOD de.lmu.ifi.dbs.elki.algorithm.outlier.OnlineLOF +de.lmu.ifi.dbs.elki.algorithm.outlier.subspace.OUTRES +de.lmu.ifi.dbs.elki.algorithm.outlier.subspace.OutRankS1 +de.lmu.ifi.dbs.elki.algorithm.outlier.subspace.SOD de.lmu.ifi.dbs.elki.algorithm.outlier.spatial.CTLuGLSBackwardSearchAlgorithm de.lmu.ifi.dbs.elki.algorithm.outlier.spatial.CTLuMeanMultipleAttributes de.lmu.ifi.dbs.elki.algorithm.outlier.spatial.CTLuMedianAlgorithm @@ -59,11 +67,13 @@ de.lmu.ifi.dbs.elki.algorithm.outlier.spatial.SOF de.lmu.ifi.dbs.elki.algorithm.outlier.spatial.TrimmedMeanApproach de.lmu.ifi.dbs.elki.algorithm.outlier.meta.ExternalDoubleOutlierScore de.lmu.ifi.dbs.elki.algorithm.outlier.meta.FeatureBagging +de.lmu.ifi.dbs.elki.algorithm.outlier.meta.HiCS de.lmu.ifi.dbs.elki.algorithm.outlier.meta.RescaleMetaOutlierAlgorithm de.lmu.ifi.dbs.elki.algorithm.outlier.trivial.ByLabelOutlier de.lmu.ifi.dbs.elki.algorithm.outlier.trivial.TrivialAllOutlier de.lmu.ifi.dbs.elki.algorithm.outlier.trivial.TrivialNoOutlier de.lmu.ifi.dbs.elki.algorithm.outlier.trivial.TrivialGeneratedOutlier +de.lmu.ifi.dbs.elki.algorithm.statistics.AddSingleScale de.lmu.ifi.dbs.elki.algorithm.statistics.AveragePrecisionAtK de.lmu.ifi.dbs.elki.algorithm.statistics.EvaluateRankingQuality de.lmu.ifi.dbs.elki.algorithm.statistics.RankingQualityHistogram @@ -74,3 +84,4 @@ de.lmu.ifi.dbs.elki.algorithm.DependencyDerivator de.lmu.ifi.dbs.elki.algorithm.KNNDistanceOrder de.lmu.ifi.dbs.elki.algorithm.KNNJoin de.lmu.ifi.dbs.elki.algorithm.MaterializeDistances +tutorial.outlier.DistanceStddevOutlier
\ No newline at end of file diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.clustering.ClusteringAlgorithm b/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.clustering.ClusteringAlgorithm index f429b53e..2ae85aa5 100644 --- a/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.clustering.ClusteringAlgorithm +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.clustering.ClusteringAlgorithm @@ -1,7 +1,11 @@ de.lmu.ifi.dbs.elki.algorithm.clustering.DBSCAN de.lmu.ifi.dbs.elki.algorithm.clustering.EM +de.lmu.ifi.dbs.elki.algorithm.clustering.gdbscan.GeneralizedDBSCAN de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMeansLloyd de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMeansMacQueen +de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMediansLloyd +de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMedoidsPAM +de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMedoidsEM de.lmu.ifi.dbs.elki.algorithm.clustering.OPTICSXi de.lmu.ifi.dbs.elki.algorithm.clustering.SNNClustering de.lmu.ifi.dbs.elki.algorithm.clustering.correlation.CASH @@ -19,3 +23,4 @@ de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.ByLabelHierarchicalClustering de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.ByModelClustering de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.TrivialAllInOne de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.TrivialAllNoise +de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.ByLabelOrAllInOneClustering
\ No newline at end of file diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.clustering.gdbscan.CorePredicate b/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.clustering.gdbscan.CorePredicate new file mode 100644 index 00000000..df33be1c --- /dev/null +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.clustering.gdbscan.CorePredicate @@ -0,0 +1 @@ +de.lmu.ifi.dbs.elki.algorithm.clustering.gdbscan.MinPtsCorePredicate
\ No newline at end of file diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.clustering.gdbscan.NeighborPredicate b/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.clustering.gdbscan.NeighborPredicate new file mode 100644 index 00000000..08ed8efa --- /dev/null +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.clustering.gdbscan.NeighborPredicate @@ -0,0 +1 @@ +de.lmu.ifi.dbs.elki.algorithm.clustering.gdbscan.EpsilonNeighborPredicate
\ No newline at end of file diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMeansInitialization b/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMeansInitialization index b5006bd4..5734e7bf 100644 --- a/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMeansInitialization +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMeansInitialization @@ -2,3 +2,4 @@ de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.RandomlyGeneratedInitialMeans de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.RandomlyChosenInitialMeans de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.FirstKInitialMeans de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMeansPlusPlusInitialMeans +de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.PAMInitialMeans
\ No newline at end of file diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMedoidsInitialization b/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMedoidsInitialization new file mode 100644 index 00000000..c75b1cf1 --- /dev/null +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMedoidsInitialization @@ -0,0 +1,4 @@ +de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.PAMInitialMeans +de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.RandomlyChosenInitialMeans +de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.FirstKInitialMeans +de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMeansPlusPlusInitialMeans
\ No newline at end of file diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.clustering.subspace.SubspaceClusteringAlgorithm b/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.clustering.subspace.SubspaceClusteringAlgorithm new file mode 100644 index 00000000..c71dd241 --- /dev/null +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.clustering.subspace.SubspaceClusteringAlgorithm @@ -0,0 +1,4 @@ +de.lmu.ifi.dbs.elki.algorithm.clustering.subspace.CLIQUE +de.lmu.ifi.dbs.elki.algorithm.clustering.subspace.DiSH +de.lmu.ifi.dbs.elki.algorithm.clustering.subspace.PROCLUS +de.lmu.ifi.dbs.elki.algorithm.clustering.subspace.SUBCLU
\ No newline at end of file diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.outlier.OutlierAlgorithm b/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.outlier.OutlierAlgorithm index 14bb81fa..18ac09f8 100644 --- a/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.outlier.OutlierAlgorithm +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.algorithm.outlier.OutlierAlgorithm @@ -1,11 +1,13 @@ de.lmu.ifi.dbs.elki.algorithm.outlier.ABOD de.lmu.ifi.dbs.elki.algorithm.outlier.AggarwalYuEvolutionary de.lmu.ifi.dbs.elki.algorithm.outlier.AggarwalYuNaive +de.lmu.ifi.dbs.elki.algorithm.outlier.ALOCI de.lmu.ifi.dbs.elki.algorithm.outlier.DBOutlierDetection de.lmu.ifi.dbs.elki.algorithm.outlier.DBOutlierScore de.lmu.ifi.dbs.elki.algorithm.outlier.EMOutlier de.lmu.ifi.dbs.elki.algorithm.outlier.GaussianModel de.lmu.ifi.dbs.elki.algorithm.outlier.GaussianUniformMixture +de.lmu.ifi.dbs.elki.algorithm.outlier.HilOut de.lmu.ifi.dbs.elki.algorithm.outlier.INFLO de.lmu.ifi.dbs.elki.algorithm.outlier.KNNOutlier de.lmu.ifi.dbs.elki.algorithm.outlier.KNNWeightOutlier @@ -14,10 +16,11 @@ de.lmu.ifi.dbs.elki.algorithm.outlier.LOCI de.lmu.ifi.dbs.elki.algorithm.outlier.LOF de.lmu.ifi.dbs.elki.algorithm.outlier.LoOP de.lmu.ifi.dbs.elki.algorithm.outlier.OPTICSOF -de.lmu.ifi.dbs.elki.algorithm.outlier.OUTRES de.lmu.ifi.dbs.elki.algorithm.outlier.ReferenceBasedOutlierDetection -de.lmu.ifi.dbs.elki.algorithm.outlier.SOD de.lmu.ifi.dbs.elki.algorithm.outlier.OnlineLOF +de.lmu.ifi.dbs.elki.algorithm.outlier.subspace.OUTRES +de.lmu.ifi.dbs.elki.algorithm.outlier.subspace.OutRankS1 +de.lmu.ifi.dbs.elki.algorithm.outlier.subspace.SOD de.lmu.ifi.dbs.elki.algorithm.outlier.spatial.CTLuGLSBackwardSearchAlgorithm de.lmu.ifi.dbs.elki.algorithm.outlier.spatial.CTLuMeanMultipleAttributes de.lmu.ifi.dbs.elki.algorithm.outlier.spatial.CTLuMedianAlgorithm @@ -31,8 +34,10 @@ de.lmu.ifi.dbs.elki.algorithm.outlier.spatial.SOF de.lmu.ifi.dbs.elki.algorithm.outlier.spatial.TrimmedMeanApproach de.lmu.ifi.dbs.elki.algorithm.outlier.meta.ExternalDoubleOutlierScore de.lmu.ifi.dbs.elki.algorithm.outlier.meta.FeatureBagging +de.lmu.ifi.dbs.elki.algorithm.outlier.meta.HiCS de.lmu.ifi.dbs.elki.algorithm.outlier.meta.RescaleMetaOutlierAlgorithm de.lmu.ifi.dbs.elki.algorithm.outlier.trivial.ByLabelOutlier de.lmu.ifi.dbs.elki.algorithm.outlier.trivial.TrivialAllOutlier de.lmu.ifi.dbs.elki.algorithm.outlier.trivial.TrivialNoOutlier de.lmu.ifi.dbs.elki.algorithm.outlier.trivial.TrivialGeneratedOutlier +tutorial.outlier.DistanceStddevOutlier
\ No newline at end of file diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.data.NumberVector b/src/META-INF/elki/de.lmu.ifi.dbs.elki.data.NumberVector index 9ee14dab..fd4cab91 100644 --- a/src/META-INF/elki/de.lmu.ifi.dbs.elki.data.NumberVector +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.data.NumberVector @@ -5,6 +5,7 @@ de.lmu.ifi.dbs.elki.data.IntegerVector de.lmu.ifi.dbs.elki.data.OneDimensionalDoubleVector de.lmu.ifi.dbs.elki.data.ParameterizationFunction de.lmu.ifi.dbs.elki.data.SparseFloatVector +de.lmu.ifi.dbs.elki.data.SparseDoubleVector # de.lmu.ifi.dbs.elki.math.linearalgebra.Vector # de.lmu.ifi.dbs.elki.math.linearalgebra.Centroid # de.lmu.ifi.dbs.elki.math.linearalgebra.ProjectedCentroid
\ No newline at end of file diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.data.SparseNumberVector b/src/META-INF/elki/de.lmu.ifi.dbs.elki.data.SparseNumberVector new file mode 100644 index 00000000..ca27b7b8 --- /dev/null +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.data.SparseNumberVector @@ -0,0 +1,2 @@ +de.lmu.ifi.dbs.elki.data.SparseFloatVector +de.lmu.ifi.dbs.elki.data.SparseDoubleVector
\ No newline at end of file diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.datasource.filter.ObjectFilter b/src/META-INF/elki/de.lmu.ifi.dbs.elki.datasource.filter.ObjectFilter index 3c159ed2..fa40e217 100644 --- a/src/META-INF/elki/de.lmu.ifi.dbs.elki.datasource.filter.ObjectFilter +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.datasource.filter.ObjectFilter @@ -6,10 +6,11 @@ de.lmu.ifi.dbs.elki.datasource.filter.ExternalIDFilter de.lmu.ifi.dbs.elki.datasource.filter.FixedDBIDsFilter de.lmu.ifi.dbs.elki.datasource.filter.NoOpFilter de.lmu.ifi.dbs.elki.datasource.filter.NoMissingValuesFilter +de.lmu.ifi.dbs.elki.datasource.filter.RandomSamplingStreamFilter de.lmu.ifi.dbs.elki.datasource.filter.ShuffleObjectsFilter de.lmu.ifi.dbs.elki.datasource.filter.SortByLabelFilter -de.lmu.ifi.dbs.elki.datasource.filter.SparseFloatVectorProjectionFilter -de.lmu.ifi.dbs.elki.datasource.filter.SparseFloatVectorRandomProjectionFilter +de.lmu.ifi.dbs.elki.datasource.filter.SparseNumberVectorProjectionFilter +de.lmu.ifi.dbs.elki.datasource.filter.SparseNumberVectorRandomProjectionFilter de.lmu.ifi.dbs.elki.datasource.filter.SparseVectorFieldFilter de.lmu.ifi.dbs.elki.datasource.filter.SplitNumberVectorFilter de.lmu.ifi.dbs.elki.datasource.filter.normalization.AttributeWiseErfNormalization @@ -19,3 +20,4 @@ de.lmu.ifi.dbs.elki.datasource.filter.normalization.LengthNormalization de.lmu.ifi.dbs.elki.datasource.filter.normalization.InverseDocumentFrequencyNormalization de.lmu.ifi.dbs.elki.datasource.filter.normalization.RankTieNormalization de.lmu.ifi.dbs.elki.datasource.filter.normalization.TFIDFNormalization +de.lmu.ifi.dbs.elki.datasource.filter.transform.GlobalPrincipalComponentAnalysisTransform
\ No newline at end of file diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.datasource.filter.StreamFilter b/src/META-INF/elki/de.lmu.ifi.dbs.elki.datasource.filter.StreamFilter new file mode 100644 index 00000000..178916cd --- /dev/null +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.datasource.filter.StreamFilter @@ -0,0 +1,10 @@ +de.lmu.ifi.dbs.elki.datasource.filter.ByLabelFilter +de.lmu.ifi.dbs.elki.datasource.filter.DoubleVectorProjectionFilter +de.lmu.ifi.dbs.elki.datasource.filter.DoubleVectorRandomProjectionFilter +de.lmu.ifi.dbs.elki.datasource.filter.FixedDBIDsFilter +de.lmu.ifi.dbs.elki.datasource.filter.NoMissingValuesFilter +de.lmu.ifi.dbs.elki.datasource.filter.NoOpFilter +de.lmu.ifi.dbs.elki.datasource.filter.RandomSamplingStreamFilter +de.lmu.ifi.dbs.elki.datasource.filter.SparseNumberVectorProjectionFilter +de.lmu.ifi.dbs.elki.datasource.filter.SparseNumberVectorRandomProjectionFilter +de.lmu.ifi.dbs.elki.datasource.filter.normalization.LengthNormalization
\ No newline at end of file diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.datasource.parser.Parser b/src/META-INF/elki/de.lmu.ifi.dbs.elki.datasource.parser.Parser index 4002f056..3d005531 100644 --- a/src/META-INF/elki/de.lmu.ifi.dbs.elki.datasource.parser.Parser +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.datasource.parser.Parser @@ -1,10 +1,10 @@ de.lmu.ifi.dbs.elki.datasource.parser.NumberVectorLabelParser de.lmu.ifi.dbs.elki.datasource.parser.ArffParser -de.lmu.ifi.dbs.elki.datasource.parser.DoubleVectorLabelParser -de.lmu.ifi.dbs.elki.datasource.parser.FloatVectorLabelParser +de.lmu.ifi.dbs.elki.datasource.parser.SparseNumberVectorLabelParser de.lmu.ifi.dbs.elki.datasource.parser.SparseBitVectorLabelParser -de.lmu.ifi.dbs.elki.datasource.parser.SparseFloatVectorLabelParser de.lmu.ifi.dbs.elki.datasource.parser.TermFrequencyParser de.lmu.ifi.dbs.elki.datasource.parser.BitVectorLabelParser -de.lmu.ifi.dbs.elki.datasource.parser.ParameterizationFunctionLabelParser de.lmu.ifi.dbs.elki.datasource.parser.SimplePolygonParser +# deprecated: de.lmu.ifi.dbs.elki.datasource.parser.DoubleVectorLabelParser +# deprecated: de.lmu.ifi.dbs.elki.datasource.parser.FloatVectorLabelParser +# deprecated: de.lmu.ifi.dbs.elki.datasource.parser.SparseFloatVectorLabelParser
\ No newline at end of file diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction b/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction index 45c31e56..249db041 100644 --- a/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction @@ -16,6 +16,10 @@ de.lmu.ifi.dbs.elki.distance.distancefunction.LocallyWeightedDistanceFunction # de.lmu.ifi.dbs.elki.distance.distancefunction.ProxyDistanceFunction # de.lmu.ifi.dbs.elki.distance.distancefunction.WeightedLPNormDistanceFunction # de.lmu.ifi.dbs.elki.distance.distancefunction.WeightedSquaredEuclideanDistanceFunction +de.lmu.ifi.dbs.elki.distance.distancefunction.SparseEuclideanDistanceFunction +de.lmu.ifi.dbs.elki.distance.distancefunction.SparseManhattanDistanceFunction +de.lmu.ifi.dbs.elki.distance.distancefunction.SparseLPNormDistanceFunction +de.lmu.ifi.dbs.elki.distance.distancefunction.SparseMaximumDistanceFunction de.lmu.ifi.dbs.elki.distance.distancefunction.adapter.SimilarityAdapterArccos de.lmu.ifi.dbs.elki.distance.distancefunction.adapter.SimilarityAdapterLn de.lmu.ifi.dbs.elki.distance.distancefunction.adapter.SimilarityAdapterLinear diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.DoubleNorm b/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.DoubleNorm index bb3847a9..2714a376 100644 --- a/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.DoubleNorm +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.DoubleNorm @@ -4,7 +4,11 @@ de.lmu.ifi.dbs.elki.distance.distancefunction.MaximumDistanceFunction de.lmu.ifi.dbs.elki.distance.distancefunction.MinimumDistanceFunction de.lmu.ifi.dbs.elki.distance.distancefunction.LPNormDistanceFunction de.lmu.ifi.dbs.elki.distance.distancefunction.SquaredEuclideanDistanceFunction -de.lmu.ifi.dbs.elki.distance.distancefunction.WeightedLPNormDistanceFunction +# de.lmu.ifi.dbs.elki.distance.distancefunction.WeightedLPNormDistanceFunction +de.lmu.ifi.dbs.elki.distance.distancefunction.SparseEuclideanDistanceFunction +de.lmu.ifi.dbs.elki.distance.distancefunction.SparseManhattanDistanceFunction +de.lmu.ifi.dbs.elki.distance.distancefunction.SparseLPNormDistanceFunction +de.lmu.ifi.dbs.elki.distance.distancefunction.SparseMaximumDistanceFunction de.lmu.ifi.dbs.elki.distance.distancefunction.subspace.SubspaceEuclideanDistanceFunction de.lmu.ifi.dbs.elki.distance.distancefunction.subspace.SubspaceLPNormDistanceFunction de.lmu.ifi.dbs.elki.distance.distancefunction.subspace.SubspaceManhattanDistanceFunction diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.LPNormDistanceFunction b/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.LPNormDistanceFunction new file mode 100644 index 00000000..4cac929a --- /dev/null +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.LPNormDistanceFunction @@ -0,0 +1,5 @@ +de.lmu.ifi.dbs.elki.distance.distancefunction.EuclideanDistanceFunction +de.lmu.ifi.dbs.elki.distance.distancefunction.ManhattanDistanceFunction +de.lmu.ifi.dbs.elki.distance.distancefunction.LPNormDistanceFunction +de.lmu.ifi.dbs.elki.distance.distancefunction.MaximumDistanceFunction +# de.lmu.ifi.dbs.elki.distance.distancefunction.WeightedLPNormDistanceFunction
\ No newline at end of file diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDistanceFunction b/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDistanceFunction index bc770e74..9dc29f22 100644 --- a/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDistanceFunction +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDistanceFunction @@ -11,6 +11,10 @@ de.lmu.ifi.dbs.elki.distance.distancefunction.JeffreyDivergenceDistanceFunction # de.lmu.ifi.dbs.elki.distance.distancefunction.WeightedDistanceFunction # de.lmu.ifi.dbs.elki.distance.distancefunction.WeightedLPNormDistanceFunction # de.lmu.ifi.dbs.elki.distance.distancefunction.WeightedSquaredEuclideanDistanceFunction +de.lmu.ifi.dbs.elki.distance.distancefunction.SparseEuclideanDistanceFunction +de.lmu.ifi.dbs.elki.distance.distancefunction.SparseManhattanDistanceFunction +de.lmu.ifi.dbs.elki.distance.distancefunction.SparseLPNormDistanceFunction +de.lmu.ifi.dbs.elki.distance.distancefunction.SparseMaximumDistanceFunction de.lmu.ifi.dbs.elki.distance.distancefunction.colorhistogram.HSBHistogramQuadraticDistanceFunction de.lmu.ifi.dbs.elki.distance.distancefunction.colorhistogram.HistogramIntersectionDistanceFunction de.lmu.ifi.dbs.elki.distance.distancefunction.colorhistogram.RGBHistogramQuadraticDistanceFunction diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDoubleDistanceFunction b/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDoubleDistanceFunction index 3bf5428e..35baef47 100644 --- a/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDoubleDistanceFunction +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDoubleDistanceFunction @@ -11,6 +11,10 @@ de.lmu.ifi.dbs.elki.distance.distancefunction.JeffreyDivergenceDistanceFunction # de.lmu.ifi.dbs.elki.distance.distancefunction.WeightedDistanceFunction # de.lmu.ifi.dbs.elki.distance.distancefunction.WeightedLPNormDistanceFunction # de.lmu.ifi.dbs.elki.distance.distancefunction.WeightedSquaredEuclideanDistanceFunction +de.lmu.ifi.dbs.elki.distance.distancefunction.SparseEuclideanDistanceFunction +de.lmu.ifi.dbs.elki.distance.distancefunction.SparseManhattanDistanceFunction +de.lmu.ifi.dbs.elki.distance.distancefunction.SparseLPNormDistanceFunction +de.lmu.ifi.dbs.elki.distance.distancefunction.SparseMaximumDistanceFunction de.lmu.ifi.dbs.elki.distance.distancefunction.colorhistogram.HSBHistogramQuadraticDistanceFunction de.lmu.ifi.dbs.elki.distance.distancefunction.colorhistogram.HistogramIntersectionDistanceFunction de.lmu.ifi.dbs.elki.distance.distancefunction.colorhistogram.RGBHistogramQuadraticDistanceFunction diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.SpatialPrimitiveDistanceFunction b/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.SpatialPrimitiveDistanceFunction index bc1c6777..01d9789a 100644 --- a/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.SpatialPrimitiveDistanceFunction +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.distance.distancefunction.SpatialPrimitiveDistanceFunction @@ -7,9 +7,9 @@ de.lmu.ifi.dbs.elki.distance.distancefunction.ArcCosineDistanceFunction de.lmu.ifi.dbs.elki.distance.distancefunction.CosineDistanceFunction de.lmu.ifi.dbs.elki.distance.distancefunction.CanberraDistanceFunction de.lmu.ifi.dbs.elki.distance.distancefunction.SquaredEuclideanDistanceFunction -de.lmu.ifi.dbs.elki.distance.distancefunction.WeightedLPNormDistanceFunction +# de.lmu.ifi.dbs.elki.distance.distancefunction.WeightedLPNormDistanceFunction de.lmu.ifi.dbs.elki.distance.distancefunction.colorhistogram.HistogramIntersectionDistanceFunction de.lmu.ifi.dbs.elki.distance.distancefunction.subspace.DimensionSelectingDistanceFunction de.lmu.ifi.dbs.elki.distance.distancefunction.subspace.SubspaceEuclideanDistanceFunction de.lmu.ifi.dbs.elki.distance.distancefunction.subspace.SubspaceLPNormDistanceFunction -de.lmu.ifi.dbs.elki.distance.distancefunction.subspace.SubspaceManhattanDistanceFunction
\ No newline at end of file +de.lmu.ifi.dbs.elki.distance.distancefunction.subspace.SubspaceManhattanDistanceFunction diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.evaluation.Evaluator b/src/META-INF/elki/de.lmu.ifi.dbs.elki.evaluation.Evaluator index f2813425..1bacadda 100644 --- a/src/META-INF/elki/de.lmu.ifi.dbs.elki.evaluation.Evaluator +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.evaluation.Evaluator @@ -1,8 +1,15 @@ +de.lmu.ifi.dbs.elki.evaluation.AutomaticEvaluation +de.lmu.ifi.dbs.elki.evaluation.NoAutomaticEvaluation de.lmu.ifi.dbs.elki.evaluation.clustering.EvaluateClustering de.lmu.ifi.dbs.elki.evaluation.clustering.pairsegments.ClusterPairSegmentAnalysis -de.lmu.ifi.dbs.elki.evaluation.roc.ComputeROCCurve de.lmu.ifi.dbs.elki.evaluation.histogram.ComputeOutlierHistogram de.lmu.ifi.dbs.elki.evaluation.index.IndexPurity de.lmu.ifi.dbs.elki.evaluation.index.IndexStatistics +de.lmu.ifi.dbs.elki.evaluation.outlier.OutlierROCCurve +de.lmu.ifi.dbs.elki.evaluation.outlier.OutlierThresholdClustering +de.lmu.ifi.dbs.elki.evaluation.outlier.OutlierPrecisionAtKCurve +de.lmu.ifi.dbs.elki.evaluation.outlier.OutlierPrecisionRecallCurve +de.lmu.ifi.dbs.elki.evaluation.outlier.OutlierSmROCCurve de.lmu.ifi.dbs.elki.evaluation.outlier.JudgeOutlierScores de.lmu.ifi.dbs.elki.evaluation.similaritymatrix.ComputeSimilarityMatrixImage +# de.lmu.ifi.dbs.elki.evaluation.roc.ComputeROCCurve
\ No newline at end of file diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.index.IndexFactory b/src/META-INF/elki/de.lmu.ifi.dbs.elki.index.IndexFactory index 6c0fff30..50052cb3 100644 --- a/src/META-INF/elki/de.lmu.ifi.dbs.elki.index.IndexFactory +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.index.IndexFactory @@ -8,6 +8,7 @@ de.lmu.ifi.dbs.elki.index.tree.metrical.mtreevariants.mktrees.mkapp.MkAppTreeFac de.lmu.ifi.dbs.elki.index.tree.metrical.mtreevariants.mktrees.mkmax.MkMaxTreeFactory de.lmu.ifi.dbs.elki.index.tree.metrical.mtreevariants.mktrees.mktab.MkTabTreeFactory de.lmu.ifi.dbs.elki.index.vafile.VAFile$Factory +de.lmu.ifi.dbs.elki.index.vafile.PartialVAFile$Factory de.lmu.ifi.dbs.elki.index.preprocessed.knn.MaterializeKNNAndRKNNPreprocessor$Factory de.lmu.ifi.dbs.elki.index.preprocessed.knn.MaterializeKNNPreprocessor$Factory de.lmu.ifi.dbs.elki.index.preprocessed.knn.KNNJoinMaterializeKNNPreprocessor$Factory diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.index.KNNIndex b/src/META-INF/elki/de.lmu.ifi.dbs.elki.index.KNNIndex index cc1e2e56..c3a3fc1b 100644 --- a/src/META-INF/elki/de.lmu.ifi.dbs.elki.index.KNNIndex +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.index.KNNIndex @@ -14,4 +14,5 @@ de.lmu.ifi.dbs.elki.index.preprocessed.knn.MetricalIndexApproximationMaterialize de.lmu.ifi.dbs.elki.index.preprocessed.knn.PartitionApproximationMaterializeKNNPreprocessor de.lmu.ifi.dbs.elki.index.preprocessed.knn.RandomSampleKNNPreprocessor de.lmu.ifi.dbs.elki.index.preprocessed.knn.SpatialApproximationMaterializeKNNPreprocessor -de.lmu.ifi.dbs.elki.index.vafile.VAFile
\ No newline at end of file +de.lmu.ifi.dbs.elki.index.vafile.VAFile +de.lmu.ifi.dbs.elki.index.vafile.PartialVAFile
\ No newline at end of file diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.index.RangeIndex b/src/META-INF/elki/de.lmu.ifi.dbs.elki.index.RangeIndex index d39d1bbb..ce559a88 100644 --- a/src/META-INF/elki/de.lmu.ifi.dbs.elki.index.RangeIndex +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.index.RangeIndex @@ -7,4 +7,5 @@ de.lmu.ifi.dbs.elki.index.tree.metrical.mtreevariants.mktrees.mkapp.MkAppTreeInd # de.lmu.ifi.dbs.elki.index.tree.metrical.mtreevariants.mktrees.mkcop.MkCoPTreeIndex de.lmu.ifi.dbs.elki.index.tree.metrical.mtreevariants.mktrees.mkmax.MkMaxTreeIndex de.lmu.ifi.dbs.elki.index.tree.metrical.mtreevariants.mktrees.mktab.MkTabTreeIndex -de.lmu.ifi.dbs.elki.index.vafile.VAFile
\ No newline at end of file +de.lmu.ifi.dbs.elki.index.vafile.VAFile +de.lmu.ifi.dbs.elki.index.vafile.PartialVAFile
\ No newline at end of file diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.math.linearalgebra.pca.EigenPairFilter b/src/META-INF/elki/de.lmu.ifi.dbs.elki.math.linearalgebra.pca.EigenPairFilter index 67050b39..b1e7c8d5 100644 --- a/src/META-INF/elki/de.lmu.ifi.dbs.elki.math.linearalgebra.pca.EigenPairFilter +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.math.linearalgebra.pca.EigenPairFilter @@ -7,3 +7,4 @@ de.lmu.ifi.dbs.elki.math.linearalgebra.pca.WeakEigenPairFilter de.lmu.ifi.dbs.elki.math.linearalgebra.pca.RelativeEigenPairFilter de.lmu.ifi.dbs.elki.math.linearalgebra.pca.SignificantEigenPairFilter de.lmu.ifi.dbs.elki.math.linearalgebra.pca.CompositeEigenPairFilter +de.lmu.ifi.dbs.elki.math.linearalgebra.pca.DropEigenPairFilter
\ No newline at end of file diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.math.linearalgebra.pca.PCARunner b/src/META-INF/elki/de.lmu.ifi.dbs.elki.math.linearalgebra.pca.PCARunner new file mode 100644 index 00000000..1f53a991 --- /dev/null +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.math.linearalgebra.pca.PCARunner @@ -0,0 +1,3 @@ +de.lmu.ifi.dbs.elki.math.linearalgebra.pca.PCARunner +# de.lmu.ifi.dbs.elki.math.linearalgebra.pca.PCAFilteredRunner +de.lmu.ifi.dbs.elki.math.linearalgebra.pca.PCAFilteredAutotuningRunner
\ No newline at end of file diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.math.statistics.tests.GoodnessOfFitTest b/src/META-INF/elki/de.lmu.ifi.dbs.elki.math.statistics.tests.GoodnessOfFitTest new file mode 100644 index 00000000..01391cbd --- /dev/null +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.math.statistics.tests.GoodnessOfFitTest @@ -0,0 +1,2 @@ +de.lmu.ifi.dbs.elki.math.statistics.tests.KolmogorovSmirnovTest +de.lmu.ifi.dbs.elki.math.statistics.tests.WelchTTest
\ No newline at end of file diff --git a/src/META-INF/elki/de.lmu.ifi.dbs.elki.visualization.visualizers.VisFactory b/src/META-INF/elki/de.lmu.ifi.dbs.elki.visualization.visualizers.VisFactory index 296024aa..06c0cc41 100644 --- a/src/META-INF/elki/de.lmu.ifi.dbs.elki.visualization.visualizers.VisFactory +++ b/src/META-INF/elki/de.lmu.ifi.dbs.elki.visualization.visualizers.VisFactory @@ -33,7 +33,7 @@ de.lmu.ifi.dbs.elki.visualization.visualizers.parallel.selection.SelectionToolLi de.lmu.ifi.dbs.elki.visualization.visualizers.pairsegments.CircleSegmentsVisualizer$Factory de.lmu.ifi.dbs.elki.visualization.visualizers.visunproj.HistogramVisFactory de.lmu.ifi.dbs.elki.visualization.visualizers.visunproj.ClusterEvaluationVisFactory -de.lmu.ifi.dbs.elki.visualization.visualizers.visunproj.CurveVisFactory +de.lmu.ifi.dbs.elki.visualization.visualizers.visunproj.XYCurveVisFactory de.lmu.ifi.dbs.elki.visualization.visualizers.visunproj.LabelVisFactory de.lmu.ifi.dbs.elki.visualization.visualizers.visunproj.PixmapVisualizer$Factory de.lmu.ifi.dbs.elki.visualization.visualizers.visunproj.SimilarityMatrixVisualizer$Factory diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/APRIORI.java b/src/de/lmu/ifi/dbs/elki/algorithm/APRIORI.java index fc346cd9..65339257 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/APRIORI.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/APRIORI.java @@ -34,7 +34,7 @@ import de.lmu.ifi.dbs.elki.data.BitVector; import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.logging.Logging; import de.lmu.ifi.dbs.elki.result.AprioriResult; @@ -127,7 +127,7 @@ public class APRIORI extends AbstractAlgorithm<AprioriResult> { * @param relation the Relation to process * @return the AprioriResult learned by this APRIORI */ - public AprioriResult run(Database database, Relation<BitVector> relation) throws IllegalStateException { + public AprioriResult run(Database database, Relation<BitVector> relation) { Map<BitSet, Integer> support = new Hashtable<BitSet, Integer>(); List<BitSet> solution = new ArrayList<BitSet>(); final int size = relation.size(); @@ -264,8 +264,8 @@ public class APRIORI extends AbstractAlgorithm<AprioriResult> { support.put(bitSet, 0); } } - for(DBID id : database.iterDBIDs()) { - BitVector bv = database.get(id); + for(DBIDIter iditer = database.iterDBIDs(); iditer.valid(); iditer.advance()) { + BitVector bv = database.get(iditer); for(BitSet bitSet : candidates) { if(bv.contains(bitSet)) { support.put(bitSet, support.get(bitSet) + 1); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/Algorithm.java b/src/de/lmu/ifi/dbs/elki/algorithm/Algorithm.java index 7c6f0dc5..ae221ca7 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/Algorithm.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/Algorithm.java @@ -53,11 +53,8 @@ public interface Algorithm extends Parameterizable { * * @param database the database to run the algorithm on * @return the Result computed by this algorithm - * @throws IllegalStateException if the algorithm has not been initialized - * properly (e.g. the setParameters(String[]) method has been failed - * to be called). */ - Result run(Database database) throws IllegalStateException; + Result run(Database database); /** * Get the input type restriction used for negotiating the data query. diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/DependencyDerivator.java b/src/de/lmu/ifi/dbs/elki/algorithm/DependencyDerivator.java index 0ecfb228..e0eabf5c 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/DependencyDerivator.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/DependencyDerivator.java @@ -149,7 +149,7 @@ public class DependencyDerivator<V extends NumberVector<V, ?>, D extends Distanc * @return the CorrelationAnalysisSolution computed by this * DependencyDerivator */ - public CorrelationAnalysisSolution<V> run(Database database, Relation<V> relation) throws IllegalStateException { + public CorrelationAnalysisSolution<V> run(Database database, Relation<V> relation) { if(logger.isVerbose()) { logger.verbose("retrieving database objects..."); } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/DummyAlgorithm.java b/src/de/lmu/ifi/dbs/elki/algorithm/DummyAlgorithm.java index 168c69f1..64188502 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/DummyAlgorithm.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/DummyAlgorithm.java @@ -27,7 +27,7 @@ import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery; import de.lmu.ifi.dbs.elki.database.relation.Relation; @@ -80,11 +80,11 @@ public class DummyAlgorithm<O extends NumberVector<?, ?>> extends AbstractAlgori DistanceQuery<O, DoubleDistance> distQuery = database.getDistanceQuery(relation, EuclideanDistanceFunction.STATIC); KNNQuery<O, DoubleDistance> knnQuery = database.getKNNQuery(distQuery, 10); - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { // Get the actual object from the database (but discard the result) - relation.get(id); + relation.get(iditer); // run a 10NN query for each point (but discard the result) - knnQuery.getKNNForDBID(id, 10); + knnQuery.getKNNForDBID(iditer, 10); } return null; } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/KNNDistanceOrder.java b/src/de/lmu/ifi/dbs/elki/algorithm/KNNDistanceOrder.java index ac1820f9..137ffadf 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/KNNDistanceOrder.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/KNNDistanceOrder.java @@ -31,7 +31,7 @@ import java.util.Random; import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery; import de.lmu.ifi.dbs.elki.database.query.knn.KNNResult; @@ -115,9 +115,9 @@ public class KNNDistanceOrder<O, D extends Distance<D>> extends AbstractDistance final Random random = new Random(); List<D> knnDistances = new ArrayList<D>(relation.size()); - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { if(random.nextDouble() < percentage) { - final KNNResult<D> neighbors = knnQuery.getKNNForDBID(id, k); + final KNNResult<D> neighbors = knnQuery.getKNNForDBID(iditer, k); knnDistances.add(neighbors.getKNNDistance()); } } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/KNNJoin.java b/src/de/lmu/ifi/dbs/elki/algorithm/KNNJoin.java index 3cbfe143..3eb789c7 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/KNNJoin.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/KNNJoin.java @@ -115,12 +115,12 @@ public class KNNJoin<V extends NumberVector<V, ?>, D extends Distance<D>, N exte /** * Joins in the given spatial database to each object its k-nearest neighbors. * - * @throws IllegalStateException if not suitable {@link SpatialIndexTree} was - * found or the specified distance function is not an instance of - * {@link SpatialPrimitiveDistanceFunction}. + * @param database Database to process + * @param relation Relation to process + * @return result */ @SuppressWarnings("unchecked") - public WritableDataStore<KNNList<D>> run(Database database, Relation<V> relation) throws IllegalStateException { + public WritableDataStore<KNNList<D>> run(Database database, Relation<V> relation) { if(!(getDistanceFunction() instanceof SpatialPrimitiveDistanceFunction)) { throw new IllegalStateException("Distance Function must be an instance of " + SpatialPrimitiveDistanceFunction.class.getName()); } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/MaterializeDistances.java b/src/de/lmu/ifi/dbs/elki/algorithm/MaterializeDistances.java index 89d2d3e0..b09f7ac2 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/MaterializeDistances.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/MaterializeDistances.java @@ -30,6 +30,7 @@ import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction; @@ -77,14 +78,14 @@ public class MaterializeDistances<O, D extends NumberDistance<D, ?>> extends Abs Collection<CTriple<DBID, DBID, Double>> r = new ArrayList<CTriple<DBID, DBID, Double>>(size * (size + 1) / 2); - for(DBID id1 : relation.iterDBIDs()) { - for(DBID id2 : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + for(DBIDIter iditer2 = relation.iterDBIDs(); iditer2.valid(); iditer2.advance()) { // skip inverted pairs - if(id2.compareTo(id1) > 0) { + if(iditer2.compareDBID(iditer) > 0) { continue; } - double d = distFunc.distance(id1, id2).doubleValue(); - r.add(new CTriple<DBID, DBID, Double>(id1, id2, d)); + double d = distFunc.distance(iditer, iditer2).doubleValue(); + r.add(new CTriple<DBID, DBID, Double>(iditer.getDBID(), iditer2.getDBID(), d)); } } return new CollectionResult<CTriple<DBID, DBID, Double>>("Distance Matrix", "distance-matrix", r); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/NullAlgorithm.java b/src/de/lmu/ifi/dbs/elki/algorithm/NullAlgorithm.java index a879c6b2..490d79fb 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/NullAlgorithm.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/NullAlgorithm.java @@ -53,7 +53,7 @@ public class NullAlgorithm extends AbstractAlgorithm<Result> { } @Override - public Result run(Database database) throws IllegalStateException { + public Result run(Database database) { return null; } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/AbstractProjectedClustering.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/AbstractProjectedClustering.java index ea441655..670a3f0f 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/AbstractProjectedClustering.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/AbstractProjectedClustering.java @@ -27,7 +27,6 @@ import de.lmu.ifi.dbs.elki.algorithm.AbstractAlgorithm; import de.lmu.ifi.dbs.elki.algorithm.clustering.subspace.PROCLUS; import de.lmu.ifi.dbs.elki.data.Clustering; import de.lmu.ifi.dbs.elki.data.NumberVector; -import de.lmu.ifi.dbs.elki.data.model.Model; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.QueryUtil; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; @@ -49,7 +48,7 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.IntParameter; * @param <R> the result we return * @param <V> the type of FeatureVector handled by this Algorithm */ -public abstract class AbstractProjectedClustering<R extends Clustering<Model>, V extends NumberVector<V, ?>> extends AbstractAlgorithm<R> implements ClusteringAlgorithm<R> { +public abstract class AbstractProjectedClustering<R extends Clustering<?>, V extends NumberVector<V, ?>> extends AbstractAlgorithm<R> implements ClusteringAlgorithm<R> { /** * Parameter to specify the number of clusters to find, must be an integer * greater than 0. diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/AbstractProjectedDBSCAN.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/AbstractProjectedDBSCAN.java index 108ba0ed..250cc70b 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/AbstractProjectedDBSCAN.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/AbstractProjectedDBSCAN.java @@ -37,6 +37,7 @@ import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; @@ -166,7 +167,14 @@ public abstract class AbstractProjectedDBSCAN<R extends Clustering<Model>, V ext this.lambda = lambda; } - public Clustering<Model> run(Database database, Relation<V> relation) throws IllegalStateException { + /** + * Run the algorithm + * + * @param database Database to process + * @param relation Relation to process + * @return Clustering result + */ + public Clustering<Model> run(Database database, Relation<V> relation) { FiniteProgress objprog = getLogger().isVerbose() ? new FiniteProgress("Processing objects", relation.size(), getLogger()) : null; IndefiniteProgress clusprog = getLogger().isVerbose() ? new IndefiniteProgress("Number of clusters", getLogger()) : null; resultList = new ArrayList<ModifiableDBIDs>(); @@ -177,9 +185,9 @@ public abstract class AbstractProjectedDBSCAN<R extends Clustering<Model>, V ext RangeQuery<V, DoubleDistance> rangeQuery = database.getRangeQuery(distFunc); if(relation.size() >= minpts) { - for(DBID id : relation.iterDBIDs()) { - if(!processedIDs.contains(id)) { - expandCluster(distFunc, rangeQuery, id, objprog, clusprog); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + if(!processedIDs.contains(iditer)) { + expandCluster(distFunc, rangeQuery, iditer.getDBID(), objprog, clusprog); if(processedIDs.size() == relation.size() && noise.size() == 0) { break; } @@ -191,8 +199,8 @@ public abstract class AbstractProjectedDBSCAN<R extends Clustering<Model>, V ext } } else { - for(DBID id : relation.iterDBIDs()) { - noise.add(id); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + noise.add(iditer); if(objprog != null && clusprog != null) { objprog.setProcessed(processedIDs.size(), getLogger()); clusprog.setProcessed(resultList.size(), getLogger()); @@ -284,28 +292,26 @@ public abstract class AbstractProjectedDBSCAN<R extends Clustering<Model>, V ext // try to expand the cluster ModifiableDBIDs currentCluster = DBIDUtil.newArray(); for(DistanceResultPair<DoubleDistance> seed : seeds) { - DBID nextID = seed.getDBID(); - - Integer nextID_corrDim = distFunc.getIndex().getLocalProjection(nextID).getCorrelationDimension(); + int nextID_corrDim = distFunc.getIndex().getLocalProjection(seed).getCorrelationDimension(); // nextID is not reachable from start object if(nextID_corrDim > lambda) { continue; } - if(!processedIDs.contains(nextID)) { - currentCluster.add(nextID); - processedIDs.add(nextID); + if(!processedIDs.contains(seed)) { + currentCluster.add(seed); + processedIDs.add(seed); } - else if(noise.contains(nextID)) { - currentCluster.add(nextID); - noise.remove(nextID); + else if(noise.contains(seed)) { + currentCluster.add(seed); + noise.remove(seed); } } seeds.remove(0); while(seeds.size() > 0) { - DBID q = seeds.remove(0).getDBID(); - Integer corrDim_q = distFunc.getIndex().getLocalProjection(q).getCorrelationDimension(); + DistanceResultPair<DoubleDistance> q = seeds.remove(0); + int corrDim_q = distFunc.getIndex().getLocalProjection(q).getCorrelationDimension(); // q forms no lambda-dim hyperplane if(corrDim_q > lambda) { continue; @@ -314,22 +320,22 @@ public abstract class AbstractProjectedDBSCAN<R extends Clustering<Model>, V ext List<DistanceResultPair<DoubleDistance>> reachables = rangeQuery.getRangeForDBID(q, epsilon); if(reachables.size() > minpts) { for(DistanceResultPair<DoubleDistance> r : reachables) { - Integer corrDim_r = distFunc.getIndex().getLocalProjection(r.getDBID()).getCorrelationDimension(); + int corrDim_r = distFunc.getIndex().getLocalProjection(r).getCorrelationDimension(); // r is not reachable from q if(corrDim_r > lambda) { continue; } - boolean inNoise = noise.contains(r.getDBID()); - boolean unclassified = !processedIDs.contains(r.getDBID()); + boolean inNoise = noise.contains(r); + boolean unclassified = !processedIDs.contains(r); if(inNoise || unclassified) { if(unclassified) { seeds.add(r); } - currentCluster.add(r.getDBID()); - processedIDs.add(r.getDBID()); + currentCluster.add(r); + processedIDs.add(r); if(inNoise) { - noise.remove(r.getDBID()); + noise.remove(r); } if(objprog != null && clusprog != null) { objprog.setProcessed(processedIDs.size(), getLogger()); @@ -349,9 +355,7 @@ public abstract class AbstractProjectedDBSCAN<R extends Clustering<Model>, V ext resultList.add(currentCluster); } else { - for(DBID id : currentCluster) { - noise.add(id); - } + noise.addDBIDs(currentCluster); noise.add(startObjectID); processedIDs.add(startObjectID); } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/ClusteringAlgorithm.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/ClusteringAlgorithm.java index 5ec59777..8f637460 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/ClusteringAlgorithm.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/ClusteringAlgorithm.java @@ -47,5 +47,5 @@ import de.lmu.ifi.dbs.elki.database.Database; */ public interface ClusteringAlgorithm<C extends Clustering<? extends Model>> extends Algorithm { @Override - C run(Database database) throws IllegalStateException; + C run(Database database); }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/DBSCAN.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/DBSCAN.java index b59af555..6bafa9e9 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/DBSCAN.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/DBSCAN.java @@ -35,6 +35,7 @@ import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.QueryUtil; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; @@ -141,9 +142,9 @@ public class DBSCAN<O, D extends Distance<D>> extends AbstractDistanceBasedAlgor noise = DBIDUtil.newHashSet(); processedIDs = DBIDUtil.newHashSet(size); if(size >= minpts) { - for(DBID id : relation.iterDBIDs()) { - if(!processedIDs.contains(id)) { - expandCluster(relation, rangeQuery, id, objprog, clusprog); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + if(!processedIDs.contains(iditer)) { + expandCluster(relation, rangeQuery, iditer.getDBID(), objprog, clusprog); } if(objprog != null && clusprog != null) { objprog.setProcessed(processedIDs.size(), logger); @@ -155,8 +156,8 @@ public class DBSCAN<O, D extends Distance<D>> extends AbstractDistanceBasedAlgor } } else { - for(DBID id : relation.iterDBIDs()) { - noise.add(id); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + noise.add(iditer); if(objprog != null && clusprog != null) { objprog.setProcessed(noise.size(), logger); clusprog.setProcessed(resultList.size(), logger); @@ -210,35 +211,33 @@ public class DBSCAN<O, D extends Distance<D>> extends AbstractDistanceBasedAlgor // try to expand the cluster ModifiableDBIDs currentCluster = DBIDUtil.newArray(); for(DistanceResultPair<D> seed : seeds) { - DBID nextID = seed.getDBID(); - if(!processedIDs.contains(nextID)) { - currentCluster.add(nextID); - processedIDs.add(nextID); + if(!processedIDs.contains(seed)) { + currentCluster.add(seed); + processedIDs.add(seed); } - else if(noise.contains(nextID)) { - currentCluster.add(nextID); - noise.remove(nextID); + else if(noise.contains(seed)) { + currentCluster.add(seed); + noise.remove(seed); } } seeds.remove(0); while(seeds.size() > 0) { - DBID o = seeds.remove(0).getDBID(); + DistanceResultPair<D> o = seeds.remove(0); List<DistanceResultPair<D>> neighborhood = rangeQuery.getRangeForDBID(o, epsilon); if(neighborhood.size() >= minpts) { for(DistanceResultPair<D> neighbor : neighborhood) { - DBID p = neighbor.getDBID(); - boolean inNoise = noise.contains(p); - boolean unclassified = !processedIDs.contains(p); + boolean inNoise = noise.contains(neighbor); + boolean unclassified = !processedIDs.contains(neighbor); if(inNoise || unclassified) { if(unclassified) { seeds.add(neighbor); } - currentCluster.add(p); - processedIDs.add(p); + currentCluster.add(neighbor); + processedIDs.add(neighbor); if(inNoise) { - noise.remove(p); + noise.remove(neighbor); } } } @@ -258,9 +257,7 @@ public class DBSCAN<O, D extends Distance<D>> extends AbstractDistanceBasedAlgor resultList.add(currentCluster); } else { - for(DBID id : currentCluster) { - noise.add(id); - } + noise.addDBIDs(currentCluster); noise.add(startObjectID); processedIDs.add(startObjectID); } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/DeLiClu.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/DeLiClu.java index f1e6c945..a0780e3d 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/DeLiClu.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/DeLiClu.java @@ -24,7 +24,6 @@ package de.lmu.ifi.dbs.elki.algorithm.clustering; */ import java.util.Collection; -import java.util.Iterator; import java.util.List; import java.util.Set; @@ -36,6 +35,7 @@ import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.datastore.DataStore; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.DistanceUtil; import de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction; @@ -201,11 +201,11 @@ public class DeLiClu<NV extends NumberVector<NV, ?>, D extends Distance<D>> exte * @return the id of the start object for the run method */ private DBID getStartObject(Relation<NV> relation) { - Iterator<DBID> it = relation.iterDBIDs(); - if(!it.hasNext()) { + DBIDIter it = relation.iterDBIDs(); + if(!it.valid()) { return null; } - return it.next(); + return it.getDBID(); } /** diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/EM.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/EM.java index a70a3f6f..63ebbabb 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/EM.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/EM.java @@ -39,7 +39,8 @@ import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.relation.Relation; @@ -170,28 +171,31 @@ public class EM<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Clusteri if(logger.isVerbose()) { logger.verbose("initializing " + k + " models"); } - List<Vector> means = initializer.chooseInitialMeans(relation, k, EuclideanDistanceFunction.STATIC); + List<Vector> means = new ArrayList<Vector>(); + for(NumberVector<?, ?> nv : initializer.chooseInitialMeans(relation, k, EuclideanDistanceFunction.STATIC)) { + means.add(nv.getColumnVector()); + } List<Matrix> covarianceMatrices = new ArrayList<Matrix>(k); - List<Double> normDistrFactor = new ArrayList<Double>(k); + double[] normDistrFactor = new double[k]; List<Matrix> invCovMatr = new ArrayList<Matrix>(k); - List<Double> clusterWeights = new ArrayList<Double>(k); + double[] clusterWeights = new double[k]; probClusterIGivenX = DataStoreUtil.makeStorage(relation.getDBIDs(), DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_SORTED, double[].class); final int dimensionality = means.get(0).getDimensionality(); for(int i = 0; i < k; i++) { Matrix m = Matrix.identity(dimensionality, dimensionality); covarianceMatrices.add(m); - normDistrFactor.add(1.0 / Math.sqrt(Math.pow(MathUtil.TWOPI, dimensionality) * m.det())); + normDistrFactor[i] = 1.0 / Math.sqrt(Math.pow(MathUtil.TWOPI, dimensionality) * m.det()); invCovMatr.add(m.inverse()); - clusterWeights.add(1.0 / k); + clusterWeights[i] = 1.0 / k; if(logger.isDebuggingFinest()) { StringBuffer msg = new StringBuffer(); msg.append(" model ").append(i).append(":\n"); msg.append(" mean: ").append(means.get(i)).append("\n"); msg.append(" m:\n").append(FormatUtil.format(m, " ")).append("\n"); msg.append(" m.det(): ").append(m.det()).append("\n"); - msg.append(" cluster weight: ").append(clusterWeights.get(i)).append("\n"); - msg.append(" normDistFact: ").append(normDistrFactor.get(i)).append("\n"); + msg.append(" cluster weight: ").append(clusterWeights[i]).append("\n"); + msg.append(" normDistFact: ").append(normDistrFactor[i]).append("\n"); logger.debugFine(msg.toString()); } } @@ -216,31 +220,31 @@ public class EM<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Clusteri double[] sumOfClusterProbabilities = new double[k]; for(int i = 0; i < k; i++) { - clusterWeights.set(i, 0.0); + clusterWeights[i] = 0.0; meanSums.add(new Vector(dimensionality)); covarianceMatrices.set(i, Matrix.zeroMatrix(dimensionality)); } // weights and means - for(DBID id : relation.iterDBIDs()) { - double[] clusterProbabilities = probClusterIGivenX.get(id); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + double[] clusterProbabilities = probClusterIGivenX.get(iditer); for(int i = 0; i < k; i++) { sumOfClusterProbabilities[i] += clusterProbabilities[i]; - Vector summand = relation.get(id).getColumnVector().timesEquals(clusterProbabilities[i]); + Vector summand = relation.get(iditer).getColumnVector().timesEquals(clusterProbabilities[i]); meanSums.get(i).plusEquals(summand); } } final int n = relation.size(); for(int i = 0; i < k; i++) { - clusterWeights.set(i, sumOfClusterProbabilities[i] / n); + clusterWeights[i] = sumOfClusterProbabilities[i] / n; Vector newMean = meanSums.get(i).timesEquals(1 / sumOfClusterProbabilities[i]); means.set(i, newMean); } // covariance matrices - for(DBID id : relation.iterDBIDs()) { - double[] clusterProbabilities = probClusterIGivenX.get(id); - Vector instance = relation.get(id).getColumnVector(); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + double[] clusterProbabilities = probClusterIGivenX.get(iditer); + Vector instance = relation.get(iditer).getColumnVector(); for(int i = 0; i < k; i++) { Vector difference = instance.minus(means.get(i)); covarianceMatrices.get(i).plusEquals(difference.timesTranspose(difference).timesEquals(clusterProbabilities[i])); @@ -250,7 +254,7 @@ public class EM<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Clusteri covarianceMatrices.set(i, covarianceMatrices.get(i).times(1 / sumOfClusterProbabilities[i]).cheatToAvoidSingularity(SINGULARITY_CHEAT)); } for(int i = 0; i < k; i++) { - normDistrFactor.set(i, 1.0 / Math.sqrt(Math.pow(MathUtil.TWOPI, dimensionality) * covarianceMatrices.get(i).det())); + normDistrFactor[i] = 1.0 / Math.sqrt(Math.pow(MathUtil.TWOPI, dimensionality) * covarianceMatrices.get(i).det()); invCovMatr.set(i, covarianceMatrices.get(i).inverse()); } // reassign probabilities @@ -269,8 +273,8 @@ public class EM<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Clusteri } // provide a hard clustering - for(DBID id : relation.iterDBIDs()) { - double[] clusterProbabilities = probClusterIGivenX.get(id); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + double[] clusterProbabilities = probClusterIGivenX.get(iditer); int maxIndex = 0; double currentMax = 0.0; for(int i = 0; i < k; i++) { @@ -279,7 +283,7 @@ public class EM<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Clusteri currentMax = clusterProbabilities[i]; } } - hardClusters.get(maxIndex).add(id); + hardClusters.get(maxIndex).add(iditer); } final V factory = DatabaseUtil.assumeVectorField(relation).getFactory(); Clustering<EMModel<V>> result = new Clustering<EMModel<V>>("EM Clustering", "em-clustering"); @@ -309,25 +313,25 @@ public class EM<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Clusteri * @param clusterWeights the weights of the current clusters * @return the expectation value of the current mixture of distributions */ - protected double assignProbabilitiesToInstances(Relation<V> database, List<Double> normDistrFactor, List<Vector> means, List<Matrix> invCovMatr, List<Double> clusterWeights, WritableDataStore<double[]> probClusterIGivenX) { + protected double assignProbabilitiesToInstances(Relation<V> database, double[] normDistrFactor, List<Vector> means, List<Matrix> invCovMatr, double[] clusterWeights, WritableDataStore<double[]> probClusterIGivenX) { double emSum = 0.0; - for(DBID id : database.iterDBIDs()) { - Vector x = database.get(id).getColumnVector(); - List<Double> probabilities = new ArrayList<Double>(k); + for(DBIDIter iditer = database.iterDBIDs(); iditer.valid(); iditer.advance()) { + Vector x = database.get(iditer).getColumnVector(); + double[] probabilities = new double[k]; for(int i = 0; i < k; i++) { Vector difference = x.minus(means.get(i)); double rowTimesCovTimesCol = difference.transposeTimesTimes(invCovMatr.get(i), difference); double power = rowTimesCovTimesCol / 2.0; - double prob = normDistrFactor.get(i) * Math.exp(-power); + double prob = normDistrFactor[i] * Math.exp(-power); if(logger.isDebuggingFinest()) { logger.debugFinest(" difference vector= ( " + difference.toString() + " )\n" + " difference:\n" + FormatUtil.format(difference, " ") + "\n" + " rowTimesCovTimesCol:\n" + rowTimesCovTimesCol + "\n" + " power= " + power + "\n" + " prob=" + prob + "\n" + " inv cov matrix: \n" + FormatUtil.format(invCovMatr.get(i), " ")); } - probabilities.add(prob); + probabilities[i] = prob; } double priorProbability = 0.0; for(int i = 0; i < k; i++) { - priorProbability += probabilities.get(i) * clusterWeights.get(i); + priorProbability += probabilities[i] * clusterWeights[i]; } double logP = Math.max(Math.log(priorProbability), MIN_LOGLIKELIHOOD); if(!Double.isNaN(logP)) { @@ -337,16 +341,16 @@ public class EM<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Clusteri double[] clusterProbabilities = new double[k]; for(int i = 0; i < k; i++) { assert (priorProbability >= 0.0); - assert (clusterWeights.get(i) >= 0.0); + assert (clusterWeights[i] >= 0.0); // do not divide by zero! if(priorProbability == 0.0) { clusterProbabilities[i] = 0.0; } else { - clusterProbabilities[i] = probabilities.get(i) / priorProbability * clusterWeights.get(i); + clusterProbabilities[i] = probabilities[i] / priorProbability * clusterWeights[i]; } } - probClusterIGivenX.put(id, clusterProbabilities); + probClusterIGivenX.put(iditer, clusterProbabilities); } return emSum; @@ -358,7 +362,7 @@ public class EM<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Clusteri * @param index Point ID * @return Probabilities of given point */ - public double[] getProbClusterIGivenX(DBID index) { + public double[] getProbClusterIGivenX(DBIDRef index) { return probClusterIGivenX.get(index); } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/OPTICS.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/OPTICS.java index 2244b07b..04b57081 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/OPTICS.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/OPTICS.java @@ -31,6 +31,7 @@ import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.QueryUtil; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; @@ -142,22 +143,22 @@ public class OPTICS<O, D extends Distance<D>> extends AbstractDistanceBasedAlgor if(getDistanceFunction() instanceof PrimitiveDoubleDistanceFunction && DoubleDistance.class.isInstance(epsilon)) { // Optimized codepath for double-based distances. Avoids Java // boxing/unboxing. - for(DBID id : relation.iterDBIDs()) { - if(!processedIDs.contains(id)) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + if(!processedIDs.contains(iditer)) { // We need to do some ugly casts to be able to run the optimized version, unfortunately. @SuppressWarnings("unchecked") final ClusterOrderResult<DoubleDistance> doubleClusterOrder = ClusterOrderResult.class.cast(clusterOrder); @SuppressWarnings("unchecked") final RangeQuery<O, DoubleDistance> doubleRangeQuery = RangeQuery.class.cast(rangeQuery); final DoubleDistance depsilon = DoubleDistance.class.cast(epsilon); - expandClusterOrderDouble(doubleClusterOrder, database, doubleRangeQuery, id, depsilon, progress); + expandClusterOrderDouble(doubleClusterOrder, database, doubleRangeQuery, iditer.getDBID(), depsilon, progress); } } } else { - for(DBID id : relation.iterDBIDs()) { - if(!processedIDs.contains(id)) { - expandClusterOrder(clusterOrder, database, rangeQuery, id, epsilon, progress); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + if(!processedIDs.contains(iditer)) { + expandClusterOrder(clusterOrder, database, rangeQuery, iditer.getDBID(), epsilon, progress); } } } @@ -194,7 +195,7 @@ public class OPTICS<O, D extends Distance<D>> extends AbstractDistanceBasedAlgor D coreDistance = last.getDistance(); for(DistanceResultPair<D> neighbor : neighbors) { - if(processedIDs.contains(neighbor.getDBID())) { + if(processedIDs.contains(neighbor)) { continue; } D reachability = DistanceUtil.max(neighbor.getDistance(), coreDistance); @@ -234,7 +235,7 @@ public class OPTICS<O, D extends Distance<D>> extends AbstractDistanceBasedAlgor double coreDistance = ((DoubleDistanceResultPair) last).getDoubleDistance(); for(DistanceResultPair<DoubleDistance> neighbor : neighbors) { - if(processedIDs.contains(neighbor.getDBID())) { + if(processedIDs.contains(neighbor)) { continue; } double reachability = Math.max(((DoubleDistanceResultPair) neighbor).getDoubleDistance(), coreDistance); @@ -247,7 +248,7 @@ public class OPTICS<O, D extends Distance<D>> extends AbstractDistanceBasedAlgor double coreDistance = last.getDistance().doubleValue(); for(DistanceResultPair<DoubleDistance> neighbor : neighbors) { - if(processedIDs.contains(neighbor.getDBID())) { + if(processedIDs.contains(neighbor)) { continue; } double reachability = Math.max(neighbor.getDistance().doubleValue(), coreDistance); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/OPTICSTypeAlgorithm.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/OPTICSTypeAlgorithm.java index d6c5872a..3ead6f3e 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/OPTICSTypeAlgorithm.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/OPTICSTypeAlgorithm.java @@ -39,7 +39,7 @@ import de.lmu.ifi.dbs.elki.result.optics.ClusterOrderResult; */ public interface OPTICSTypeAlgorithm<D extends Distance<D>> extends Algorithm { @Override - ClusterOrderResult<D> run(Database database) throws IllegalStateException; + ClusterOrderResult<D> run(Database database); /** * Get the minpts value used. Needed for OPTICS Xi etc. diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/SLINK.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/SLINK.java index 45b12c43..2aa38bdd 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/SLINK.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/SLINK.java @@ -27,7 +27,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -49,6 +48,8 @@ import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore; import de.lmu.ifi.dbs.elki.database.datastore.WritableRecordStore; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDMIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; @@ -144,7 +145,8 @@ public class SLINK<O, D extends Distance<D>> extends AbstractDistanceBasedAlgori ModifiableDBIDs processedIDs = DBIDUtil.newArray(relation.size()); // apply the algorithm - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); step1(id); step2(id, processedIDs, distQuery, m); step3(id, processedIDs, m); @@ -200,7 +202,8 @@ public class SLINK<O, D extends Distance<D>> extends AbstractDistanceBasedAlgori * @param distFunc Distance function to use */ private void step2(DBID newID, DBIDs processedIDs, DistanceQuery<O, D> distFunc, WritableDataStore<D> m) { - for(DBID id : processedIDs) { + for(DBIDIter it = processedIDs.iter(); it.valid(); it.advance()) { + DBID id = it.getDBID(); // M(i) = dist(i, n+1) m.put(id, distFunc.distance(id, newID)); } @@ -215,7 +218,8 @@ public class SLINK<O, D extends Distance<D>> extends AbstractDistanceBasedAlgori */ private void step3(DBID newID, DBIDs processedIDs, WritableDataStore<D> m) { // for i = 1..n - for(DBID id : processedIDs) { + for(DBIDIter it = processedIDs.iter(); it.valid(); it.advance()) { + DBID id = it.getDBID(); D l_i = lambda.get(id); D m_i = m.get(id); DBID p_i = pi.get(id); @@ -247,7 +251,8 @@ public class SLINK<O, D extends Distance<D>> extends AbstractDistanceBasedAlgori */ private void step4(DBID newID, DBIDs processedIDs) { // for i = 1..n - for(DBID id : processedIDs) { + for(DBIDIter it = processedIDs.iter(); it.valid(); it.advance()) { + DBID id = it.getDBID(); D l_i = lambda.get(id); D lp_i = lambda.get(pi.get(id)); @@ -303,7 +308,8 @@ public class SLINK<O, D extends Distance<D>> extends AbstractDistanceBasedAlgori // extract the child clusters Map<DBID, ModifiableDBIDs> cluster_ids = new HashMap<DBID, ModifiableDBIDs>(); Map<DBID, D> cluster_distances = new HashMap<DBID, D>(); - for(DBID id : ids) { + for(DBIDIter it = ids.iter(); it.valid(); it.advance()) { + DBID id = it.getDBID(); DBID lastObjectInCluster = lastObjectInCluster(id, stopdist, pi, lambda); ModifiableDBIDs cluster = cluster_ids.get(lastObjectInCluster); if(cluster == null) { @@ -387,7 +393,7 @@ public class SLINK<O, D extends Distance<D>> extends AbstractDistanceBasedAlgori } // right child DBID rightID = pi.get(leftID); - if(leftID.equals(rightID)) { + if(leftID.sameDBID(rightID)) { break; } Cluster<DendrogramModel<D>> right = nodes.get(rightID); @@ -472,11 +478,12 @@ public class SLINK<O, D extends Distance<D>> extends AbstractDistanceBasedAlgori FiniteProgress progress = logger.isVerbose() ? new FiniteProgress("Extracting clusters", ids.size(), logger) : null; - for(DBID cur : order) { - DBID dest = pi.get(cur); - D l = lambda.get(cur); + for(DBIDIter it = order.iter(); it.valid(); it.advance()) { + DBID dest = pi.get(it); + D l = lambda.get(it); // logger.debugFine("DBID " + cur.toString() + " dist: " + l.toString()); if(stopdist != null && stopdist.compareTo(l) > 0) { + DBID cur = it.getDBID(); ModifiableDBIDs curset = cids.remove(cur); ModifiableDBIDs destset = cids.get(dest); if(destset == null) { @@ -511,13 +518,11 @@ public class SLINK<O, D extends Distance<D>> extends AbstractDistanceBasedAlgori Cluster<Model> cluster = new Cluster<Model>(cname, clusids, ClusterModel.CLUSTER, hier); // Collect child clusters and clean up the cluster ids, keeping only // "new" objects. - Iterator<DBID> iter = clusids.iterator(); - while(iter.hasNext()) { - DBID child = iter.next(); - Cluster<Model> chiclus = clusters.get(child); + for(DBIDMIter iter = clusids.iter(); iter.valid(); iter.advance()) { + Cluster<Model> chiclus = clusters.get(iter); if(chiclus != null) { hier.add(cluster, chiclus); - clusters.remove(child); + clusters.remove(iter); iter.remove(); } } @@ -545,7 +550,7 @@ public class SLINK<O, D extends Distance<D>> extends AbstractDistanceBasedAlgori cids.put(dest, destset); destset.add(dest); } - destset.add(cur); + destset.add(it); } } // Decrement counter diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/SNNClustering.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/SNNClustering.java index 7c3a13c9..ae612b2a 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/SNNClustering.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/SNNClustering.java @@ -37,6 +37,7 @@ import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.query.similarity.SimilarityQuery; @@ -154,9 +155,9 @@ public class SNNClustering<O> extends AbstractAlgorithm<Clustering<Model>> imple noise = DBIDUtil.newHashSet(); processedIDs = DBIDUtil.newHashSet(relation.size()); if(relation.size() >= minpts) { - for(DBID id : snnInstance.getRelation().iterDBIDs()) { + for(DBIDIter id = snnInstance.getRelation().iterDBIDs(); id.valid(); id.advance()) { if(!processedIDs.contains(id)) { - expandCluster(snnInstance, id, objprog, clusprog); + expandCluster(snnInstance, id.getDBID(), objprog, clusprog); if(processedIDs.size() == relation.size() && noise.size() == 0) { break; } @@ -168,7 +169,7 @@ public class SNNClustering<O> extends AbstractAlgorithm<Clustering<Model>> imple } } else { - for(DBID id : snnInstance.getRelation().iterDBIDs()) { + for(DBIDIter id = snnInstance.getRelation().iterDBIDs(); id.valid(); id.advance()) { noise.add(id); if(objprog != null && clusprog != null) { objprog.setProcessed(noise.size(), logger); @@ -202,9 +203,9 @@ public class SNNClustering<O> extends AbstractAlgorithm<Clustering<Model>> imple */ protected ArrayModifiableDBIDs findSNNNeighbors(SimilarityQuery<O, IntegerDistance> snnInstance, DBID queryObject) { ArrayModifiableDBIDs neighbors = DBIDUtil.newArray(); - for(DBID id : snnInstance.getRelation().iterDBIDs()) { - if(snnInstance.similarity(queryObject, id).compareTo(epsilon) >= 0) { - neighbors.add(id); + for(DBIDIter iditer = snnInstance.getRelation().iterDBIDs(); iditer.valid(); iditer.advance()) { + if(snnInstance.similarity(queryObject, iditer).compareTo(epsilon) >= 0) { + neighbors.add(iditer); } } return neighbors; @@ -237,7 +238,7 @@ public class SNNClustering<O> extends AbstractAlgorithm<Clustering<Model>> imple // try to expand the cluster ModifiableDBIDs currentCluster = DBIDUtil.newArray(); - for(DBID seed : seeds) { + for(DBIDIter seed = seeds.iter(); seed.valid(); seed.advance()) { if(!processedIDs.contains(seed)) { currentCluster.add(seed); processedIDs.add(seed); @@ -253,7 +254,8 @@ public class SNNClustering<O> extends AbstractAlgorithm<Clustering<Model>> imple ArrayModifiableDBIDs neighborhood = findSNNNeighbors(snnInstance, o); if(neighborhood.size() >= minpts) { - for(DBID p : neighborhood) { + for(DBIDIter iter = neighborhood.iter(); iter.valid(); iter.advance()) { + DBID p = iter.getDBID(); boolean inNoise = noise.contains(p); boolean unclassified = !processedIDs.contains(p); if(inNoise || unclassified) { @@ -283,9 +285,7 @@ public class SNNClustering<O> extends AbstractAlgorithm<Clustering<Model>> imple resultList.add(currentCluster); } else { - for(DBID id : currentCluster) { - noise.add(id); - } + noise.addDBIDs(currentCluster); noise.add(startObjectID); processedIDs.add(startObjectID); } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/CASH.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/CASH.java index b877415e..e4c6a123 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/CASH.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/CASH.java @@ -48,7 +48,7 @@ import de.lmu.ifi.dbs.elki.data.type.VectorFieldTypeInformation; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.ProxyDatabase; import de.lmu.ifi.dbs.elki.database.QueryUtil; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; @@ -86,9 +86,9 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.IntParameter; * Provides the CASH algorithm, an subspace clustering algorithm based on the * Hough transform. * - * <b>Note:</b> CASH requires explicitly setting the input parser other than default to - * {@link de.lmu.ifi.dbs.elki.datasource.parser.ParameterizationFunctionLabelParser}: - * (in the MiniGui, set option: dbc.parser ParameterizationFunctionLabelParser). + * <b>Note:</b> CASH requires explicitly setting the input vector type to + * {@link ParameterizationFunction}: + * (in the MiniGui, set option: parser.vector-type ParameterizationFunction). * * <p> * Reference: E. Achtert, C. Böhm, J. David, P. Kröger, A. Zimek: Robust @@ -503,9 +503,9 @@ public class CASH extends AbstractAlgorithm<Clustering<Model>> implements Cluste proxy.addRelation(prep); // Project - for(DBID id : ids) { - ParameterizationFunction f = project(basis, relation.get(id)); - prep.set(id, f); + for(DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + ParameterizationFunction f = project(basis, relation.get(iter)); + prep.set(iter, f); } if(logger.isDebugging()) { @@ -662,8 +662,8 @@ public class CASH extends AbstractAlgorithm<Clustering<Model>> implements Cluste double d_min = Double.POSITIVE_INFINITY; double d_max = Double.NEGATIVE_INFINITY; - for(DBID id : relation.iterDBIDs()) { - ParameterizationFunction f = relation.get(id); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + ParameterizationFunction f = relation.get(iditer); HyperBoundingBox minMax = f.determineAlphaMinMax(box); double f_min = f.function(SpatialUtil.getMin(minMax)); double f_max = f.function(SpatialUtil.getMax(minMax)); @@ -709,11 +709,11 @@ public class CASH extends AbstractAlgorithm<Clustering<Model>> implements Cluste ids.addDBIDs(interval.getIDs()); // Search for nearby vectors in original database - for(DBID id : relation.iterDBIDs()) { - DoubleVector v = new DoubleVector(relation.get(id).getColumnVector().getArrayRef()); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DoubleVector v = new DoubleVector(relation.get(iditer).getColumnVector().getArrayRef()); DoubleDistance d = df.distance(v, centroid); if(d.compareTo(eps) < 0) { - ids.add(id); + ids.add(iditer); } } @@ -735,15 +735,15 @@ public class CASH extends AbstractAlgorithm<Clustering<Model>> implements Cluste private Database buildDerivatorDB(Relation<ParameterizationFunction> relation, CASHInterval interval) throws UnableToComplyException { DBIDs ids = interval.getIDs(); ProxyDatabase proxy = new ProxyDatabase(ids); - int dim = relation.get(ids.iterator().next()).getDimensionality(); + int dim = DatabaseUtil.dimensionality(relation); SimpleTypeInformation<DoubleVector> type = new VectorFieldTypeInformation<DoubleVector>(DoubleVector.class, dim, new DoubleVector(new double[dim])); MaterializedRelation<DoubleVector> prep = new MaterializedRelation<DoubleVector>(proxy, type, ids); proxy.addRelation(prep); // Project - for(DBID id : ids) { - DoubleVector v = new DoubleVector(relation.get(id).getColumnVector().getArrayRef()); - prep.set(id, v); + for(DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + DoubleVector v = new DoubleVector(relation.get(iter).getColumnVector().getArrayRef()); + prep.set(iter, v); } if(logger.isDebugging()) { @@ -800,15 +800,15 @@ public class CASH extends AbstractAlgorithm<Clustering<Model>> implements Cluste */ private Database buildDerivatorDB(Relation<ParameterizationFunction> relation, DBIDs ids) throws UnableToComplyException { ProxyDatabase proxy = new ProxyDatabase(ids); - int dim = relation.get(ids.iterator().next()).getDimensionality(); + int dim = DatabaseUtil.dimensionality(relation); SimpleTypeInformation<DoubleVector> type = new VectorFieldTypeInformation<DoubleVector>(DoubleVector.class, dim, new DoubleVector(new double[dim])); MaterializedRelation<DoubleVector> prep = new MaterializedRelation<DoubleVector>(proxy, type, ids); proxy.addRelation(prep); // Project - for(DBID id : ids) { - DoubleVector v = new DoubleVector(relation.get(id).getColumnVector().getArrayRef()); - prep.set(id, v); + for(DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + DoubleVector v = new DoubleVector(relation.get(iter).getColumnVector().getArrayRef()); + prep.set(iter, v); } return proxy; diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/COPAC.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/COPAC.java index 575bf117..1d41d37e 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/COPAC.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/COPAC.java @@ -41,6 +41,7 @@ import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.ProxyDatabase; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; @@ -176,7 +177,7 @@ public class COPAC<V extends NumberVector<V, ?>, D extends Distance<D>> extends * @return Clustering result */ @SuppressWarnings("unchecked") - public Clustering<Model> run(Relation<V> relation) throws IllegalStateException { + public Clustering<Model> run(Relation<V> relation) { if(logger.isVerbose()) { logger.verbose("Running COPAC on db size = " + relation.size() + " with dimensionality = " + DatabaseUtil.dimensionality(relation)); } @@ -189,14 +190,14 @@ public class COPAC<V extends NumberVector<V, ?>, D extends Distance<D>> extends FiniteProgress partitionProgress = logger.isVerbose() ? new FiniteProgress("Partitioning", relation.size(), logger) : null; int processed = 1; - for(DBID id : relation.iterDBIDs()) { - Integer corrdim = preprocin.getLocalProjection(id).getCorrelationDimension(); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + int corrdim = preprocin.getLocalProjection(iditer).getCorrelationDimension(); if(!partitionMap.containsKey(corrdim)) { partitionMap.put(corrdim, DBIDUtil.newArray()); } - partitionMap.get(corrdim).add(id); + partitionMap.get(corrdim).add(iditer); if(partitionProgress != null) { partitionProgress.setProcessed(processed++, logger); } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/ERiC.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/ERiC.java index af4f677f..b57a6e29 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/ERiC.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/ERiC.java @@ -118,7 +118,7 @@ public class ERiC<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Cluste * @param relation Relation to process * @return Clustering result */ - public Clustering<CorrelationModel<V>> run(Relation<V> relation) throws IllegalStateException { + public Clustering<CorrelationModel<V>> run(Relation<V> relation) { final int dimensionality = DatabaseUtil.dimensionality(relation); StepProgress stepprog = logger.isVerbose() ? new StepProgress(3) : null; @@ -291,7 +291,7 @@ public class ERiC<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Cluste return parameters; } - private void buildHierarchy(SortedMap<Integer, List<Cluster<CorrelationModel<V>>>> clusterMap, DistanceQuery<V, IntegerDistance> query) throws IllegalStateException { + private void buildHierarchy(SortedMap<Integer, List<Cluster<CorrelationModel<V>>>> clusterMap, DistanceQuery<V, IntegerDistance> query) { StringBuffer msg = new StringBuffer(); DBSCAN<V, DoubleDistance> dbscan = ClassGenericsUtil.castWithGenericsOrNull(DBSCAN.class, copacAlgorithm.getPartitionAlgorithm(query)); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/LMCLUS.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/LMCLUS.java index 41ee1f69..b8942de8 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/LMCLUS.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/LMCLUS.java @@ -35,7 +35,7 @@ import de.lmu.ifi.dbs.elki.data.model.Model; import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; @@ -175,9 +175,9 @@ public class LMCLUS extends AbstractAlgorithm<Clustering<Model>> { break; } ModifiableDBIDs subset = DBIDUtil.newArray(current.size()); - for(DBID id : current) { - if(deviation(relation.get(id).getColumnVector().minusEquals(separation.originV), separation.basis) < separation.threshold) { - subset.add(id); + for(DBIDIter iter = current.iter(); iter.valid(); iter.advance()) { + if(deviation(relation.get(iter).getColumnVector().minusEquals(separation.originV), separation.basis) < separation.threshold) { + subset.add(iter); } } // logger.verbose("size:"+subset.size()); @@ -265,16 +265,16 @@ public class LMCLUS extends AbstractAlgorithm<Clustering<Model>> { int remaining_retries = 100; for(int i = 1; i <= samples; i++) { DBIDs sample = DBIDUtil.randomSample(currentids, dimension + 1, r.nextLong()); - final Iterator<DBID> iter = sample.iterator(); + final DBIDIter iter = sample.iter(); // Use first as origin - DBID origin = iter.next(); - Vector originV = relation.get(origin).getColumnVector(); + Vector originV = relation.get(iter).getColumnVector(); + iter.advance(); // Build orthogonal basis from remainder Matrix basis; { List<Vector> vectors = new ArrayList<Vector>(sample.size() - 1); - while(iter.hasNext()) { - Vector vec = relation.get(iter.next()).getColumnVector(); + for(;iter.valid(); iter.advance()) { + Vector vec = relation.get(iter).getColumnVector(); vectors.add(vec.minusEquals(originV)); } // generate orthogonal basis @@ -292,12 +292,12 @@ public class LMCLUS extends AbstractAlgorithm<Clustering<Model>> { // Generate and fill a histogram. FlexiHistogram<Double, Double> histogram = FlexiHistogram.DoubleSumHistogram(BINS); double w = 1.0 / currentids.size(); - for(DBID point : currentids) { + for(DBIDIter iter2 = currentids.iter(); iter2.valid(); iter2.advance()) { // Skip sampled points - if(sample.contains(point)) { + if(sample.contains(iter2)) { continue; } - Vector vec = relation.get(point).getColumnVector().minusEquals(originV); + Vector vec = relation.get(iter2).getColumnVector().minusEquals(originV); final double distance = deviation(vec, basis); histogram.aggregate(distance, w); } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/ORCLUS.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/ORCLUS.java index eb5608fc..2e9f4a9b 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/ORCLUS.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/ORCLUS.java @@ -38,6 +38,7 @@ import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; @@ -139,8 +140,11 @@ public class ORCLUS<V extends NumberVector<V, ?>> extends AbstractProjectedClust /** * Performs the ORCLUS algorithm on the given database. + * + * @param database Database + * @param relation Relation */ - public Clustering<Model> run(Database database, Relation<V> relation) throws IllegalStateException { + public Clustering<Model> run(Database database, Relation<V> relation) { try { DistanceQuery<V, DoubleDistance> distFunc = this.getDistanceQuery(database); // current dimensionality associated with each seed @@ -211,8 +215,8 @@ public class ORCLUS<V extends NumberVector<V, ?>> extends AbstractProjectedClust DBIDs randomSample = DBIDUtil.randomSample(database.getDBIDs(), k, seed); V factory = DatabaseUtil.assumeVectorField(database).getFactory(); List<ORCLUSCluster> seeds = new ArrayList<ORCLUSCluster>(); - for(DBID id : randomSample) { - seeds.add(new ORCLUSCluster(database.get(id), id, factory)); + for(DBIDIter iter = randomSample.iter(); iter.valid(); iter.advance()) { + seeds.add(new ORCLUSCluster(database.get(iter), iter.getDBID(), factory)); } return seeds; } @@ -240,10 +244,8 @@ public class ORCLUS<V extends NumberVector<V, ?>> extends AbstractProjectedClust } // for each data point o do - Iterator<DBID> it = database.iterDBIDs(); - while(it.hasNext()) { - DBID id = it.next(); - V o = database.get(id); + for (DBIDIter it = database.iterDBIDs(); it.valid(); it.advance()) { + V o = database.get(it); DoubleDistance minDist = null; ORCLUSCluster minCluster = null; @@ -260,7 +262,7 @@ public class ORCLUS<V extends NumberVector<V, ?>> extends AbstractProjectedClust } // add p to the cluster with the least value of projected distance assert minCluster != null; - minCluster.objectIDs.add(id); + minCluster.objectIDs.add(it); } // recompute the seed in each clusters @@ -285,10 +287,9 @@ public class ORCLUS<V extends NumberVector<V, ?>> extends AbstractProjectedClust // covariance matrix of cluster // Matrix covariance = Util.covarianceMatrix(database, cluster.objectIDs); List<DistanceResultPair<DoubleDistance>> results = new ArrayList<DistanceResultPair<DoubleDistance>>(cluster.objectIDs.size()); - for(Iterator<DBID> it = cluster.objectIDs.iterator(); it.hasNext();) { - DBID id = it.next(); - DoubleDistance distance = distFunc.distance(cluster.centroid, database.get(id)); - DistanceResultPair<DoubleDistance> qr = new GenericDistanceResultPair<DoubleDistance>(distance, id); + for(DBIDIter it = cluster.objectIDs.iter(); it.valid(); it.advance()) { + DoubleDistance distance = distFunc.distance(cluster.centroid, database.get(it)); + DistanceResultPair<DoubleDistance> qr = new GenericDistanceResultPair<DoubleDistance>(distance, it.getDBID()); results.add(qr); } Collections.sort(results); @@ -407,9 +408,8 @@ public class ORCLUS<V extends NumberVector<V, ?>> extends AbstractProjectedClust DoubleDistance sum = getDistanceFunction().getDistanceFactory().nullDistance(); V c_proj = projection(c_ij, c_ij.centroid, factory); - for(DBID id : c_ij.objectIDs) { - V o = database.get(id); - V o_proj = projection(c_ij, o, factory); + for(DBIDIter iter = c_ij.objectIDs.iter(); iter.valid(); iter.advance()) { + V o_proj = projection(c_ij, database.get(iter), factory); DoubleDistance dist = distFunc.distance(o_proj, c_proj); sum = sum.plus(dist.times(dist)); } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/cash/CASHIntervalSplit.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/cash/CASHIntervalSplit.java index 86e045cb..b0a12832 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/cash/CASHIntervalSplit.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/cash/CASHIntervalSplit.java @@ -30,6 +30,7 @@ import de.lmu.ifi.dbs.elki.data.HyperBoundingBox; import de.lmu.ifi.dbs.elki.data.ParameterizationFunction; import de.lmu.ifi.dbs.elki.data.spatial.SpatialUtil; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; @@ -114,7 +115,8 @@ public class CASHIntervalSplit { f_maxima.put(interval, maxima); } - for(DBID id : superSetIDs) { + for(DBIDIter iter = superSetIDs.iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); Double f_min = minima.get(id); Double f_max = maxima.get(id); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/gdbscan/CorePredicate.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/gdbscan/CorePredicate.java new file mode 100644 index 00000000..e75a89dc --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/gdbscan/CorePredicate.java @@ -0,0 +1,80 @@ +package de.lmu.ifi.dbs.elki.algorithm.clustering.gdbscan; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; +import de.lmu.ifi.dbs.elki.database.Database; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; + +/** + * Predicate for GeneralizedDBSCAN to evaluate whether a point is a core point + * or not. + * + * Note the Factory/Instance split of this interface. + * + * @author Erich Schubert + * + * @apiviz.has Instance + */ +public interface CorePredicate { + /** + * Constant for the generic type {@code List<? extends DistanceResultPair<?>>} + */ + public static final String NEIGHBOR_LIST = "neighborhood-list"; + + /** + * Instantiate for a database. + * + * @param database Database to instantiate for + * @param type Type to instantiate for + * @return Instance + */ + public <T> Instance<T> instantiate(Database database, SimpleTypeInformation<?> type); + + /** + * Test whether the neighborhood type T is accepted by this predicate. + * + * @param type Type information + * @return true when the type is accepted + */ + public boolean acceptsType(SimpleTypeInformation<?> type); + + /** + * Instance for a particular data set. + * + * @author Erich Schubert + * + * @param <T> actual type + */ + public static interface Instance<T> { + /** + * Decide whether the point is a core point, based on its neighborhood. + * + * @param point Query point + * @param neighbors Neighbors + * @return core point property + */ + public boolean isCorePoint(DBIDRef point, T neighbors); + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/gdbscan/EpsilonNeighborPredicate.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/gdbscan/EpsilonNeighborPredicate.java new file mode 100644 index 00000000..cb24e8f1 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/gdbscan/EpsilonNeighborPredicate.java @@ -0,0 +1,268 @@ +package de.lmu.ifi.dbs.elki.algorithm.clustering.gdbscan; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import java.util.List; + +import de.lmu.ifi.dbs.elki.algorithm.AbstractDistanceBasedAlgorithm; +import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; +import de.lmu.ifi.dbs.elki.data.type.TypeInformation; +import de.lmu.ifi.dbs.elki.data.type.TypeUtil; +import de.lmu.ifi.dbs.elki.database.Database; +import de.lmu.ifi.dbs.elki.database.QueryUtil; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; +import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; +import de.lmu.ifi.dbs.elki.database.ids.DBIDs; +import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; +import de.lmu.ifi.dbs.elki.database.query.DistanceDBIDResult; +import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; +import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; +import de.lmu.ifi.dbs.elki.database.query.range.RangeQuery; +import de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction; +import de.lmu.ifi.dbs.elki.distance.distancefunction.EuclideanDistanceFunction; +import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; +import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; +import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.DistanceParameter; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter; + +/** + * The default DBSCAN and OPTICS neighbor predicate, using an + * epsilon-neighborhood. + * + * <p> + * Reference: <br> + * M. Ester, H.-P. Kriegel, J. Sander, and X. Xu: A Density-Based Algorithm for + * Discovering Clusters in Large Spatial Databases with Noise. <br> + * In Proc. 2nd Int. Conf. on Knowledge Discovery and Data Mining (KDD '96), + * Portland, OR, 1996. + * </p> + * + * @author Erich Schubert + * + * @param <D> Distance type + */ +@Reference(authors = "M. Ester, H.-P. Kriegel, J. Sander, and X. Xu", title = "A Density-Based Algorithm for Discovering Clusters in Large Spatial Databases with Noise", booktitle = "Proc. 2nd Int. Conf. on Knowledge Discovery and Data Mining (KDD '96), Portland, OR, 1996", url = "http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.71.1980") +public class EpsilonNeighborPredicate<O, D extends Distance<D>> implements NeighborPredicate { + /** + * Range to query with + */ + D epsilon; + + /** + * Distance function to use + */ + DistanceFunction<O, D> distFunc; + + /** + * Full constructor. + * + * @param epsilon Epsilon value + * @param distFunc Distance function to use + */ + public EpsilonNeighborPredicate(D epsilon, DistanceFunction<O, D> distFunc) { + super(); + this.epsilon = epsilon; + this.distFunc = distFunc; + } + + @SuppressWarnings("unchecked") + @Override + public <T> Instance<T> instantiate(Database database, SimpleTypeInformation<?> type) { + if(TypeUtil.DBIDS.isAssignableFromType(type)) { + DistanceQuery<O, D> dq = QueryUtil.getDistanceQuery(database, distFunc); + RangeQuery<O, D> rq = database.getRangeQuery(dq); + return (Instance<T>) new DBIDInstance<D>(epsilon, rq, dq.getRelation().getDBIDs()); + } + if(TypeUtil.NEIGHBORLIST.isAssignableFromType(type)) { + DistanceQuery<O, D> dq = QueryUtil.getDistanceQuery(database, distFunc); + RangeQuery<O, D> rq = database.getRangeQuery(dq); + return (Instance<T>) new NeighborListInstance<D>(epsilon, rq, dq.getRelation().getDBIDs()); + } + throw new AbortException("Incompatible predicate types"); + } + + @Override + public SimpleTypeInformation<?>[] getOutputType() { + return new SimpleTypeInformation<?>[] { TypeUtil.DBIDS, TypeUtil.NEIGHBORLIST }; + } + + @Override + public TypeInformation getInputTypeRestriction() { + return distFunc.getInputTypeRestriction(); + } + + /** + * Instance for a particular data set. + * + * @author Erich Schubert + */ + public static class DBIDInstance<D extends Distance<D>> implements NeighborPredicate.Instance<DBIDs> { + /** + * Range to query with + */ + D epsilon; + + /** + * Range query to use on the database. + */ + RangeQuery<?, D> rq; + + /** + * DBIDs to process + */ + DBIDs ids; + + /** + * Constructor. + * + * @param epsilon Epsilon + * @param rq Range query to use + * @param ids DBIDs to process + */ + public DBIDInstance(D epsilon, RangeQuery<?, D> rq, DBIDs ids) { + super(); + this.epsilon = epsilon; + this.rq = rq; + this.ids = ids; + } + + @Override + public DBIDs getIDs() { + return ids; + } + + @Override + public DBIDs getNeighbors(DBIDRef reference) { + List<DistanceResultPair<D>> res = rq.getRangeForDBID(reference, epsilon); + // Throw away the actual distance values ... + ModifiableDBIDs neighbors = DBIDUtil.newHashSet(res.size()); + for(DistanceResultPair<D> dr : res) { + neighbors.add(dr); + } + return neighbors; + } + + @Override + public void addDBIDs(ModifiableDBIDs ids, DBIDs neighbors) { + ids.addDBIDs(neighbors); + } + } + + /** + * Instance for a particular data set. + * + * @author Erich Schubert + */ + public static class NeighborListInstance<D extends Distance<D>> implements NeighborPredicate.Instance<DistanceDBIDResult<D>> { + /** + * Range to query with + */ + D epsilon; + + /** + * Range query to use on the database. + */ + RangeQuery<?, D> rq; + + /** + * DBIDs to process + */ + DBIDs ids; + + /** + * Constructor. + * + * @param epsilon Epsilon + * @param rq Range query to use + * @param ids DBIDs to process + */ + public NeighborListInstance(D epsilon, RangeQuery<?, D> rq, DBIDs ids) { + super(); + this.epsilon = epsilon; + this.rq = rq; + this.ids = ids; + } + + @Override + public DBIDs getIDs() { + return ids; + } + + @Override + public DistanceDBIDResult<D> getNeighbors(DBIDRef reference) { + return rq.getRangeForDBID(reference, epsilon); + } + + @Override + public void addDBIDs(ModifiableDBIDs ids, DistanceDBIDResult<D> neighbors) { + for(DistanceResultPair<D> neighbor : neighbors) { + ids.add(neighbor); + } + } + } + + /** + * Parameterization class + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer<O, D extends Distance<D>> extends AbstractParameterizer { + /** + * Range to query with + */ + D epsilon; + + /** + * Distance function to use + */ + DistanceFunction<O, D> distfun = null; + + @Override + protected void makeOptions(Parameterization config) { + super.makeOptions(config); + // Get a distance function. + ObjectParameter<DistanceFunction<O, D>> distanceP = new ObjectParameter<DistanceFunction<O, D>>(AbstractDistanceBasedAlgorithm.DISTANCE_FUNCTION_ID, DistanceFunction.class, EuclideanDistanceFunction.class); + D distanceFactory = null; + if(config.grab(distanceP)) { + distfun = distanceP.instantiateClass(config); + distanceFactory = distfun.getDistanceFactory(); + } + // Get the epsilon parameter + DistanceParameter<D> epsilonP = new DistanceParameter<D>(de.lmu.ifi.dbs.elki.algorithm.clustering.DBSCAN.EPSILON_ID, distanceFactory); + if(config.grab(epsilonP)) { + epsilon = epsilonP.getValue(); + } + } + + @Override + protected EpsilonNeighborPredicate<O, D> makeInstance() { + return new EpsilonNeighborPredicate<O, D>(epsilon, distfun); + } + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/gdbscan/GeneralizedDBSCAN.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/gdbscan/GeneralizedDBSCAN.java new file mode 100644 index 00000000..2e1c2093 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/gdbscan/GeneralizedDBSCAN.java @@ -0,0 +1,323 @@ +package de.lmu.ifi.dbs.elki.algorithm.clustering.gdbscan; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import gnu.trove.list.array.TIntArrayList; + +import java.util.ArrayList; + +import de.lmu.ifi.dbs.elki.algorithm.AbstractAlgorithm; +import de.lmu.ifi.dbs.elki.algorithm.clustering.ClusteringAlgorithm; +import de.lmu.ifi.dbs.elki.data.Cluster; +import de.lmu.ifi.dbs.elki.data.Clustering; +import de.lmu.ifi.dbs.elki.data.model.ClusterModel; +import de.lmu.ifi.dbs.elki.data.model.Model; +import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; +import de.lmu.ifi.dbs.elki.data.type.TypeInformation; +import de.lmu.ifi.dbs.elki.data.type.TypeUtil; +import de.lmu.ifi.dbs.elki.database.Database; +import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; +import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; +import de.lmu.ifi.dbs.elki.database.datastore.WritableIntegerDataStore; +import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; +import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; +import de.lmu.ifi.dbs.elki.database.ids.DBIDs; +import de.lmu.ifi.dbs.elki.logging.Logging; +import de.lmu.ifi.dbs.elki.logging.progress.FiniteProgress; +import de.lmu.ifi.dbs.elki.logging.progress.IndefiniteProgress; +import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; +import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter; + +/** + * Generalized DBSCAN, density-based clustering with noise. + * <p> + * Reference:<br /> + * Jörg Sander, Martin Ester, Hans-Peter Kriegel, Xiaowei Xu:<br /> + * Density-Based Clustering in Spatial Databases: The Algorithm GDBSCAN and Its + * Applications<br /> + * In: Data Mining and Knowledge Discovery, 1998. + * </p> + * + * @author Erich Schubert + * @author Arthur Zimek + * + * @apiviz.has Instance + */ +@Reference(authors = "Jörg Sander, Martin Ester, Hans-Peter Kriegel, Xiaowei Xu", title = "Density-Based Clustering in Spatial Databases: The Algorithm GDBSCAN and Its Applications", booktitle = "Data Mining and Knowledge Discovery", url = "http://dx.doi.org/10.1023/A:1009745219419") +public class GeneralizedDBSCAN extends AbstractAlgorithm<Clustering<Model>> implements ClusteringAlgorithm<Clustering<Model>> { + /** + * Get a logger for this algorithm + */ + final static Logging logger = Logging.getLogger(GeneralizedDBSCAN.class); + + /** + * The neighborhood predicate factory. + */ + NeighborPredicate npred; + + /** + * The core predicate factory. + */ + CorePredicate corepred; + + /** + * Constructor for parameterized algorithm. + * + * @param npred Neighbor predicate + * @param corepred Core point predicate + */ + public GeneralizedDBSCAN(NeighborPredicate npred, CorePredicate corepred) { + super(); + this.npred = npred; + this.corepred = corepred; + } + + @Override + public Clustering<Model> run(Database database) { + for (SimpleTypeInformation<?> t : npred.getOutputType()) { + if (corepred.acceptsType(t)) { + return new Instance<Object>(npred.instantiate(database, t), corepred.instantiate(database, t)).run(); + } + } + throw new AbortException("No compatible types found."); + } + + @Override + public TypeInformation[] getInputTypeRestriction() { + return TypeUtil.array(npred.getInputTypeRestriction()); + } + + @Override + protected Logging getLogger() { + return logger; + } + + /** + * Instance for a particular data set. + * + * @author Erich Schubert + */ + public class Instance<T> { + /** + * The neighborhood predicate + */ + final NeighborPredicate.Instance<T> npred; + + /** + * The core object property + */ + final CorePredicate.Instance<T> corepred; + + /** + * Full Constructor + * + * @param npred Neighborhood predicate + * @param corepred Core object predicate + */ + public Instance(NeighborPredicate.Instance<T> npred, CorePredicate.Instance<T> corepred) { + super(); + this.npred = npred; + this.corepred = corepred; + } + + /** + * Run the actual DBSCAN algorithm. + * + * @return Clustering result + */ + public Clustering<Model> run() { + final DBIDs ids = npred.getIDs(); + // Setup progress logging + final FiniteProgress progress = logger.isVerbose() ? new FiniteProgress("Clustering", ids.size(), logger) : null; + final IndefiniteProgress clusprogress = logger.isVerbose() ? new IndefiniteProgress("Clusters", logger) : null; + // (Temporary) store the cluster ID assigned. + final WritableIntegerDataStore clusterids = DataStoreUtil.makeIntegerStorage(ids, DataStoreFactory.HINT_TEMP, -2); + // Note: these are not exact! + final TIntArrayList clustersizes = new TIntArrayList(); + + // Implementation Note: using Integer objects should result in + // reduced memory use in the HashMap! + final int noiseid = -1; + int clusterid = 0; + int clustersize = 0; + int noisesize = 0; + // Iterate over all objects in the database. + for(DBIDIter id = ids.iter(); id.valid(); id.advance()) { + // Skip already processed ids. + if(clusterids.intValue(id) > -2) { + continue; + } + // Evaluate Neighborhood predicate + final T neighbors = npred.getNeighbors(id); + // Evaluate Core-Point predicate: + if(corepred.isCorePoint(id, neighbors)) { + clusterids.putInt(id, clusterid); + clustersize = 1 + setbasedExpandCluster(clusterid, clusterids, neighbors, progress); + // start next cluster on next iteration. + clustersizes.add(clustersize); + clustersize = 0; + clusterid += 1; + if(clusprogress != null) { + clusprogress.setProcessed(clusterid, logger); + } + } + else { + // otherwise, it's a noise point + clusterids.putInt(id, noiseid); + noisesize += 1; + } + // We've completed this element + if(progress != null) { + progress.incrementProcessed(logger); + } + } + // Finish progress logging. + if(progress != null) { + progress.ensureCompleted(logger); + } + if(clusprogress != null) { + clusprogress.setCompleted(logger); + } + + // Transform cluster ID mapping into a clustering result: + ArrayList<ArrayModifiableDBIDs> clusterlists = new ArrayList<ArrayModifiableDBIDs>(clusterid + 1); + // add noise cluster storage + clusterlists.add(DBIDUtil.newArray(noisesize)); + // add storage containers for clusters + for(int i = 0; i < clustersizes.size(); i++) { + clusterlists.add(DBIDUtil.newArray(clustersizes.get(i))); + } + // do the actual inversion + for(DBIDIter id = ids.iter(); id.valid(); id.advance()) { + int cluster = clusterids.intValue(id); + clusterlists.get(cluster + 1).add(id); + } + clusterids.destroy(); + + Clustering<Model> result = new Clustering<Model>("GDBSCAN", "gdbscan-clustering"); + int cid = 0; + for(ArrayModifiableDBIDs res : clusterlists) { + boolean isNoise = (cid == 0); + Cluster<Model> c = new Cluster<Model>(res, isNoise, ClusterModel.CLUSTER); + result.addCluster(c); + cid++; + } + return result; + } + + /** + * Set-based expand cluster implementation. + * + * @param clusterid ID of the current cluster. + * @param clusterids Current object to cluster mapping. + * @param neighbors Neighbors acquired by initial getNeighbors call. + * @param progress Progress logging + * + * @return cluster size; + */ + protected int setbasedExpandCluster(final int clusterid, final WritableIntegerDataStore clusterids, final T neighbors, final FiniteProgress progress) { + int clustersize = 0; + final ArrayModifiableDBIDs activeSet = DBIDUtil.newArray(); + npred.addDBIDs(activeSet, neighbors); + // run expandCluster as long as this set is non-empty (non-recursive + // implementation) + while(!activeSet.isEmpty()) { + final DBID id = activeSet.remove(activeSet.size() - 1); + clustersize += 1; + // Assign object to cluster + final int oldclus = clusterids.putInt(id, clusterid); + if(oldclus == -2) { + // expandCluster again: + // Evaluate Neighborhood predicate + final T newneighbors = npred.getNeighbors(id); + // Evaluate Core-Point predicate + if(corepred.isCorePoint(id, newneighbors)) { + // Note: the recursion is unrolled into iteration over the active + // set. + npred.addDBIDs(activeSet, newneighbors); + } + if(progress != null) { + progress.incrementProcessed(logger); + } + } + } + return clustersize; + } + } + + /** + * Parameterization class + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer extends AbstractParameterizer { + /** + * Neighborhood predicate + */ + NeighborPredicate npred = null; + + /** + * Core point predicate + */ + CorePredicate corepred = null; + + /** + * Parameter for neighborhood predicate + */ + public final static OptionID NEIGHBORHOODPRED_ID = OptionID.getOrCreateOptionID("gdbscan.neighborhood", "Neighborhood predicate for GDBSCAN"); + + /** + * Parameter for core predicate + */ + public final static OptionID COREPRED_ID = OptionID.getOrCreateOptionID("gdbscan.core", "Core point predicate for GDBSCAN"); + + @Override + protected void makeOptions(Parameterization config) { + // Neighborhood predicate + ObjectParameter<NeighborPredicate> npredOpt = new ObjectParameter<NeighborPredicate>(NEIGHBORHOODPRED_ID, NeighborPredicate.class, EpsilonNeighborPredicate.class); + if(config.grab(npredOpt)) { + npred = npredOpt.instantiateClass(config); + } + + // Core point predicate + ObjectParameter<CorePredicate> corepredOpt = new ObjectParameter<CorePredicate>(COREPRED_ID, CorePredicate.class, MinPtsCorePredicate.class); + if(config.grab(corepredOpt)) { + corepred = corepredOpt.instantiateClass(config); + } + } + + @Override + protected GeneralizedDBSCAN makeInstance() { + return new GeneralizedDBSCAN(npred, corepred); + } + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/gdbscan/MinPtsCorePredicate.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/gdbscan/MinPtsCorePredicate.java new file mode 100644 index 00000000..b9852eca --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/gdbscan/MinPtsCorePredicate.java @@ -0,0 +1,178 @@ +package de.lmu.ifi.dbs.elki.algorithm.clustering.gdbscan; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import java.util.List; + +import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; +import de.lmu.ifi.dbs.elki.data.type.TypeUtil; +import de.lmu.ifi.dbs.elki.database.Database; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; +import de.lmu.ifi.dbs.elki.database.ids.DBIDs; +import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; +import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; +import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.IntParameter; + +/** + * The DBSCAN default core point predicate -- having at least {@link #minpts} + * neighbors. + * + * <p> + * Reference: <br> + * M. Ester, H.-P. Kriegel, J. Sander, and X. Xu: A Density-Based Algorithm for + * Discovering Clusters in Large Spatial Databases with Noise. <br> + * In Proc. 2nd Int. Conf. on Knowledge Discovery and Data Mining (KDD '96), + * Portland, OR, 1996. + * </p> + * + * @author Erich Schubert + * + * @apiviz.has Instance + */ +@Reference(authors = "M. Ester, H.-P. Kriegel, J. Sander, and X. Xu", title = "A Density-Based Algorithm for Discovering Clusters in Large Spatial Databases with Noise", booktitle = "Proc. 2nd Int. Conf. on Knowledge Discovery and Data Mining (KDD '96), Portland, OR, 1996", url = "http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.71.1980") +public class MinPtsCorePredicate implements CorePredicate { + /** + * The minpts parameter. + */ + int minpts; + + /** + * Default constructor. + * + * @param minpts Minimum number of neighbors to be a core point. + */ + public MinPtsCorePredicate(int minpts) { + super(); + this.minpts = minpts; + } + + @SuppressWarnings("unchecked") + @Override + public <T> Instance<T> instantiate(Database database, SimpleTypeInformation<?> type) { + if(TypeUtil.DBIDS.isAssignableFromType(type)) { + return (Instance<T>) new DBIDsInstance(minpts); + } + if(TypeUtil.NEIGHBORLIST.isAssignableFromType(type)) { + return (Instance<T>) new NeighborListInstance(minpts); + } + throw new AbortException("Incompatible predicate types"); + } + + @Override + public boolean acceptsType(SimpleTypeInformation<?> type) { + if(TypeUtil.DBIDS.isAssignableFromType(type)) { + return true; + } + if(TypeUtil.NEIGHBORLIST.isAssignableFromType(type)) { + return true; + } + return false; + } + + /** + * Instance for a particular data set. + * + * @author Erich Schubert + */ + public static class DBIDsInstance implements CorePredicate.Instance<DBIDs> { + /** + * The minpts parameter. + */ + int minpts; + + /** + * Constructor for this predicate. + * + * @param minpts MinPts parameter + */ + public DBIDsInstance(int minpts) { + super(); + this.minpts = minpts; + } + + @Override + public boolean isCorePoint(DBIDRef point, DBIDs neighbors) { + return neighbors.size() >= minpts; + } + } + + /** + * Instance for a particular data set. + * + * @author Erich Schubert + */ + public static class NeighborListInstance implements CorePredicate.Instance<List<? extends DistanceResultPair<?>>> { + /** + * The minpts parameter. + */ + int minpts; + + /** + * Constructor for this predicate. + * + * @param minpts MinPts parameter + */ + public NeighborListInstance(int minpts) { + super(); + this.minpts = minpts; + } + + @Override + public boolean isCorePoint(DBIDRef point, List<? extends DistanceResultPair<?>> neighbors) { + return neighbors.size() >= minpts; + } + } + + /** + * Parameterization class + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer extends AbstractParameterizer { + /** + * Minpts value + */ + int minpts; + + @Override + protected void makeOptions(Parameterization config) { + super.makeOptions(config); + // Get the minpts parameter + IntParameter minptsP = new IntParameter(de.lmu.ifi.dbs.elki.algorithm.clustering.DBSCAN.MINPTS_ID); + if(config.grab(minptsP)) { + minpts = minptsP.getValue(); + } + } + + @Override + protected MinPtsCorePredicate makeInstance() { + return new MinPtsCorePredicate(minpts); + } + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/gdbscan/NeighborPredicate.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/gdbscan/NeighborPredicate.java new file mode 100644 index 00000000..4f9eca27 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/gdbscan/NeighborPredicate.java @@ -0,0 +1,94 @@ +package de.lmu.ifi.dbs.elki.algorithm.clustering.gdbscan; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; +import de.lmu.ifi.dbs.elki.data.type.TypeInformation; +import de.lmu.ifi.dbs.elki.database.Database; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; +import de.lmu.ifi.dbs.elki.database.ids.DBIDs; +import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; + +/** + * Get the neighbors of an object + * + * Note the Factory/Instance split of this interface. + * + * @author Erich Schubert + * + * @apiviz.has Instance + */ +public interface NeighborPredicate { + /** + * Instantiate for a database. + * + * @param database Database to instantiate for + * @return Instance + */ + public <T> Instance<T> instantiate(Database database, SimpleTypeInformation<?> type); + + /** + * Input data type restriction. + * + * @return Type restriction + */ + public TypeInformation getInputTypeRestriction(); + + /** + * Output data type information. + * + * @return Type information + */ + public SimpleTypeInformation<?>[] getOutputType(); + + /** + * Instance for a particular data set. + * + * @author Erich Schubert + */ + public static interface Instance<T> { + /** + * Get the neighbors of a reference object for DBSCAN. + * + * @param reference Reference object + * @return Neighborhood + */ + public T getNeighbors(DBIDRef reference); + + /** + * Get the IDs the predicate is defined for. + * + * @return Database ids + */ + public DBIDs getIDs(); + + /** + * Add the neighbors to a DBID set + * + * @param ids ID set + * @param neighbors Neighbors to add + */ + public void addDBIDs(ModifiableDBIDs ids, T neighbors); + } +} diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/gdbscan/package-info.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/gdbscan/package-info.java new file mode 100644 index 00000000..8be23c7d --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/gdbscan/package-info.java @@ -0,0 +1,43 @@ +/** + * <p>Generalized DBSCAN.</p> + * + * Generalized DBSCAN is an abstraction of the original DBSCAN idea, + * that allows the use of arbitrary "neighborhood" and "core point" predicates. + * + * For each object, the neighborhood as defined by the "neighborhood" predicate + * is retrieved - in original DBSCAN, this is the objects within an epsilon sphere + * around the query object. Then the core point predicate is evaluated to decide if + * the object is considered dense. If so, a cluster is started (or extended) to + * include the neighbors as well. + * + * <p> + * Reference:<br /> + * Jörg Sander, Martin Ester, Hans-Peter Kriegel, Xiaowei Xu:<br /> + * Density-Based Clustering in Spatial Databases: The Algorithm GDBSCAN and Its + * Applications<br /> + * In: Data Mining and Knowledge Discovery, 1998. + * </p> + */ +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +package de.lmu.ifi.dbs.elki.algorithm.clustering.gdbscan;
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/AbstractKMeans.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/AbstractKMeans.java index d3c73b53..92862909 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/AbstractKMeans.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/AbstractKMeans.java @@ -1,24 +1,5 @@ package de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import de.lmu.ifi.dbs.elki.algorithm.AbstractPrimitiveDistanceBasedAlgorithm; -import de.lmu.ifi.dbs.elki.data.Clustering; -import de.lmu.ifi.dbs.elki.data.NumberVector; -import de.lmu.ifi.dbs.elki.data.model.MeanModel; -import de.lmu.ifi.dbs.elki.data.type.TypeInformation; -import de.lmu.ifi.dbs.elki.data.type.TypeUtil; -import de.lmu.ifi.dbs.elki.database.ids.DBID; -import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; -import de.lmu.ifi.dbs.elki.database.relation.Relation; -import de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDistanceFunction; -import de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDoubleDistanceFunction; -import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; -import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; -import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; - /* This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures @@ -42,37 +23,39 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; along with this program. If not, see <http://www.gnu.org/licenses/>. */ +import java.util.ArrayList; +import java.util.List; + +import de.lmu.ifi.dbs.elki.algorithm.AbstractPrimitiveDistanceBasedAlgorithm; +import de.lmu.ifi.dbs.elki.data.Clustering; +import de.lmu.ifi.dbs.elki.data.NumberVector; +import de.lmu.ifi.dbs.elki.data.VectorUtil.SortDBIDsBySingleDimension; +import de.lmu.ifi.dbs.elki.data.model.MeanModel; +import de.lmu.ifi.dbs.elki.data.type.TypeInformation; +import de.lmu.ifi.dbs.elki.data.type.TypeUtil; +import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; +import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; +import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; +import de.lmu.ifi.dbs.elki.database.relation.Relation; +import de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDistanceFunction; +import de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDoubleDistanceFunction; +import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; +import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; +import de.lmu.ifi.dbs.elki.utilities.datastructures.QuickSelect; + /** * Abstract base class for k-means implementations. * * @author Erich Schubert * + * @apiviz.composedOf KMeansInitialization + * * @param <V> Vector type * @param <D> Distance type */ -public abstract class AbstractKMeans<V extends NumberVector<V, ?>, D extends Distance<D>> extends AbstractPrimitiveDistanceBasedAlgorithm<NumberVector<?, ?>, D, Clustering<MeanModel<V>>> { - /** - * Parameter to specify the number of clusters to find, must be an integer - * greater than 0. - */ - public static final OptionID K_ID = OptionID.getOrCreateOptionID("kmeans.k", "The number of clusters to find."); - - /** - * Parameter to specify the number of clusters to find, must be an integer - * greater or equal to 0, where 0 means no limit. - */ - public static final OptionID MAXITER_ID = OptionID.getOrCreateOptionID("kmeans.maxiter", "The maximum number of iterations to do. 0 means no limit."); - - /** - * Parameter to specify the random generator seed. - */ - public static final OptionID SEED_ID = OptionID.getOrCreateOptionID("kmeans.seed", "The random number generator seed."); - - /** - * Parameter to specify the initialization method - */ - public static final OptionID INIT_ID = OptionID.getOrCreateOptionID("kmeans.initialization", "Method to choose the initial means."); - +public abstract class AbstractKMeans<V extends NumberVector<V, ?>, D extends Distance<D>> extends AbstractPrimitiveDistanceBasedAlgorithm<NumberVector<?, ?>, D, Clustering<MeanModel<V>>> implements KMeans { /** * Holds the value of {@link #K_ID}. */ @@ -94,6 +77,7 @@ public abstract class AbstractKMeans<V extends NumberVector<V, ?>, D extends Dis * @param distanceFunction distance function * @param k k parameter * @param maxiter Maxiter parameter + * @param initializer Function to generate the initial means */ public AbstractKMeans(PrimitiveDistanceFunction<NumberVector<?, ?>, D> distanceFunction, int k, int maxiter, KMeansInitialization<V> initializer) { super(distanceFunction); @@ -111,15 +95,15 @@ public abstract class AbstractKMeans<V extends NumberVector<V, ?>, D extends Dis * @param clusters cluster assignment * @return true when the object was reassigned */ - protected boolean assignToNearestCluster(Relation<V> relation, List<Vector> means, List<? extends ModifiableDBIDs> clusters) { + protected boolean assignToNearestCluster(Relation<V> relation, List<? extends NumberVector<?, ?>> means, List<? extends ModifiableDBIDs> clusters) { boolean changed = false; if(getDistanceFunction() instanceof PrimitiveDoubleDistanceFunction) { @SuppressWarnings("unchecked") final PrimitiveDoubleDistanceFunction<? super NumberVector<?, ?>> df = (PrimitiveDoubleDistanceFunction<? super NumberVector<?, ?>>) getDistanceFunction(); - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { double mindist = Double.POSITIVE_INFINITY; - V fv = relation.get(id); + V fv = relation.get(iditer); int minIndex = 0; for(int i = 0; i < k; i++) { double dist = df.doubleDistance(fv, means.get(i)); @@ -128,13 +112,13 @@ public abstract class AbstractKMeans<V extends NumberVector<V, ?>, D extends Dis mindist = dist; } } - if(clusters.get(minIndex).add(id)) { + if(clusters.get(minIndex).add(iditer)) { changed = true; // Remove from previous cluster // TODO: keep a list of cluster assignments to save this search? for(int i = 0; i < k; i++) { if(i != minIndex) { - if(clusters.get(i).remove(id)) { + if(clusters.get(i).remove(iditer)) { break; } } @@ -144,9 +128,9 @@ public abstract class AbstractKMeans<V extends NumberVector<V, ?>, D extends Dis } else { final PrimitiveDistanceFunction<? super NumberVector<?, ?>, D> df = getDistanceFunction(); - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { D mindist = df.getDistanceFactory().infiniteDistance(); - V fv = relation.get(id); + V fv = relation.get(iditer); int minIndex = 0; for(int i = 0; i < k; i++) { D dist = df.distance(fv, means.get(i)); @@ -155,13 +139,13 @@ public abstract class AbstractKMeans<V extends NumberVector<V, ?>, D extends Dis mindist = dist; } } - if(clusters.get(minIndex).add(id)) { + if(clusters.get(minIndex).add(iditer)) { changed = true; // Remove from previous cluster // TODO: keep a list of cluster assignments to save this search? for(int i = 0; i < k; i++) { if(i != minIndex) { - if(clusters.get(i).remove(id)) { + if(clusters.get(i).remove(iditer)) { break; } } @@ -185,25 +169,23 @@ public abstract class AbstractKMeans<V extends NumberVector<V, ?>, D extends Dis * @param database the database containing the vectors * @return the mean vectors of the given clusters in the given database */ - protected List<Vector> means(List<? extends ModifiableDBIDs> clusters, List<Vector> means, Relation<V> database) { + protected List<Vector> means(List<? extends ModifiableDBIDs> clusters, List<? extends NumberVector<?, ?>> means, Relation<V> database) { List<Vector> newMeans = new ArrayList<Vector>(k); for(int i = 0; i < k; i++) { ModifiableDBIDs list = clusters.get(i); Vector mean = null; - for(Iterator<DBID> clusterIter = list.iterator(); clusterIter.hasNext();) { - if(mean == null) { - mean = database.get(clusterIter.next()).getColumnVector(); - } - else { - mean.plusEquals(database.get(clusterIter.next()).getColumnVector()); - } - } if(list.size() > 0) { - assert mean != null; - mean.timesEquals(1.0 / list.size()); + double s = 1.0 / list.size(); + DBIDIter iter = list.iter(); + assert (iter.valid()); + mean = database.get(iter).getColumnVector().timesEquals(s); + iter.advance(); + for(; iter.valid(); iter.advance()) { + mean.plusTimesEquals(database.get(iter).getColumnVector(), s); + } } else { - mean = means.get(i); + mean = means.get(i).getColumnVector(); } newMeans.add(mean); } @@ -211,6 +193,36 @@ public abstract class AbstractKMeans<V extends NumberVector<V, ?>, D extends Dis } /** + * Returns the median vectors of the given clusters in the given database. + * + * @param clusters the clusters to compute the means + * @param medians the recent medians + * @param database the database containing the vectors + * @return the mean vectors of the given clusters in the given database + */ + protected List<NumberVector<?, ?>> medians(List<? extends ModifiableDBIDs> clusters, List<? extends NumberVector<?, ?>> medians, Relation<V> database) { + final int dim = medians.get(0).getDimensionality(); + final SortDBIDsBySingleDimension sorter = new SortDBIDsBySingleDimension(database); + List<NumberVector<?, ?>> newMedians = new ArrayList<NumberVector<?, ?>>(k); + for(int i = 0; i < k; i++) { + ArrayModifiableDBIDs list = DBIDUtil.newArray(clusters.get(i)); + if(list.size() > 0) { + Vector mean = new Vector(dim); + for(int d = 0; d < dim; d++) { + sorter.setDimension(d + 1); + DBID id = QuickSelect.median(list, sorter); + mean.set(d, database.get(id).doubleValue(d + 1)); + } + newMedians.add(mean); + } + else { + newMedians.add((NumberVector<?, ?>) medians.get(i)); + } + } + return newMedians; + } + + /** * Compute an incremental update for the mean * * @param mean Mean to update @@ -239,16 +251,16 @@ public abstract class AbstractKMeans<V extends NumberVector<V, ?>, D extends Dis */ protected boolean macQueenIterate(Relation<V> relation, List<Vector> means, List<ModifiableDBIDs> clusters) { boolean changed = false; - + if(getDistanceFunction() instanceof PrimitiveDoubleDistanceFunction) { // Raw distance function @SuppressWarnings("unchecked") final PrimitiveDoubleDistanceFunction<? super NumberVector<?, ?>> df = (PrimitiveDoubleDistanceFunction<? super NumberVector<?, ?>>) getDistanceFunction(); - + // Incremental update - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { double mindist = Double.POSITIVE_INFINITY; - V fv = relation.get(id); + V fv = relation.get(iditer); int minIndex = 0; for(int i = 0; i < k; i++) { double dist = df.doubleDistance(fv, means.get(i)); @@ -261,13 +273,13 @@ public abstract class AbstractKMeans<V extends NumberVector<V, ?>, D extends Dis for(int i = 0; i < k; i++) { ModifiableDBIDs ci = clusters.get(i); if(i == minIndex) { - if(ci.add(id)) { - incrementalUpdateMean(means.get(i), relation.get(id), ci.size(), +1); + if(ci.add(iditer)) { + incrementalUpdateMean(means.get(i), fv, ci.size(), +1); changed = true; } } - else if(ci.remove(id)) { - incrementalUpdateMean(means.get(i), relation.get(id), ci.size() + 1, -1); + else if(ci.remove(iditer)) { + incrementalUpdateMean(means.get(i), fv, ci.size() + 1, -1); changed = true; } } @@ -276,11 +288,11 @@ public abstract class AbstractKMeans<V extends NumberVector<V, ?>, D extends Dis else { // Raw distance function final PrimitiveDistanceFunction<? super NumberVector<?, ?>, D> df = getDistanceFunction(); - + // Incremental update - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { D mindist = df.getDistanceFactory().infiniteDistance(); - V fv = relation.get(id); + V fv = relation.get(iditer); int minIndex = 0; for(int i = 0; i < k; i++) { D dist = df.distance(fv, means.get(i)); @@ -293,13 +305,13 @@ public abstract class AbstractKMeans<V extends NumberVector<V, ?>, D extends Dis for(int i = 0; i < k; i++) { ModifiableDBIDs ci = clusters.get(i); if(i == minIndex) { - if(ci.add(id)) { - incrementalUpdateMean(means.get(i), relation.get(id), ci.size(), +1); + if(ci.add(iditer)) { + incrementalUpdateMean(means.get(i), fv, ci.size(), +1); changed = true; } } - else if(ci.remove(id)) { - incrementalUpdateMean(means.get(i), relation.get(id), ci.size() + 1, -1); + else if(ci.remove(iditer)) { + incrementalUpdateMean(means.get(i), fv, ci.size() + 1, -1); changed = true; } } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/AbstractKMeansInitialization.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/AbstractKMeansInitialization.java index b5f088fb..a8effecd 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/AbstractKMeansInitialization.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/AbstractKMeansInitialization.java @@ -22,7 +22,6 @@ package de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans; You should have received a copy of the GNU Affero General Public License along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.LongParameter; @@ -34,9 +33,9 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.LongParameter; * * @param <V> Vector type */ -public abstract class AbstractKMeansInitialization<V extends NumberVector<V, ?>> implements KMeansInitialization<V> { +public abstract class AbstractKMeansInitialization<V> implements KMeansInitialization<V> { /** - * Holds the value of {@link KMeansLloyd#SEED_ID}. + * Holds the value of {@link KMeans#SEED_ID}. */ protected Long seed; @@ -56,13 +55,13 @@ public abstract class AbstractKMeansInitialization<V extends NumberVector<V, ?>> * * @apiviz.exclude */ - public abstract static class Parameterizer<V extends NumberVector<V, ?>> extends AbstractParameterizer { + public abstract static class Parameterizer<V> extends AbstractParameterizer { protected Long seed; @Override protected void makeOptions(Parameterization config) { super.makeOptions(config); - LongParameter seedP = new LongParameter(AbstractKMeans.SEED_ID, true); + LongParameter seedP = new LongParameter(KMeans.SEED_ID, true); if(config.grab(seedP)) { seed = seedP.getValue(); } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/FirstKInitialMeans.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/FirstKInitialMeans.java index 78ccd426..7a7f2867 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/FirstKInitialMeans.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/FirstKInitialMeans.java @@ -23,14 +23,16 @@ package de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans; along with this program. If not, see <http://www.gnu.org/licenses/>. */ import java.util.ArrayList; -import java.util.Iterator; import java.util.List; import de.lmu.ifi.dbs.elki.data.NumberVector; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; +import de.lmu.ifi.dbs.elki.database.ids.DBIDs; +import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDistanceFunction; -import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; /** @@ -40,20 +42,30 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; * * @param <V> Vector type */ -public class FirstKInitialMeans<V extends NumberVector<V, ?>> extends AbstractKMeansInitialization<V> { +public class FirstKInitialMeans<V> implements KMeansInitialization<V>, KMedoidsInitialization<V> { /** * Constructor. */ public FirstKInitialMeans() { - super(null); + super(); } @Override - public List<Vector> chooseInitialMeans(Relation<V> relation, int k, PrimitiveDistanceFunction<? super V, ?> distanceFunction) { - Iterator<DBID> iter = relation.iterDBIDs(); - List<Vector> means = new ArrayList<Vector>(k); - for(int i = 0; i < k && iter.hasNext(); i++) { - means.add(relation.get(iter.next()).getColumnVector()); + public List<V> chooseInitialMeans(Relation<V> relation, int k, PrimitiveDistanceFunction<? super V, ?> distanceFunction) { + DBIDIter iter = relation.iterDBIDs(); + List<V> means = new ArrayList<V>(k); + for(int i = 0; i < k && iter.valid(); i++, iter.advance()) { + means.add(relation.get(iter)); + } + return means; + } + + @Override + public DBIDs chooseInitialMedoids(int k, DistanceQuery<? super V, ?> distanceFunction) { + DBIDIter iter = distanceFunction.getRelation().iterDBIDs(); + ArrayModifiableDBIDs means = DBIDUtil.newArray(k); + for(int i = 0; i < k && iter.valid(); i++, iter.advance()) { + means.add(iter); } return means; } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMeans.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMeans.java new file mode 100644 index 00000000..37171d4a --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMeans.java @@ -0,0 +1,55 @@ +package de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans; + +import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +/** + * Some constants and options shared among kmeans family algorithms. + * + * @author Erich Schubert + */ +public interface KMeans { + /** + * Parameter to specify the initialization method + */ + public static final OptionID INIT_ID = OptionID.getOrCreateOptionID("kmeans.initialization", "Method to choose the initial means."); + + /** + * Parameter to specify the number of clusters to find, must be an integer + * greater than 0. + */ + public static final OptionID K_ID = OptionID.getOrCreateOptionID("kmeans.k", "The number of clusters to find."); + + /** + * Parameter to specify the number of clusters to find, must be an integer + * greater or equal to 0, where 0 means no limit. + */ + public static final OptionID MAXITER_ID = OptionID.getOrCreateOptionID("kmeans.maxiter", "The maximum number of iterations to do. 0 means no limit."); + + /** + * Parameter to specify the random generator seed. + */ + public static final OptionID SEED_ID = OptionID.getOrCreateOptionID("kmeans.seed", "The random number generator seed."); +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMeansInitialization.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMeansInitialization.java index f4c0d9c7..9e5d69f0 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMeansInitialization.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMeansInitialization.java @@ -24,19 +24,17 @@ package de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans; */ import java.util.List; -import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDistanceFunction; -import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; /** * Interface for initializing K-Means * * @author Erich Schubert * - * @param <V> Vector type + * @param <V> Object type */ -public interface KMeansInitialization<V extends NumberVector<V, ?>> { +public interface KMeansInitialization<V> { /** * Choose initial means * @@ -45,5 +43,5 @@ public interface KMeansInitialization<V extends NumberVector<V, ?>> { * @param distanceFunction Distance function * @return List of chosen means for k-means */ - public abstract List<Vector> chooseInitialMeans(Relation<V> relation, int k, PrimitiveDistanceFunction<? super V, ?> distanceFunction); + public abstract List<V> chooseInitialMeans(Relation<V> relation, int k, PrimitiveDistanceFunction<? super V, ?> distanceFunction); }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMeansLloyd.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMeansLloyd.java index fda1d6c0..b1b40632 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMeansLloyd.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMeansLloyd.java @@ -39,7 +39,6 @@ import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDistanceFunction; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; import de.lmu.ifi.dbs.elki.logging.Logging; -import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; import de.lmu.ifi.dbs.elki.utilities.documentation.Description; import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; @@ -94,14 +93,13 @@ public class KMeansLloyd<V extends NumberVector<V, ?>, D extends Distance<D>> ex * @param database Database * @param relation relation to use * @return result - * @throws IllegalStateException */ - public Clustering<MeanModel<V>> run(Database database, Relation<V> relation) throws IllegalStateException { + public Clustering<MeanModel<V>> run(Database database, Relation<V> relation) { if(relation.size() <= 0) { return new Clustering<MeanModel<V>>("k-Means Clustering", "kmeans-clustering"); } // Choose initial means - List<Vector> means = initializer.chooseInitialMeans(relation, k, getDistanceFunction()); + List<? extends NumberVector<?, ?>> means = initializer.chooseInitialMeans(relation, k, getDistanceFunction()); // Setup cluster assignment store List<ModifiableDBIDs> clusters = new ArrayList<ModifiableDBIDs>(); for(int i = 0; i < k; i++) { @@ -124,7 +122,7 @@ public class KMeansLloyd<V extends NumberVector<V, ?>, D extends Distance<D>> ex final V factory = DatabaseUtil.assumeVectorField(relation).getFactory(); Clustering<MeanModel<V>> result = new Clustering<MeanModel<V>>("k-Means Clustering", "kmeans-clustering"); for(int i = 0; i < clusters.size(); i++) { - MeanModel<V> model = new MeanModel<V>(factory.newNumberVector(means.get(i).getArrayRef())); + MeanModel<V> model = new MeanModel<V>(factory.newNumberVector(means.get(i).getColumnVector().getArrayRef())); result.addCluster(new Cluster<MeanModel<V>>(clusters.get(i), model)); } return result; diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMeansMacQueen.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMeansMacQueen.java index 56492dd0..c729eb10 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMeansMacQueen.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMeansMacQueen.java @@ -93,15 +93,17 @@ public class KMeansMacQueen<V extends NumberVector<V, ?>, D extends Distance<D>> * * @param database Database * @param relation relation to use - * @return result - * @throws IllegalStateException + * @return Clustering result */ - public Clustering<MeanModel<V>> run(Database database, Relation<V> relation) throws IllegalStateException { + public Clustering<MeanModel<V>> run(Database database, Relation<V> relation) { if(relation.size() <= 0) { return new Clustering<MeanModel<V>>("k-Means Clustering", "kmeans-clustering"); } // Choose initial means - List<Vector> means = initializer.chooseInitialMeans(relation, k, getDistanceFunction()); + List<Vector> means = new ArrayList<Vector>(k); + for(NumberVector<?, ?> nv : initializer.chooseInitialMeans(relation, k, getDistanceFunction())) { + means.add(nv.getColumnVector()); + } // Initialize cluster and assign objects List<ModifiableDBIDs> clusters = new ArrayList<ModifiableDBIDs>(); for(int i = 0; i < k; i++) { diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMeansPlusPlusInitialMeans.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMeansPlusPlusInitialMeans.java index c7a2fa1d..9afeff6c 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMeansPlusPlusInitialMeans.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMeansPlusPlusInitialMeans.java @@ -26,19 +26,18 @@ import java.util.ArrayList; import java.util.List; import java.util.Random; -import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; +import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; -import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; +import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDistanceFunction; import de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDoubleDistanceFunction; import de.lmu.ifi.dbs.elki.distance.distancevalue.NumberDistance; import de.lmu.ifi.dbs.elki.logging.LoggingUtil; -import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; @@ -59,7 +58,7 @@ import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; * @param <D> Distance type */ @Reference(authors = "D. Arthur, S. Vassilvitskii", title = "k-means++: the advantages of careful seeding", booktitle = "Proc. of the Eighteenth Annual ACM-SIAM Symposium on Discrete Algorithms, SODA 2007", url = "http://dx.doi.org/10.1145/1283383.1283494") -public class KMeansPlusPlusInitialMeans<V extends NumberVector<V, ?>, D extends NumberDistance<D, ?>> extends AbstractKMeansInitialization<V> { +public class KMeansPlusPlusInitialMeans<V, D extends NumberDistance<D, ?>> extends AbstractKMeansInitialization<V> implements KMedoidsInitialization<V> { /** * Constructor. * @@ -70,7 +69,7 @@ public class KMeansPlusPlusInitialMeans<V extends NumberVector<V, ?>, D extends } @Override - public List<Vector> chooseInitialMeans(Relation<V> relation, int k, PrimitiveDistanceFunction<? super V, ?> distanceFunction) { + public List<V> chooseInitialMeans(Relation<V> relation, int k, PrimitiveDistanceFunction<? super V, ?> distanceFunction) { // Get a distance query if(!(distanceFunction.getDistanceFactory() instanceof NumberDistance)) { throw new AbortException("K-Means++ initialization can only be used with numerical distances."); @@ -80,14 +79,12 @@ public class KMeansPlusPlusInitialMeans<V extends NumberVector<V, ?>, D extends DistanceQuery<V, D> distQ = relation.getDatabase().getDistanceQuery(relation, distF); // Chose first mean - List<Vector> means = new ArrayList<Vector>(k); + List<V> means = new ArrayList<V>(k); Random random = (seed != null) ? new Random(seed) : new Random(); - DBID first = DBIDUtil.randomSample(relation.getDBIDs(), 1, random.nextLong()).iterator().next(); - means.add(relation.get(first).getColumnVector()); + DBID first = DBIDUtil.randomSample(relation.getDBIDs(), 1, random.nextLong()).iter().getDBID(); + means.add(relation.get(first)); - ModifiableDBIDs chosen = DBIDUtil.newHashSet(k); - chosen.add(first); ArrayDBIDs ids = DBIDUtil.ensureArray(relation.getDBIDs()); // Initialize weights double[] weights = new double[ids.size()]; @@ -107,16 +104,16 @@ public class KMeansPlusPlusInitialMeans<V extends NumberVector<V, ?>, D extends } // Add new mean: DBID newmean = ids.get(pos); - means.add(relation.get(newmean).getColumnVector()); - chosen.add(newmean); + means.add(relation.get(newmean)); // Update weights: weights[pos] = 0.0; // Choose optimized version for double distances, if applicable. - if (distF instanceof PrimitiveDoubleDistanceFunction) { + if(distF instanceof PrimitiveDoubleDistanceFunction) { @SuppressWarnings("unchecked") PrimitiveDoubleDistanceFunction<V> ddist = (PrimitiveDoubleDistanceFunction<V>) distF; weightsum = updateWeights(weights, ids, newmean, ddist, relation); - } else { + } + else { weightsum = updateWeights(weights, ids, newmean, distQ); } } @@ -124,6 +121,48 @@ public class KMeansPlusPlusInitialMeans<V extends NumberVector<V, ?>, D extends return means; } + @Override + public DBIDs chooseInitialMedoids(int k, DistanceQuery<? super V, ?> distQ2) { + if(!(distQ2.getDistanceFactory() instanceof NumberDistance)) { + throw new AbortException("PAM initialization can only be used with numerical distances."); + } + @SuppressWarnings("unchecked") + DistanceQuery<? super V, D> distQ = (DistanceQuery<? super V, D>) distQ2; + // Chose first mean + ArrayModifiableDBIDs means = DBIDUtil.newArray(k); + + Random random = (seed != null) ? new Random(seed) : new Random(); + DBID first = DBIDUtil.randomSample(distQ.getRelation().getDBIDs(), 1, random.nextLong()).iter().getDBID(); + means.add(first); + + ArrayDBIDs ids = DBIDUtil.ensureArray(distQ.getRelation().getDBIDs()); + // Initialize weights + double[] weights = new double[ids.size()]; + double weightsum = initialWeights(weights, ids, first, distQ); + while(means.size() < k) { + if(weightsum > Double.MAX_VALUE) { + LoggingUtil.warning("Could not choose a reasonable mean for k-means++ - too many data points, too large squared distances?"); + } + if(weightsum < Double.MIN_NORMAL) { + LoggingUtil.warning("Could not choose a reasonable mean for k-means++ - to few data points?"); + } + double r = random.nextDouble() * weightsum; + int pos = 0; + while(r > 0 && pos < weights.length) { + r -= weights[pos]; + pos++; + } + // Add new mean: + DBID newmean = ids.get(pos); + means.add(newmean); + // Update weights: + weights[pos] = 0.0; + weightsum = updateWeights(weights, ids, newmean, distQ); + } + + return means; + } + /** * Initialize the weight list. * @@ -133,16 +172,15 @@ public class KMeansPlusPlusInitialMeans<V extends NumberVector<V, ?>, D extends * @param distQ Distance query * @return Weight sum */ - protected double initialWeights(double[] weights, ArrayDBIDs ids, DBID latest, DistanceQuery<V, D> distQ) { + protected double initialWeights(double[] weights, ArrayDBIDs ids, DBID latest, DistanceQuery<? super V, D> distQ) { double weightsum = 0.0; DBIDIter it = ids.iter(); for(int i = 0; i < weights.length; i++, it.advance()) { - DBID id = it.getDBID(); - if(latest.equals(id)) { + if(latest.sameDBID(it)) { weights[i] = 0.0; } else { - double d = distQ.distance(latest, id).doubleValue(); + double d = distQ.distance(latest, it).doubleValue(); weights[i] = d * d; } weightsum += weights[i]; @@ -159,13 +197,12 @@ public class KMeansPlusPlusInitialMeans<V extends NumberVector<V, ?>, D extends * @param distQ Distance query * @return Weight sum */ - protected double updateWeights(double[] weights, ArrayDBIDs ids, DBID latest, DistanceQuery<V, D> distQ) { + protected double updateWeights(double[] weights, ArrayDBIDs ids, DBID latest, DistanceQuery<? super V, D> distQ) { double weightsum = 0.0; DBIDIter it = ids.iter(); for(int i = 0; i < weights.length; i++, it.advance()) { - DBID id = it.getDBID(); if(weights[i] > 0.0) { - double d = distQ.distance(latest, id).doubleValue(); + double d = distQ.distance(latest, it).doubleValue(); weights[i] = Math.min(weights[i], d * d); weightsum += weights[i]; } @@ -187,9 +224,8 @@ public class KMeansPlusPlusInitialMeans<V extends NumberVector<V, ?>, D extends double weightsum = 0.0; DBIDIter it = ids.iter(); for(int i = 0; i < weights.length; i++, it.advance()) { - DBID id = it.getDBID(); if(weights[i] > 0.0) { - double d = distF.doubleDistance(lv, rel.get(id)); + double d = distF.doubleDistance(lv, rel.get(it)); weights[i] = Math.min(weights[i], d * d); weightsum += weights[i]; } @@ -204,7 +240,7 @@ public class KMeansPlusPlusInitialMeans<V extends NumberVector<V, ?>, D extends * * @apiviz.exclude */ - public static class Parameterizer<V extends NumberVector<V, ?>, D extends NumberDistance<D, ?>> extends AbstractKMeansInitialization.Parameterizer<V> { + public static class Parameterizer<V, D extends NumberDistance<D, ?>> extends AbstractKMeansInitialization.Parameterizer<V> { @Override protected KMeansPlusPlusInitialMeans<V, D> makeInstance() { return new KMeansPlusPlusInitialMeans<V, D>(seed); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMediansLloyd.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMediansLloyd.java new file mode 100644 index 00000000..8c284981 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMediansLloyd.java @@ -0,0 +1,172 @@ +package de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import java.util.ArrayList; +import java.util.List; + +import de.lmu.ifi.dbs.elki.algorithm.AbstractPrimitiveDistanceBasedAlgorithm; +import de.lmu.ifi.dbs.elki.algorithm.clustering.ClusteringAlgorithm; +import de.lmu.ifi.dbs.elki.data.Cluster; +import de.lmu.ifi.dbs.elki.data.Clustering; +import de.lmu.ifi.dbs.elki.data.NumberVector; +import de.lmu.ifi.dbs.elki.data.model.MeanModel; +import de.lmu.ifi.dbs.elki.database.Database; +import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; +import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; +import de.lmu.ifi.dbs.elki.database.relation.Relation; +import de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDistanceFunction; +import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; +import de.lmu.ifi.dbs.elki.logging.Logging; +import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; +import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; +import de.lmu.ifi.dbs.elki.utilities.documentation.Title; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.GreaterConstraint; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.GreaterEqualConstraint; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.IntParameter; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter; + +/** + * Provides the k-medians clustering algorithm, using Lloyd-style bulk + * iterations. + * + * Reference: + * <p> + * Clustering via Concave Minimization<br /> + * P. S. Bradley, O. L. Mangasarian, W. N. Street<br /> + * in: Advances in neural information processing systems + * </p> + * + * @author Erich Schubert + * + * @apiviz.has MeanModel + * + * @param <V> vector datatype + * @param <D> distance value type + */ +@Title("K-Medians") +@Reference(title = "Clustering via Concave Minimization", authors = "P. S. Bradley, O. L. Mangasarian, W. N. Street", booktitle = "Advances in neural information processing systems", url="http://nips.djvuzone.org/djvu/nips09/0368.djvu") +public class KMediansLloyd<V extends NumberVector<V, ?>, D extends Distance<D>> extends AbstractKMeans<V, D> implements ClusteringAlgorithm<Clustering<MeanModel<V>>> { + /** + * The logger for this class. + */ + private static final Logging logger = Logging.getLogger(KMediansLloyd.class); + + /** + * Constructor. + * + * @param distanceFunction distance function + * @param k k parameter + * @param maxiter Maxiter parameter + */ + public KMediansLloyd(PrimitiveDistanceFunction<NumberVector<?, ?>, D> distanceFunction, int k, int maxiter, KMeansInitialization<V> initializer) { + super(distanceFunction, k, maxiter, initializer); + } + + /** + * Run k-medians + * + * @param database Database + * @param relation relation to use + * @return result + */ + public Clustering<MeanModel<V>> run(Database database, Relation<V> relation) { + if(relation.size() <= 0) { + return new Clustering<MeanModel<V>>("k-Medians Clustering", "kmedians-clustering"); + } + // Choose initial medians + List<? extends NumberVector<?, ?>> medians = initializer.chooseInitialMeans(relation, k, getDistanceFunction()); + // Setup cluster assignment store + List<ModifiableDBIDs> clusters = new ArrayList<ModifiableDBIDs>(); + for(int i = 0; i < k; i++) { + clusters.add(DBIDUtil.newHashSet(relation.size() / k)); + } + + for(int iteration = 0; maxiter <= 0 || iteration < maxiter; iteration++) { + if(logger.isVerbose()) { + logger.verbose("K-Medians iteration " + (iteration + 1)); + } + boolean changed = assignToNearestCluster(relation, medians, clusters); + // Stop if no cluster assignment changed. + if(!changed) { + break; + } + // Recompute medians. + medians = medians(clusters, medians, relation); + } + // Wrap result + final V factory = DatabaseUtil.assumeVectorField(relation).getFactory(); + Clustering<MeanModel<V>> result = new Clustering<MeanModel<V>>("k-Medians Clustering", "kmedians-clustering"); + for(int i = 0; i < clusters.size(); i++) { + MeanModel<V> model = new MeanModel<V>(factory.newNumberVector(medians.get(i).getColumnVector().getArrayRef())); + result.addCluster(new Cluster<MeanModel<V>>(clusters.get(i), model)); + } + return result; + } + + @Override + protected Logging getLogger() { + return logger; + } + + /** + * Parameterization class. + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer<V extends NumberVector<V, ?>, D extends Distance<D>> extends AbstractPrimitiveDistanceBasedAlgorithm.Parameterizer<NumberVector<?, ?>, D> { + protected int k; + + protected int maxiter; + + protected KMeansInitialization<V> initializer; + + @Override + protected void makeOptions(Parameterization config) { + super.makeOptions(config); + IntParameter kP = new IntParameter(K_ID, new GreaterConstraint(0)); + if(config.grab(kP)) { + k = kP.getValue(); + } + + ObjectParameter<KMeansInitialization<V>> initialP = new ObjectParameter<KMeansInitialization<V>>(INIT_ID, KMeansInitialization.class, RandomlyGeneratedInitialMeans.class); + if(config.grab(initialP)) { + initializer = initialP.instantiateClass(config); + } + + IntParameter maxiterP = new IntParameter(MAXITER_ID, new GreaterEqualConstraint(0), 0); + if(config.grab(maxiterP)) { + maxiter = maxiterP.getValue(); + } + } + + @Override + protected AbstractKMeans<V, D> makeInstance() { + return new KMediansLloyd<V, D>(distanceFunction, k, maxiter, initializer); + } + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMedoidsEM.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMedoidsEM.java new file mode 100644 index 00000000..a5c3d675 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMedoidsEM.java @@ -0,0 +1,271 @@ +package de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import java.util.ArrayList; +import java.util.List; + +import de.lmu.ifi.dbs.elki.algorithm.AbstractDistanceBasedAlgorithm; +import de.lmu.ifi.dbs.elki.algorithm.AbstractPrimitiveDistanceBasedAlgorithm; +import de.lmu.ifi.dbs.elki.algorithm.clustering.ClusteringAlgorithm; +import de.lmu.ifi.dbs.elki.data.Cluster; +import de.lmu.ifi.dbs.elki.data.Clustering; +import de.lmu.ifi.dbs.elki.data.model.MedoidModel; +import de.lmu.ifi.dbs.elki.data.type.TypeInformation; +import de.lmu.ifi.dbs.elki.data.type.TypeUtil; +import de.lmu.ifi.dbs.elki.database.Database; +import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; +import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; +import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; +import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; +import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; +import de.lmu.ifi.dbs.elki.database.relation.Relation; +import de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDistanceFunction; +import de.lmu.ifi.dbs.elki.distance.distancevalue.NumberDistance; +import de.lmu.ifi.dbs.elki.logging.Logging; +import de.lmu.ifi.dbs.elki.math.Mean; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.GreaterConstraint; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.GreaterEqualConstraint; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.IntParameter; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter; + +/** + * Provides the k-medoids clustering algorithm, using a "bulk" variation of the + * "Partitioning Around Medoids" approach. + * + * In contrast to PAM, which will in each iteration update one medoid with one + * (arbitrary) non-medoid, this implementation follows the EM pattern. In the + * expectation step, the best medoid from the cluster members is chosen; in the + * M-step, the objects are reassigned to their nearest medoid. + * + * We do not have a reference for this algorithm. It borrows ideas from EM and + * PAM. If needed, you are welcome cite it using the latest ELKI publication + * (this variation is likely not worth publishing on its own). + * + * @author Erich Schubert + * + * @apiviz.has MedoidModel + * @apiviz.composedOf KMedoidsInitialization + * + * @param <V> vector datatype + * @param <D> distance value type + */ +public class KMedoidsEM<V, D extends NumberDistance<D, ?>> extends AbstractDistanceBasedAlgorithm<V, D, Clustering<MedoidModel>> implements ClusteringAlgorithm<Clustering<MedoidModel>> { + /** + * The logger for this class. + */ + private static final Logging logger = Logging.getLogger(KMedoidsEM.class); + + /** + * Holds the value of {@link AbstractKMeans#K_ID}. + */ + protected int k; + + /** + * Holds the value of {@link AbstractKMeans#MAXITER_ID}. + */ + protected int maxiter; + + /** + * Method to choose initial means. + */ + protected KMedoidsInitialization<V> initializer; + + /** + * Constructor. + * + * @param distanceFunction distance function + * @param k k parameter + * @param maxiter Maxiter parameter + * @param initializer Function to generate the initial means + */ + public KMedoidsEM(PrimitiveDistanceFunction<? super V, D> distanceFunction, int k, int maxiter, KMedoidsInitialization<V> initializer) { + super(distanceFunction); + this.k = k; + this.maxiter = maxiter; + this.initializer = initializer; + } + + /** + * Run k-medoids + * + * @param database Database + * @param relation relation to use + * @return result + */ + public Clustering<MedoidModel> run(Database database, Relation<V> relation) { + if(relation.size() <= 0) { + return new Clustering<MedoidModel>("k-Medoids Clustering", "kmedoids-clustering"); + } + DistanceQuery<V, D> distQ = database.getDistanceQuery(relation, getDistanceFunction()); + // Choose initial medoids + ArrayModifiableDBIDs medoids = DBIDUtil.newArray(initializer.chooseInitialMedoids(k, distQ)); + // Setup cluster assignment store + List<ModifiableDBIDs> clusters = new ArrayList<ModifiableDBIDs>(); + for(int i = 0; i < k; i++) { + clusters.add(DBIDUtil.newHashSet(relation.size() / k)); + } + Mean[] mdists = Mean.newArray(k); + + // Initial assignment to nearest medoids + // TODO: reuse this information, from the build phase, when possible? + assignToNearestCluster(medoids, mdists, clusters, distQ); + + // Swap phase + boolean changed = true; + while(changed) { + changed = false; + // Try to swap the medoid with a better cluster member: + for(int i = 0; i < k; i++) { + DBID med = medoids.get(i); + DBID best = null; + Mean bestm = mdists[i]; + for(DBIDIter iter = clusters.get(i).iter(); iter.valid(); iter.advance()) { + if(med.sameDBID(iter)) { + continue; + } + Mean mdist = new Mean(); + for(DBIDIter iter2 = clusters.get(i).iter(); iter2.valid(); iter2.advance()) { + mdist.put(distQ.distance(iter, iter2).doubleValue()); + } + if(mdist.getMean() < bestm.getMean()) { + best = iter.getDBID(); + bestm = mdist; + } + } + if(best != null && !med.sameDBID(best)) { + changed = true; + medoids.set(i, best); + mdists[i] = bestm; + } + } + // Reassign + if(changed) { + assignToNearestCluster(medoids, mdists, clusters, distQ); + } + } + + // Wrap result + Clustering<MedoidModel> result = new Clustering<MedoidModel>("k-Medoids Clustering", "kmedoids-clustering"); + for(int i = 0; i < clusters.size(); i++) { + MedoidModel model = new MedoidModel(medoids.get(i)); + result.addCluster(new Cluster<MedoidModel>(clusters.get(i), model)); + } + return result; + } + + /** + * Returns a list of clusters. The k<sup>th</sup> cluster contains the ids of + * those FeatureVectors, that are nearest to the k<sup>th</sup> mean. + * + * @param means a list of k means + * @param mdist Mean distances + * @param clusters cluster assignment + * @param distQ distance query + * @return true when the object was reassigned + */ + protected boolean assignToNearestCluster(ArrayDBIDs means, Mean[] mdist, List<? extends ModifiableDBIDs> clusters, DistanceQuery<V, D> distQ) { + boolean changed = false; + + double[] dists = new double[k]; + for(DBIDIter iditer = distQ.getRelation().iterDBIDs(); iditer.valid(); iditer.advance()) { + int minIndex = 0; + double mindist = Double.POSITIVE_INFINITY; + for(int i = 0; i < k; i++) { + dists[i] = distQ.distance(iditer, means.get(i)).doubleValue(); + if(dists[i] < mindist) { + minIndex = i; + mindist = dists[i]; + } + } + if(clusters.get(minIndex).add(iditer)) { + changed = true; + mdist[minIndex].put(mindist); + // Remove from previous cluster + // TODO: keep a list of cluster assignments to save this search? + for(int i = 0; i < k; i++) { + if(i != minIndex) { + if(clusters.get(i).remove(iditer)) { + mdist[minIndex].put(dists[i], -1); + break; + } + } + } + } + } + return changed; + } + + @Override + public TypeInformation[] getInputTypeRestriction() { + return TypeUtil.array(getDistanceFunction().getInputTypeRestriction()); + } + + @Override + protected Logging getLogger() { + return logger; + } + + /** + * Parameterization class. + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer<V, D extends NumberDistance<D, ?>> extends AbstractPrimitiveDistanceBasedAlgorithm.Parameterizer<V, D> { + protected int k; + + protected int maxiter; + + protected KMedoidsInitialization<V> initializer; + + @Override + protected void makeOptions(Parameterization config) { + super.makeOptions(config); + IntParameter kP = new IntParameter(KMeans.K_ID, new GreaterConstraint(0)); + if(config.grab(kP)) { + k = kP.getValue(); + } + + ObjectParameter<KMedoidsInitialization<V>> initialP = new ObjectParameter<KMedoidsInitialization<V>>(KMeans.INIT_ID, KMedoidsInitialization.class, PAMInitialMeans.class); + if(config.grab(initialP)) { + initializer = initialP.instantiateClass(config); + } + + IntParameter maxiterP = new IntParameter(KMeans.MAXITER_ID, new GreaterEqualConstraint(0), 0); + if(config.grab(maxiterP)) { + maxiter = maxiterP.getValue(); + } + } + + @Override + protected KMedoidsEM<V, D> makeInstance() { + return new KMedoidsEM<V, D>(distanceFunction, k, maxiter, initializer); + } + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMedoidsInitialization.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMedoidsInitialization.java new file mode 100644 index 00000000..269e7e9e --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMedoidsInitialization.java @@ -0,0 +1,45 @@ +package de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans; +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import de.lmu.ifi.dbs.elki.database.ids.DBIDs; +import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; + +/** + * Interface for initializing K-Medoids. In contrast to k-means initializers, + * this initialization will only return members of the original data set. + * + * @author Erich Schubert + * + * @param <V> Object type + */ +public interface KMedoidsInitialization<V> { + /** + * Choose initial means + * + * @param k Parameter k + * @param distanceFunction Distance function + * @return List of chosen means for k-means + */ + public abstract DBIDs chooseInitialMedoids(int k, DistanceQuery<? super V, ?> distanceFunction); +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMedoidsPAM.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMedoidsPAM.java new file mode 100644 index 00000000..30c80084 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/KMedoidsPAM.java @@ -0,0 +1,310 @@ +package de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import java.util.ArrayList; +import java.util.List; + +import de.lmu.ifi.dbs.elki.algorithm.AbstractDistanceBasedAlgorithm; +import de.lmu.ifi.dbs.elki.algorithm.AbstractPrimitiveDistanceBasedAlgorithm; +import de.lmu.ifi.dbs.elki.algorithm.clustering.ClusteringAlgorithm; +import de.lmu.ifi.dbs.elki.data.Cluster; +import de.lmu.ifi.dbs.elki.data.Clustering; +import de.lmu.ifi.dbs.elki.data.model.MedoidModel; +import de.lmu.ifi.dbs.elki.data.type.TypeInformation; +import de.lmu.ifi.dbs.elki.data.type.TypeUtil; +import de.lmu.ifi.dbs.elki.database.Database; +import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; +import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; +import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; +import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; +import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; +import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; +import de.lmu.ifi.dbs.elki.database.ids.DBIDs; +import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; +import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; +import de.lmu.ifi.dbs.elki.database.relation.Relation; +import de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDistanceFunction; +import de.lmu.ifi.dbs.elki.distance.distancevalue.NumberDistance; +import de.lmu.ifi.dbs.elki.logging.Logging; +import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; +import de.lmu.ifi.dbs.elki.utilities.documentation.Title; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.GreaterConstraint; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.GreaterEqualConstraint; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.IntParameter; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter; + +/** + * Provides the k-medoids clustering algorithm, using the + * "Partitioning Around Medoids" approach. + * + * Reference: + * <p> + * Clustering my means of Medoids<br /> + * Kaufman, L. and Rousseeuw, P.J.<br /> + * in: Statistical Data Analysis Based on the L_1–Norm and Related Methods + * </p> + * + * @author Erich Schubert + * + * @apiviz.has MedoidModel + * @apiviz.composedOf KMedoidsInitialization + * + * @param <V> vector datatype + * @param <D> distance value type + */ +@Title("Partioning Around Medoids") +@Reference(title = "Clustering my means of Medoids", authors = "Kaufman, L. and Rousseeuw, P.J.", booktitle = "Statistical Data Analysis Based on the L_1–Norm and Related Methods") +public class KMedoidsPAM<V, D extends NumberDistance<D, ?>> extends AbstractDistanceBasedAlgorithm<V, D, Clustering<MedoidModel>> implements ClusteringAlgorithm<Clustering<MedoidModel>> { + /** + * The logger for this class. + */ + private static final Logging logger = Logging.getLogger(KMedoidsPAM.class); + + /** + * Holds the value of {@link AbstractKMeans#K_ID}. + */ + protected int k; + + /** + * Holds the value of {@link AbstractKMeans#MAXITER_ID}. + */ + protected int maxiter; + + /** + * Method to choose initial means. + */ + protected KMedoidsInitialization<V> initializer; + + /** + * Constructor. + * + * @param distanceFunction distance function + * @param k k parameter + * @param maxiter Maxiter parameter + * @param initializer Function to generate the initial means + */ + public KMedoidsPAM(PrimitiveDistanceFunction<? super V, D> distanceFunction, int k, int maxiter, KMedoidsInitialization<V> initializer) { + super(distanceFunction); + this.k = k; + this.maxiter = maxiter; + this.initializer = initializer; + } + + /** + * Run k-medoids + * + * @param database Database + * @param relation relation to use + * @return result + */ + public Clustering<MedoidModel> run(Database database, Relation<V> relation) { + if(relation.size() <= 0) { + return new Clustering<MedoidModel>("k-Medoids Clustering", "kmedoids-clustering"); + } + DistanceQuery<V, D> distQ = database.getDistanceQuery(relation, getDistanceFunction()); + DBIDs ids = relation.getDBIDs(); + // Choose initial medoids + ArrayModifiableDBIDs medoids = DBIDUtil.newArray(initializer.chooseInitialMedoids(k, distQ)); + // Setup cluster assignment store + List<ModifiableDBIDs> clusters = new ArrayList<ModifiableDBIDs>(); + for(int i = 0; i < k; i++) { + clusters.add(DBIDUtil.newHashSet(relation.size() / k)); + } + + WritableDoubleDataStore second = DataStoreUtil.makeDoubleStorage(ids, DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_TEMP); + // Initial assignment to nearest medoids + // TODO: reuse this information, from the build phase, when possible? + assignToNearestCluster(medoids, ids, second, clusters, distQ); + + // Swap phase + boolean changed = true; + while(changed) { + changed = false; + // Try to swap the medoid with a better cluster member: + double best = 0; + DBID bestid = null; + int bestcluster = -1; + for(int i = 0; i < k; i++) { + DBID med = medoids.get(i); + for(DBIDIter iter = clusters.get(i).iter(); iter.valid(); iter.advance()) { + if(med.sameDBID(iter)) { + continue; + } + // double disti = distQ.distance(id, med).doubleValue(); + double cost = 0; + for(int j = 0; j < k; j++) { + for(DBIDIter iter2 = clusters.get(j).iter(); iter2.valid(); iter2.advance()) { + double distcur = distQ.distance(iter2, medoids.get(j)).doubleValue(); + double distnew = distQ.distance(iter2, iter).doubleValue(); + if(j == i) { + // Cases 1 and 2. + double distsec = second.doubleValue(iter2); + if(distcur > distsec) { + // Case 1, other would switch to a third medoid + cost += distsec - distcur; // Always positive! + } + else { // Would remain with the candidate + cost += distnew - distcur; // Could be negative + } + } + else { + // Cases 3-4: objects from other clusters + if (distcur < distnew) { + // Case 3: no change + } else { + // Case 4: would switch to new medoid + cost += distnew - distcur; // Always negative + } + } + } + } + if (cost < best) { + best = cost; + bestid = iter.getDBID(); + bestcluster = i; + } + } + } + if(logger.isDebugging()) { + logger.debug("Best cost: " + best); + } + if(bestid != null) { + changed = true; + medoids.set(bestcluster, bestid); + } + // Reassign + if(changed) { + // TODO: can we save some of these recomputations? + assignToNearestCluster(medoids, ids, second, clusters, distQ); + } + } + + // Wrap result + Clustering<MedoidModel> result = new Clustering<MedoidModel>("k-Medoids Clustering", "kmedoids-clustering"); + for(int i = 0; i < clusters.size(); i++) { + MedoidModel model = new MedoidModel(medoids.get(i)); + result.addCluster(new Cluster<MedoidModel>(clusters.get(i), model)); + } + return result; + } + + /** + * Returns a list of clusters. The k<sup>th</sup> cluster contains the ids of + * those FeatureVectors, that are nearest to the k<sup>th</sup> mean. + * + * @param means Object centroids + * @param ids Object ids + * @param second Distance to second nearest medoid + * @param clusters cluster assignment + * @param distQ distance query + * @return true when any object was reassigned + */ + protected boolean assignToNearestCluster(ArrayDBIDs means, DBIDs ids, WritableDoubleDataStore second, List<? extends ModifiableDBIDs> clusters, DistanceQuery<V, D> distQ) { + boolean changed = false; + + for(DBIDIter iditer = distQ.getRelation().iterDBIDs(); iditer.valid(); iditer.advance()) { + int minIndex = 0; + double mindist = Double.POSITIVE_INFINITY; + double mindist2 = Double.POSITIVE_INFINITY; + for(int i = 0; i < k; i++) { + double dist = distQ.distance(iditer, means.get(i)).doubleValue(); + if(dist < mindist) { + minIndex = i; + mindist2 = mindist; + mindist = dist; + } + else if(dist < mindist2) { + mindist2 = dist; + } + } + if(clusters.get(minIndex).add(iditer)) { + changed = true; + // Remove from previous cluster + // TODO: keep a list of cluster assignments to save this search? + for(int i = 0; i < k; i++) { + if(i != minIndex) { + if(clusters.get(i).remove(iditer)) { + break; + } + } + } + } + second.put(iditer, mindist2); + } + return changed; + } + + @Override + public TypeInformation[] getInputTypeRestriction() { + return TypeUtil.array(getDistanceFunction().getInputTypeRestriction()); + } + + @Override + protected Logging getLogger() { + return logger; + } + + /** + * Parameterization class. + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer<V, D extends NumberDistance<D, ?>> extends AbstractPrimitiveDistanceBasedAlgorithm.Parameterizer<V, D> { + protected int k; + + protected int maxiter; + + protected KMedoidsInitialization<V> initializer; + + @Override + protected void makeOptions(Parameterization config) { + super.makeOptions(config); + IntParameter kP = new IntParameter(KMeans.K_ID, new GreaterConstraint(0)); + if(config.grab(kP)) { + k = kP.getValue(); + } + + ObjectParameter<KMedoidsInitialization<V>> initialP = new ObjectParameter<KMedoidsInitialization<V>>(KMeans.INIT_ID, KMedoidsInitialization.class, PAMInitialMeans.class); + if(config.grab(initialP)) { + initializer = initialP.instantiateClass(config); + } + + IntParameter maxiterP = new IntParameter(KMeans.MAXITER_ID, new GreaterEqualConstraint(0), 0); + if(config.grab(maxiterP)) { + maxiter = maxiterP.getValue(); + } + } + + @Override + protected KMedoidsPAM<V, D> makeInstance() { + return new KMedoidsPAM<V, D>(distanceFunction, k, maxiter, initializer); + } + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/PAMInitialMeans.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/PAMInitialMeans.java new file mode 100644 index 00000000..094c37bb --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/PAMInitialMeans.java @@ -0,0 +1,187 @@ +package de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +import java.util.ArrayList; +import java.util.List; + +import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; +import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; +import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; +import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; +import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; +import de.lmu.ifi.dbs.elki.database.ids.DBIDs; +import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; +import de.lmu.ifi.dbs.elki.database.relation.Relation; +import de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDistanceFunction; +import de.lmu.ifi.dbs.elki.distance.distancevalue.NumberDistance; +import de.lmu.ifi.dbs.elki.math.Mean; +import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; +import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; + +/** + * PAM initialization for k-means (and of course, PAM). + * + * Reference: + * <p> + * Clustering my means of Medoids<br /> + * Kaufman, L. and Rousseeuw, P.J.<br /> + * in: Statistical Data Analysis Based on the L_1–Norm and Related Methods + * </p> + * + * TODO: enforce using a distance matrix? + * + * @author Erich Schubert + * + * @param <V> Vector type + * @param <D> Distance type + */ +@Reference(title = "Clustering my means of Medoids", authors = "Kaufman, L. and Rousseeuw, P.J.", booktitle = "Statistical Data Analysis Based on the L_1–Norm and Related Methods") +public class PAMInitialMeans<V, D extends NumberDistance<D, ?>> implements KMeansInitialization<V>, KMedoidsInitialization<V> { + /** + * Constructor. + */ + public PAMInitialMeans() { + super(); + } + + @Override + public List<V> chooseInitialMeans(Relation<V> relation, int k, PrimitiveDistanceFunction<? super V, ?> distanceFunction) { + // Get a distance query + if(!(distanceFunction.getDistanceFactory() instanceof NumberDistance)) { + throw new AbortException("PAM initialization can only be used with numerical distances."); + } + @SuppressWarnings("unchecked") + final PrimitiveDistanceFunction<? super V, D> distF = (PrimitiveDistanceFunction<? super V, D>) distanceFunction; + final DistanceQuery<V, D> distQ = relation.getDatabase().getDistanceQuery(relation, distF); + DBIDs medids = chooseInitialMedoids(k, distQ); + List<V> medoids = new ArrayList<V>(k); + for(DBIDIter iter = medids.iter(); iter.valid(); iter.advance()) { + medoids.add(relation.get(iter)); + } + return medoids; + } + + @Override + public DBIDs chooseInitialMedoids(int k, DistanceQuery<? super V, ?> distQ2) { + if(!(distQ2.getDistanceFactory() instanceof NumberDistance)) { + throw new AbortException("PAM initialization can only be used with numerical distances."); + } + @SuppressWarnings("unchecked") + DistanceQuery<? super V, D> distQ = (DistanceQuery<? super V, D>) distQ2; + final DBIDs ids = distQ.getRelation().getDBIDs(); + + ArrayModifiableDBIDs medids = DBIDUtil.newArray(k); + double best = Double.POSITIVE_INFINITY; + Mean mean = new Mean(); // Mean is numerically more stable than sum. + WritableDoubleDataStore mindist = null; + + // First mean is chosen by having the smallest distance sum to all others. + { + DBID bestid = null; + WritableDoubleDataStore bestd = null; + for(DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + WritableDoubleDataStore newd = DataStoreUtil.makeDoubleStorage(ids, DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_TEMP); + mean.reset(); + for(DBIDIter iter2 = ids.iter(); iter2.valid(); iter2.advance()) { + double d = distQ.distance(iter, iter2).doubleValue(); + mean.put(d); + newd.putDouble(iter2, d); + } + if(mean.getMean() < best) { + best = mean.getMean(); + bestid = iter.getDBID(); + if(bestd != null) { + bestd.destroy(); + } + bestd = newd; + } + else { + newd.destroy(); + } + } + medids.add(bestid); + mindist = bestd; + } + assert (mindist != null); + + // Subsequent means optimize the full criterion. + for(int i = 1; i < k; i++) { + DBID bestid = null; + WritableDoubleDataStore bestd = null; + for(DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); + if(medids.contains(id)) { + continue; + } + WritableDoubleDataStore newd = DataStoreUtil.makeDoubleStorage(ids, DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_TEMP); + mean.reset(); + for(DBIDIter iter2 = ids.iter(); iter2.valid(); iter2.advance()) { + DBID other = iter2.getDBID(); + double dn = distQ.distance(id, other).doubleValue(); + double v = Math.min(dn, mindist.doubleValue(other)); + mean.put(v); + newd.put(other, v); + } + assert (mean.getCount() == ids.size()); + if(mean.getMean() < best) { + best = mean.getMean(); + bestid = id; + if(bestd != null) { + bestd.destroy(); + } + bestd = newd; + } + else { + newd.destroy(); + } + } + if(bestid == null) { + throw new AbortException("No median found that improves the criterion function?!?"); + } + medids.add(bestid); + mindist.destroy(); + mindist = bestd; + } + + mindist.destroy(); + return medids; + } + + /** + * Parameterization class. + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer<V, D extends NumberDistance<D, ?>> extends AbstractParameterizer { + @Override + protected PAMInitialMeans<V, D> makeInstance() { + return new PAMInitialMeans<V, D>(); + } + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/RandomlyChosenInitialMeans.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/RandomlyChosenInitialMeans.java index 30e59453..5b9da923 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/RandomlyChosenInitialMeans.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/RandomlyChosenInitialMeans.java @@ -25,13 +25,12 @@ package de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans; import java.util.ArrayList; import java.util.List; -import de.lmu.ifi.dbs.elki.data.NumberVector; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; +import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDistanceFunction; -import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; /** * Initialize K-means by randomly choosing k exsiting elements as cluster @@ -41,7 +40,7 @@ import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; * * @param <V> Vector type */ -public class RandomlyChosenInitialMeans<V extends NumberVector<V, ?>> extends AbstractKMeansInitialization<V> { +public class RandomlyChosenInitialMeans<V> extends AbstractKMeansInitialization<V> implements KMedoidsInitialization<V> { /** * Constructor. * @@ -52,15 +51,20 @@ public class RandomlyChosenInitialMeans<V extends NumberVector<V, ?>> extends Ab } @Override - public List<Vector> chooseInitialMeans(Relation<V> relation, int k, PrimitiveDistanceFunction<? super V, ?> distanceFunction) { + public List<V> chooseInitialMeans(Relation<V> relation, int k, PrimitiveDistanceFunction<? super V, ?> distanceFunction) { DBIDs ids = DBIDUtil.randomSample(relation.getDBIDs(), k, seed); - List<Vector> means = new ArrayList<Vector>(k); - for(DBID id : ids) { - means.add(relation.get(id).getColumnVector()); + List<V> means = new ArrayList<V>(k); + for(DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + means.add(relation.get(iter)); } return means; } + @Override + public DBIDs chooseInitialMedoids(int k, DistanceQuery<? super V, ?> distanceFunction) { + return DBIDUtil.randomSample(distanceFunction.getRelation().getDBIDs(), k, seed); + } + /** * Parameterization class. * @@ -68,7 +72,7 @@ public class RandomlyChosenInitialMeans<V extends NumberVector<V, ?>> extends Ab * * @apiviz.exclude */ - public static class Parameterizer<V extends NumberVector<V, ?>> extends AbstractKMeansInitialization.Parameterizer<V> { + public static class Parameterizer<V> extends AbstractKMeansInitialization.Parameterizer<V> { @Override protected RandomlyChosenInitialMeans<V> makeInstance() { diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/RandomlyGeneratedInitialMeans.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/RandomlyGeneratedInitialMeans.java index e8a466dd..00ed08c4 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/RandomlyGeneratedInitialMeans.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/RandomlyGeneratedInitialMeans.java @@ -30,7 +30,6 @@ import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDistanceFunction; import de.lmu.ifi.dbs.elki.math.MathUtil; -import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; import de.lmu.ifi.dbs.elki.utilities.pairs.Pair; @@ -53,10 +52,10 @@ public class RandomlyGeneratedInitialMeans<V extends NumberVector<V, ?>> extends } @Override - public List<Vector> chooseInitialMeans(Relation<V> relation, int k, PrimitiveDistanceFunction<? super V, ?> distanceFunction) { + public List<V> chooseInitialMeans(Relation<V> relation, int k, PrimitiveDistanceFunction<? super V, ?> distanceFunction) { final int dim = DatabaseUtil.dimensionality(relation); Pair<V, V> minmax = DatabaseUtil.computeMinMax(relation); - List<Vector> means = new ArrayList<Vector>(k); + List<V> means = new ArrayList<V>(k); final Random random = (this.seed != null) ? new Random(this.seed) : new Random(); for(int i = 0; i < k; i++) { double[] r = MathUtil.randomDoubleArray(dim, random); @@ -64,12 +63,11 @@ public class RandomlyGeneratedInitialMeans<V extends NumberVector<V, ?>> extends for(int d = 0; d < dim; d++) { r[d] = minmax.first.doubleValue(d + 1) + (minmax.second.doubleValue(d + 1) - minmax.first.doubleValue(d + 1)) * r[d]; } - means.add(new Vector(r)); + means.add(minmax.first.newNumberVector(r)); } return means; } - /** * Parameterization class. * @@ -78,7 +76,6 @@ public class RandomlyGeneratedInitialMeans<V extends NumberVector<V, ?>> extends * @apiviz.exclude */ public static class Parameterizer<V extends NumberVector<V, ?>> extends AbstractKMeansInitialization.Parameterizer<V> { - @Override protected RandomlyGeneratedInitialMeans<V> makeInstance() { return new RandomlyGeneratedInitialMeans<V>(seed); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/CLIQUE.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/CLIQUE.java index e3b274a6..01a693e4 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/CLIQUE.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/CLIQUE.java @@ -27,14 +27,12 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; import de.lmu.ifi.dbs.elki.algorithm.AbstractAlgorithm; -import de.lmu.ifi.dbs.elki.algorithm.clustering.ClusteringAlgorithm; import de.lmu.ifi.dbs.elki.algorithm.clustering.subspace.clique.CLIQUESubspace; import de.lmu.ifi.dbs.elki.algorithm.clustering.subspace.clique.CLIQUEUnit; import de.lmu.ifi.dbs.elki.data.Cluster; @@ -46,6 +44,7 @@ import de.lmu.ifi.dbs.elki.data.model.SubspaceModel; import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.logging.Logging; @@ -97,7 +96,7 @@ import de.lmu.ifi.dbs.elki.utilities.pairs.Pair; @Title("CLIQUE: Automatic Subspace Clustering of High Dimensional Data for Data Mining Applications") @Description("Grid-based algorithm to identify dense clusters in subspaces of maximum dimensionality.") @Reference(authors = "R. Agrawal, J. Gehrke, D. Gunopulos, P. Raghavan", title = "Automatic Subspace Clustering of High Dimensional Data for Data Mining Applications", booktitle = "Proc. SIGMOD Conference, Seattle, WA, 1998", url = "http://dx.doi.org/10.1145/276304.276314") -public class CLIQUE<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Clustering<SubspaceModel<V>>> implements ClusteringAlgorithm<Clustering<SubspaceModel<V>>> { +public class CLIQUE<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Clustering<SubspaceModel<V>>> implements SubspaceClusteringAlgorithm<SubspaceModel<V>> { /** * The logger for this class. */ @@ -299,8 +298,8 @@ public class CLIQUE<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Clus minima[d] = Double.MAX_VALUE; } // update minima and maxima - for(Iterator<DBID> it = database.iterDBIDs(); it.hasNext();) { - V featureVector = database.get(it.next()); + for(DBIDIter it = database.iterDBIDs(); it.valid(); it.advance()) { + V featureVector = database.get(it); updateMinMax(featureVector, minima, maxima); } for(int i = 0; i < maxima.length; i++) { @@ -393,13 +392,15 @@ public class CLIQUE<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Clus // identify dense units double total = database.size(); - for(Iterator<DBID> it = database.iterDBIDs(); it.hasNext();) { - final DBID id = it.next(); - V featureVector = database.get(id); + for(DBIDIter it = database.iterDBIDs(); it.valid();) { + V featureVector = database.get(it); + final DBID id = it.getDBID(); + it.advance(); for(CLIQUEUnit<V> unit : units) { unit.addFeatureVector(id, featureVector); // unit is a dense unit - if(!it.hasNext() && unit.selectivity(total) >= tau) { + // FIXME: why it.valid()? + if(!it.valid() && unit.selectivity(total) >= tau) { denseUnits.add(unit); // add the dense unit to its subspace int dim = unit.getIntervals().iterator().next().getDimension(); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/DiSH.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/DiSH.java index c4c1687b..df3fe8b5 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/DiSH.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/DiSH.java @@ -34,7 +34,6 @@ import java.util.List; import java.util.Map; import de.lmu.ifi.dbs.elki.algorithm.AbstractAlgorithm; -import de.lmu.ifi.dbs.elki.algorithm.clustering.ClusteringAlgorithm; import de.lmu.ifi.dbs.elki.algorithm.clustering.OPTICS; import de.lmu.ifi.dbs.elki.data.Cluster; import de.lmu.ifi.dbs.elki.data.Clustering; @@ -100,7 +99,7 @@ import de.lmu.ifi.dbs.elki.utilities.pairs.Pair; @Title("DiSH: Detecting Subspace cluster Hierarchies") @Description("Algorithm to find hierarchical correlation clusters in subspaces.") @Reference(authors = "E. Achtert, C. Böhm, H.-P. Kriegel, P. Kröger, I. Müller-Gorman, A. Zimek", title = "Detection and Visualization of Subspace Cluster Hierarchies", booktitle = "Proc. 12th International Conference on Database Systems for Advanced Applications (DASFAA), Bangkok, Thailand, 2007", url = "http://dx.doi.org/10.1007/978-3-540-71703-4_15") -public class DiSH<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Clustering<SubspaceModel<V>>> implements ClusteringAlgorithm<Clustering<SubspaceModel<V>>> { +public class DiSH<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Clustering<SubspaceModel<V>>> implements SubspaceClusteringAlgorithm<SubspaceModel<V>> { /** * The logger for this class. */ @@ -162,8 +161,11 @@ public class DiSH<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Cluste /** * Performs the DiSH algorithm on the given database. + * + * @param database Database to process + * @param relation Relation to process */ - public Clustering<SubspaceModel<V>> run(Database database, Relation<V> relation) throws IllegalStateException { + public Clustering<SubspaceModel<V>> run(Database database, Relation<V> relation) { // Instantiate DiSH distance (and thus run the preprocessor) if(logger.isVerbose()) { logger.verbose("*** Run DiSH preprocessor."); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/PROCLUS.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/PROCLUS.java index 3f16e907..4eedbecd 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/PROCLUS.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/PROCLUS.java @@ -28,7 +28,6 @@ import java.util.BitSet; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Random; @@ -39,13 +38,13 @@ import de.lmu.ifi.dbs.elki.data.Cluster; import de.lmu.ifi.dbs.elki.data.Clustering; import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.data.Subspace; -import de.lmu.ifi.dbs.elki.data.model.Model; import de.lmu.ifi.dbs.elki.data.model.SubspaceModel; import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; @@ -87,11 +86,11 @@ import de.lmu.ifi.dbs.elki.utilities.pairs.Pair; * * @param <V> the type of NumberVector handled by this Algorithm */ +// TODO: optimize by creating much less objects @Title("PROCLUS: PROjected CLUStering") @Description("Algorithm to find subspace clusters in high dimensional spaces.") @Reference(authors = "C. C. Aggarwal, C. Procopiuc, J. L. Wolf, P. S. Yu, J. S. Park", title = "Fast Algorithms for Projected Clustering", booktitle = "Proc. ACM SIGMOD Int. Conf. on Management of Data (SIGMOD '99)", url = "http://dx.doi.org/10.1145/304181.304188") -// TODO: make the generics reflect the SubspaceModel -public class PROCLUS<V extends NumberVector<V, ?>> extends AbstractProjectedClustering<Clustering<Model>, V> { +public class PROCLUS<V extends NumberVector<V, ?>> extends AbstractProjectedClustering<Clustering<SubspaceModel<V>>, V> implements SubspaceClusteringAlgorithm<SubspaceModel<V>> { /** * The logger for this class. */ @@ -141,8 +140,11 @@ public class PROCLUS<V extends NumberVector<V, ?>> extends AbstractProjectedClus /** * Performs the PROCLUS algorithm on the given database. + * + * @param database Database to process + * @param relation Relation to process */ - public Clustering<Model> run(Database database, Relation<V> relation) throws IllegalStateException { + public Clustering<SubspaceModel<V>> run(Database database, Relation<V> relation) { DistanceQuery<V, DoubleDistance> distFunc = this.getDistanceQuery(database); RangeQuery<V, DoubleDistance> rangeQuery = database.getRangeQuery(distFunc); final Random random = new Random(); @@ -193,6 +195,7 @@ public class PROCLUS<V extends NumberVector<V, ?>> extends AbstractProjectedClus IndefiniteProgress cprogress = logger.isVerbose() ? new IndefiniteProgress("Current number of clusters:", logger) : null; + // TODO: Use DataStore and Trove for performance Map<DBID, PROCLUSCluster> clusters = null; int loops = 0; while(loops < 10) { @@ -229,9 +232,9 @@ public class PROCLUS<V extends NumberVector<V, ?>> extends AbstractProjectedClus // build result int numClusters = 1; - Clustering<Model> result = new Clustering<Model>("ProClus clustering", "proclus-clustering"); + Clustering<SubspaceModel<V>> result = new Clustering<SubspaceModel<V>>("ProClus clustering", "proclus-clustering"); for(PROCLUSCluster c : finalClusters) { - Cluster<Model> cluster = new Cluster<Model>(c.objectIDs); + Cluster<SubspaceModel<V>> cluster = new Cluster<SubspaceModel<V>>(c.objectIDs); cluster.setModel(new SubspaceModel<V>(new Subspace<V>(c.getDimensions()), c.centroid)); cluster.setName("cluster_" + numClusters++); @@ -262,7 +265,8 @@ public class PROCLUS<V extends NumberVector<V, ?>> extends AbstractProjectedClus // compute distances between each point in S and m_i Map<DBID, DistanceResultPair<DoubleDistance>> distances = new HashMap<DBID, DistanceResultPair<DoubleDistance>>(); - for(DBID id : s) { + for(DBIDIter iter = s.iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); DoubleDistance dist = distFunc.distance(id, m_i); distances.put(id, new GenericDistanceResultPair<DoubleDistance>(dist, id)); } @@ -278,7 +282,8 @@ public class PROCLUS<V extends NumberVector<V, ?>> extends AbstractProjectedClus distances.remove(m_i); // compute distances of each point to closest medoid - for(DBID id : s) { + for(DBIDIter iter = s.iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); DoubleDistance dist_new = distFunc.distance(id, m_i); DoubleDistance dist_old = distances.get(id).getDistance(); @@ -323,12 +328,11 @@ public class PROCLUS<V extends NumberVector<V, ?>> extends AbstractProjectedClus */ private ModifiableDBIDs computeM_current(DBIDs m, DBIDs m_best, DBIDs m_bad, Random random) { ArrayModifiableDBIDs m_list = DBIDUtil.newArray(m); - for(DBID m_i : m_best) { - m_list.remove(m_i); - } + m_list.removeDBIDs(m_best); ModifiableDBIDs m_current = DBIDUtil.newHashSet(); - for(DBID m_i : m_best) { + for(DBIDIter iter = m_best.iter(); iter.valid(); iter.advance()) { + DBID m_i = iter.getDBID(); if(m_bad.contains(m_i)) { int currentSize = m_current.size(); while(m_current.size() == currentSize) { @@ -358,11 +362,13 @@ public class PROCLUS<V extends NumberVector<V, ?>> extends AbstractProjectedClus private Map<DBID, List<DistanceResultPair<DoubleDistance>>> getLocalities(DBIDs medoids, Relation<V> database, DistanceQuery<V, DoubleDistance> distFunc, RangeQuery<V, DoubleDistance> rangeQuery) { Map<DBID, List<DistanceResultPair<DoubleDistance>>> result = new HashMap<DBID, List<DistanceResultPair<DoubleDistance>>>(); - for(DBID m : medoids) { + for(DBIDIter iter = medoids.iter(); iter.valid(); iter.advance()) { + DBID m = iter.getDBID(); // determine minimum distance between current medoid m and any other // medoid m_i DoubleDistance minDist = null; - for(DBID m_i : medoids) { + for(DBIDIter iter2 = medoids.iter(); iter2.valid(); iter2.advance()) { + DBID m_i = iter2.getDBID(); if(m_i == m) { continue; } @@ -399,7 +405,8 @@ public class PROCLUS<V extends NumberVector<V, ?>> extends AbstractProjectedClus int dim = DatabaseUtil.dimensionality(database); Map<DBID, double[]> averageDistances = new HashMap<DBID, double[]>(); - for(DBID m_i : medoids) { + for(DBIDIter iter = medoids.iter(); iter.valid(); iter.advance()) { + DBID m_i = iter.getDBID(); V medoid_i = database.get(m_i); List<DistanceResultPair<DoubleDistance>> l_i = localities.get(m_i); double[] x_i = new double[dim]; @@ -417,7 +424,8 @@ public class PROCLUS<V extends NumberVector<V, ?>> extends AbstractProjectedClus Map<DBID, Set<Integer>> dimensionMap = new HashMap<DBID, Set<Integer>>(); List<CTriple<Double, DBID, Integer>> z_ijs = new ArrayList<CTriple<Double, DBID, Integer>>(); - for(DBID m_i : medoids) { + for(DBIDIter iter = medoids.iter(); iter.valid(); iter.advance()) { + DBID m_i = iter.getDBID(); Set<Integer> dims_i = new HashSet<Integer>(); dimensionMap.put(m_i, dims_i); @@ -478,8 +486,8 @@ public class PROCLUS<V extends NumberVector<V, ?>> extends AbstractProjectedClus for(int i = 0; i < clusters.size(); i++) { PROCLUSCluster c_i = clusters.get(i); double[] x_i = new double[dim]; - for(DBID id : c_i.objectIDs) { - V o = database.get(id); + for(DBIDIter iter = c_i.objectIDs.iter(); iter.valid(); iter.advance()) { + V o = database.get(iter); for(int d = 0; d < dim; d++) { x_i[d] += Math.abs(c_i.centroid.doubleValue(d + 1) - o.doubleValue(d + 1)); } @@ -560,8 +568,8 @@ public class PROCLUS<V extends NumberVector<V, ?>> extends AbstractProjectedClus clusterIDs.put(m_i, DBIDUtil.newHashSet()); } - for(Iterator<DBID> it = database.iterDBIDs(); it.hasNext();) { - DBID p_id = it.next(); + for(DBIDIter it = database.iterDBIDs(); it.valid(); it.advance()) { + DBID p_id = it.getDBID(); V p = database.get(p_id); DistanceResultPair<DoubleDistance> minDist = null; for(DBID m_i : dimensions.keySet()) { @@ -610,8 +618,8 @@ public class PROCLUS<V extends NumberVector<V, ?>> extends AbstractProjectedClus clusterIDs.put(i, DBIDUtil.newHashSet()); } - for(Iterator<DBID> it = database.iterDBIDs(); it.hasNext();) { - DBID p_id = it.next(); + for(DBIDIter it = database.iterDBIDs(); it.valid(); it.advance()) { + DBID p_id = it.getDBID(); V p = database.get(p_id); Pair<DoubleDistance, Integer> minDist = null; for(int i = 0; i < dimensions.size(); i++) { @@ -707,8 +715,8 @@ public class PROCLUS<V extends NumberVector<V, ?>> extends AbstractProjectedClus */ private double avgDistance(V centroid, DBIDs objectIDs, Relation<V> database, int dimension) { double avg = 0; - for(DBID objectID : objectIDs) { - V o = database.get(objectID); + for(DBIDIter iter = objectIDs.iter(); iter.valid(); iter.advance()) { + V o = database.get(iter); avg += Math.abs(centroid.doubleValue(dimension) - o.doubleValue(dimension)); } return avg / objectIDs.size(); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/SUBCLU.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/SUBCLU.java index 963c0922..c47c74b6 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/SUBCLU.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/SUBCLU.java @@ -30,7 +30,6 @@ import java.util.List; import java.util.TreeMap; import de.lmu.ifi.dbs.elki.algorithm.AbstractAlgorithm; -import de.lmu.ifi.dbs.elki.algorithm.clustering.ClusteringAlgorithm; import de.lmu.ifi.dbs.elki.algorithm.clustering.DBSCAN; import de.lmu.ifi.dbs.elki.data.Cluster; import de.lmu.ifi.dbs.elki.data.Clustering; @@ -77,7 +76,7 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter; * @author Elke Achtert * * @apiviz.uses DBSCAN - * @apiviz.uses DimensionsSelectingEuclideanDistanceFunction + * @apiviz.uses AbstractDimensionsSelectingDoubleDistanceFunction * @apiviz.has SubspaceModel * * @param <V> the type of FeatureVector handled by this Algorithm @@ -85,7 +84,7 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter; @Title("SUBCLU: Density connected Subspace Clustering") @Description("Algorithm to detect arbitrarily shaped and positioned clusters in subspaces. SUBCLU delivers for each subspace the same clusters DBSCAN would have found, when applied to this subspace seperately.") @Reference(authors = "K. Kailing, H.-P. Kriegel, P. Kröger", title = "Density connected Subspace Clustering for High Dimensional Data. ", booktitle = "Proc. SIAM Int. Conf. on Data Mining (SDM'04), Lake Buena Vista, FL, 2004") -public class SUBCLU<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Clustering<SubspaceModel<V>>> implements ClusteringAlgorithm<Clustering<SubspaceModel<V>>> { +public class SUBCLU<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Clustering<SubspaceModel<V>>> implements SubspaceClusteringAlgorithm<SubspaceModel<V>> { /** * The logger for this class. */ @@ -162,7 +161,7 @@ public class SUBCLU<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Clus * @param relation Relation to process * @return Clustering result */ - public Clustering<SubspaceModel<V>> run(Relation<V> relation) throws IllegalStateException { + public Clustering<SubspaceModel<V>> run(Relation<V> relation) { final int dimensionality = DatabaseUtil.dimensionality(relation); StepProgress stepprog = logger.isVerbose() ? new StepProgress(dimensionality) : null; diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/SubspaceClusteringAlgorithm.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/SubspaceClusteringAlgorithm.java new file mode 100644 index 00000000..17eb3c19 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/SubspaceClusteringAlgorithm.java @@ -0,0 +1,39 @@ +package de.lmu.ifi.dbs.elki.algorithm.clustering.subspace; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +import de.lmu.ifi.dbs.elki.algorithm.clustering.ClusteringAlgorithm; +import de.lmu.ifi.dbs.elki.data.Clustering; +import de.lmu.ifi.dbs.elki.data.model.SubspaceModel; + +/** + * Interface for subspace clustering algorithms that use a model derived from + * {@link SubspaceModel}, that can then be post-processed for outlier detection. + * + * @author Erich Schubert + * + * @param <M> Model type + */ +public interface SubspaceClusteringAlgorithm<M extends SubspaceModel<?>> extends ClusteringAlgorithm<Clustering<M>> { + // No additional constraints +} diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/trivial/ByLabelClustering.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/trivial/ByLabelClustering.java index 43c6a218..ee42a59f 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/trivial/ByLabelClustering.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/trivial/ByLabelClustering.java @@ -39,7 +39,11 @@ import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; +import de.lmu.ifi.dbs.elki.database.ids.DBIDs; +import de.lmu.ifi.dbs.elki.database.ids.HashSetModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.logging.Logging; @@ -137,12 +141,12 @@ public class ByLabelClustering extends AbstractAlgorithm<Clustering<Model>> impl * @param relation The data input we use */ public Clustering<Model> run(Relation<?> relation) { - HashMap<String, ModifiableDBIDs> labelMap = multiple ? multipleAssignment(relation) : singleAssignment(relation); + HashMap<String, DBIDs> labelMap = multiple ? multipleAssignment(relation) : singleAssignment(relation); ModifiableDBIDs noiseids = DBIDUtil.newArray(); Clustering<Model> result = new Clustering<Model>("By Label Clustering", "bylabel-clustering"); - for(Entry<String, ModifiableDBIDs> entry : labelMap.entrySet()) { - ModifiableDBIDs ids = entry.getValue(); + for(Entry<String, DBIDs> entry : labelMap.entrySet()) { + DBIDs ids = entry.getValue(); if(ids.size() <= 1) { noiseids.addDBIDs(ids); continue; @@ -170,12 +174,13 @@ public class ByLabelClustering extends AbstractAlgorithm<Clustering<Model>> impl * @param data the database storing the objects * @return a mapping of labels to ids */ - private HashMap<String, ModifiableDBIDs> singleAssignment(Relation<?> data) { - HashMap<String, ModifiableDBIDs> labelMap = new HashMap<String, ModifiableDBIDs>(); + private HashMap<String, DBIDs> singleAssignment(Relation<?> data) { + HashMap<String, DBIDs> labelMap = new HashMap<String, DBIDs>(); - for(DBID id : data.iterDBIDs()) { - String label = data.get(id).toString(); - assign(labelMap, label, id); + for(DBIDIter iditer = data.iterDBIDs(); iditer.valid(); iditer.advance()) { + final Object val = data.get(iditer); + String label = (val != null) ? val.toString() : null; + assign(labelMap, label, iditer); } return labelMap; } @@ -187,13 +192,13 @@ public class ByLabelClustering extends AbstractAlgorithm<Clustering<Model>> impl * @param data the database storing the objects * @return a mapping of labels to ids */ - private HashMap<String, ModifiableDBIDs> multipleAssignment(Relation<?> data) { - HashMap<String, ModifiableDBIDs> labelMap = new HashMap<String, ModifiableDBIDs>(); + private HashMap<String, DBIDs> multipleAssignment(Relation<?> data) { + HashMap<String, DBIDs> labelMap = new HashMap<String, DBIDs>(); - for(DBID id : data.iterDBIDs()) { - String[] labels = data.get(id).toString().split(" "); + for(DBIDIter iditer = data.iterDBIDs(); iditer.valid(); iditer.advance()) { + String[] labels = data.get(iditer).toString().split(" "); for(String label : labels) { - assign(labelMap, label, id); + assign(labelMap, label, iditer); } } return labelMap; @@ -206,14 +211,22 @@ public class ByLabelClustering extends AbstractAlgorithm<Clustering<Model>> impl * @param label the label of the object to be assigned * @param id the id of the object to be assigned */ - private void assign(HashMap<String, ModifiableDBIDs> labelMap, String label, DBID id) { + private void assign(HashMap<String, DBIDs> labelMap, String label, DBIDRef id) { if(labelMap.containsKey(label)) { - labelMap.get(label).add(id); + DBIDs exist = labelMap.get(label); + if (exist instanceof DBID) { + ModifiableDBIDs n = DBIDUtil.newHashSet(); + n.add((DBID)exist); + n.add(id); + labelMap.put(label, n); + } else { + assert(exist instanceof HashSetModifiableDBIDs); + assert (exist.size() > 1); + ((ModifiableDBIDs)exist).add(id); + } } else { - ModifiableDBIDs n = DBIDUtil.newHashSet(); - n.add(id); - labelMap.put(label, n); + labelMap.put(label, id.getDBID()); } } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/trivial/ByLabelHierarchicalClustering.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/trivial/ByLabelHierarchicalClustering.java index 228cc7e7..26bf525a 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/trivial/ByLabelHierarchicalClustering.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/trivial/ByLabelHierarchicalClustering.java @@ -39,7 +39,11 @@ import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; +import de.lmu.ifi.dbs.elki.database.ids.DBIDs; +import de.lmu.ifi.dbs.elki.database.ids.HashSetModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.logging.Logging; @@ -96,28 +100,26 @@ public class ByLabelHierarchicalClustering extends AbstractAlgorithm<Clustering< * * @param relation The data input to use */ - public Clustering<Model> run(Relation<?> relation) throws IllegalStateException { - HashMap<String, ModifiableDBIDs> labelmap = new HashMap<String, ModifiableDBIDs>(); + public Clustering<Model> run(Relation<?> relation) { + HashMap<String, DBIDs> labelmap = new HashMap<String, DBIDs>(); ModifiableDBIDs noiseids = DBIDUtil.newArray(); - for(DBID id : relation.iterDBIDs()) { - String label = relation.get(id).toString(); - - if(labelmap.containsKey(label)) { - labelmap.get(label).add(id); - } - else { - ModifiableDBIDs n = DBIDUtil.newHashSet(); - n.add(id); - labelmap.put(label, n); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + final Object val = relation.get(iditer); + if(val == null) { + noiseids.add(iditer); + continue; } + String label = val.toString(); + + assign(labelmap, label, iditer); } ArrayList<Cluster<Model>> clusters = new ArrayList<Cluster<Model>>(labelmap.size()); - for(Entry<String, ModifiableDBIDs> entry : labelmap.entrySet()) { - ModifiableDBIDs ids = entry.getValue(); - if(ids.size() <= 1) { - noiseids.addDBIDs(ids); + for(Entry<String, DBIDs> entry : labelmap.entrySet()) { + DBIDs ids = entry.getValue(); + if(ids instanceof DBID) { + noiseids.add((DBID) ids); continue; } Cluster<Model> clus = new Cluster<Model>(entry.getKey(), ids, ClusterModel.CLUSTER, new ArrayList<Cluster<Model>>(), new ArrayList<Cluster<Model>>()); @@ -153,6 +155,33 @@ public class ByLabelHierarchicalClustering extends AbstractAlgorithm<Clustering< return new Clustering<Model>("By Label Hierarchical Clustering", "bylabel-clustering", rootclusters); } + /** + * Assigns the specified id to the labelMap according to its label + * + * @param labelMap the mapping of label to ids + * @param label the label of the object to be assigned + * @param id the id of the object to be assigned + */ + private void assign(HashMap<String, DBIDs> labelMap, String label, DBIDRef id) { + if(labelMap.containsKey(label)) { + DBIDs exist = labelMap.get(label); + if(exist instanceof DBID) { + ModifiableDBIDs n = DBIDUtil.newHashSet(); + n.add((DBID) exist); + n.add(id); + labelMap.put(label, n); + } + else { + assert (exist instanceof HashSetModifiableDBIDs); + assert (exist.size() > 1); + ((ModifiableDBIDs) exist).add(id); + } + } + else { + labelMap.put(label, id.getDBID()); + } + } + @Override public TypeInformation[] getInputTypeRestriction() { return TypeUtil.array(TypeUtil.GUESSED_LABEL); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/trivial/ByLabelOrAllInOneClustering.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/trivial/ByLabelOrAllInOneClustering.java new file mode 100644 index 00000000..f082db9c --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/trivial/ByLabelOrAllInOneClustering.java @@ -0,0 +1,74 @@ +package de.lmu.ifi.dbs.elki.algorithm.clustering.trivial; + +import de.lmu.ifi.dbs.elki.data.ClassLabel; +import de.lmu.ifi.dbs.elki.data.Cluster; +import de.lmu.ifi.dbs.elki.data.Clustering; +import de.lmu.ifi.dbs.elki.data.model.ClusterModel; +import de.lmu.ifi.dbs.elki.data.model.Model; +import de.lmu.ifi.dbs.elki.data.type.NoSupportedDataTypeException; +import de.lmu.ifi.dbs.elki.data.type.TypeUtil; +import de.lmu.ifi.dbs.elki.database.Database; +import de.lmu.ifi.dbs.elki.database.ids.DBIDs; +import de.lmu.ifi.dbs.elki.database.relation.Relation; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +/** + * Trivial class that will try to cluster by label, and fall back to an + * "all-in-one" clustering. + * + * @author Erich Schubert + */ +public class ByLabelOrAllInOneClustering extends ByLabelClustering { + /** + * Constructor. + */ + public ByLabelOrAllInOneClustering() { + super(); + } + + @Override + public Clustering<Model> run(Database database) { + // Prefer a true class label + try { + Relation<ClassLabel> relation = database.getRelation(TypeUtil.CLASSLABEL); + return run(relation); + } + catch(NoSupportedDataTypeException e) { + // Ignore. + } + try { + Relation<ClassLabel> relation = database.getRelation(TypeUtil.GUESSED_LABEL); + return run(relation); + } + catch(NoSupportedDataTypeException e) { + // Ignore. + } + final DBIDs ids = database.getRelation(TypeUtil.ANY).getDBIDs(); + Clustering<Model> result = new Clustering<Model>("All-in-one trivial Clustering", "allinone-clustering"); + Cluster<Model> c = new Cluster<Model>(ids, ClusterModel.CLUSTER); + result.addCluster(c); + return result; + } +} diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/trivial/ByModelClustering.java b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/trivial/ByModelClustering.java index cd45cda2..90ca3625 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/clustering/trivial/ByModelClustering.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/clustering/trivial/ByModelClustering.java @@ -35,7 +35,7 @@ import de.lmu.ifi.dbs.elki.data.model.Model; import de.lmu.ifi.dbs.elki.data.synthetic.bymodel.GeneratorInterface; import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.relation.Relation; @@ -102,14 +102,14 @@ public class ByModelClustering extends AbstractAlgorithm<Clustering<Model>> impl public Clustering<Model> run(Relation<Model> relation) { // Build model mapping HashMap<Model, ModifiableDBIDs> modelMap = new HashMap<Model, ModifiableDBIDs>(); - for(DBID id : relation.iterDBIDs()) { - Model model = relation.get(id); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + Model model = relation.get(iditer); ModifiableDBIDs modelids = modelMap.get(model); if(modelids == null) { modelids = DBIDUtil.newHashSet(); modelMap.put(model, modelids); } - modelids.add(id); + modelids.add(iditer); } Clustering<Model> result = new Clustering<Model>("By Model Clustering", "bymodel-clustering"); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/ABOD.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/ABOD.java index f0b31d32..88a62e38 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/ABOD.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/ABOD.java @@ -38,6 +38,8 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; @@ -186,20 +188,21 @@ public class ABOD<V extends NumberVector<V, ?>> extends AbstractDistanceBasedAlg assert (k == this.k); KNNQuery<V, DoubleDistance> knnQuery = QueryUtil.getKNNQuery(relation, getDistanceFunction(), k); - for(DBID objKey : relation.iterDBIDs()) { - MeanVariance s = new MeanVariance(); + MeanVariance s = new MeanVariance(); + for(DBIDIter objKey = relation.iterDBIDs(); objKey.valid(); objKey.advance()) { + s.reset(); // System.out.println("Processing: " +objKey); KNNResult<DoubleDistance> neighbors = knnQuery.getKNNForDBID(objKey, k); Iterator<DistanceResultPair<DoubleDistance>> iter = neighbors.iterator(); while(iter.hasNext()) { - DBID key1 = iter.next().getDBID(); + DistanceResultPair<DoubleDistance> key1 = iter.next(); // Iterator iter2 = data.keyIterator(); Iterator<DistanceResultPair<DoubleDistance>> iter2 = neighbors.iterator(); // PriorityQueue best = new PriorityQueue(false, k); while(iter2.hasNext()) { - DBID key2 = iter2.next().getDBID(); - if(key2.equals(key1) || key1.equals(objKey) || key2.equals(objKey)) { + DistanceResultPair<DoubleDistance> key2 = iter2.next(); + if(key2.sameDBID(key1) || key1.sameDBID(objKey) || key2.sameDBID(objKey)) { continue; } double nenner = calcDenominator(kernelMatrix, objKey, key1, key2); @@ -214,7 +217,7 @@ public class ABOD<V extends NumberVector<V, ?>> extends AbstractDistanceBasedAlg } // Sample variance probably would be correct, however the numerical // instabilities can actually break ABOD here. - pq.add(new DoubleObjPair<DBID>(s.getNaiveVariance(), objKey)); + pq.add(new DoubleObjPair<DBID>(s.getNaiveVariance(), objKey.getDBID())); } DoubleMinMax minmaxabod = new DoubleMinMax(); @@ -238,16 +241,18 @@ public class ABOD<V extends NumberVector<V, ?>> extends AbstractDistanceBasedAlg * @return result */ public OutlierResult getFastRanking(Relation<V> relation, int k, int sampleSize) { + final DBIDs ids = relation.getDBIDs(); // Fix a static set of IDs - staticids = DBIDUtil.newArray(relation.getDBIDs()); + // TODO: add a DBIDUtil.ensureSorted? + staticids = DBIDUtil.newArray(ids); staticids.sort(); KernelMatrix kernelMatrix = new KernelMatrix(primitiveKernelFunction, relation, staticids); Heap<DoubleObjPair<DBID>> pq = new Heap<DoubleObjPair<DBID>>(relation.size(), Collections.reverseOrder()); // get Candidate Ranking - for(DBID aKey : relation.iterDBIDs()) { - HashMap<DBID, Double> dists = new HashMap<DBID, Double>(relation.size()); + for(DBIDIter aKey = relation.iterDBIDs(); aKey.valid(); aKey.advance()) { + WritableDoubleDataStore dists = DataStoreUtil.makeDoubleStorage(ids, DataStoreFactory.HINT_TEMP | DataStoreFactory.HINT_HOT); // determine kNearestNeighbors and pairwise distances Heap<DoubleObjPair<DBID>> nn; if(!useRNDSample) { @@ -259,7 +264,7 @@ public class ABOD<V extends NumberVector<V, ?>> extends AbstractDistanceBasedAlg } // get normalization - double[] counter = calcFastNormalization(aKey, dists); + double[] counter = calcFastNormalization(aKey, dists, staticids); // System.out.println(counter[0] + " " + counter2[0] + " " + counter[1] + // " " + counter2[1]); // umsetzen von Pq zu list @@ -269,13 +274,14 @@ public class ABOD<V extends NumberVector<V, ?>> extends AbstractDistanceBasedAlg } // getFilter double var = getAbofFilter(kernelMatrix, aKey, dists, counter[1], counter[0], neighbors); - pq.add(new DoubleObjPair<DBID>(var, aKey)); + pq.add(new DoubleObjPair<DBID>(var, aKey.getDBID())); // System.out.println("prog "+(prog++)); } // refine Candidates Heap<DoubleObjPair<DBID>> resqueue = new Heap<DoubleObjPair<DBID>>(k); // System.out.println(pq.size() + " objects ordered into candidate list."); // int v = 0; + MeanVariance s = new MeanVariance(); while(!pq.isEmpty()) { if(resqueue.size() == k && pq.peek().first > resqueue.peek().first) { break; @@ -290,13 +296,13 @@ public class ABOD<V extends NumberVector<V, ?>> extends AbstractDistanceBasedAlg // + " worst result: " + Double.MAX_VALUE); // } // v++; - MeanVariance s = new MeanVariance(); - for(DBID bKey : relation.iterDBIDs()) { - if(bKey.equals(aKey)) { + s.reset(); + for(DBIDIter bKey = relation.iterDBIDs(); bKey.valid(); bKey.advance()) { + if(bKey.sameDBID(aKey)) { continue; } - for(DBID cKey : relation.iterDBIDs()) { - if(cKey.equals(aKey)) { + for(DBIDIter cKey = relation.iterDBIDs(); cKey.valid(); cKey.advance()) { + if(cKey.sameDBID(aKey)) { continue; } // double nenner = dists[y]*dists[z]; @@ -325,64 +331,60 @@ public class ABOD<V extends NumberVector<V, ?>> extends AbstractDistanceBasedAlg } // System.out.println(v + " Punkte von " + data.size() + " verfeinert !!"); DoubleMinMax minmaxabod = new DoubleMinMax(); - WritableDoubleDataStore abodvalues = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC); + WritableDoubleDataStore abodvalues = DataStoreUtil.makeDoubleStorage(ids, DataStoreFactory.HINT_STATIC); for(DoubleObjPair<DBID> pair : pq) { abodvalues.putDouble(pair.getSecond(), pair.first); minmaxabod.put(pair.first); } // Build result representation. - Relation<Double> scoreResult = new MaterializedRelation<Double>("Angle-based Outlier Detection", "abod-outlier", TypeUtil.DOUBLE, abodvalues, relation.getDBIDs()); + Relation<Double> scoreResult = new MaterializedRelation<Double>("Angle-based Outlier Detection", "abod-outlier", TypeUtil.DOUBLE, abodvalues, ids); OutlierScoreMeta scoreMeta = new InvertedOutlierScoreMeta(minmaxabod.getMin(), minmaxabod.getMax(), 0.0, Double.POSITIVE_INFINITY); return new OutlierResult(scoreMeta, scoreResult); } - private double[] calcFastNormalization(DBID x, HashMap<DBID, Double> dists) { + private double[] calcFastNormalization(DBIDRef x, WritableDoubleDataStore dists, DBIDs ids) { double[] result = new double[2]; double sum = 0; double sumF = 0; - for(DBID yKey : dists.keySet()) { - if(dists.get(yKey) != 0) { - double tmp = 1 / Math.sqrt(dists.get(yKey)); + for (DBIDIter yKey = ids.iter(); yKey.valid(); yKey.advance()) { + if(dists.doubleValue(yKey) != 0) { + double tmp = 1 / Math.sqrt(dists.doubleValue(yKey)); sum += tmp; - sumF += (1 / dists.get(yKey)) * tmp; + sumF += (1 / dists.doubleValue(yKey)) * tmp; } } double sofar = 0; double sofarF = 0; - for(DBID zKey : dists.keySet()) { - if(dists.get(zKey) != 0) { - double tmp = 1 / Math.sqrt(dists.get(zKey)); + for (DBIDIter zKey = ids.iter(); zKey.valid(); zKey.advance()) { + if(dists.doubleValue(zKey) != 0) { + double tmp = 1 / Math.sqrt(dists.doubleValue(zKey)); sofar += tmp; double rest = sum - sofar; result[0] += tmp * rest; - sofarF += (1 / dists.get(zKey)) * tmp; + sofarF += (1 / dists.doubleValue(zKey)) * tmp; double restF = sumF - sofarF; - result[1] += (1 / dists.get(zKey)) * tmp * restF; + result[1] += (1 / dists.doubleValue(zKey)) * tmp * restF; } } return result; } - private double getAbofFilter(KernelMatrix kernelMatrix, DBID aKey, HashMap<DBID, Double> dists, double fulCounter, double counter, DBIDs neighbors) { + private double getAbofFilter(KernelMatrix kernelMatrix, DBIDRef aKey, WritableDoubleDataStore dists, double fulCounter, double counter, DBIDs neighbors) { double sum = 0.0; double sqrSum = 0.0; double partCounter = 0; - Iterator<DBID> iter = neighbors.iterator(); - while(iter.hasNext()) { - DBID bKey = iter.next(); - if(bKey.equals(aKey)) { + for(DBIDIter bKey = neighbors.iter(); bKey.valid(); bKey.advance()) { + if(bKey.sameDBID(aKey)) { continue; } - Iterator<DBID> iter2 = neighbors.iterator(); - while(iter2.hasNext()) { - DBID cKey = iter2.next(); - if(cKey.equals(aKey)) { + for(DBIDIter cKey = neighbors.iter(); cKey.valid(); cKey.advance()) { + if(cKey.sameDBID(aKey)) { continue; } - if(bKey.compareTo(cKey) > 0) { - double nenner = dists.get(bKey).doubleValue() * dists.get(cKey).doubleValue(); + if(bKey.compareDBID(cKey) > 0) { + double nenner = dists.doubleValue(bKey) * dists.doubleValue(cKey); if(nenner != 0) { double tmp = calcNumerator(kernelMatrix, aKey, bKey, cKey) / nenner; double sqrtNenner = Math.sqrt(nenner); @@ -406,13 +408,13 @@ public class ABOD<V extends NumberVector<V, ?>> extends AbstractDistanceBasedAlg * @param bKey * @return cosinus value */ - private double calcCos(KernelMatrix kernelMatrix, DBID aKey, DBID bKey) { + private double calcCos(KernelMatrix kernelMatrix, DBIDRef aKey, DBIDRef bKey) { final int ai = mapDBID(aKey); final int bi = mapDBID(bKey); return kernelMatrix.getDistance(ai, ai) + kernelMatrix.getDistance(bi, bi) - 2 * kernelMatrix.getDistance(ai, bi); } - private int mapDBID(DBID aKey) { + private int mapDBID(DBIDRef aKey) { // TODO: this is not the most efficient... int off = staticids.binarySearch(aKey); if(off < 0) { @@ -421,44 +423,44 @@ public class ABOD<V extends NumberVector<V, ?>> extends AbstractDistanceBasedAlg return off + 1; } - private double calcDenominator(KernelMatrix kernelMatrix, DBID aKey, DBID bKey, DBID cKey) { + private double calcDenominator(KernelMatrix kernelMatrix, DBIDRef aKey, DBIDRef bKey, DBIDRef cKey) { return calcCos(kernelMatrix, aKey, bKey) * calcCos(kernelMatrix, aKey, cKey); } - private double calcNumerator(KernelMatrix kernelMatrix, DBID aKey, DBID bKey, DBID cKey) { + private double calcNumerator(KernelMatrix kernelMatrix, DBIDRef aKey, DBIDRef bKey, DBIDRef cKey) { final int ai = mapDBID(aKey); final int bi = mapDBID(bKey); final int ci = mapDBID(cKey); return (kernelMatrix.getDistance(ai, ai) + kernelMatrix.getDistance(bi, ci) - kernelMatrix.getDistance(ai, ci) - kernelMatrix.getDistance(ai, bi)); } - private Heap<DoubleObjPair<DBID>> calcDistsandNN(Relation<V> data, KernelMatrix kernelMatrix, int sampleSize, DBID aKey, HashMap<DBID, Double> dists) { + private Heap<DoubleObjPair<DBID>> calcDistsandNN(Relation<V> data, KernelMatrix kernelMatrix, int sampleSize, DBIDRef aKey, WritableDoubleDataStore dists) { Heap<DoubleObjPair<DBID>> nn = new Heap<DoubleObjPair<DBID>>(sampleSize); - for(DBID bKey : data.iterDBIDs()) { + for(DBIDIter bKey = data.iterDBIDs(); bKey.valid(); bKey.advance()) { double val = calcCos(kernelMatrix, aKey, bKey); - dists.put(bKey, val); + dists.putDouble(bKey, val); if(nn.size() < sampleSize) { - nn.add(new DoubleObjPair<DBID>(val, bKey)); + nn.add(new DoubleObjPair<DBID>(val, bKey.getDBID())); } else { if(val < nn.peek().first) { nn.remove(); - nn.add(new DoubleObjPair<DBID>(val, bKey)); + nn.add(new DoubleObjPair<DBID>(val, bKey.getDBID())); } } } return nn; } - private Heap<DoubleObjPair<DBID>> calcDistsandRNDSample(Relation<V> data, KernelMatrix kernelMatrix, int sampleSize, DBID aKey, HashMap<DBID, Double> dists) { + private Heap<DoubleObjPair<DBID>> calcDistsandRNDSample(Relation<V> data, KernelMatrix kernelMatrix, int sampleSize, DBIDRef aKey, WritableDoubleDataStore dists) { Heap<DoubleObjPair<DBID>> nn = new Heap<DoubleObjPair<DBID>>(sampleSize); int step = (int) ((double) data.size() / (double) sampleSize); int counter = 0; - for(DBID bKey : data.iterDBIDs()) { + for(DBIDIter bKey = data.iterDBIDs(); bKey.valid(); bKey.advance()) { double val = calcCos(kernelMatrix, aKey, bKey); - dists.put(bKey, val); + dists.putDouble(bKey, val); if(counter % step == 0) { - nn.add(new DoubleObjPair<DBID>(val, bKey)); + nn.add(new DoubleObjPair<DBID>(val, bKey.getDBID())); } counter++; } @@ -477,24 +479,21 @@ public class ABOD<V extends NumberVector<V, ?>> extends AbstractDistanceBasedAlg Heap<DoubleObjPair<DBID>> pq = new Heap<DoubleObjPair<DBID>>(data.size(), Collections.reverseOrder()); HashMap<DBID, DBIDs> explaintab = new HashMap<DBID, DBIDs>(); // test all objects - for(DBID objKey : data.iterDBIDs()) { - MeanVariance s = new MeanVariance(); + MeanVariance s = new MeanVariance(), s2 = new MeanVariance(); + for(DBIDIter objKey = data.iterDBIDs(); objKey.valid(); objKey.advance()) { + s.reset(); // Queue for the best explanation Heap<DoubleObjPair<DBID>> explain = new Heap<DoubleObjPair<DBID>>(); // determine Object // for each pair of other objects - Iterator<DBID> iter = data.iterDBIDs(); + for (DBIDIter key1 = data.iterDBIDs(); key1.valid(); key1.advance()) { // Collect Explanation Vectors - while(iter.hasNext()) { - MeanVariance s2 = new MeanVariance(); - DBID key1 = iter.next(); - Iterator<DBID> iter2 = data.iterDBIDs(); - if(objKey.equals(key1)) { + s2.reset(); + if(objKey.sameDBID(key1)) { continue; } - while(iter2.hasNext()) { - DBID key2 = iter2.next(); - if(key2.equals(key1) || objKey.equals(key2)) { + for (DBIDIter key2 = data.iterDBIDs(); key2.valid(); key2.advance()) { + if(key2.sameDBID(key1) || objKey.sameDBID(key2)) { continue; } double nenner = calcDenominator(kernelMatrix, objKey, key1, key2); @@ -504,22 +503,22 @@ public class ABOD<V extends NumberVector<V, ?>> extends AbstractDistanceBasedAlg s2.put(tmp, 1 / sqr); } } - explain.add(new DoubleObjPair<DBID>(s2.getSampleVariance(), key1)); + explain.add(new DoubleObjPair<DBID>(s2.getSampleVariance(), key1.getDBID())); s.put(s2); } // build variance of the observed vectors - pq.add(new DoubleObjPair<DBID>(s.getSampleVariance(), objKey)); + pq.add(new DoubleObjPair<DBID>(s.getSampleVariance(), objKey.getDBID())); // ModifiableDBIDs expList = DBIDUtil.newArray(); expList.add(explain.remove().getSecond()); while(!explain.isEmpty()) { DBID nextKey = explain.remove().getSecond(); - if(nextKey.equals(objKey)) { + if(nextKey.sameDBID(objKey)) { continue; } double max = Double.MIN_VALUE; - for(DBID exp : expList) { - if(exp.equals(objKey) || nextKey.equals(exp)) { + for(DBIDIter exp = expList.iter(); exp.valid(); exp.advance()) { + if(exp.sameDBID(objKey) || nextKey.sameDBID(exp)) { continue; } double nenner = Math.sqrt(calcCos(kernelMatrix, objKey, nextKey)) * Math.sqrt(calcCos(kernelMatrix, objKey, exp)); @@ -530,7 +529,7 @@ public class ABOD<V extends NumberVector<V, ?>> extends AbstractDistanceBasedAlg expList.add(nextKey); } } - explaintab.put(objKey, expList); + explaintab.put(objKey.getDBID(), expList); } System.out.println("--------------------------------------------"); System.out.println("Result: ABOD"); @@ -552,10 +551,9 @@ public class ABOD<V extends NumberVector<V, ?>> extends AbstractDistanceBasedAlg private void generateExplanation(Relation<V> data, DBID key, DBIDs expList) { Vector vect1 = data.get(key).getColumnVector(); - Iterator<DBID> iter = expList.iterator(); - while(iter.hasNext()) { + for(DBIDIter iter = expList.iter(); iter.valid(); iter.advance()) { System.out.println("Outlier: " + vect1); - Vector exp = data.get(iter.next()).getColumnVector(); + Vector exp = data.get(iter).getColumnVector(); System.out.println("Most common neighbor: " + exp); // determine difference Vector Vector vals = exp.minus(vect1); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/ALOCI.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/ALOCI.java new file mode 100644 index 00000000..39c3db60 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/ALOCI.java @@ -0,0 +1,724 @@ +package de.lmu.ifi.dbs.elki.algorithm.outlier;
+
+/*
+ This file is part of ELKI:
+ Environment for Developing KDD-Applications Supported by Index-Structures
+
+ Copyright (C) 2012
+ Ludwig-Maximilians-Universität München
+ Lehr- und Forschungseinheit für Datenbanksysteme
+ ELKI Development Team
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import de.lmu.ifi.dbs.elki.algorithm.AbstractAlgorithm;
+import de.lmu.ifi.dbs.elki.data.NumberVector;
+import de.lmu.ifi.dbs.elki.data.type.TypeInformation;
+import de.lmu.ifi.dbs.elki.data.type.TypeUtil;
+import de.lmu.ifi.dbs.elki.database.Database;
+import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory;
+import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil;
+import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore;
+import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil;
+import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation;
+import de.lmu.ifi.dbs.elki.database.relation.Relation;
+import de.lmu.ifi.dbs.elki.distance.distancefunction.EuclideanDistanceFunction;
+import de.lmu.ifi.dbs.elki.distance.distancefunction.NumberVectorDistanceFunction;
+import de.lmu.ifi.dbs.elki.distance.distancevalue.NumberDistance;
+import de.lmu.ifi.dbs.elki.logging.Logging;
+import de.lmu.ifi.dbs.elki.logging.progress.FiniteProgress;
+import de.lmu.ifi.dbs.elki.math.DoubleMinMax;
+import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector;
+import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult;
+import de.lmu.ifi.dbs.elki.result.outlier.OutlierScoreMeta;
+import de.lmu.ifi.dbs.elki.result.outlier.QuotientOutlierScoreMeta;
+import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil;
+import de.lmu.ifi.dbs.elki.utilities.documentation.Description;
+import de.lmu.ifi.dbs.elki.utilities.documentation.Reference;
+import de.lmu.ifi.dbs.elki.utilities.documentation.Title;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.IntParameter;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.LongParameter;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter;
+import de.lmu.ifi.dbs.elki.utilities.pairs.Pair;
+
+/**
+ * Fast Outlier Detection Using the "approximate Local Correlation Integral".
+ *
+ * Outlier detection using multiple epsilon neighborhoods.
+ *
+ * Reference:
+ * <p>
+ * S. Papadimitriou, H. Kitagawa, P. B. Gibbons and C. Faloutsos:<br />
+ * LOCI: Fast Outlier Detection Using the Local Correlation Integral.<br />
+ * In: Proc. 19th IEEE Int. Conf. on Data Engineering (ICDE '03), Bangalore,
+ * India, 2003.
+ * </p>
+ *
+ * @author Jonathan von Brünken
+ * @author Erich Schubert
+ *
+ * @param <O> Object type
+ * @param <D> Distance type
+ */
+@Title("LOCI: Fast Outlier Detection Using the Local Correlation Integral")
+@Description("Algorithm to compute outliers based on the Local Correlation Integral")
+@Reference(authors = "S. Papadimitriou, H. Kitagawa, P. B. Gibbons, C. Faloutsos", title = "LOCI: Fast Outlier Detection Using the Local Correlation Integral", booktitle = "Proc. 19th IEEE Int. Conf. on Data Engineering (ICDE '03), Bangalore, India, 2003", url = "http://dx.doi.org/10.1109/ICDE.2003.1260802")
+public class ALOCI<O extends NumberVector<O, ?>, D extends NumberDistance<D, ?>> extends AbstractAlgorithm<OutlierResult> implements OutlierAlgorithm {
+ /**
+ * The logger for this class.
+ */
+ private static final Logging logger = Logging.getLogger(ALOCI.class);
+
+ /**
+ * Minimum size for a leaf.
+ */
+ private int nmin;
+
+ /**
+ * Alpha (level difference of sampling and counting neighborhoods)
+ */
+ private int alpha;
+
+ /**
+ * Number of trees to generate (forest size)
+ */
+ private int g;
+
+ /**
+ * Random generator
+ */
+ private Random random;
+
+ /**
+ * Distance function
+ */
+ private NumberVectorDistanceFunction<D> distFunc;
+
+ /**
+ * Constructor.
+ *
+ * @param distanceFunction Distance function
+ * @param nmin Minimum neighborhood size
+ * @param alpha Alpha value
+ * @param g Number of grids to use
+ * @param seed Random generator seed.
+ */
+ public ALOCI(NumberVectorDistanceFunction<D> distanceFunction, int nmin, int alpha, int g, Long seed) {
+ super();
+ this.distFunc = distanceFunction;
+ this.nmin = nmin;
+ this.alpha = alpha;
+ this.g = g;
+ this.random = (seed != null) ? new Random(seed) : new Random(0);
+ }
+
+ public OutlierResult run(Database database, Relation<O> relation) {
+ final int dim = DatabaseUtil.dimensionality(relation);
+ FiniteProgress progressPreproc = logger.isVerbose() ? new FiniteProgress("Build aLOCI quadtress", g, logger) : null;
+
+ // Compute extend of dataset.
+ double[] min, max;
+ {
+ Pair<O, O> hbbs = DatabaseUtil.computeMinMax(relation);
+ double maxd = 0;
+ min = new double[dim];
+ max = new double[dim];
+ for(int i = 0; i < dim; i++) {
+ min[i] = hbbs.first.doubleValue(i + 1);
+ max[i] = hbbs.second.doubleValue(i + 1);
+ maxd = Math.max(maxd, max[i] - min[i]);
+ }
+ // Enlarge bounding box to have equal lengths.
+ for(int i = 0; i < dim; i++) {
+ double diff = (maxd - (max[i] - min[i])) / 2;
+ min[i] -= diff;
+ max[i] += diff;
+ }
+ }
+
+ List<ALOCIQuadTree> qts = new ArrayList<ALOCIQuadTree>(g);
+
+ double[] nshift = new double[dim];
+ ALOCIQuadTree qt = new ALOCIQuadTree(min, max, nshift, nmin, relation);
+ qts.add(qt);
+ if(progressPreproc != null) {
+ progressPreproc.incrementProcessed(logger);
+ }
+ /*
+ * create the remaining g-1 shifted QuadTrees. This not clearly described in
+ * the paper and therefore implemented in a way that achieves good results
+ * with the test data.
+ */
+ for(int shift = 1; shift < g; shift++) {
+ double[] svec = new double[dim];
+ for(int i = 0; i < dim; i++) {
+ svec[i] = random.nextDouble() * (max[i] - min[i]);
+ }
+ qt = new ALOCIQuadTree(min, max, svec, nmin, relation);
+ qts.add(qt);
+ if(progressPreproc != null) {
+ progressPreproc.incrementProcessed(logger);
+ }
+ }
+ if(progressPreproc != null) {
+ progressPreproc.ensureCompleted(logger);
+ }
+
+ // aLOCI main loop: evaluate
+ FiniteProgress progressLOCI = logger.isVerbose() ? new FiniteProgress("Compute aLOCI scores", relation.size(), logger) : null;
+ WritableDoubleDataStore mdef_norm = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);
+ DoubleMinMax minmax = new DoubleMinMax();
+
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + final O obj = relation.get(iditer);
+
+ double maxmdefnorm = 0;
+ // For each level
+ for(int l = 0;; l++) {
+ // Find the closest C_i
+ Node ci = null;
+ for(int i = 0; i < g; i++) {
+ Node ci2 = qts.get(i).findClosestNode(obj, l);
+ if(ci2.getLevel() != l) {
+ continue;
+ }
+ // TODO: always use manhattan?
+ if(ci == null || distFunc.distance(ci.getCenter(), obj).compareTo(distFunc.distance(ci2.getCenter(), obj)) > 0) {
+ ci = ci2;
+ }
+ }
+ // logger.debug("level:" + (ci != null ? ci.getLevel() : -1) +" l:"+l);
+ if(ci == null) {
+ break; // no matching tree for this level.
+ }
+
+ // Find the closest C_j
+ Node cj = null;
+ for(int i = 0; i < g; i++) {
+ Node cj2 = qts.get(i).findClosestNode(ci.getCenter(), l - alpha);
+ // TODO: allow higher levels or not?
+ if(cj != null && cj2.getLevel() < cj.getLevel()) {
+ continue;
+ }
+ // TODO: always use manhattan?
+ if(cj == null || distFunc.distance(cj.getCenter(), ci.getCenter()).compareTo(distFunc.distance(cj2.getCenter(), ci.getCenter())) > 0) {
+ cj = cj2;
+ }
+ }
+ // logger.debug("level:" + (cj != null ? cj.getLevel() : -1) +" l:"+l);
+ if(cj == null) {
+ continue; // no matching tree for this level.
+ }
+ double mdefnorm = calculate_MDEF_norm(cj, ci);
+ // logger.warning("level:" + ci.getLevel() + "/" + cj.getLevel() +
+ // " mdef: " + mdefnorm);
+ maxmdefnorm = Math.max(maxmdefnorm, mdefnorm);
+ }
+ // Store results
+ mdef_norm.putDouble(iditer, maxmdefnorm);
+ minmax.put(maxmdefnorm);
+ if(progressLOCI != null) {
+ progressLOCI.incrementProcessed(logger);
+ }
+ }
+ if(progressLOCI != null) {
+ progressLOCI.ensureCompleted(logger);
+ }
+ Relation<Double> scoreResult = new MaterializedRelation<Double>("aLOCI normalized MDEF", "aloci-mdef-outlier", TypeUtil.DOUBLE, mdef_norm, relation.getDBIDs());
+ OutlierScoreMeta scoreMeta = new QuotientOutlierScoreMeta(minmax.getMin(), minmax.getMax(), 0.0, Double.POSITIVE_INFINITY);
+ OutlierResult result = new OutlierResult(scoreMeta, scoreResult);
+ return result;
+ }
+
+ /**
+ * Method for the MDEF calculation
+ *
+ * @param sn Sampling Neighborhood
+ * @param cg Counting Neighborhood
+ *
+ * @return MDEF norm
+ */
+ private static double calculate_MDEF_norm(Node sn, Node cg) {
+ // get the square sum of the counting neighborhoods box counts
+ long sq = sn.getSquareSum(cg.getLevel() - sn.getLevel());
+ /*
+ * if the square sum is equal to box count of the sampling Neighborhood then
+ * n_hat is equal one, and as cg needs to have at least one Element mdef
+ * would get zero or lower than zero. This is the case when all of the
+ * counting Neighborhoods contain one or zero Objects. Additionally, the
+ * cubic sum, square sum and sampling Neighborhood box count are all equal,
+ * which leads to sig_n_hat being zero and thus mdef_norm is either negative
+ * infinite or undefined. As the distribution of the Objects seem quite
+ * uniform, a mdef_norm value of zero ( = no outlier) is appropriate and
+ * circumvents the problem of undefined values.
+ */
+ if(sq == sn.getCount()) {
+ return 0.0;
+ }
+ // calculation of mdef according to the paper and standardization as done in
+ // LOCI
+ long cb = sn.getCubicSum(cg.getLevel() - sn.getLevel());
+ double n_hat = (double) sq / sn.getCount();
+ double sig_n_hat = java.lang.Math.sqrt(cb * sn.getCount() - (sq * sq)) / sn.getCount();
+ // Avoid NaN - correct result 0.0?
+ if(sig_n_hat < Double.MIN_NORMAL) {
+ return 0.0;
+ }
+ double mdef = n_hat - cg.getCount();
+ return mdef / sig_n_hat;
+ }
+
+ @Override
+ protected Logging getLogger() {
+ return logger;
+ }
+
+ @Override
+ public TypeInformation[] getInputTypeRestriction() {
+ return TypeUtil.array(distFunc.getInputTypeRestriction());
+ }
+
+ /**
+ * Simple quadtree for ALOCI. Not storing the actual objects, just the counts.
+ *
+ * Furthermore, the quadtree can be shifted by a specified vector, wrapping
+ * around min/max
+ *
+ * @author Jonathan von Brünken
+ * @author Erich Schubert
+ *
+ * @apiviz.composedOf Node
+ */
+ static class ALOCIQuadTree {
+ /**
+ * Tree parameters
+ */
+ private double[] shift, min, width;
+
+ /**
+ * Maximum fill for a page before splitting
+ */
+ private int nmin;
+
+ /**
+ * Tree root
+ */
+ Node root;
+
+ /**
+ * Relation indexed.
+ */
+ private Relation<? extends NumberVector<?, ?>> relation;
+
+ /**
+ * Constructor.
+ *
+ * @param min Minimum coordinates
+ * @param max Maximum coordinates
+ * @param shift Tree shift offset
+ * @param nmin Maximum size for a page to split
+ * @param relation Relation to index
+ */
+ public ALOCIQuadTree(double[] min, double[] max, double[] shift, int nmin, Relation<? extends NumberVector<?, ?>> relation) {
+ super();
+ assert (min.length <= 32) : "Quadtrees are only supported for up to 32 dimensions";
+ this.shift = shift;
+ this.nmin = nmin;
+ this.min = min;
+ this.width = new double[min.length];
+ for(int d = 0; d < min.length; d++) {
+ width[d] = max[d] - min[d];
+ if(width[d] <= 0) {
+ width[d] = 1;
+ }
+ }
+ double[] center = new double[min.length];
+ for(int d = 0; d < min.length; d++) {
+ if(shift[d] < width[d] * .5) {
+ center[d] = min[d] + shift[d] + width[d] * .5;
+ }
+ else {
+ center[d] = min[d] + shift[d] - width[d] * .5;
+ }
+ }
+ this.relation = relation;
+ ArrayModifiableDBIDs ids = DBIDUtil.newArray(relation.getDBIDs());
+ List<Node> children = new ArrayList<Node>();
+ bulkLoad(min.clone(), max.clone(), children, ids, 0, ids.size(), 0, 0, 0);
+ this.root = new Node(0, new Vector(center), ids.size(), -1, children);
+ }
+
+ /**
+ * Bulk load the tree
+ *
+ * @param lmin Subtree minimum (unshifted, will be modified)
+ * @param lmax Subtree maximum (unshifted, will be modified)
+ * @param children List of children for current parent
+ * @param ids IDs to process
+ * @param start Start of ids subinterval
+ * @param end End of ids subinterval
+ * @param dim Current dimension
+ * @param level Current tree level
+ * @param code Bit code of node position
+ */
+ private void bulkLoad(double[] lmin, double[] lmax, List<Node> children, ArrayModifiableDBIDs ids, int start, int end, int dim, int level, int code) {
+ // logger.warning(FormatUtil.format(lmin)+" "+FormatUtil.format(lmax)+" "+start+"->"+end+" "+(end-start));
+ // Hack: Check degenerate cases that won't split
+ if(dim == 0) {
+ NumberVector<?, ?> first = relation.get(ids.get(start));
+ boolean degenerate = true;
+ loop: for(int pos = start + 1; pos < end; pos++) {
+ NumberVector<?, ?> other = relation.get(ids.get(pos));
+ for(int d = 1; d <= lmin.length; d++) {
+ if(Math.abs(first.doubleValue(d) - other.doubleValue(d)) > 1E-15) {
+ degenerate = false;
+ break loop;
+ }
+ }
+ }
+ if(degenerate) {
+ double[] center = new double[lmin.length];
+ for(int d = 0; d < lmin.length; d++) {
+ center[d] = lmin[d] * .5 + lmax[d] * .5 + shift[d];
+ if(center[d] > min[d] + width[d]) {
+ center[d] -= width[d];
+ }
+ }
+ children.add(new Node(code, new Vector(center), end - start, level, null));
+ return;
+ }
+ }
+ // Complete level
+ if(dim == lmin.length) {
+ double[] center = new double[lmin.length];
+ for(int d = 0; d < lmin.length; d++) {
+ center[d] = lmin[d] * .5 + lmax[d] * .5 + shift[d];
+ if(center[d] > min[d] + width[d]) {
+ center[d] -= width[d];
+ }
+ }
+ if(end - start < nmin) {
+ children.add(new Node(code, new Vector(center), end - start, level, null));
+ return;
+ }
+ else {
+ List<Node> newchildren = new ArrayList<Node>();
+ bulkLoad(lmin, lmax, newchildren, ids, start, end, 0, level + 1, 0);
+ children.add(new Node(code, new Vector(center), end - start, level, newchildren));
+ return;
+ }
+ }
+ else {
+ // Partially sort data, by dimension dim < mid
+ int spos = start, epos = end;
+ while(spos < epos) {
+ if(getShiftedDim(relation.get(ids.get(spos)), dim, level) <= .5) {
+ spos++;
+ continue;
+ }
+ if(getShiftedDim(relation.get(ids.get(epos - 1)), dim, level) > 0.5) {
+ epos--;
+ continue;
+ }
+ ids.swap(spos, epos - 1);
+ spos++;
+ epos--;
+ }
+ if(start < spos) {
+ final double tmp = lmax[dim];
+ lmax[dim] = lmax[dim] * .5 + lmin[dim] * .5;
+ bulkLoad(lmin, lmax, children, ids, start, spos, dim + 1, level, code);
+ lmax[dim] = tmp; // Restore
+ }
+ if(spos < end) {
+ final double tmp = lmin[dim];
+ lmin[dim] = lmax[dim] * .5 + lmin[dim] * .5;
+ bulkLoad(lmin, lmax, children, ids, spos, end, dim + 1, level, code | (1 << dim));
+ lmin[dim] = tmp; // Restore
+ }
+ }
+ }
+
+ /**
+ * Shift and wrap a single dimension.
+ *
+ * @param obj Object
+ * @param dim Dimension
+ * @param level Level (controls scaling/wraping!)
+ * @return Shifted position
+ */
+ private double getShiftedDim(NumberVector<?, ?> obj, int dim, int level) {
+ double pos = obj.doubleValue(dim + 1) + shift[dim];
+ pos = (pos - min[dim]) / width[dim] * (1 + level);
+ return pos - Math.floor(pos);
+ }
+
+ /**
+ * Find the closest node (of depth tlevel or above, if there is no node at
+ * this depth) for the given vector.
+ *
+ * @param vec Query vector
+ * @param tlevel Target level
+ * @return Node
+ */
+ public Node findClosestNode(NumberVector<?, ?> vec, int tlevel) {
+ Node cur = root;
+ for(int level = 0; level <= tlevel; level++) {
+ if(cur.children == null) {
+ break;
+ }
+ int code = 0;
+ for(int d = 0; d < min.length; d++) {
+ if(getShiftedDim(vec, d, level) > .5) {
+ code |= 1 << d;
+ }
+ }
+ boolean found = false;
+ for(Node child : cur.children) {
+ if(child.code == code) {
+ cur = child;
+ found = true;
+ break;
+ }
+ }
+ if(!found) {
+ break; // Do not descend
+ }
+ }
+ return cur;
+ }
+ }
+
+ /**
+ * Node of the ALOCI Quadtree
+ *
+ * @author Erich Schubert
+ */
+ static class Node {
+ /**
+ * Position code
+ */
+ final int code;
+
+ /**
+ * Number of elements
+ */
+ final int count;
+
+ /**
+ * Level of node
+ */
+ final int level;
+
+ /**
+ * Child nodes, may be null
+ */
+ List<Node> children;
+
+ /**
+ * Parent node
+ */
+ Node parent = null;
+
+ /**
+ * Center vector
+ */
+ Vector center;
+
+ /**
+ * Constructor.
+ *
+ * @param code Node code
+ * @param center Center vector
+ * @param count Element count
+ * @param level Node level
+ * @param children Children list
+ */
+ protected Node(int code, Vector center, int count, int level, List<Node> children) {
+ this.code = code;
+ this.center = center;
+ this.count = count;
+ this.level = level;
+ this.children = children;
+ if(children != null) {
+ for(Node child : children) {
+ child.parent = this;
+ }
+ }
+ }
+
+ /**
+ * Get level of node.
+ *
+ * @return Level of node
+ */
+ public int getLevel() {
+ return level;
+ }
+
+ /**
+ * Get count of subtree
+ *
+ * @return subtree count
+ */
+ public int getCount() {
+ return count;
+ }
+
+ /**
+ * Return center vector
+ *
+ * @return center vector
+ */
+ public Vector getCenter() {
+ return center;
+ }
+
+ /**
+ * Get sum of squares, recursively
+ *
+ * @param levels Depth to collect
+ * @return Sum of squares
+ */
+ public long getSquareSum(int levels) {
+ if(levels <= 0 || children == null) {
+ return ((long) count) * ((long) count);
+ }
+ long agg = 0;
+ for(Node child : children) {
+ agg += child.getSquareSum(levels - 1);
+ }
+ return agg;
+ }
+
+ /**
+ * Get cubic sum.
+ *
+ * @param levels Level to collect
+ * @return sum of cubes
+ */
+ public long getCubicSum(int levels) {
+ if(levels <= 0 || children == null) {
+ return ((long) count) * ((long) count) * ((long) count);
+ }
+ long agg = 0;
+ for(Node child : children) {
+ agg += child.getCubicSum(levels - 1);
+ }
+ return agg;
+ }
+ }
+
+ /**
+ * Parameterization class.
+ *
+ * @author Erich Schubert
+ *
+ * @apiviz.exclude
+ */
+ public static class Parameterizer<O extends NumberVector<O, ?>, D extends NumberDistance<D, ?>> extends AbstractParameterizer {
+ /**
+ * Parameter to specify the minimum neighborhood size
+ */
+ public static final OptionID NMIN_ID = OptionID.getOrCreateOptionID("loci.nmin", "Minimum neighborhood size to be considered.");
+
+ /**
+ * Parameter to specify the averaging neighborhood scaling.
+ */
+ public static final OptionID ALPHA_ID = OptionID.getOrCreateOptionID("loci.alpha", "Scaling factor for averaging neighborhood");
+
+ /**
+ * Parameter to specify the number of Grids to use.
+ */
+ public static final OptionID GRIDS_ID = OptionID.getOrCreateOptionID("loci.g", "The number of Grids to use.");
+
+ /**
+ * Parameter to specify the seed to initialize Random.
+ */
+ public static final OptionID SEED_ID = OptionID.getOrCreateOptionID("loci.seed", "The seed to use for initializing Random.");
+
+ /**
+ * Neighborhood minimum size
+ */
+ protected int nmin = 0;
+
+ /**
+ * Alpha: number of levels difference to use in comparison
+ */
+ protected int alpha = 4;
+
+ /**
+ * G: number of shifted trees to create.
+ */
+ protected int g = 1;
+
+ /**
+ * Random generator seed
+ */
+ protected Long seed = null;
+
+ /**
+ * The distance function
+ */
+ private NumberVectorDistanceFunction<D> distanceFunction;
+
+ @Override
+ protected void makeOptions(Parameterization config) {
+ super.makeOptions(config);
+
+ ObjectParameter<NumberVectorDistanceFunction<D>> distanceFunctionP = makeParameterDistanceFunction(EuclideanDistanceFunction.class, NumberVectorDistanceFunction.class);
+ if(config.grab(distanceFunctionP)) {
+ distanceFunction = distanceFunctionP.instantiateClass(config);
+ }
+
+ final IntParameter nminP = new IntParameter(NMIN_ID, 20);
+ if(config.grab(nminP)) {
+ nmin = nminP.getValue();
+ }
+
+ final IntParameter g = new IntParameter(GRIDS_ID, 1);
+ if(config.grab(g)) {
+ this.g = g.getValue();
+ }
+
+ final LongParameter seedP = new LongParameter(SEED_ID, true);
+ if(config.grab(seedP)) {
+ this.seed = seedP.getValue();
+ }
+
+ final IntParameter alphaP = new IntParameter(ALPHA_ID, 4);
+ if(config.grab(alphaP)) {
+ this.alpha = alphaP.getValue();
+ if(this.alpha < 1) {
+ this.alpha = 1;
+ }
+ }
+ }
+
+ @Override
+ protected ALOCI<O, D> makeInstance() {
+ return new ALOCI<O, D>(distanceFunction, nmin, alpha, g, seed);
+ }
+ }
+}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/AbstractAggarwalYuOutlier.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/AbstractAggarwalYuOutlier.java index 994ce8e2..9c1a216a 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/AbstractAggarwalYuOutlier.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/AbstractAggarwalYuOutlier.java @@ -33,6 +33,7 @@ import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.HashSetModifiableDBIDs; @@ -45,7 +46,7 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.GreaterEqualConstraint; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.IntParameter; -import de.lmu.ifi.dbs.elki.utilities.pairs.FCPair; +import de.lmu.ifi.dbs.elki.utilities.pairs.DoubleObjPair; import de.lmu.ifi.dbs.elki.utilities.pairs.IntIntPair; /** @@ -121,22 +122,23 @@ public abstract class AbstractAggarwalYuOutlier<V extends NumberVector<?, ?>> ex final ArrayList<ArrayList<DBIDs>> ranges = new ArrayList<ArrayList<DBIDs>>(); // Temporary projection storage of the database - final ArrayList<ArrayList<FCPair<Double, DBID>>> dbAxis = new ArrayList<ArrayList<FCPair<Double, DBID>>>(dim); + final ArrayList<ArrayList<DoubleObjPair<DBID>>> dbAxis = new ArrayList<ArrayList<DoubleObjPair<DBID>>>(dim); for(int i = 0; i < dim; i++) { - ArrayList<FCPair<Double, DBID>> axis = new ArrayList<FCPair<Double, DBID>>(size); + ArrayList<DoubleObjPair<DBID>> axis = new ArrayList<DoubleObjPair<DBID>>(size); dbAxis.add(i, axis); } // Project - for(DBID id : allids) { + for(DBIDIter iter = allids.iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); final V obj = database.get(id); for(int d = 1; d <= dim; d++) { - dbAxis.get(d - 1).add(new FCPair<Double, DBID>(obj.doubleValue(d), id)); + dbAxis.get(d - 1).add(new DoubleObjPair<DBID>(obj.doubleValue(d), id)); } } // Split into cells final double part = size * 1.0 / phi; for(int d = 1; d <= dim; d++) { - ArrayList<FCPair<Double, DBID>> axis = dbAxis.get(d - 1); + ArrayList<DoubleObjPair<DBID>> axis = dbAxis.get(d - 1); Collections.sort(axis); ArrayList<DBIDs> dimranges = new ArrayList<DBIDs>(phi + 1); dimranges.add(allids); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/AbstractDBOutlier.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/AbstractDBOutlier.java index 1d77af3a..a5ccce3a 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/AbstractDBOutlier.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/AbstractDBOutlier.java @@ -77,8 +77,11 @@ public abstract class AbstractDBOutlier<O, D extends Distance<D>> extends Abstra /**
* Runs the algorithm in the timed evaluation part.
*
+ * @param database Database to process
+ * @param relation Relation to process
+ * @return Outlier result
*/
- public OutlierResult run(Database database, Relation<O> relation) throws IllegalStateException {
+ public OutlierResult run(Database database, Relation<O> relation) {
// Run the actual score process
DataStore<Double> dbodscore = computeOutlierScores(database, relation, d);
diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/AggarwalYuEvolutionary.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/AggarwalYuEvolutionary.java index 5d357744..1d02e865 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/AggarwalYuEvolutionary.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/AggarwalYuEvolutionary.java @@ -38,6 +38,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil;
import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore;
import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDs;
import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation;
import de.lmu.ifi.dbs.elki.database.relation.Relation;
@@ -139,9 +140,8 @@ public class AggarwalYuEvolutionary<V extends NumberVector<?, ?>> extends Abstra * @param database Database
* @param relation Relation
* @return Result
- * @throws IllegalStateException
*/
- public OutlierResult run(Database database, Relation<V> relation) throws IllegalStateException {
+ public OutlierResult run(Database database, Relation<V> relation) {
final int dbsize = relation.size();
ArrayList<ArrayList<DBIDs>> ranges = buildRanges(relation);
@@ -151,7 +151,8 @@ public class AggarwalYuEvolutionary<V extends NumberVector<?, ?>> extends Abstra for(Individuum ind : individuums) {
DBIDs ids = computeSubspaceForGene(ind.getGene(), ranges);
double sparsityC = sparsity(ids.size(), dbsize, k);
- for(DBID id : ids) {
+ for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) {
+ DBID id = iter.getDBID();
double prev = outlierScore.doubleValue(id);
if(Double.isNaN(prev) || sparsityC < prev) {
outlierScore.putDouble(id, sparsityC);
@@ -160,7 +161,8 @@ public class AggarwalYuEvolutionary<V extends NumberVector<?, ?>> extends Abstra }
DoubleMinMax minmax = new DoubleMinMax();
- for(DBID id : relation.iterDBIDs()) {
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID();
double val = outlierScore.doubleValue(id);
if(Double.isNaN(val)) {
outlierScore.putDouble(id, 0.0);
diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/AggarwalYuNaive.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/AggarwalYuNaive.java index 190211c3..0bb73aba 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/AggarwalYuNaive.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/AggarwalYuNaive.java @@ -31,7 +31,7 @@ import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory;
import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil;
import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore;
-import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDs;
import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation;
import de.lmu.ifi.dbs.elki.database.relation.Relation;
@@ -147,19 +147,19 @@ public class AggarwalYuNaive<V extends NumberVector<?, ?>> extends AbstractAggar final double sparsityC = sparsity(ids.size(), size, k);
if(sparsityC < 0) {
- for(DBID id : ids) {
- double prev = sparsity.doubleValue(id);
+ for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) {
+ double prev = sparsity.doubleValue(iter);
if(Double.isNaN(prev) || sparsityC < prev) {
- sparsity.putDouble(id, sparsityC);
+ sparsity.putDouble(iter, sparsityC);
}
}
}
}
DoubleMinMax minmax = new DoubleMinMax();
- for(DBID id : relation.iterDBIDs()) {
- double val = sparsity.doubleValue(id);
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + double val = sparsity.doubleValue(iditer);
if(Double.isNaN(val)) {
- sparsity.putDouble(id, 0.0);
+ sparsity.putDouble(iditer, 0.0);
val = 0.0;
}
minmax.put(val);
diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/DBOutlierDetection.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/DBOutlierDetection.java index f4b0ba35..dbaf8a5a 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/DBOutlierDetection.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/DBOutlierDetection.java @@ -23,14 +23,12 @@ package de.lmu.ifi.dbs.elki.algorithm.outlier; along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-import java.util.Iterator;
-
import de.lmu.ifi.dbs.elki.database.Database;
import de.lmu.ifi.dbs.elki.database.datastore.DataStore;
import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory;
import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil;
import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore;
-import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.query.DatabaseQuery;
import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery;
import de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery;
@@ -117,19 +115,19 @@ public class DBOutlierDetection<O, D extends Distance<D>> extends AbstractDBOutl // if index exists, kNN query. if the distance to the mth nearest neighbor
// is more than d -> object is outlier
if(knnQuery != null) {
- for(DBID id : distFunc.getRelation().iterDBIDs()) {
+ for(DBIDIter iditer = distFunc.getRelation().iterDBIDs(); iditer.valid(); iditer.advance()) { counter++;
- final KNNResult<D> knns = knnQuery.getKNNForDBID(id, m);
+ final KNNResult<D> knns = knnQuery.getKNNForDBID(iditer, m);
if(logger.isDebugging()) {
logger.debugFine("distance to mth nearest neighbour" + knns.toString());
}
if(knns.get(Math.min(m, knns.size()) - 1).getDistance().compareTo(neighborhoodSize) <= 0) {
// flag as outlier
- scores.putDouble(id, 1.0);
+ scores.putDouble(iditer, 1.0);
}
else {
// flag as no outlier
- scores.putDouble(id, 0.0);
+ scores.putDouble(iditer, 0.0);
}
}
if(progressOFlags != null) {
@@ -138,27 +136,16 @@ public class DBOutlierDetection<O, D extends Distance<D>> extends AbstractDBOutl }
else {
// range query for each object. stop if m objects are found
- for(DBID id : distFunc.getRelation().iterDBIDs()) {
+ for(DBIDIter iditer = distFunc.getRelation().iterDBIDs(); iditer.valid(); iditer.advance()) { counter++;
- Iterator<DBID> iterator = distFunc.getRelation().iterDBIDs();
int count = 0;
- while(iterator.hasNext() && count < m) {
- DBID currentID = iterator.next();
- D currentDistance = distFunc.distance(id, currentID);
-
+ for (DBIDIter iterator = distFunc.getRelation().iterDBIDs(); iterator.valid() && count < m; iterator.advance()) {
+ D currentDistance = distFunc.distance(iditer, iterator);
if(currentDistance.compareTo(neighborhoodSize) <= 0) {
count++;
}
}
-
- if(count < m) {
- // flag as outlier
- scores.putDouble(id, 1.0);
- }
- else {
- // flag as no outlier
- scores.putDouble(id, 0.0);
- }
+ scores.putDouble(iditer, (count < m) ? 1.0 : 0);
}
if(progressOFlags != null) {
diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/DBOutlierScore.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/DBOutlierScore.java index ec83a2a2..419b9a0e 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/DBOutlierScore.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/DBOutlierScore.java @@ -28,7 +28,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStore; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory;
import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil;
import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore;
-import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery;
import de.lmu.ifi.dbs.elki.database.query.range.RangeQuery;
import de.lmu.ifi.dbs.elki.database.relation.Relation;
@@ -80,10 +80,10 @@ public class DBOutlierScore<O, D extends Distance<D>> extends AbstractDBOutlier< WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(distFunc.getRelation().getDBIDs(), DataStoreFactory.HINT_STATIC);
// TODO: use bulk when implemented.
- for(DBID id : distFunc.getRelation().iterDBIDs()) {
+ for(DBIDIter iditer = distFunc.getRelation().iterDBIDs(); iditer.valid(); iditer.advance()) { // compute percentage of neighbors in the given neighborhood with size d
- double n = (rangeQuery.getRangeForDBID(id, d).size()) / size;
- scores.putDouble(id, 1.0 - n);
+ double n = (rangeQuery.getRangeForDBID(iditer, d).size()) / size;
+ scores.putDouble(iditer, 1.0 - n);
}
return scores;
}
diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/EMOutlier.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/EMOutlier.java index 92d92036..db4b7782 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/EMOutlier.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/EMOutlier.java @@ -1,26 +1,27 @@ package de.lmu.ifi.dbs.elki.algorithm.outlier;
-/* -This file is part of ELKI: -Environment for Developing KDD-Applications Supported by Index-Structures - -Copyright (C) 2012 -Ludwig-Maximilians-Universität München -Lehr- und Forschungseinheit für Datenbanksysteme -ELKI Development Team - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ +
+/*
+ This file is part of ELKI:
+ Environment for Developing KDD-Applications Supported by Index-Structures +
+ Copyright (C) 2012
+ Ludwig-Maximilians-Universität München
+ Lehr- und Forschungseinheit für Datenbanksysteme
+ ELKI Development Team +
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version. +
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details. +
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
import de.lmu.ifi.dbs.elki.algorithm.AbstractAlgorithm;
import de.lmu.ifi.dbs.elki.algorithm.clustering.EM;
@@ -33,7 +34,7 @@ import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory;
import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil;
import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore;
-import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation;
import de.lmu.ifi.dbs.elki.database.relation.Relation;
import de.lmu.ifi.dbs.elki.logging.Logging;
@@ -84,19 +85,23 @@ public class EMOutlier<V extends NumberVector<V, ?>> extends AbstractAlgorithm<O /**
* Runs the algorithm in the timed evaluation part.
+ *
+ * @param database Database to process
+ * @param relation Relation to process
+ * @return Outlier result
*/
- public OutlierResult run(Database database, Relation<V> relation) throws IllegalStateException {
+ public OutlierResult run(Database database, Relation<V> relation) {
Clustering<EMModel<V>> emresult = emClustering.run(database, relation);
double globmax = 0.0;
WritableDoubleDataStore emo_score = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_TEMP | DataStoreFactory.HINT_HOT);
- for(DBID id : relation.iterDBIDs()) {
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { double maxProb = Double.POSITIVE_INFINITY;
- double[] probs = emClustering.getProbClusterIGivenX(id);
+ double[] probs = emClustering.getProbClusterIGivenX(iditer);
for(double prob : probs) {
maxProb = Math.min(1 - prob, maxProb);
}
- emo_score.putDouble(id, maxProb);
+ emo_score.putDouble(iditer, maxProb);
globmax = Math.max(maxProb, globmax);
}
Relation<Double> scoreres = new MaterializedRelation<Double>("EM outlier scores", "em-outlier", TypeUtil.DOUBLE, emo_score, relation.getDBIDs());
diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/GaussianModel.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/GaussianModel.java index ae47c100..51833c8b 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/GaussianModel.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/GaussianModel.java @@ -30,6 +30,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil;
import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore;
import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation;
import de.lmu.ifi.dbs.elki.database.relation.Relation;
import de.lmu.ifi.dbs.elki.logging.Logging;
@@ -92,7 +93,13 @@ public class GaussianModel<V extends NumberVector<V, ?>> extends AbstractAlgorit this.invert = invert;
}
- public OutlierResult run(Relation<V> relation) throws IllegalStateException {
+ /**
+ * Run the algorithm
+ *
+ * @param relation Data relation
+ * @return Outlier result
+ */
+ public OutlierResult run(Relation<V> relation) {
DoubleMinMax mm = new DoubleMinMax();
// resulting scores
WritableDoubleDataStore oscores = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_TEMP | DataStoreFactory.HINT_HOT);
@@ -109,20 +116,21 @@ public class GaussianModel<V extends NumberVector<V, ?>> extends AbstractAlgorit final double fakt = (1.0 / (Math.sqrt(Math.pow(MathUtil.TWOPI, DatabaseUtil.dimensionality(relation)) * covarianceMatrix.det())));
// for each object compute Mahalanobis distance
- for(DBID id : relation.iterDBIDs()) {
- Vector x = relation.get(id).getColumnVector().minusEquals(mean);
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + Vector x = relation.get(iditer).getColumnVector().minusEquals(mean);
// Gaussian PDF
final double mDist = x.transposeTimesTimes(covarianceTransposed, x);
final double prob = fakt * Math.exp(-mDist / 2.0);
mm.put(prob);
- oscores.putDouble(id, prob);
+ oscores.putDouble(iditer, prob);
}
final OutlierScoreMeta meta;
if(invert) {
double max = mm.getMax() != 0 ? mm.getMax() : 1.;
- for(DBID id : relation.iterDBIDs()) {
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID();
oscores.putDouble(id, (max - oscores.doubleValue(id)) / max);
}
meta = new BasicOutlierScoreMeta(0.0, 1.0);
diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/GaussianUniformMixture.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/GaussianUniformMixture.java index aa352582..1cd31442 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/GaussianUniformMixture.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/GaussianUniformMixture.java @@ -33,6 +33,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore;
import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs;
import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil;
import de.lmu.ifi.dbs.elki.database.ids.DBIDs;
import de.lmu.ifi.dbs.elki.database.ids.generic.MaskedDBIDs;
@@ -41,6 +42,7 @@ import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.logging.Logging;
import de.lmu.ifi.dbs.elki.math.DoubleMinMax;
import de.lmu.ifi.dbs.elki.math.MathUtil;
+import de.lmu.ifi.dbs.elki.math.linearalgebra.CovarianceMatrix;
import de.lmu.ifi.dbs.elki.math.linearalgebra.Matrix;
import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector;
import de.lmu.ifi.dbs.elki.result.outlier.BasicOutlierScoreMeta;
@@ -127,7 +129,13 @@ public class GaussianUniformMixture<V extends NumberVector<V, ?>> extends Abstra this.c = c;
}
- public OutlierResult run(Relation<V> relation) throws IllegalStateException {
+ /**
+ * Run the algorithm
+ *
+ * @param relation Data relation
+ * @return Outlier result
+ */
+ public OutlierResult run(Relation<V> relation) {
// Use an array list of object IDs for fast random access by an offset
ArrayDBIDs objids = DBIDUtil.ensureArray(relation.getDBIDs());
// A bit set to flag objects as anomalous, none at the beginning
@@ -205,9 +213,9 @@ public class GaussianUniformMixture<V extends NumberVector<V, ?>> extends Abstra if(objids.isEmpty()) {
return 0;
}
- double prob = 0;
- Vector mean = DatabaseUtil.centroid(database, objids).getColumnVector();
- Matrix covarianceMatrix = DatabaseUtil.covarianceMatrix(database, objids);
+ CovarianceMatrix builder = CovarianceMatrix.make(database, objids);
+ Vector mean = builder.getMeanVector();
+ Matrix covarianceMatrix = builder.destroyToSampleMatrix();
// test singulaere matrix
Matrix covInv = covarianceMatrix.cheatToAvoidSingularity(SINGULARITY_CHEAT).inverse();
@@ -215,8 +223,9 @@ public class GaussianUniformMixture<V extends NumberVector<V, ?>> extends Abstra double covarianceDet = covarianceMatrix.det();
double fakt = 1.0 / Math.sqrt(Math.pow(MathUtil.TWOPI, DatabaseUtil.dimensionality(database)) * covarianceDet);
// for each object compute probability and sum
- for(DBID id : objids) {
- Vector x = database.get(id).getColumnVector().minusEquals(mean);
+ double prob = 0;
+ for (DBIDIter iter = objids.iter(); iter.valid(); iter.advance()) {
+ Vector x = database.get(iter).getColumnVector().minusEquals(mean);
double mDist = x.transposeTimesTimes(covInv, x);
prob += Math.log(fakt * Math.exp(-mDist / 2.0));
}
diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/HilOut.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/HilOut.java new file mode 100644 index 00000000..4ed56e1a --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/HilOut.java @@ -0,0 +1,988 @@ +package de.lmu.ifi.dbs.elki.algorithm.outlier;
+/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.Set;
+
+import de.lmu.ifi.dbs.elki.algorithm.AbstractDistanceBasedAlgorithm;
+import de.lmu.ifi.dbs.elki.data.NumberVector;
+import de.lmu.ifi.dbs.elki.data.type.TypeInformation;
+import de.lmu.ifi.dbs.elki.data.type.TypeUtil;
+import de.lmu.ifi.dbs.elki.database.Database;
+import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory;
+import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil;
+import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore;
+import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil;
+import de.lmu.ifi.dbs.elki.database.ids.HashSetModifiableDBIDs;
+import de.lmu.ifi.dbs.elki.database.query.DoubleDistanceResultPair;
+import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery;
+import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation;
+import de.lmu.ifi.dbs.elki.database.relation.Relation;
+import de.lmu.ifi.dbs.elki.distance.distancefunction.EuclideanDistanceFunction;
+import de.lmu.ifi.dbs.elki.distance.distancefunction.LPNormDistanceFunction;
+import de.lmu.ifi.dbs.elki.distance.distancevalue.DoubleDistance;
+import de.lmu.ifi.dbs.elki.logging.Logging;
+import de.lmu.ifi.dbs.elki.logging.progress.FiniteProgress;
+import de.lmu.ifi.dbs.elki.math.DoubleMinMax;
+import de.lmu.ifi.dbs.elki.math.spacefillingcurves.HilbertSpatialSorter;
+import de.lmu.ifi.dbs.elki.result.outlier.BasicOutlierScoreMeta;
+import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult;
+import de.lmu.ifi.dbs.elki.result.outlier.OutlierScoreMeta;
+import de.lmu.ifi.dbs.elki.utilities.BitsUtil;
+import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil;
+import de.lmu.ifi.dbs.elki.utilities.datastructures.heap.Heap;
+import de.lmu.ifi.dbs.elki.utilities.documentation.Description;
+import de.lmu.ifi.dbs.elki.utilities.documentation.Reference;
+import de.lmu.ifi.dbs.elki.utilities.documentation.Title;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.EnumParameter;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.IntParameter;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter;
+import de.lmu.ifi.dbs.elki.utilities.pairs.Pair;
+
+/**
+ * Fast Outlier Detection in High Dimensional Spaces
+ *
+ * Outlier Detection using Hilbert space filling curves
+ *
+ * Reference:
+ * <p>
+ * F. Angiulli, C. Pizzuti:<br />
+ * Fast Outlier Detection in High Dimensional Spaces.<br />
+ * In: Proc. European Conference on Principles of Knowledge Discovery and Data
+ * Mining (PKDD'02), Helsinki, Finland, 2002.
+ * </p>
+ *
+ * @author Jonathan von Brünken
+ * @author Erich Schubert
+ *
+ * @apiviz.composedOf HilbertFeatures
+ * @apiviz.uses HilFeature
+ *
+ * @param <O> Object type
+ */
+@Title("Fast Outlier Detection in High Dimensional Spaces")
+@Description("Algorithm to compute outliers using Hilbert space filling curves")
+@Reference(authors = "F. Angiulli, C. Pizzuti", title = "Fast Outlier Detection in High Dimensional Spaces", booktitle = "Proc. European Conference on Principles of Knowledge Discovery and Data Mining (PKDD'02)", url = "http://dx.doi.org/10.1145/375663.375668")
+public class HilOut<O extends NumberVector<O, ?>> extends AbstractDistanceBasedAlgorithm<O, DoubleDistance, OutlierResult> implements OutlierAlgorithm {
+ /**
+ * The logger for this class.
+ */
+ private static final Logging logger = Logging.getLogger(HilOut.class);
+
+ /**
+ * Number of nearest neighbors
+ */
+ private int k;
+
+ /**
+ * Number of outliers to compute exactly
+ */
+ private int n;
+
+ /**
+ * Hilbert precision
+ */
+ private int h;
+
+ /**
+ * LPNorm p parameter
+ */
+ private double t;
+
+ /**
+ * Reporting mode: exact (top n) only, or all
+ */
+ private Enum<ScoreType> tn;
+
+ /**
+ * Distance query
+ */
+ private DistanceQuery<O, DoubleDistance> distq;
+
+ /**
+ * Set sizes, total and current iteration
+ */
+ private int capital_n, n_star, capital_n_star, d;
+
+ /**
+ * Outlier threshold
+ */
+ private double omega_star;
+
+ /**
+ * Type of output: all scores (upper bounds) or top n only
+ *
+ * @author Jonathan von Brünken
+ *
+ * @apiviz.exclude
+ */
+ public static enum ScoreType {
+ All, TopN
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param k Number of Next Neighbors
+ * @param n Number of Outlier
+ * @param h Number of Bits for precision to use - max 32
+ * @param tn TopN or All Outlier Rank to return
+ */
+ protected HilOut(LPNormDistanceFunction distfunc, int k, int n, int h, Enum<ScoreType> tn) {
+ super(distfunc);
+ this.n = n;
+ // HilOut does not count the object itself. We do in KNNWeightOutlier.
+ this.k = k - 1;
+ this.h = h;
+ this.tn = tn;
+ this.t = distfunc.getP();
+ this.n_star = 0;
+ this.omega_star = 0.0;
+ }
+
+ public OutlierResult run(Database database, Relation<O> relation) {
+ distq = database.getDistanceQuery(relation, getDistanceFunction());
+ d = DatabaseUtil.dimensionality(relation);
+ WritableDoubleDataStore hilout_weight = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);
+
+ // Compute extend of dataset.
+ double[] min;
+ double diameter = 0; // Actually "length of edge"
+ {
+ Pair<O, O> hbbs = DatabaseUtil.computeMinMax(relation);
+ min = new double[d];
+ double[] max = new double[d];
+ for(int i = 0; i < d; i++) {
+ min[i] = hbbs.first.doubleValue(i + 1);
+ max[i] = hbbs.second.doubleValue(i + 1);
+ diameter = Math.max(diameter, max[i] - min[i]);
+ }
+ // Enlarge bounding box to have equal lengths.
+ for(int i = 0; i < d; i++) {
+ double diff = (diameter - (max[i] - min[i])) / 2;
+ min[i] -= diff;
+ max[i] += diff;
+ }
+ if(logger.isVerbose()) {
+ logger.verbose("Rescaling dataset by " + (1 / diameter) + " to fit the unit cube.");
+ }
+ }
+
+ // Initialization part
+ capital_n_star = capital_n = relation.size();
+ HilbertFeatures h = new HilbertFeatures(relation, min, diameter);
+
+ FiniteProgress progressHilOut = logger.isVerbose() ? new FiniteProgress("HilOut iterations", d + 1, logger) : null;
+ FiniteProgress progressTrueOut = logger.isVerbose() ? new FiniteProgress("True outliers found", n, logger) : null;
+ // Main part: 1. Phase max. d+1 loops
+ for(int j = 0; j <= d && n_star < n; j++) {
+ // initialize (clear) out and wlb - not 100% clear in the paper
+ h.out.clear();
+ h.wlb.clear();
+ // Initialize Hilbert values in pf according to current shift
+ h.initialize(.5 * j / (d + 1));
+ // scan the Data according to the current shift; build out and wlb
+ scan(h, (int) (k * capital_n / (double) capital_n_star));
+ // determine the true outliers (n_star)
+ trueOutliers(h);
+ if(progressTrueOut != null) {
+ progressTrueOut.setProcessed(n_star, logger);
+ }
+ // Build the top Set as out + wlb
+ h.top.clear();
+ HashSetModifiableDBIDs top_keys = DBIDUtil.newHashSet(h.out.size());
+ for(HilFeature entry : h.out) {
+ top_keys.add(entry.id);
+ h.top.add(entry);
+ }
+ for(HilFeature entry : h.wlb) {
+ if(!top_keys.contains(entry.id)) {
+ // No need to update top_keys - discarded
+ h.top.add(entry);
+ }
+ }
+ if(progressHilOut != null) {
+ progressHilOut.incrementProcessed(logger);
+ }
+ }
+ // 2. Phase: Additional Scan if less than n true outliers determined
+ if(n_star < n) {
+ h.out.clear();
+ h.wlb.clear();
+ // TODO: reinitialize shift to 0?
+ scan(h, capital_n);
+ }
+ if(progressHilOut != null) {
+ progressHilOut.setProcessed(d, logger);
+ progressHilOut.ensureCompleted(logger);
+ }
+ if(progressTrueOut != null) {
+ progressTrueOut.setProcessed(n, logger);
+ progressTrueOut.ensureCompleted(logger);
+ }
+ DoubleMinMax minmax = new DoubleMinMax();
+ // Return weights in out
+ if(tn == ScoreType.TopN) {
+ minmax.put(0.0);
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + hilout_weight.putDouble(iditer, 0.0);
+ }
+ for(HilFeature ent : h.out) {
+ minmax.put(ent.ubound);
+ hilout_weight.putDouble(ent.id, ent.ubound);
+ }
+ }
+ // Return all weights in pf
+ else {
+ for(HilFeature ent : h.pf) {
+ minmax.put(ent.ubound);
+ hilout_weight.putDouble(ent.id, ent.ubound);
+ }
+ }
+ Relation<Double> scoreResult = new MaterializedRelation<Double>("HilOut weight", "hilout-weight", TypeUtil.DOUBLE, hilout_weight, relation.getDBIDs());
+ OutlierScoreMeta scoreMeta = new BasicOutlierScoreMeta(minmax.getMin(), minmax.getMax(), 0.0, Double.POSITIVE_INFINITY);
+ OutlierResult result = new OutlierResult(scoreMeta, scoreResult);
+ return result;
+ }
+
+ /**
+ * Scan function performs a squential scan over the data.
+ *
+ * @param hf the hilbert features
+ * @param k0
+ */
+ private void scan(HilbertFeatures hf, int k0) {
+ final int mink0 = Math.min(2 * k0, capital_n - 1);
+ if(logger.isDebuggingFine()) {
+ logger.debugFine("Scanning with k0=" + k0 + " (" + mink0 + ")" + " N*=" + capital_n_star);
+ }
+ for(int i = 0; i < hf.pf.length; i++) {
+ if(hf.pf[i].ubound < omega_star) {
+ continue;
+ }
+ if(hf.pf[i].lbound < hf.pf[i].ubound) {
+ double omega = hf.fastUpperBound(i);
+ if(omega < omega_star) {
+ hf.pf[i].ubound = omega;
+ }
+ else {
+ int maxcount;
+ // capital_n-1 instead of capital_n: all, except self
+ if(hf.top.contains(hf.pf[i])) {
+ maxcount = capital_n - 1;
+ }
+ else {
+ maxcount = mink0;
+ }
+ innerScan(hf, i, maxcount);
+ }
+ }
+ if(hf.pf[i].ubound > 0) {
+ hf.updateOUT(i);
+ }
+ if(hf.pf[i].lbound > 0) {
+ hf.updateWLB(i);
+ }
+ if(hf.wlb.size() >= n) {
+ omega_star = Math.max(omega_star, hf.wlb.peek().lbound);
+ }
+ }
+ }
+
+ /**
+ * innerScan function calculates new upper and lower bounds and inserts the
+ * points of the neighborhood the bounds are based on in the NN Set
+ *
+ * @param i position in pf of the feature for which the bounds should be
+ * calculated
+ * @param maxcount maximal size of the neighborhood
+ */
+ private void innerScan(HilbertFeatures hf, final int i, final int maxcount) {
+ final O p = hf.relation.get(hf.pf[i].id); // Get only once for performance
+ int a = i, b = i;
+ int level = h, levela = h, levelb = h;
+ // Explore up to "maxcount" neighbors in this pass
+ for(int count = 0; count < maxcount; count++) {
+ final int c; // Neighbor to explore
+ if(a == 0) { // At left end, explore right
+ // assert (b < capital_n - 1);
+ levelb = Math.min(levelb, hf.pf[b].level);
+ b++;
+ c = b;
+ }
+ else if(b >= capital_n - 1) { // At right end, explore left
+ // assert (a > 0);
+ a--;
+ levela = Math.min(levela, hf.pf[a].level);
+ c = a;
+ }
+ else if(hf.pf[a - 1].level >= hf.pf[b].level) { // Prefer higher level
+ a--;
+ levela = Math.min(levela, hf.pf[a].level);
+ c = a;
+ }
+ else {
+ // assert (b < capital_n - 1);
+ levelb = Math.min(levelb, hf.pf[b].level);
+ b++;
+ c = b;
+ }
+ if(!hf.pf[i].nn_keys.contains(hf.pf[c].id)) {
+ // hf.distcomp ++;
+ hf.pf[i].insert(hf.pf[c].id, distq.distance(p, hf.pf[c].id).doubleValue(), k);
+ if(hf.pf[i].nn.size() == k) {
+ if(hf.pf[i].sum_nn < omega_star) {
+ break; // stop = true
+ }
+ final int mlevel = Math.max(levela, levelb);
+ if(mlevel < level) {
+ level = mlevel;
+ final double delta = hf.minDistLevel(hf.pf[i].id, level);
+ if(delta >= hf.pf[i].nn.peek().getDoubleDistance()) {
+ break; // stop = true
+ }
+ }
+ }
+ }
+ }
+ double br = hf.boxRadius(i, a - 1, b + 1);
+ double newlb = 0.0;
+ double newub = 0.0;
+ for(DoubleDistanceResultPair entry : hf.pf[i].nn) {
+ newub += entry.getDoubleDistance();
+ if(entry.getDoubleDistance() <= br) {
+ newlb += entry.getDoubleDistance();
+ }
+ }
+ if(newlb > hf.pf[i].lbound) {
+ hf.pf[i].lbound = newlb;
+ }
+ if(newub < hf.pf[i].ubound) {
+ hf.pf[i].ubound = newub;
+ }
+ }
+
+ /**
+ * trueOutliers function updates n_star
+ *
+ * @param h the HilberFeatures
+ *
+ */
+
+ private void trueOutliers(HilbertFeatures h) {
+ n_star = 0;
+ for(HilFeature entry : h.out) {
+ if(entry.ubound >= omega_star && (entry.ubound - entry.lbound < 1E-10)) {
+ n_star++;
+ }
+ }
+ }
+
+ @Override
+ protected Logging getLogger() {
+ return logger;
+ }
+
+ @Override
+ public TypeInformation[] getInputTypeRestriction() {
+ return TypeUtil.array(new LPNormDistanceFunction(t).getInputTypeRestriction());
+ }
+
+ /**
+ * Class organizing the data points along a hilbert curve.
+ *
+ * @author Jonathan von Brünken
+ *
+ * @apiviz.composedOf HilFeature
+ */
+ class HilbertFeatures {
+ // public int distcomp = 1;
+
+ /**
+ * Relation indexed
+ */
+ Relation<O> relation;
+
+ /**
+ * Hilbert representation ("point features")
+ */
+ HilFeature[] pf;
+
+ /**
+ * Data space minimums
+ */
+ double[] min;
+
+ /**
+ * Data space diameter
+ */
+ double diameter;
+
+ /**
+ * Current curve shift
+ */
+ double shift;
+
+ /**
+ * Top candidates
+ */
+ private Set<HilFeature> top;
+
+ /**
+ * "OUT"
+ */
+ private Heap<HilFeature> out;
+
+ /**
+ * "WLB"
+ */
+ private Heap<HilFeature> wlb;
+
+ /**
+ * Constructor.
+ *
+ * @param relation Relation to index
+ * @param min Minimums for data space
+ * @param diameter Diameter of data space
+ */
+ public HilbertFeatures(Relation<O> relation, double[] min, double diameter) {
+ super();
+ this.relation = relation;
+ this.min = min;
+ this.diameter = diameter;
+ this.pf = new HilFeature[relation.size()];
+
+ int pos = 0;
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + pf[pos++] = new HilFeature(iditer.getDBID(), new Heap<DoubleDistanceResultPair>(k, Collections.reverseOrder()));
+ }
+ this.out = new Heap<HilFeature>(n, new Comparator<HilFeature>() {
+ @Override
+ public int compare(HilFeature o1, HilFeature o2) {
+ return Double.compare(o1.ubound, o2.ubound);
+ }
+ });
+ this.wlb = new Heap<HilFeature>(n, new Comparator<HilFeature>() {
+ @Override
+ public int compare(HilFeature o1, HilFeature o2) {
+ return Double.compare(o1.lbound, o2.lbound);
+ }
+ });
+ this.top = new HashSet<HilFeature>(2 * n);
+ }
+
+ /**
+ * Hilbert function to fill pf with shifted Hilbert values. Also calculates
+ * the number current Outlier candidates capital_n_star
+ *
+ * @param shift the new shift factor
+ */
+ private void initialize(double shift) {
+ this.shift = shift;
+ // FIXME: 64 bit mode untested - sign bit is tricky to handle correctly
+ // with the rescaling. 63 bit should be fine. The sign bit probably needs
+ // to be handled differently, or at least needs careful testing of the API
+ if(h >= 32) { // 32 to 63 bit
+ final long scale = Long.MAX_VALUE; // = 63 bits
+ for(int i = 0; i < pf.length; i++) {
+ NumberVector<?, ?> obj = relation.get(pf[i].id);
+ long[] coord = new long[d];
+ for(int dim = 0; dim < d; dim++) {
+ coord[dim] = (long) (getDimForObject(obj, dim) * .5 * scale);
+ }
+ pf[i].hilbert = HilbertSpatialSorter.coordinatesToHilbert(coord, h, 1);
+ }
+ }
+ else if(h >= 16) { // 16-31 bit
+ final int scale = ~1 >>> 1;
+ for(int i = 0; i < pf.length; i++) {
+ NumberVector<?, ?> obj = relation.get(pf[i].id);
+ int[] coord = new int[d];
+ for(int dim = 0; dim < d; dim++) {
+ coord[dim] = (int) (getDimForObject(obj, dim) * .5 * scale);
+ }
+ pf[i].hilbert = HilbertSpatialSorter.coordinatesToHilbert(coord, h, 1);
+ }
+ }
+ else if(h >= 8) { // 8-15 bit
+ final int scale = ~1 >>> 16;
+ for(int i = 0; i < pf.length; i++) {
+ NumberVector<?, ?> obj = relation.get(pf[i].id);
+ short[] coord = new short[d];
+ for(int dim = 0; dim < d; dim++) {
+ coord[dim] = (short) (getDimForObject(obj, dim) * .5 * scale);
+ }
+ pf[i].hilbert = HilbertSpatialSorter.coordinatesToHilbert(coord, h, 16);
+ }
+ }
+ else { // 1-7 bit
+ final int scale = ~1 >>> 8;
+ for(int i = 0; i < pf.length; i++) {
+ NumberVector<?, ?> obj = relation.get(pf[i].id);
+ byte[] coord = new byte[d];
+ for(int dim = 0; dim < d; dim++) {
+ coord[dim] = (byte) (getDimForObject(obj, dim) * .5 * scale);
+ }
+ pf[i].hilbert = HilbertSpatialSorter.coordinatesToHilbert(coord, h, 24);
+ }
+ }
+ java.util.Arrays.sort(pf);
+ // Update levels
+ for(int i = 0; i < pf.length - 1; i++) {
+ pf[i].level = minRegLevel(i, i + 1);
+ }
+ // Count candidates
+ capital_n_star = 0;
+ for(int i = 0; i < pf.length; i++) {
+ if(pf[i].ubound >= omega_star) {
+ capital_n_star++;
+ }
+ }
+ }
+
+ /**
+ * updateOUT function inserts pf[i] in out.
+ *
+ * @param i position in pf of the feature to be inserted
+ */
+ private void updateOUT(int i) {
+ if(out.size() < n) {
+ out.offer(pf[i]);
+ }
+ else {
+ HilFeature head = out.peek();
+ if(pf[i].ubound > head.ubound) {
+ // replace smallest
+ out.poll();
+ // assert (out.peek().ubound >= head.ubound);
+ out.offer(pf[i]);
+ }
+ }
+ }
+
+ /**
+ * updateWLB function inserts pf[i] in wlb.
+ *
+ * @param i position in pf of the feature to be inserted
+ */
+ private void updateWLB(int i) {
+ if(wlb.size() < n) {
+ wlb.offer(pf[i]);
+ }
+ else {
+ HilFeature head = wlb.peek();
+ if(pf[i].lbound > head.lbound) {
+ // replace smallest
+ wlb.poll();
+ // assert (wlb.peek().lbound >= head.lbound);
+ wlb.offer(pf[i]);
+ }
+ }
+ }
+
+ /**
+ * fastUpperBound function calculates an upper Bound as k*maxDist(pf[i],
+ * smallest neighborhood)
+ *
+ * @param i position in pf of the feature for which the bound should be
+ * calculated
+ */
+ private double fastUpperBound(int i) {
+ int pre = i;
+ int post = i;
+ while(post - pre < k) {
+ int pre_level = (pre - 1 >= 0) ? pf[pre - 1].level : -2;
+ int post_level = (post < capital_n - 1) ? pf[post].level : -2;
+ if(post_level >= pre_level) {
+ post++;
+ }
+ else {
+ pre--;
+ }
+ }
+ return k * maxDistLevel(pf[i].id, minRegLevel(pre, post));
+ }
+
+ /**
+ * minDist function calculate the minimal Distance from Vector p to the
+ * border of the corresponding r-region at the given level
+ *
+ * @param id Object ID
+ * @param level Level of the corresponding r-region
+ */
+ private double minDistLevel(DBID id, int level) {
+ final NumberVector<?, ?> obj = relation.get(id);
+ // level 1 is supposed to have r=1 as in the original publication
+ // 2 ^ - (level - 1)
+ final double r = 1.0 / (1 << (level - 1));
+ double dist = Double.POSITIVE_INFINITY;
+ for(int dim = 0; dim < d; dim++) {
+ final double p_m_r = getDimForObject(obj, dim) % r;
+ dist = Math.min(dist, Math.min(p_m_r, r - p_m_r));
+ }
+ return dist * diameter;
+ }
+
+ /**
+ * maxDist function calculate the maximal Distance from Vector p to the
+ * border of the corresponding r-region at the given level
+ *
+ * @param id Object ID
+ * @param level Level of the corresponding r-region
+ */
+ private double maxDistLevel(DBID id, int level) {
+ final NumberVector<?, ?> obj = relation.get(id);
+ // level 1 is supposed to have r=1 as in the original publication
+ final double r = 1.0 / (1 << (level - 1));
+ double dist;
+ if(t == 1.0) {
+ dist = 0.0;
+ for(int dim = 0; dim < d; dim++) {
+ final double p_m_r = getDimForObject(obj, dim) % r;
+ // assert (p_m_r >= 0);
+ dist += Math.max(p_m_r, r - p_m_r);
+ }
+ }
+ else if(t == 2.0) {
+ dist = 0.0;
+ for(int dim = 0; dim < d; dim++) {
+ final double p_m_r = getDimForObject(obj, dim) % r;
+ // assert (p_m_r >= 0);
+ double a = Math.max(p_m_r, r - p_m_r);
+ dist += a * a;
+ }
+ dist = Math.sqrt(dist);
+ }
+ else if(!Double.isInfinite(t)) {
+ dist = 0.0;
+ for(int dim = 0; dim < d; dim++) {
+ final double p_m_r = getDimForObject(obj, dim) % r;
+ dist += Math.pow(Math.max(p_m_r, r - p_m_r), t);
+ }
+ dist = Math.pow(dist, 1.0 / t);
+ }
+ else {
+ dist = Double.NEGATIVE_INFINITY;
+ for(int dim = 0; dim < d; dim++) {
+ final double p_m_r = getDimForObject(obj, dim) % r;
+ dist = Math.max(dist, Math.max(p_m_r, r - p_m_r));
+ }
+ }
+ return dist * diameter;
+ }
+
+ /**
+ * Number of levels shared
+ *
+ * @param a First bitset
+ * @param b Second bitset
+ * @return Number of level shared
+ */
+ private int numberSharedLevels(long[] a, long[] b) {
+ for(int i = 0, j = a.length - 1; i < a.length; i++, j--) {
+ final long diff = a[j] ^ b[j];
+ if(diff != 0) {
+ // expected unused = available - used
+ final int expected = (a.length * Long.SIZE) - (d * h);
+ return ((BitsUtil.numberOfLeadingZeros(diff) + i * Long.SIZE) - expected) / d;
+ }
+ }
+ return h - 1;
+ }
+
+ /**
+ * minReg function calculate the minimal r-region level containing two
+ * points
+ *
+ * @param a index of first point in pf
+ * @param b index of second point in pf
+ *
+ * @return Level of the r-region
+ */
+ private int minRegLevel(int a, int b) {
+ // Sanity test: first level different -> region of level 0, r=2
+ // all same: level h - 1
+ return numberSharedLevels(pf[a].hilbert, pf[b].hilbert);
+ }
+
+ /**
+ * Level of the maximum region containing ref but not q
+ *
+ * @param ref Reference point
+ * @param q Query point
+ * @return Number of bits shared across all dimensions
+ */
+ private int maxRegLevel(int ref, int q) {
+ // Sanity test: first level different -> region of level 1, r=1
+ // all same: level h
+ return numberSharedLevels(pf[ref].hilbert, pf[q].hilbert) + 1;
+ }
+
+ /**
+ * boxRadius function calculate the Boxradius
+ *
+ * @param i index of first point
+ * @param a index of second point
+ * @param b index of third point
+ *
+ * @return box radius
+ */
+ private double boxRadius(int i, int a, int b) {
+ // level are inversely ordered to box sizes. min -> max
+ final int level;
+ if(a < 0) {
+ if(b >= pf.length) {
+ return Double.POSITIVE_INFINITY;
+ }
+ level = maxRegLevel(i, b);
+ }
+ else if(b >= pf.length) {
+ level = maxRegLevel(i, a);
+ }
+ else {
+ level = Math.max(maxRegLevel(i, a), maxRegLevel(i, b));
+ }
+ return minDistLevel(pf[i].id, level);
+ }
+
+ /**
+ * Get the (projected) position of the object in dimension dim.
+ *
+ * @param obj Object
+ * @param dim Dimension
+ * @return Projected and shifted position
+ */
+ private double getDimForObject(NumberVector<?, ?> obj, int dim) {
+ return (obj.doubleValue(dim + 1) - min[dim]) / diameter + shift;
+ }
+ }
+
+ /**
+ * Hilbert representation of a single object.
+ *
+ * Details of this representation are discussed in the main HilOut
+ * publication, see "point features".
+ *
+ * @author Jonathan von Brünken
+ */
+ final static class HilFeature implements Comparable<HilFeature> {
+ /**
+ * Object ID
+ */
+ public DBID id;
+
+ /**
+ * Hilbert representation
+ *
+ * TODO: use byte[] to save some memory, but slower?
+ */
+ public long[] hilbert = null;
+
+ /**
+ * Object level
+ */
+ public int level = 0;
+
+ /**
+ * Upper bound for object
+ */
+ public double ubound = Double.POSITIVE_INFINITY;
+
+ /**
+ * Lower bound of object
+ */
+ public double lbound = 0.0;
+
+ /**
+ * Heap with the nearest known neighbors
+ */
+ public Heap<DoubleDistanceResultPair> nn;
+
+ /**
+ * Set representation of the nearest neighbors for faster lookups
+ */
+ public HashSetModifiableDBIDs nn_keys = DBIDUtil.newHashSet();
+
+ /**
+ * Current weight (sum of nn distances)
+ */
+ public double sum_nn = 0.0;
+
+ /**
+ * Constructor.
+ *
+ * @param id Object ID
+ * @param nn Heap for neighbors
+ */
+ public HilFeature(DBID id, Heap<DoubleDistanceResultPair> nn) {
+ super();
+ this.id = id;
+ this.nn = nn;
+ }
+
+ @Override
+ public int compareTo(HilFeature o) {
+ return BitsUtil.compare(this.hilbert, o.hilbert);
+ }
+
+ /**
+ * insert function inserts a nearest neighbor into a features nn list and
+ * its distance
+ *
+ * @param id DBID of the nearest neighbor
+ * @param dt distance or the neighbor to the features position
+ * @param k K
+ */
+ protected void insert(DBID id, double dt, int k) {
+ // assert (!nn_keys.contains(id));
+ if(nn.size() < k) {
+ DoubleDistanceResultPair entry = new DoubleDistanceResultPair(dt, id);
+ nn.offer(entry);
+ nn_keys.add(id);
+ sum_nn += dt;
+ }
+ else {
+ DoubleDistanceResultPair head = nn.peek();
+ if(dt < head.getDoubleDistance()) {
+ head = nn.poll(); // Remove worst
+ sum_nn -= head.getDoubleDistance();
+ nn_keys.remove(head.getDBID());
+
+ // assert (nn.peek().getDoubleDistance() <= head.getDoubleDistance());
+
+ DoubleDistanceResultPair entry = new DoubleDistanceResultPair(dt, id);
+ nn.offer(entry);
+ nn_keys.add(id);
+ sum_nn += dt;
+ }
+ }
+
+ }
+ }
+
+ /**
+ * Parameterization class
+ *
+ * @author Jonathan von Brünken
+ *
+ * @apiviz.exclude
+ *
+ * @param <O> Vector type
+ */
+ public static class Parameterizer<O extends NumberVector<O, ?>> extends AbstractParameterizer {
+ /**
+ * Parameter to specify how many next neighbors should be used in the
+ * computation
+ */
+ public static final OptionID K_ID = OptionID.getOrCreateOptionID("HilOut.k", "Compute up to k next neighbors");
+
+ /**
+ * Parameter to specify how many outliers should be computed
+ */
+ public static final OptionID N_ID = OptionID.getOrCreateOptionID("HilOut.n", "Compute n outliers");
+
+ /**
+ * Parameter to specify the maximum Hilbert-Level
+ */
+ public static final OptionID H_ID = OptionID.getOrCreateOptionID("HilOut.h", "Max. Hilbert-Level");
+
+ /**
+ * Parameter to specify p of LP-NormDistance
+ */
+ public static final OptionID T_ID = OptionID.getOrCreateOptionID("HilOut.t", "t of Lt Metric");
+
+ /**
+ * Parameter to specify if only the Top n, or also approximations for the
+ * other elements, should be returned
+ */
+ public static final OptionID TN_ID = OptionID.getOrCreateOptionID("HilOut.tn", "output of Top n or all elements");
+
+ /**
+ * Neighborhood size
+ */
+ protected int k = 5;
+
+ /**
+ * Top-n candidates to compute exactly
+ */
+ protected int n = 10;
+
+ /**
+ * Hilbert curve precision
+ */
+ protected int h = 32;
+
+ /**
+ * LPNorm distance function
+ */
+ protected LPNormDistanceFunction distfunc;
+
+ /**
+ * Scores to report: all or top-n only
+ */
+ protected Enum<ScoreType> tn;
+
+ @Override
+ protected void makeOptions(Parameterization config) {
+ super.makeOptions(config);
+
+ final IntParameter kP = new IntParameter(K_ID, 5);
+ if(config.grab(kP)) {
+ k = kP.getValue();
+ }
+
+ final IntParameter nP = new IntParameter(N_ID, 10);
+ if(config.grab(nP)) {
+ n = nP.getValue();
+ }
+
+ final IntParameter hP = new IntParameter(H_ID, 32);
+ if(config.grab(hP)) {
+ h = hP.getValue();
+ }
+
+ ObjectParameter<LPNormDistanceFunction> distP = AbstractDistanceBasedAlgorithm.makeParameterDistanceFunction(EuclideanDistanceFunction.class, LPNormDistanceFunction.class);
+ if (config.grab(distP)) {
+ distfunc = distP.instantiateClass(config);
+ }
+
+ final EnumParameter<ScoreType> tnP = new EnumParameter<ScoreType>(TN_ID, ScoreType.class, ScoreType.TopN);
+ if(config.grab(tnP)) {
+ tn = tnP.getValue();
+ }
+ }
+
+ @Override
+ protected HilOut<O> makeInstance() {
+ return new HilOut<O>(distfunc, k, n, h, tn);
+ }
+ }
+}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/INFLO.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/INFLO.java index 083a72a6..1fe5fe71 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/INFLO.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/INFLO.java @@ -30,7 +30,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil;
import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore;
import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore;
-import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil;
import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs;
import de.lmu.ifi.dbs.elki.database.query.DatabaseQuery;
@@ -43,6 +43,7 @@ import de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction; import de.lmu.ifi.dbs.elki.distance.distancevalue.NumberDistance;
import de.lmu.ifi.dbs.elki.logging.Logging;
import de.lmu.ifi.dbs.elki.math.DoubleMinMax;
+import de.lmu.ifi.dbs.elki.math.Mean;
import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult;
import de.lmu.ifi.dbs.elki.result.outlier.OutlierScoreMeta;
import de.lmu.ifi.dbs.elki.result.outlier.QuotientOutlierScoreMeta;
@@ -120,9 +121,14 @@ public class INFLO<O, D extends NumberDistance<D, ?>> extends AbstractDistanceBa this.k = k;
}
- @Override
- public OutlierResult run(Database database) throws IllegalStateException {
- Relation<O> relation = database.getRelation(getInputTypeRestriction()[0]);
+ /**
+ * Run the algorithm
+ *
+ * @param database Database to process
+ * @param relation Relation to process
+ * @return Outlier result
+ */
+ public OutlierResult run(Database database, Relation<O> relation) {
DistanceQuery<O, D> distFunc = database.getDistanceQuery(relation, getDistanceFunction());
ModifiableDBIDs processedIDs = DBIDUtil.newHashSet(relation.size());
@@ -134,15 +140,15 @@ public class INFLO<O, D extends NumberDistance<D, ?>> extends AbstractDistanceBa // density
WritableDoubleDataStore density = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_TEMP | DataStoreFactory.HINT_HOT);
// init knns and rnns
- for(DBID id : relation.iterDBIDs()) {
- knns.put(id, DBIDUtil.newArray());
- rnns.put(id, DBIDUtil.newArray());
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + knns.put(iditer, DBIDUtil.newArray());
+ rnns.put(iditer, DBIDUtil.newArray());
}
// TODO: use kNN preprocessor?
KNNQuery<O, D> knnQuery = database.getKNNQuery(distFunc, k, DatabaseQuery.HINT_HEAVY_USE);
- for(DBID id : relation.iterDBIDs()) {
+ for(DBIDIter id = relation.iterDBIDs(); id.valid(); id.advance()) { // if not visited count=0
int count = rnns.get(id).size();
ModifiableDBIDs s;
@@ -158,7 +164,7 @@ public class INFLO<O, D extends NumberDistance<D, ?>> extends AbstractDistanceBa else {
s = knns.get(id);
}
- for(DBID q : s) {
+ for (DBIDIter q = s.iter(); q.valid(); q.advance()) {
if(!processedIDs.contains(q)) {
// TODO: use exactly k neighbors?
KNNResult<D> listQ = knnQuery.getKNNForDBID(q, k);
@@ -182,20 +188,18 @@ public class INFLO<O, D extends NumberDistance<D, ?>> extends AbstractDistanceBa // IF Object is pruned INFLO=1.0
DoubleMinMax inflominmax = new DoubleMinMax();
WritableDoubleDataStore inflos = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);
- for(DBID id : relation.iterDBIDs()) {
+ for(DBIDIter id = relation.iterDBIDs(); id.valid(); id.advance()) { if(!pruned.contains(id)) {
ModifiableDBIDs knn = knns.get(id);
ModifiableDBIDs rnn = rnns.get(id);
double denP = density.doubleValue(id);
knn.addDBIDs(rnn);
- double den = 0;
- for(DBID q : knn) {
- double denQ = density.doubleValue(q);
- den = den + denQ;
+ Mean mean = new Mean();
+ for (DBIDIter iter = knn.iter(); iter.valid(); iter.advance()) {
+ mean.put(density.doubleValue(iter));
}
- den = den / rnn.size();
- den = den / denP;
+ double den = mean.getMean() / denP;
inflos.putDouble(id, den);
// update minimum and maximum
inflominmax.put(den);
diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/KNNOutlier.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/KNNOutlier.java index ee748f99..08be944a 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/KNNOutlier.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/KNNOutlier.java @@ -29,7 +29,7 @@ import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory;
import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil;
import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore;
-import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery;
import de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery;
import de.lmu.ifi.dbs.elki.database.query.knn.KNNResult;
@@ -115,11 +115,11 @@ public class KNNOutlier<O, D extends NumberDistance<D, ?>> extends AbstractDista DoubleMinMax minmax = new DoubleMinMax();
WritableDoubleDataStore knno_score = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);
// compute distance to the k nearest neighbor.
- for(DBID id : relation.iterDBIDs()) {
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { // distance to the kth nearest neighbor
- final KNNResult<D> knns = knnQuery.getKNNForDBID(id, k);
+ final KNNResult<D> knns = knnQuery.getKNNForDBID(iditer, k);
double dkn = knns.getKNNDistance().doubleValue();
- knno_score.putDouble(id, dkn);
+ knno_score.putDouble(iditer, dkn);
minmax.put(dkn);
diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/KNNWeightOutlier.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/KNNWeightOutlier.java index e9657e12..cb3ca2f1 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/KNNWeightOutlier.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/KNNWeightOutlier.java @@ -30,7 +30,7 @@ import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory;
import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil;
import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore;
-import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair;
import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery;
import de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery;
@@ -119,15 +119,15 @@ public class KNNWeightOutlier<O, D extends NumberDistance<D, ?>> extends Abstrac // compute distance to the k nearest neighbor. n objects with the highest
// distance are flagged as outliers
WritableDoubleDataStore knnw_score = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);
- for(DBID id : relation.iterDBIDs()) {
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { // compute sum of the distances to the k nearest neighbors
- final KNNResult<D> knn = knnQuery.getKNNForDBID(id, k);
+ final KNNResult<D> knn = knnQuery.getKNNForDBID(iditer, k);
double skn = 0;
for(DistanceResultPair<D> r : knn) {
skn += r.getDistance().doubleValue();
}
- knnw_score.putDouble(id, skn);
+ knnw_score.putDouble(iditer, skn);
minmax.put(skn);
if(progressKNNWeight != null) {
diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/LDOF.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/LDOF.java index d9256428..84f5dcc6 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/LDOF.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/LDOF.java @@ -30,7 +30,7 @@ import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory;
import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil;
import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore;
-import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair;
import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery;
import de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery;
@@ -42,6 +42,7 @@ import de.lmu.ifi.dbs.elki.distance.distancevalue.NumberDistance; import de.lmu.ifi.dbs.elki.logging.Logging;
import de.lmu.ifi.dbs.elki.logging.progress.FiniteProgress;
import de.lmu.ifi.dbs.elki.math.DoubleMinMax;
+import de.lmu.ifi.dbs.elki.math.Mean;
import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult;
import de.lmu.ifi.dbs.elki.result.outlier.OutlierScoreMeta;
import de.lmu.ifi.dbs.elki.result.outlier.QuotientOutlierScoreMeta;
@@ -110,7 +111,14 @@ public class LDOF<O, D extends NumberDistance<D, ?>> extends AbstractDistanceBas this.k = k;
}
- public OutlierResult run(Database database, Relation<O> relation) throws IllegalStateException {
+ /**
+ * Run the algorithm
+ *
+ * @param database Database to process
+ * @param relation Relation to process
+ * @return Outlier result
+ */
+ public OutlierResult run(Database database, Relation<O> relation) {
DistanceQuery<O, D> distFunc = database.getDistanceQuery(relation, getDistanceFunction());
KNNQuery<O, D> knnQuery = database.getKNNQuery(distFunc, k);
@@ -125,29 +133,26 @@ public class LDOF<O, D extends NumberDistance<D, ?>> extends AbstractDistanceBas }
FiniteProgress progressLDOFs = logger.isVerbose() ? new FiniteProgress("LDOF_SCORE for objects", relation.size(), logger) : null;
- for(DBID id : relation.iterDBIDs()) {
- KNNResult<D> neighbors = knnQuery.getKNNForDBID(id, k);
- int nsize = neighbors.size() - 1;
+ Mean dxp = new Mean(), Dxp = new Mean();
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + KNNResult<D> neighbors = knnQuery.getKNNForDBID(iditer, k);
// skip the point itself
- double dxp = 0;
- double Dxp = 0;
+ dxp.reset(); Dxp.reset();
for(DistanceResultPair<D> neighbor1 : neighbors) {
- if(!neighbor1.getDBID().equals(id)) {
- dxp += neighbor1.getDistance().doubleValue();
+ if(!neighbor1.sameDBID(iditer)) {
+ dxp.put(neighbor1.getDistance().doubleValue());
for(DistanceResultPair<D> neighbor2 : neighbors) {
- if(!neighbor1.getDBID().equals(neighbor2.getDBID()) && !neighbor2.getDBID().equals(id)) {
- Dxp += distFunc.distance(neighbor1.getDBID(), neighbor2.getDBID()).doubleValue();
+ if(!neighbor1.sameDBID(neighbor2) && !neighbor2.sameDBID(iditer)) {
+ Dxp.put(distFunc.distance(neighbor1, neighbor2).doubleValue());
}
}
}
}
- dxp /= nsize;
- Dxp /= (nsize * (nsize - 1));
- Double ldof = dxp / Dxp;
- if(ldof.isNaN() || ldof.isInfinite()) {
+ double ldof = dxp.getMean() / Dxp.getMean();
+ if(Double.isNaN(ldof) || Double.isInfinite(ldof)) {
ldof = 1.0;
}
- ldofs.putDouble(id, ldof);
+ ldofs.putDouble(iditer, ldof);
// update maximum
ldofminmax.put(ldof);
diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/LOCI.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/LOCI.java index cfd8623c..a04aa041 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/LOCI.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/LOCI.java @@ -35,7 +35,8 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.query.DistanceDBIDResult; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.query.range.RangeQuery; @@ -136,19 +137,21 @@ public class LOCI<O, D extends NumberDistance<D, ?>> extends AbstractDistanceBas } /** - * Runs the algorithm in the timed evaluation part. + * Run the algorithm + * + * @param database Database to process + * @param relation Relation to process + * @return Outlier result */ - @Override - public OutlierResult run(Database database) throws IllegalStateException { - Relation<O> relation = database.getRelation(getInputTypeRestriction()[0]); + public OutlierResult run(Database database, Relation<O> relation) { DistanceQuery<O, D> distFunc = database.getDistanceQuery(relation, getDistanceFunction()); RangeQuery<O, D> rangeQuery = database.getRangeQuery(distFunc); FiniteProgress progressPreproc = logger.isVerbose() ? new FiniteProgress("LOCI preprocessing", relation.size(), logger) : null; // LOCI preprocessing step WritableDataStore<ArrayList<DoubleIntPair>> interestingDistances = DataStoreUtil.makeStorage(relation.getDBIDs(), DataStoreFactory.HINT_TEMP | DataStoreFactory.HINT_SORTED, ArrayList.class); - for(DBID id : relation.iterDBIDs()) { - List<DistanceResultPair<D>> neighbors = rangeQuery.getRangeForDBID(id, rmax); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DistanceDBIDResult<D> neighbors = rangeQuery.getRangeForDBID(iditer, rmax); // build list of critical distances ArrayList<DoubleIntPair> cdist = new ArrayList<DoubleIntPair>(neighbors.size() * 2); { @@ -177,7 +180,7 @@ public class LOCI<O, D extends NumberDistance<D, ?>> extends AbstractDistanceBas } } - interestingDistances.put(id, cdist); + interestingDistances.put(iditer, cdist); if(progressPreproc != null) { progressPreproc.incrementProcessed(logger); } @@ -191,8 +194,8 @@ public class LOCI<O, D extends NumberDistance<D, ?>> extends AbstractDistanceBas WritableDoubleDataStore mdef_radius = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC); DoubleMinMax minmax = new DoubleMinMax(); - for(DBID id : relation.iterDBIDs()) { - final List<DoubleIntPair> cdist = interestingDistances.get(id); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + final List<DoubleIntPair> cdist = interestingDistances.get(iditer); final double maxdist = cdist.get(cdist.size() - 1).first; final int maxneig = cdist.get(cdist.size() - 1).second; @@ -201,7 +204,7 @@ public class LOCI<O, D extends NumberDistance<D, ?>> extends AbstractDistanceBas if(maxneig >= nmin) { D range = distFunc.getDistanceFactory().fromDouble(maxdist); // Compute the largest neighborhood we will need. - List<DistanceResultPair<D>> maxneighbors = rangeQuery.getRangeForDBID(id, range); + List<DistanceResultPair<D>> maxneighbors = rangeQuery.getRangeForDBID(iditer, range); // Ensure the set is sorted. Should be a no-op with most indexes. Collections.sort(maxneighbors); // For any critical distance, compute the normalized MDEF score. @@ -221,7 +224,7 @@ public class LOCI<O, D extends NumberDistance<D, ?>> extends AbstractDistanceBas if(ne.getDistance().doubleValue() > r) { break; } - int rn_alphar = elementsAtRadius(interestingDistances.get(ne.getDBID()), alpha_r); + int rn_alphar = elementsAtRadius(interestingDistances.get(ne), alpha_r); mv_n_r_alpha.put(rn_alphar); } // We only use the average and standard deviation @@ -244,8 +247,8 @@ public class LOCI<O, D extends NumberDistance<D, ?>> extends AbstractDistanceBas maxmdefnorm = 1.0; maxnormr = maxdist; } - mdef_norm.putDouble(id, maxmdefnorm); - mdef_radius.putDouble(id, maxnormr); + mdef_norm.putDouble(iditer, maxmdefnorm); + mdef_radius.putDouble(iditer, maxnormr); minmax.put(maxmdefnorm); if(progressLOCI != null) { progressLOCI.incrementProcessed(logger); @@ -255,7 +258,7 @@ public class LOCI<O, D extends NumberDistance<D, ?>> extends AbstractDistanceBas progressLOCI.ensureCompleted(logger); } Relation<Double> scoreResult = new MaterializedRelation<Double>("LOCI normalized MDEF", "loci-mdef-outlier", TypeUtil.DOUBLE, mdef_norm, relation.getDBIDs()); - OutlierScoreMeta scoreMeta = new QuotientOutlierScoreMeta(minmax.getMin(), minmax.getMax(), Double.POSITIVE_INFINITY, 0.0); + OutlierScoreMeta scoreMeta = new QuotientOutlierScoreMeta(minmax.getMin(), minmax.getMax(), 0.0, Double.POSITIVE_INFINITY, 0.0); OutlierResult result = new OutlierResult(scoreMeta, scoreResult); result.addChildResult(new MaterializedRelation<Double>("LOCI MDEF Radius", "loci-critical-radius", TypeUtil.DOUBLE, mdef_radius, relation.getDBIDs())); return result; diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/LOF.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/LOF.java index 85e1aef2..5aba41ec 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/LOF.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/LOF.java @@ -33,7 +33,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStore; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.query.DatabaseQuery; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; @@ -51,6 +51,7 @@ import de.lmu.ifi.dbs.elki.logging.Logging; import de.lmu.ifi.dbs.elki.logging.progress.FiniteProgress; import de.lmu.ifi.dbs.elki.logging.progress.StepProgress; import de.lmu.ifi.dbs.elki.math.DoubleMinMax; +import de.lmu.ifi.dbs.elki.math.Mean; import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; import de.lmu.ifi.dbs.elki.result.outlier.OutlierScoreMeta; import de.lmu.ifi.dbs.elki.result.outlier.QuotientOutlierScoreMeta; @@ -174,13 +175,15 @@ public class LOF<O, D extends NumberDistance<D, ?>> extends AbstractAlgorithm<Ou * @param k the value of k * @param distanceFunction the distance function * - * Uses the same distance function for neighborhood computation and reachability distance (standard as in the original publication), - * same as {@link #LOF(int, DistanceFunction, DistanceFunction) LOF(int, distanceFunction, distanceFunction)}. + * Uses the same distance function for neighborhood computation and + * reachability distance (standard as in the original publication), + * same as {@link #LOF(int, DistanceFunction, DistanceFunction) + * LOF(int, distanceFunction, distanceFunction)}. */ public LOF(int k, DistanceFunction<? super O, D> distanceFunction) { this(k, distanceFunction, distanceFunction); } - + /** * Performs the Generalized LOF_SCORE algorithm on the given database by * calling {@link #doRunInTime}. @@ -239,11 +242,14 @@ public class LOF<O, D extends NumberDistance<D, ?>> extends AbstractAlgorithm<Ou * returns a {@link LOF.LOFResult} encapsulating information that may be * needed by an OnlineLOF algorithm. * + * @param ids Object ids * @param kNNRefer the kNN query w.r.t. reference neighborhood distance * function * @param kNNReach the kNN query w.r.t. reachability distance function + * @param stepprog Progress logger + * @return LOF result */ - protected LOFResult<O, D> doRunInTime(DBIDs ids, KNNQuery<O, D> kNNRefer, KNNQuery<O, D> kNNReach, StepProgress stepprog) throws IllegalStateException { + protected LOFResult<O, D> doRunInTime(DBIDs ids, KNNQuery<O, D> kNNRefer, KNNQuery<O, D> kNNReach, StepProgress stepprog) { // Assert we got something if(kNNRefer == null) { throw new AbortException("No kNN queries supported by database for reference neighborhood distance function."); @@ -290,19 +296,19 @@ public class LOF<O, D extends NumberDistance<D, ?>> extends AbstractAlgorithm<Ou protected WritableDoubleDataStore computeLRDs(DBIDs ids, KNNQuery<O, D> knnReach) { WritableDoubleDataStore lrds = DataStoreUtil.makeDoubleStorage(ids, DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_TEMP); FiniteProgress lrdsProgress = logger.isVerbose() ? new FiniteProgress("LRD", ids.size(), logger) : null; - for(DBID id : ids) { - double sum = 0; - KNNResult<D> neighbors = knnReach.getKNNForDBID(id, k); - int nsize = neighbors.size() - (objectIsInKNN ? 0 : 1); + Mean mean = new Mean(); + for(DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + mean.reset(); + KNNResult<D> neighbors = knnReach.getKNNForDBID(iter, k); for(DistanceResultPair<D> neighbor : neighbors) { - if(objectIsInKNN || !neighbor.getDBID().equals(id)) { - KNNResult<D> neighborsNeighbors = knnReach.getKNNForDBID(neighbor.getDBID(), k); - sum += Math.max(neighbor.getDistance().doubleValue(), neighborsNeighbors.getKNNDistance().doubleValue()); + if(objectIsInKNN || !neighbor.sameDBID(iter)) { + KNNResult<D> neighborsNeighbors = knnReach.getKNNForDBID(neighbor, k); + mean.put(Math.max(neighbor.getDistance().doubleValue(), neighborsNeighbors.getKNNDistance().doubleValue())); } } // Avoid division by 0 - double lrd = (sum > 0) ? nsize / sum : 0.0; - lrds.putDouble(id, lrd); + final double lrd = (mean.getCount() > 0) ? 1 / mean.getMean() : 0.0; + lrds.putDouble(iter, lrd); if(lrdsProgress != null) { lrdsProgress.incrementProcessed(logger); } @@ -328,26 +334,25 @@ public class LOF<O, D extends NumberDistance<D, ?>> extends AbstractAlgorithm<Ou DoubleMinMax lofminmax = new DoubleMinMax(); FiniteProgress progressLOFs = logger.isVerbose() ? new FiniteProgress("LOF_SCORE for objects", ids.size(), logger) : null; - for(DBID id : ids) { - double lrdp = lrds.get(id); + Mean mean = new Mean(); + for(DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + double lrdp = lrds.get(iter); final double lof; if(lrdp > 0) { - final KNNResult<D> neighbors = knnRefer.getKNNForDBID(id, k); - int nsize = neighbors.size() - (objectIsInKNN ? 0 : 1); - // skip the point itself - // neighbors.remove(0); - double sum = 0; + final KNNResult<D> neighbors = knnRefer.getKNNForDBID(iter, k); + mean.reset(); for(DistanceResultPair<D> neighbor : neighbors) { - if(objectIsInKNN || !neighbor.getDBID().equals(id)) { - sum += lrds.get(neighbor.getDBID()); + // skip the point itself + if(objectIsInKNN || !neighbor.sameDBID(iter)) { + mean.put(lrds.get(neighbor)); } } - lof = (sum / nsize) / lrdp; + lof = mean.getMean() / lrdp; } else { lof = 1.0; } - lofs.putDouble(id, lof); + lofs.putDouble(iter, lof); // update minimum and maximum lofminmax.put(lof); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/LoOP.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/LoOP.java index f1c273f6..dc0d26a4 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/LoOP.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/LoOP.java @@ -32,7 +32,7 @@ import de.lmu.ifi.dbs.elki.database.QueryUtil; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.query.DatabaseQuery; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; @@ -47,6 +47,7 @@ import de.lmu.ifi.dbs.elki.index.preprocessed.knn.MaterializeKNNPreprocessor; import de.lmu.ifi.dbs.elki.logging.Logging; import de.lmu.ifi.dbs.elki.logging.progress.FiniteProgress; import de.lmu.ifi.dbs.elki.logging.progress.StepProgress; +import de.lmu.ifi.dbs.elki.math.Mean; import de.lmu.ifi.dbs.elki.math.MeanVariance; import de.lmu.ifi.dbs.elki.math.statistics.distribution.NormalDistribution; import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; @@ -206,8 +207,12 @@ public class LoOP<O, D extends NumberDistance<D, ?>> extends AbstractAlgorithm<O /** * Performs the LoOP algorithm on the given database. + * + * @param database Database to process + * @param relation Relation to process + * @return Outlier result */ - public OutlierResult run(Database database, Relation<O> relation) throws IllegalStateException { + public OutlierResult run(Database database, Relation<O> relation) { final double sqrt2 = Math.sqrt(2.0); StepProgress stepprog = logger.isVerbose() ? new StepProgress(5) : null; @@ -226,28 +231,29 @@ public class LoOP<O, D extends NumberDistance<D, ?>> extends AbstractAlgorithm<O // Probabilistic distances WritableDoubleDataStore pdists = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_TEMP); + Mean mean = new Mean(); {// computing PRDs if(stepprog != null) { stepprog.beginStep(3, "Computing pdists", logger); } FiniteProgress prdsProgress = logger.isVerbose() ? new FiniteProgress("pdists", relation.size(), logger) : null; - for(DBID id : relation.iterDBIDs()) { - final KNNResult<D> neighbors = knnReach.getKNNForDBID(id, kreach); - double sqsum = 0.0; + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + final KNNResult<D> neighbors = knnReach.getKNNForDBID(iditer, kreach); + mean.reset(); // use first kref neighbors as reference set int ks = 0; for(DistanceResultPair<D> neighbor : neighbors) { - if(objectIsInKNN || !neighbor.getDBID().equals(id)) { + if(objectIsInKNN || !neighbor.sameDBID(iditer)) { double d = neighbor.getDistance().doubleValue(); - sqsum += d * d; + mean.put(d * d); ks++; if(ks >= kreach) { break; } } } - double pdist = lambda * Math.sqrt(sqsum / ks); - pdists.putDouble(id, pdist); + double pdist = lambda * Math.sqrt(mean.getMean()); + pdists.putDouble(iditer, pdist); if(prdsProgress != null) { prdsProgress.incrementProcessed(logger); } @@ -262,25 +268,26 @@ public class LoOP<O, D extends NumberDistance<D, ?>> extends AbstractAlgorithm<O } FiniteProgress progressPLOFs = logger.isVerbose() ? new FiniteProgress("PLOFs for objects", relation.size(), logger) : null; - for(DBID id : relation.iterDBIDs()) { - final KNNResult<D> neighbors = knnComp.getKNNForDBID(id, kcomp); - MeanVariance mv = new MeanVariance(); + MeanVariance mv = new MeanVariance(); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + final KNNResult<D> neighbors = knnComp.getKNNForDBID(iditer, kcomp); + mv.reset(); // use first kref neighbors as comparison set. int ks = 0; for(DistanceResultPair<D> neighbor1 : neighbors) { - if(objectIsInKNN || !neighbor1.getDBID().equals(id)) { - mv.put(pdists.doubleValue(neighbor1.getDBID())); + if(objectIsInKNN || !neighbor1.sameDBID(iditer)) { + mv.put(pdists.doubleValue(neighbor1)); ks++; if(ks >= kcomp) { break; } } } - double plof = Math.max(pdists.doubleValue(id) / mv.getMean(), 1.0); + double plof = Math.max(pdists.doubleValue(iditer) / mv.getMean(), 1.0); if(Double.isNaN(plof) || Double.isInfinite(plof)) { plof = 1.0; } - plofs.putDouble(id, plof); + plofs.putDouble(iditer, plof); mvplof.put((plof - 1.0) * (plof - 1.0)); if(progressPLOFs != null) { @@ -302,8 +309,8 @@ public class LoOP<O, D extends NumberDistance<D, ?>> extends AbstractAlgorithm<O } FiniteProgress progressLOOPs = logger.isVerbose() ? new FiniteProgress("LoOP for objects", relation.size(), logger) : null; - for(DBID id : relation.iterDBIDs()) { - loops.putDouble(id, NormalDistribution.erf((plofs.doubleValue(id) - 1) / (nplof * sqrt2))); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + loops.putDouble(iditer, NormalDistribution.erf((plofs.doubleValue(iditer) - 1) / (nplof * sqrt2))); if(progressLOOPs != null) { progressLOOPs.incrementProcessed(logger); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/OPTICSOF.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/OPTICSOF.java index 2f120c44..b3d24463 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/OPTICSOF.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/OPTICSOF.java @@ -34,19 +34,20 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil;
import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore;
import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore;
-import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.datastore.WritableIntegerDataStore;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDs;
import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair;
import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery;
import de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery;
import de.lmu.ifi.dbs.elki.database.query.knn.KNNResult;
import de.lmu.ifi.dbs.elki.database.query.range.RangeQuery;
+import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation;
import de.lmu.ifi.dbs.elki.database.relation.Relation;
import de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction;
import de.lmu.ifi.dbs.elki.distance.distancevalue.NumberDistance;
import de.lmu.ifi.dbs.elki.logging.Logging;
import de.lmu.ifi.dbs.elki.math.DoubleMinMax;
-import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation;
import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult;
import de.lmu.ifi.dbs.elki.result.outlier.OutlierScoreMeta;
import de.lmu.ifi.dbs.elki.result.outlier.QuotientOutlierScoreMeta;
@@ -116,51 +117,49 @@ public class OPTICSOF<O, D extends NumberDistance<D, ?>> extends AbstractDistanc // FIXME: implicit preprocessor.
WritableDataStore<KNNResult<D>> nMinPts = DataStoreUtil.makeStorage(ids, DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_TEMP, KNNResult.class);
WritableDoubleDataStore coreDistance = DataStoreUtil.makeDoubleStorage(ids, DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_TEMP);
- WritableDataStore<Integer> minPtsNeighborhoodSize = DataStoreUtil.makeStorage(ids, DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_TEMP, Integer.class);
+ WritableIntegerDataStore minPtsNeighborhoodSize = DataStoreUtil.makeIntegerStorage(ids, DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_TEMP, -1);
// Pass 1
// N_minpts(id) and core-distance(id)
- for(DBID id : relation.iterDBIDs()) {
- KNNResult<D> minptsNeighbours = knnQuery.getKNNForDBID(id, minpts);
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + KNNResult<D> minptsNeighbours = knnQuery.getKNNForDBID(iditer, minpts);
D d = minptsNeighbours.getKNNDistance();
- nMinPts.put(id, minptsNeighbours);
- coreDistance.putDouble(id, d.doubleValue());
- minPtsNeighborhoodSize.put(id, rangeQuery.getRangeForDBID(id, d).size());
+ nMinPts.put(iditer, minptsNeighbours);
+ coreDistance.putDouble(iditer, d.doubleValue());
+ minPtsNeighborhoodSize.put(iditer, rangeQuery.getRangeForDBID(iditer, d).size());
}
// Pass 2
WritableDataStore<List<Double>> reachDistance = DataStoreUtil.makeStorage(ids, DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_TEMP, List.class);
WritableDoubleDataStore lrds = DataStoreUtil.makeDoubleStorage(ids, DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_TEMP);
- for(DBID id : relation.iterDBIDs()) {
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { List<Double> core = new ArrayList<Double>();
double lrd = 0;
- for(DistanceResultPair<D> neighPair : nMinPts.get(id)) {
- DBID idN = neighPair.getDBID();
- double coreDist = coreDistance.doubleValue(idN);
- double dist = distQuery.distance(id, idN).doubleValue();
- Double rd = Math.max(coreDist, dist);
+ for(DistanceResultPair<D> neighPair : nMinPts.get(iditer)) {
+ double coreDist = coreDistance.doubleValue(neighPair);
+ double dist = distQuery.distance(iditer, neighPair).doubleValue();
+ double rd = Math.max(coreDist, dist);
lrd = rd + lrd;
core.add(rd);
}
- lrd = (minPtsNeighborhoodSize.get(id) / lrd);
- reachDistance.put(id, core);
- lrds.putDouble(id, lrd);
+ lrd = minPtsNeighborhoodSize.intValue(iditer) / lrd;
+ reachDistance.put(iditer, core);
+ lrds.putDouble(iditer, lrd);
}
// Pass 3
DoubleMinMax ofminmax = new DoubleMinMax();
WritableDoubleDataStore ofs = DataStoreUtil.makeDoubleStorage(ids, DataStoreFactory.HINT_STATIC);
- for(DBID id : relation.iterDBIDs()) {
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { double of = 0;
- for(DistanceResultPair<D> pair : nMinPts.get(id)) {
- DBID idN = pair.getDBID();
- double lrd = lrds.doubleValue(id);
- double lrdN = lrds.doubleValue(idN);
+ for(DistanceResultPair<D> pair : nMinPts.get(iditer)) {
+ double lrd = lrds.doubleValue(iditer);
+ double lrdN = lrds.doubleValue(pair);
of = of + lrdN / lrd;
}
- of = of / minPtsNeighborhoodSize.get(id);
- ofs.putDouble(id, of);
+ of = of / minPtsNeighborhoodSize.intValue(iditer);
+ ofs.putDouble(iditer, of);
// update minimum and maximum
ofminmax.put(of);
}
diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/OnlineLOF.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/OnlineLOF.java index ad17398c..9b974ad9 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/OnlineLOF.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/OnlineLOF.java @@ -29,7 +29,7 @@ import de.lmu.ifi.dbs.elki.database.QueryUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore;
import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs;
import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs;
-import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil;
import de.lmu.ifi.dbs.elki.database.ids.DBIDs;
import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs;
@@ -67,7 +67,6 @@ import de.lmu.ifi.dbs.elki.utilities.pairs.Pair; * @author Elke Achtert
*
* @apiviz.has LOF.LOFResult oneway - - updates
- * @apiviz.composedOf OnlineLOF.LOFKNNListener
*/
// TODO: related to publication?
public class OnlineLOF<O, D extends NumberDistance<D, ?>> extends LOF<O, D> {
@@ -170,6 +169,10 @@ public class OnlineLOF<O, D extends NumberDistance<D, ?>> extends LOF<O, D> { /**
* Encapsulates a listener for changes of kNNs used in the online LOF
* algorithm.
+ *
+ * @author Elke Achtert
+ *
+ * @apiviz.exclude
*/
private class LOFKNNListener implements KNNListener {
/**
@@ -269,12 +272,12 @@ public class OnlineLOF<O, D extends NumberDistance<D, ?>> extends LOF<O, D> { ArrayDBIDs affected_lrd_id_candidates = mergeIDs(reachDistRKNNs, lrd_ids);
ArrayModifiableDBIDs affected_lrd_ids = DBIDUtil.newArray(affected_lrd_id_candidates.size());
WritableDoubleDataStore new_lrds = computeLRDs(affected_lrd_id_candidates, lofResult.getKNNReach());
- for(DBID id : affected_lrd_id_candidates) {
- double new_lrd = new_lrds.doubleValue(id);
- double old_lrd = lofResult.getLrds().doubleValue(id);
+ for (DBIDIter iter = affected_lrd_id_candidates.iter(); iter.valid(); iter.advance()) {
+ double new_lrd = new_lrds.doubleValue(iter);
+ double old_lrd = lofResult.getLrds().doubleValue(iter);
if(Double.isNaN(old_lrd) || old_lrd != new_lrd) {
- lofResult.getLrds().putDouble(id, new_lrd);
- affected_lrd_ids.add(id);
+ lofResult.getLrds().putDouble(iter, new_lrd);
+ affected_lrd_ids.add(iter);
}
}
@@ -314,9 +317,9 @@ public class OnlineLOF<O, D extends NumberDistance<D, ?>> extends LOF<O, D> { if(stepprog != null) {
stepprog.beginStep(1, "Delete old LRDs and LOFs.", logger);
}
- for(DBID id : deletions) {
- lofResult.getLrds().delete(id);
- lofResult.getLofs().delete(id);
+ for (DBIDIter iter = deletions.iter(); iter.valid(); iter.advance()) {
+ lofResult.getLrds().delete(iter);
+ lofResult.getLofs().delete(iter);
}
// recompute lrds
@@ -328,12 +331,12 @@ public class OnlineLOF<O, D extends NumberDistance<D, ?>> extends LOF<O, D> { ArrayDBIDs affected_lrd_id_candidates = mergeIDs(reachDistRKNNs, lrd_ids);
ArrayModifiableDBIDs affected_lrd_ids = DBIDUtil.newArray(affected_lrd_id_candidates.size());
WritableDoubleDataStore new_lrds = computeLRDs(affected_lrd_id_candidates, lofResult.getKNNReach());
- for(DBID id : affected_lrd_id_candidates) {
- double new_lrd = new_lrds.doubleValue(id);
- double old_lrd = lofResult.getLrds().doubleValue(id);
+ for (DBIDIter iter = affected_lrd_id_candidates.iter(); iter.valid(); iter.advance()) {
+ double new_lrd = new_lrds.doubleValue(iter);
+ double old_lrd = lofResult.getLrds().doubleValue(iter);
if(old_lrd != new_lrd) {
- lofResult.getLrds().putDouble(id, new_lrd);
- affected_lrd_ids.add(id);
+ lofResult.getLrds().putDouble(iter, new_lrd);
+ affected_lrd_ids.add(iter);
}
}
@@ -371,7 +374,7 @@ public class OnlineLOF<O, D extends NumberDistance<D, ?>> extends LOF<O, D> { }
for(List<DistanceResultPair<D>> queryResult : queryResults) {
for(DistanceResultPair<D> qr : queryResult) {
- result.add(qr.getDBID());
+ result.add(qr);
}
}
return DBIDUtil.newArray(result);
@@ -386,8 +389,8 @@ public class OnlineLOF<O, D extends NumberDistance<D, ?>> extends LOF<O, D> { private void recomputeLOFs(DBIDs ids, LOFResult<O, D> lofResult) {
Pair<WritableDoubleDataStore, DoubleMinMax> lofsAndMax = computeLOFs(ids, lofResult.getLrds(), lofResult.getKNNRefer());
WritableDoubleDataStore new_lofs = lofsAndMax.getFirst();
- for(DBID id : ids) {
- lofResult.getLofs().putDouble(id, new_lofs.doubleValue(id));
+ for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) {
+ lofResult.getLofs().putDouble(iter, new_lofs.doubleValue(iter));
}
// track the maximum value for normalization.
DoubleMinMax new_lofminmax = lofsAndMax.getSecond();
diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/OutlierAlgorithm.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/OutlierAlgorithm.java index 2b122183..d8322d8b 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/OutlierAlgorithm.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/OutlierAlgorithm.java @@ -38,5 +38,5 @@ public interface OutlierAlgorithm extends Algorithm { // Note: usually you won't override this method directly, but instead // Use the magic in AbstractAlgorithm and just implement a run method for your input data @Override - OutlierResult run(Database database) throws IllegalStateException; + OutlierResult run(Database database); }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/ReferenceBasedOutlierDetection.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/ReferenceBasedOutlierDetection.java index befd03ed..dd1d37a3 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/ReferenceBasedOutlierDetection.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/ReferenceBasedOutlierDetection.java @@ -37,7 +37,7 @@ import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory;
import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil;
import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore;
-import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDs;
import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair;
import de.lmu.ifi.dbs.elki.database.query.GenericDistanceResultPair;
@@ -87,7 +87,7 @@ import de.lmu.ifi.dbs.elki.utilities.referencepoints.ReferencePointsHeuristic; */
@Title("An Efficient Reference-based Approach to Outlier Detection in Large Datasets")
@Description("Computes kNN distances approximately, using reference points with various reference point strategies.")
-@Reference(authors = "Y. Pei, O.R. Zaiane, Y. Gao", title = "An Efficient Reference-based Approach to Outlier Detection in Large Datasets", booktitle = "Proc. 19th IEEE Int. Conf. on Data Engineering (ICDE '03), Bangalore, India, 2003", url = "http://dx.doi.org/10.1109/ICDM.2006.17")
+@Reference(authors = "Y. Pei, O.R. Zaiane, Y. Gao", title = "An Efficient Reference-based Approach to Outlier Detection in Large Datasets", booktitle = "Proc. 6th IEEE Int. Conf. on Data Mining (ICDM '06), Hong Kong, China, 2006", url = "http://dx.doi.org/10.1109/ICDM.2006.17")
public class ReferenceBasedOutlierDetection<V extends NumberVector<?, ?>, D extends NumberDistance<D, ?>> extends AbstractAlgorithm<OutlierResult> implements OutlierAlgorithm {
/**
* The logger for this class.
@@ -164,7 +164,7 @@ public class ReferenceBasedOutlierDetection<V extends NumberVector<?, ?>, D exte for(int l = 0; l < firstReferenceDists.size(); l++) {
double density = computeDensity(firstReferenceDists, l);
// Initial value
- rbod_score.putDouble(firstReferenceDists.get(l).getDBID(), density);
+ rbod_score.putDouble(firstReferenceDists.get(l), density);
}
// compute density values for all remaining reference points
while(iter.hasNext()) {
@@ -174,24 +174,24 @@ public class ReferenceBasedOutlierDetection<V extends NumberVector<?, ?>, D exte for(int l = 0; l < referenceDists.size(); l++) {
double density = computeDensity(referenceDists, l);
// Update minimum
- if(density < rbod_score.doubleValue(referenceDists.get(l).getDBID())) {
- rbod_score.putDouble(referenceDists.get(l).getDBID(), density);
+ if(density < rbod_score.doubleValue(referenceDists.get(l))) {
+ rbod_score.putDouble(referenceDists.get(l), density);
}
}
}
}
// compute maximum density
double maxDensity = 0.0;
- for(DBID id : relation.iterDBIDs()) {
- double dens = rbod_score.doubleValue(id);
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + double dens = rbod_score.doubleValue(iditer);
if(dens > maxDensity) {
maxDensity = dens;
}
}
// compute ROS
- for(DBID id : relation.iterDBIDs()) {
- double score = 1 - (rbod_score.doubleValue(id) / maxDensity);
- rbod_score.putDouble(id, score);
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + double score = 1 - (rbod_score.doubleValue(iditer) / maxDensity);
+ rbod_score.putDouble(iditer, score);
}
// adds reference points to the result. header information for the
@@ -218,9 +218,9 @@ public class ReferenceBasedOutlierDetection<V extends NumberVector<?, ?>, D exte protected List<DistanceResultPair<D>> computeDistanceVector(V refPoint, Relation<V> database, DistanceQuery<V, D> distFunc) {
// TODO: optimize for double distances?
List<DistanceResultPair<D>> referenceDists = new ArrayList<DistanceResultPair<D>>(database.size());
- for(DBID id : database.iterDBIDs()) {
- final D distance = distFunc.distance(id, refPoint);
- referenceDists.add(new GenericDistanceResultPair<D>(distance, id));
+ for(DBIDIter iditer = database.iterDBIDs(); iditer.valid(); iditer.advance()) { + final D distance = distFunc.distance(iditer, refPoint);
+ referenceDists.add(new GenericDistanceResultPair<D>(distance, iditer.getDBID()));
}
Collections.sort(referenceDists);
return referenceDists;
diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/meta/ExternalDoubleOutlierScore.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/meta/ExternalDoubleOutlierScore.java index 22447454..1542b8e3 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/meta/ExternalDoubleOutlierScore.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/meta/ExternalDoubleOutlierScore.java @@ -40,7 +40,7 @@ import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation; import de.lmu.ifi.dbs.elki.database.relation.Relation; @@ -142,7 +142,7 @@ public class ExternalDoubleOutlierScore extends AbstractAlgorithm<OutlierResult> public OutlierResult run(Database database, Relation<?> relation) { WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC); - Pattern colSep = Pattern.compile(AbstractParser.WHITESPACE_PATTERN); + Pattern colSep = Pattern.compile(AbstractParser.DEFAULT_SEPARATOR); DoubleMinMax minmax = new DoubleMinMax(); InputStream in; try { @@ -210,10 +210,10 @@ public class ExternalDoubleOutlierScore extends AbstractAlgorithm<OutlierResult> ((OutlierScalingFunction) scaling).prepare(or); } DoubleMinMax mm = new DoubleMinMax(); - for(DBID id : relation.iterDBIDs()) { - double val = scoresult.get(id); // scores.get(id); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + double val = scoresult.get(iditer); val = scaling.getScaled(val); - scores.putDouble(id, val); + scores.putDouble(iditer, val); mm.put(val); } meta = new BasicOutlierScoreMeta(mm.getMin(), mm.getMax()); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/meta/FeatureBagging.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/meta/FeatureBagging.java index c8da9501..407b7400 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/meta/FeatureBagging.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/meta/FeatureBagging.java @@ -36,7 +36,7 @@ import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.subspace.SubspaceEuclideanDistanceFunction; @@ -50,7 +50,6 @@ import de.lmu.ifi.dbs.elki.result.outlier.OutlierScoreMeta; import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; import de.lmu.ifi.dbs.elki.utilities.documentation.Title; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.GreaterConstraint; @@ -163,14 +162,14 @@ public class FeatureBagging extends AbstractAlgorithm<OutlierResult> implements DoubleMinMax minmax = new DoubleMinMax(); if(breadth) { FiniteProgress cprog = logger.isVerbose() ? new FiniteProgress("Combining results", relation.size(), logger) : null; - Pair<IterableIterator<DBID>, Relation<Double>>[] IDVectorOntoScoreVector = Pair.newPairArray(results.size()); + Pair<DBIDIter, Relation<Double>>[] IDVectorOntoScoreVector = Pair.newPairArray(results.size()); // Mapping score-sorted DBID-Iterators onto their corresponding scores. // We need to initialize them now be able to iterate them "in parallel". { int i = 0; for(OutlierResult r : results) { - IDVectorOntoScoreVector[i] = new Pair<IterableIterator<DBID>, Relation<Double>>(r.getOrdering().iter(relation.getDBIDs()), r.getScores()); + IDVectorOntoScoreVector[i] = new Pair<DBIDIter, Relation<Double>>(r.getOrdering().iter(relation.getDBIDs()).iter(), r.getScores()); i++; } } @@ -178,17 +177,17 @@ public class FeatureBagging extends AbstractAlgorithm<OutlierResult> implements // Iterating over the *lines* of the AS_t(i)-matrix. for(int i = 0; i < relation.size(); i++) { // Iterating over the elements of a line (breadth-first). - for(Pair<IterableIterator<DBID>, Relation<Double>> pair : IDVectorOntoScoreVector) { - IterableIterator<DBID> iter = pair.first; + for(Pair<DBIDIter, Relation<Double>> pair : IDVectorOntoScoreVector) { + DBIDIter iter = pair.first; // Always true if every algorithm returns a complete result (one score // for every DBID). - if(iter.hasNext()) { - DBID tmpID = iter.next(); - double score = pair.second.get(tmpID); - if(Double.isNaN(scores.doubleValue(tmpID))) { - scores.putDouble(tmpID, score); + if(iter.valid()) { + double score = pair.second.get(iter); + if(Double.isNaN(scores.doubleValue(iter))) { + scores.putDouble(iter, score); minmax.put(score); } + iter.advance(); } else { logger.warning("Incomplete result: Iterator does not contain |DB| DBIDs"); @@ -205,15 +204,15 @@ public class FeatureBagging extends AbstractAlgorithm<OutlierResult> implements } else { FiniteProgress cprog = logger.isVerbose() ? new FiniteProgress("Combining results", relation.size(), logger) : null; - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iter = relation.iterDBIDs(); iter.valid(); iter.advance()) { double sum = 0.0; for(OutlierResult r : results) { - final Double s = r.getScores().get(id); + final Double s = r.getScores().get(iter); if (s != null && !Double.isNaN(s)) { sum += s; } } - scores.putDouble(id, sum); + scores.putDouble(iter, sum); minmax.put(sum); if(cprog != null) { cprog.incrementProcessed(logger); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/meta/HiCS.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/meta/HiCS.java new file mode 100644 index 00000000..73d4156a --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/meta/HiCS.java @@ -0,0 +1,633 @@ +package de.lmu.ifi.dbs.elki.algorithm.outlier.meta;
+
+/*
+ This file is part of ELKI:
+ Environment for Developing KDD-Applications Supported by Index-Structures
+
+ Copyright (C) 2012
+ Ludwig-Maximilians-Universität München
+ Lehr- und Forschungseinheit für Datenbanksysteme
+ ELKI Development Team
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeSet;
+
+import de.lmu.ifi.dbs.elki.algorithm.AbstractAlgorithm;
+import de.lmu.ifi.dbs.elki.algorithm.outlier.LOF;
+import de.lmu.ifi.dbs.elki.algorithm.outlier.OutlierAlgorithm;
+import de.lmu.ifi.dbs.elki.data.NumberVector;
+import de.lmu.ifi.dbs.elki.data.VectorUtil;
+import de.lmu.ifi.dbs.elki.data.VectorUtil.SortDBIDsBySingleDimension;
+import de.lmu.ifi.dbs.elki.data.projection.NumericalFeatureSelection;
+import de.lmu.ifi.dbs.elki.data.projection.Projection;
+import de.lmu.ifi.dbs.elki.data.type.TypeInformation;
+import de.lmu.ifi.dbs.elki.data.type.TypeUtil;
+import de.lmu.ifi.dbs.elki.database.ProxyDatabase;
+import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory;
+import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil;
+import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore;
+import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs;
+import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDs;
+import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation;
+import de.lmu.ifi.dbs.elki.database.relation.ProjectedView;
+import de.lmu.ifi.dbs.elki.database.relation.Relation;
+import de.lmu.ifi.dbs.elki.logging.Logging;
+import de.lmu.ifi.dbs.elki.logging.progress.FiniteProgress;
+import de.lmu.ifi.dbs.elki.logging.progress.IndefiniteProgress;
+import de.lmu.ifi.dbs.elki.math.DoubleMinMax;
+import de.lmu.ifi.dbs.elki.math.statistics.tests.GoodnessOfFitTest;
+import de.lmu.ifi.dbs.elki.math.statistics.tests.KolmogorovSmirnovTest;
+import de.lmu.ifi.dbs.elki.result.outlier.BasicOutlierScoreMeta;
+import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult;
+import de.lmu.ifi.dbs.elki.result.outlier.OutlierScoreMeta;
+import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil;
+import de.lmu.ifi.dbs.elki.utilities.datastructures.heap.TopBoundedHeap;
+import de.lmu.ifi.dbs.elki.utilities.documentation.Description;
+import de.lmu.ifi.dbs.elki.utilities.documentation.Reference;
+import de.lmu.ifi.dbs.elki.utilities.documentation.Title;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.GreaterConstraint;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.DoubleParameter;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.IntParameter;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.LongParameter;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter;
+
+/**
+ * Algorithm to compute High Contrast Subspaces for Density-Based Outlier
+ * Ranking.
+ *
+ * Reference:
+ * <p>
+ * Fabian Keller, Emmanuel Müller, Klemens Böhm:<br />
+ * HiCS: High Contrast Subspaces for Density-Based Outlier Ranking<br />
+ * in: Proc. IEEE 28th Int. Conf. on Data Engineering (ICDE 2012), Washington,
+ * DC, USA
+ * </p>
+ *
+ * @author Jan Brusis
+ * @author Erich Schubert
+ *
+ * @apiviz.composedOf GoodnessOfFitTest
+ * @apiviz.composedOf OutlierAlgorithm
+ *
+ * @param <V> vector type
+ */
+@Title("HiCS: High Contrast Subspaces for Density-Based Outlier Ranking")
+@Description("Algorithm to compute High Contrast Subspaces in a database as a pre-processing step for for density-based outlier ranking methods.")
+@Reference(authors = "Fabian Keller, Emmanuel Müller, Klemens Böhm", title = "HiCS: High Contrast Subspaces for Density-Based Outlier Ranking", booktitle = "Proc. IEEE 28th International Conference on Data Engineering (ICDE 2012)")
+public class HiCS<V extends NumberVector<V, ?>> extends AbstractAlgorithm<OutlierResult> implements OutlierAlgorithm {
+ /**
+ * The Logger for this class
+ */
+ private static final Logging logger = Logging.getLogger(HiCS.class);
+
+ /**
+ * Maximum number of retries.
+ */
+ private static final int MAX_RETRIES = 100;
+
+ /**
+ * Monte-Carlo iterations
+ */
+ private int m;
+
+ /**
+ * Alpha threshold
+ */
+ private double alpha;
+
+ /**
+ * Outlier detection algorithm
+ */
+ private OutlierAlgorithm outlierAlgorithm;
+
+ /**
+ * Statistical test to use
+ */
+ private GoodnessOfFitTest statTest;
+
+ /**
+ * Candidates limit
+ */
+ private int cutoff;
+
+ /**
+ * Random generator
+ */
+ private Random random;
+
+ /**
+ * Constructor
+ *
+ * @param m value of m
+ * @param alpha value of alpha
+ * @param outlierAlgorithm Inner outlier detection algorithm
+ * @param statTest Test to use
+ * @param cutoff Candidate limit
+ * @param seed Random seed
+ */
+ public HiCS(int m, double alpha, OutlierAlgorithm outlierAlgorithm, GoodnessOfFitTest statTest, int cutoff, Long seed) {
+ super();
+ this.m = m;
+ this.alpha = alpha;
+ this.outlierAlgorithm = outlierAlgorithm;
+ this.statTest = statTest;
+ this.cutoff = cutoff;
+ this.random = (seed != null) ? new Random(seed) : new Random();
+ }
+
+ /**
+ * Perform HiCS on a given database
+ *
+ * @param relation the database
+ * @return The aggregated resulting scores that were assigned by the given
+ * outlier detection algorithm
+ */
+ public OutlierResult run(Relation<V> relation) {
+ final DBIDs ids = relation.getDBIDs();
+ final V factory = DatabaseUtil.assumeVectorField(relation).getFactory();
+
+ ArrayList<ArrayDBIDs> subspaceIndex = buildOneDimIndexes(relation);
+ Set<HiCSSubspace> subspaces = calculateSubspaces(relation, subspaceIndex);
+
+ if(logger.isVerbose()) {
+ logger.verbose("Number of high-contrast subspaces: " + subspaces.size());
+ }
+ List<Relation<Double>> results = new ArrayList<Relation<Double>>();
+ FiniteProgress prog = logger.isVerbose() ? new FiniteProgress("Calculating Outlier scores for high Contrast subspaces", subspaces.size(), logger) : null;
+
+ // run outlier detection and collect the result
+ // TODO extend so that any outlierAlgorithm can be used (use materialized
+ // relation instead of SubspaceEuclideanDistanceFunction?)
+ for(HiCSSubspace dimset : subspaces) {
+ if(logger.isVerbose()) {
+ logger.verbose("Performing outlier detection in subspace " + dimset);
+ }
+
+ ProxyDatabase pdb = new ProxyDatabase(ids);
+ Projection<V, V> proj = new NumericalFeatureSelection<V>(dimset, factory);
+ pdb.addRelation(new ProjectedView<V, V>(relation, proj));
+
+ // run LOF and collect the result
+ OutlierResult result = outlierAlgorithm.run(pdb);
+ results.add(result.getScores());
+ if(prog != null) {
+ prog.incrementProcessed(logger);
+ }
+ }
+ if(prog != null) {
+ prog.ensureCompleted(logger);
+ }
+
+ WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);
+ DoubleMinMax minmax = new DoubleMinMax();
+
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + double sum = 0.0;
+ for(Relation<Double> r : results) {
+ final Double s = r.get(iditer);
+ if(s != null && !Double.isNaN(s)) {
+ sum += s;
+ }
+ }
+ scores.putDouble(iditer, sum);
+ minmax.put(sum);
+ }
+ OutlierScoreMeta meta = new BasicOutlierScoreMeta(minmax.getMin(), minmax.getMax());
+ Relation<Double> scoreres = new MaterializedRelation<Double>("HiCS", "HiCS-outlier", TypeUtil.DOUBLE, scores, relation.getDBIDs());
+
+ return new OutlierResult(meta, scoreres);
+ }
+
+ /**
+ * Calculates "index structures" for every attribute, i.e. sorts a
+ * ModifiableArray of every DBID in the database for every dimension and
+ * stores them in a list
+ *
+ * @param relation Relation to index
+ * @return List of sorted objects
+ */
+ private ArrayList<ArrayDBIDs> buildOneDimIndexes(Relation<? extends NumberVector<?, ?>> relation) {
+ final int dim = DatabaseUtil.dimensionality(relation);
+ ArrayList<ArrayDBIDs> subspaceIndex = new ArrayList<ArrayDBIDs>(dim + 1);
+
+ SortDBIDsBySingleDimension comp = new VectorUtil.SortDBIDsBySingleDimension(relation);
+ for(int i = 1; i <= dim; i++) {
+ ArrayModifiableDBIDs amDBIDs = DBIDUtil.newArray(relation.getDBIDs());
+ comp.setDimension(i);
+ amDBIDs.sort(comp);
+ subspaceIndex.add(amDBIDs);
+ }
+
+ return subspaceIndex;
+ }
+
+ /**
+ * Identifies high contrast subspaces in a given full-dimensional database
+ *
+ * @param relation the relation the HiCS should be evaluated for
+ * @param subspaceIndex Subspace indexes
+ * @return a set of high contrast subspaces
+ */
+ private Set<HiCSSubspace> calculateSubspaces(Relation<? extends NumberVector<?, ?>> relation, ArrayList<ArrayDBIDs> subspaceIndex) {
+ final int dbdim = DatabaseUtil.dimensionality(relation);
+
+ FiniteProgress dprog = logger.isVerbose() ? new FiniteProgress("Subspace dimensionality", dbdim, logger) : null;
+ if(dprog != null) {
+ dprog.setProcessed(2, logger);
+ }
+
+ TreeSet<HiCSSubspace> subspaceList = new TreeSet<HiCSSubspace>(HiCSSubspace.SORT_BY_SUBSPACE);
+ TopBoundedHeap<HiCSSubspace> dDimensionalList = new TopBoundedHeap<HiCSSubspace>(cutoff, HiCSSubspace.SORT_BY_CONTRAST_ASC);
+ FiniteProgress prog = logger.isVerbose() ? new FiniteProgress("Generating two-element subsets", dbdim * (dbdim - 1) / 2, logger) : null;
+ // compute two-element sets of subspaces
+ for(int i = 0; i < dbdim; i++) {
+ for(int j = i + 1; j < dbdim; j++) {
+ HiCSSubspace ts = new HiCSSubspace();
+ ts.set(i);
+ ts.set(j);
+ calculateContrast(relation, ts, subspaceIndex);
+ dDimensionalList.add(ts);
+ if(prog != null) {
+ prog.incrementProcessed(logger);
+ }
+ }
+ }
+ if(prog != null) {
+ prog.ensureCompleted(logger);
+ }
+
+ IndefiniteProgress qprog = logger.isVerbose() ? new IndefiniteProgress("Testing subspace candidates", logger) : null;
+ for(int d = 3; !dDimensionalList.isEmpty(); d++) {
+ if(dprog != null) {
+ dprog.setProcessed(d, logger);
+ }
+ subspaceList.addAll(dDimensionalList);
+ // result now contains all d-dimensional sets of subspaces
+
+ ArrayList<HiCSSubspace> candidateList = new ArrayList<HiCSSubspace>(dDimensionalList);
+ dDimensionalList.clear();
+ // candidateList now contains the *m* best d-dimensional sets
+ Collections.sort(candidateList, HiCSSubspace.SORT_BY_SUBSPACE);
+
+ // TODO: optimize APRIORI style, by not even computing the bit set or?
+ for(int i = 0; i < candidateList.size() - 1; i++) {
+ for(int j = i + 1; j < candidateList.size(); j++) {
+ HiCSSubspace set1 = candidateList.get(i);
+ HiCSSubspace set2 = candidateList.get(j);
+
+ HiCSSubspace joinedSet = new HiCSSubspace();
+ joinedSet.or(set1);
+ joinedSet.or(set2);
+ if(joinedSet.cardinality() != d) {
+ continue;
+ }
+
+ calculateContrast(relation, joinedSet, subspaceIndex);
+ dDimensionalList.add(joinedSet);
+ if(qprog != null) {
+ qprog.incrementProcessed(logger);
+ }
+ }
+ }
+ // Prune
+ for(HiCSSubspace cand : candidateList) {
+ for(HiCSSubspace nextSet : dDimensionalList) {
+ if(nextSet.contrast > cand.contrast) {
+ subspaceList.remove(cand);
+ break;
+ }
+ }
+ }
+ }
+ if(qprog != null) {
+ qprog.setCompleted(logger);
+ }
+ if(dprog != null) {
+ dprog.setProcessed(dbdim, logger);
+ dprog.ensureCompleted(logger);
+ }
+ return subspaceList;
+ }
+
+ /**
+ * Calculates the actual contrast of a given subspace
+ *
+ * @param relation
+ * @param subspace
+ * @param subspaceIndex Subspace indexes
+ */
+ private void calculateContrast(Relation<? extends NumberVector<?, ?>> relation, HiCSSubspace subspace, ArrayList<ArrayDBIDs> subspaceIndex) {
+ final int card = subspace.cardinality();
+ final double alpha1 = Math.pow(alpha, (1.0 / card));
+ final int windowsize = (int) (relation.size() * alpha1);
+ final FiniteProgress prog = logger.isDebugging() ? new FiniteProgress("Monte-Carlo iterations", m, logger) : null;
+
+ int retries = 0;
+ double deviationSum = 0.0;
+ for(int i = 0; i < m; i++) {
+ // Choose a random set bit.
+ int chosen = -1;
+ for(int tmp = random.nextInt(card); tmp >= 0; tmp--) {
+ chosen = subspace.nextSetBit(chosen + 1);
+ }
+ // initialize sample
+ DBIDs conditionalSample = relation.getDBIDs();
+
+ for(int j = subspace.nextSetBit(0); j >= 0; j = subspace.nextSetBit(j + 1)) {
+ if(j == chosen) {
+ continue;
+ }
+ ArrayDBIDs sortedIndices = subspaceIndex.get(j);
+ ArrayModifiableDBIDs indexBlock = DBIDUtil.newArray();
+ // initialize index block
+ int start = random.nextInt(relation.size() - windowsize);
+ for(int k = start; k < start + windowsize; k++) {
+ indexBlock.add(sortedIndices.get(k)); // select index block
+ }
+
+ conditionalSample = DBIDUtil.intersection(conditionalSample, indexBlock);
+ }
+ if(conditionalSample.size() < 10) {
+ retries++;
+ if(logger.isDebugging()) {
+ logger.debug("Sample size very small. Retry no. " + retries);
+ }
+ if(retries >= MAX_RETRIES) {
+ logger.warning("Too many retries, for small samples: " + retries);
+ }
+ else {
+ i--;
+ continue;
+ }
+ }
+ // Project conditional set
+ double[] sampleValues = new double[conditionalSample.size()];
+ {
+ int l = 0;
+ for (DBIDIter iter = conditionalSample.iter(); iter.valid(); iter.advance()) {
+ sampleValues[l] = relation.get(iter).doubleValue(chosen + 1);
+ l++;
+ }
+ }
+ // Project full set
+ double[] fullValues = new double[relation.size()];
+ {
+ int l = 0;
+ for (DBIDIter iter = subspaceIndex.get(chosen).iter(); iter.valid(); iter.advance()) {
+ fullValues[l] = relation.get(iter).doubleValue(chosen + 1);
+ l++;
+ }
+ }
+ double contrast = statTest.deviation(fullValues, sampleValues);
+ if(Double.isNaN(contrast)) {
+ i--;
+ logger.warning("Contrast was NaN");
+ continue;
+ }
+ deviationSum += contrast;
+ if(prog != null) {
+ prog.incrementProcessed(logger);
+ }
+ }
+ if(prog != null) {
+ prog.ensureCompleted(logger);
+ }
+ subspace.contrast = deviationSum / m;
+ }
+
+ @Override
+ public TypeInformation[] getInputTypeRestriction() {
+ return TypeUtil.array(TypeUtil.NUMBER_VECTOR_FIELD);
+ }
+
+ @Override
+ protected Logging getLogger() {
+ return logger;
+ }
+
+ /**
+ * BitSet that holds a contrast value as field. Used for the representation of
+ * a subspace in HiCS
+ *
+ * @author Erich Schubert
+ *
+ * @apiviz.exclude
+ */
+ public static class HiCSSubspace extends BitSet {
+ /**
+ * Serial version
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * The HiCS contrast value
+ */
+ protected double contrast;
+
+ /**
+ * Constructor.
+ */
+ public HiCSSubspace() {
+ super();
+ }
+
+ @Override
+ public String toString() {
+ StringBuffer buf = new StringBuffer();
+ buf.append("[contrast=").append(contrast);
+ for(int i = nextSetBit(0); i >= 0; i = nextSetBit(i + 1)) {
+ buf.append(" ").append(i + 1);
+ }
+ buf.append("]");
+ return buf.toString();
+ }
+
+ /**
+ * Sort subspaces by their actual subspace.
+ */
+ public static Comparator<HiCSSubspace> SORT_BY_CONTRAST_ASC = new Comparator<HiCSSubspace>() {
+ @Override
+ public int compare(HiCSSubspace o1, HiCSSubspace o2) {
+ if(o1.contrast == o2.contrast) {
+ return 0;
+ }
+ return o1.contrast > o2.contrast ? 1 : -1;
+ }
+ };
+
+ /**
+ * Sort subspaces by their actual subspace.
+ */
+ public static Comparator<HiCSSubspace> SORT_BY_CONTRAST_DESC = new Comparator<HiCSSubspace>() {
+ @Override
+ public int compare(HiCSSubspace o1, HiCSSubspace o2) {
+ if(o1.contrast == o2.contrast) {
+ return 0;
+ }
+ return o1.contrast < o2.contrast ? 1 : -1;
+ }
+ };
+
+ /**
+ * Sort subspaces by their actual subspace.
+ */
+ public static Comparator<HiCSSubspace> SORT_BY_SUBSPACE = new Comparator<HiCSSubspace>() {
+ @Override
+ public int compare(HiCSSubspace o1, HiCSSubspace o2) {
+ int dim1 = o1.nextSetBit(0);
+ int dim2 = o2.nextSetBit(0);
+ while(dim1 >= 0 && dim2 >= 0) {
+ if(dim1 < dim2) {
+ return -1;
+ }
+ else if(dim1 > dim2) {
+ return 1;
+ }
+ dim1 = o1.nextSetBit(dim1 + 1);
+ dim2 = o2.nextSetBit(dim2 + 1);
+ }
+ return 0;
+ }
+ };
+ }
+
+ /**
+ * Parameterization class
+ *
+ * @author Jan Brusis
+ *
+ * @apiviz.exclude
+ *
+ * @param <V> vector type
+ */
+ public static class Parameterizer<V extends NumberVector<V, ?>> extends AbstractParameterizer {
+ /**
+ * Parameter that specifies the number of iterations in the Monte-Carlo
+ * process of identifying high contrast subspaces
+ */
+ public static final OptionID M_ID = OptionID.getOrCreateOptionID("hics.m", "The number of iterations in the Monte-Carlo processing.");
+
+ /**
+ * Parameter that determines the size of the test statistic during the
+ * Monte-Carlo iteration
+ */
+ public static final OptionID ALPHA_ID = OptionID.getOrCreateOptionID("hics.alpha", "The discriminance value that determines the size of the test statistic .");
+
+ /**
+ * Parameter that specifies which outlier detection algorithm to use on the
+ * resulting set of high contrast subspaces
+ */
+ public static final OptionID ALGO_ID = OptionID.getOrCreateOptionID("hics.algo", "The Algorithm that performs the actual outlier detection on the resulting set of subspace");
+
+ /**
+ * Parameter that specifies which statistical test to use in order to
+ * calculate the deviation of two given data samples
+ */
+ public static final OptionID TEST_ID = OptionID.getOrCreateOptionID("hics.test", "The statistical test that is used to calculate the deviation of two data samples");
+
+ /**
+ * Parameter that specifies the candidate_cutoff
+ */
+ public static final OptionID LIMIT_ID = OptionID.getOrCreateOptionID("hics.limit", "The threshold that determines how many d-dimensional subspace candidates to retain in each step of the generation");
+
+ /**
+ * Parameter that specifies the random seed
+ */
+ public static final OptionID SEED_ID = OptionID.getOrCreateOptionID("hics.seed", "The random seed.");
+
+ /**
+ * Holds the value of {@link #M_ID}.
+ */
+ private int m = 50;
+
+ /**
+ * Holds the value of {@link #ALPHA_ID}.
+ */
+ private double alpha = 0.1;
+
+ /**
+ * Holds the value of {@link #ALGO_ID}.
+ */
+ private OutlierAlgorithm outlierAlgorithm;
+
+ /**
+ * Holds the value of {@link #TEST_ID}.
+ */
+ private GoodnessOfFitTest statTest;
+
+ /**
+ * Holds the value of {@link #LIMIT_ID}
+ */
+ private int cutoff = 400;
+
+ /**
+ * Random seed (optional)
+ */
+ private Long seed = null;
+
+ @Override
+ protected void makeOptions(Parameterization config) {
+ super.makeOptions(config);
+ final IntParameter mP = new IntParameter(M_ID, new GreaterConstraint(1), 50);
+ if(config.grab(mP)) {
+ m = mP.getValue();
+ }
+
+ final DoubleParameter alphaP = new DoubleParameter(ALPHA_ID, new GreaterConstraint(0), 0.1);
+ if(config.grab(alphaP)) {
+ alpha = alphaP.getValue();
+ }
+
+ final ObjectParameter<OutlierAlgorithm> algoP = new ObjectParameter<OutlierAlgorithm>(ALGO_ID, OutlierAlgorithm.class, LOF.class);
+ if(config.grab(algoP)) {
+ outlierAlgorithm = algoP.instantiateClass(config);
+ }
+
+ final ObjectParameter<GoodnessOfFitTest> testP = new ObjectParameter<GoodnessOfFitTest>(TEST_ID, GoodnessOfFitTest.class, KolmogorovSmirnovTest.class);
+ if(config.grab(testP)) {
+ statTest = testP.instantiateClass(config);
+ }
+
+ final IntParameter cutoffP = new IntParameter(LIMIT_ID, new GreaterConstraint(1), 100);
+ if(config.grab(cutoffP)) {
+ cutoff = cutoffP.getValue();
+ }
+
+ final LongParameter seedP = new LongParameter(SEED_ID, true);
+ if(config.grab(seedP)) {
+ seed = seedP.getValue();
+ }
+}
+
+ @Override
+ protected HiCS<V> makeInstance() {
+ return new HiCS<V>(m, alpha, outlierAlgorithm, statTest, cutoff, seed);
+ }
+ }
+}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/meta/RescaleMetaOutlierAlgorithm.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/meta/RescaleMetaOutlierAlgorithm.java index 9634cd59..a4db7e3d 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/meta/RescaleMetaOutlierAlgorithm.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/meta/RescaleMetaOutlierAlgorithm.java @@ -34,7 +34,7 @@ import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.logging.Logging; @@ -55,6 +55,8 @@ import de.lmu.ifi.dbs.elki.utilities.scaling.outlier.OutlierScalingFunction; * Scale another outlier score using the given scaling function. * * @author Erich Schubert + * + * @apiviz.composedOf OutlierAlgorithm */ public class RescaleMetaOutlierAlgorithm extends AbstractAlgorithm<OutlierResult> implements OutlierAlgorithm { /** @@ -93,7 +95,7 @@ public class RescaleMetaOutlierAlgorithm extends AbstractAlgorithm<OutlierResult } @Override - public OutlierResult run(Database database) throws IllegalStateException { + public OutlierResult run(Database database) { Result innerresult = algorithm.run(database); OutlierResult or = getOutlierResult(innerresult); @@ -105,10 +107,9 @@ public class RescaleMetaOutlierAlgorithm extends AbstractAlgorithm<OutlierResult WritableDoubleDataStore scaledscores = DataStoreUtil.makeDoubleStorage(scores.getDBIDs(), DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_STATIC); DoubleMinMax minmax = new DoubleMinMax(); - for(DBID id : scores.iterDBIDs()) { - double val = scores.get(id); - val = scaling.getScaled(val); - scaledscores.putDouble(id, val); + for(DBIDIter iditer = scores.iterDBIDs(); iditer.valid(); iditer.advance()) { + double val = scaling.getScaled(scores.get(iditer)); + scaledscores.putDouble(iditer, val); minmax.put(val); } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuGLSBackwardSearchAlgorithm.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuGLSBackwardSearchAlgorithm.java index b4070e0c..7f3bac29 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuGLSBackwardSearchAlgorithm.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuGLSBackwardSearchAlgorithm.java @@ -34,6 +34,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; @@ -129,7 +130,7 @@ public class CTLuGLSBackwardSearchAlgorithm<V extends NumberVector<?, ?>, D exte ModifiableDBIDs idview = DBIDUtil.newHashSet(relationx.getDBIDs()); ProxyView<V> proxy = new ProxyView<V>(relationx.getDatabase(), idview, relationx); - double phialpha = NormalDistribution.standardNormalProbit(1.0 - alpha / 2); + double phialpha = NormalDistribution.standardNormalQuantile(1.0 - alpha / 2); // Detect outliers while significant. while(true) { Pair<DBID, Double> candidate = singleIteration(proxy, relationy); @@ -144,8 +145,8 @@ public class CTLuGLSBackwardSearchAlgorithm<V extends NumberVector<?, ?>, D exte } // Remaining objects are inliers - for(DBID id : idview) { - scores.putDouble(id, 0.0); + for (DBIDIter iter = idview.iter(); iter.valid(); iter.advance()) { + scores.putDouble(iter.getDBID(), 0.0); } } @@ -204,7 +205,7 @@ public class CTLuGLSBackwardSearchAlgorithm<V extends NumberVector<?, ?>, D exte KNNResult<D> neighbors = knnQuery.getKNNForDBID(id, k + 1); ModifiableDBIDs neighborhood = DBIDUtil.newArray(neighbors.size()); for(DistanceResultPair<D> dpair : neighbors) { - if(id.equals(dpair.getDBID())) { + if(id.sameDBID(dpair.getDBID())) { continue; } neighborhood.add(dpair.getDBID()); @@ -213,8 +214,8 @@ public class CTLuGLSBackwardSearchAlgorithm<V extends NumberVector<?, ?>, D exte F.set(i, i, 1.0); final int nweight = -1 / neighborhood.size(); // We need to find the index positions of the neighbors, unfortunately. - for(DBID nid : neighborhood) { - int pos = ids.binarySearch(nid); + for (DBIDIter iter = neighborhood.iter(); iter.valid(); iter.advance()) { + int pos = ids.binarySearch(iter.getDBID()); assert (pos >= 0); F.set(pos, i, nweight); } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuMeanMultipleAttributes.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuMeanMultipleAttributes.java index 68e58ffa..a0c09057 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuMeanMultipleAttributes.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuMeanMultipleAttributes.java @@ -32,6 +32,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation; import de.lmu.ifi.dbs.elki.database.relation.Relation; @@ -99,7 +100,8 @@ public class CTLuMeanMultipleAttributes<N, O extends NumberVector<?, ?>> extends CovarianceMatrix covmaker = new CovarianceMatrix(DatabaseUtil.dimensionality(attributes)); WritableDataStore<Vector> deltas = DataStoreUtil.makeStorage(attributes.getDBIDs(), DataStoreFactory.HINT_TEMP, Vector.class); - for(DBID id : attributes.iterDBIDs()) { + for(DBIDIter iditer = attributes.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); final O obj = attributes.get(id); final DBIDs neighbors = npred.getNeighborDBIDs(id); // TODO: remove object itself from neighbors? @@ -117,7 +119,8 @@ public class CTLuMeanMultipleAttributes<N, O extends NumberVector<?, ?>> extends DoubleMinMax minmax = new DoubleMinMax(); WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(attributes.getDBIDs(), DataStoreFactory.HINT_STATIC); - for(DBID id : attributes.iterDBIDs()) { + for(DBIDIter iditer = attributes.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); Vector temp = deltas.get(id).minus(mean); final double score = temp.transposeTimesTimes(cmati, temp); minmax.put(score); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuMedianAlgorithm.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuMedianAlgorithm.java index 9b4534fe..20ab9a00 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuMedianAlgorithm.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuMedianAlgorithm.java @@ -31,6 +31,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil;
import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore;
import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDs;
import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation;
import de.lmu.ifi.dbs.elki.database.relation.Relation;
@@ -94,18 +95,19 @@ public class CTLuMedianAlgorithm<N> extends AbstractNeighborhoodOutlier<N> { WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);
MeanVariance mv = new MeanVariance();
- for(DBID id : relation.iterDBIDs()) {
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID();
DBIDs neighbors = npred.getNeighborDBIDs(id);
final double median;
{
double[] fi = new double[neighbors.size()];
// calculate and store Median of neighborhood
int c = 0;
- for(DBID n : neighbors) {
- if(id.equals(n)) {
+ for(DBIDIter iter = neighbors.iter(); iter.valid(); iter.advance()) {
+ if(id.sameDBID(iter)) {
continue;
}
- fi[c] = relation.get(n).doubleValue(1);
+ fi[c] = relation.get(iter).doubleValue(1);
c++;
}
@@ -125,7 +127,8 @@ public class CTLuMedianAlgorithm<N> extends AbstractNeighborhoodOutlier<N> { final double mean = mv.getMean();
final double stddev = mv.getNaiveStddev();
DoubleMinMax minmax = new DoubleMinMax();
- for(DBID id : relation.iterDBIDs()) {
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID();
double score = Math.abs((scores.doubleValue(id) - mean) / stddev);
minmax.put(score);
scores.putDouble(id, score);
diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuMedianMultipleAttributes.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuMedianMultipleAttributes.java index cbf61c38..c8bcba74 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuMedianMultipleAttributes.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuMedianMultipleAttributes.java @@ -32,6 +32,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation; import de.lmu.ifi.dbs.elki.database.relation.Relation; @@ -108,7 +109,8 @@ public class CTLuMedianMultipleAttributes<N, O extends NumberVector<?, ?>> exten CovarianceMatrix covmaker = new CovarianceMatrix(dim); WritableDataStore<Vector> deltas = DataStoreUtil.makeStorage(attributes.getDBIDs(), DataStoreFactory.HINT_TEMP, Vector.class); - for(DBID id : attributes.iterDBIDs()) { + for(DBIDIter iditer = attributes.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); final O obj = attributes.get(id); final DBIDs neighbors = npred.getNeighborDBIDs(id); // Compute the median vector @@ -117,9 +119,9 @@ public class CTLuMedianMultipleAttributes<N, O extends NumberVector<?, ?>> exten double[][] data = new double[dim][neighbors.size()]; int i = 0; // Load data - for(DBID n : neighbors) { + for(DBIDIter iter = neighbors.iter(); iter.valid(); iter.advance()) { // TODO: skip object itself within neighbors? - O nobj = attributes.get(n); + O nobj = attributes.get(iter); for(int d = 0; d < dim; d++) { data[d][i] = nobj.doubleValue(d + 1); } @@ -143,7 +145,8 @@ public class CTLuMedianMultipleAttributes<N, O extends NumberVector<?, ?>> exten DoubleMinMax minmax = new DoubleMinMax(); WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(attributes.getDBIDs(), DataStoreFactory.HINT_STATIC); - for(DBID id : attributes.iterDBIDs()) { + for(DBIDIter iditer = attributes.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); Vector temp = deltas.get(id).minus(mean); final double score = temp.transposeTimesTimes(cmati, temp); minmax.put(score); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuMoranScatterplotOutlier.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuMoranScatterplotOutlier.java index 9f19757d..7b88ae66 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuMoranScatterplotOutlier.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuMoranScatterplotOutlier.java @@ -33,6 +33,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.logging.Logging; @@ -98,7 +99,8 @@ public class CTLuMoranScatterplotOutlier<N> extends AbstractNeighborhoodOutlier< // Compute the global mean and variance MeanVariance globalmv = new MeanVariance(); - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); globalmv.put(relation.get(id).doubleValue(1)); } @@ -107,12 +109,14 @@ public class CTLuMoranScatterplotOutlier<N> extends AbstractNeighborhoodOutlier< // calculate normalized attribute values // calculate neighborhood average of normalized attribute values. - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); // Compute global z score final double globalZ = (relation.get(id).doubleValue(1) - globalmv.getMean()) / globalmv.getNaiveStddev(); // Compute local average z score Mean localm = new Mean(); - for(DBID n : npred.getNeighborDBIDs(id)) { + for(DBIDIter iter = npred.getNeighborDBIDs(id).iter(); iter.valid(); iter.advance()) { + DBID n = iter.getDBID(); if(id.equals(n)) { continue; } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuRandomWalkEC.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuRandomWalkEC.java index a6425d43..852c4be4 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuRandomWalkEC.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuRandomWalkEC.java @@ -34,6 +34,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore;
import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs;
import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil;
import de.lmu.ifi.dbs.elki.database.ids.DBIDs;
import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs;
@@ -208,7 +209,8 @@ public class CTLuRandomWalkEC<N, D extends NumberDistance<D, ?>> extends Abstrac DBID id = ids.get(i);
double gmean = 1.0;
int cnt = 0;
- for(DBID n : neighbors.get(id)) {
+ for(DBIDIter iter = neighbors.get(id).iter(); iter.valid(); iter.advance()) {
+ DBID n = iter.getDBID();
if(id.equals(n)) {
continue;
}
diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuScatterplotOutlier.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuScatterplotOutlier.java index 8e4ab32c..4f11cb38 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuScatterplotOutlier.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuScatterplotOutlier.java @@ -32,6 +32,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation; import de.lmu.ifi.dbs.elki.database.relation.Relation; @@ -102,12 +103,14 @@ public class CTLuScatterplotOutlier<N> extends AbstractNeighborhoodOutlier<N> { // Calculate average of neighborhood for each object and perform a linear // regression using the covariance matrix CovarianceMatrix covm = new CovarianceMatrix(2); - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); final double local = relation.get(id).doubleValue(1); // Compute mean of neighbors Mean mean = new Mean(); DBIDs neighbors = npred.getNeighborDBIDs(id); - for(DBID n : neighbors) { + for(DBIDIter iter = neighbors.iter(); iter.valid(); iter.advance()) { + DBID n = iter.getDBID(); if(id.equals(n)) { continue; } @@ -139,7 +142,8 @@ public class CTLuScatterplotOutlier<N> extends AbstractNeighborhoodOutlier<N> { // calculate mean and variance for error WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC); MeanVariance mv = new MeanVariance(); - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); // Compute the error from the linear regression double y_i = relation.get(id).doubleValue(1); double e = means.doubleValue(id) - (slope * y_i + inter); @@ -152,7 +156,8 @@ public class CTLuScatterplotOutlier<N> extends AbstractNeighborhoodOutlier<N> { { final double mean = mv.getMean(); final double variance = mv.getNaiveStddev(); - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); double score = Math.abs((scores.doubleValue(id) - mean) / variance); minmax.put(score); scores.putDouble(id, score); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuZTestOutlier.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuZTestOutlier.java index 573e1526..05729481 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuZTestOutlier.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/CTLuZTestOutlier.java @@ -33,6 +33,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation; import de.lmu.ifi.dbs.elki.database.relation.Relation; @@ -102,17 +103,17 @@ public class CTLuZTestOutlier<N> extends AbstractNeighborhoodOutlier<N> { WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC); MeanVariance zmv = new MeanVariance(); - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); DBIDs neighbors = npred.getNeighborDBIDs(id); // Compute Mean of neighborhood Mean localmean = new Mean(); - for(DBID n : neighbors) { + for(DBIDIter iter = neighbors.iter(); iter.valid(); iter.advance()) { + DBID n = iter.getDBID(); if(id.equals(n)) { continue; } - else { - localmean.put(relation.get(n).doubleValue(1)); - } + localmean.put(relation.get(n).doubleValue(1)); } final double localdiff; if(localmean.getCount() > 0) { @@ -127,7 +128,8 @@ public class CTLuZTestOutlier<N> extends AbstractNeighborhoodOutlier<N> { // Normalize scores using mean and variance DoubleMinMax minmax = new DoubleMinMax(); - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); double score = Math.abs(scores.doubleValue(id) - zmv.getMean()) / zmv.getSampleStddev(); minmax.put(score); scores.putDouble(id, score); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/SLOM.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/SLOM.java index e69d46d4..8ae23229 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/SLOM.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/SLOM.java @@ -31,6 +31,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation; @@ -53,7 +54,7 @@ import de.lmu.ifi.dbs.elki.utilities.documentation.Title; * Reference:<br> * Sanjay Chawla and Pei Sun<br> * SLOM: a new measure for local spatial outliers<br> - * in Knowledge and Information Systems 2005 + * in Knowledge and Information Systems 9(4), 412-429, 2006 * </p> * * This implementation works around some corner cases in SLOM, in particular @@ -68,7 +69,7 @@ import de.lmu.ifi.dbs.elki.utilities.documentation.Title; */ @Title("SLOM: a new measure for local spatial outliers") @Description("Spatial local outlier measure (SLOM), which captures the local behaviour of datum in their spatial neighbourhood") -@Reference(authors = "Sanjay Chawla and Pei Sun", title = "SLOM: a new measure for local spatial outliers", booktitle = "Knowledge and Information Systems 2005", url = "http://rp-www.cs.usyd.edu.au/~chawlarg/papers/KAIS_online.pdf") +@Reference(authors = "Sanjay Chawla and Pei Sun", title = "SLOM: a new measure for local spatial outliers", booktitle = "Knowledge and Information Systems 9(4), 412-429, 2006", url = "http://dx.doi.org/10.1007/s10115-005-0200-2") public class SLOM<N, O, D extends NumberDistance<D, ?>> extends AbstractDistanceBasedSpatialOutlier<N, O, D> { /** * The logger for this class. @@ -98,13 +99,15 @@ public class SLOM<N, O, D extends NumberDistance<D, ?>> extends AbstractDistance WritableDoubleDataStore modifiedDistance = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_TEMP); // calculate D-Tilde - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); double sum = 0; double maxDist = 0; int cnt = 0; final DBIDs neighbors = npred.getNeighborDBIDs(id); - for(DBID neighbor : neighbors) { + for(DBIDIter iter = neighbors.iter(); iter.valid(); iter.advance()) { + DBID neighbor = iter.getDBID(); if(id.equals(neighbor)) { continue; } @@ -127,12 +130,14 @@ public class SLOM<N, O, D extends NumberDistance<D, ?>> extends AbstractDistance DoubleMinMax slomminmax = new DoubleMinMax(); WritableDoubleDataStore sloms = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC); - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); double sum = 0; int cnt = 0; final DBIDs neighbors = npred.getNeighborDBIDs(id); - for(DBID neighbor : neighbors) { + for(DBIDIter iter = neighbors.iter(); iter.valid(); iter.advance()) { + DBID neighbor = iter.getDBID(); if(neighbor.equals(id)) { continue; } @@ -146,7 +151,8 @@ public class SLOM<N, O, D extends NumberDistance<D, ?>> extends AbstractDistance double avg = sum / cnt; double beta = 0; - for(DBID neighbor : neighbors) { + for(DBIDIter iter = neighbors.iter(); iter.valid(); iter.advance()) { + DBID neighbor = iter.getDBID(); final double dist = modifiedDistance.doubleValue(neighbor); if(dist > avgPlus) { beta += 1; diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/SOF.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/SOF.java index abc3c481..e9987bf0 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/SOF.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/SOF.java @@ -30,6 +30,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil;
import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore;
import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDs;
import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery;
import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation;
@@ -108,11 +109,12 @@ public class SOF<N, O, D extends NumberDistance<D, ?>> extends AbstractDistanceB DoubleMinMax lofminmax = new DoubleMinMax();
// Compute densities
- for(DBID id : relation.iterDBIDs()) {
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID();
DBIDs neighbors = npred.getNeighborDBIDs(id);
double avg = 0;
- for(DBID n : neighbors) {
- avg += distFunc.distance(id, n).doubleValue();
+ for(DBIDIter iter = neighbors.iter(); iter.valid(); iter.advance()) {
+ avg += distFunc.distance(id, iter.getDBID()).doubleValue();
}
double lrd = 1 / (avg / neighbors.size());
if (Double.isNaN(lrd)) {
@@ -122,11 +124,12 @@ public class SOF<N, O, D extends NumberDistance<D, ?>> extends AbstractDistanceB }
// Compute density quotients
- for(DBID id : relation.iterDBIDs()) {
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID();
DBIDs neighbors = npred.getNeighborDBIDs(id);
double avg = 0;
- for(DBID n : neighbors) {
- avg += lrds.doubleValue(n);
+ for(DBIDIter iter = neighbors.iter(); iter.valid(); iter.advance()) {
+ avg += lrds.doubleValue(iter.getDBID());
}
final double lrd = (avg / neighbors.size()) / lrds.doubleValue(id);
if (!Double.isNaN(lrd)) {
diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/TrimmedMeanApproach.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/TrimmedMeanApproach.java index 75700bca..41022414 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/TrimmedMeanApproach.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/TrimmedMeanApproach.java @@ -34,6 +34,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil;
import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore;
import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDs;
import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation;
import de.lmu.ifi.dbs.elki.database.relation.Relation;
@@ -116,13 +117,14 @@ public class TrimmedMeanApproach<N> extends AbstractNeighborhoodOutlier<N> { WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC);
FiniteProgress progress = logger.isVerbose() ? new FiniteProgress("Computing trimmed means", relation.size(), logger) : null;
- for(DBID id : relation.iterDBIDs()) {
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID();
DBIDs neighbors = npred.getNeighborDBIDs(id);
int num = 0;
double[] values = new double[neighbors.size()];
// calculate trimmedMean
- for(DBID n : neighbors) {
- values[num] = relation.get(n).doubleValue(1);
+ for(DBIDIter iter = neighbors.iter(); iter.valid(); iter.advance()) {
+ values[num] = relation.get(iter).doubleValue(1);
num++;
}
@@ -161,7 +163,8 @@ public class TrimmedMeanApproach<N> extends AbstractNeighborhoodOutlier<N> { double[] ei = new double[relation.size()];
{
int i = 0;
- for(DBID id : relation.iterDBIDs()) {
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID();
ei[i] = errors.doubleValue(id);
i++;
}
@@ -180,7 +183,8 @@ public class TrimmedMeanApproach<N> extends AbstractNeighborhoodOutlier<N> { }
// calculate score
DoubleMinMax minmax = new DoubleMinMax();
- for(DBID id : relation.iterDBIDs()) {
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID();
double score = Math.abs(errors.doubleValue(id)) * 0.6745 / median_dev_from_median;
scores.putDouble(id, score);
minmax.put(score);
diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/neighborhood/ExtendedNeighborhood.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/neighborhood/ExtendedNeighborhood.java index 9ee92d35..7a2fda52 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/neighborhood/ExtendedNeighborhood.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/neighborhood/ExtendedNeighborhood.java @@ -29,6 +29,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.HashSetModifiableDBIDs; @@ -132,15 +133,17 @@ public class ExtendedNeighborhood extends AbstractPrecomputedNeighborhood { // Expand multiple steps FiniteProgress progress = logger.isVerbose() ? new FiniteProgress("Expanding neighborhoods", database.size(), logger) : null; - for(final DBID id : database.iterDBIDs()) { + for(DBIDIter iter = database.iterDBIDs(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); HashSetModifiableDBIDs res = DBIDUtil.newHashSet(id); DBIDs todo = id; for(int i = 0; i < steps; i++) { ModifiableDBIDs ntodo = DBIDUtil.newHashSet(); - for(final DBID oid : todo) { - DBIDs add = innerinst.getNeighborDBIDs(oid); + for(DBIDIter iter2 = todo.iter(); iter2.valid(); iter2.advance()) { + DBIDs add = innerinst.getNeighborDBIDs(iter2.getDBID()); if(add != null) { - for(DBID nid : add) { + for(DBIDIter iter3 = add.iter(); iter.valid(); iter.advance()) { + DBID nid = iter3.getDBID(); if(res.contains(nid)) { continue; } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/neighborhood/ExternalNeighborhood.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/neighborhood/ExternalNeighborhood.java index f2586e2e..74e5bbcf 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/neighborhood/ExternalNeighborhood.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/neighborhood/ExternalNeighborhood.java @@ -42,6 +42,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.relation.Relation; @@ -149,7 +150,8 @@ public class ExternalNeighborhood extends AbstractPrecomputedNeighborhood { { Relation<LabelList> olq = database.getDatabase().getRelation(TypeUtil.LABELLIST); Relation<ExternalID> eidq = database.getDatabase().getRelation(TypeUtil.EXTERNALID); - for(DBID id : database.iterDBIDs()) { + for(DBIDIter iditer = database.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); if(eidq != null) { ExternalID eid = eidq.get(id); if(eid != null) { diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/neighborhood/PrecomputedKNearestNeighborNeighborhood.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/neighborhood/PrecomputedKNearestNeighborNeighborhood.java index f5ea7e15..9dd2dee1 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/neighborhood/PrecomputedKNearestNeighborNeighborhood.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/neighborhood/PrecomputedKNearestNeighborNeighborhood.java @@ -30,6 +30,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore;
import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs;
import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil;
import de.lmu.ifi.dbs.elki.database.ids.DBIDs;
import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair;
@@ -119,7 +120,8 @@ public class PrecomputedKNearestNeighborNeighborhood<D extends Distance<D>> exte // TODO: use bulk?
WritableDataStore<DBIDs> s = DataStoreUtil.makeStorage(relation.getDBIDs(), DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_STATIC, DBIDs.class);
- for(DBID id : relation.iterDBIDs()) {
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID();
KNNResult<D> neighbors = knnQuery.getKNNForDBID(id, k);
ArrayModifiableDBIDs neighbours = DBIDUtil.newArray(neighbors.size());
for(DistanceResultPair<D> dpair : neighbors) {
diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/neighborhood/weighted/LinearWeightedExtendedNeighborhood.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/neighborhood/weighted/LinearWeightedExtendedNeighborhood.java index 52fc2c46..d170571f 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/neighborhood/weighted/LinearWeightedExtendedNeighborhood.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/neighborhood/weighted/LinearWeightedExtendedNeighborhood.java @@ -30,6 +30,7 @@ import java.util.List; import de.lmu.ifi.dbs.elki.algorithm.outlier.spatial.neighborhood.NeighborSetPredicate; import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; @@ -99,8 +100,10 @@ public class LinearWeightedExtendedNeighborhood implements WeightedNeighborSetPr final double weight = computeWeight(i); // Collect newly discovered IDs ModifiableDBIDs add = DBIDUtil.newHashSet(); - for(DBID id : cur) { - for(DBID nid : inner.getNeighborDBIDs(id)) { + for(DBIDIter iter = cur.iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); + for(DBIDIter iter2 = inner.getNeighborDBIDs(id).iter(); iter2.valid(); iter2.advance()) { + DBID nid = iter2.getDBID(); // Seen before? if(seen.contains(nid)) { continue; diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/neighborhood/weighted/UnweightedNeighborhoodAdapter.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/neighborhood/weighted/UnweightedNeighborhoodAdapter.java index 4378aa2e..ce0666df 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/neighborhood/weighted/UnweightedNeighborhoodAdapter.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/spatial/neighborhood/weighted/UnweightedNeighborhoodAdapter.java @@ -29,6 +29,7 @@ import java.util.Collection; import de.lmu.ifi.dbs.elki.algorithm.outlier.spatial.neighborhood.NeighborSetPredicate; import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; @@ -63,7 +64,8 @@ public class UnweightedNeighborhoodAdapter implements WeightedNeighborSetPredica public Collection<DoubleObjPair<DBID>> getWeightedNeighbors(DBID reference) { DBIDs neighbors = inner.getNeighborDBIDs(reference); ArrayList<DoubleObjPair<DBID>> adapted = new ArrayList<DoubleObjPair<DBID>>(neighbors.size()); - for(DBID id : neighbors) { + for(DBIDIter iter = neighbors.iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); adapted.add(new DoubleObjPair<DBID>(1.0, id)); } return adapted; diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/OUTRES.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/subspace/OUTRES.java index 912f878a..573233a7 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/OUTRES.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/subspace/OUTRES.java @@ -1,4 +1,4 @@ -package de.lmu.ifi.dbs.elki.algorithm.outlier; +package de.lmu.ifi.dbs.elki.algorithm.outlier.subspace; /* This file is part of ELKI: @@ -23,6 +23,7 @@ package de.lmu.ifi.dbs.elki.algorithm.outlier; along with this program. If not, see <http://www.gnu.org/licenses/>. */ +import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; import java.util.List; @@ -37,11 +38,14 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.database.query.DoubleDistanceResultPair; import de.lmu.ifi.dbs.elki.database.query.range.RangeQuery; import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation; import de.lmu.ifi.dbs.elki.database.relation.Relation; +import de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDoubleDistanceFunction; import de.lmu.ifi.dbs.elki.distance.distancefunction.subspace.SubspaceEuclideanDistanceFunction; import de.lmu.ifi.dbs.elki.distance.distancevalue.DoubleDistance; import de.lmu.ifi.dbs.elki.logging.Logging; @@ -77,8 +81,12 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.DoubleParameter; * management * </p> * - * @author Pleintinger Viktoria + * @author Viktoria Pleintinger * @author Erich Schubert + * + * @apiviz.composedOf KernelDensityEstimator + * + * @param <V> vector type */ @Reference(authors = "E. Müller, M. Schiffer, T. Seidl", title = "Adaptive outlierness for subspace outlier ranking", booktitle = "Proc. 19th ACM International Conference on Information and knowledge management") public class OUTRES<V extends NumberVector<V, ?>> extends AbstractAlgorithm<OutlierResult> implements OutlierAlgorithm { @@ -122,10 +130,10 @@ public class OUTRES<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Outl FiniteProgress progress = logger.isVerbose() ? new FiniteProgress("OutRank scores", relation.size(), logger) : null; - for(DBID object : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { subspace.clear(); - double score = outresScore(0, subspace, object, kernel); - ranks.putDouble(object, score); + double score = outresScore(0, subspace, iditer, kernel); + ranks.putDouble(iditer, score); minmax.put(score); if(progress != null) { progress.incrementProcessed(logger); @@ -149,7 +157,7 @@ public class OUTRES<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Outl * @param kernel Kernel * @return Score */ - public double outresScore(final int s, BitSet subspace, DBID id, KernelDensityEstimator kernel) { + public double outresScore(final int s, BitSet subspace, DBIDRef id, KernelDensityEstimator kernel) { double score = 1.0; // Initial score is 1.0 for(int i = s; i < kernel.dim; i++) { @@ -158,10 +166,14 @@ public class OUTRES<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Outl } subspace.set(i); final SubspaceEuclideanDistanceFunction df = new SubspaceEuclideanDistanceFunction(subspace); - final DoubleDistance range = new DoubleDistance(kernel.adjustedEps(kernel.dim)); + final double adjustedEps = kernel.adjustedEps(kernel.dim); + // Query with a larger window, to also get neighbors of neighbors + // Subspace euclidean is metric! + final DoubleDistance range = new DoubleDistance(adjustedEps * 2); RangeQuery<V, DoubleDistance> rq = QueryUtil.getRangeQuery(kernel.relation, df, range); - List<DistanceResultPair<DoubleDistance>> neigh = rq.getRangeForDBID(id, range); + List<DistanceResultPair<DoubleDistance>> neighc = rq.getRangeForDBID(id, range); + List<DoubleDistanceResultPair> neigh = refineRange(neighc, adjustedEps); if(neigh.size() > 2) { // Relevance test if(relevantSubspace(subspace, neigh, kernel)) { @@ -169,8 +181,8 @@ public class OUTRES<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Outl final double deviation; // Compute mean and standard deviation for densities of neighbors. MeanVariance meanv = new MeanVariance(); - for(DistanceResultPair<DoubleDistance> pair : neigh) { - List<DistanceResultPair<DoubleDistance>> n2 = rq.getRangeForDBID(pair.getDBID(), range); + for(DoubleDistanceResultPair pair : neigh) { + List<DoubleDistanceResultPair> n2 = subsetNeighborhoodQuery(neighc, pair.getDBID(), df, adjustedEps, kernel); meanv.put(kernel.subspaceDensity(subspace, n2)); } deviation = (meanv.getMean() - density) / (2. * meanv.getSampleStddev()); @@ -188,11 +200,62 @@ public class OUTRES<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Outl } /** + * Refine a range query. + * + * @param neighc Original result + * @param adjustedEps New epsilon + * @return refined list + */ + private List<DoubleDistanceResultPair> refineRange(List<DistanceResultPair<DoubleDistance>> neighc, double adjustedEps) { + List<DoubleDistanceResultPair> n = new ArrayList<DoubleDistanceResultPair>(neighc.size()); + // We don't have a guarantee for this list to be sorted + for(DistanceResultPair<DoubleDistance> p : neighc) { + if(p instanceof DoubleDistanceResultPair) { + if(((DoubleDistanceResultPair) p).getDoubleDistance() <= adjustedEps) { + n.add((DoubleDistanceResultPair) p); + } + } + else { + double dist = p.getDistance().doubleValue(); + if(dist <= adjustedEps) { + n.add(new DoubleDistanceResultPair(dist, p.getDBID())); + } + } + } + return n; + } + + /** + * Refine neighbors within a subset. * - * @param test: subspace that will be tested about scattering - * @return if the subspace is scattered return will be 0, else 1 + * @param neighc Neighbor candidates + * @param dbid Query object + * @param df distance function + * @param adjustedEps Epsilon range + * @param kernel Kernel + * @return Neighbors of neighbor object */ - protected boolean relevantSubspace(BitSet subspace, List<DistanceResultPair<DoubleDistance>> neigh, KernelDensityEstimator kernel) { + private List<DoubleDistanceResultPair> subsetNeighborhoodQuery(List<DistanceResultPair<DoubleDistance>> neighc, DBID dbid, PrimitiveDoubleDistanceFunction<? super V> df, double adjustedEps, KernelDensityEstimator kernel) { + List<DoubleDistanceResultPair> n = new ArrayList<DoubleDistanceResultPair>(neighc.size()); + V query = kernel.relation.get(dbid); + for(DistanceResultPair<DoubleDistance> p : neighc) { + double dist = df.doubleDistance(query, kernel.relation.get(p)); + if(dist <= adjustedEps) { + n.add(new DoubleDistanceResultPair(dist, p.getDBID())); + } + } + return n; + } + + /** + * Subspace relevance test. + * + * @param subspace Subspace to test + * @param neigh Neighbor list + * @param kernel Kernel density estimator + * @return relevance test result + */ + protected boolean relevantSubspace(BitSet subspace, List<DoubleDistanceResultPair> neigh, KernelDensityEstimator kernel) { Relation<V> relation = kernel.relation; final double crit = K_S_CRITICAL001 / Math.sqrt(neigh.size()); @@ -201,7 +264,7 @@ public class OUTRES<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Outl double[] data = new double[neigh.size()]; { int count = 0; - for(DistanceResultPair<DoubleDistance> object : neigh) { + for(DoubleDistanceResultPair object : neigh) { V vector = relation.get(object.getDBID()); data[count] = vector.doubleValue(dim + 1); count++; @@ -257,7 +320,7 @@ public class OUTRES<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Outl /** * Constructor. - * + * * @param relation Relation to apply to */ public KernelDensityEstimator(Relation<V> relation) { @@ -277,17 +340,14 @@ public class OUTRES<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Outl * @param neighbours Neighbor distance list * @return Density */ - protected double subspaceDensity(BitSet subspace, List<DistanceResultPair<DoubleDistance>> neighbours) { + protected double subspaceDensity(BitSet subspace, List<DoubleDistanceResultPair> neighbours) { final double bandwidth = optimalBandwidth(subspace.cardinality()); - // TODO: optimize by moving instanceof outside? double density = 0; - for(DistanceResultPair<DoubleDistance> pair : neighbours) { - if(pair instanceof DoubleDistanceResultPair) { - density += kernel.density(((DoubleDistanceResultPair) pair).getDoubleDistance() / bandwidth); - } - else { - density += kernel.density(pair.getDistance().doubleValue() / bandwidth); + for(DoubleDistanceResultPair pair : neighbours) { + double v = pair.getDoubleDistance() / bandwidth; + if(v < 1) { + density += 1 - (v * v); } } @@ -302,7 +362,7 @@ public class OUTRES<V extends NumberVector<V, ?>> extends AbstractAlgorithm<Outl */ protected double optimalBandwidth(int dim) { // Pi in the publication is redundant and cancels out! - double hopt = 8 * Math.exp(GammaDistribution.logGamma(dim / 2.0 + 1)) * (dim + 4) * Math.pow(2, dim); + double hopt = 8 * GammaDistribution.gamma(dim / 2.0 + 1) * (dim + 4) * Math.pow(2, dim); return hopt * Math.pow(relation.size(), (-1 / (dim + 4))); } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/subspace/OutRankS1.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/subspace/OutRankS1.java new file mode 100644 index 00000000..e370d2bf --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/subspace/OutRankS1.java @@ -0,0 +1,199 @@ +package de.lmu.ifi.dbs.elki.algorithm.outlier.subspace; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import de.lmu.ifi.dbs.elki.algorithm.AbstractAlgorithm; +import de.lmu.ifi.dbs.elki.algorithm.clustering.subspace.SubspaceClusteringAlgorithm; +import de.lmu.ifi.dbs.elki.algorithm.outlier.OutlierAlgorithm; +import de.lmu.ifi.dbs.elki.data.Cluster; +import de.lmu.ifi.dbs.elki.data.Clustering; +import de.lmu.ifi.dbs.elki.data.model.SubspaceModel; +import de.lmu.ifi.dbs.elki.data.type.TypeInformation; +import de.lmu.ifi.dbs.elki.data.type.TypeUtil; +import de.lmu.ifi.dbs.elki.database.Database; +import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; +import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; +import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDs; +import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation; +import de.lmu.ifi.dbs.elki.database.relation.Relation; +import de.lmu.ifi.dbs.elki.logging.Logging; +import de.lmu.ifi.dbs.elki.math.DoubleMinMax; +import de.lmu.ifi.dbs.elki.result.outlier.InvertedOutlierScoreMeta; +import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; +import de.lmu.ifi.dbs.elki.result.outlier.OutlierScoreMeta; +import de.lmu.ifi.dbs.elki.utilities.documentation.Description; +import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; +import de.lmu.ifi.dbs.elki.utilities.documentation.Title; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.GreaterConstraint; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.DoubleParameter; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter; + +/** + * OutRank: ranking outliers in high dimensional data. + * + * Algorithm to score outliers based on a subspace clustering result. This class + * implements score 1 of the OutRank publication, which is a score based on + * cluster sizes and cluster dimensionality. + * + * Reference: + * <p> + * Emmanuel Müller, Ira Assent, Uwe Steinhausen, Thomas Seidl<br /> + * OutRank: ranking outliers in high dimensional data<br /> + * In Proceedings 24th International Conference on Data Engineering (ICDE) + * Workshop on Ranking in Databases (DBRank), Cancun, Mexico + * </p> + * + * @author Erich Schubert + */ +@Title("OutRank: ranking outliers in high dimensional data") +@Description("Ranking outliers in high dimensional data - score 1") +@Reference(authors = "Emmanuel Müller, Ira Assent, Uwe Steinhausen, Thomas Seidl", title = "OutRank: ranking outliers in high dimensional data", booktitle = "Proc. 24th Int. Conf. on Data Engineering (ICDE) Workshop on Ranking in Databases (DBRank), Cancun, Mexico", url = "http://dx.doi.org/10.1109/ICDEW.2008.4498387") +public class OutRankS1 extends AbstractAlgorithm<OutlierResult> implements OutlierAlgorithm { + /** + * The logger for this class. + */ + private static final Logging logger = Logging.getLogger(OutRankS1.class); + + /** + * Clustering algorithm to run. + */ + protected SubspaceClusteringAlgorithm<? extends SubspaceModel<?>> clusteralg; + + /** + * Weighting parameter of size vs. dimensionality score. + */ + double alpha; + + /** + * Constructor. + * + * @param clusteralg Clustering algorithm to use (must implement + * {@link SubspaceClusteringAlgorithm}!) + * @param alpha Alpha parameter to balance size and dimensionality. + */ + public OutRankS1(SubspaceClusteringAlgorithm<? extends SubspaceModel<?>> clusteralg, double alpha) { + super(); + this.clusteralg = clusteralg; + this.alpha = alpha; + } + + @Override + public OutlierResult run(Database database) { + DBIDs ids = database.getRelation(TypeUtil.DBID).getDBIDs(); + // Run the primary algorithm + Clustering<? extends SubspaceModel<?>> clustering = clusteralg.run(database); + + WritableDoubleDataStore score = DataStoreUtil.makeDoubleStorage(ids, DataStoreFactory.HINT_HOT); + for(DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + score.putDouble(iter, 0); + } + + int maxdim = 0, maxsize = 0; + // Find maximum dimensionality and cluster size + for(Cluster<? extends SubspaceModel<?>> cluster : clustering.getAllClusters()) { + maxsize = Math.max(maxsize, cluster.size()); + maxdim = Math.max(maxdim, cluster.getModel().getDimensions().cardinality()); + } + // Iterate over all clusters: + DoubleMinMax minmax = new DoubleMinMax(); + for(Cluster<? extends SubspaceModel<?>> cluster : clustering.getAllClusters()) { + double relsize = cluster.size() / (double) maxsize; + double reldim = cluster.getModel().getDimensions().cardinality() / (double) maxdim; + // Process objects in the cluster + for(DBIDIter iter = cluster.getIDs().iter(); iter.valid(); iter.advance()) { + double newscore = score.doubleValue(iter) + alpha * relsize + (1 - alpha) * reldim; + score.putDouble(iter, newscore); + minmax.put(newscore); + } + } + + Relation<Double> scoreResult = new MaterializedRelation<Double>("OutRank-S1", "OUTRANK_S1", TypeUtil.DOUBLE, score, ids); + OutlierScoreMeta meta = new InvertedOutlierScoreMeta(minmax.getMin(), minmax.getMax(), 0, Double.POSITIVE_INFINITY); + OutlierResult res = new OutlierResult(meta, scoreResult); + res.addChildResult(clustering); + return res; + } + + @Override + public TypeInformation[] getInputTypeRestriction() { + return clusteralg.getInputTypeRestriction(); + } + + @Override + protected Logging getLogger() { + return logger; + } + + /** + * Parameterization class. + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer extends AbstractParameterizer { + /** + * Clustering algorithm to use. + */ + public static final OptionID ALGORITHM_ID = OptionID.getOrCreateOptionID("outrank.algorithm", "Subspace clustering algorithm to use."); + + /** + * Alpha parameter for S1 + */ + public static final OptionID ALPHA_ID = OptionID.getOrCreateOptionID("outrank.s1.alpha", "Alpha parameter for S1 score."); + + /** + * Clustering algorithm to run. + */ + protected SubspaceClusteringAlgorithm<? extends SubspaceModel<?>> algorithm = null; + + /** + * Alpha parameter to balance parameters + */ + protected double alpha = 0.25; + + @Override + protected void makeOptions(Parameterization config) { + super.makeOptions(config); + ObjectParameter<SubspaceClusteringAlgorithm<? extends SubspaceModel<?>>> algP = new ObjectParameter<SubspaceClusteringAlgorithm<? extends SubspaceModel<?>>>(ALGORITHM_ID, SubspaceClusteringAlgorithm.class); + if(config.grab(algP)) { + algorithm = algP.instantiateClass(config); + } + DoubleParameter alphaP = new DoubleParameter(ALPHA_ID, new GreaterConstraint(0), 0.25); + if(config.grab(alphaP)) { + alpha = alphaP.getValue(); + } + } + + @Override + protected OutRankS1 makeInstance() { + return new OutRankS1(algorithm, alpha); + } + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/SOD.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/subspace/SOD.java index a09bbcfd..7fef95e0 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/SOD.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/subspace/SOD.java @@ -1,4 +1,4 @@ -package de.lmu.ifi.dbs.elki.algorithm.outlier; +package de.lmu.ifi.dbs.elki.algorithm.outlier.subspace; /* This file is part of ELKI: @@ -24,9 +24,9 @@ package de.lmu.ifi.dbs.elki.algorithm.outlier; */ import java.util.BitSet; -import java.util.Iterator; import de.lmu.ifi.dbs.elki.algorithm.AbstractAlgorithm; +import de.lmu.ifi.dbs.elki.algorithm.outlier.OutlierAlgorithm; import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeInformation; @@ -37,6 +37,8 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.query.similarity.SimilarityQuery; @@ -61,8 +63,6 @@ import de.lmu.ifi.dbs.elki.utilities.datastructures.heap.TiedTopBoundedHeap; import de.lmu.ifi.dbs.elki.utilities.documentation.Description; import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; import de.lmu.ifi.dbs.elki.utilities.documentation.Title; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableUtil; import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.GreaterConstraint; @@ -73,6 +73,16 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter; import de.lmu.ifi.dbs.elki.utilities.pairs.DoubleObjPair; /** + * Subspace Outlier Degree. Outlier detection method for axis-parallel subspaces. + * + * Reference: + * <p> + * * H.-P. Kriegel, P. Kröger, E. Schubert, A. Zimek:<br /> + * Outlier Detection in Axis-Parallel Subspaces of High Dimensional Data<br /> + * In: Proceedings of the 13th Pacific-Asia Conference on Knowledge Discovery + * and Data Mining (PAKDD), Bangkok, Thailand, 2009 + * </p> + * * @author Arthur Zimek * * @apiviz.has SODModel oneway - - computes @@ -141,20 +151,20 @@ public class SOD<V extends NumberVector<V, ?>, D extends NumberDistance<D, ?>> e * Performs the SOD algorithm on the given database. * * @param relation Data relation to process + * @return Outlier result */ - public OutlierResult run(Relation<V> relation) throws IllegalStateException { + public OutlierResult run(Relation<V> relation) { SimilarityQuery<V, D> snnInstance = similarityFunction.instantiate(relation); FiniteProgress progress = logger.isVerbose() ? new FiniteProgress("Assigning Subspace Outlier Degree", relation.size(), logger) : null; WritableDataStore<SODModel<?>> sod_models = DataStoreUtil.makeStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC, SODModel.class); DoubleMinMax minmax = new DoubleMinMax(); - for(Iterator<DBID> iter = relation.iterDBIDs(); iter.hasNext();) { - DBID queryObject = iter.next(); + for(DBIDIter iter = relation.iterDBIDs(); iter.valid(); iter.advance()) { if(progress != null) { progress.incrementProcessed(logger); } - DBIDs knnList = getNearestNeighbors(relation, snnInstance, queryObject); - SODModel<V> model = new SODModel<V>(relation, knnList, alpha, relation.get(queryObject)); - sod_models.put(queryObject, model); + DBIDs knnList = getNearestNeighbors(relation, snnInstance, iter); + SODModel<V> model = new SODModel<V>(relation, knnList, alpha, relation.get(iter)); + sod_models.put(iter, model); minmax.put(model.getSod()); } if(progress != null) { @@ -181,14 +191,14 @@ public class SOD<V extends NumberVector<V, ?>, D extends NumberDistance<D, ?>> e * @return the k nearest neighbors in terms of the shared nearest neighbor * distance without the query object */ - private DBIDs getNearestNeighbors(Relation<V> relation, SimilarityQuery<V, D> simQ, DBID queryObject) { + private DBIDs getNearestNeighbors(Relation<V> relation, SimilarityQuery<V, D> simQ, DBIDRef queryObject) { // similarityFunction.getPreprocessor().getParameters(); Heap<DoubleObjPair<DBID>> nearestNeighbors = new TiedTopBoundedHeap<DoubleObjPair<DBID>>(knn); - for(DBID id : relation.iterDBIDs()) { - if(!id.equals(queryObject)) { - double sim = simQ.similarity(queryObject, id).doubleValue(); + for(DBIDIter iter = relation.iterDBIDs(); iter.valid(); iter.advance()) { + if(!iter.sameDBID(queryObject)) { + double sim = simQ.similarity(queryObject, iter).doubleValue(); if(sim > 0) { - nearestNeighbors.add(new DoubleObjPair<DBID>(sim, id)); + nearestNeighbors.add(new DoubleObjPair<DBID>(sim, iter.getDBID())); } } } @@ -244,8 +254,8 @@ public class SOD<V extends NumberVector<V, ?>, D extends NumberDistance<D, ?>> e // TODO: store database link? centerValues = new double[DatabaseUtil.dimensionality(relation)]; variances = new double[centerValues.length]; - for(DBID id : neighborhood) { - V databaseObject = relation.get(id); + for(DBIDIter iter = neighborhood.iter(); iter.valid(); iter.advance()) { + V databaseObject = relation.get(iter); for(int d = 0; d < centerValues.length; d++) { centerValues[d] += databaseObject.doubleValue(d + 1); } @@ -253,8 +263,8 @@ public class SOD<V extends NumberVector<V, ?>, D extends NumberDistance<D, ?>> e for(int d = 0; d < centerValues.length; d++) { centerValues[d] /= neighborhood.size(); } - for(DBID id : neighborhood) { - V databaseObject = relation.get(id); + for(DBIDIter iter = neighborhood.iter(); iter.valid(); iter.advance()) { + V databaseObject = relation.get(iter); for(int d = 0; d < centerValues.length; d++) { // distance double distance = centerValues[d] - databaseObject.doubleValue(d + 1); @@ -359,7 +369,7 @@ public class SOD<V extends NumberVector<V, ?>, D extends NumberDistance<D, ?>> e } @Override - public Double get(DBID objID) { + public Double get(DBIDRef objID) { return models.get(objID).getSod(); } @@ -379,8 +389,8 @@ public class SOD<V extends NumberVector<V, ?>, D extends NumberDistance<D, ?>> e } @Override - public IterableIterator<DBID> iterDBIDs() { - return IterableUtil.fromIterator(dbids.iterator()); + public DBIDIter iterDBIDs() { + return dbids.iter(); } @Override @@ -389,12 +399,12 @@ public class SOD<V extends NumberVector<V, ?>, D extends NumberDistance<D, ?>> e } @Override - public void set(DBID id, Double val) { + public void set(DBIDRef id, Double val) { throw new UnsupportedOperationException(); } @Override - public void delete(DBID id) { + public void delete(DBIDRef id) { throw new UnsupportedOperationException(); } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/subspace/package-info.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/subspace/package-info.java new file mode 100644 index 00000000..8b1c80df --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/subspace/package-info.java @@ -0,0 +1,28 @@ +/** + * <p>Subspace outlier detection methods.</p> + * + * Methods that detect outliers in subspaces (projections) of the data set. + */ +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +package de.lmu.ifi.dbs.elki.algorithm.outlier.subspace;
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/trivial/ByLabelOutlier.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/trivial/ByLabelOutlier.java index 86730404..66a89cf5 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/trivial/ByLabelOutlier.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/trivial/ByLabelOutlier.java @@ -35,7 +35,7 @@ import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.logging.Logging; @@ -112,15 +112,10 @@ public class ByLabelOutlier extends AbstractAlgorithm<OutlierResult> implements */ public OutlierResult run(Relation<?> relation) { WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_HOT); - for(DBID id : relation.iterDBIDs()) { - String label = relation.get(id).toString(); - final double score; - if (pattern.matcher(label).matches()) { - score = 1.0; - } else { - score = 0.0; - } - scores.putDouble(id, score); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + String label = relation.get(iditer).toString(); + final double score = (pattern.matcher(label).matches()) ? 1 : 0; + scores.putDouble(iditer, score); } Relation<Double> scoreres = new MaterializedRelation<Double>("By label outlier scores", "label-outlier", TypeUtil.DOUBLE, scores, relation.getDBIDs()); OutlierScoreMeta meta = new ProbabilisticOutlierScore(); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/trivial/TrivialAllOutlier.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/trivial/TrivialAllOutlier.java index 509e35e9..b50226f1 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/trivial/TrivialAllOutlier.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/trivial/TrivialAllOutlier.java @@ -30,7 +30,7 @@ import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.logging.Logging; @@ -70,8 +70,8 @@ public class TrivialAllOutlier extends AbstractAlgorithm<OutlierResult> implemen */ public OutlierResult run(Relation<?> relation) { WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_HOT); - for(DBID id : relation.iterDBIDs()) { - scores.putDouble(id, 1.0); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + scores.putDouble(iditer, 1.0); } Relation<Double> scoreres = new MaterializedRelation<Double>("Trivial all-outlier score", "all-outlier", TypeUtil.DOUBLE, scores, relation.getDBIDs()); OutlierScoreMeta meta = new ProbabilisticOutlierScore(); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/trivial/TrivialGeneratedOutlier.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/trivial/TrivialGeneratedOutlier.java index db40ff30..d1c2e076 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/trivial/TrivialGeneratedOutlier.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/trivial/TrivialGeneratedOutlier.java @@ -37,7 +37,7 @@ import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.logging.Logging; @@ -100,7 +100,7 @@ public class TrivialGeneratedOutlier extends AbstractAlgorithm<OutlierResult> im } @Override - public OutlierResult run(Database database) throws IllegalStateException { + public OutlierResult run(Database database) { Relation<NumberVector<?, ?>> vecs = database.getRelation(TypeUtil.NUMBER_VECTOR_FIELD); Relation<Model> models = database.getRelation(new SimpleTypeInformation<Model>(Model.class)); // Prefer a true class label @@ -129,8 +129,8 @@ public class TrivialGeneratedOutlier extends AbstractAlgorithm<OutlierResult> im final double minscore = expect / (expect + 1); HashSet<GeneratorSingleCluster> generators = new HashSet<GeneratorSingleCluster>(); - for(DBID id : models.iterDBIDs()) { - Model model = models.get(id); + for(DBIDIter iditer = models.iterDBIDs(); iditer.valid(); iditer.advance()) { + Model model = models.get(iditer); if(model instanceof GeneratorSingleCluster) { generators.add((GeneratorSingleCluster) model); } @@ -139,10 +139,10 @@ public class TrivialGeneratedOutlier extends AbstractAlgorithm<OutlierResult> im logger.warning("No generator models found for dataset - all points will be considered outliers."); } - for(DBID id : models.iterDBIDs()) { + for(DBIDIter iditer = models.iterDBIDs(); iditer.valid(); iditer.advance()) { double score = 0.0; // Convert to a math vector - Vector v = vecs.get(id).getColumnVector(); + Vector v = vecs.get(iditer).getColumnVector(); for(GeneratorSingleCluster gen : generators) { Vector tv = v; // Transform backwards @@ -170,7 +170,7 @@ public class TrivialGeneratedOutlier extends AbstractAlgorithm<OutlierResult> im score = expect / (expect + score); // adjust to 0 to 1 range: score = (score - minscore) / (1 - minscore); - scores.putDouble(id, score); + scores.putDouble(iditer, score); } Relation<Double> scoreres = new MaterializedRelation<Double>("Model outlier scores", "model-outlier", TypeUtil.DOUBLE, scores, models.getDBIDs()); OutlierScoreMeta meta = new ProbabilisticOutlierScore(0., 1.); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/trivial/TrivialNoOutlier.java b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/trivial/TrivialNoOutlier.java index cff2ad2c..6d8e9f46 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/outlier/trivial/TrivialNoOutlier.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/outlier/trivial/TrivialNoOutlier.java @@ -30,7 +30,7 @@ import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.logging.Logging; @@ -68,10 +68,10 @@ public class TrivialNoOutlier extends AbstractAlgorithm<OutlierResult> implement * @param relation Relation * @return Result */ - public OutlierResult run(Relation<?> relation) throws IllegalStateException { + public OutlierResult run(Relation<?> relation) { WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_HOT); - for(DBID id : relation.iterDBIDs()) { - scores.putDouble(id, 0.0); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + scores.putDouble(iditer, 0.0); } Relation<Double> scoreres = new MaterializedRelation<Double>("Trivial no-outlier score", "no-outlier", TypeUtil.DOUBLE, scores, relation.getDBIDs()); OutlierScoreMeta meta = new ProbabilisticOutlierScore(); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/statistics/AddSingleScale.java b/src/de/lmu/ifi/dbs/elki/algorithm/statistics/AddSingleScale.java new file mode 100644 index 00000000..481261b3 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/algorithm/statistics/AddSingleScale.java @@ -0,0 +1,97 @@ +package de.lmu.ifi.dbs.elki.algorithm.statistics; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +import de.lmu.ifi.dbs.elki.algorithm.Algorithm; +import de.lmu.ifi.dbs.elki.data.NumberVector; +import de.lmu.ifi.dbs.elki.data.type.TypeInformation; +import de.lmu.ifi.dbs.elki.data.type.TypeUtil; +import de.lmu.ifi.dbs.elki.database.Database; +import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.relation.Relation; +import de.lmu.ifi.dbs.elki.math.DoubleMinMax; +import de.lmu.ifi.dbs.elki.math.scales.LinearScale; +import de.lmu.ifi.dbs.elki.result.Result; +import de.lmu.ifi.dbs.elki.result.ResultUtil; +import de.lmu.ifi.dbs.elki.result.ScalesResult; +import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; +import de.lmu.ifi.dbs.elki.utilities.documentation.Description; + +/** + * Pseudo "algorith" that computes the global min/max for a relation across all + * attributes. + * + * @author Erich Schubert + */ +@Description("Setup a scaling so that all dimensions are scaled equally in visualization.") +public class AddSingleScale implements Algorithm { + /** + * Constructor. + */ + public AddSingleScale() { + super(); + } + + @SuppressWarnings("unchecked") + @Override + public Result run(Database database) { + for(Relation<?> rel : database.getRelations()) { + if(TypeUtil.NUMBER_VECTOR_FIELD.isAssignableFromType(rel.getDataTypeInformation())) { + ScalesResult res = run((Relation<? extends NumberVector<?, ?>>) rel); + ResultUtil.addChildResult(rel, res); + } + } + return null; + } + + /** + * Add scales to a single vector relation. + * + * @param rel Relation + * @return Scales + */ + private ScalesResult run(Relation<? extends NumberVector<?, ?>> rel) { + final int dim = DatabaseUtil.dimensionality(rel); + DoubleMinMax minmax = new DoubleMinMax(); + for(DBIDIter iditer = rel.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); + NumberVector<?, ?> vec = rel.get(id); + for(int d = 1; d <= dim; d++) { + minmax.put(vec.doubleValue(d)); + } + } + LinearScale scale = new LinearScale(minmax.getMin(), minmax.getMax()); + LinearScale[] scales = new LinearScale[dim]; + for(int i = 0; i < dim; i++) { + scales[i] = scale; + } + ScalesResult res = new ScalesResult(scales); + return res; + } + + @Override + public TypeInformation[] getInputTypeRestriction() { + return TypeUtil.array(TypeUtil.NUMBER_VECTOR_FIELD); + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/statistics/AveragePrecisionAtK.java b/src/de/lmu/ifi/dbs/elki/algorithm/statistics/AveragePrecisionAtK.java index 1c74621b..f6f1d16f 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/statistics/AveragePrecisionAtK.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/statistics/AveragePrecisionAtK.java @@ -34,6 +34,7 @@ import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; @@ -101,7 +102,7 @@ public class AveragePrecisionAtK<V extends Object, D extends NumberDistance<D, ? } @Override - public HistogramResult<DoubleVector> run(Database database) throws IllegalStateException { + public HistogramResult<DoubleVector> run(Database database) { final Relation<V> relation = database.getRelation(getInputTypeRestriction()[0]); final Relation<Object> lrelation = database.getRelation(getInputTypeRestriction()[1]); final DistanceQuery<V, D> distQuery = database.getDistanceQuery(relation, getDistanceFunction()); @@ -122,7 +123,8 @@ public class AveragePrecisionAtK<V extends Object, D extends NumberDistance<D, ? } FiniteProgress objloop = logger.isVerbose() ? new FiniteProgress("Computing nearest neighbors", ids.size(), logger) : null; // sort neighbors - for(DBID id : ids) { + for(DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); KNNResult<D> knn = knnQuery.getKNNForDBID(id, k); Object label = lrelation.get(id); diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/statistics/DistanceStatisticsWithClasses.java b/src/de/lmu/ifi/dbs/elki/algorithm/statistics/DistanceStatisticsWithClasses.java index 78bbf5f4..d6ce6a15 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/statistics/DistanceStatisticsWithClasses.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/statistics/DistanceStatisticsWithClasses.java @@ -31,7 +31,7 @@ import java.util.Random; import java.util.TreeSet; import de.lmu.ifi.dbs.elki.algorithm.AbstractDistanceBasedAlgorithm; -import de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.ByLabelClustering; +import de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.ByLabelOrAllInOneClustering; import de.lmu.ifi.dbs.elki.data.Cluster; import de.lmu.ifi.dbs.elki.data.DoubleVector; import de.lmu.ifi.dbs.elki.data.model.Model; @@ -40,6 +40,7 @@ import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; @@ -66,7 +67,6 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.Flag; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.IntParameter; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.Parameter; import de.lmu.ifi.dbs.elki.utilities.pairs.DoubleObjPair; -import de.lmu.ifi.dbs.elki.utilities.pairs.FCPair; import de.lmu.ifi.dbs.elki.utilities.pairs.Pair; /** @@ -131,11 +131,8 @@ public class DistanceStatisticsWithClasses<O, D extends NumberDistance<D, ?>> ex this.sampling = sampling; } - /** - * Iterates over all points in the database. - */ @Override - public HistogramResult<DoubleVector> run(Database database) throws IllegalStateException { + public HistogramResult<DoubleVector> run(Database database) { final Relation<O> relation = database.getRelation(getInputTypeRestriction()[0]); final DistanceQuery<O, D> distFunc = database.getDistanceQuery(relation, getDistanceFunction()); @@ -145,7 +142,7 @@ public class DistanceStatisticsWithClasses<O, D extends NumberDistance<D, ?>> ex DoubleMinMax gminmax = new DoubleMinMax(); // Cluster by labels - Collection<Cluster<Model>> split = (new ByLabelClustering()).run(database).getAllClusters(); + Collection<Cluster<Model>> split = (new ByLabelOrAllInOneClustering()).run(database).getAllClusters(); // global in-cluster min/max DoubleMinMax giminmax = new DoubleMinMax(); @@ -184,12 +181,14 @@ public class DistanceStatisticsWithClasses<O, D extends NumberDistance<D, ?>> ex final Pair<Long, Long> incFirst = new Pair<Long, Long>(1L, 0L); final Pair<Long, Long> incSecond = new Pair<Long, Long>(0L, 1L); for(Cluster<?> c1 : split) { - for(DBID id1 : c1.getIDs()) { + for(DBIDIter iter = c1.getIDs().iter(); iter.valid(); iter.advance()) { + DBID id1 = iter.getDBID(); // in-cluster distances DoubleMinMax iminmax = new DoubleMinMax(); - for(DBID id2 : c1.getIDs()) { + for(DBIDIter iter2 = c1.getIDs().iter(); iter2.valid(); iter2.advance()) { + DBID id2 = iter2.getDBID(); // skip the point itself. - if(id1 == id2) { + if(id1.sameDBID(id2)) { continue; } double d = distFunc.distance(id1, id2).doubleValue(); @@ -212,9 +211,10 @@ public class DistanceStatisticsWithClasses<O, D extends NumberDistance<D, ?>> ex if(c2 == c1) { continue; } - for(DBID id2 : c2.getIDs()) { + for(DBIDIter iter2 = c2.getIDs().iter(); iter2.valid(); iter2.advance()) { + DBID id2 = iter2.getDBID(); // skip the point itself (shouldn't happen though) - if(id1 == id2) { + if(id1.sameDBID(id2)) { continue; } double d = distFunc.distance(id1, id2).doubleValue(); @@ -255,8 +255,6 @@ public class DistanceStatisticsWithClasses<O, D extends NumberDistance<D, ?>> ex onum += ppair.getSecond().getSecond(); } long bnum = inum + onum; - // Note: when full sampling is added, this assertion won't hold anymore. - assert (bnum == relation.size() * (relation.size() - 1)); Collection<DoubleVector> binstat = new ArrayList<DoubleVector>(numbin); for(DoubleObjPair<Pair<Long, Long>> ppair : histogram) { @@ -285,58 +283,62 @@ public class DistanceStatisticsWithClasses<O, D extends NumberDistance<D, ?>> ex Random rnd = new Random(); // estimate minimum and maximum. int k = (int) Math.max(25, Math.pow(database.size(), 0.2)); - TreeSet<FCPair<Double, DBID>> minhotset = new TreeSet<FCPair<Double, DBID>>(); - TreeSet<FCPair<Double, DBID>> maxhotset = new TreeSet<FCPair<Double, DBID>>(Collections.reverseOrder()); + TreeSet<DoubleObjPair<DBID>> minhotset = new TreeSet<DoubleObjPair<DBID>>(); + TreeSet<DoubleObjPair<DBID>> maxhotset = new TreeSet<DoubleObjPair<DBID>>(Collections.reverseOrder()); int randomsize = (int) Math.max(25, Math.pow(database.size(), 0.2)); double rprob = ((double) randomsize) / size; ArrayModifiableDBIDs randomset = DBIDUtil.newArray(randomsize); - Iterator<DBID> iter = database.iterDBIDs(); - if(!iter.hasNext()) { + DBIDIter iter = database.iterDBIDs(); + if(!iter.valid()) { throw new IllegalStateException(ExceptionMessages.DATABASE_EMPTY); } - DBID firstid = iter.next(); - minhotset.add(new FCPair<Double, DBID>(Double.MAX_VALUE, firstid)); - maxhotset.add(new FCPair<Double, DBID>(Double.MIN_VALUE, firstid)); - while(iter.hasNext()) { - DBID id1 = iter.next(); + DBID firstid = iter.getDBID(); + iter.advance(); + minhotset.add(new DoubleObjPair<DBID>(Double.MAX_VALUE, firstid)); + maxhotset.add(new DoubleObjPair<DBID>(Double.MIN_VALUE, firstid)); + while(iter.valid()) { + DBID id1 = iter.getDBID(); + iter.advance(); // generate candidates for min distance. - ArrayList<FCPair<Double, DBID>> np = new ArrayList<FCPair<Double, DBID>>(k * 2 + randomsize * 2); - for(FCPair<Double, DBID> pair : minhotset) { + ArrayList<DoubleObjPair<DBID>> np = new ArrayList<DoubleObjPair<DBID>>(k * 2 + randomsize * 2); + for(DoubleObjPair<DBID> pair : minhotset) { DBID id2 = pair.getSecond(); // skip the object itself if(id1.compareTo(id2) == 0) { continue; } double d = distFunc.distance(id1, id2).doubleValue(); - np.add(new FCPair<Double, DBID>(d, id1)); - np.add(new FCPair<Double, DBID>(d, id2)); + np.add(new DoubleObjPair<DBID>(d, id1)); + np.add(new DoubleObjPair<DBID>(d, id2)); } - for(DBID id2 : randomset) { + for(DBIDIter iter2 = randomset.iter(); iter2.valid(); iter2.advance()) { + DBID id2 = iter2.getDBID(); double d = distFunc.distance(id1, id2).doubleValue(); - np.add(new FCPair<Double, DBID>(d, id1)); - np.add(new FCPair<Double, DBID>(d, id2)); + np.add(new DoubleObjPair<DBID>(d, id1)); + np.add(new DoubleObjPair<DBID>(d, id2)); } minhotset.addAll(np); shrinkHeap(minhotset, k); // generate candidates for max distance. - ArrayList<FCPair<Double, DBID>> np2 = new ArrayList<FCPair<Double, DBID>>(k * 2 + randomsize * 2); - for(FCPair<Double, DBID> pair : minhotset) { + ArrayList<DoubleObjPair<DBID>> np2 = new ArrayList<DoubleObjPair<DBID>>(k * 2 + randomsize * 2); + for(DoubleObjPair<DBID> pair : minhotset) { DBID id2 = pair.getSecond(); // skip the object itself if(id1.compareTo(id2) == 0) { continue; } double d = distFunc.distance(id1, id2).doubleValue(); - np2.add(new FCPair<Double, DBID>(d, id1)); - np2.add(new FCPair<Double, DBID>(d, id2)); + np2.add(new DoubleObjPair<DBID>(d, id1)); + np2.add(new DoubleObjPair<DBID>(d, id2)); } - for(DBID id2 : randomset) { + for(DBIDIter iter2 = randomset.iter(); iter2.valid(); iter2.advance()) { + DBID id2 = iter2.getDBID(); double d = distFunc.distance(id1, id2).doubleValue(); - np.add(new FCPair<Double, DBID>(d, id1)); - np.add(new FCPair<Double, DBID>(d, id2)); + np.add(new DoubleObjPair<DBID>(d, id1)); + np.add(new DoubleObjPair<DBID>(d, id2)); } maxhotset.addAll(np2); shrinkHeap(maxhotset, k); @@ -349,14 +351,16 @@ public class DistanceStatisticsWithClasses<O, D extends NumberDistance<D, ?>> ex randomset.set((int) Math.floor(rnd.nextDouble() * randomsize), id1); } } - return new DoubleMinMax(minhotset.first().getFirst(), maxhotset.first().getFirst()); + return new DoubleMinMax(minhotset.first().first, maxhotset.first().first); } private DoubleMinMax exactMinMax(Relation<O> database, DistanceQuery<O, D> distFunc) { DoubleMinMax minmax = new DoubleMinMax(); // find exact minimum and maximum first. - for(DBID id1 : database.iterDBIDs()) { - for(DBID id2 : database.iterDBIDs()) { + for(DBIDIter iditer = database.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id1 = iditer.getDBID(); + for(DBIDIter iditer2 = database.iterDBIDs(); iditer2.valid(); iditer2.advance()) { + DBID id2 = iditer2.getDBID(); // skip the point itself. if(id1.compareTo(id2) == 0) { continue; @@ -368,12 +372,12 @@ public class DistanceStatisticsWithClasses<O, D extends NumberDistance<D, ?>> ex return minmax; } - private void shrinkHeap(TreeSet<FCPair<Double, DBID>> hotset, int k) { + private void shrinkHeap(TreeSet<DoubleObjPair<DBID>> hotset, int k) { // drop duplicates ModifiableDBIDs seenids = DBIDUtil.newHashSet(2 * k); int cnt = 0; - for(Iterator<FCPair<Double, DBID>> i = hotset.iterator(); i.hasNext();) { - FCPair<Double, DBID> p = i.next(); + for(Iterator<DoubleObjPair<DBID>> i = hotset.iterator(); i.hasNext();) { + DoubleObjPair<DBID> p = i.next(); if(cnt > k || seenids.contains(p.getSecond())) { i.remove(); } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/statistics/EvaluateRankingQuality.java b/src/de/lmu/ifi/dbs/elki/algorithm/statistics/EvaluateRankingQuality.java index c1eb118d..353c1b02 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/statistics/EvaluateRankingQuality.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/statistics/EvaluateRankingQuality.java @@ -29,7 +29,7 @@ import java.util.Collections; import java.util.HashMap; import de.lmu.ifi.dbs.elki.algorithm.AbstractDistanceBasedAlgorithm; -import de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.ByLabelClustering; +import de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.ByLabelOrAllInOneClustering; import de.lmu.ifi.dbs.elki.data.Cluster; import de.lmu.ifi.dbs.elki.data.DoubleVector; import de.lmu.ifi.dbs.elki.data.NumberVector; @@ -39,6 +39,7 @@ import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery; import de.lmu.ifi.dbs.elki.database.query.knn.KNNResult; @@ -114,11 +115,8 @@ public class EvaluateRankingQuality<V extends NumberVector<V, ?>, D extends Numb */ int numbins = 20; - /** - * Run the algorithm. - */ @Override - public HistogramResult<DoubleVector> run(Database database) throws IllegalStateException { + public HistogramResult<DoubleVector> run(Database database) { final Relation<V> relation = database.getRelation(getInputTypeRestriction()[0]); final DistanceQuery<V, D> distQuery = database.getDistanceQuery(relation, getDistanceFunction()); final KNNQuery<V, D> knnQuery = database.getKNNQuery(distQuery, relation.size()); @@ -127,7 +125,7 @@ public class EvaluateRankingQuality<V extends NumberVector<V, ?>, D extends Numb logger.verbose("Preprocessing clusters..."); } // Cluster by labels - Collection<Cluster<Model>> split = (new ByLabelClustering()).run(database).getAllClusters(); + Collection<Cluster<Model>> split = (new ByLabelOrAllInOneClustering()).run(database).getAllClusters(); // Compute cluster averages and covariance matrix HashMap<Cluster<?>, V> averages = new HashMap<Cluster<?>, V>(split.size()); @@ -150,7 +148,8 @@ public class EvaluateRankingQuality<V extends NumberVector<V, ?>, D extends Numb Vector av = averages.get(clus).getColumnVector(); Matrix covm = covmats.get(clus); - for(DBID i1 : clus.getIDs()) { + for(DBIDIter iter = clus.getIDs().iter(); iter.valid(); iter.advance()) { + DBID i1 = iter.getDBID(); Double d = MathUtil.mahalanobisDistance(covm, av.minus(relation.get(i1).getColumnVector())); cmem.add(new FCPair<Double, DBID>(d, i1)); } diff --git a/src/de/lmu/ifi/dbs/elki/algorithm/statistics/RankingQualityHistogram.java b/src/de/lmu/ifi/dbs/elki/algorithm/statistics/RankingQualityHistogram.java index 6d64dc55..4305bbca 100644 --- a/src/de/lmu/ifi/dbs/elki/algorithm/statistics/RankingQualityHistogram.java +++ b/src/de/lmu/ifi/dbs/elki/algorithm/statistics/RankingQualityHistogram.java @@ -27,7 +27,7 @@ import java.util.ArrayList; import java.util.Collection; import de.lmu.ifi.dbs.elki.algorithm.AbstractDistanceBasedAlgorithm; -import de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.ByLabelClustering; +import de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.ByLabelOrAllInOneClustering; import de.lmu.ifi.dbs.elki.data.Cluster; import de.lmu.ifi.dbs.elki.data.DoubleVector; import de.lmu.ifi.dbs.elki.data.model.Model; @@ -35,6 +35,7 @@ import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery; import de.lmu.ifi.dbs.elki.database.query.knn.KNNResult; @@ -107,7 +108,7 @@ public class RankingQualityHistogram<O, D extends NumberDistance<D, ?>> extends logger.verbose("Preprocessing clusters..."); } // Cluster by labels - Collection<Cluster<Model>> split = (new ByLabelClustering()).run(database).getAllClusters(); + Collection<Cluster<Model>> split = (new ByLabelOrAllInOneClustering()).run(database).getAllClusters(); AggregatingHistogram<Double, Double> hist = AggregatingHistogram.DoubleSumHistogram(numbins, 0.0, 1.0); @@ -119,7 +120,8 @@ public class RankingQualityHistogram<O, D extends NumberDistance<D, ?>> extends MeanVariance mv = new MeanVariance(); // sort neighbors for(Cluster<?> clus : split) { - for(DBID i1 : clus.getIDs()) { + for(DBIDIter iter = clus.getIDs().iter(); iter.valid(); iter.advance()) { + DBID i1 = iter.getDBID(); KNNResult<D> knn = knnQuery.getKNNForDBID(i1, relation.size()); double result = ROC.computeROCAUCDistanceResult(relation.size(), clus, knn); diff --git a/src/de/lmu/ifi/dbs/elki/application/GeneratorXMLSpec.java b/src/de/lmu/ifi/dbs/elki/application/GeneratorXMLSpec.java index ccee6d39..19207c5c 100644 --- a/src/de/lmu/ifi/dbs/elki/application/GeneratorXMLSpec.java +++ b/src/de/lmu/ifi/dbs/elki/application/GeneratorXMLSpec.java @@ -44,7 +44,7 @@ import de.lmu.ifi.dbs.elki.datasource.GeneratorXMLDatabaseConnection; import de.lmu.ifi.dbs.elki.datasource.bundle.MultipleObjectsBundle; import de.lmu.ifi.dbs.elki.logging.Logging; import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; -import de.lmu.ifi.dbs.elki.math.statistics.distribution.Distribution; +import de.lmu.ifi.dbs.elki.math.statistics.distribution.DistributionWithRandom; import de.lmu.ifi.dbs.elki.utilities.FormatUtil; import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; import de.lmu.ifi.dbs.elki.utilities.exceptions.UnableToComplyException; @@ -186,7 +186,7 @@ public class GeneratorXMLSpec extends AbstractApplication { outStream.write("## Density correction factor: " + cursclus.getDensityCorrection() + LINE_SEPARATOR); outStream.write("## Generators:" + LINE_SEPARATOR); for(int i = 0; i < cursclus.getDim(); i++) { - Distribution gen = cursclus.getDistribution(i); + DistributionWithRandom gen = cursclus.getDistribution(i); outStream.write("## " + gen.toString() + LINE_SEPARATOR); } if(cursclus.getTransformation() != null && cursclus.getTransformation().getTransformation() != null) { diff --git a/src/de/lmu/ifi/dbs/elki/application/cache/CacheDoubleDistanceInOnDiskMatrix.java b/src/de/lmu/ifi/dbs/elki/application/cache/CacheDoubleDistanceInOnDiskMatrix.java index c2e575be..19a22699 100644 --- a/src/de/lmu/ifi/dbs/elki/application/cache/CacheDoubleDistanceInOnDiskMatrix.java +++ b/src/de/lmu/ifi/dbs/elki/application/cache/CacheDoubleDistanceInOnDiskMatrix.java @@ -30,6 +30,7 @@ import de.lmu.ifi.dbs.elki.application.AbstractApplication; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.StaticArrayDatabase; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction; @@ -119,7 +120,8 @@ public class CacheDoubleDistanceInOnDiskMatrix<O, D extends NumberDistance<D, ?> DistanceQuery<O, D> distanceQuery = database.getDistanceQuery(relation, distance); int matrixsize = 0; - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); matrixsize = Math.max(matrixsize, id.getIntegerID() + 1); if(id.getIntegerID() < 0) { throw new AbortException("OnDiskMatrixCache does not allow negative DBIDs."); @@ -134,8 +136,10 @@ public class CacheDoubleDistanceInOnDiskMatrix<O, D extends NumberDistance<D, ?> throw new AbortException("Error creating output matrix.", e); } - for(DBID id1 : relation.iterDBIDs()) { - for(DBID id2 : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id1 = iditer.getDBID(); + for(DBIDIter iditer2 = relation.iterDBIDs(); iditer2.valid(); iditer2.advance()) { + DBID id2 = iditer2.getDBID(); if(id2.getIntegerID() >= id1.getIntegerID()) { double d = distanceQuery.distance(id1, id2).doubleValue(); if(debugExtraCheckSymmetry) { diff --git a/src/de/lmu/ifi/dbs/elki/application/cache/CacheFloatDistanceInOnDiskMatrix.java b/src/de/lmu/ifi/dbs/elki/application/cache/CacheFloatDistanceInOnDiskMatrix.java index 237aedb9..78a64442 100644 --- a/src/de/lmu/ifi/dbs/elki/application/cache/CacheFloatDistanceInOnDiskMatrix.java +++ b/src/de/lmu/ifi/dbs/elki/application/cache/CacheFloatDistanceInOnDiskMatrix.java @@ -30,6 +30,7 @@ import de.lmu.ifi.dbs.elki.application.AbstractApplication; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.StaticArrayDatabase; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction; @@ -124,7 +125,8 @@ public class CacheFloatDistanceInOnDiskMatrix<O, D extends NumberDistance<D, ?>> DistanceQuery<O, D> distanceQuery = database.getDistanceQuery(relation, distance); int matrixsize = 0; - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); matrixsize = Math.max(matrixsize, id.getIntegerID() + 1); if(id.getIntegerID() < 0) { throw new AbortException("OnDiskMatrixCache does not allow negative DBIDs."); @@ -139,8 +141,10 @@ public class CacheFloatDistanceInOnDiskMatrix<O, D extends NumberDistance<D, ?>> throw new AbortException("Error creating output matrix.", e); } - for(DBID id1 : relation.iterDBIDs()) { - for(DBID id2 : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id1 = iditer.getDBID(); + for(DBIDIter iditer2 = relation.iterDBIDs(); iditer2.valid(); iditer2.advance()) { + DBID id2 = iditer2.getDBID(); if(id2.getIntegerID() >= id1.getIntegerID()) { float d = distanceQuery.distance(id1, id2).floatValue(); if(debugExtraCheckSymmetry) { diff --git a/src/de/lmu/ifi/dbs/elki/application/greedyensemble/ComputeKNNOutlierScores.java b/src/de/lmu/ifi/dbs/elki/application/greedyensemble/ComputeKNNOutlierScores.java index f6506fcc..de1ddbec 100644 --- a/src/de/lmu/ifi/dbs/elki/application/greedyensemble/ComputeKNNOutlierScores.java +++ b/src/de/lmu/ifi/dbs/elki/application/greedyensemble/ComputeKNNOutlierScores.java @@ -44,6 +44,7 @@ import de.lmu.ifi.dbs.elki.data.DoubleVector; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.QueryUtil; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery; import de.lmu.ifi.dbs.elki.database.query.knn.PreprocessorKNNQuery; @@ -185,7 +186,8 @@ public class ComputeKNNOutlierScores<O, D extends NumberDistance<D, ?>> extends { try { MessageDigest md = MessageDigest.getInstance("MD5"); - for(DBID id : ids) { + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); md.update(" ".getBytes()); md.update(id.toString().getBytes()); } @@ -330,8 +332,8 @@ public class ComputeKNNOutlierScores<O, D extends NumberDistance<D, ?>> extends void writeResult(PrintStream out, DBIDs ids, OutlierResult result, ScalingFunction scaling, String label) { out.append(label); Relation<Double> scores = result.getScores(); - for(DBID id : ids) { - final double value = scaling.getScaled(scores.get(id)); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + final double value = scaling.getScaled(scores.get(iter)); out.append(" ").append(FormatUtil.format(value, FormatUtil.NF8)); } out.append(FormatUtil.NEWLINE); diff --git a/src/de/lmu/ifi/dbs/elki/application/greedyensemble/GreedyEnsembleExperiment.java b/src/de/lmu/ifi/dbs/elki/application/greedyensemble/GreedyEnsembleExperiment.java index cfd768c1..6b2f9e13 100644 --- a/src/de/lmu/ifi/dbs/elki/application/greedyensemble/GreedyEnsembleExperiment.java +++ b/src/de/lmu/ifi/dbs/elki/application/greedyensemble/GreedyEnsembleExperiment.java @@ -33,6 +33,7 @@ import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.HashSetModifiableDBIDs; @@ -43,6 +44,7 @@ import de.lmu.ifi.dbs.elki.distance.distancefunction.correlation.WeightedPearson import de.lmu.ifi.dbs.elki.evaluation.roc.ROC; import de.lmu.ifi.dbs.elki.logging.Logging; import de.lmu.ifi.dbs.elki.math.MeanVariance; +import de.lmu.ifi.dbs.elki.math.geometry.XYCurve; import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; import de.lmu.ifi.dbs.elki.utilities.datastructures.heap.TiedTopBoundedHeap; import de.lmu.ifi.dbs.elki.utilities.datastructures.heap.TopBoundedHeap; @@ -114,7 +116,7 @@ public class GreedyEnsembleExperiment extends AbstractApplication { final Database database = inputstep.getDatabase(); final Relation<NumberVector<?, ?>> relation = database.getRelation(TypeUtil.NUMBER_VECTOR_FIELD); final Relation<String> labels = DatabaseUtil.guessLabelRepresentation(database); - final DBID firstid = labels.iterDBIDs().next(); + final DBID firstid = labels.iterDBIDs().getDBID(); final String firstlabel = labels.get(firstid); if(!firstlabel.matches("bylabel")) { throw new AbortException("No 'by label' reference outlier found, which is needed for weighting!"); @@ -137,9 +139,10 @@ public class GreedyEnsembleExperiment extends AbstractApplication { final int[] outliers_seen = new int[dim]; // Find the top-k for each ensemble member { - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); // Skip "by label", obviously - if(firstid.equals(id)) { + if(firstid.sameDBID(id)) { continue; } final NumberVector<?, ?> vec = relation.get(id); @@ -174,7 +177,8 @@ public class GreedyEnsembleExperiment extends AbstractApplication { // Build the naive ensemble: final double[] naiveensemble = new double[dim]; { - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); if(firstid.equals(id)) { continue; } @@ -199,7 +203,8 @@ public class GreedyEnsembleExperiment extends AbstractApplication { double bestest = Double.POSITIVE_INFINITY; { // Compute individual scores - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); if(firstid.equals(id)) { continue; } @@ -250,7 +255,8 @@ public class GreedyEnsembleExperiment extends AbstractApplication { final int heapsize = enscands.size(); TopBoundedHeap<DoubleObjPair<DBID>> heap = new TopBoundedHeap<DoubleObjPair<DBID>>(heapsize, Collections.reverseOrder()); - for(DBID id : enscands) { + for (DBIDIter iter = enscands.iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); final NumberVector<?, ?> vec = relation.get(id); double diversity = wdist.doubleDistance(vec, greedyvec); heap.add(new DoubleObjPair<DBID>(diversity, id)); @@ -301,11 +307,11 @@ public class GreedyEnsembleExperiment extends AbstractApplication { // Build the improved ensemble: StringBuffer greedylbl = new StringBuffer(); { - for(DBID id : ensemble) { + for (DBIDIter iter = ensemble.iter(); iter.valid(); iter.advance()) { if(greedylbl.length() > 0) { greedylbl.append(" "); } - greedylbl.append(labels.get(id)); + greedylbl.append(labels.get(iter)); } } NumberVector<?, ?> greedyvec = refvec.newNumberVector(greedyensemble); @@ -340,7 +346,8 @@ public class GreedyEnsembleExperiment extends AbstractApplication { final double[] randomensemble = new double[dim]; { DBIDs random = DBIDUtil.randomSample(candidates, ensemble.size(), (long)i); - for(DBID id : random) { + for (DBIDIter iter = random.iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); assert (!firstid.equals(id)); // logger.verbose("Using: "+labels.get(id)); final NumberVector<?, ?> vec = relation.get(id); @@ -395,7 +402,7 @@ public class GreedyEnsembleExperiment extends AbstractApplication { scores[d] = new DoubleIntPair(vec.doubleValue(d + 1), d); } Arrays.sort(scores, Collections.reverseOrder(DoubleIntPair.BYFIRST_COMPARATOR)); - return ROC.computeAUC(ROC.materializeROC(dim, positive, Arrays.asList(scores).iterator())); + return XYCurve.areaUnderCurve(ROC.materializeROC(dim, positive, Arrays.asList(scores).iterator())); } double gain(double score, double ref, double optimal) { diff --git a/src/de/lmu/ifi/dbs/elki/application/greedyensemble/VisualizePairwiseGainMatrix.java b/src/de/lmu/ifi/dbs/elki/application/greedyensemble/VisualizePairwiseGainMatrix.java index 2c728878..105eeabc 100644 --- a/src/de/lmu/ifi/dbs/elki/application/greedyensemble/VisualizePairwiseGainMatrix.java +++ b/src/de/lmu/ifi/dbs/elki/application/greedyensemble/VisualizePairwiseGainMatrix.java @@ -45,6 +45,7 @@ import de.lmu.ifi.dbs.elki.evaluation.similaritymatrix.ComputeSimilarityMatrixIm import de.lmu.ifi.dbs.elki.evaluation.similaritymatrix.ComputeSimilarityMatrixImage.SimilarityMatrix; import de.lmu.ifi.dbs.elki.logging.Logging; import de.lmu.ifi.dbs.elki.math.DoubleMinMax; +import de.lmu.ifi.dbs.elki.math.geometry.XYCurve; import de.lmu.ifi.dbs.elki.result.ResultUtil; import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; @@ -115,7 +116,7 @@ public class VisualizePairwiseGainMatrix extends AbstractApplication { final Database database = inputstep.getDatabase(); final Relation<NumberVector<?, ?>> relation = database.getRelation(TypeUtil.NUMBER_VECTOR_FIELD); final Relation<String> labels = DatabaseUtil.guessLabelRepresentation(database); - final DBID firstid = labels.iterDBIDs().next(); + final DBID firstid = labels.iterDBIDs().getDBID(); final String firstlabel = labels.get(firstid); if(!firstlabel.matches("bylabel")) { throw new AbortException("No 'by label' reference outlier found, which is needed for weighting!"); @@ -153,7 +154,7 @@ public class VisualizePairwiseGainMatrix extends AbstractApplication { combined[d].second = d; } Arrays.sort(combined, Collections.reverseOrder(DoubleIntPair.BYFIRST_COMPARATOR)); - double auc = ROC.computeAUC(ROC.materializeROC(dim, pos, Arrays.asList(combined).iterator())); + double auc = XYCurve.areaUnderCurve(ROC.materializeROC(dim, pos, Arrays.asList(combined).iterator())); data[a][a] = auc; // minmax.put(auc); // logger.verbose(auc + " " + labels.get(ids.get(a))); @@ -166,7 +167,7 @@ public class VisualizePairwiseGainMatrix extends AbstractApplication { combined[d].second = d; } Arrays.sort(combined, Collections.reverseOrder(DoubleIntPair.BYFIRST_COMPARATOR)); - double auc = ROC.computeAUC(ROC.materializeROC(dim, pos, Arrays.asList(combined).iterator())); + double auc = XYCurve.areaUnderCurve(ROC.materializeROC(dim, pos, Arrays.asList(combined).iterator())); // logger.verbose(auc + " " + labels.get(ids.get(a)) + " " + // labels.get(ids.get(b))); data[a][b] = auc; diff --git a/src/de/lmu/ifi/dbs/elki/application/internal/CheckELKIServices.java b/src/de/lmu/ifi/dbs/elki/application/internal/CheckELKIServices.java index 21ed047b..21b8c698 100644 --- a/src/de/lmu/ifi/dbs/elki/application/internal/CheckELKIServices.java +++ b/src/de/lmu/ifi/dbs/elki/application/internal/CheckELKIServices.java @@ -60,7 +60,7 @@ public class CheckELKIServices { /** * Pattern to strip comments, while keeping commented class names. */ - private Pattern strip = Pattern.compile("^[\\s#]*(.*?)[\\s]*$"); + private Pattern strip = Pattern.compile("^[\\s#]*(?:deprecated:\\s*)?(.*?)[\\s]*$"); /** * Package to skip matches in - unreleased code. diff --git a/src/de/lmu/ifi/dbs/elki/application/jsmap/JSONWebServer.java b/src/de/lmu/ifi/dbs/elki/application/jsmap/JSONWebServer.java index 5b210795..2f51c743 100644 --- a/src/de/lmu/ifi/dbs/elki/application/jsmap/JSONWebServer.java +++ b/src/de/lmu/ifi/dbs/elki/application/jsmap/JSONWebServer.java @@ -41,6 +41,7 @@ import de.lmu.ifi.dbs.elki.data.spatial.Polygon; import de.lmu.ifi.dbs.elki.data.spatial.PolygonsObject; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.relation.Relation; @@ -274,8 +275,8 @@ public class JSONWebServer implements HttpHandler { DBIDs neighbors = pred.getNeighborDBIDs(id); re.appendKeyValue("DBID", id); re.appendKeyArray("neighbors"); - for(DBID nid : neighbors) { - re.appendString(nid.toString()); + for (DBIDIter iter = neighbors.iter(); iter.valid(); iter.advance()) { + re.appendString(iter.toString()); } re.closeArray(); return; @@ -316,12 +317,12 @@ public class JSONWebServer implements HttpHandler { re.appendKeyArray("scores"); Relation<Double> scores = or.getScores(); - Iterator<DBID> iter = or.getOrdering().iter(scores.getDBIDs()).iterator(); - for(int i = 0; i < offset && iter.hasNext(); i++) { - iter.next(); + DBIDIter iter = or.getOrdering().iter(scores.getDBIDs()).iter(); + for(int i = 0; i < offset && iter.valid(); i++) { + iter.advance(); } - for(int i = 0; i < pagesize && iter.hasNext(); i++) { - DBID id = iter.next(); + for(int i = 0; i < pagesize && iter.valid(); i++, iter.advance()) { + DBID id = iter.getDBID(); re.startHash(); bundleToJSON(re, id); final Double val = scores.get(id); diff --git a/src/de/lmu/ifi/dbs/elki/application/visualization/KNNExplorer.java b/src/de/lmu/ifi/dbs/elki/application/visualization/KNNExplorer.java index 0419f791..c7f6d266 100644 --- a/src/de/lmu/ifi/dbs/elki/application/visualization/KNNExplorer.java +++ b/src/de/lmu/ifi/dbs/elki/application/visualization/KNNExplorer.java @@ -57,6 +57,7 @@ import de.lmu.ifi.dbs.elki.data.VectorUtil; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.StaticArrayDatabase; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery; @@ -375,7 +376,8 @@ public class KNNExplorer<O extends NumberVector<?, ?>, D extends NumberDistance< double min = Double.MAX_VALUE; double max = Double.MIN_VALUE; - for(DBID objID : data.iterDBIDs()) { + for(DBIDIter iditer = data.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID objID = iditer.getDBID(); O vec = data.get(objID); DoubleMinMax mm = VectorUtil.getRangeDouble(vec); min = Math.min(min, mm.getMin()); @@ -408,7 +410,8 @@ public class KNNExplorer<O extends NumberVector<?, ?>, D extends NumberDistance< svgCanvas.setPlot(plot); DefaultListModel m = new DefaultListModel(); - for(DBID dbid : data.iterDBIDs()) { + for(DBIDIter iditer = data.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID dbid = iditer.getDBID(); m.addElement(dbid); } seriesList.setModel(m); @@ -449,7 +452,7 @@ public class KNNExplorer<O extends NumberVector<?, ?>, D extends NumberDistance< double dist = pair.getDistance().doubleValue() / maxdist; Color color = getColor(dist); String colstr = "#" + Integer.toHexString(color.getRGB()).substring(2); - String width = (pair.getDBID().equals(idx)) ? "0.5%" : "0.2%"; + String width = (pair.getDBID().sameDBID(idx)) ? "0.5%" : "0.2%"; SVGUtil.setStyle(line, "stroke: " + colstr + "; stroke-width: " + width + "; fill: none"); newe.appendChild(line); // put into cache diff --git a/src/de/lmu/ifi/dbs/elki/data/Cluster.java b/src/de/lmu/ifi/dbs/elki/data/Cluster.java index 6863e995..88f610ea 100644 --- a/src/de/lmu/ifi/dbs/elki/data/Cluster.java +++ b/src/de/lmu/ifi/dbs/elki/data/Cluster.java @@ -26,6 +26,7 @@ package de.lmu.ifi.dbs.elki.data; import java.util.ArrayList; import java.util.Comparator; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Set; @@ -37,12 +38,11 @@ import de.lmu.ifi.dbs.elki.utilities.datastructures.hierarchy.Hierarchical; import de.lmu.ifi.dbs.elki.utilities.datastructures.hierarchy.Hierarchy; import de.lmu.ifi.dbs.elki.utilities.datastructures.hierarchy.HierarchyReferenceLists; import de.lmu.ifi.dbs.elki.utilities.iterator.EmptyIterator; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; /** * Generic cluster class, that may or not have hierarchical information. Note - * that every cluster MUST have a DBIDs, since it implements the - * interface, too. Calls to the interface are proxied to the inner group object. + * that every cluster MUST have a DBIDs, since it implements the interface, too. + * Calls to the interface are proxied to the inner group object. * * A hierarchy object of class SimpleHierarchy will be created automatically * when a list of parents and children is provided. Alternatively, a @@ -271,7 +271,7 @@ public class Cluster<M extends Model> implements Hierarchical<Cluster<M>>, TextW * Delegate to hierarchy object */ @Override - public IterableIterator<Cluster<M>> iterDescendants() { + public Iterator<Cluster<M>> iterDescendants() { if(hierarchy == null) { return EmptyIterator.STATIC(); } @@ -286,8 +286,8 @@ public class Cluster<M extends Model> implements Hierarchical<Cluster<M>>, TextW public Set<Cluster<M>> getDescendants() { HashSet<Cluster<M>> set = new HashSet<Cluster<M>>(); // add all - for (Cluster<M> c : iterDescendants()) { - set.add(c); + for(Iterator<Cluster<M>> iter = iterDescendants(); iter.hasNext();) { + set.add(iter.next()); } return set; } @@ -318,7 +318,7 @@ public class Cluster<M extends Model> implements Hierarchical<Cluster<M>>, TextW * Delegate to hierarchy object */ @Override - public IterableIterator<Cluster<M>> iterAncestors() { + public Iterator<Cluster<M>> iterAncestors() { if(hierarchy == null) { return EmptyIterator.STATIC(); } @@ -520,6 +520,6 @@ public class Cluster<M extends Model> implements Hierarchical<Cluster<M>>, TextW public String toString() { String mstr = (model == null) ? "null" : model.toString(); String nstr = noise ? ",noise" : ""; - return "Cluster(size="+size()+",model="+mstr+nstr+")"; + return "Cluster(size=" + size() + ",model=" + mstr + nstr + ")"; } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/data/Clustering.java b/src/de/lmu/ifi/dbs/elki/data/Clustering.java index eaac4418..f4312c4c 100644 --- a/src/de/lmu/ifi/dbs/elki/data/Clustering.java +++ b/src/de/lmu/ifi/dbs/elki/data/Clustering.java @@ -26,6 +26,7 @@ package de.lmu.ifi.dbs.elki.data; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Set; @@ -105,8 +106,8 @@ public class Clustering<M extends Model> extends BasicResult { for(Cluster<M> rc : toplevelclusters) { if(!clu.contains(rc)) { clu.add(rc); - for (Cluster<M> c : rc.iterDescendants()) { - clu.add(c); + for (Iterator<Cluster<M>> iter = rc.iterDescendants(); iter.hasNext(); ) { + clu.add(iter.next()); } } } diff --git a/src/de/lmu/ifi/dbs/elki/data/NumberVector.java b/src/de/lmu/ifi/dbs/elki/data/NumberVector.java index 1964277c..dcb869f8 100644 --- a/src/de/lmu/ifi/dbs/elki/data/NumberVector.java +++ b/src/de/lmu/ifi/dbs/elki/data/NumberVector.java @@ -38,7 +38,6 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.Parameterizable; * @param <N> the type of the attribute values * * @apiviz.landmark - * @apiviz.has Matrix * @apiviz.has Vector */ public interface NumberVector<V extends NumberVector<? extends V, N>, N extends Number> extends FeatureVector<V, N>, SpatialComparable, Parameterizable { diff --git a/src/de/lmu/ifi/dbs/elki/data/SparseDoubleVector.java b/src/de/lmu/ifi/dbs/elki/data/SparseDoubleVector.java new file mode 100644 index 00000000..10536058 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/data/SparseDoubleVector.java @@ -0,0 +1,341 @@ +package de.lmu.ifi.dbs.elki.data; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import gnu.trove.impl.unmodifiable.TUnmodifiableIntDoubleMap; +import gnu.trove.iterator.TIntDoubleIterator; +import gnu.trove.map.TIntDoubleMap; +import gnu.trove.map.hash.TIntDoubleHashMap; + +import java.util.Arrays; +import java.util.BitSet; + +import de.lmu.ifi.dbs.elki.datasource.parser.SparseNumberVectorLabelParser; +import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; +import de.lmu.ifi.dbs.elki.utilities.datastructures.arraylike.ArrayAdapter; +import de.lmu.ifi.dbs.elki.utilities.datastructures.arraylike.NumberArrayAdapter; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; + +/** + * <p> + * A SparseDoubleVector is to store real values as double values. + * </p> + * + * A SparseDoubleVector only requires storage for those attribute values that are + * non-zero. + * + * @author Arthur Zimek + */ +// TODO: implement ByteArraySerializer<SparseDoubleVector> +public class SparseDoubleVector extends AbstractNumberVector<SparseDoubleVector, Double> implements SparseNumberVector<SparseDoubleVector, Double> { + /** + * Static instance + */ + public static final SparseDoubleVector STATIC = new SparseDoubleVector(new int[0], new double[0], -1); + + /** + * Indexes of values + */ + private int[] indexes; + + /** + * Stored values + */ + private double[] values; + + /** + * The dimensionality of this feature vector. + */ + private int dimensionality; + + /** + * Direct constructor. + * + * @param indexes Indexes Must be sorted! + * @param values Associated value. + * @param dimensionality "true" dimensionality + */ + public SparseDoubleVector(int[] indexes, double[] values, int dimensionality) { + super(); + this.indexes = indexes; + this.values = values; + this.dimensionality = dimensionality; + } + + /** + * Provides a SparseDoubleVector consisting of double values according to the + * specified mapping of indices and values. + * + * @param values the values to be set as values of the real vector + * @param dimensionality the dimensionality of this feature vector + * @throws IllegalArgumentException if the given dimensionality is too small + * to cover the given values (i.e., the maximum index of any value not + * zero is bigger than the given dimensionality) + */ + public SparseDoubleVector(TIntDoubleMap values, int dimensionality) throws IllegalArgumentException { + if(values.size() > dimensionality) { + throw new IllegalArgumentException("values.size() > dimensionality!"); + } + + this.indexes = new int[values.size()]; + this.values = new double[values.size()]; + // Import and sort the indexes + { + TIntDoubleIterator iter = values.iterator(); + for (int i = 0; iter.hasNext(); i++) { + iter.advance(); + this.indexes[i] = iter.key(); + } + Arrays.sort(this.indexes); + } + // Import the values accordingly + { + for(int i = 0; i < values.size(); i++) { + this.values[i] = values.get(this.indexes[i]); + } + } + this.dimensionality = dimensionality; + final int maxdim = getMaxDim(); + if(maxdim > dimensionality) { + throw new IllegalArgumentException("Given dimensionality " + dimensionality + " is too small w.r.t. the given values (occurring maximum: " + maxdim + ")."); + } + } + + /** + * Get the maximum dimensionality. + * + * @return the maximum dimensionality seen + */ + private int getMaxDim() { + if(this.indexes.length == 0) { + return 0; + } + else { + return this.indexes[this.indexes.length - 1]; + } + } + + /** + * Provides a SparseDoubleVector consisting of double values according to the + * specified mapping of indices and values. + * + * @param values the values to be set as values of the real vector + * @throws IllegalArgumentException if the given dimensionality is too small + * to cover the given values (i.e., the maximum index of any value not + * zero is bigger than the given dimensionality) + */ + public SparseDoubleVector(double[] values) throws IllegalArgumentException { + this.dimensionality = values.length; + + // Count the number of non-zero entries + int size = 0; + { + for(int i = 0; i < values.length; i++) { + if(values[i] != 0.0f) { + size++; + } + } + } + this.indexes = new int[size]; + this.values = new double[size]; + + // Copy the values + { + int pos = 0; + for(int i = 0; i < values.length; i++) { + double value = values[i]; + if(value != 0.0f) { + this.indexes[pos] = i + 1; + this.values[pos] = value; + pos++; + } + } + } + } + + @Override + public int getDimensionality() { + return dimensionality; + } + + /** + * Sets the dimensionality to the new value. + * + * + * @param dimensionality the new dimensionality + * @throws IllegalArgumentException if the given dimensionality is too small + * to cover the given values (i.e., the maximum index of any value not + * zero is bigger than the given dimensionality) + */ + @Override + public void setDimensionality(int dimensionality) throws IllegalArgumentException { + final int maxdim = getMaxDim(); + if(maxdim > dimensionality) { + throw new IllegalArgumentException("Given dimensionality " + dimensionality + " is too small w.r.t. the given values (occurring maximum: " + maxdim + ")."); + } + this.dimensionality = dimensionality; + } + + @Override + public Double getValue(int dimension) { + int pos = Arrays.binarySearch(this.indexes, dimension); + if(pos >= 0) { + return values[pos]; + } + else { + return 0.0; + } + } + + @Override + public double doubleValue(int dimension) { + int pos = Arrays.binarySearch(this.indexes, dimension); + if(pos >= 0) { + return values[pos]; + } + else { + return 0.0; + } + } + + @Override + public long longValue(int dimension) { + int pos = Arrays.binarySearch(this.indexes, dimension); + if(pos >= 0) { + return (long) values[pos]; + } + else { + return 0; + } + } + + @Override + public Vector getColumnVector() { + double[] values = getValues(); + return new Vector(values); + } + + /** + * <p> + * Provides a String representation of this SparseDoubleVector as suitable for + * {@link SparseNumberVectorLabelParser}. + * </p> + * + * <p> + * The returned String is a single line with entries separated by + * {@link AbstractNumberVector#ATTRIBUTE_SEPARATOR}. The first entry gives the + * number of values actually not zero. Following entries are pairs of Integer + * and Double where the Integer gives the index of the dimensionality and the + * Double gives the corresponding value. + * </p> + * + * <p> + * Example: a vector (0,1.2,1.3,0)<sup>T</sup> would result in the String<br> + * <code>2 2 1.2 3 1.3</code><br> + * </p> + * + * @return a String representation of this SparseDoubleVector + */ + @Override + public String toString() { + StringBuilder featureLine = new StringBuilder(); + featureLine.append(this.indexes.length); + for(int i = 0; i < this.indexes.length; i++) { + featureLine.append(ATTRIBUTE_SEPARATOR); + featureLine.append(this.indexes[i]); + featureLine.append(ATTRIBUTE_SEPARATOR); + featureLine.append(this.values[i]); + } + + return featureLine.toString(); + } + + /** + * Returns an array consisting of the values of this feature vector. + * + * @return an array consisting of the values of this feature vector + */ + private double[] getValues() { + double[] values = new double[dimensionality]; + for(int i = 0; i < indexes.length; i++) { + values[this.indexes[i]] = this.values[i]; + } + return values; + } + + @Override + public <A> SparseDoubleVector newFeatureVector(A array, ArrayAdapter<Double, A> adapter) { + int dim = adapter.size(array); + double[] values = new double[dim]; + for(int i = 0; i < dim; i++) { + values[i] = adapter.get(array, i); + } + // TODO: inefficient + return new SparseDoubleVector(values); + } + + @Override + public <A> SparseDoubleVector newNumberVector(A array, NumberArrayAdapter<?, A> adapter) { + int dim = adapter.size(array); + double[] values = new double[dim]; + for(int i = 0; i < dim; i++) { + values[i] = adapter.getDouble(array, i); + } + // TODO: inefficient + return new SparseDoubleVector(values); + } + + @Override + public SparseDoubleVector newNumberVector(TIntDoubleMap values, int maxdim) { + return new SparseDoubleVector(values, maxdim); + } + + @Override + public BitSet getNotNullMask() { + BitSet b = new BitSet(); + for(int key : indexes) { + b.set(key); + } + return b; + } + + /** + * Parameterization class + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer extends AbstractParameterizer { + @Override + protected SparseDoubleVector makeInstance() { + return STATIC; + } + } + + /** + * Empty map. + */ + public static final TIntDoubleMap EMPTYMAP = new TUnmodifiableIntDoubleMap(new TIntDoubleHashMap()); +} diff --git a/src/de/lmu/ifi/dbs/elki/data/SparseFloatVector.java b/src/de/lmu/ifi/dbs/elki/data/SparseFloatVector.java index 1ce8c5f7..36a4e171 100644 --- a/src/de/lmu/ifi/dbs/elki/data/SparseFloatVector.java +++ b/src/de/lmu/ifi/dbs/elki/data/SparseFloatVector.java @@ -24,14 +24,16 @@ package de.lmu.ifi.dbs.elki.data; */ import gnu.trove.impl.unmodifiable.TUnmodifiableIntFloatMap; +import gnu.trove.iterator.TIntDoubleIterator; import gnu.trove.iterator.TIntFloatIterator; +import gnu.trove.map.TIntDoubleMap; import gnu.trove.map.TIntFloatMap; import gnu.trove.map.hash.TIntFloatHashMap; import java.util.Arrays; import java.util.BitSet; -import de.lmu.ifi.dbs.elki.datasource.parser.SparseFloatVectorLabelParser; +import de.lmu.ifi.dbs.elki.datasource.parser.SparseNumberVectorLabelParser; import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; import de.lmu.ifi.dbs.elki.utilities.datastructures.arraylike.ArrayAdapter; import de.lmu.ifi.dbs.elki.utilities.datastructures.arraylike.NumberArrayAdapter; @@ -103,9 +105,9 @@ public class SparseFloatVector extends AbstractNumberVector<SparseFloatVector, F // Import and sort the indexes { TIntFloatIterator iter = values.iterator(); - for (int i = 0; iter.hasNext(); i++) { - iter.advance(); + for(int i = 0; iter.hasNext(); i++) { this.indexes[i] = iter.key(); + iter.advance(); } Arrays.sort(this.indexes); } @@ -188,6 +190,7 @@ public class SparseFloatVector extends AbstractNumberVector<SparseFloatVector, F * to cover the given values (i.e., the maximum index of any value not * zero is bigger than the given dimensionality) */ + @Override public void setDimensionality(int dimensionality) throws IllegalArgumentException { final int maxdim = getMaxDim(); if(maxdim > dimensionality) { @@ -238,7 +241,7 @@ public class SparseFloatVector extends AbstractNumberVector<SparseFloatVector, F /** * <p> * Provides a String representation of this SparseFloatVector as suitable for - * {@link SparseFloatVectorLabelParser}. + * {@link SparseNumberVectorLabelParser}. * </p> * * <p> @@ -306,6 +309,24 @@ public class SparseFloatVector extends AbstractNumberVector<SparseFloatVector, F } @Override + public SparseFloatVector newNumberVector(TIntDoubleMap dvalues, int maxdim) { + int[] indexes = new int[dvalues.size()]; + float[] values = new float[dvalues.size()]; + // Import and sort the indexes + TIntDoubleIterator iter = dvalues.iterator(); + for(int i = 0; iter.hasNext(); i++) { + iter.advance(); + indexes[i] = iter.key(); + } + Arrays.sort(indexes); + // Import the values accordingly + for(int i = 0; i < dvalues.size(); i++) { + values[i] = (float) dvalues.get(indexes[i]); + } + return new SparseFloatVector(indexes, values, maxdim); + } + + @Override public BitSet getNotNullMask() { BitSet b = new BitSet(); for(int key : indexes) { @@ -327,9 +348,9 @@ public class SparseFloatVector extends AbstractNumberVector<SparseFloatVector, F return STATIC; } } - + /** * Empty map. */ - public static final TIntFloatMap EMPTYMAP = new TUnmodifiableIntFloatMap(new TIntFloatHashMap()); -} + public static final TIntFloatMap EMPTYMAP = new TUnmodifiableIntFloatMap(new TIntFloatHashMap()); +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/data/SparseNumberVector.java b/src/de/lmu/ifi/dbs/elki/data/SparseNumberVector.java index d337d08b..b1f1c095 100644 --- a/src/de/lmu/ifi/dbs/elki/data/SparseNumberVector.java +++ b/src/de/lmu/ifi/dbs/elki/data/SparseNumberVector.java @@ -23,16 +23,30 @@ package de.lmu.ifi.dbs.elki.data; along with this program. If not, see <http://www.gnu.org/licenses/>. */ - +import gnu.trove.map.TIntDoubleMap; /** * Combines the SparseFeatureVector and NumberVector * * @author Erich Schubert - * - * @param <V> - * @param <N> + * + * @param <V> Vector type number type + * @param <N> Number type */ public interface SparseNumberVector<V extends SparseNumberVector<V, N>, N extends Number> extends NumberVector<V, N>, SparseFeatureVector<V, N> { - // Empty combination interface + /** + * Returns a new NumberVector of N for the given values. + * + * @param values the values of the NumberVector + * @param maxdim Maximum dimensionality. + * @return a new NumberVector of N for the given values + */ + V newNumberVector(TIntDoubleMap values, int maxdim); + + /** + * Update the vector space dimensionality. + * + * @param maxdim New dimensionality + */ + void setDimensionality(int maxdim); } diff --git a/src/de/lmu/ifi/dbs/elki/data/VectorUtil.java b/src/de/lmu/ifi/dbs/elki/data/VectorUtil.java index 1607b482..c8bf2c02 100644 --- a/src/de/lmu/ifi/dbs/elki/data/VectorUtil.java +++ b/src/de/lmu/ifi/dbs/elki/data/VectorUtil.java @@ -24,12 +24,20 @@ package de.lmu.ifi.dbs.elki.data; */ import java.util.BitSet; +import java.util.Comparator; import java.util.Random; import de.lmu.ifi.dbs.elki.data.spatial.SpatialComparable; +import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; +import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; +import de.lmu.ifi.dbs.elki.database.ids.DBIDs; +import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.math.DoubleMinMax; import de.lmu.ifi.dbs.elki.math.MathUtil; import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; +import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; +import de.lmu.ifi.dbs.elki.utilities.datastructures.QuickSelect; /** * Utility functions for use with vectors. @@ -268,4 +276,115 @@ public final class VectorUtil { } return result; } + + /** + * Compute medoid for a given subset. + * + * @param relation Relation to process + * @param sample Sample set + * @return Medoid vector + */ + public static Vector computeMedoid(Relation<? extends NumberVector<?, ?>> relation, DBIDs sample) { + final int dim = DatabaseUtil.dimensionality(relation); + ArrayModifiableDBIDs mids = DBIDUtil.newArray(sample); + SortDBIDsBySingleDimension s = new SortDBIDsBySingleDimension(relation); + Vector medoid = new Vector(dim); + for (int d = 0; d < dim; d++) { + s.setDimension(d + 1); + medoid.set(d, relation.get(QuickSelect.median(mids, s)).doubleValue(d + 1)); + } + return medoid; + } + + /** + * Compare number vectors by a single dimension + * + * @author Erich Schubert + */ + public static class SortDBIDsBySingleDimension implements Comparator<DBID> { + /** + * Dimension to sort with + */ + public int d; + + /** + * The relation to sort. + */ + private Relation<? extends NumberVector<?, ?>> data; + + /** + * Constructor. + * + * @param data Vector data source + */ + public SortDBIDsBySingleDimension(Relation<? extends NumberVector<?, ?>> data) { + super(); + this.data = data; + }; + + /** + * Get the dimension to sort by + * + * @return Dimension to sort with + */ + public int getDimension() { + return this.d; + } + + /** + * Set the dimension to sort by + * + * @param d Dimension to sort with + */ + public void setDimension(int d) { + this.d = d; + } + + @Override + public int compare(DBID id1, DBID id2) { + return Double.compare(data.get(id1).doubleValue(d), data.get(id2).doubleValue(d)); + } + } + + /** + * Compare number vectors by a single dimension + * + * @author Erich Schubert + */ + public static class SortVectorsBySingleDimension implements Comparator<NumberVector<?, ?>> { + /** + * Dimension to sort with + */ + public int d; + + /** + * Constructor. + */ + public SortVectorsBySingleDimension() { + super(); + }; + + /** + * Get the dimension to sort by + * + * @return Dimension to sort with + */ + public int getDimension() { + return this.d; + } + + /** + * Set the dimension to sort by + * + * @param d Dimension to sort with + */ + public void setDimension(int d) { + this.d = d; + } + + @Override + public int compare(NumberVector<?, ?> o1, NumberVector<?, ?> o2) { + return Double.compare(o1.doubleValue(d), o2.doubleValue(d)); + } + } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/data/model/CorrelationAnalysisSolution.java b/src/de/lmu/ifi/dbs/elki/data/model/CorrelationAnalysisSolution.java index e6cb893f..56fe8f9d 100644 --- a/src/de/lmu/ifi/dbs/elki/data/model/CorrelationAnalysisSolution.java +++ b/src/de/lmu/ifi/dbs/elki/data/model/CorrelationAnalysisSolution.java @@ -27,15 +27,15 @@ import java.text.NumberFormat; import java.util.Locale; import de.lmu.ifi.dbs.elki.data.NumberVector; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.relation.Relation; +import de.lmu.ifi.dbs.elki.datasource.filter.normalization.NonNumericFeaturesException; +import de.lmu.ifi.dbs.elki.datasource.filter.normalization.Normalization; import de.lmu.ifi.dbs.elki.logging.LoggingUtil; import de.lmu.ifi.dbs.elki.math.linearalgebra.LinearEquationSystem; import de.lmu.ifi.dbs.elki.math.linearalgebra.Matrix; import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; -import de.lmu.ifi.dbs.elki.datasource.filter.normalization.NonNumericFeaturesException; -import de.lmu.ifi.dbs.elki.datasource.filter.normalization.Normalization; import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.textwriter.TextWriteable; import de.lmu.ifi.dbs.elki.result.textwriter.TextWriterStream; @@ -141,8 +141,8 @@ public class CorrelationAnalysisSolution<V extends NumberVector<V, ?>> implement // determine standard deviation double variance = 0; DBIDs ids = db.getDBIDs(); - for (DBID id : ids) { - double distance = distance(db.get(id).getColumnVector()); + for(DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + double distance = distance(db.get(iter).getColumnVector()); variance += distance * distance; } standardDeviation = Math.sqrt(variance / ids.size()); @@ -213,7 +213,7 @@ public class CorrelationAnalysisSolution<V extends NumberVector<V, ?>> implement * @return the error vectors */ public Vector errorVector(V p) { - return p.getColumnVector().minus(centroid).projection(weakEigenvectors); + return p.getColumnVector().minusEquals(centroid).projection(weakEigenvectors); } /** @@ -223,7 +223,7 @@ public class CorrelationAnalysisSolution<V extends NumberVector<V, ?>> implement * @return the data projections */ public Matrix dataProjections(V p) { - Vector centered = p.getColumnVector().minus(centroid); + Vector centered = p.getColumnVector().minusEquals(centroid); Matrix sum = new Matrix(p.getDimensionality(), strongEigenvectors.getColumnDimensionality()); for(int i = 0; i < strongEigenvectors.getColumnDimensionality(); i++) { Vector v_i = strongEigenvectors.getCol(i); @@ -240,7 +240,7 @@ public class CorrelationAnalysisSolution<V extends NumberVector<V, ?>> implement * @return the error vectors */ public Vector dataVector(V p) { - return p.getColumnVector().minus(centroid).projection(strongEigenvectors); + return p.getColumnVector().minusEquals(centroid).projection(strongEigenvectors); } /** @@ -254,21 +254,21 @@ public class CorrelationAnalysisSolution<V extends NumberVector<V, ?>> implement } /** - * Returns a copy of the strong eigenvectors. + * Returns the strong eigenvectors. * - * @return a copy of the strong eigenvectors + * @return the strong eigenvectors */ public Matrix getStrongEigenvectors() { - return strongEigenvectors.copy(); + return strongEigenvectors; } /** - * Returns a copy of the weak eigenvectors. + * Returns the weak eigenvectors. * - * @return a copy of the weak eigenvectors + * @return the weak eigenvectors */ public Matrix getWeakEigenvectors() { - return weakEigenvectors.copy(); + return weakEigenvectors; } /** diff --git a/src/de/lmu/ifi/dbs/elki/data/model/MedoidModel.java b/src/de/lmu/ifi/dbs/elki/data/model/MedoidModel.java new file mode 100644 index 00000000..5540b931 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/data/model/MedoidModel.java @@ -0,0 +1,76 @@ +package de.lmu.ifi.dbs.elki.data.model; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.result.textwriter.TextWriteable; +import de.lmu.ifi.dbs.elki.result.textwriter.TextWriterStream; + +/** + * Cluster model that stores a mean for the cluster. + * + * @author Erich Schubert + */ +public class MedoidModel extends BaseModel implements TextWriteable { + /** + * Cluster medoid + */ + private DBID medoid; + + /** + * Constructor with medoid + * + * @param medoid Cluster medoid + */ + public MedoidModel(DBID medoid) { + super(); + this.medoid = medoid; + } + + /** + * @return medoid + */ + public DBID getMedoid() { + return medoid; + } + + /** + * @param medoid Medoid object + */ + public void setMedoid(DBID medoid) { + this.medoid = medoid; + } + + /** + * Implementation of {@link TextWriteable} interface. + */ + @Override + public void writeToText(TextWriterStream out, String label) { + if(label != null) { + out.commentPrintLn(label); + } + out.commentPrintLn(TextWriterStream.SER_MARKER + " " + getClass().getName()); + out.commentPrintLn("Cluster Medoid: " + medoid.toString()); + } +} diff --git a/src/de/lmu/ifi/dbs/elki/data/projection/FeatureSelection.java b/src/de/lmu/ifi/dbs/elki/data/projection/FeatureSelection.java index fca75e3f..04878aea 100644 --- a/src/de/lmu/ifi/dbs/elki/data/projection/FeatureSelection.java +++ b/src/de/lmu/ifi/dbs/elki/data/projection/FeatureSelection.java @@ -39,7 +39,7 @@ import de.lmu.ifi.dbs.elki.utilities.datastructures.arraylike.SubsetArrayAdapter * @param <V> Vector type * @param <F> Feature type */ -public class FeatureSelection<V extends FeatureVector<V, F>, F> extends AbstractFeatureSelection<V, F> { +public class FeatureSelection<V extends FeatureVector<V, F>, F> implements Projection<V, V> { /** * Minimum dimensionality required for projection */ @@ -56,13 +56,18 @@ public class FeatureSelection<V extends FeatureVector<V, F>, F> extends Abstract private int dimensionality; /** + * Array adapter + */ + protected ArrayAdapter<F, V> adapter; + + /** * Constructor. * * @param dims Dimensions * @param factory Object factory */ public FeatureSelection(int[] dims, V factory) { - super(new SubsetArrayAdapter<F, V>(getAdapter(factory), dims)); + this.adapter = new SubsetArrayAdapter<F, V>(getAdapter(factory), dims); this.factory = factory; this.dimensionality = dims.length; @@ -73,6 +78,11 @@ public class FeatureSelection<V extends FeatureVector<V, F>, F> extends Abstract this.mindim = mindim; } + @Override + public V project(V data) { + return data.newFeatureVector(data, adapter); + } + /** * Choose the best adapter for this. * diff --git a/src/de/lmu/ifi/dbs/elki/data/projection/NumericalFeatureSelection.java b/src/de/lmu/ifi/dbs/elki/data/projection/NumericalFeatureSelection.java index 7dfd580f..bd41e1cf 100644 --- a/src/de/lmu/ifi/dbs/elki/data/projection/NumericalFeatureSelection.java +++ b/src/de/lmu/ifi/dbs/elki/data/projection/NumericalFeatureSelection.java @@ -1,4 +1,5 @@ package de.lmu.ifi.dbs.elki.data.projection; + /* This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures @@ -22,13 +23,12 @@ package de.lmu.ifi.dbs.elki.data.projection; along with this program. If not, see <http://www.gnu.org/licenses/>. */ +import java.util.BitSet; + import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.data.type.VectorTypeInformation; -import de.lmu.ifi.dbs.elki.utilities.datastructures.arraylike.ArrayLikeUtil; -import de.lmu.ifi.dbs.elki.utilities.datastructures.arraylike.NumberArrayAdapter; -import de.lmu.ifi.dbs.elki.utilities.datastructures.arraylike.SubsetArrayAdapter; /** * Projection class for number vectors. @@ -36,9 +36,8 @@ import de.lmu.ifi.dbs.elki.utilities.datastructures.arraylike.SubsetArrayAdapter * @author Erich Schubert * * @param <V> Vector type - * @param <N> Number type */ -public class NumericalFeatureSelection<V extends NumberVector<V, N>, N extends Number> extends AbstractFeatureSelection<V, N> { +public class NumericalFeatureSelection<V extends NumberVector<V, ?>> implements Projection<V, V> { /** * Minimum dimensionality required for projection */ @@ -55,37 +54,36 @@ public class NumericalFeatureSelection<V extends NumberVector<V, N>, N extends N private int dimensionality; /** + * Subspace + */ + private BitSet bits; + + /** * Constructor. * - * @param dims Dimensions + * @param bits Dimensions * @param factory Object factory */ - public NumericalFeatureSelection(int[] dims, V factory) { - super(new SubsetArrayAdapter<N, V>(getAdapter(factory), dims)); + public NumericalFeatureSelection(BitSet bits, V factory) { + super(); + this.bits = bits; this.factory = factory; - this.dimensionality = dims.length; + this.dimensionality = bits.cardinality(); int mindim = 0; - for(int dim : dims) { - mindim = Math.max(mindim, dim + 1); + for(int i = bits.nextSetBit(0); i >= 0; i = bits.nextSetBit(i + 1)) { + mindim = Math.max(mindim, i + 1); } this.mindim = mindim; } - /** - * Choose the best adapter for this. - * - * @param factory Object factory, for type inference - * @return Adapter - */ - private static <V extends NumberVector<V, N>, N extends Number> NumberArrayAdapter<N, ? super V> getAdapter(V factory) { - return ArrayLikeUtil.numberVectorAdapter(factory); - } - - @SuppressWarnings("unchecked") @Override public V project(V data) { - return factory.newNumberVector(data, (NumberArrayAdapter<N, ? super V>) adapter); + double[] dbl = new double[dimensionality]; + for(int i = bits.nextSetBit(0), j = 0; i >= 0; i = bits.nextSetBit(i + 1), j++) { + dbl[j] = data.doubleValue(i + 1); + } + return factory.newNumberVector(dbl); } @Override diff --git a/src/de/lmu/ifi/dbs/elki/data/synthetic/bymodel/GeneratorMain.java b/src/de/lmu/ifi/dbs/elki/data/synthetic/bymodel/GeneratorMain.java index 6870fcaa..4d93e1ea 100644 --- a/src/de/lmu/ifi/dbs/elki/data/synthetic/bymodel/GeneratorMain.java +++ b/src/de/lmu/ifi/dbs/elki/data/synthetic/bymodel/GeneratorMain.java @@ -70,7 +70,12 @@ public class GeneratorMain { /** * List of clusters to generate */ - private LinkedList<GeneratorInterface> generators = new LinkedList<GeneratorInterface>(); + protected LinkedList<GeneratorInterface> generators = new LinkedList<GeneratorInterface>(); + + /** + * Controls whether points are tested against the model during generation + */ + protected boolean testAgainstModel = true; /** * Add a cluster to the cluster list. @@ -82,11 +87,6 @@ public class GeneratorMain { } /** - * Controls whether points are tested against the model during generation - */ - private boolean testAgainstModel = true; - - /** * Main loop to generate data set. * * @throws UnableToComplyException when model not satisfiable or no clusters @@ -154,6 +154,13 @@ public class GeneratorMain { cursclus.incrementDiscarded(); } } + } else { + // Keep all. + for (Vector p : newp) { + DoubleVector dv = new DoubleVector(p); + bundle.appendSimple(dv, l, model); + ++kept; + } } } } diff --git a/src/de/lmu/ifi/dbs/elki/data/synthetic/bymodel/GeneratorSingleCluster.java b/src/de/lmu/ifi/dbs/elki/data/synthetic/bymodel/GeneratorSingleCluster.java index f9818916..d2970de7 100644 --- a/src/de/lmu/ifi/dbs/elki/data/synthetic/bymodel/GeneratorSingleCluster.java +++ b/src/de/lmu/ifi/dbs/elki/data/synthetic/bymodel/GeneratorSingleCluster.java @@ -30,7 +30,7 @@ import java.util.Random; import de.lmu.ifi.dbs.elki.data.model.Model; import de.lmu.ifi.dbs.elki.math.linearalgebra.AffineTransformation; import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; -import de.lmu.ifi.dbs.elki.math.statistics.distribution.Distribution; +import de.lmu.ifi.dbs.elki.math.statistics.distribution.DistributionWithRandom; import de.lmu.ifi.dbs.elki.utilities.exceptions.UnableToComplyException; /** @@ -40,14 +40,14 @@ import de.lmu.ifi.dbs.elki.utilities.exceptions.UnableToComplyException; * * @author Erich Schubert * - * @apiviz.composedOf Distribution + * @apiviz.composedOf DistributionWithRandom * @apiviz.composedOf AffineTransformation */ public class GeneratorSingleCluster implements GeneratorInterfaceDynamic, Model { /** * The distribution generators for each axis */ - private List<Distribution> axes = new ArrayList<Distribution>(); + private List<DistributionWithRandom> axes = new ArrayList<DistributionWithRandom>(); /** * The transformation matrix @@ -121,7 +121,7 @@ public class GeneratorSingleCluster implements GeneratorInterfaceDynamic, Model * @throws UnableToComplyException thrown when no new generators may be added * anymore */ - public void addGenerator(Distribution gen) throws UnableToComplyException { + public void addGenerator(DistributionWithRandom gen) throws UnableToComplyException { if(trans != null) { throw new UnableToComplyException("Generators may no longer be added when transformations have been applied."); } @@ -235,7 +235,7 @@ public class GeneratorSingleCluster implements GeneratorInterfaceDynamic, Model while(result.size() < count) { double[] d = new double[dim]; int i = 0; - for(Distribution axis : axes) { + for(DistributionWithRandom axis : axes) { d[i] = axis.nextRandom(); i++; } @@ -269,7 +269,7 @@ public class GeneratorSingleCluster implements GeneratorInterfaceDynamic, Model double density = 1.0; int i = 0; - for(Distribution axis : axes) { + for(DistributionWithRandom axis : axes) { density = density * axis.pdf(o.get(i)); i++; } @@ -389,6 +389,7 @@ public class GeneratorSingleCluster implements GeneratorInterfaceDynamic, Model * * @return Model */ + @Override public Model makeModel() { return this; } @@ -399,7 +400,7 @@ public class GeneratorSingleCluster implements GeneratorInterfaceDynamic, Model * @param i Generator axis i * @return Distribution */ - public Distribution getDistribution(int i) { + public DistributionWithRandom getDistribution(int i) { return axes.get(i); } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/data/type/NoSupportedDataTypeException.java b/src/de/lmu/ifi/dbs/elki/data/type/NoSupportedDataTypeException.java index 9865e96f..188f962c 100644 --- a/src/de/lmu/ifi/dbs/elki/data/type/NoSupportedDataTypeException.java +++ b/src/de/lmu/ifi/dbs/elki/data/type/NoSupportedDataTypeException.java @@ -23,6 +23,7 @@ package de.lmu.ifi.dbs.elki.data.type; along with this program. If not, see <http://www.gnu.org/licenses/>. */ +import java.util.Collection; /** * Exception thrown when no supported data type was found. @@ -38,18 +39,40 @@ public class NoSupportedDataTypeException extends IllegalStateException { private static final long serialVersionUID = 1L; /** + * Available types + */ + private Collection<TypeInformation> types = null; + + /** * Constructor. + * + * @param type Requested type + * @param types Available types. */ - public NoSupportedDataTypeException(TypeInformation type) { + public NoSupportedDataTypeException(TypeInformation type, Collection<TypeInformation> types) { super("No data type found satisfying: " + type.toString()); + this.types = types; } /** - * Constructor with string message. If possible, use the type parameter instead! - * + * Constructor with string message. If possible, use the type parameter + * instead! + * * @param string Error message */ public NoSupportedDataTypeException(String string) { super(string); } + + @Override + public String getMessage() { + StringBuffer buf = new StringBuffer(super.getMessage()); + if(types != null) { + buf.append("\nAvailable types:"); + for(TypeInformation type : types) { + buf.append(" ").append(type.toString()); + } + } + return buf.toString(); + } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/data/type/TypeUtil.java b/src/de/lmu/ifi/dbs/elki/data/type/TypeUtil.java index 91257906..4d62a453 100644 --- a/src/de/lmu/ifi/dbs/elki/data/type/TypeUtil.java +++ b/src/de/lmu/ifi/dbs/elki/data/type/TypeUtil.java @@ -27,13 +27,18 @@ import de.lmu.ifi.dbs.elki.data.BitVector; import de.lmu.ifi.dbs.elki.data.ClassLabel; import de.lmu.ifi.dbs.elki.data.DoubleVector; import de.lmu.ifi.dbs.elki.data.ExternalID; +import de.lmu.ifi.dbs.elki.data.FloatVector; import de.lmu.ifi.dbs.elki.data.LabelList; import de.lmu.ifi.dbs.elki.data.NumberVector; +import de.lmu.ifi.dbs.elki.data.SparseDoubleVector; import de.lmu.ifi.dbs.elki.data.SparseFloatVector; +import de.lmu.ifi.dbs.elki.data.SparseNumberVector; import de.lmu.ifi.dbs.elki.data.model.Model; import de.lmu.ifi.dbs.elki.data.spatial.PolygonsObject; import de.lmu.ifi.dbs.elki.database.ids.DBID; import de.lmu.ifi.dbs.elki.database.ids.DBIDFactory; +import de.lmu.ifi.dbs.elki.database.ids.DBIDs; +import de.lmu.ifi.dbs.elki.database.query.DistanceDBIDResult; import de.lmu.ifi.dbs.elki.math.linearalgebra.Matrix; import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; import de.lmu.ifi.dbs.elki.persistent.ByteArrayUtil; @@ -58,6 +63,11 @@ public final class TypeUtil { public static final SimpleTypeInformation<DBID> DBID = new SimpleTypeInformation<DBID>(DBID.class, DBIDFactory.FACTORY.getDBIDSerializer()); /** + * Database ID lists + */ + public static final SimpleTypeInformation<DBIDs> DBIDS = new SimpleTypeInformation<DBIDs>(DBIDs.class); + + /** * A string */ public static final SimpleTypeInformation<String> STRING = new SimpleTypeInformation<String>(String.class, ByteArrayUtil.STRING_SERIALIZER); @@ -73,6 +83,11 @@ public final class TypeUtil { public static final SimpleTypeInformation<LabelList> LABELLIST = new SimpleTypeInformation<LabelList>(LabelList.class); /** + * A list of neighbors + */ + public static final SimpleTypeInformation<DistanceDBIDResult<?>> NEIGHBORLIST = new SimpleTypeInformation<DistanceDBIDResult<?>>(DistanceDBIDResult.class); + + /** * Either class label, object labels or a string - anything that will be * accepted by * {@link de.lmu.ifi.dbs.elki.utilities.DatabaseUtil#guessObjectLabelRepresentation} @@ -98,15 +113,41 @@ public final class TypeUtil { /** * Input type for algorithms that require number vector fields. + * + * If possible, please use {@link #NUMBER_VECTOR_FIELD}! + */ + public static final VectorFieldTypeInformation<FloatVector> FLOAT_VECTOR_FIELD = new VectorFieldTypeInformation<FloatVector>(FloatVector.class, FloatVector.STATIC); + + /** + * Input type for algorithms that require number vector fields. */ public static final VectorFieldTypeInformation<BitVector> BIT_VECTOR_FIELD = new VectorFieldTypeInformation<BitVector>(BitVector.class); /** * Sparse float vector field. */ + public static final SimpleTypeInformation<SparseNumberVector<?, ?>> SPARSE_VECTOR_VARIABLE_LENGTH = new SimpleTypeInformation<SparseNumberVector<?, ?>>(SparseNumberVector.class); + + /** + * Sparse vector field. + */ + public static final VectorFieldTypeInformation<SparseNumberVector<?, ?>> SPARSE_VECTOR_FIELD = new VectorFieldTypeInformation<SparseNumberVector<?, ?>>(SparseNumberVector.class); + + /** + * Sparse float vector field. + * + * If possible, please use {@link #SPARSE_VECTOR_FIELD} instead! + */ public static final VectorFieldTypeInformation<SparseFloatVector> SPARSE_FLOAT_FIELD = new VectorFieldTypeInformation<SparseFloatVector>(SparseFloatVector.class); /** + * Sparse double vector field. + * + * If possible, please use {@link #SPARSE_VECTOR_FIELD} instead! + */ + public static final VectorFieldTypeInformation<SparseDoubleVector> SPARSE_DOUBLE_FIELD = new VectorFieldTypeInformation<SparseDoubleVector>(SparseDoubleVector.class); + + /** * External ID type */ public static final SimpleTypeInformation<ExternalID> EXTERNALID = new SimpleTypeInformation<ExternalID>(ExternalID.class); diff --git a/src/de/lmu/ifi/dbs/elki/data/type/VectorFieldTypeInformation.java b/src/de/lmu/ifi/dbs/elki/data/type/VectorFieldTypeInformation.java index 69e0c969..9e6c2470 100644 --- a/src/de/lmu/ifi/dbs/elki/data/type/VectorFieldTypeInformation.java +++ b/src/de/lmu/ifi/dbs/elki/data/type/VectorFieldTypeInformation.java @@ -234,12 +234,20 @@ public class VectorFieldTypeInformation<V extends FeatureVector<?, ?>> extends V @Override public String toString() { + StringBuffer buf = new StringBuffer(getRestrictionClass().getSimpleName()); if(mindim == maxdim) { - return getRestrictionClass().getSimpleName() + ",dim=" + mindim; + buf.append(",dim=").append(mindim); } else { - return super.toString(); + buf.append(",field"); + if(mindim >= 0) { + buf.append(",mindim=" + mindim); + } + if(maxdim < Integer.MAX_VALUE) { + buf.append(",maxdim=" + maxdim); + } } + return buf.toString(); } /** diff --git a/src/de/lmu/ifi/dbs/elki/data/type/VectorTypeInformation.java b/src/de/lmu/ifi/dbs/elki/data/type/VectorTypeInformation.java index 012cff48..a1db6d1c 100644 --- a/src/de/lmu/ifi/dbs/elki/data/type/VectorTypeInformation.java +++ b/src/de/lmu/ifi/dbs/elki/data/type/VectorTypeInformation.java @@ -180,6 +180,7 @@ public class VectorTypeInformation<V extends FeatureVector<?, ?>> extends Simple @Override public String toString() { StringBuffer buf = new StringBuffer(super.toString()); + buf.append(",variable"); if (mindim >= 0) { buf.append(",mindim="+mindim); } diff --git a/src/de/lmu/ifi/dbs/elki/database/AbstractDatabase.java b/src/de/lmu/ifi/dbs/elki/database/AbstractDatabase.java index f9582039..e0fc6bf2 100644 --- a/src/de/lmu/ifi/dbs/elki/database/AbstractDatabase.java +++ b/src/de/lmu/ifi/dbs/elki/database/AbstractDatabase.java @@ -23,6 +23,7 @@ package de.lmu.ifi.dbs.elki.database; along with this program. If not, see <http://www.gnu.org/licenses/>. */ +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -56,7 +57,8 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; /** * Abstract base class for database API implementations. Provides default - * management of relations, indexes and events as well as default query matching. + * management of relations, indexes and events as well as default query + * matching. * * @author Erich Schubert * @@ -137,7 +139,7 @@ public abstract class AbstractDatabase extends AbstractHierarchicalResult implem throw e; } } - + @Override public Collection<Relation<?>> getRelations() { return Collections.unmodifiableCollection(relations); @@ -152,15 +154,11 @@ public abstract class AbstractDatabase extends AbstractHierarchicalResult implem return (Relation<O>) relation; } } - if (getLogger().isDebugging()) { - StringBuffer buf = new StringBuffer(); - buf.append("No matching relation for type ").append(restriction.toString()).append(":\n"); - for(Relation<?> relation : relations) { - buf.append(relation.getDataTypeInformation().toString()).append(","); - } - getLogger().debug(buf); + List<TypeInformation> types = new ArrayList<TypeInformation>(relations.size()); + for(Relation<?> relation : relations) { + types.add(relation.getDataTypeInformation()); } - throw new NoSupportedDataTypeException(restriction); + throw new NoSupportedDataTypeException(restriction, types); } @Override diff --git a/src/de/lmu/ifi/dbs/elki/database/HashmapDatabase.java b/src/de/lmu/ifi/dbs/elki/database/HashmapDatabase.java index e94bcfb1..a92c4427 100644 --- a/src/de/lmu/ifi/dbs/elki/database/HashmapDatabase.java +++ b/src/de/lmu/ifi/dbs/elki/database/HashmapDatabase.java @@ -31,6 +31,7 @@ import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; import de.lmu.ifi.dbs.elki.database.ids.DBIDFactory; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.HashSetModifiableDBIDs; @@ -62,7 +63,7 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter; * @author Erich Schubert * * @apiviz.landmark - * @apiviz.composedOf TreeSetModifiableDBIDs + * @apiviz.composedOf HashSetModifiableDBIDs */ @Description("Database using an in-memory hashtable and at least providing linear scans.") public class HashmapDatabase extends AbstractDatabase implements UpdatableDatabase, Parameterizable { @@ -246,13 +247,14 @@ public class HashmapDatabase extends AbstractDatabase implements UpdatableDataba MultipleObjectsBundle bundle = new MultipleObjectsBundle(); for(Relation<?> relation : relations) { ArrayList<Object> data = new ArrayList<Object>(ids.size()); - for(DBID id : ids) { - data.add(relation.get(id)); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + data.add(relation.get(iter)); } bundle.appendColumn(relation.getDataTypeInformation(), data); } // remove from db - for(DBID id : ids) { + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); doDelete(id); } // Remove from indexes diff --git a/src/de/lmu/ifi/dbs/elki/database/datastore/DataStore.java b/src/de/lmu/ifi/dbs/elki/database/datastore/DataStore.java index 8d7f73ce..a6d5a704 100644 --- a/src/de/lmu/ifi/dbs/elki/database/datastore/DataStore.java +++ b/src/de/lmu/ifi/dbs/elki/database/datastore/DataStore.java @@ -24,6 +24,7 @@ package de.lmu.ifi.dbs.elki.database.datastore; */ import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.result.Result; /** @@ -40,5 +41,5 @@ public interface DataStore<T> extends Result { * @param id Database ID. * @return Object or {@code null} */ - public T get(DBID id); -} + public T get(DBIDRef id); +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/datastore/DataStoreFactory.java b/src/de/lmu/ifi/dbs/elki/database/datastore/DataStoreFactory.java index 65a1265d..2ed9f536 100644 --- a/src/de/lmu/ifi/dbs/elki/database/datastore/DataStoreFactory.java +++ b/src/de/lmu/ifi/dbs/elki/database/datastore/DataStoreFactory.java @@ -91,6 +91,38 @@ public interface DataStoreFactory { public WritableDoubleDataStore makeDoubleStorage(DBIDs ids, int hints); /** + * Make a new storage, to associate the given ids with an object of class + * dataclass. + * + * @param ids DBIDs to store data for + * @param hints Hints for the storage manager + * @param def Default value + * @return new data store + */ + public WritableDoubleDataStore makeDoubleStorage(DBIDs ids, int hints, double def); + + /** + * Make a new storage, to associate the given ids with an object of class + * dataclass. + * + * @param ids DBIDs to store data for + * @param hints Hints for the storage manager + * @return new data store + */ + public WritableIntegerDataStore makeIntegerStorage(DBIDs ids, int hints); + + /** + * Make a new storage, to associate the given ids with an object of class + * dataclass. + * + * @param ids DBIDs to store data for + * @param hints Hints for the storage manager + * @param def Default value + * @return new data store + */ + public WritableIntegerDataStore makeIntegerStorage(DBIDs ids, int hints, int def); + + /** * Make a new record storage, to associate the given ids with an object of * class dataclass. * diff --git a/src/de/lmu/ifi/dbs/elki/database/datastore/DataStoreIDMap.java b/src/de/lmu/ifi/dbs/elki/database/datastore/DataStoreIDMap.java index ead75709..dada881a 100644 --- a/src/de/lmu/ifi/dbs/elki/database/datastore/DataStoreIDMap.java +++ b/src/de/lmu/ifi/dbs/elki/database/datastore/DataStoreIDMap.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.database.datastore; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; /** * Interface to map DBIDs to integer record ids for use in storage. @@ -37,5 +37,5 @@ public interface DataStoreIDMap { * @param dbid DBID * @return record id {@code id >= 0} */ - public int map(DBID dbid); + public int map(DBIDRef dbid); } diff --git a/src/de/lmu/ifi/dbs/elki/database/datastore/DataStoreUtil.java b/src/de/lmu/ifi/dbs/elki/database/datastore/DataStoreUtil.java index f299c1e8..a8afeaec 100644 --- a/src/de/lmu/ifi/dbs/elki/database/datastore/DataStoreUtil.java +++ b/src/de/lmu/ifi/dbs/elki/database/datastore/DataStoreUtil.java @@ -59,6 +59,41 @@ public final class DataStoreUtil { } /** + * Make a new storage, to associate the given ids with an object of class dataclass. + * + * @param ids DBIDs to store data for + * @param hints Hints for the storage manager + * @param def Default value + * @return new data store + */ + public static WritableDoubleDataStore makeDoubleStorage(DBIDs ids, int hints, double def) { + return DataStoreFactory.FACTORY.makeDoubleStorage(ids, hints, def); + } + + /** + * Make a new storage, to associate the given ids with an object of class dataclass. + * + * @param ids DBIDs to store data for + * @param hints Hints for the storage manager + * @return new data store + */ + public static WritableIntegerDataStore makeIntegerStorage(DBIDs ids, int hints) { + return DataStoreFactory.FACTORY.makeIntegerStorage(ids, hints); + } + + /** + * Make a new storage, to associate the given ids with an object of class dataclass. + * + * @param ids DBIDs to store data for + * @param hints Hints for the storage manager + * @param def Default value + * @return new data store + */ + public static WritableIntegerDataStore makeIntegerStorage(DBIDs ids, int hints, int def) { + return DataStoreFactory.FACTORY.makeIntegerStorage(ids, hints, def); + } + + /** * Make a new record storage, to associate the given ids with an object of class dataclass. * * @param ids DBIDs to store data for diff --git a/src/de/lmu/ifi/dbs/elki/database/datastore/DoubleDataStore.java b/src/de/lmu/ifi/dbs/elki/database/datastore/DoubleDataStore.java index ee315a0c..3348a246 100644 --- a/src/de/lmu/ifi/dbs/elki/database/datastore/DoubleDataStore.java +++ b/src/de/lmu/ifi/dbs/elki/database/datastore/DoubleDataStore.java @@ -1,7 +1,5 @@ package de.lmu.ifi.dbs.elki.database.datastore; -import de.lmu.ifi.dbs.elki.database.ids.DBID; - /* This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures @@ -25,14 +23,22 @@ import de.lmu.ifi.dbs.elki.database.ids.DBID; along with this program. If not, see <http://www.gnu.org/licenses/>. */ +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; + /** * Double-valued data store (avoids boxing/unboxing). * * @author Erich Schubert */ public interface DoubleDataStore extends DataStore<Double> { + /** + * Getter, but using objects. + * + * @deprecated Use {@link #doubleValue} instead, to avoid boxing/unboxing cost. + */ + @Override @Deprecated - public Double get(DBID id); + public Double get(DBIDRef id); /** * Retrieves an object from the storage. @@ -40,5 +46,5 @@ public interface DoubleDataStore extends DataStore<Double> { * @param id Database ID. * @return Double value */ - public double doubleValue(DBID id); + public double doubleValue(DBIDRef id); }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/utilities/iterator/IterableUtil.java b/src/de/lmu/ifi/dbs/elki/database/datastore/IntegerDataStore.java index 8e4afd80..e450c11b 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/iterator/IterableUtil.java +++ b/src/de/lmu/ifi/dbs/elki/database/datastore/IntegerDataStore.java @@ -1,4 +1,4 @@ -package de.lmu.ifi.dbs.elki.utilities.iterator; +package de.lmu.ifi.dbs.elki.database.datastore; /* This file is part of ELKI: @@ -23,33 +23,28 @@ package de.lmu.ifi.dbs.elki.utilities.iterator; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.Iterator; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; /** - * Utility function to wrap an Iterator as iterable. + * Integer-valued data store (avoids boxing/unboxing). * * @author Erich Schubert */ -public final class IterableUtil { +public interface IntegerDataStore extends DataStore<Integer> { /** - * Wrap an iterator as Iterable. + * Getter, but using objects. * - * @param <C> Type restriction - * @param iter Iterator - * @return Iterable wrapper + * @deprecated Use {@link #intValue} instead, to avoid boxing/unboxing cost. */ - public static <C> IterableIterator<C> fromIterator(Iterator<C> iter) { - return new IterableIteratorAdapter<C>(iter); - } + @Override + @Deprecated + public Integer get(DBIDRef id); /** - * Wrap an Iterable as IterableIterator + * Retrieves an object from the storage. * - * @param <C> Type restriction - * @param iter Iterator - * @return Iterable wrapper + * @param id Database ID. + * @return Double value */ - public static <C> IterableIterator<C> fromIterable(Iterable<C> iter) { - return new IterableIteratorAdapter<C>(iter); - } -} + public int intValue(DBIDRef id); +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/datastore/RangeIDMap.java b/src/de/lmu/ifi/dbs/elki/database/datastore/RangeIDMap.java index 7fc0ed7f..e00f9ff8 100644 --- a/src/de/lmu/ifi/dbs/elki/database/datastore/RangeIDMap.java +++ b/src/de/lmu/ifi/dbs/elki/database/datastore/RangeIDMap.java @@ -23,8 +23,8 @@ package de.lmu.ifi.dbs.elki.database.datastore; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import de.lmu.ifi.dbs.elki.database.ids.DBID; import de.lmu.ifi.dbs.elki.database.ids.DBIDRange; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; /** * Mapping a static DBID range to storage IDs. @@ -47,7 +47,7 @@ public class RangeIDMap implements DataStoreIDMap { } @Override - public int map(DBID dbid) { + public int map(DBIDRef dbid) { return range.getOffset(dbid); } } diff --git a/src/de/lmu/ifi/dbs/elki/database/datastore/WritableDataStore.java b/src/de/lmu/ifi/dbs/elki/database/datastore/WritableDataStore.java index 7f1fbf52..93176445 100644 --- a/src/de/lmu/ifi/dbs/elki/database/datastore/WritableDataStore.java +++ b/src/de/lmu/ifi/dbs/elki/database/datastore/WritableDataStore.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.database.datastore; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; /** * Writable data store. @@ -32,7 +32,7 @@ import de.lmu.ifi.dbs.elki.database.ids.DBID; * * @apiviz.landmark * - * @param <T> + * @param <T> Data type */ public interface WritableDataStore<T> extends DataStore<T> { /** @@ -44,7 +44,7 @@ public interface WritableDataStore<T> extends DataStore<T> { * @param value Value to store. * @return previous value */ - public T put(DBID id, T value); + public T put(DBIDRef id, T value); /** * Deallocate the storage, freeing the memory and notifies the registered @@ -58,5 +58,5 @@ public interface WritableDataStore<T> extends DataStore<T> { * * @param id Database ID. */ - public void delete(DBID id); + public void delete(DBIDRef id); } diff --git a/src/de/lmu/ifi/dbs/elki/database/datastore/WritableDoubleDataStore.java b/src/de/lmu/ifi/dbs/elki/database/datastore/WritableDoubleDataStore.java index 313e4adc..19cc54c7 100644 --- a/src/de/lmu/ifi/dbs/elki/database/datastore/WritableDoubleDataStore.java +++ b/src/de/lmu/ifi/dbs/elki/database/datastore/WritableDoubleDataStore.java @@ -1,7 +1,5 @@ package de.lmu.ifi.dbs.elki.database.datastore; -import de.lmu.ifi.dbs.elki.database.ids.DBID; - /* This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures @@ -25,15 +23,22 @@ import de.lmu.ifi.dbs.elki.database.ids.DBID; along with this program. If not, see <http://www.gnu.org/licenses/>. */ +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; + /** * Data store specialized for doubles. Avoids boxing/unboxing. * * @author Erich Schubert */ public interface WritableDoubleDataStore extends DoubleDataStore, WritableDataStore<Double> { + /** + * Setter, but using objects. + * + * @deprecated Use {@link #putDouble} instead, to avoid boxing/unboxing cost. + */ @Override @Deprecated - public Double put(DBID id, Double value); + public Double put(DBIDRef id, Double value); /** * Associates the specified value with the specified id in this storage. If @@ -44,7 +49,7 @@ public interface WritableDoubleDataStore extends DoubleDataStore, WritableDataSt * @param value Value to store. * @return previous value */ - public double putDouble(DBID id, double value); + public double putDouble(DBIDRef id, double value); /** * Associates the specified value with the specified id in this storage. If @@ -55,5 +60,5 @@ public interface WritableDoubleDataStore extends DoubleDataStore, WritableDataSt * @param value Value to store. * @return previous value */ - public double put(DBID id, double value); + public double put(DBIDRef id, double value); }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/datastore/WritableIntegerDataStore.java b/src/de/lmu/ifi/dbs/elki/database/datastore/WritableIntegerDataStore.java new file mode 100644 index 00000000..b8bf1348 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/database/datastore/WritableIntegerDataStore.java @@ -0,0 +1,64 @@ +package de.lmu.ifi.dbs.elki.database.datastore; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; + +/** + * Data store specialized for doubles. Avoids boxing/unboxing. + * + * @author Erich Schubert + */ +public interface WritableIntegerDataStore extends IntegerDataStore, WritableDataStore<Integer> { + /** + * Setter, but using objects. + * + * @deprecated Use {@link #putInt} instead, to avoid boxing/unboxing cost. + */ + @Override + @Deprecated + public Integer put(DBIDRef id, Integer value); + + /** + * Associates the specified value with the specified id in this storage. If + * the storage previously contained a value for the id, the previous value is + * replaced by the specified value. + * + * @param id Database ID. + * @param value Value to store. + * @return previous value + */ + public int putInt(DBIDRef id, int value); + + /** + * Associates the specified value with the specified id in this storage. If + * the storage previously contained a value for the id, the previous value is + * replaced by the specified value. + * + * @param id Database ID. + * @param value Value to store. + * @return previous value + */ + public int put(DBIDRef id, int value); +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/datastore/WritableRecordStore.java b/src/de/lmu/ifi/dbs/elki/database/datastore/WritableRecordStore.java index 775d201b..0799fdfe 100644 --- a/src/de/lmu/ifi/dbs/elki/database/datastore/WritableRecordStore.java +++ b/src/de/lmu/ifi/dbs/elki/database/datastore/WritableRecordStore.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.database.datastore; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; /** * Represents a storage which stores multiple values per object in a record fashion. @@ -52,5 +52,5 @@ public interface WritableRecordStore extends RecordStore { * @param id object ID to remove * @return success code */ - public boolean remove(DBID id); + public boolean remove(DBIDRef id); } diff --git a/src/de/lmu/ifi/dbs/elki/database/datastore/memory/ArrayDoubleStore.java b/src/de/lmu/ifi/dbs/elki/database/datastore/memory/ArrayDoubleStore.java index 433547a5..de22a6b3 100644 --- a/src/de/lmu/ifi/dbs/elki/database/datastore/memory/ArrayDoubleStore.java +++ b/src/de/lmu/ifi/dbs/elki/database/datastore/memory/ArrayDoubleStore.java @@ -23,9 +23,11 @@ package de.lmu.ifi.dbs.elki.database.datastore.memory; along with this program. If not, see <http://www.gnu.org/licenses/>. */ +import java.util.Arrays; + import de.lmu.ifi.dbs.elki.database.datastore.DataStoreIDMap; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; /** * A class to answer representation queries using the stored Array. @@ -52,14 +54,28 @@ public class ArrayDoubleStore implements WritableDoubleDataStore { * @param idmap ID map */ public ArrayDoubleStore(int size, DataStoreIDMap idmap) { + this(size, idmap, Double.NaN); + } + + /** + * Constructor. + * + * @param size Size + * @param idmap ID map + * @param def Default value + */ + public ArrayDoubleStore(int size, DataStoreIDMap idmap, double def) { super(); this.data = new double[size]; + if(def != 0) { + Arrays.fill(this.data, def); + } this.idmap = idmap; } @Override @Deprecated - public Double get(DBID id) { + public Double get(DBIDRef id) { try { return data[idmap.map(id)]; } @@ -70,20 +86,20 @@ public class ArrayDoubleStore implements WritableDoubleDataStore { @Override @Deprecated - public Double put(DBID id, Double value) { + public Double put(DBIDRef id, Double value) { final int off = idmap.map(id); double ret = data[off]; data[off] = value; return ret; } - + @Override - public double doubleValue(DBID id) { + public double doubleValue(DBIDRef id) { return data[idmap.map(id)]; } @Override - public double putDouble(DBID id, double value) { + public double putDouble(DBIDRef id, double value) { final int off = idmap.map(id); final double ret = data[off]; data[off] = value; @@ -91,7 +107,7 @@ public class ArrayDoubleStore implements WritableDoubleDataStore { } @Override - public double put(DBID id, double value) { + public double put(DBIDRef id, double value) { final int off = idmap.map(id); final double ret = data[off]; data[off] = value; @@ -105,7 +121,7 @@ public class ArrayDoubleStore implements WritableDoubleDataStore { } @Override - public void delete(DBID id) { + public void delete(DBIDRef id) { throw new UnsupportedOperationException("Can't delete from a static array storage."); } diff --git a/src/de/lmu/ifi/dbs/elki/database/datastore/memory/ArrayIntegerStore.java b/src/de/lmu/ifi/dbs/elki/database/datastore/memory/ArrayIntegerStore.java new file mode 100644 index 00000000..8caa7ec3 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/database/datastore/memory/ArrayIntegerStore.java @@ -0,0 +1,137 @@ +package de.lmu.ifi.dbs.elki.database.datastore.memory; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import java.util.Arrays; + +import de.lmu.ifi.dbs.elki.database.datastore.DataStoreIDMap; +import de.lmu.ifi.dbs.elki.database.datastore.WritableIntegerDataStore; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; + +/** + * A class to answer representation queries using the stored Array. + * + * @author Erich Schubert + * + * @apiviz.composedOf de.lmu.ifi.dbs.elki.database.datastore.DataStoreIDMap + */ +public class ArrayIntegerStore implements WritableIntegerDataStore { + /** + * Data array + */ + private int[] data; + + /** + * DBID to index map + */ + private DataStoreIDMap idmap; + + /** + * Constructor. + * + * @param size Size + * @param idmap ID map + */ + public ArrayIntegerStore(int size, DataStoreIDMap idmap) { + this(size, idmap, 0); + } + + /** + * Constructor. + * + * @param size Size + * @param idmap ID map + * @param def Default value + */ + public ArrayIntegerStore(int size, DataStoreIDMap idmap, int def) { + super(); + this.data = new int[size]; + if (def != 0) { + Arrays.fill(this.data, def); + } + this.idmap = idmap; + } + + @Override + @Deprecated + public Integer get(DBIDRef id) { + try { + return data[idmap.map(id)]; + } + catch(ArrayIndexOutOfBoundsException e) { + return null; + } + } + + @Override + @Deprecated + public Integer put(DBIDRef id, Integer value) { + final int off = idmap.map(id); + int ret = data[off]; + data[off] = value; + return ret; + } + + @Override + public int intValue(DBIDRef id) { + return data[idmap.map(id)]; + } + + @Override + public int putInt(DBIDRef id, int value) { + final int off = idmap.map(id); + final int ret = data[off]; + data[off] = value; + return ret; + } + + @Override + public int put(DBIDRef id, int value) { + final int off = idmap.map(id); + final int ret = data[off]; + data[off] = value; + return ret; + } + + @Override + public void destroy() { + data = null; + idmap = null; + } + + @Override + public void delete(DBIDRef id) { + throw new UnsupportedOperationException("Can't delete from a static array storage."); + } + + @Override + public String getLongName() { + return "raw"; + } + + @Override + public String getShortName() { + return "raw"; + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/datastore/memory/ArrayRecordStore.java b/src/de/lmu/ifi/dbs/elki/database/datastore/memory/ArrayRecordStore.java index 7be68c97..6e578b61 100644 --- a/src/de/lmu/ifi/dbs/elki/database/datastore/memory/ArrayRecordStore.java +++ b/src/de/lmu/ifi/dbs/elki/database/datastore/memory/ArrayRecordStore.java @@ -26,7 +26,7 @@ package de.lmu.ifi.dbs.elki.database.datastore.memory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreIDMap; import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore; import de.lmu.ifi.dbs.elki.database.datastore.WritableRecordStore; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; /** * A class to answer representation queries using the stored Array. @@ -73,7 +73,7 @@ public class ArrayRecordStore implements WritableRecordStore { * @return current value */ @SuppressWarnings("unchecked") - protected <T> T get(DBID id, int index) { + protected <T> T get(DBIDRef id, int index) { try { return (T) data[idmap.map(id)][index]; } @@ -97,7 +97,7 @@ public class ArrayRecordStore implements WritableRecordStore { * @return old value */ @SuppressWarnings("unchecked") - protected <T> T set(DBID id, int index, T value) { + protected <T> T set(DBIDRef id, int index, T value) { T ret = (T) data[idmap.map(id)][index]; data[idmap.map(id)][index] = value; return ret; @@ -128,12 +128,12 @@ public class ArrayRecordStore implements WritableRecordStore { @SuppressWarnings("unchecked") @Override - public T get(DBID id) { + public T get(DBIDRef id) { return (T) ArrayRecordStore.this.get(id, index); } @Override - public T put(DBID id, T value) { + public T put(DBIDRef id, T value) { return ArrayRecordStore.this.set(id, index, value); } @@ -143,7 +143,7 @@ public class ArrayRecordStore implements WritableRecordStore { } @Override - public void delete(DBID id) { + public void delete(DBIDRef id) { throw new UnsupportedOperationException("ArrayStore record values cannot be deleted."); } @@ -159,7 +159,7 @@ public class ArrayRecordStore implements WritableRecordStore { } @Override - public boolean remove(DBID id) { + public boolean remove(DBIDRef id) { throw new UnsupportedOperationException("ArrayStore records cannot be removed."); } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/datastore/memory/ArrayStore.java b/src/de/lmu/ifi/dbs/elki/database/datastore/memory/ArrayStore.java index a41a444d..a7ce310b 100644 --- a/src/de/lmu/ifi/dbs/elki/database/datastore/memory/ArrayStore.java +++ b/src/de/lmu/ifi/dbs/elki/database/datastore/memory/ArrayStore.java @@ -25,7 +25,7 @@ package de.lmu.ifi.dbs.elki.database.datastore.memory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreIDMap; import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; /** * A class to answer representation queries using the stored Array. @@ -58,7 +58,7 @@ public class ArrayStore<T> implements WritableDataStore<T> { @SuppressWarnings("unchecked") @Override - public T get(DBID id) { + public T get(DBIDRef id) { try { return (T) data[idmap.map(id)]; } @@ -74,7 +74,7 @@ public class ArrayStore<T> implements WritableDataStore<T> { } @Override - public T put(DBID id, T value) { + public T put(DBIDRef id, T value) { T ret = get(id); data[idmap.map(id)] = value; return ret; @@ -87,7 +87,7 @@ public class ArrayStore<T> implements WritableDataStore<T> { } @Override - public void delete(DBID id) { + public void delete(DBIDRef id) { throw new UnsupportedOperationException("Can't delete from a static array storage."); } diff --git a/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapIntegerDBIDDoubleStore.java b/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapIntegerDBIDDoubleStore.java index ae06dc00..f9f8d48a 100644 --- a/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapIntegerDBIDDoubleStore.java +++ b/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapIntegerDBIDDoubleStore.java @@ -1,4 +1,5 @@ package de.lmu.ifi.dbs.elki.database.datastore.memory; + /* This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures @@ -25,7 +26,7 @@ package de.lmu.ifi.dbs.elki.database.datastore.memory; import gnu.trove.map.TIntDoubleMap; import gnu.trove.map.hash.TIntDoubleHashMap; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; /** * Writable data store for double values. @@ -37,25 +38,35 @@ public class MapIntegerDBIDDoubleStore implements WritableDoubleDataStore { * Data storage */ private TIntDoubleMap map; - + /** * Constructor. - * + * * @param size Expected size */ public MapIntegerDBIDDoubleStore(int size) { + this(size, Double.NaN); + } + + /** + * Constructor. + * + * @param size Expected size + * @param def Default value + */ + public MapIntegerDBIDDoubleStore(int size, double def) { super(); - map = new TIntDoubleHashMap(size, 0.5f, Integer.MIN_VALUE, Double.NaN); + map = new TIntDoubleHashMap(size, 0.5f, Integer.MIN_VALUE, def); } @Override @Deprecated - public Double get(DBID id) { + public Double get(DBIDRef id) { return map.get(id.getIntegerID()); } @Override - public double doubleValue(DBID id) { + public double doubleValue(DBIDRef id) { return map.get(id.getIntegerID()); } @@ -71,7 +82,7 @@ public class MapIntegerDBIDDoubleStore implements WritableDoubleDataStore { @Override @Deprecated - public Double put(DBID id, Double value) { + public Double put(DBIDRef id, Double value) { return map.put(id.getIntegerID(), value); } @@ -82,17 +93,17 @@ public class MapIntegerDBIDDoubleStore implements WritableDoubleDataStore { } @Override - public void delete(DBID id) { + public void delete(DBIDRef id) { map.remove(id.getIntegerID()); } @Override - public double putDouble(DBID id, double value) { + public double putDouble(DBIDRef id, double value) { return map.put(id.getIntegerID(), value); } @Override - public double put(DBID id, double value) { + public double put(DBIDRef id, double value) { return map.put(id.getIntegerID(), value); } } diff --git a/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapIntegerDBIDIntegerStore.java b/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapIntegerDBIDIntegerStore.java new file mode 100644 index 00000000..f7aea633 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapIntegerDBIDIntegerStore.java @@ -0,0 +1,109 @@ +package de.lmu.ifi.dbs.elki.database.datastore.memory; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import gnu.trove.map.TIntIntMap; +import gnu.trove.map.hash.TIntIntHashMap; +import de.lmu.ifi.dbs.elki.database.datastore.WritableIntegerDataStore; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; + +/** + * Writable data store for double values. + * + * @author Erich Schubert + */ +public class MapIntegerDBIDIntegerStore implements WritableIntegerDataStore { + /** + * Data storage + */ + private TIntIntMap map; + + /** + * Constructor. + * + * @param size Expected size + */ + public MapIntegerDBIDIntegerStore(int size) { + this(size, 0); + } + + /** + * Constructor. + * + * @param size Expected size + * @param def Default value + */ + public MapIntegerDBIDIntegerStore(int size, int def) { + super(); + map = new TIntIntHashMap(size, 0.5f, Integer.MIN_VALUE, def); + } + + @Override + @Deprecated + public Integer get(DBIDRef id) { + return map.get(id.getIntegerID()); + } + + @Override + public int intValue(DBIDRef id) { + return map.get(id.getIntegerID()); + } + + @Override + public String getLongName() { + return "raw"; + } + + @Override + public String getShortName() { + return "raw"; + } + + @Override + @Deprecated + public Integer put(DBIDRef id, Integer value) { + return map.put(id.getIntegerID(), value); + } + + @Override + public void destroy() { + map.clear(); + map = null; + } + + @Override + public void delete(DBIDRef id) { + map.remove(id.getIntegerID()); + } + + @Override + public int putInt(DBIDRef id, int value) { + return map.put(id.getIntegerID(), value); + } + + @Override + public int put(DBIDRef id, int value) { + return map.put(id.getIntegerID(), value); + } +} diff --git a/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapIntegerDBIDRecordStore.java b/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapIntegerDBIDRecordStore.java index 8272fb2e..805c6de3 100644 --- a/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapIntegerDBIDRecordStore.java +++ b/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapIntegerDBIDRecordStore.java @@ -27,7 +27,7 @@ import gnu.trove.map.TIntObjectMap; import gnu.trove.map.hash.TIntObjectHashMap; import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore; import de.lmu.ifi.dbs.elki.database.datastore.WritableRecordStore; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; /** * A class to answer representation queries using a map and an index within the @@ -93,7 +93,7 @@ public class MapIntegerDBIDRecordStore implements WritableRecordStore { * @return current value */ @SuppressWarnings("unchecked") - protected <T> T get(DBID id, int index) { + protected <T> T get(DBIDRef id, int index) { Object[] d = data.get(id.getIntegerID()); if(d == null) { return null; @@ -118,7 +118,7 @@ public class MapIntegerDBIDRecordStore implements WritableRecordStore { * @return previous value */ @SuppressWarnings("unchecked") - protected <T> T set(DBID id, int index, T value) { + protected <T> T set(DBIDRef id, int index, T value) { Object[] d = data.get(id.getIntegerID()); if(d == null) { d = new Object[rlen]; @@ -154,12 +154,12 @@ public class MapIntegerDBIDRecordStore implements WritableRecordStore { @SuppressWarnings("unchecked") @Override - public T get(DBID id) { + public T get(DBIDRef id) { return (T) MapIntegerDBIDRecordStore.this.get(id, index); } @Override - public T put(DBID id, T value) { + public T put(DBIDRef id, T value) { return MapIntegerDBIDRecordStore.this.set(id, index, value); } @@ -169,7 +169,7 @@ public class MapIntegerDBIDRecordStore implements WritableRecordStore { } @Override - public void delete(DBID id) { + public void delete(DBIDRef id) { throw new UnsupportedOperationException("Record storage values cannot be deleted."); } @@ -185,7 +185,7 @@ public class MapIntegerDBIDRecordStore implements WritableRecordStore { } @Override - public boolean remove(DBID id) { + public boolean remove(DBIDRef id) { return data.remove(id.getIntegerID()) != null; } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapIntegerDBIDStore.java b/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapIntegerDBIDStore.java index 4deb929d..e04027d0 100644 --- a/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapIntegerDBIDStore.java +++ b/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapIntegerDBIDStore.java @@ -26,7 +26,7 @@ package de.lmu.ifi.dbs.elki.database.datastore.memory; import gnu.trove.map.TIntObjectMap; import gnu.trove.map.hash.TIntObjectHashMap; import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; /** * A class to answer representation queries using a map. Basically, it is just a @@ -70,12 +70,12 @@ public class MapIntegerDBIDStore<T> implements WritableDataStore<T> { } @Override - public T get(DBID id) { + public T get(DBIDRef id) { return data.get(id.getIntegerID()); } @Override - public T put(DBID id, T value) { + public T put(DBIDRef id, T value) { if(value == null) { return data.remove(id.getIntegerID()); } @@ -88,7 +88,7 @@ public class MapIntegerDBIDStore<T> implements WritableDataStore<T> { } @Override - public void delete(DBID id) { + public void delete(DBIDRef id) { data.remove(id.getIntegerID()); } diff --git a/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapRecordStore.java b/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapRecordStore.java index 5a98966f..05cf3697 100644 --- a/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapRecordStore.java +++ b/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapRecordStore.java @@ -29,6 +29,7 @@ import java.util.concurrent.ConcurrentHashMap; import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore; import de.lmu.ifi.dbs.elki.database.datastore.WritableRecordStore; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; /** * A class to answer representation queries using a map and an index within the @@ -47,6 +48,7 @@ public class MapRecordStore implements WritableRecordStore { /** * Storage Map */ + // TODO: Use trove maps? private final Map<DBID, Object[]> data; /** @@ -84,8 +86,8 @@ public class MapRecordStore implements WritableRecordStore { * @return current value */ @SuppressWarnings("unchecked") - protected <T> T get(DBID id, int index) { - Object[] d = data.get(id); + protected <T> T get(DBIDRef id, int index) { + Object[] d = data.get(id.getDBID()); if(d == null) { return null; } @@ -109,11 +111,11 @@ public class MapRecordStore implements WritableRecordStore { * @return previous value */ @SuppressWarnings("unchecked") - protected <T> T set(DBID id, int index, T value) { - Object[] d = data.get(id); + protected <T> T set(DBIDRef id, int index, T value) { + Object[] d = data.get(id.getDBID()); if(d == null) { d = new Object[rlen]; - data.put(id, d); + data.put(id.getDBID(), d); } T ret = (T) d[index]; d[index] = value; @@ -145,12 +147,12 @@ public class MapRecordStore implements WritableRecordStore { @SuppressWarnings("unchecked") @Override - public T get(DBID id) { + public T get(DBIDRef id) { return (T) MapRecordStore.this.get(id, index); } @Override - public T put(DBID id, T value) { + public T put(DBIDRef id, T value) { return MapRecordStore.this.set(id, index, value); } @@ -160,7 +162,7 @@ public class MapRecordStore implements WritableRecordStore { } @Override - public void delete(DBID id) { + public void delete(DBIDRef id) { throw new UnsupportedOperationException("Record storage values cannot be deleted."); } @@ -176,7 +178,7 @@ public class MapRecordStore implements WritableRecordStore { } @Override - public boolean remove(DBID id) { + public boolean remove(DBIDRef id) { return data.remove(id) != null; } diff --git a/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapStore.java b/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapStore.java index 27cd9f63..90742993 100644 --- a/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapStore.java +++ b/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MapStore.java @@ -28,6 +28,7 @@ import java.util.Map; import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; /** * A class to answer representation queries using a map. Basically, it is just a @@ -41,6 +42,7 @@ public class MapStore<T> implements WritableDataStore<T> { /** * Storage Map */ + // TODO: use trove maps? private Map<DBID, T> data; /** @@ -62,16 +64,16 @@ public class MapStore<T> implements WritableDataStore<T> { } @Override - public T get(DBID id) { - return data.get(id); + public T get(DBIDRef id) { + return data.get(id.getDBID()); } @Override - public T put(DBID id, T value) { + public T put(DBIDRef id, T value) { if(value == null) { - return data.remove(id); + return data.remove(id.getDBID()); } - return data.put(id, value); + return data.put(id.getDBID(), value); } @Override @@ -80,7 +82,7 @@ public class MapStore<T> implements WritableDataStore<T> { } @Override - public void delete(DBID id) { + public void delete(DBIDRef id) { data.remove(id); } diff --git a/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MemoryDataStoreFactory.java b/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MemoryDataStoreFactory.java index 3e3ce017..683e4c5d 100644 --- a/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MemoryDataStoreFactory.java +++ b/src/de/lmu/ifi/dbs/elki/database/datastore/memory/MemoryDataStoreFactory.java @@ -27,6 +27,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.RangeIDMap; import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore; import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; +import de.lmu.ifi.dbs.elki.database.datastore.WritableIntegerDataStore; import de.lmu.ifi.dbs.elki.database.datastore.WritableRecordStore; import de.lmu.ifi.dbs.elki.database.ids.DBIDRange; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; @@ -47,8 +48,15 @@ import de.lmu.ifi.dbs.elki.database.ids.DBIDs; * @apiviz.uses MapRecordStore oneway - - «create» */ public class MemoryDataStoreFactory implements DataStoreFactory { + @SuppressWarnings("unchecked") @Override public <T> WritableDataStore<T> makeStorage(DBIDs ids, int hints, Class<? super T> dataclass) { + if (Double.class.equals(dataclass)) { + return (WritableDataStore<T>) makeDoubleStorage(ids, hints); + } + if (Integer.class.equals(dataclass)) { + return (WritableDataStore<T>) makeIntegerStorage(ids, hints); + } if(ids instanceof DBIDRange) { DBIDRange range = (DBIDRange) ids; Object[] data = new Object[range.size()]; @@ -71,6 +79,39 @@ public class MemoryDataStoreFactory implements DataStoreFactory { } @Override + public WritableDoubleDataStore makeDoubleStorage(DBIDs ids, int hints, double def) { + if(ids instanceof DBIDRange) { + DBIDRange range = (DBIDRange) ids; + return new ArrayDoubleStore(range.size(), new RangeIDMap(range), def); + } + else { + return new MapIntegerDBIDDoubleStore(ids.size(), def); + } + } + + @Override + public WritableIntegerDataStore makeIntegerStorage(DBIDs ids, int hints) { + if(ids instanceof DBIDRange) { + DBIDRange range = (DBIDRange) ids; + return new ArrayIntegerStore(range.size(), new RangeIDMap(range)); + } + else { + return new MapIntegerDBIDIntegerStore(ids.size()); + } + } + + @Override + public WritableIntegerDataStore makeIntegerStorage(DBIDs ids, int hints, int def) { + if(ids instanceof DBIDRange) { + DBIDRange range = (DBIDRange) ids; + return new ArrayIntegerStore(range.size(), new RangeIDMap(range), def); + } + else { + return new MapIntegerDBIDIntegerStore(ids.size(), def); + } + } + + @Override public WritableRecordStore makeRecordStorage(DBIDs ids, int hints, Class<?>... dataclasses) { if(ids instanceof DBIDRange) { DBIDRange range = (DBIDRange) ids; diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/ArrayDBIDs.java b/src/de/lmu/ifi/dbs/elki/database/ids/ArrayDBIDs.java index 13d6ea58..68bbb83d 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/ArrayDBIDs.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/ArrayDBIDs.java @@ -42,6 +42,7 @@ public interface ArrayDBIDs extends DBIDs { * * @return Iterator */ + @Override public DBIDIter iter(); /** @@ -49,6 +50,7 @@ public interface ArrayDBIDs extends DBIDs { * * @return size */ + @Override public int size(); /** @@ -61,5 +63,5 @@ public interface ArrayDBIDs extends DBIDs { * @param key Key to search for * @return Offset of key */ - public int binarySearch(DBID key); + public int binarySearch(DBIDRef key); } diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/ArrayModifiableDBIDs.java b/src/de/lmu/ifi/dbs/elki/database/ids/ArrayModifiableDBIDs.java index e9a6c8e0..95bcc2f7 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/ArrayModifiableDBIDs.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/ArrayModifiableDBIDs.java @@ -59,4 +59,12 @@ public interface ArrayModifiableDBIDs extends ModifiableDBIDs, ArrayDBIDs { * @return previous value */ public DBID set(int i, DBID newval); + + /** + * Swap DBIDs add positions a and b. + * + * @param a First position + * @param b Second position + */ + public void swap(int a, int b); }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/DBID.java b/src/de/lmu/ifi/dbs/elki/database/ids/DBID.java index 1391b75b..8d98893d 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/DBID.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/DBID.java @@ -37,11 +37,67 @@ package de.lmu.ifi.dbs.elki.database.ids; * * @apiviz.landmark */ -public interface DBID extends Comparable<DBID>, ArrayDBIDs { +public interface DBID extends DBIDRef, Comparable<DBIDRef>, ArrayDBIDs { /** - * Return the integer value of the object ID, if possible. + * Compare the <em>current</em> value of two referenced DBIDs. * - * @return integer id + * @param other Other DBID reference (or DBID) + * @return {@code true} when the references <em>currently</em> refer to the same. */ - public int getIntegerID(); + @Override + public boolean sameDBID(DBIDRef other); + + /** + * Compare two objects by the value of the referenced DBID. + * + * @param other Other DBID or object + * @return -1, 0 or +1 + */ + @Override + public int compareDBID(DBIDRef other); + + /** + * In contrast to {@link DBIDRef}, the DBID interface is supposed to have a + * stable hash code. However, it is generally preferred to use optimized + * storage classes instead of Java collections! + * + * @return hash code + */ + @Override + public int hashCode(); + + /** + * In contrast to {@link DBIDRef}, the DBID interface is supposed to have a + * stable equals for other DBIDs. + * + * Yet, {@link #sameDBID} is more type safe and explicit. + * + * @return true when the object is the same DBID. + */ + @Override + public boolean equals(Object obj); + + /** + * Part of the DBIDRef API, this <em>must</em> return {@code this} for an + * actual DBID. + * + * @return {@code this} + * @deprecated When the object is known to be a DBID, the usage of this method + * is pointless, therefore it is marked as deprecated to cause a + * warning. + */ + @Deprecated + @Override + public DBID getDBID(); + + /** + * Compare two DBIDs for ordering. + * + * Consider using {@link #compareDBID}, which is more explicit. + * + * @param other Other DBID object + * @return Comparison result + */ + @Override + public int compareTo(DBIDRef other); }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/DBIDFactory.java b/src/de/lmu/ifi/dbs/elki/database/ids/DBIDFactory.java index f35f390b..6063508b 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/DBIDFactory.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/DBIDFactory.java @@ -40,7 +40,7 @@ import de.lmu.ifi.dbs.elki.persistent.FixedSizeByteBufferSerializer; * @apiviz.uses DBIDRange oneway - - «create» * @apiviz.uses ArrayModifiableDBIDs oneway - - «create» * @apiviz.uses HashSetModifiableDBIDs oneway - - «create» - * @apiviz.uses TreeSetModifiableDBIDs oneway - - «create» + * @apiviz.uses HashSetModifiableDBIDs oneway - - «create» * @apiviz.has ByteBufferSerializer oneway - - provides */ public interface DBIDFactory { @@ -89,12 +89,12 @@ public interface DBIDFactory { /** * Make a DBID pair from two existing DBIDs. * - * @param first first DBID - * @param second second DBID + * @param id1 first DBID + * @param id2 second DBID * * @return new pair. */ - public DBIDPair makePair(DBID first, DBID second); + public DBIDPair makePair(DBIDRef id1, DBIDRef id2); /** * Make a new (modifiable) array of DBIDs. diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/DBIDIter.java b/src/de/lmu/ifi/dbs/elki/database/ids/DBIDIter.java index a41284f4..f2d0ae91 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/DBIDIter.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/DBIDIter.java @@ -22,6 +22,9 @@ package de.lmu.ifi.dbs.elki.database.ids; You should have received a copy of the GNU Affero General Public License along with this program. If not, see <http://www.gnu.org/licenses/>. */ + +import de.lmu.ifi.dbs.elki.utilities.iterator.Iter; + /** * Iterator for DBIDs. * @@ -30,11 +33,15 @@ package de.lmu.ifi.dbs.elki.database.ids; * with Java, but at the same time, the syntax is much more compatible with for * loops. * - * Usage example: <blockquote><code>{@code + * Usage example: + * + * <pre> + * {@code * for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { * iter.getDBID(); * } - * }</code></blockquote> + * } + * </pre> * * We list some fundamental differences. * <ul> @@ -47,33 +54,15 @@ package de.lmu.ifi.dbs.elki.database.ids; * * @author Erich Schubert */ -public interface DBIDIter { - /** - * Returns true if the iterator currently points to a valid object. - * - * @return a <code>boolean</code> value - */ - public boolean valid(); - +public interface DBIDIter extends DBIDRef, Iter { /** - * Moves the iterator forward to the next entry. + * Get the referenced {@link DBID}. * - * @throws java.util.NoSuchElementException if the iterator is already - * exhausted - */ - public void advance(); - - /** - * Return the integer value of the object ID, if possible. - * - * @return integer id - */ - public int getIntegerID(); - - /** - * Get the current DBID. + * Efficiency note: this may require materialization of a DBID object - if + * possible, use DBIDRef based APIs instead. * - * @return current DBID + * @return referenced DBID */ + @Override public DBID getDBID(); }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/DBIDMIter.java b/src/de/lmu/ifi/dbs/elki/database/ids/DBIDMIter.java new file mode 100644 index 00000000..fe3b182c --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/database/ids/DBIDMIter.java @@ -0,0 +1,38 @@ +package de.lmu.ifi.dbs.elki.database.ids; +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +/** + * Modifiable DBID iterator. + * + * @author Erich Schubert + */ +public interface DBIDMIter extends DBIDIter { + /** + * Remove the object the iterator currently points to. + * + * Subsequent calls to {@link #getDBID} may return a different element. + * Call {@link #advance()} to advance the iterator to the next element for further processing. + */ + void remove(); +} diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/DBIDRange.java b/src/de/lmu/ifi/dbs/elki/database/ids/DBIDRange.java index 48a1f0ba..1b9ccc3b 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/DBIDRange.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/DBIDRange.java @@ -23,7 +23,6 @@ package de.lmu.ifi.dbs.elki.database.ids; along with this program. If not, see <http://www.gnu.org/licenses/>. */ - /** * Static DBID range. * @@ -39,5 +38,5 @@ public interface DBIDRange extends ArrayStaticDBIDs { * @param dbid ID to compute index for * @return index */ - public int getOffset(DBID dbid); + public int getOffset(DBIDRef dbid); }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/DBIDRef.java b/src/de/lmu/ifi/dbs/elki/database/ids/DBIDRef.java new file mode 100644 index 00000000..0934b10b --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/database/ids/DBIDRef.java @@ -0,0 +1,92 @@ +package de.lmu.ifi.dbs.elki.database.ids; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +/** + * Some object referencing a {@link DBID}. Could be a {@link DBID}, a + * {@link DBIDIter}, for example. + * + * Important note: <em>do not assume this reference to be stable</em>. Iterators + * are a good example how the DBIDRef may change. + * + * @author Erich Schubert + */ +public interface DBIDRef { + /** + * Get the referenced {@link DBID}. + * + * Efficiency note: this may require materialization of a DBID object. + * + * @return referenced DBID + */ + public DBID getDBID(); + + /** + * Return the integer value of the object ID, if possible. + * + * @return integer id + */ + public int getIntegerID(); + + /** + * WARNING: Hash codes of this interface <b>might not be stable</b> (e.g. for + * iterators). + * + * @return current hash code (<b>may change!</b>) + * + * @deprecated Do not use this hash code. Some implementations will not offer + * stable hash codes! + */ + @Override + @Deprecated + public int hashCode(); + + /** + * WARNING: calling equality on a reference may be an indicator of incorrect + * usage, as it is not clear whether the programmer meant the references to be + * the same or the DBIDs. + * + * @param obj Object to compare with + * @return True when they are the same object + */ + @Override + @Deprecated + public boolean equals(Object obj); + + /** + * Compare the <em>current</em> value of two referenced DBIDs. + * + * @param other Other DBID reference (or DBID) + * @return {@code true} when the references <em>currently</em> refer to the same. + */ + public boolean sameDBID(DBIDRef other); + + /** + * Compare two objects by the value of the referenced DBID. + * + * @param other Other DBID or object + * @return -1, 0 or +1 + */ + public int compareDBID(DBIDRef other); +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/DBIDUtil.java b/src/de/lmu/ifi/dbs/elki/database/ids/DBIDUtil.java index cd2bd4e0..000659b9 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/DBIDUtil.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/DBIDUtil.java @@ -25,6 +25,7 @@ package de.lmu.ifi.dbs.elki.database.ids; import java.util.Random; +import de.lmu.ifi.dbs.elki.database.ids.generic.UnmodifiableArrayDBIDs; import de.lmu.ifi.dbs.elki.database.ids.generic.UnmodifiableDBIDs; import de.lmu.ifi.dbs.elki.persistent.ByteBufferSerializer; @@ -185,15 +186,45 @@ public final class DBIDUtil { return intersection(second, first); } ModifiableDBIDs inter = newHashSet(first.size()); - for(DBID id : first) { - if(second.contains(id)) { - inter.add(id); + for(DBIDIter it = first.iter(); it.valid(); it.advance()) { + if(second.contains(it)) { + inter.add(it); } } return inter; } /** + * Compute the set symmetric intersection of two sets. + * + * @param first First set + * @param second Second set + * @param firstonly OUTPUT: elements only in first. MUST BE EMPTY + * @param intersection OUTPUT: elements in intersection. MUST BE EMPTY + * @param secondonly OUTPUT: elements only in second. MUST BE EMPTY + */ + // TODO: optimize? + public static void symmetricIntersection(DBIDs first, DBIDs second, HashSetModifiableDBIDs firstonly, HashSetModifiableDBIDs intersection, HashSetModifiableDBIDs secondonly) { + if(first.size() > second.size()) { + symmetricIntersection(second, first, secondonly, intersection, firstonly); + return; + } + assert(firstonly.size() == 0) : "OUTPUT set should be empty!"; + assert(intersection.size() == 0) : "OUTPUT set should be empty!"; + assert(secondonly.size() == 0) : "OUTPUT set should be empty!"; + // Initialize with second + secondonly.addDBIDs(second); + for(DBIDIter it = first.iter(); it.valid(); it.advance()) { + // Try to remove + if(secondonly.remove(it)) { + intersection.add(it); + } else { + firstonly.add(it); + } + } + } + + /** * Returns the union of the two specified collection of IDs. * * @param ids1 the first collection @@ -201,7 +232,7 @@ public final class DBIDUtil { * @return the union of ids1 and ids2 without duplicates */ public static ModifiableDBIDs union(DBIDs ids1, DBIDs ids2) { - ModifiableDBIDs result = DBIDUtil.newHashSet(); + ModifiableDBIDs result = DBIDUtil.newHashSet(Math.max(ids1.size(), ids2.size())); result.addDBIDs(ids1); result.addDBIDs(ids2); return result; @@ -215,8 +246,7 @@ public final class DBIDUtil { * @return the difference of ids1 minus ids2 */ public static ModifiableDBIDs difference(DBIDs ids1, DBIDs ids2) { - ModifiableDBIDs result = DBIDUtil.newHashSet(); - result.addDBIDs(ids1); + ModifiableDBIDs result = DBIDUtil.newHashSet(ids1); result.removeDBIDs(ids2); return result; } @@ -231,7 +261,11 @@ public final class DBIDUtil { if(existing instanceof StaticDBIDs) { return (StaticDBIDs) existing; } - return new UnmodifiableDBIDs(existing); + if (existing instanceof ArrayDBIDs) { + return new UnmodifiableArrayDBIDs((ArrayDBIDs)existing); + } else { + return new UnmodifiableDBIDs(existing); + } } /** @@ -293,7 +327,7 @@ public final class DBIDUtil { * * @return DBID pair */ - public static DBIDPair newPair(DBID id1, DBID id2) { + public static DBIDPair newPair(DBIDRef id1, DBIDRef id2) { return DBIDFactory.FACTORY.makePair(id1, id2); } @@ -319,7 +353,7 @@ public final class DBIDUtil { */ public static ModifiableDBIDs randomSample(DBIDs source, int k, Long seed) { if(k <= 0 || k > source.size()) { - throw new IllegalArgumentException("Illegal value for size of random sample: " + k); + throw new IllegalArgumentException("Illegal value for size of random sample: " + k+ " > "+source.size()+" or < 0"); } final Random random; if(seed != null) { diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/DBIDs.java b/src/de/lmu/ifi/dbs/elki/database/ids/DBIDs.java index ef83ba69..e9a3e0ab 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/DBIDs.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/DBIDs.java @@ -32,18 +32,21 @@ import java.util.Iterator; * * @apiviz.landmark * @apiviz.composedOf DBID + * @apiviz.has DBIDIter */ public interface DBIDs extends Iterable<DBID> { /** - * Retrieve Iterator access to the IDs. + * Get a DBID iterator (a more efficient API). * - * @return an iterator for the IDs - */ - @Override - public Iterator<DBID> iterator(); - - /** - * Get a DBIDIterator (a more efficient API). + * usage example: + * + * <pre> + * {@code + * for(DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + * DBID id = iter.getDBID(); + * } + * } + * </pre> * * @return iterator */ @@ -62,7 +65,7 @@ public interface DBIDs extends Iterable<DBID> { * @param o object to test * @return true when contained */ - public boolean contains(DBID o); + public boolean contains(DBIDRef o); /** * Test for an empty DBID collection. @@ -70,4 +73,13 @@ public interface DBIDs extends Iterable<DBID> { * @return true when empty. */ public boolean isEmpty(); + + /** + * Classic iterator. + * + * @deprecated Use {@link DBIDIter} API instead. + */ + @Override + @Deprecated + public Iterator<DBID> iterator(); } diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/EmptyDBIDs.java b/src/de/lmu/ifi/dbs/elki/database/ids/EmptyDBIDs.java index 5a7d6991..a85b8954 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/EmptyDBIDs.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/EmptyDBIDs.java @@ -26,12 +26,15 @@ package de.lmu.ifi.dbs.elki.database.ids; import java.util.Iterator; import java.util.NoSuchElementException; +import de.lmu.ifi.dbs.elki.logging.LoggingUtil; import de.lmu.ifi.dbs.elki.utilities.iterator.EmptyIterator; /** * Empty DBID collection. * * @author Erich Schubert + * + * @apiviz.composedOf EmptyDBIDIterator */ public class EmptyDBIDs implements ArrayStaticDBIDs, SetDBIDs { /** @@ -47,7 +50,7 @@ public class EmptyDBIDs implements ArrayStaticDBIDs, SetDBIDs { } @Override - public boolean contains(DBID o) { + public boolean contains(DBIDRef o) { return false; } @@ -77,7 +80,7 @@ public class EmptyDBIDs implements ArrayStaticDBIDs, SetDBIDs { } @Override - public int binarySearch(DBID key) { + public int binarySearch(DBIDRef key) { return -1; // Not found } @@ -106,5 +109,23 @@ public class EmptyDBIDs implements ArrayStaticDBIDs, SetDBIDs { public DBID getDBID() { throw new NoSuchElementException(); } + + @Override + public boolean equals(Object other) { + if (other instanceof DBID) { + LoggingUtil.warning("Programming error detected: DBIDItr.equals(DBID). Use sameDBID()!", new Throwable()); + } + return super.equals(other); + } + + @Override + public boolean sameDBID(DBIDRef other) { + return false; + } + + @Override + public int compareDBID(DBIDRef other) { + throw new UnsupportedOperationException("Invalid iterator position. Cannot compare."); + } } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/ModifiableDBIDs.java b/src/de/lmu/ifi/dbs/elki/database/ids/ModifiableDBIDs.java index 97646902..c92caef0 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/ModifiableDBIDs.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/ModifiableDBIDs.java @@ -54,17 +54,36 @@ public interface ModifiableDBIDs extends DBIDs { * * @param id ID to add */ - boolean add(DBID id); + boolean add(DBIDRef id); /** * Remove a single DBID from the collection. * * @param id ID to remove */ - boolean remove(DBID id); + boolean remove(DBIDRef id); /** * Clear this collection. */ void clear(); + + /** + * Get a <em>modifiable</em> DBID iterator (a more efficient API). + * + * usage example: + * + * <pre> + * {@code + * for(DBIDMIter iter = ids.iter(); iter.valid(); iter.advance()) { + * DBID id = iter.getDBID(); + * iter.remove(); + * } + * } + * </pre> + * + * @return modifiable iterator + */ + @Override + DBIDMIter iter(); }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/generic/DBIDIterAdapter.java b/src/de/lmu/ifi/dbs/elki/database/ids/generic/DBIDIterAdapter.java index c517ea9f..91e307c2 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/generic/DBIDIterAdapter.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/generic/DBIDIterAdapter.java @@ -1,10 +1,5 @@ package de.lmu.ifi.dbs.elki.database.ids.generic; -import java.util.Iterator; - -import de.lmu.ifi.dbs.elki.database.ids.DBID; -import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; - /* This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures @@ -28,12 +23,18 @@ import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; along with this program. If not, see <http://www.gnu.org/licenses/>. */ +import java.util.Iterator; + +import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDMIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; + /** * Iterator for classic collections. * * @author Erich Schubert */ -public class DBIDIterAdapter implements DBIDIter { +public class DBIDIterAdapter implements DBIDMIter { /** * Current DBID */ @@ -79,4 +80,21 @@ public class DBIDIterAdapter implements DBIDIter { public DBID getDBID() { return cur; } + + @Override + public void remove() { + iter.remove(); + } + + @Override + public boolean sameDBID(DBIDRef other) { + return cur.getIntegerID() == other.getIntegerID(); + } + + @Override + public int compareDBID(DBIDRef o) { + final int thisVal = cur.getIntegerID(); + final int anotherVal = o.getIntegerID(); + return (thisVal < anotherVal ? -1 : (thisVal == anotherVal ? 0 : 1)); + } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/generic/GenericArrayModifiableDBIDs.java b/src/de/lmu/ifi/dbs/elki/database/ids/generic/GenericArrayModifiableDBIDs.java index f271180d..f02c5064 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/generic/GenericArrayModifiableDBIDs.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/generic/GenericArrayModifiableDBIDs.java @@ -30,6 +30,8 @@ import java.util.Comparator; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDMIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; /** @@ -80,7 +82,7 @@ public class GenericArrayModifiableDBIDs extends ArrayList<DBID> implements Arra public boolean addDBIDs(DBIDs ids) { super.ensureCapacity(size() + ids.size()); boolean changed = false; - for(DBID id : ids) { + for(DBIDIter id = ids.iter(); id.valid(); id.advance()) { changed |= add(id); } return changed; @@ -89,15 +91,20 @@ public class GenericArrayModifiableDBIDs extends ArrayList<DBID> implements Arra @Override public boolean removeDBIDs(DBIDs ids) { boolean changed = false; - for(DBID id : ids) { + for(DBIDIter id = ids.iter(); id.valid(); id.advance()) { changed |= super.remove(id); } return changed; } @Override - public boolean remove(DBID id) { - return super.remove(id); + public boolean add(DBIDRef id) { + return add(id.getDBID()); + } + + @Override + public boolean remove(DBIDRef id) { + return super.remove(id.getDBID()); } @Override @@ -111,17 +118,22 @@ public class GenericArrayModifiableDBIDs extends ArrayList<DBID> implements Arra } @Override - public DBIDIter iter() { + public DBIDMIter iter() { return new DBIDIterAdapter(iterator()); } @Override - public int binarySearch(DBID key) { - return Collections.binarySearch(this, key); + public int binarySearch(DBIDRef key) { + return Collections.binarySearch(this, key.getDBID()); } @Override - public boolean contains(DBID o) { + public boolean contains(DBIDRef o) { return super.contains(o); } + + @Override + public void swap(int a, int b) { + set(a, set(b, get(a))); + } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/generic/GenericHashSetModifiableDBIDs.java b/src/de/lmu/ifi/dbs/elki/database/ids/generic/GenericHashSetModifiableDBIDs.java index 31a8602c..6eadf0b1 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/generic/GenericHashSetModifiableDBIDs.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/generic/GenericHashSetModifiableDBIDs.java @@ -24,10 +24,11 @@ package de.lmu.ifi.dbs.elki.database.ids.generic; */ import java.util.HashSet; -import java.util.Iterator; import de.lmu.ifi.dbs.elki.database.ids.DBID; import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDMIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.HashSetModifiableDBIDs; @@ -72,7 +73,7 @@ public class GenericHashSetModifiableDBIDs extends HashSet<DBID> implements Hash */ public GenericHashSetModifiableDBIDs(DBIDs c) { super(c.size()); - for(DBID id : c) { + for(DBIDIter id = c.iter(); id.valid(); id.advance()) { add(id); } } @@ -80,7 +81,7 @@ public class GenericHashSetModifiableDBIDs extends HashSet<DBID> implements Hash @Override public boolean addDBIDs(DBIDs ids) { boolean changed = false; - for(DBID id : ids) { + for(DBIDIter id = ids.iter(); id.valid(); id.advance()) { changed |= add(id); } return changed; @@ -89,23 +90,27 @@ public class GenericHashSetModifiableDBIDs extends HashSet<DBID> implements Hash @Override public boolean removeDBIDs(DBIDs ids) { boolean changed = false; - for(DBID id : ids) { + for(DBIDIter id = ids.iter(); id.valid(); id.advance()) { changed |= super.remove(id); } return changed; } @Override - public boolean remove(DBID id) { - return super.remove(id); + public boolean add(DBIDRef id) { + return super.add(id.getDBID()); + } + + @Override + public boolean remove(DBIDRef id) { + return super.remove(id.getDBID()); } @Override public boolean retainAll(DBIDs ids) { boolean modified = false; - Iterator<DBID> it = iterator(); - while(it.hasNext()) { - if(!ids.contains(it.next())) { + for(DBIDMIter it = iter(); it.valid(); it.advance()) { + if(!ids.contains(it)) { it.remove(); modified = true; } @@ -114,12 +119,12 @@ public class GenericHashSetModifiableDBIDs extends HashSet<DBID> implements Hash } @Override - public DBIDIter iter() { + public DBIDMIter iter() { return new DBIDIterAdapter(iterator()); } @Override - public boolean contains(DBID o) { + public boolean contains(DBIDRef o) { return super.contains(o); } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/generic/MaskedDBIDs.java b/src/de/lmu/ifi/dbs/elki/database/ids/generic/MaskedDBIDs.java index 67c84290..13a6cc21 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/generic/MaskedDBIDs.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/generic/MaskedDBIDs.java @@ -29,6 +29,7 @@ import java.util.Iterator; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; /** @@ -36,7 +37,7 @@ import de.lmu.ifi.dbs.elki.database.ids.DBIDs; * * @author Erich Schubert * - * @apiviz.uses de.lmu.ifi.dbs.elki.database.ids.DBIDs + * @apiviz.uses DBIDs */ public class MaskedDBIDs implements DBIDs { /** @@ -99,10 +100,10 @@ public class MaskedDBIDs implements DBIDs { } @Override - public boolean contains(DBID o) { + public boolean contains(DBIDRef o) { // TODO: optimize. for(DBID id : this) { - if(id.equals(o)) { + if(id.sameDBID(o)) { return true; } } @@ -191,6 +192,18 @@ public class MaskedDBIDs implements DBIDs { public DBID getDBID() { return data.get(pos); } + + @Override + public boolean sameDBID(DBIDRef other) { + return getIntegerID() == other.getIntegerID(); + } + + @Override + public int compareDBID(DBIDRef o) { + final int thisVal = getIntegerID(); + final int anotherVal = o.getIntegerID(); + return (thisVal < anotherVal ? -1 : (thisVal == anotherVal ? 0 : 1)); + } } /** @@ -253,7 +266,7 @@ public class MaskedDBIDs implements DBIDs { @Override public boolean valid() { - return pos >= 0; + return (pos >= 0) && (pos < data.size()); } @Override @@ -270,5 +283,17 @@ public class MaskedDBIDs implements DBIDs { public DBID getDBID() { return data.get(pos); } + + @Override + public boolean sameDBID(DBIDRef other) { + return getIntegerID() == other.getIntegerID(); + } + + @Override + public int compareDBID(DBIDRef o) { + final int thisVal = getIntegerID(); + final int anotherVal = o.getIntegerID(); + return (thisVal < anotherVal ? -1 : (thisVal == anotherVal ? 0 : 1)); + } } -} +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/generic/MergedDBIDs.java b/src/de/lmu/ifi/dbs/elki/database/ids/generic/MergedDBIDs.java index 97743867..16d83a56 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/generic/MergedDBIDs.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/generic/MergedDBIDs.java @@ -27,6 +27,7 @@ import java.util.Iterator; import de.lmu.ifi.dbs.elki.database.ids.DBID; import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; @@ -79,7 +80,7 @@ public class MergedDBIDs implements DBIDs { } @Override - public boolean contains(DBID o) { + public boolean contains(DBIDRef o) { for(DBIDs child : childs) { if(child.contains(o)) { return true; diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/generic/UnmodifiableArrayDBIDs.java b/src/de/lmu/ifi/dbs/elki/database/ids/generic/UnmodifiableArrayDBIDs.java new file mode 100644 index 00000000..35b092c2 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/database/ids/generic/UnmodifiableArrayDBIDs.java @@ -0,0 +1,101 @@ +package de.lmu.ifi.dbs.elki.database.ids.generic; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import java.util.Iterator; + +import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; +import de.lmu.ifi.dbs.elki.database.ids.ArrayStaticDBIDs; +import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; +import de.lmu.ifi.dbs.elki.utilities.iterator.UnmodifiableIterator; + +/** + * Unmodifiable wrapper for DBIDs. + * + * @author Erich Schubert + * + * @apiviz.uses de.lmu.ifi.dbs.elki.database.ids.DBIDs + */ +public class UnmodifiableArrayDBIDs implements ArrayStaticDBIDs { + /** + * The DBIDs we wrap. + */ + final private ArrayDBIDs inner; + + /** + * Constructor. + * + * @param inner Inner DBID collection. + */ + public UnmodifiableArrayDBIDs(ArrayDBIDs inner) { + super(); + this.inner = inner; + } + + @Override + public boolean contains(DBIDRef o) { + return inner.contains(o); + } + + @Override + public boolean isEmpty() { + return inner.isEmpty(); + } + + @SuppressWarnings("deprecation") + @Override + public Iterator<DBID> iterator() { + return new UnmodifiableIterator<DBID>(inner.iterator()); + } + + @Override + public DBIDIter iter() { + return inner.iter(); + } + + @Override + public int size() { + return inner.size(); + } + + /** + * Returns a string representation of the inner DBID collection. + */ + @Override + public String toString() { + return inner.toString(); + } + + @Override + public DBID get(int i) { + return inner.get(i); + } + + @Override + public int binarySearch(DBIDRef key) { + return inner.binarySearch(key); + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/generic/UnmodifiableDBIDs.java b/src/de/lmu/ifi/dbs/elki/database/ids/generic/UnmodifiableDBIDs.java index bd68e2fb..682b2e4b 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/generic/UnmodifiableDBIDs.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/generic/UnmodifiableDBIDs.java @@ -27,6 +27,7 @@ import java.util.Iterator; import de.lmu.ifi.dbs.elki.database.ids.DBID; import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.StaticDBIDs; import de.lmu.ifi.dbs.elki.utilities.iterator.UnmodifiableIterator; @@ -55,7 +56,7 @@ public class UnmodifiableDBIDs implements StaticDBIDs { } @Override - public boolean contains(DBID o) { + public boolean contains(DBIDRef o) { return inner.contains(o); } @@ -64,6 +65,7 @@ public class UnmodifiableDBIDs implements StaticDBIDs { return inner.isEmpty(); } + @SuppressWarnings("deprecation") @Override public Iterator<DBID> iterator() { return new UnmodifiableIterator<DBID>(inner.iterator()); diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/integer/IntegerArrayStaticDBIDs.java b/src/de/lmu/ifi/dbs/elki/database/ids/integer/IntegerArrayStaticDBIDs.java index ef4aee94..90ab5a6a 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/integer/IntegerArrayStaticDBIDs.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/integer/IntegerArrayStaticDBIDs.java @@ -31,6 +31,8 @@ import de.lmu.ifi.dbs.elki.database.ids.ArrayStaticDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; import de.lmu.ifi.dbs.elki.database.ids.DBIDFactory; import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; +import de.lmu.ifi.dbs.elki.logging.LoggingUtil; /** * Static (no modifications allowed) set of Database Object IDs. @@ -123,6 +125,24 @@ public class IntegerArrayStaticDBIDs extends AbstractList<DBID> implements Array return new IntegerDBID(ids[pos]); } + @Override + public boolean sameDBID(DBIDRef other) { + return ids[pos] == other.getIntegerID(); + } + + @Override + public boolean equals(Object other) { + if (other instanceof DBID) { + LoggingUtil.warning("Programming error detected: DBIDItr.equals(DBID). Use sameDBID()!", new Throwable()); + } + return super.equals(other); + } + + @Override + public int compareDBID(DBIDRef o) { + int anotherVal = o.getIntegerID(); + return (ids[pos] < anotherVal ? -1 : (ids[pos] == anotherVal ? 0 : 1)); + } } @Override @@ -131,7 +151,7 @@ public class IntegerArrayStaticDBIDs extends AbstractList<DBID> implements Array } @Override - public boolean contains(DBID o) { + public boolean contains(DBIDRef o) { final int oid = o.getIntegerID(); for(int i = 0; i < ids.length; i++) { if(ids[i] == oid) { @@ -164,7 +184,7 @@ public class IntegerArrayStaticDBIDs extends AbstractList<DBID> implements Array } @Override - public int binarySearch(DBID key) { + public int binarySearch(DBIDRef key) { return Arrays.binarySearch(ids, key.getIntegerID()); } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/integer/IntegerDBID.java b/src/de/lmu/ifi/dbs/elki/database/ids/integer/IntegerDBID.java index f9e83294..66c1f980 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/integer/IntegerDBID.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/integer/IntegerDBID.java @@ -28,6 +28,8 @@ import java.util.Iterator; import de.lmu.ifi.dbs.elki.database.ids.DBID; import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; +import de.lmu.ifi.dbs.elki.logging.LoggingUtil; import de.lmu.ifi.dbs.elki.persistent.ByteArrayUtil; import de.lmu.ifi.dbs.elki.persistent.ByteBufferSerializer; import de.lmu.ifi.dbs.elki.persistent.FixedSizeByteBufferSerializer; @@ -49,7 +51,7 @@ import de.lmu.ifi.dbs.elki.persistent.FixedSizeByteBufferSerializer; * @apiviz.composedOf DynamicSerializer * @apiviz.composedOf StaticSerializer */ -class IntegerDBID implements DBID { +final class IntegerDBID implements DBID { /** * The actual object ID. */ @@ -75,6 +77,11 @@ class IntegerDBID implements DBID { this.id = id; } + @Override + public DBID getDBID() { + return this; + } + /** * Return the integer value of the object ID. * @@ -98,14 +105,29 @@ class IntegerDBID implements DBID { @Override public boolean equals(Object obj) { if(!(obj instanceof IntegerDBID)) { + if (obj instanceof DBIDRef) { + LoggingUtil.warning("Programming error: DBID.equals(DBIDRef) is not well-defined. use sameDBID!", new Throwable()); + } return false; } IntegerDBID other = (IntegerDBID) obj; return this.id == other.id; } + + @Override + public boolean sameDBID(DBIDRef other) { + return this.id == other.getIntegerID(); + } @Override - public int compareTo(DBID o) { + public int compareTo(DBIDRef o) { + int thisVal = this.id; + int anotherVal = o.getIntegerID(); + return (thisVal < anotherVal ? -1 : (thisVal == anotherVal ? 0 : 1)); + } + + @Override + public int compareDBID(DBIDRef o) { int thisVal = this.id; int anotherVal = o.getIntegerID(); return (thisVal < anotherVal ? -1 : (thisVal == anotherVal ? 0 : 1)); @@ -130,7 +152,7 @@ class IntegerDBID implements DBID { } @Override - public boolean contains(DBID o) { + public boolean contains(DBIDRef o) { return o.getIntegerID() == id; } @@ -140,8 +162,8 @@ class IntegerDBID implements DBID { } @Override - public int binarySearch(DBID key) { - return equals(key) ? 0 : -1; + public int binarySearch(DBIDRef key) { + return (id == key.getIntegerID()) ? 0 : -1; } /** @@ -207,6 +229,25 @@ class IntegerDBID implements DBID { public boolean valid() { return first; } + + @Override + public boolean equals(Object other) { + if (other instanceof DBID) { + LoggingUtil.warning("Programming error detected: DBIDItr.equals(DBID). Use sameDBID()!", new Throwable()); + } + return super.equals(other); + } + + @Override + public boolean sameDBID(DBIDRef other) { + return id == other.getIntegerID(); + } + + @Override + public int compareDBID(DBIDRef o) { + int anotherVal = o.getIntegerID(); + return (id < anotherVal ? -1 : (id == anotherVal ? 0 : 1)); + } } @Override diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/integer/IntegerDBIDRange.java b/src/de/lmu/ifi/dbs/elki/database/ids/integer/IntegerDBIDRange.java index debe39a4..cad771d9 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/integer/IntegerDBIDRange.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/integer/IntegerDBIDRange.java @@ -30,6 +30,7 @@ import de.lmu.ifi.dbs.elki.database.ids.DBID; import de.lmu.ifi.dbs.elki.database.ids.DBIDFactory; import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDRange; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; /** * Representing a DBID range allocation @@ -134,10 +135,25 @@ class IntegerDBIDRange extends AbstractList<DBID> implements DBIDRange { return new IntegerDBID(start + pos); } + @Override + public boolean equals(Object other) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean sameDBID(DBIDRef other) { + return start + pos == other.getIntegerID(); + } + + @Override + public int compareDBID(DBIDRef o) { + int anotherVal = o.getIntegerID(); + return (start + pos < anotherVal ? -1 : (start + pos == anotherVal ? 0 : 1)); + } } @Override - public boolean contains(DBID o) { + public boolean contains(DBIDRef o) { int oid = o.getIntegerID(); if(oid < start) { return false; @@ -180,12 +196,12 @@ class IntegerDBIDRange extends AbstractList<DBID> implements DBIDRange { * @return array offset */ @Override - public int getOffset(DBID dbid) { + public int getOffset(DBIDRef dbid) { return dbid.getIntegerID() - start; } @Override - public int binarySearch(DBID key) { + public int binarySearch(DBIDRef key) { int keyid = key.getIntegerID(); if(keyid < start) { return -1; diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/integer/SimpleDBIDFactory.java b/src/de/lmu/ifi/dbs/elki/database/ids/integer/SimpleDBIDFactory.java index 59fa34b5..8003e4ae 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/integer/SimpleDBIDFactory.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/integer/SimpleDBIDFactory.java @@ -28,6 +28,7 @@ import de.lmu.ifi.dbs.elki.database.ids.DBID; import de.lmu.ifi.dbs.elki.database.ids.DBIDFactory; import de.lmu.ifi.dbs.elki.database.ids.DBIDPair; import de.lmu.ifi.dbs.elki.database.ids.DBIDRange; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.HashSetModifiableDBIDs; import de.lmu.ifi.dbs.elki.persistent.ByteBufferSerializer; @@ -132,7 +133,7 @@ public class SimpleDBIDFactory implements DBIDFactory { } @Override - public DBIDPair makePair(DBID first, DBID second) { + public DBIDPair makePair(DBIDRef first, DBIDRef second) { return new IntegerDBIDPair(first.getIntegerID(), second.getIntegerID()); } diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/integer/TrivialDBIDFactory.java b/src/de/lmu/ifi/dbs/elki/database/ids/integer/TrivialDBIDFactory.java index b38286fe..80f65f29 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/integer/TrivialDBIDFactory.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/integer/TrivialDBIDFactory.java @@ -30,6 +30,7 @@ import de.lmu.ifi.dbs.elki.database.ids.DBID; import de.lmu.ifi.dbs.elki.database.ids.DBIDFactory; import de.lmu.ifi.dbs.elki.database.ids.DBIDPair; import de.lmu.ifi.dbs.elki.database.ids.DBIDRange; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.HashSetModifiableDBIDs; import de.lmu.ifi.dbs.elki.persistent.ByteBufferSerializer; @@ -130,7 +131,7 @@ public class TrivialDBIDFactory implements DBIDFactory { } @Override - public DBIDPair makePair(DBID first, DBID second) { + public DBIDPair makePair(DBIDRef first, DBIDRef second) { return new IntegerDBIDPair(first.getIntegerID(), second.getIntegerID()); } diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/integer/TroveArrayDBIDs.java b/src/de/lmu/ifi/dbs/elki/database/ids/integer/TroveArrayDBIDs.java index a060c6b8..e2bfbcd5 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/integer/TroveArrayDBIDs.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/integer/TroveArrayDBIDs.java @@ -29,7 +29,9 @@ import java.util.Iterator; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; -import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDMIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; +import de.lmu.ifi.dbs.elki.logging.LoggingUtil; /** * Abstract base class for GNU Trove array based lists. @@ -53,7 +55,7 @@ public abstract class TroveArrayDBIDs implements ArrayDBIDs { } @Override - public DBIDIter iter() { + public DBIDMIter iter() { return new DBIDItr(getStore()); } @@ -73,12 +75,12 @@ public abstract class TroveArrayDBIDs implements ArrayDBIDs { } @Override - public boolean contains(DBID o) { + public boolean contains(DBIDRef o) { return getStore().contains(o.getIntegerID()); } @Override - public int binarySearch(DBID key) { + public int binarySearch(DBIDRef key) { return getStore().binarySearch(key.getIntegerID()); } @@ -89,7 +91,7 @@ public abstract class TroveArrayDBIDs implements ArrayDBIDs { * * @apiviz.exclude */ - protected static class DBIDItr implements DBIDIter { + protected static class DBIDItr implements DBIDMIter { /** * Current position */ @@ -129,5 +131,31 @@ public abstract class TroveArrayDBIDs implements ArrayDBIDs { public DBID getDBID() { return new IntegerDBID(store.get(pos)); } + + @Override + public void remove() { + store.removeAt(pos); + pos--; + } + + @Override + public boolean sameDBID(DBIDRef other) { + return store.get(pos) == other.getIntegerID(); + } + + @Override + public boolean equals(Object other) { + if (other instanceof DBID) { + LoggingUtil.warning("Programming error detected: DBIDItr.equals(DBID). Use sameDBID()!", new Throwable()); + } + return super.equals(other); + } + + @Override + public int compareDBID(DBIDRef o) { + final int thisVal = store.get(pos); + final int anotherVal = o.getIntegerID(); + return (thisVal < anotherVal ? -1 : (thisVal == anotherVal ? 0 : 1)); + } } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/integer/TroveArrayModifiableDBIDs.java b/src/de/lmu/ifi/dbs/elki/database/ids/integer/TroveArrayModifiableDBIDs.java index abcbba14..379ea8a6 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/integer/TroveArrayModifiableDBIDs.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/integer/TroveArrayModifiableDBIDs.java @@ -31,6 +31,7 @@ import java.util.Comparator; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; /** @@ -96,12 +97,12 @@ class TroveArrayModifiableDBIDs extends TroveArrayDBIDs implements ArrayModifiab } @Override - public boolean add(DBID e) { + public boolean add(DBIDRef e) { return store.add(e.getIntegerID()); } @Override - public boolean remove(DBID o) { + public boolean remove(DBIDRef o) { return store.remove(o.getIntegerID()); } @@ -141,4 +142,9 @@ class TroveArrayModifiableDBIDs extends TroveArrayDBIDs implements ArrayModifiab store.set(i, data[i].getIntegerID()); } } + + @Override + public void swap(int a, int b) { + store.set(a, store.set(b, store.get(a))); + } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/ids/integer/TroveHashSetModifiableDBIDs.java b/src/de/lmu/ifi/dbs/elki/database/ids/integer/TroveHashSetModifiableDBIDs.java index 11fe669c..12656f7b 100644 --- a/src/de/lmu/ifi/dbs/elki/database/ids/integer/TroveHashSetModifiableDBIDs.java +++ b/src/de/lmu/ifi/dbs/elki/database/ids/integer/TroveHashSetModifiableDBIDs.java @@ -7,6 +7,8 @@ import gnu.trove.impl.hash.TIntHash; import gnu.trove.set.hash.TIntHashSet; import de.lmu.ifi.dbs.elki.database.ids.DBID; import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDMIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.HashSetModifiableDBIDs; @@ -76,7 +78,7 @@ class TroveHashSetModifiableDBIDs implements HashSetModifiableDBIDs { } @Override - public DBIDIter iter() { + public DBIDMIter iter() { return new DBIDItr(store); } @@ -99,12 +101,12 @@ class TroveHashSetModifiableDBIDs implements HashSetModifiableDBIDs { } @Override - public boolean add(DBID e) { + public boolean add(DBIDRef e) { return store.add(e.getIntegerID()); } @Override - public boolean remove(DBID o) { + public boolean remove(DBIDRef o) { return store.remove(o.getIntegerID()); } @@ -142,7 +144,7 @@ class TroveHashSetModifiableDBIDs implements HashSetModifiableDBIDs { } @Override - public boolean contains(DBID o) { + public boolean contains(DBIDRef o) { return store.contains(o.getIntegerID()); } @@ -153,7 +155,7 @@ class TroveHashSetModifiableDBIDs implements HashSetModifiableDBIDs { * * @apiviz.exclude */ - protected static class DBIDItr extends THashPrimitiveIterator implements DBIDIter { + protected static class DBIDItr extends THashPrimitiveIterator implements DBIDMIter { /** * The has we access */ @@ -189,5 +191,17 @@ class TroveHashSetModifiableDBIDs implements HashSetModifiableDBIDs { public DBID getDBID() { return new IntegerDBID(hash._set[_index]); } + + @Override + public boolean sameDBID(DBIDRef other) { + return getIntegerID() == other.getIntegerID(); + } + + @Override + public int compareDBID(DBIDRef o) { + final int thisVal = hash._set[_index]; + final int anotherVal = o.getIntegerID(); + return (thisVal < anotherVal ? -1 : (thisVal == anotherVal ? 0 : 1)); + } } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/utilities/iterator/IterableIterator.java b/src/de/lmu/ifi/dbs/elki/database/query/DistanceDBIDResult.java index d416faa6..ead3eebc 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/iterator/IterableIterator.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/DistanceDBIDResult.java @@ -1,4 +1,4 @@ -package de.lmu.ifi.dbs.elki.utilities.iterator; +package de.lmu.ifi.dbs.elki.database.query; /* This file is part of ELKI: @@ -23,21 +23,19 @@ package de.lmu.ifi.dbs.elki.utilities.iterator; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.Iterator; +import java.util.List; + +import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; /** - * Interface that is both Iterable and an Iterator. - * - * Calling {@code iterator()} repeatedly MAY return the same iterator, - * e.g. the IterableIterator itself. In fact, this is the expected behavior, - * since this is just meant to allow the use of this Iterator in a {@code foreach} statement. + * Collection of objects and their distances. * * @author Erich Schubert * - * @apiviz.landmark - * - * @param <T> Data type + * @apiviz.composedOf DistanceResultPair + * + * @param <D> Distance type */ -public interface IterableIterator<T> extends Iterable<T>, Iterator<T> { - // no extra conditions +public interface DistanceDBIDResult<D extends Distance<D>> extends List<DistanceResultPair<D>> { + // Empty. TODO: add "sorted" property? } diff --git a/src/de/lmu/ifi/dbs/elki/database/query/DistanceResultPair.java b/src/de/lmu/ifi/dbs/elki/database/query/DistanceResultPair.java index e7403628..b6cc129b 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/DistanceResultPair.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/DistanceResultPair.java @@ -24,6 +24,7 @@ package de.lmu.ifi.dbs.elki.database.query; */ import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; import de.lmu.ifi.dbs.elki.utilities.pairs.PairInterface; @@ -35,7 +36,7 @@ import de.lmu.ifi.dbs.elki.utilities.pairs.PairInterface; * * @param <D> Distance type */ -public interface DistanceResultPair<D extends Distance<?>> extends PairInterface<D, DBID>, Comparable<DistanceResultPair<D>> { +public interface DistanceResultPair<D extends Distance<?>> extends PairInterface<D, DBID>, Comparable<DistanceResultPair<D>>, DBIDRef { /** * Getter for first * @@ -51,13 +52,6 @@ public interface DistanceResultPair<D extends Distance<?>> extends PairInterface public void setDistance(D first); /** - * Getter for second element in pair - * - * @return second element in pair - */ - public DBID getDBID(); - - /** * Setter for second * * @param second new value for second element diff --git a/src/de/lmu/ifi/dbs/elki/database/query/DoubleDistanceResultPair.java b/src/de/lmu/ifi/dbs/elki/database/query/DoubleDistanceResultPair.java index e0be7c40..23abc327 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/DoubleDistanceResultPair.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/DoubleDistanceResultPair.java @@ -24,6 +24,7 @@ package de.lmu.ifi.dbs.elki.database.query; */ import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.distance.distancevalue.DoubleDistance; /** @@ -71,6 +72,21 @@ public class DoubleDistanceResultPair implements DistanceResultPair<DoubleDistan } @Override + public int getIntegerID() { + return id.getIntegerID(); + } + + @Override + public boolean sameDBID(DBIDRef other) { + return id.sameDBID(other); + } + + @Override + public int compareDBID(DBIDRef other) { + return id.compareDBID(other); + } + + @Override public void setID(DBID id) { this.id = id; } @@ -127,10 +143,10 @@ public class DoubleDistanceResultPair implements DistanceResultPair<DoubleDistan } if(obj instanceof DoubleDistanceResultPair) { DoubleDistanceResultPair ddrp = (DoubleDistanceResultPair) obj; - return distance == ddrp.distance && id.equals(ddrp.id); + return distance == ddrp.distance && id.sameDBID(ddrp.id); } DistanceResultPair<?> other = (DistanceResultPair<?>) obj; - return other.getDistance().equals(distance) && id.equals(other.getDBID()); + return other.getDistance().equals(distance) && id.sameDBID(other.getDBID()); } /** diff --git a/src/de/lmu/ifi/dbs/elki/data/projection/AbstractFeatureSelection.java b/src/de/lmu/ifi/dbs/elki/database/query/GenericDistanceDBIDList.java index fcd5fd84..2283c401 100644 --- a/src/de/lmu/ifi/dbs/elki/data/projection/AbstractFeatureSelection.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/GenericDistanceDBIDList.java @@ -1,6 +1,5 @@ -package de.lmu.ifi.dbs.elki.data.projection; +package de.lmu.ifi.dbs.elki.database.query; -import de.lmu.ifi.dbs.elki.data.FeatureVector; /* This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures @@ -23,42 +22,47 @@ import de.lmu.ifi.dbs.elki.data.FeatureVector; You should have received a copy of the GNU Affero General Public License along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; -import de.lmu.ifi.dbs.elki.data.type.TypeInformation; -import de.lmu.ifi.dbs.elki.utilities.datastructures.arraylike.SubsetArrayAdapter; + +import java.util.ArrayList; +import java.util.Collection; + +import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; /** - * Abstract feature selection projection + * Default class to keep a list of distance-object pairs. * * @author Erich Schubert * - * @param <V> Vector type - * @param <F> Feature type + * @param <D> Distance type */ -public abstract class AbstractFeatureSelection<V extends FeatureVector<V, F>, F> implements Projection<V, V> { +public class GenericDistanceDBIDList<D extends Distance<D>> extends ArrayList<DistanceResultPair<D>> implements DistanceDBIDResult<D> { /** - * Array adapter + * Serialization Version */ - protected SubsetArrayAdapter<F, V> adapter; + private static final long serialVersionUID = 1L; /** * Constructor. - * - * @param adapter Data adapter */ - public AbstractFeatureSelection(SubsetArrayAdapter<F, V> adapter) { + public GenericDistanceDBIDList() { super(); - this.adapter = adapter; } - @Override - public V project(V data) { - return data.newFeatureVector(data, adapter); + /** + * Constructor. + * + * @param c existing collection + */ + public GenericDistanceDBIDList(Collection<? extends DistanceResultPair<D>> c) { + super(c); } - @Override - abstract public SimpleTypeInformation<V> getOutputDataTypeInformation(); - - @Override - abstract public TypeInformation getInputDataTypeInformation(); + /** + * Constructor. + * + * @param initialCapacity Capacity + */ + public GenericDistanceDBIDList(int initialCapacity) { + super(initialCapacity); + } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/query/GenericDistanceResultPair.java b/src/de/lmu/ifi/dbs/elki/database/query/GenericDistanceResultPair.java index 4b838560..414f145a 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/GenericDistanceResultPair.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/GenericDistanceResultPair.java @@ -24,6 +24,7 @@ package de.lmu.ifi.dbs.elki.database.query; */ import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; import de.lmu.ifi.dbs.elki.utilities.pairs.Pair; @@ -74,6 +75,21 @@ public class GenericDistanceResultPair<D extends Distance<D>> extends Pair<D, DB public final DBID getDBID() { return second; } + + @Override + public int getIntegerID() { + return second.getIntegerID(); + } + + @Override + public boolean sameDBID(DBIDRef other) { + return second.sameDBID(other); + } + + @Override + public int compareDBID(DBIDRef other) { + return second.compareDBID(other); + } /** * Setter for second @@ -105,7 +121,7 @@ public class GenericDistanceResultPair<D extends Distance<D>> extends Pair<D, DB return false; } DistanceResultPair<?> other = (DistanceResultPair<?>) obj; - return first.equals(other.getDistance()) && second.equals(other.getDBID()); + return first.equals(other.getDistance()) && second.sameDBID(other.getDBID()); } @Override diff --git a/src/de/lmu/ifi/dbs/elki/database/query/distance/AbstractDatabaseDistanceQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/distance/AbstractDatabaseDistanceQuery.java index 92c116e3..ed1ee50c 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/distance/AbstractDatabaseDistanceQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/distance/AbstractDatabaseDistanceQuery.java @@ -24,6 +24,7 @@ package de.lmu.ifi.dbs.elki.database.query.distance; */ import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; @@ -46,7 +47,7 @@ public abstract class AbstractDatabaseDistanceQuery<O, D extends Distance<D>> ex } @Override - public D distance(O o1, DBID id2) { + public D distance(O o1, DBIDRef id2) { if(o1 instanceof DBID) { return distance((DBID) o1, id2); } @@ -54,7 +55,7 @@ public abstract class AbstractDatabaseDistanceQuery<O, D extends Distance<D>> ex } @Override - public D distance(DBID id1, O o2) { + public D distance(DBIDRef id1, O o2) { if(o2 instanceof DBID) { return distance(id1, (DBID) o2); } diff --git a/src/de/lmu/ifi/dbs/elki/database/query/distance/AbstractDistanceQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/distance/AbstractDistanceQuery.java index 10dde0ad..b85ddef0 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/distance/AbstractDistanceQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/distance/AbstractDistanceQuery.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.database.query.distance; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.AbstractDataBasedQuery; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; @@ -55,7 +55,7 @@ public abstract class AbstractDistanceQuery<O, D extends Distance<D>> extends Ab * @return the distance between the two objects specified by their object ids */ @Override - public abstract D distance(DBID id1, DBID id2); + public abstract D distance(DBIDRef id1, DBIDRef id2); /** * Returns the distance between the two objects specified by their object ids. @@ -65,7 +65,7 @@ public abstract class AbstractDistanceQuery<O, D extends Distance<D>> extends Ab * @return the distance between the two objects specified by their object ids */ @Override - public abstract D distance(O o1, DBID id2); + public abstract D distance(O o1, DBIDRef id2); /** * Returns the distance between the two objects specified by their object ids. @@ -75,7 +75,7 @@ public abstract class AbstractDistanceQuery<O, D extends Distance<D>> extends Ab * @return the distance between the two objects specified by their object ids */ @Override - public abstract D distance(DBID id1, O o2); + public abstract D distance(DBIDRef id1, O o2); /** * Returns the distance between the two objects specified by their object ids. diff --git a/src/de/lmu/ifi/dbs/elki/database/query/distance/DBIDDistanceQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/distance/DBIDDistanceQuery.java index 2bb45623..28ca9b16 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/distance/DBIDDistanceQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/distance/DBIDDistanceQuery.java @@ -24,6 +24,7 @@ package de.lmu.ifi.dbs.elki.database.query.distance; */ import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.DBIDDistanceFunction; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; @@ -56,7 +57,7 @@ public class DBIDDistanceQuery<D extends Distance<D>> extends AbstractDatabaseDi } @Override - public D distance(DBID id1, DBID id2) { + public D distance(DBIDRef id1, DBIDRef id2) { if(id1 == null) { throw new UnsupportedOperationException("This distance function can only be used for objects stored in the database."); } diff --git a/src/de/lmu/ifi/dbs/elki/database/query/distance/DistanceQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/distance/DistanceQuery.java index 1986b246..c026162d 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/distance/DistanceQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/distance/DistanceQuery.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.database.query.distance; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.DatabaseQuery; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction; @@ -48,7 +48,7 @@ public interface DistanceQuery<O, D extends Distance<?>> extends DatabaseQuery { * @param id2 second object id * @return the distance between the two objects specified by their object ids */ - D distance(DBID id1, DBID id2); + D distance(DBIDRef id1, DBIDRef id2); /** * Returns the distance between the two objects specified by their object ids. @@ -57,7 +57,7 @@ public interface DistanceQuery<O, D extends Distance<?>> extends DatabaseQuery { * @param id2 second object id * @return the distance between the two objects specified by their object ids */ - D distance(O o1, DBID id2); + D distance(O o1, DBIDRef id2); /** * Returns the distance between the two objects specified by their object ids. @@ -66,7 +66,7 @@ public interface DistanceQuery<O, D extends Distance<?>> extends DatabaseQuery { * @param o2 second object * @return the distance between the two objects specified by their object ids */ - D distance(DBID id1, O o2); + D distance(DBIDRef id1, O o2); /** * Returns the distance between the two objects specified by their object ids. diff --git a/src/de/lmu/ifi/dbs/elki/database/query/distance/PrimitiveDistanceQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/distance/PrimitiveDistanceQuery.java index 5015dee1..75bd9e39 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/distance/PrimitiveDistanceQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/distance/PrimitiveDistanceQuery.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.database.query.distance; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDistanceFunction; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; @@ -57,20 +57,20 @@ public class PrimitiveDistanceQuery<O, D extends Distance<D>> extends AbstractDi } @Override - public D distance(DBID id1, DBID id2) { + public D distance(DBIDRef id1, DBIDRef id2) { O o1 = relation.get(id1); O o2 = relation.get(id2); return distance(o1, o2); } @Override - public D distance(O o1, DBID id2) { + public D distance(O o1, DBIDRef id2) { O o2 = relation.get(id2); return distance(o1, o2); } @Override - public D distance(DBID id1, O o2) { + public D distance(DBIDRef id1, O o2) { O o1 = relation.get(id1); return distance(o1, o2); } diff --git a/src/de/lmu/ifi/dbs/elki/database/query/distance/PrimitiveDistanceSimilarityQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/distance/PrimitiveDistanceSimilarityQuery.java index 4988c925..930d195c 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/distance/PrimitiveDistanceSimilarityQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/distance/PrimitiveDistanceSimilarityQuery.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.database.query.distance; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.DistanceSimilarityQuery; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDistanceFunction; @@ -61,20 +61,20 @@ public class PrimitiveDistanceSimilarityQuery<O, D extends Distance<D>> extends } @Override - public D similarity(DBID id1, DBID id2) { + public D similarity(DBIDRef id1, DBIDRef id2) { O o1 = relation.get(id1); O o2 = relation.get(id2); return similarity(o1, o2); } @Override - public D similarity(O o1, DBID id2) { + public D similarity(O o1, DBIDRef id2) { O o2 = relation.get(id2); return similarity(o1, o2); } @Override - public D similarity(DBID id1, O o2) { + public D similarity(DBIDRef id1, O o2) { O o1 = relation.get(id1); return similarity(o1, o2); } diff --git a/src/de/lmu/ifi/dbs/elki/database/query/knn/AbstractDistanceKNNQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/knn/AbstractDistanceKNNQuery.java index 731148e8..e5df1452 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/knn/AbstractDistanceKNNQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/knn/AbstractDistanceKNNQuery.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.database.query.knn; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.AbstractDataBasedQuery; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; @@ -50,7 +50,7 @@ public abstract class AbstractDistanceKNNQuery<O, D extends Distance<D>> extends } @Override - abstract public KNNResult<D> getKNNForDBID(DBID id, int k); + abstract public KNNResult<D> getKNNForDBID(DBIDRef id, int k); @Override abstract public KNNResult<D> getKNNForObject(O obj, int k); diff --git a/src/de/lmu/ifi/dbs/elki/database/query/knn/KNNQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/knn/KNNQuery.java index 80d91180..13bf58c2 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/knn/KNNQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/knn/KNNQuery.java @@ -28,6 +28,7 @@ import java.util.Map; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.DatabaseQuery; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; import de.lmu.ifi.dbs.elki.utilities.datastructures.heap.KNNHeap; @@ -51,7 +52,7 @@ public interface KNNQuery<O, D extends Distance<D>> extends DatabaseQuery { * @param k Number of neighbors requested * @return neighbors */ - public KNNResult<D> getKNNForDBID(DBID id, int k); + public KNNResult<D> getKNNForDBID(DBIDRef id, int k); /** * Bulk query method diff --git a/src/de/lmu/ifi/dbs/elki/database/query/knn/KNNResult.java b/src/de/lmu/ifi/dbs/elki/database/query/knn/KNNResult.java index d45fd8d7..e7612fcc 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/knn/KNNResult.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/knn/KNNResult.java @@ -23,10 +23,10 @@ package de.lmu.ifi.dbs.elki.database.query.knn; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.Collection; import java.util.List; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; +import de.lmu.ifi.dbs.elki.database.query.DistanceDBIDResult; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; @@ -39,7 +39,7 @@ import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; * * @apiviz.composedOf DistanceResultPair */ -public interface KNNResult<D extends Distance<D>> extends Collection<DistanceResultPair<D>> { +public interface KNNResult<D extends Distance<D>> extends DistanceDBIDResult<D> { /** * Size */ @@ -58,6 +58,7 @@ public interface KNNResult<D extends Distance<D>> extends Collection<DistanceRes * * @param index */ + @Override public DistanceResultPair<D> get(int index); /** diff --git a/src/de/lmu/ifi/dbs/elki/database/query/knn/KNNUtil.java b/src/de/lmu/ifi/dbs/elki/database/query/knn/KNNUtil.java index 1179edb5..01deeaa0 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/knn/KNNUtil.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/knn/KNNUtil.java @@ -23,7 +23,6 @@ package de.lmu.ifi.dbs.elki.database.query.knn; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.AbstractCollection; import java.util.AbstractList; import java.util.Iterator; import java.util.List; @@ -31,6 +30,7 @@ import java.util.List; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; @@ -49,7 +49,7 @@ public final class KNNUtil { * * @param <D> Distance */ - protected static class KNNSubList<D extends Distance<D>> extends AbstractCollection<DistanceResultPair<D>> implements KNNResult<D> { + protected static class KNNSubList<D extends Distance<D>> extends AbstractList<DistanceResultPair<D>> implements KNNResult<D> { /** * Parameter k */ @@ -240,6 +240,18 @@ public final class KNNUtil { public DBID getDBID() { return cur.getDBID(); } + + @Override + public boolean sameDBID(DBIDRef other) { + return getIntegerID() == other.getIntegerID(); + } + + @Override + public int compareDBID(DBIDRef o) { + final int thisVal = getIntegerID(); + final int anotherVal = o.getIntegerID(); + return (thisVal < anotherVal ? -1 : (thisVal == anotherVal ? 0 : 1)); + } } /** @@ -284,9 +296,9 @@ public final class KNNUtil { } @Override - public boolean contains(DBID o) { - for(DBID id : this) { - if(id.equals(o)) { + public boolean contains(DBIDRef o) { + for (DBIDIter iter = iter(); iter.valid(); iter.advance()) { + if(iter.sameDBID(o)) { return true; } } @@ -304,7 +316,7 @@ public final class KNNUtil { */ @Override @Deprecated - public int binarySearch(DBID key) { + public int binarySearch(DBIDRef key) { throw new UnsupportedOperationException("Since the result is usually not sorted, a binary Search does not make sense!"); } } diff --git a/src/de/lmu/ifi/dbs/elki/database/query/knn/LinearScanKNNQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/knn/LinearScanKNNQuery.java index d6492f43..b51dad4c 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/knn/LinearScanKNNQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/knn/LinearScanKNNQuery.java @@ -31,6 +31,8 @@ import java.util.Map.Entry; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.query.LinearScanQuery; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; @@ -64,28 +66,29 @@ public class LinearScanKNNQuery<O, D extends Distance<D>> extends AbstractDistan */ private void linearScanBatchKNN(ArrayDBIDs ids, List<KNNHeap<D>> heaps) { // The distance is computed on database IDs - for(DBID candidateID : relation.iterDBIDs()) { - Integer index = -1; - for(DBID id : ids) { - index++; + for(DBIDIter iter = relation.getDBIDs().iter(); iter.valid(); iter.advance()) { + DBID candidateID = iter.getDBID(); + int index = 0; + for (DBIDIter iter2 = ids.iter(); iter2.valid(); iter2.advance()) { KNNHeap<D> heap = heaps.get(index); - heap.add(distanceQuery.distance(id, candidateID), candidateID); + heap.add(distanceQuery.distance(iter2, candidateID), candidateID); + index++; } } } @Override - public KNNResult<D> getKNNForDBID(DBID id, int k) { + public KNNResult<D> getKNNForDBID(DBIDRef id, int k) { KNNHeap<D> heap = new KNNHeap<D>(k); if(PrimitiveDistanceQuery.class.isInstance(distanceQuery)) { O obj = relation.get(id); - for(DBID candidateID : relation.iterDBIDs()) { - heap.add(distanceQuery.distance(obj, relation.get(candidateID)), candidateID); + for(DBIDIter iter = relation.getDBIDs().iter(); iter.valid(); iter.advance()) { + heap.add(distanceQuery.distance(obj, relation.get(iter)), iter); } } else { - for(DBID candidateID : relation.iterDBIDs()) { - heap.add(distanceQuery.distance(id, candidateID), candidateID); + for(DBIDIter iter = relation.getDBIDs().iter(); iter.valid(); iter.advance()) { + heap.add(distanceQuery.distance(id, iter), iter); } } return heap.toKNNList(); @@ -122,9 +125,8 @@ public class LinearScanKNNQuery<O, D extends Distance<D>> extends AbstractDistan @Override public KNNResult<D> getKNNForObject(O obj, int k) { KNNHeap<D> heap = new KNNHeap<D>(k); - for(DBID candidateID : relation.iterDBIDs()) { - O candidate = relation.get(candidateID); - heap.add(distanceQuery.distance(obj, candidate), candidateID); + for(DBIDIter iter = relation.getDBIDs().iter(); iter.valid(); iter.advance()) { + heap.add(distanceQuery.distance(obj, iter), iter); } return heap.toKNNList(); } diff --git a/src/de/lmu/ifi/dbs/elki/database/query/knn/LinearScanPrimitiveDistanceKNNQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/knn/LinearScanPrimitiveDistanceKNNQuery.java index 07632a34..59987282 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/knn/LinearScanPrimitiveDistanceKNNQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/knn/LinearScanPrimitiveDistanceKNNQuery.java @@ -30,6 +30,8 @@ import java.util.Map.Entry; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.distance.PrimitiveDistanceQuery; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; import de.lmu.ifi.dbs.elki.utilities.datastructures.heap.KNNHeap; @@ -63,7 +65,8 @@ public class LinearScanPrimitiveDistanceKNNQuery<O, D extends Distance<D>> exten protected void linearScanBatchKNN(List<O> objs, List<KNNHeap<D>> heaps) { final int size = objs.size(); // Linear scan style KNN. - for(DBID candidateID : relation.iterDBIDs()) { + for(DBIDIter iter = relation.getDBIDs().iter(); iter.valid(); iter.advance()) { + DBID candidateID = iter.getDBID(); O candidate = relation.get(candidateID); for(int index = 0; index < size; index++) { O object = objs.get(index); @@ -74,7 +77,7 @@ public class LinearScanPrimitiveDistanceKNNQuery<O, D extends Distance<D>> exten } @Override - public KNNResult<D> getKNNForDBID(DBID id, int k) { + public KNNResult<D> getKNNForDBID(DBIDRef id, int k) { return getKNNForObject(relation.get(id), k); } @@ -83,9 +86,9 @@ public class LinearScanPrimitiveDistanceKNNQuery<O, D extends Distance<D>> exten final int size = ids.size(); final List<KNNHeap<D>> heaps = new ArrayList<KNNHeap<D>>(size); List<O> objs = new ArrayList<O>(size); - for(DBID id : ids) { + for(DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { heaps.add(new KNNHeap<D>(k)); - objs.add(relation.get(id)); + objs.add(relation.get(iter)); } linearScanBatchKNN(objs, heaps); diff --git a/src/de/lmu/ifi/dbs/elki/database/query/knn/LinearScanRawDoubleDistanceKNNQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/knn/LinearScanRawDoubleDistanceKNNQuery.java index 3f7dc04f..a10aaac5 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/knn/LinearScanRawDoubleDistanceKNNQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/knn/LinearScanRawDoubleDistanceKNNQuery.java @@ -26,7 +26,8 @@ package de.lmu.ifi.dbs.elki.database.query.knn; import java.util.Arrays; import java.util.List; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.DoubleDistanceResultPair; import de.lmu.ifi.dbs.elki.database.query.distance.PrimitiveDistanceQuery; import de.lmu.ifi.dbs.elki.distance.distancefunction.PrimitiveDoubleDistanceFunction; @@ -56,7 +57,7 @@ public class LinearScanRawDoubleDistanceKNNQuery<O> extends LinearScanPrimitiveD } @Override - public KNNResult<DoubleDistance> getKNNForDBID(DBID id, int k) { + public KNNResult<DoubleDistance> getKNNForDBID(DBIDRef id, int k) { return getKNNForObject(relation.get(id), k); } @@ -67,10 +68,10 @@ public class LinearScanRawDoubleDistanceKNNQuery<O> extends LinearScanPrimitiveD // Optimization for double distances. final KNNHeap<DoubleDistance> heap = new KNNHeap<DoubleDistance>(k); double max = Double.POSITIVE_INFINITY; - for(DBID candidateID : relation.iterDBIDs()) { - final double doubleDistance = rawdist.doubleDistance(obj, relation.get(candidateID)); + for(DBIDIter iter = relation.getDBIDs().iter(); iter.valid(); iter.advance()) { + final double doubleDistance = rawdist.doubleDistance(obj, relation.get(iter)); if(doubleDistance <= max) { - heap.add(new DoubleDistanceResultPair(doubleDistance, candidateID)); + heap.add(new DoubleDistanceResultPair(doubleDistance, iter.getDBID())); // Update cutoff if(heap.size() >= heap.getK()) { max = ((DoubleDistanceResultPair) heap.peek()).getDoubleDistance(); @@ -91,13 +92,13 @@ public class LinearScanRawDoubleDistanceKNNQuery<O> extends LinearScanPrimitiveD // The distance is computed on arbitrary vectors, we can reduce object // loading by working on the actual vectors. - for(DBID candidateID : relation.iterDBIDs()) { - O candidate = relation.get(candidateID); + for(DBIDIter iter = relation.getDBIDs().iter(); iter.valid(); iter.advance()) { + O candidate = relation.get(iter); for(int index = 0; index < size; index++) { final KNNHeap<DoubleDistance> heap = heaps.get(index); double doubleDistance = rawdist.doubleDistance(objs.get(index), candidate); if(doubleDistance <= max[index]) { - heap.add(new DoubleDistanceResultPair(doubleDistance, candidateID)); + heap.add(new DoubleDistanceResultPair(doubleDistance, iter.getDBID())); if(heap.size() >= heap.getK()) { max[index] = ((DoubleDistanceResultPair) heap.peek()).getDoubleDistance(); } diff --git a/src/de/lmu/ifi/dbs/elki/database/query/knn/PreprocessorKNNQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/knn/PreprocessorKNNQuery.java index 93321609..09f93945 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/knn/PreprocessorKNNQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/knn/PreprocessorKNNQuery.java @@ -30,6 +30,8 @@ import java.util.Map.Entry; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.AbstractDataBasedQuery; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.database.relation.Relation; @@ -77,7 +79,7 @@ public class PreprocessorKNNQuery<O, D extends Distance<D>, T extends KNNResult< } @Override - public KNNResult<D> getKNNForDBID(DBID id, int k) { + public KNNResult<D> getKNNForDBID(DBIDRef id, int k) { if(!warned && k > preprocessor.getK()) { LoggingUtil.warning("Requested more neighbors than preprocessed!"); } @@ -112,8 +114,8 @@ public class PreprocessorKNNQuery<O, D extends Distance<D>, T extends KNNResult< } List<KNNResult<D>> result = new ArrayList<KNNResult<D>>(ids.size()); if(k < preprocessor.getK()) { - for(DBID id : ids) { - KNNResult<D> dr = preprocessor.get(id); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + KNNResult<D> dr = preprocessor.get(iter); int subk = k; D kdist = dr.get(subk - 1).getDistance(); while(subk < dr.size()) { @@ -135,8 +137,8 @@ public class PreprocessorKNNQuery<O, D extends Distance<D>, T extends KNNResult< } } else { - for(DBID id : ids) { - result.add(preprocessor.get(id)); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + result.add(preprocessor.get(iter)); } } return result; diff --git a/src/de/lmu/ifi/dbs/elki/database/query/range/AbstractDistanceRangeQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/range/AbstractDistanceRangeQuery.java index 60426034..871c5e9e 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/range/AbstractDistanceRangeQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/range/AbstractDistanceRangeQuery.java @@ -23,11 +23,9 @@ package de.lmu.ifi.dbs.elki.database.query.range; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.List; - -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.AbstractDataBasedQuery; -import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; +import de.lmu.ifi.dbs.elki.database.query.DistanceDBIDResult; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; @@ -57,8 +55,8 @@ public abstract class AbstractDistanceRangeQuery<O, D extends Distance<D>> exten } @Override - abstract public List<DistanceResultPair<D>> getRangeForDBID(DBID id, D range); + abstract public DistanceDBIDResult<D> getRangeForDBID(DBIDRef id, D range); @Override - abstract public List<DistanceResultPair<D>> getRangeForObject(O obj, D range); + abstract public DistanceDBIDResult<D> getRangeForObject(O obj, D range); }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/query/range/LinearScanPrimitiveDistanceRangeQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/range/LinearScanPrimitiveDistanceRangeQuery.java index dfe0b581..758d603e 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/range/LinearScanPrimitiveDistanceRangeQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/range/LinearScanPrimitiveDistanceRangeQuery.java @@ -23,10 +23,8 @@ package de.lmu.ifi.dbs.elki.database.query.range; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.List; - -import de.lmu.ifi.dbs.elki.database.ids.DBID; -import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; +import de.lmu.ifi.dbs.elki.database.query.DistanceDBIDResult; import de.lmu.ifi.dbs.elki.database.query.distance.PrimitiveDistanceQuery; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; @@ -54,7 +52,8 @@ public class LinearScanPrimitiveDistanceRangeQuery<O, D extends Distance<D>> ext } @Override - public List<DistanceResultPair<D>> getRangeForDBID(DBID id, D range) { + public DistanceDBIDResult<D> getRangeForDBID(DBIDRef id, D range) { + // Note: subtle optimization. Get "id" only once! return getRangeForObject(relation.get(id), range); } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/query/range/LinearScanRangeQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/range/LinearScanRangeQuery.java index a7bb7db9..c8ddb1ec 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/range/LinearScanRangeQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/range/LinearScanRangeQuery.java @@ -23,12 +23,12 @@ package de.lmu.ifi.dbs.elki.database.query.range; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.ArrayList; import java.util.Collections; -import java.util.List; -import de.lmu.ifi.dbs.elki.database.ids.DBID; -import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; +import de.lmu.ifi.dbs.elki.database.query.DistanceDBIDResult; +import de.lmu.ifi.dbs.elki.database.query.GenericDistanceDBIDList; import de.lmu.ifi.dbs.elki.database.query.GenericDistanceResultPair; import de.lmu.ifi.dbs.elki.database.query.LinearScanQuery; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; @@ -56,12 +56,12 @@ public class LinearScanRangeQuery<O, D extends Distance<D>> extends AbstractDist } @Override - public List<DistanceResultPair<D>> getRangeForDBID(DBID id, D range) { - List<DistanceResultPair<D>> result = new ArrayList<DistanceResultPair<D>>(); - for(DBID currentID : relation.iterDBIDs()) { - D currentDistance = distanceQuery.distance(id, currentID); + public DistanceDBIDResult<D> getRangeForDBID(DBIDRef id, D range) { + GenericDistanceDBIDList<D> result = new GenericDistanceDBIDList<D>(); + for(DBIDIter iter = relation.getDBIDs().iter(); iter.valid(); iter.advance()) { + D currentDistance = distanceQuery.distance(id, iter); if(currentDistance.compareTo(range) <= 0) { - result.add(new GenericDistanceResultPair<D>(currentDistance, currentID)); + result.add(new GenericDistanceResultPair<D>(currentDistance, iter.getDBID())); } } Collections.sort(result); @@ -69,12 +69,12 @@ public class LinearScanRangeQuery<O, D extends Distance<D>> extends AbstractDist } @Override - public List<DistanceResultPair<D>> getRangeForObject(O obj, D range) { - List<DistanceResultPair<D>> result = new ArrayList<DistanceResultPair<D>>(); - for(DBID currentID : relation.iterDBIDs()) { - D currentDistance = distanceQuery.distance(currentID, obj); + public DistanceDBIDResult<D> getRangeForObject(O obj, D range) { + GenericDistanceDBIDList<D> result = new GenericDistanceDBIDList<D>(); + for(DBIDIter iter = relation.getDBIDs().iter(); iter.valid(); iter.advance()) { + D currentDistance = distanceQuery.distance(obj, iter); if(currentDistance.compareTo(range) <= 0) { - result.add(new GenericDistanceResultPair<D>(currentDistance, currentID)); + result.add(new GenericDistanceResultPair<D>(currentDistance, iter.getDBID())); } } Collections.sort(result); diff --git a/src/de/lmu/ifi/dbs/elki/database/query/range/LinearScanRawDoubleDistanceRangeQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/range/LinearScanRawDoubleDistanceRangeQuery.java index 18d1d173..c5ce2f55 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/range/LinearScanRawDoubleDistanceRangeQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/range/LinearScanRawDoubleDistanceRangeQuery.java @@ -23,13 +23,13 @@ package de.lmu.ifi.dbs.elki.database.query.range; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.ArrayList; import java.util.Collections; -import java.util.List; -import de.lmu.ifi.dbs.elki.database.ids.DBID; -import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; +import de.lmu.ifi.dbs.elki.database.query.DistanceDBIDResult; import de.lmu.ifi.dbs.elki.database.query.DoubleDistanceResultPair; +import de.lmu.ifi.dbs.elki.database.query.GenericDistanceDBIDList; import de.lmu.ifi.dbs.elki.database.query.LinearScanQuery; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.query.distance.PrimitiveDistanceQuery; @@ -56,18 +56,18 @@ public class LinearScanRawDoubleDistanceRangeQuery<O> extends LinearScanRangeQue } @Override - public List<DistanceResultPair<DoubleDistance>> getRangeForDBID(DBID id, DoubleDistance range) { + public DistanceDBIDResult<DoubleDistance> getRangeForDBID(DBIDRef id, DoubleDistance range) { if(distanceQuery instanceof PrimitiveDistanceQuery && distanceQuery.getDistanceFunction() instanceof PrimitiveDoubleDistanceFunction) { @SuppressWarnings("unchecked") PrimitiveDoubleDistanceFunction<O> rawdist = (PrimitiveDoubleDistanceFunction<O>) distanceQuery.getDistanceFunction(); double epsilon = range.doubleValue(); O qo = relation.get(id); - List<DistanceResultPair<DoubleDistance>> result = new ArrayList<DistanceResultPair<DoubleDistance>>(); - for(DBID currentID : relation.iterDBIDs()) { - double doubleDistance = rawdist.doubleDistance(qo, relation.get(currentID)); + GenericDistanceDBIDList<DoubleDistance> result = new GenericDistanceDBIDList<DoubleDistance>(); + for(DBIDIter iter = relation.getDBIDs().iter(); iter.valid(); iter.advance()) { + double doubleDistance = rawdist.doubleDistance(qo, relation.get(iter)); if(doubleDistance <= epsilon) { - result.add(new DoubleDistanceResultPair(doubleDistance, currentID)); + result.add(new DoubleDistanceResultPair(doubleDistance, iter.getDBID())); } } Collections.sort(result); @@ -79,17 +79,17 @@ public class LinearScanRawDoubleDistanceRangeQuery<O> extends LinearScanRangeQue } @Override - public List<DistanceResultPair<DoubleDistance>> getRangeForObject(O obj, DoubleDistance range) { + public DistanceDBIDResult<DoubleDistance> getRangeForObject(O obj, DoubleDistance range) { if(distanceQuery instanceof PrimitiveDistanceQuery && distanceQuery.getDistanceFunction() instanceof PrimitiveDoubleDistanceFunction) { @SuppressWarnings("unchecked") PrimitiveDoubleDistanceFunction<O> rawdist = (PrimitiveDoubleDistanceFunction<O>) distanceQuery.getDistanceFunction(); double epsilon = range.doubleValue(); - List<DistanceResultPair<DoubleDistance>> result = new ArrayList<DistanceResultPair<DoubleDistance>>(); - for(DBID currentID : relation.iterDBIDs()) { - double doubleDistance = rawdist.doubleDistance(obj, relation.get(currentID)); + GenericDistanceDBIDList<DoubleDistance> result = new GenericDistanceDBIDList<DoubleDistance>(); + for(DBIDIter iter = relation.getDBIDs().iter(); iter.valid(); iter.advance()) { + double doubleDistance = rawdist.doubleDistance(obj, relation.get(iter)); if(doubleDistance <= epsilon) { - result.add(new DoubleDistanceResultPair(doubleDistance, currentID)); + result.add(new DoubleDistanceResultPair(doubleDistance, iter.getDBID())); } } Collections.sort(result); diff --git a/src/de/lmu/ifi/dbs/elki/database/query/range/RangeQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/range/RangeQuery.java index c2dbecd6..2c6842bf 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/range/RangeQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/range/RangeQuery.java @@ -23,11 +23,9 @@ package de.lmu.ifi.dbs.elki.database.query.range; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.List; - -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.DatabaseQuery; -import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; +import de.lmu.ifi.dbs.elki.database.query.DistanceDBIDResult; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; /** @@ -36,7 +34,7 @@ import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; * @author Erich Schubert * * @apiviz.landmark - * @apiviz.uses DistanceResultPair oneway - - «create» + * @apiviz.uses DistanceDBIDResult oneway - - «create» * * @param <O> Object type * @param <D> Distance type @@ -49,7 +47,7 @@ public interface RangeQuery<O, D extends Distance<D>> extends DatabaseQuery { * @param range Query range * @return neighbors */ - public List<DistanceResultPair<D>> getRangeForDBID(DBID id, D range); + public DistanceDBIDResult<D> getRangeForDBID(DBIDRef id, D range); /** * Get the nearest neighbors for a particular object in a given query range @@ -58,5 +56,5 @@ public interface RangeQuery<O, D extends Distance<D>> extends DatabaseQuery { * @param range Query range * @return neighbors */ - public List<DistanceResultPair<D>> getRangeForObject(O obj, D range); + public DistanceDBIDResult<D> getRangeForObject(O obj, D range); }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/query/rknn/AbstractRKNNQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/rknn/AbstractRKNNQuery.java index 86e82c7e..fe4f49e1 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/rknn/AbstractRKNNQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/rknn/AbstractRKNNQuery.java @@ -25,7 +25,7 @@ package de.lmu.ifi.dbs.elki.database.query.rknn; import java.util.List; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.AbstractDataBasedQuery; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; @@ -53,5 +53,5 @@ public abstract class AbstractRKNNQuery<O, D extends Distance<D>> extends Abstra } @Override - abstract public List<DistanceResultPair<D>> getRKNNForDBID(DBID id, int k); + abstract public List<DistanceResultPair<D>> getRKNNForDBID(DBIDRef id, int k); }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/database/query/rknn/LinearScanRKNNQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/rknn/LinearScanRKNNQuery.java index 1449d7f0..1f901828 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/rknn/LinearScanRKNNQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/rknn/LinearScanRKNNQuery.java @@ -29,6 +29,8 @@ import java.util.List; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.database.query.GenericDistanceResultPair; @@ -76,12 +78,12 @@ public class LinearScanRKNNQuery<O, D extends Distance<D>> extends AbstractRKNNQ List<? extends KNNResult<D>> kNNLists = knnQuery.getKNNForBulkDBIDs(allIDs, k); int i = 0; - for(DBID qid : allIDs) { + for (DBIDIter iter = allIDs.iter(); iter.valid(); iter.advance()) { KNNResult<D> knn = kNNLists.get(i); int last = Math.min(k - 1, knn.size() - 1); - D dist = distanceQuery.distance(obj, qid); + D dist = distanceQuery.distance(obj, iter); if(last < k - 1 || dist.compareTo(knn.get(last).getDistance()) < 1) { - rNNlist.add(new GenericDistanceResultPair<D>(dist, qid)); + rNNlist.add(new GenericDistanceResultPair<D>(dist, iter.getDBID())); } i++; } @@ -90,8 +92,24 @@ public class LinearScanRKNNQuery<O, D extends Distance<D>> extends AbstractRKNNQ } @Override - public List<DistanceResultPair<D>> getRKNNForDBID(DBID id, int k) { - return getRKNNForBulkDBIDs(id, k).get(0); + public List<DistanceResultPair<D>> getRKNNForDBID(DBIDRef id, int k) { + ArrayList<DistanceResultPair<D>> rNNList = new ArrayList<DistanceResultPair<D>>(); + + ArrayDBIDs allIDs = DBIDUtil.ensureArray(relation.getDBIDs()); + List<? extends KNNResult<D>> kNNList = knnQuery.getKNNForBulkDBIDs(allIDs, k); + + int i = 0; + for (DBIDIter iter = allIDs.iter(); iter.valid(); iter.advance()) { + KNNResult<D> knn = kNNList.get(i); + for(DistanceResultPair<D> n : knn) { + if(n.sameDBID(id)) { + rNNList.add(new GenericDistanceResultPair<D>(n.getDistance(), iter.getDBID())); + } + } + i++; + } + Collections.sort(rNNList); + return rNNList; } @Override @@ -105,12 +123,13 @@ public class LinearScanRKNNQuery<O, D extends Distance<D>> extends AbstractRKNNQ List<? extends KNNResult<D>> kNNList = knnQuery.getKNNForBulkDBIDs(allIDs, k); int i = 0; - for(DBID qid : allIDs) { + for (DBIDIter iter = allIDs.iter(); iter.valid(); iter.advance()) { + DBID qid = iter.getDBID(); KNNResult<D> knn = kNNList.get(i); for(DistanceResultPair<D> n : knn) { int j = 0; - for(DBID id : ids) { - if(n.getDBID().equals(id)) { + for (DBIDIter iter2 = ids.iter(); iter2.valid(); iter2.advance()) { + if(n.getDBID().sameDBID(iter2)) { List<DistanceResultPair<D>> rNN = rNNList.get(j); rNN.add(new GenericDistanceResultPair<D>(n.getDistance(), qid)); } diff --git a/src/de/lmu/ifi/dbs/elki/database/query/rknn/PreprocessorRKNNQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/rknn/PreprocessorRKNNQuery.java index 2f8e841a..ab8c3b30 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/rknn/PreprocessorRKNNQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/rknn/PreprocessorRKNNQuery.java @@ -27,7 +27,8 @@ import java.util.ArrayList; import java.util.List; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.AbstractDataBasedQuery; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.database.relation.Relation; @@ -74,7 +75,7 @@ public class PreprocessorRKNNQuery<O, D extends Distance<D>> extends AbstractDat } @Override - public List<DistanceResultPair<D>> getRKNNForDBID(DBID id, int k) { + public List<DistanceResultPair<D>> getRKNNForDBID(DBIDRef id, int k) { if(!warned && k != preprocessor.getK()) { LoggingUtil.warning("Requested more neighbors than preprocessed!"); } @@ -92,8 +93,8 @@ public class PreprocessorRKNNQuery<O, D extends Distance<D>> extends AbstractDat LoggingUtil.warning("Requested more neighbors than preprocessed!"); } List<List<DistanceResultPair<D>>> result = new ArrayList<List<DistanceResultPair<D>>>(ids.size()); - for(DBID id : ids) { - result.add(preprocessor.getRKNN(id)); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + result.add(preprocessor.getRKNN(iter.getDBID())); } return result; } diff --git a/src/de/lmu/ifi/dbs/elki/database/query/rknn/RKNNQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/rknn/RKNNQuery.java index e707b6ce..9ff6f790 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/rknn/RKNNQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/rknn/RKNNQuery.java @@ -26,7 +26,7 @@ package de.lmu.ifi.dbs.elki.database.query.rknn; import java.util.List; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.DatabaseQuery; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; @@ -49,7 +49,7 @@ public interface RKNNQuery<O, D extends Distance<D>> extends DatabaseQuery { * @param k number of neighbors requested * @return reverse k nearest neighbors */ - public List<DistanceResultPair<D>> getRKNNForDBID(DBID id, int k); + public List<DistanceResultPair<D>> getRKNNForDBID(DBIDRef id, int k); /** * Get the reverse k nearest neighbors for a particular object. diff --git a/src/de/lmu/ifi/dbs/elki/database/query/similarity/AbstractDBIDSimilarityQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/similarity/AbstractDBIDSimilarityQuery.java index 1af1c624..3d5ff71f 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/similarity/AbstractDBIDSimilarityQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/similarity/AbstractDBIDSimilarityQuery.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.database.query.similarity; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; @@ -46,12 +46,12 @@ public abstract class AbstractDBIDSimilarityQuery<O, D extends Distance<D>> exte } @Override - public D similarity(O o1, DBID id2) { + public D similarity(O o1, DBIDRef id2) { throw new UnsupportedOperationException("This distance function can only be used for objects when referenced by ID."); } @Override - public D similarity(DBID id1, O o2) { + public D similarity(DBIDRef id1, O o2) { throw new UnsupportedOperationException("This distance function can only be used for objects when referenced by ID."); } diff --git a/src/de/lmu/ifi/dbs/elki/database/query/similarity/AbstractSimilarityQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/similarity/AbstractSimilarityQuery.java index 316d155c..21abe219 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/similarity/AbstractSimilarityQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/similarity/AbstractSimilarityQuery.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.database.query.similarity; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.AbstractDataBasedQuery; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; @@ -55,7 +55,7 @@ public abstract class AbstractSimilarityQuery<O, D extends Distance<D>> extends * @return the distance between the two objects specified by their object ids */ @Override - public abstract D similarity(DBID id1, DBID id2); + public abstract D similarity(DBIDRef id1, DBIDRef id2); /** * Returns the distance between the two objects specified by their object ids. @@ -65,7 +65,7 @@ public abstract class AbstractSimilarityQuery<O, D extends Distance<D>> extends * @return the distance between the two objects specified by their object ids */ @Override - public abstract D similarity(O o1, DBID id2); + public abstract D similarity(O o1, DBIDRef id2); /** * Returns the distance between the two objects specified by their object ids. @@ -75,7 +75,7 @@ public abstract class AbstractSimilarityQuery<O, D extends Distance<D>> extends * @return the distance between the two objects specified by their object ids */ @Override - public abstract D similarity(DBID id1, O o2); + public abstract D similarity(DBIDRef id1, O o2); /** * Returns the distance between the two objects specified by their object ids. diff --git a/src/de/lmu/ifi/dbs/elki/database/query/similarity/PrimitiveSimilarityQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/similarity/PrimitiveSimilarityQuery.java index f57e0bd5..5a604d0c 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/similarity/PrimitiveSimilarityQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/similarity/PrimitiveSimilarityQuery.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.database.query.similarity; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; import de.lmu.ifi.dbs.elki.distance.similarityfunction.PrimitiveSimilarityFunction; @@ -54,20 +54,20 @@ public class PrimitiveSimilarityQuery<O, D extends Distance<D>> extends Abstract } @Override - public D similarity(DBID id1, DBID id2) { + public D similarity(DBIDRef id1, DBIDRef id2) { O o1 = relation.get(id1); O o2 = relation.get(id2); return similarity(o1, o2); } @Override - public D similarity(O o1, DBID id2) { + public D similarity(O o1, DBIDRef id2) { O o2 = relation.get(id2); return similarity(o1, o2); } @Override - public D similarity(DBID id1, O o2) { + public D similarity(DBIDRef id1, O o2) { O o1 = relation.get(id1); return similarity(o1, o2); } diff --git a/src/de/lmu/ifi/dbs/elki/database/query/similarity/SimilarityQuery.java b/src/de/lmu/ifi/dbs/elki/database/query/similarity/SimilarityQuery.java index c3fdaaa4..b25d13fc 100644 --- a/src/de/lmu/ifi/dbs/elki/database/query/similarity/SimilarityQuery.java +++ b/src/de/lmu/ifi/dbs/elki/database/query/similarity/SimilarityQuery.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.database.query.similarity; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.DatabaseQuery; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; @@ -50,7 +50,7 @@ public interface SimilarityQuery<O, D extends Distance<?>> extends DatabaseQuery * @return the similarity between the two objects specified by their object * ids */ - public abstract D similarity(DBID id1, DBID id2); + public abstract D similarity(DBIDRef id1, DBIDRef id2); /** * Returns the similarity between the two objects specified by their object @@ -61,7 +61,7 @@ public interface SimilarityQuery<O, D extends Distance<?>> extends DatabaseQuery * @return the similarity between the two objects specified by their object * ids */ - public abstract D similarity(O o1, DBID id2); + public abstract D similarity(O o1, DBIDRef id2); /** * Returns the similarity between the two objects specified by their object @@ -72,7 +72,7 @@ public interface SimilarityQuery<O, D extends Distance<?>> extends DatabaseQuery * @return the similarity between the two objects specified by their object * ids */ - public abstract D similarity(DBID id1, O o2); + public abstract D similarity(DBIDRef id1, O o2); /** * Returns the similarity between the two objects specified by their object diff --git a/src/de/lmu/ifi/dbs/elki/database/relation/ConvertToStringView.java b/src/de/lmu/ifi/dbs/elki/database/relation/ConvertToStringView.java index 9d9ad672..4c25405b 100644 --- a/src/de/lmu/ifi/dbs/elki/database/relation/ConvertToStringView.java +++ b/src/de/lmu/ifi/dbs/elki/database/relation/ConvertToStringView.java @@ -26,10 +26,10 @@ package de.lmu.ifi.dbs.elki.database.relation; import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.result.AbstractHierarchicalResult; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; /** * Representation adapter that uses toString() to produce a string @@ -59,17 +59,17 @@ public class ConvertToStringView extends AbstractHierarchicalResult implements R } @Override - public String get(DBID id) { + public String get(DBIDRef id) { return existing.get(id).toString(); } @Override - public void set(DBID id, String val) { + public void set(DBIDRef id, String val) { throw new UnsupportedOperationException("Covnersion representations are not writable!"); } @Override - public void delete(DBID id) { + public void delete(DBIDRef id) { throw new UnsupportedOperationException("Covnersion representations are not writable!"); } @@ -79,7 +79,7 @@ public class ConvertToStringView extends AbstractHierarchicalResult implements R } @Override - public IterableIterator<DBID> iterDBIDs() { + public DBIDIter iterDBIDs() { return existing.iterDBIDs(); } diff --git a/src/de/lmu/ifi/dbs/elki/database/relation/DBIDView.java b/src/de/lmu/ifi/dbs/elki/database/relation/DBIDView.java index 62a6fd36..4898c518 100644 --- a/src/de/lmu/ifi/dbs/elki/database/relation/DBIDView.java +++ b/src/de/lmu/ifi/dbs/elki/database/relation/DBIDView.java @@ -28,11 +28,11 @@ import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.UpdatableDatabase; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.result.AbstractHierarchicalResult; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableUtil; /** * Pseudo-representation that is the object ID itself. @@ -68,20 +68,21 @@ public class DBIDView extends AbstractHierarchicalResult implements Relation<DBI } @Override - public DBID get(DBID id) { + public DBID get(DBIDRef id) { assert (ids.contains(id)); - return id; + return id.getDBID(); } @Override - public void set(DBID id, DBID val) { + public void set(DBIDRef id, DBID val) { throw new UnsupportedOperationException("DBIDs cannot be changed."); } @Override - public void delete(DBID id) { + public void delete(DBIDRef id) { if(database instanceof UpdatableDatabase) { - ((UpdatableDatabase) database).delete(id); + // TODO: skip getDBID() + ((UpdatableDatabase) database).delete(id.getDBID()); } else { throw new UnsupportedOperationException("Deletions are not supported."); @@ -99,8 +100,8 @@ public class DBIDView extends AbstractHierarchicalResult implements Relation<DBI } @Override - public IterableIterator<DBID> iterDBIDs() { - return IterableUtil.fromIterable(ids); + public DBIDIter iterDBIDs() { + return ids.iter(); } @Override diff --git a/src/de/lmu/ifi/dbs/elki/database/relation/MaterializedRelation.java b/src/de/lmu/ifi/dbs/elki/database/relation/MaterializedRelation.java index af65e84d..2d090e99 100644 --- a/src/de/lmu/ifi/dbs/elki/database/relation/MaterializedRelation.java +++ b/src/de/lmu/ifi/dbs/elki/database/relation/MaterializedRelation.java @@ -29,13 +29,12 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStore; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.StaticDBIDs; import de.lmu.ifi.dbs.elki.result.AbstractHierarchicalResult; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableUtil; /** * Represents a single representation. This is attached to a DBIDs object, which @@ -152,12 +151,12 @@ public class MaterializedRelation<O> extends AbstractHierarchicalResult implemen } @Override - public O get(DBID id) { + public O get(DBIDRef id) { return content.get(id); } @Override - public void set(DBID id, O val) { + public void set(DBIDRef id, O val) { assert (ids.contains(id)); if(content instanceof WritableDataStore) { ((WritableDataStore<O>) content).put(id, val); @@ -170,7 +169,7 @@ public class MaterializedRelation<O> extends AbstractHierarchicalResult implemen * @param id ID to delete */ @Override - public void delete(DBID id) { + public void delete(DBIDRef id) { assert (!ids.contains(id)); if(content instanceof WritableDataStore) { ((WritableDataStore<O>) content).delete(id); @@ -183,8 +182,8 @@ public class MaterializedRelation<O> extends AbstractHierarchicalResult implemen } @Override - public IterableIterator<DBID> iterDBIDs() { - return IterableUtil.fromIterable(ids); + public DBIDIter iterDBIDs() { + return ids.iter(); } @Override diff --git a/src/de/lmu/ifi/dbs/elki/database/relation/ProjectedView.java b/src/de/lmu/ifi/dbs/elki/database/relation/ProjectedView.java index 190b15c6..8f2b8c92 100644 --- a/src/de/lmu/ifi/dbs/elki/database/relation/ProjectedView.java +++ b/src/de/lmu/ifi/dbs/elki/database/relation/ProjectedView.java @@ -3,10 +3,10 @@ package de.lmu.ifi.dbs.elki.database.relation; import de.lmu.ifi.dbs.elki.data.projection.Projection; import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; import de.lmu.ifi.dbs.elki.database.Database; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.result.AbstractHierarchicalResult; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; /* This file is part of ELKI: @@ -78,17 +78,17 @@ public class ProjectedView<IN, OUT> extends AbstractHierarchicalResult implement } @Override - public OUT get(DBID id) { + public OUT get(DBIDRef id) { return projection.project(inner.get(id)); } @Override - public void set(DBID id, OUT val) { + public void set(DBIDRef id, OUT val) { throw new UnsupportedOperationException("Projections are read-only."); } @Override - public void delete(DBID id) { + public void delete(DBIDRef id) { inner.delete(id); } @@ -103,7 +103,7 @@ public class ProjectedView<IN, OUT> extends AbstractHierarchicalResult implement } @Override - public IterableIterator<DBID> iterDBIDs() { + public DBIDIter iterDBIDs() { return inner.iterDBIDs(); } diff --git a/src/de/lmu/ifi/dbs/elki/database/relation/ProxyView.java b/src/de/lmu/ifi/dbs/elki/database/relation/ProxyView.java index 1ae761f3..9937c8ee 100644 --- a/src/de/lmu/ifi/dbs/elki/database/relation/ProxyView.java +++ b/src/de/lmu/ifi/dbs/elki/database/relation/ProxyView.java @@ -25,12 +25,11 @@ package de.lmu.ifi.dbs.elki.database.relation; import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; import de.lmu.ifi.dbs.elki.database.Database; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.result.AbstractHierarchicalResult; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableUtil; /** * A virtual partitioning of the database. For the accepted DBIDs, access is @@ -87,19 +86,19 @@ public class ProxyView<O> extends AbstractHierarchicalResult implements Relation } @Override - public O get(DBID id) { + public O get(DBIDRef id) { assert (idview.contains(id)) : "Accessing object not included in view."; return inner.get(id); } - + @Override - public void set(DBID id, O val) { + public void set(DBIDRef id, O val) { assert (idview.contains(id)) : "Accessing object not included in view."; inner.set(id, val); } @Override - public void delete(DBID id) { + public void delete(DBIDRef id) { throw new UnsupportedOperationException("Semantics of 'delete-from-virtual-partition' are not uniquely defined. Delete from IDs or from underlying data, please!"); } @@ -109,8 +108,8 @@ public class ProxyView<O> extends AbstractHierarchicalResult implements Relation } @Override - public IterableIterator<DBID> iterDBIDs() { - return IterableUtil.fromIterable(idview); + public DBIDIter iterDBIDs() { + return idview.iter(); } @Override diff --git a/src/de/lmu/ifi/dbs/elki/database/relation/Relation.java b/src/de/lmu/ifi/dbs/elki/database/relation/Relation.java index ebe1be3c..4969ad50 100644 --- a/src/de/lmu/ifi/dbs/elki/database/relation/Relation.java +++ b/src/de/lmu/ifi/dbs/elki/database/relation/Relation.java @@ -25,17 +25,19 @@ package de.lmu.ifi.dbs.elki.database.relation; import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; import de.lmu.ifi.dbs.elki.database.Database; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.query.DatabaseQuery; import de.lmu.ifi.dbs.elki.result.HierarchicalResult; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; /** * An object representation from a database * * @author Erich Schubert - * + * + * @apiviz.uses DBIDRef + * * @param <O> Object type */ public interface Relation<O> extends DatabaseQuery, HierarchicalResult { @@ -47,15 +49,15 @@ public interface Relation<O> extends DatabaseQuery, HierarchicalResult { * @return Database */ public Database getDatabase(); - + /** * Get the representation of an object. * * @param id Object ID * @return object instance */ - public O get(DBID id); - + public O get(DBIDRef id); + /** * Set an object representation. * @@ -63,14 +65,14 @@ public interface Relation<O> extends DatabaseQuery, HierarchicalResult { * @param val Value */ // TODO: remove / move to a writable API? - public void set(DBID id, O val); + public void set(DBIDRef id, O val); /** * Delete an objects values. * * @param id ID to delete */ - public void delete(DBID id); + public void delete(DBIDRef id); /** * Get the data type of this representation @@ -78,7 +80,7 @@ public interface Relation<O> extends DatabaseQuery, HierarchicalResult { * @return Data type */ public SimpleTypeInformation<O> getDataTypeInformation(); - + /** * Get the IDs the query is defined for. * @@ -89,9 +91,19 @@ public interface Relation<O> extends DatabaseQuery, HierarchicalResult { /** * Get an iterator access to the DBIDs. * + * To iterate over all IDs, use the following code fragment: + * + * <pre> + * {@code + * for(DBIDIter iter = relation.iterDBIDs(); iter.valid(); iter.advance()) { + * DBID id = iter.getDBID(); + * } + * } + * </pre> + * * @return iterator for the DBIDs. */ - public IterableIterator<DBID> iterDBIDs(); + public DBIDIter iterDBIDs(); /** * Get the number of DBIDs. diff --git a/src/de/lmu/ifi/dbs/elki/datasource/ArrayAdapterDatabaseConnection.java b/src/de/lmu/ifi/dbs/elki/datasource/ArrayAdapterDatabaseConnection.java index 4a126d65..d632a3fb 100644 --- a/src/de/lmu/ifi/dbs/elki/datasource/ArrayAdapterDatabaseConnection.java +++ b/src/de/lmu/ifi/dbs/elki/datasource/ArrayAdapterDatabaseConnection.java @@ -24,12 +24,17 @@ package de.lmu.ifi.dbs.elki.datasource; */ import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import de.lmu.ifi.dbs.elki.data.DoubleVector; import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; +import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.data.type.VectorFieldTypeInformation; +import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.datasource.bundle.MultipleObjectsBundle; +import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; /** * Import an existing data matrix (<code>double[rows][cols]</code>) into an ELKI @@ -46,17 +51,59 @@ public class ArrayAdapterDatabaseConnection implements DatabaseConnection { double[][] data; /** + * Object labels + */ + String[] labels; + + /** + * Starting ID for fixed object ids. + */ + Integer startid = null; + + /** * Constructor. * * @param data Existing data matrix */ public ArrayAdapterDatabaseConnection(double[][] data) { + this(data, null, null); + } + + /** + * Constructor. + * + * @param data Existing data matrix + * @param labels Object labels + */ + public ArrayAdapterDatabaseConnection(double[][] data, String[] labels) { + this(data, labels, null); + } + + /** + * Constructor. + * + * @param data Existing data matrix + * @param labels Object labels + * @param startid Starting object ID + */ + public ArrayAdapterDatabaseConnection(double[][] data, String[] labels, Integer startid) { super(); + this.data = data; + this.labels = labels; + this.startid = startid; } @Override public MultipleObjectsBundle loadData() { MultipleObjectsBundle b = new MultipleObjectsBundle(); + if(startid != null) { + List<DBID> ids = new ArrayList<DBID>(data.length); + for(int i = 0; i < data.length; i++) { + ids.add(DBIDUtil.importInteger(startid + i)); + } + b.appendColumn(TypeUtil.DBID, Arrays.asList(labels)); + } + int mind = Integer.MAX_VALUE; int maxd = 0; List<DoubleVector> vecs = new ArrayList<DoubleVector>(data.length); @@ -73,6 +120,12 @@ public class ArrayAdapterDatabaseConnection implements DatabaseConnection { type = new SimpleTypeInformation<DoubleVector>(DoubleVector.class); } b.appendColumn(type, vecs); + if(labels != null) { + if(labels.length != data.length) { + throw new AbortException("Label and DBID columns must have the same size."); + } + b.appendColumn(TypeUtil.STRING, Arrays.asList(labels)); + } return b; } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/datasource/GeneratorXMLDatabaseConnection.java b/src/de/lmu/ifi/dbs/elki/datasource/GeneratorXMLDatabaseConnection.java index 5d3dbbf9..faf557f0 100644 --- a/src/de/lmu/ifi/dbs/elki/datasource/GeneratorXMLDatabaseConnection.java +++ b/src/de/lmu/ifi/dbs/elki/datasource/GeneratorXMLDatabaseConnection.java @@ -52,7 +52,7 @@ import de.lmu.ifi.dbs.elki.data.synthetic.bymodel.GeneratorStatic; import de.lmu.ifi.dbs.elki.datasource.bundle.MultipleObjectsBundle; import de.lmu.ifi.dbs.elki.logging.Logging; import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; -import de.lmu.ifi.dbs.elki.math.statistics.distribution.Distribution; +import de.lmu.ifi.dbs.elki.math.statistics.distribution.DistributionWithRandom; import de.lmu.ifi.dbs.elki.math.statistics.distribution.GammaDistribution; import de.lmu.ifi.dbs.elki.math.statistics.distribution.NormalDistribution; import de.lmu.ifi.dbs.elki.math.statistics.distribution.UniformDistribution; @@ -73,6 +73,8 @@ import de.lmu.ifi.dbs.elki.utilities.xml.XMLNodeIterator; * supported) data sets that satisfy a given specification file. * * @author Erich Schubert + * + * @apiviz.composedOf GeneratorMain */ public class GeneratorXMLDatabaseConnection implements DatabaseConnection { /** @@ -237,7 +239,9 @@ public class GeneratorXMLDatabaseConnection implements DatabaseConnection { testAgainstModel = (Integer.valueOf(testmod) != 0); } // TODO: check for unknown attributes. - for(Node child : new XMLNodeIterator(cur.getFirstChild())) { + XMLNodeIterator iter = new XMLNodeIterator(cur.getFirstChild()); + while(iter.hasNext()) { + Node child = iter.next(); if(child.getNodeName() == "cluster") { processElementCluster(gen, child); } @@ -285,7 +289,9 @@ public class GeneratorXMLDatabaseConnection implements DatabaseConnection { GeneratorSingleCluster cluster = new GeneratorSingleCluster(name, size, overweight, newRand); // TODO: check for unknown attributes. - for(Node child : new XMLNodeIterator(cur.getFirstChild())) { + XMLNodeIterator iter = new XMLNodeIterator(cur.getFirstChild()); + while(iter.hasNext()) { + Node child = iter.next(); if(child.getNodeName() == "uniform") { processElementUniform(cluster, child); } @@ -334,11 +340,13 @@ public class GeneratorXMLDatabaseConnection implements DatabaseConnection { // *** new uniform generator Random random = cluster.getNewRandomGenerator(); - Distribution generator = new UniformDistribution(min, max, random); + DistributionWithRandom generator = new UniformDistribution(min, max, random); cluster.addGenerator(generator); // TODO: check for unknown attributes. - for(Node child : new XMLNodeIterator(cur.getFirstChild())) { + XMLNodeIterator iter = new XMLNodeIterator(cur.getFirstChild()); + while(iter.hasNext()) { + Node child = iter.next(); if(child.getNodeType() == Node.ELEMENT_NODE) { logger.warning("Unknown element in XML specification file: " + child.getNodeName()); } @@ -366,11 +374,13 @@ public class GeneratorXMLDatabaseConnection implements DatabaseConnection { // *** New normal distribution generator Random random = cluster.getNewRandomGenerator(); - Distribution generator = new NormalDistribution(mean, stddev, random); + DistributionWithRandom generator = new NormalDistribution(mean, stddev, random); cluster.addGenerator(generator); // TODO: check for unknown attributes. - for(Node child : new XMLNodeIterator(cur.getFirstChild())) { + XMLNodeIterator iter = new XMLNodeIterator(cur.getFirstChild()); + while(iter.hasNext()) { + Node child = iter.next(); if(child.getNodeType() == Node.ELEMENT_NODE) { logger.warning("Unknown element in XML specification file: " + child.getNodeName()); } @@ -398,11 +408,13 @@ public class GeneratorXMLDatabaseConnection implements DatabaseConnection { // *** New normal distribution generator Random random = cluster.getNewRandomGenerator(); - Distribution generator = new GammaDistribution(k, theta, random); + DistributionWithRandom generator = new GammaDistribution(k, theta, random); cluster.addGenerator(generator); // TODO: check for unknown attributes. - for(Node child : new XMLNodeIterator(cur.getFirstChild())) { + XMLNodeIterator iter = new XMLNodeIterator(cur.getFirstChild()); + while(iter.hasNext()) { + Node child = iter.next(); if(child.getNodeType() == Node.ELEMENT_NODE) { logger.warning("Unknown element in XML specification file: " + child.getNodeName()); } @@ -447,7 +459,9 @@ public class GeneratorXMLDatabaseConnection implements DatabaseConnection { cluster.addRotation(axis1 - 1, axis2 - 1, Math.toRadians(angle)); // TODO: check for unknown attributes. - for(Node child : new XMLNodeIterator(cur.getFirstChild())) { + XMLNodeIterator iter = new XMLNodeIterator(cur.getFirstChild()); + while(iter.hasNext()) { + Node child = iter.next(); if(child.getNodeType() == Node.ELEMENT_NODE) { logger.warning("Unknown element in XML specification file: " + child.getNodeName()); } @@ -475,7 +489,9 @@ public class GeneratorXMLDatabaseConnection implements DatabaseConnection { cluster.addTranslation(offset); // TODO: check for unknown attributes. - for(Node child : new XMLNodeIterator(cur.getFirstChild())) { + XMLNodeIterator iter = new XMLNodeIterator(cur.getFirstChild()); + while(iter.hasNext()) { + Node child = iter.next(); if(child.getNodeType() == Node.ELEMENT_NODE) { logger.warning("Unknown element in XML specification file: " + child.getNodeName()); } @@ -509,7 +525,9 @@ public class GeneratorXMLDatabaseConnection implements DatabaseConnection { cluster.setClipping(cmin, cmax); // TODO: check for unknown attributes. - for(Node child : new XMLNodeIterator(cur.getFirstChild())) { + XMLNodeIterator iter = new XMLNodeIterator(cur.getFirstChild()); + while(iter.hasNext()) { + Node child = iter.next(); if(child.getNodeType() == Node.ELEMENT_NODE) { logger.warning("Unknown element in XML specification file: " + child.getNodeName()); } @@ -531,7 +549,9 @@ public class GeneratorXMLDatabaseConnection implements DatabaseConnection { ArrayList<Vector> points = new ArrayList<Vector>(); // TODO: check for unknown attributes. - for(Node child : new XMLNodeIterator(cur.getFirstChild())) { + XMLNodeIterator iter = new XMLNodeIterator(cur.getFirstChild()); + while(iter.hasNext()) { + Node child = iter.next(); if(child.getNodeName() == "point") { processElementPoint(points, child); } @@ -569,7 +589,9 @@ public class GeneratorXMLDatabaseConnection implements DatabaseConnection { points.add(point); // TODO: check for unknown attributes. - for(Node child : new XMLNodeIterator(cur.getFirstChild())) { + XMLNodeIterator iter = new XMLNodeIterator(cur.getFirstChild()); + while(iter.hasNext()) { + Node child = iter.next(); if(child.getNodeType() == Node.ELEMENT_NODE) { logger.warning("Unknown element in XML specification file: " + child.getNodeName()); } diff --git a/src/de/lmu/ifi/dbs/elki/datasource/filter/AbstractRandomFeatureSelectionFilter.java b/src/de/lmu/ifi/dbs/elki/datasource/filter/AbstractRandomFeatureSelectionFilter.java index b52c7887..6c16abfc 100644 --- a/src/de/lmu/ifi/dbs/elki/datasource/filter/AbstractRandomFeatureSelectionFilter.java +++ b/src/de/lmu/ifi/dbs/elki/datasource/filter/AbstractRandomFeatureSelectionFilter.java @@ -36,6 +36,7 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.GreaterEqualConstraint; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.IntParameter; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.LongParameter; /** * <p> @@ -74,6 +75,16 @@ public abstract class AbstractRandomFeatureSelectionFilter<V extends FeatureVect public static final OptionID NUMBER_SELECTED_ATTRIBUTES_ID = OptionID.getOrCreateOptionID("randomprojection.numberselected", "number of selected attributes"); /** + * Optional parameter to specify a seed for random projection. + * If unused, system time is used as seed. + * <p> + * Key: {@code -randomprojection.seed} + * </p> + */ + public static final OptionID SEED_ID = OptionID.getOrCreateOptionID("randomprojection.seed", "Seed for random selection of projection attributes."); + + + /** * Holds the desired cardinality of the subset of attributes selected for * projection. */ @@ -82,7 +93,7 @@ public abstract class AbstractRandomFeatureSelectionFilter<V extends FeatureVect /** * Holds a random object. */ - protected final Random random = new Random(); + protected final Random random; /** * Constructor. @@ -92,6 +103,19 @@ public abstract class AbstractRandomFeatureSelectionFilter<V extends FeatureVect public AbstractRandomFeatureSelectionFilter(int dim) { super(); this.k = dim; + this.random = new Random(); + } + + /** + * Constructor. + * + * @param dim dimensionality + * @param seed seed for random + */ + public AbstractRandomFeatureSelectionFilter(int dim, long seed) { + super(); + this.k = dim; + this.random = new Random(seed); } /** @@ -115,6 +139,8 @@ public abstract class AbstractRandomFeatureSelectionFilter<V extends FeatureVect */ public static abstract class Parameterizer<V extends NumberVector<V, ?>> extends AbstractParameterizer { protected int k = 0; + + protected long seed = System.currentTimeMillis(); @Override protected void makeOptions(Parameterization config) { @@ -123,6 +149,10 @@ public abstract class AbstractRandomFeatureSelectionFilter<V extends FeatureVect if(config.grab(kP)) { k = kP.getValue(); } + LongParameter seedP = new LongParameter(SEED_ID, true); + if(config.grab(seedP)) { + seed = seedP.getValue(); + } } } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/datasource/filter/DoubleVectorRandomProjectionFilter.java b/src/de/lmu/ifi/dbs/elki/datasource/filter/DoubleVectorRandomProjectionFilter.java index 5aa31967..b21e7cea 100644 --- a/src/de/lmu/ifi/dbs/elki/datasource/filter/DoubleVectorRandomProjectionFilter.java +++ b/src/de/lmu/ifi/dbs/elki/datasource/filter/DoubleVectorRandomProjectionFilter.java @@ -44,9 +44,10 @@ public class DoubleVectorRandomProjectionFilter extends AbstractRandomFeatureSel * Constructor.
*
* @param dim
+ * @param seed
*/
- public DoubleVectorRandomProjectionFilter(int dim) {
- super(dim);
+ public DoubleVectorRandomProjectionFilter(int dim, long seed) {
+ super(dim, seed);
}
@Override
@@ -80,7 +81,7 @@ public class DoubleVectorRandomProjectionFilter extends AbstractRandomFeatureSel @Override
protected DoubleVectorRandomProjectionFilter makeInstance() {
- return new DoubleVectorRandomProjectionFilter(k);
+ return new DoubleVectorRandomProjectionFilter(k,seed);
}
}
}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/datasource/filter/FilterUtil.java b/src/de/lmu/ifi/dbs/elki/datasource/filter/FilterUtil.java new file mode 100644 index 00000000..0015dce1 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/datasource/filter/FilterUtil.java @@ -0,0 +1,63 @@ +package de.lmu.ifi.dbs.elki.datasource.filter; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import java.lang.reflect.Field; + +import de.lmu.ifi.dbs.elki.data.NumberVector; +import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; +import de.lmu.ifi.dbs.elki.data.type.VectorFieldTypeInformation; +import de.lmu.ifi.dbs.elki.logging.LoggingUtil; + +/** + * Utilities for implementing filters. + * + * @author Erich Schubert + */ +public final class FilterUtil { + /** + * Try to guess the factory + * + * @param in Input type + * @return Factory + */ + @SuppressWarnings("unchecked") + protected static <V extends NumberVector<?, ?>> V guessFactory(SimpleTypeInformation<V> in) { + V factory = null; + if(in instanceof VectorFieldTypeInformation) { + factory = ((VectorFieldTypeInformation<V>) in).getFactory(); + } + if(factory == null) { + // FIXME: hack. Add factories to simple type information, too? + try { + Field f = in.getRestrictionClass().getField("STATIC"); + factory = (V) f.get(null); + } + catch(Exception e) { + LoggingUtil.warning("Cannot determine factory for type " + in.getRestrictionClass()); + } + } + return factory; + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/datasource/filter/NoMissingValuesFilter.java b/src/de/lmu/ifi/dbs/elki/datasource/filter/NoMissingValuesFilter.java index da5f066f..7d5b6c44 100644 --- a/src/de/lmu/ifi/dbs/elki/datasource/filter/NoMissingValuesFilter.java +++ b/src/de/lmu/ifi/dbs/elki/datasource/filter/NoMissingValuesFilter.java @@ -25,6 +25,7 @@ package de.lmu.ifi.dbs.elki.datasource.filter; import java.util.ArrayList; +import de.lmu.ifi.dbs.elki.datasource.bundle.BundleMeta; import de.lmu.ifi.dbs.elki.datasource.bundle.MultipleObjectsBundle; import de.lmu.ifi.dbs.elki.logging.Logging; import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; @@ -34,13 +35,18 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; * * @author Erich Schubert */ -public class NoMissingValuesFilter implements ObjectFilter { +public class NoMissingValuesFilter extends AbstractStreamFilter { /** * Class logger */ private static final Logging logger = Logging.getLogger(NoMissingValuesFilter.class); /** + * Number of columns + */ + private int cols = 0; + + /** * Constructor. */ public NoMissingValuesFilter() { @@ -48,6 +54,42 @@ public class NoMissingValuesFilter implements ObjectFilter { } @Override + public BundleMeta getMeta() { + return source.getMeta(); + } + + @Override + public Object data(int rnum) { + return source.data(rnum); + } + + @Override + public Event nextEvent() { + while(true) { + Event ev = source.nextEvent(); + switch(ev){ + case END_OF_STREAM: + return ev; + case META_CHANGED: + cols = source.getMeta().size(); + return ev; + case NEXT_OBJECT: + boolean good = true; + for(int j = 0; j < cols; j++) { + if(source.data(j) == null) { + good = false; + break; + } + } + if(good) { + return ev; + } + continue; + } + } + } + + @Override public MultipleObjectsBundle filter(final MultipleObjectsBundle objects) { if(logger.isDebugging()) { logger.debug("Filtering the data set"); diff --git a/src/de/lmu/ifi/dbs/elki/datasource/filter/RandomSamplingStreamFilter.java b/src/de/lmu/ifi/dbs/elki/datasource/filter/RandomSamplingStreamFilter.java new file mode 100644 index 00000000..56509d8a --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/datasource/filter/RandomSamplingStreamFilter.java @@ -0,0 +1,139 @@ +package de.lmu.ifi.dbs.elki.datasource.filter; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import java.util.Random; + +import de.lmu.ifi.dbs.elki.datasource.bundle.BundleMeta; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.IntervalConstraint; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.IntervalConstraint.IntervalBoundary; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.DoubleParameter; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.LongParameter; + +/** + * Subsampling stream filter. + * + * @author Erich Schubert + */ +public class RandomSamplingStreamFilter extends AbstractStreamFilter { + /** + * Probability + */ + protected double prob; + + /** + * Random generator + */ + protected Random random; + + /** + * Constructor. + * + * @param prob Probability + * @param seed Random seed + */ + public RandomSamplingStreamFilter(double prob, Long seed) { + super(); + this.prob = prob; + this.random = (seed != null) ? new Random(seed) : new Random(); + } + + @Override + public BundleMeta getMeta() { + return source.getMeta(); + } + + @Override + public Object data(int rnum) { + return source.data(rnum); + } + + @Override + public Event nextEvent() { + while(true) { + Event ev = source.nextEvent(); + switch(ev){ + case END_OF_STREAM: + return ev; + case META_CHANGED: + return ev; + case NEXT_OBJECT: + if(random.nextDouble() < prob) { + return ev; + } + continue; + } + } + } + + /** + * Parameterization class + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer extends AbstractParameterizer { + /** + * Option ID for sampling probability + */ + private static final OptionID PROB_ID = OptionID.getOrCreateOptionID("sampling.p", "Sampling probability. Each object has a chance of being samples with this probability."); + + /** + * Option ID for random seed + */ + private static final OptionID SEED_ID = OptionID.getOrCreateOptionID("sampling.seed", "Random generator seed for sampling."); + + /** + * Probability + */ + protected double prob; + + /** + * Random seed + */ + protected Long seed = null; + + @Override + protected void makeOptions(Parameterization config) { + super.makeOptions(config); + DoubleParameter probP = new DoubleParameter(PROB_ID, new IntervalConstraint(0, IntervalBoundary.CLOSE, 1.0, IntervalBoundary.CLOSE)); + if(config.grab(probP)) { + prob = probP.getValue(); + } + LongParameter seedP = new LongParameter(SEED_ID, true); + if(config.grab(seedP)) { + seed = seedP.getValue(); + } + } + + @Override + protected RandomSamplingStreamFilter makeInstance() { + return new RandomSamplingStreamFilter(prob, seed); + } + } +} diff --git a/src/de/lmu/ifi/dbs/elki/datasource/filter/SparseFloatVectorProjectionFilter.java b/src/de/lmu/ifi/dbs/elki/datasource/filter/SparseNumberVectorProjectionFilter.java index 06e686c3..79a671c5 100644 --- a/src/de/lmu/ifi/dbs/elki/datasource/filter/SparseFloatVectorProjectionFilter.java +++ b/src/de/lmu/ifi/dbs/elki/datasource/filter/SparseNumberVectorProjectionFilter.java @@ -22,11 +22,9 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import gnu.trove.map.hash.TIntFloatHashMap;
-
import java.util.BitSet;
-import de.lmu.ifi.dbs.elki.data.SparseFloatVector;
+import de.lmu.ifi.dbs.elki.data.SparseNumberVector;
import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation;
import de.lmu.ifi.dbs.elki.data.type.TypeUtil;
import de.lmu.ifi.dbs.elki.data.type.VectorFieldTypeInformation;
@@ -41,30 +39,30 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameteriz *
* @author Arthur Zimek
*/
-public class SparseFloatVectorProjectionFilter extends AbstractFeatureSelectionFilter<SparseFloatVector> {
+public class SparseNumberVectorProjectionFilter<V extends SparseNumberVector<V, ?>> extends AbstractFeatureSelectionFilter<V> {
/**
* Constructor.
*
* @param selectedAttributes
*/
- public SparseFloatVectorProjectionFilter(BitSet selectedAttributes) {
+ public SparseNumberVectorProjectionFilter(BitSet selectedAttributes) {
super(selectedAttributes);
}
@Override
- protected SparseFloatVector filterSingleObject(SparseFloatVector obj) {
+ protected V filterSingleObject(V obj) {
return Util.project(obj, getSelectedAttributes());
}
@Override
- protected SimpleTypeInformation<? super SparseFloatVector> getInputTypeRestriction() {
- return TypeUtil.SPARSE_FLOAT_FIELD;
+ protected SimpleTypeInformation<? super V> getInputTypeRestriction() {
+ return TypeUtil.SPARSE_VECTOR_FIELD;
}
@Override
- protected SimpleTypeInformation<? super SparseFloatVector> convertedType(SimpleTypeInformation<SparseFloatVector> in) {
- final TIntFloatHashMap emptyMap = new TIntFloatHashMap();
- return new VectorFieldTypeInformation<SparseFloatVector>(SparseFloatVector.class, getDimensionality(), new SparseFloatVector(emptyMap, getDimensionality()));
+ protected SimpleTypeInformation<? super V> convertedType(SimpleTypeInformation<V> in) {
+ V factory = FilterUtil.guessFactory(in);
+ return new VectorFieldTypeInformation<V>(in.getRestrictionClass(), getDimensionality(), factory);
}
/**
@@ -74,15 +72,15 @@ public class SparseFloatVectorProjectionFilter extends AbstractFeatureSelectionF * * @apiviz.exclude */
- public static class Parameterizer extends AbstractFeatureSelectionFilter.Parameterizer<SparseFloatVector> {
+ public static class Parameterizer<V extends SparseNumberVector<V, ?>> extends AbstractFeatureSelectionFilter.Parameterizer<V> {
@Override
protected void makeOptions(Parameterization config) {
super.makeOptions(config);
}
@Override
- protected SparseFloatVectorProjectionFilter makeInstance() {
- return new SparseFloatVectorProjectionFilter(selectedAttributes);
+ protected SparseNumberVectorProjectionFilter<V> makeInstance() {
+ return new SparseNumberVectorProjectionFilter<V>(selectedAttributes);
}
}
}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/datasource/filter/SparseFloatVectorRandomProjectionFilter.java b/src/de/lmu/ifi/dbs/elki/datasource/filter/SparseNumberVectorRandomProjectionFilter.java index fbf26eea..8597d659 100644 --- a/src/de/lmu/ifi/dbs/elki/datasource/filter/SparseFloatVectorRandomProjectionFilter.java +++ b/src/de/lmu/ifi/dbs/elki/datasource/filter/SparseNumberVectorRandomProjectionFilter.java @@ -22,7 +22,7 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import de.lmu.ifi.dbs.elki.data.SparseFloatVector;
+import de.lmu.ifi.dbs.elki.data.SparseNumberVector;
import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation;
import de.lmu.ifi.dbs.elki.data.type.TypeUtil;
import de.lmu.ifi.dbs.elki.data.type.VectorFieldTypeInformation;
@@ -35,30 +35,31 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameteriz *
* @author Arthur Zimek
*/
-public class SparseFloatVectorRandomProjectionFilter extends AbstractRandomFeatureSelectionFilter<SparseFloatVector> {
+public class SparseNumberVectorRandomProjectionFilter<V extends SparseNumberVector<V, ?>> extends AbstractRandomFeatureSelectionFilter<V> {
/**
* Constructor.
*
* @param dim
*/
- public SparseFloatVectorRandomProjectionFilter(int dim) {
+ public SparseNumberVectorRandomProjectionFilter(int dim) {
super(dim);
}
@Override
- protected SparseFloatVector filterSingleObject(SparseFloatVector obj) {
+ protected V filterSingleObject(V obj) {
return Util.project(obj, selectedAttributes);
}
@Override
- protected SimpleTypeInformation<? super SparseFloatVector> getInputTypeRestriction() {
- return TypeUtil.SPARSE_FLOAT_FIELD;
+ protected SimpleTypeInformation<? super V> getInputTypeRestriction() {
+ return TypeUtil.SPARSE_VECTOR_FIELD;
}
@Override
- protected SimpleTypeInformation<? super SparseFloatVector> convertedType(SimpleTypeInformation<SparseFloatVector> in) {
+ protected SimpleTypeInformation<? super V> convertedType(SimpleTypeInformation<V> in) {
initializeRandomAttributes(in);
- return new VectorFieldTypeInformation<SparseFloatVector>(SparseFloatVector.class, k, new SparseFloatVector(SparseFloatVector.EMPTYMAP, k));
+ V factory = FilterUtil.guessFactory(in);
+ return new VectorFieldTypeInformation<V>(in.getRestrictionClass(), k, factory);
}
/**
@@ -68,15 +69,15 @@ public class SparseFloatVectorRandomProjectionFilter extends AbstractRandomFeatu * * @apiviz.exclude */
- public static class Parameterizer extends AbstractRandomFeatureSelectionFilter.Parameterizer<SparseFloatVector> {
+ public static class Parameterizer<V extends SparseNumberVector<V, ?>> extends AbstractRandomFeatureSelectionFilter.Parameterizer<V> {
@Override
protected void makeOptions(Parameterization config) {
super.makeOptions(config);
}
@Override
- protected SparseFloatVectorRandomProjectionFilter makeInstance() {
- return new SparseFloatVectorRandomProjectionFilter(k);
+ protected SparseNumberVectorRandomProjectionFilter<V> makeInstance() {
+ return new SparseNumberVectorRandomProjectionFilter<V>(k);
}
}
}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/datasource/filter/SparseVectorFieldFilter.java b/src/de/lmu/ifi/dbs/elki/datasource/filter/SparseVectorFieldFilter.java index 482fc498..9d34057b 100644 --- a/src/de/lmu/ifi/dbs/elki/datasource/filter/SparseVectorFieldFilter.java +++ b/src/de/lmu/ifi/dbs/elki/datasource/filter/SparseVectorFieldFilter.java @@ -1,4 +1,5 @@ package de.lmu.ifi.dbs.elki.datasource.filter; + /* This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures @@ -22,7 +23,7 @@ package de.lmu.ifi.dbs.elki.datasource.filter; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import de.lmu.ifi.dbs.elki.data.SparseFloatVector; +import de.lmu.ifi.dbs.elki.data.SparseNumberVector; import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.data.type.VectorFieldTypeInformation; @@ -33,7 +34,7 @@ import de.lmu.ifi.dbs.elki.data.type.VectorFieldTypeInformation; * * @author Erich Schubert */ -public class SparseVectorFieldFilter extends AbstractConversionFilter<SparseFloatVector, SparseFloatVector> { +public class SparseVectorFieldFilter<V extends SparseNumberVector<V, ?>> extends AbstractConversionFilter<V, V> { /** * Maximum dimension */ @@ -47,29 +48,30 @@ public class SparseVectorFieldFilter extends AbstractConversionFilter<SparseFloa } @Override - protected boolean prepareStart(SimpleTypeInformation<SparseFloatVector> in) { + protected boolean prepareStart(SimpleTypeInformation<V> in) { return true; } @Override - protected void prepareProcessInstance(SparseFloatVector obj) { + protected void prepareProcessInstance(V obj) { maxdim = Math.max(maxdim, obj.getDimensionality()); } @Override - protected SparseFloatVector filterSingleObject(SparseFloatVector obj) { - assert(maxdim > 0); + protected V filterSingleObject(V obj) { + assert (maxdim > 0); obj.setDimensionality(maxdim); return obj; } @Override - protected SimpleTypeInformation<? super SparseFloatVector> getInputTypeRestriction() { - return TypeUtil.SPARSE_FLOAT_FIELD; + protected SimpleTypeInformation<? super V> getInputTypeRestriction() { + return TypeUtil.SPARSE_VECTOR_VARIABLE_LENGTH; } @Override - protected SimpleTypeInformation<? super SparseFloatVector> convertedType(SimpleTypeInformation<SparseFloatVector> in) { - return new VectorFieldTypeInformation<SparseFloatVector>(SparseFloatVector.class, maxdim, SparseFloatVector.STATIC); + protected SimpleTypeInformation<? super V> convertedType(SimpleTypeInformation<V> in) { + V factory = FilterUtil.guessFactory(in); + return new VectorFieldTypeInformation<V>(in.getRestrictionClass(), maxdim, factory); } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/datasource/filter/normalization/AbstractNormalization.java b/src/de/lmu/ifi/dbs/elki/datasource/filter/normalization/AbstractNormalization.java index 3a629760..5b6c02e3 100644 --- a/src/de/lmu/ifi/dbs/elki/datasource/filter/normalization/AbstractNormalization.java +++ b/src/de/lmu/ifi/dbs/elki/datasource/filter/normalization/AbstractNormalization.java @@ -49,8 +49,8 @@ public abstract class AbstractNormalization<O> extends AbstractConversionFilter< } @Override - public final MultipleObjectsBundle normalizeObjects(MultipleObjectsBundle objects) { - return normalizeObjects(objects); + public MultipleObjectsBundle normalizeObjects(MultipleObjectsBundle objects) throws NonNumericFeaturesException { + return super.filter(objects); } @Override diff --git a/src/de/lmu/ifi/dbs/elki/datasource/filter/normalization/AbstractStreamNormalization.java b/src/de/lmu/ifi/dbs/elki/datasource/filter/normalization/AbstractStreamNormalization.java new file mode 100644 index 00000000..c1524788 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/datasource/filter/normalization/AbstractStreamNormalization.java @@ -0,0 +1,68 @@ +package de.lmu.ifi.dbs.elki.datasource.filter.normalization; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; +import de.lmu.ifi.dbs.elki.datasource.bundle.MultipleObjectsBundle; +import de.lmu.ifi.dbs.elki.datasource.filter.AbstractStreamConversionFilter; +import de.lmu.ifi.dbs.elki.math.linearalgebra.LinearEquationSystem; + +/** + * Abstract super class for all normalizations. + * + * @author Erich Schubert + * + * @param <O> Object type processed + */ +public abstract class AbstractStreamNormalization<O> extends AbstractStreamConversionFilter<O, O> implements Normalization<O> { + /** + * Initializes the option handler and the parameter map. + */ + protected AbstractStreamNormalization() { + super(); + } + + @Override + protected SimpleTypeInformation<? super O> convertedType(SimpleTypeInformation<O> in) { + return in; + } + + @Override + public MultipleObjectsBundle normalizeObjects(MultipleObjectsBundle objects) throws NonNumericFeaturesException { + return super.filter(objects); + } + + @Override + public LinearEquationSystem transform(LinearEquationSystem linearEquationSystem) { + // FIXME: implement. + throw new UnsupportedOperationException("Not yet implemented!"); + } + + @Override + public String toString() { + StringBuffer result = new StringBuffer(); + result.append("normalization class: ").append(getClass().getName()); + return result.toString(); + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/datasource/filter/normalization/InverseDocumentFrequencyNormalization.java b/src/de/lmu/ifi/dbs/elki/datasource/filter/normalization/InverseDocumentFrequencyNormalization.java index 41cce2b9..9350426b 100644 --- a/src/de/lmu/ifi/dbs/elki/datasource/filter/normalization/InverseDocumentFrequencyNormalization.java +++ b/src/de/lmu/ifi/dbs/elki/datasource/filter/normalization/InverseDocumentFrequencyNormalization.java @@ -26,11 +26,10 @@ package de.lmu.ifi.dbs.elki.datasource.filter.normalization; import gnu.trove.iterator.TIntDoubleIterator; import gnu.trove.map.TIntDoubleMap; import gnu.trove.map.hash.TIntDoubleHashMap; -import gnu.trove.map.hash.TIntFloatHashMap; import java.util.BitSet; -import de.lmu.ifi.dbs.elki.data.SparseFloatVector; +import de.lmu.ifi.dbs.elki.data.SparseNumberVector; import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; @@ -40,9 +39,9 @@ import de.lmu.ifi.dbs.elki.data.type.TypeUtil; * * @author Erich Schubert * - * @apiviz.uses SparseFloatVector + * @apiviz.uses SparseNumberVector */ -public class InverseDocumentFrequencyNormalization extends AbstractNormalization<SparseFloatVector> { +public class InverseDocumentFrequencyNormalization<V extends SparseNumberVector<V, ?>> extends AbstractNormalization<V> { /** * The IDF storage */ @@ -61,7 +60,7 @@ public class InverseDocumentFrequencyNormalization extends AbstractNormalization } @Override - protected boolean prepareStart(SimpleTypeInformation<SparseFloatVector> in) { + protected boolean prepareStart(SimpleTypeInformation<V> in) { if(idf.size() > 0) { throw new UnsupportedOperationException("This normalization may only be used once!"); } @@ -70,7 +69,7 @@ public class InverseDocumentFrequencyNormalization extends AbstractNormalization } @Override - protected void prepareProcessInstance(SparseFloatVector featureVector) { + protected void prepareProcessInstance(V featureVector) { BitSet b = featureVector.getNotNullMask(); for(int i = b.nextSetBit(0); i >= 0; i = b.nextSetBit(i + 1)) { if(featureVector.doubleValue(i) >= 0.0) { @@ -98,27 +97,27 @@ public class InverseDocumentFrequencyNormalization extends AbstractNormalization } @Override - protected SparseFloatVector filterSingleObject(SparseFloatVector featureVector) { + protected V filterSingleObject(V featureVector) { BitSet b = featureVector.getNotNullMask(); - TIntFloatHashMap vals = new TIntFloatHashMap(); + TIntDoubleHashMap vals = new TIntDoubleHashMap(); for(int i = b.nextSetBit(0); i >= 0; i = b.nextSetBit(i + 1)) { vals.put(i, (float) (featureVector.doubleValue(i) * idf.get(i))); } - return new SparseFloatVector(vals, featureVector.getDimensionality()); + return featureVector.newNumberVector(vals, featureVector.getDimensionality()); } @Override - public SparseFloatVector restore(SparseFloatVector featureVector) { + public V restore(V featureVector) { BitSet b = featureVector.getNotNullMask(); - TIntFloatHashMap vals = new TIntFloatHashMap(); + TIntDoubleHashMap vals = new TIntDoubleHashMap(); for(int i = b.nextSetBit(0); i >= 0; i = b.nextSetBit(i + 1)) { vals.put(i, (float) (featureVector.doubleValue(i) / idf.get(i))); } - return new SparseFloatVector(vals, featureVector.getDimensionality()); + return featureVector.newNumberVector(vals, featureVector.getDimensionality()); } @Override - protected SimpleTypeInformation<? super SparseFloatVector> getInputTypeRestriction() { - return TypeUtil.SPARSE_FLOAT_FIELD; + protected SimpleTypeInformation<? super V> getInputTypeRestriction() { + return TypeUtil.SPARSE_VECTOR_FIELD; } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/datasource/filter/normalization/LengthNormalization.java b/src/de/lmu/ifi/dbs/elki/datasource/filter/normalization/LengthNormalization.java index 6de7eaba..2edeebf9 100644 --- a/src/de/lmu/ifi/dbs/elki/datasource/filter/normalization/LengthNormalization.java +++ b/src/de/lmu/ifi/dbs/elki/datasource/filter/normalization/LengthNormalization.java @@ -42,7 +42,7 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter; *
* @param <V> vector type
*/
-public class LengthNormalization<V extends NumberVector<V, ?>> extends AbstractNormalization<V> {
+public class LengthNormalization<V extends NumberVector<V, ?>> extends AbstractStreamNormalization<V> {
/**
* Norm to use
*/
diff --git a/src/de/lmu/ifi/dbs/elki/datasource/filter/normalization/TFIDFNormalization.java b/src/de/lmu/ifi/dbs/elki/datasource/filter/normalization/TFIDFNormalization.java index e279c42c..031cfb4c 100644 --- a/src/de/lmu/ifi/dbs/elki/datasource/filter/normalization/TFIDFNormalization.java +++ b/src/de/lmu/ifi/dbs/elki/datasource/filter/normalization/TFIDFNormalization.java @@ -23,11 +23,11 @@ package de.lmu.ifi.dbs.elki.datasource.filter.normalization; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import gnu.trove.map.hash.TIntFloatHashMap; +import gnu.trove.map.hash.TIntDoubleHashMap; import java.util.BitSet; -import de.lmu.ifi.dbs.elki.data.SparseFloatVector; +import de.lmu.ifi.dbs.elki.data.SparseNumberVector; /** * Perform full TF-IDF Normalization as commonly used in text mining. @@ -40,7 +40,7 @@ import de.lmu.ifi.dbs.elki.data.SparseFloatVector; * * @author Erich Schubert */ -public class TFIDFNormalization extends InverseDocumentFrequencyNormalization { +public class TFIDFNormalization<V extends SparseNumberVector<V, ?>> extends InverseDocumentFrequencyNormalization<V> { /** * Constructor. */ @@ -49,7 +49,7 @@ public class TFIDFNormalization extends InverseDocumentFrequencyNormalization { } @Override - protected SparseFloatVector filterSingleObject(SparseFloatVector featureVector) { + protected V filterSingleObject(V featureVector) { BitSet b = featureVector.getNotNullMask(); double sum = 0.0; for(int i = b.nextSetBit(0); i >= 0; i = b.nextSetBit(i + 1)) { @@ -58,10 +58,10 @@ public class TFIDFNormalization extends InverseDocumentFrequencyNormalization { if(sum <= 0) { sum = 1.0; } - TIntFloatHashMap vals = new TIntFloatHashMap(); + TIntDoubleHashMap vals = new TIntDoubleHashMap(); for(int i = b.nextSetBit(0); i >= 0; i = b.nextSetBit(i + 1)) { vals.put(i, (float) (featureVector.doubleValue(i) / sum * idf.get(i))); } - return new SparseFloatVector(vals, featureVector.getDimensionality()); + return featureVector.newNumberVector(vals, featureVector.getDimensionality()); } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/datasource/filter/transform/GlobalPrincipalComponentAnalysisTransform.java b/src/de/lmu/ifi/dbs/elki/datasource/filter/transform/GlobalPrincipalComponentAnalysisTransform.java new file mode 100644 index 00000000..afa21fa6 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/datasource/filter/transform/GlobalPrincipalComponentAnalysisTransform.java @@ -0,0 +1,135 @@ +package de.lmu.ifi.dbs.elki.datasource.filter.transform; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import de.lmu.ifi.dbs.elki.data.NumberVector; +import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; +import de.lmu.ifi.dbs.elki.data.type.TypeUtil; +import de.lmu.ifi.dbs.elki.data.type.VectorFieldTypeInformation; +import de.lmu.ifi.dbs.elki.datasource.filter.AbstractConversionFilter; +import de.lmu.ifi.dbs.elki.math.linearalgebra.CovarianceMatrix; +import de.lmu.ifi.dbs.elki.math.linearalgebra.EigenPair; +import de.lmu.ifi.dbs.elki.math.linearalgebra.SortedEigenPairs; +import de.lmu.ifi.dbs.elki.math.linearalgebra.VMath; +import de.lmu.ifi.dbs.elki.math.linearalgebra.pca.PCAResult; +import de.lmu.ifi.dbs.elki.math.linearalgebra.pca.PCARunner; +import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; + +/** + * Apply principal component analysis to the data set. + * + * TODO: add dimensionality reduction + * + * @author Erich Schubert + * + * @param <O> Vector type + */ +public class GlobalPrincipalComponentAnalysisTransform<O extends NumberVector<O, ?>> extends AbstractConversionFilter<O, O> { + int dim = -1; + + CovarianceMatrix covmat = null; + + double[][] proj = null; + + double[] buf = null; + + double[] mean = null; + + /** + * Constructor. + */ + public GlobalPrincipalComponentAnalysisTransform() { + super(); + } + + @Override + protected boolean prepareStart(SimpleTypeInformation<O> in) { + if(!(in instanceof VectorFieldTypeInformation)) { + throw new AbortException("PCA can only applied to fixed dimensionality vectors"); + } + dim = ((VectorFieldTypeInformation<?>) in).dimensionality(); + covmat = new CovarianceMatrix(dim); + return true; + } + + @Override + protected void prepareProcessInstance(O obj) { + covmat.put(obj); + } + + @Override + protected void prepareComplete() { + mean = covmat.getMeanVector().getArrayRef(); + PCAResult pcares = (new PCARunner<O>(null)).processCovarMatrix(covmat.destroyToSampleMatrix()); + SortedEigenPairs eps = pcares.getEigenPairs(); + covmat = null; + + proj = new double[dim][dim]; + for(int d = 0; d < dim; d++) { + EigenPair ep = eps.getEigenPair(d); + double[] ev = ep.getEigenvector().getArrayRef(); + double eval = Math.sqrt(ep.getEigenvalue()); + // Fill weighted and transposed: + for(int i = 0; i < dim; i++) { + proj[d][i] = ev[i] / eval; + } + } + buf = new double[dim]; + } + + @Override + protected O filterSingleObject(O obj) { + // Shift by mean and copy + for(int i = 0; i < dim; i++) { + buf[i] = obj.doubleValue(i + 1) - mean[i]; + } + double[] p = VMath.times(proj, buf); + return obj.newNumberVector(p); + } + + @Override + protected SimpleTypeInformation<? super O> getInputTypeRestriction() { + return TypeUtil.NUMBER_VECTOR_FIELD; + } + + @Override + protected SimpleTypeInformation<? super O> convertedType(SimpleTypeInformation<O> in) { + return in; + } + + /** + * Parameterization class. + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer<O extends NumberVector<O, ?>> extends AbstractParameterizer { + @Override + protected Object makeInstance() { + return new GlobalPrincipalComponentAnalysisTransform<O>(); + } + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/datasource/filter/transform/package-info.java b/src/de/lmu/ifi/dbs/elki/datasource/filter/transform/package-info.java new file mode 100644 index 00000000..9f8d3262 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/datasource/filter/transform/package-info.java @@ -0,0 +1,26 @@ +/** + * <p>Data space transformations.</p> + */ +/* +This file is part of ELKI: +Environment for Developing KDD-Applications Supported by Index-Structures + +Copyright (C) 2012 +Ludwig-Maximilians-Universität München +Lehr- und Forschungseinheit für Datenbanksysteme +ELKI Development Team + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ +package de.lmu.ifi.dbs.elki.datasource.filter.transform;
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/datasource/parser/AbstractParser.java b/src/de/lmu/ifi/dbs/elki/datasource/parser/AbstractParser.java index dcfb8245..3c294ca4 100644 --- a/src/de/lmu/ifi/dbs/elki/datasource/parser/AbstractParser.java +++ b/src/de/lmu/ifi/dbs/elki/datasource/parser/AbstractParser.java @@ -46,12 +46,12 @@ public abstract class AbstractParser { /** * A pattern defining whitespace. */ - public static final String WHITESPACE_PATTERN = "\\s+"; + public static final String DEFAULT_SEPARATOR = "(\\s+|\\s*[,;]\\s*)"; /** * A quote pattern */ - public static final String QUOTE_CHAR = "\""; + public static final char QUOTE_CHAR = '\"'; /** * A pattern catching most numbers that can be parsed using Double.parseDouble: @@ -63,7 +63,7 @@ public abstract class AbstractParser { /** * OptionID for the column separator parameter (defaults to whitespace as in - * {@link #WHITESPACE_PATTERN}. + * {@link #DEFAULT_SEPARATOR}. */ public static final OptionID COLUMN_SEPARATOR_ID = OptionID.getOrCreateOptionID("parser.colsep", "Column separator pattern. The default assumes whitespace separated data."); @@ -81,7 +81,7 @@ public abstract class AbstractParser { /** * Stores the quotation character */ - protected char quoteChar = QUOTE_CHAR.charAt(0); + protected char quoteChar = QUOTE_CHAR; /** * The comment character. @@ -205,16 +205,16 @@ public abstract class AbstractParser { /** * Stores the quotation character */ - protected char quoteChar = QUOTE_CHAR.charAt(0); + protected char quoteChar = QUOTE_CHAR; @Override protected void makeOptions(Parameterization config) { super.makeOptions(config); - PatternParameter colParam = new PatternParameter(COLUMN_SEPARATOR_ID, WHITESPACE_PATTERN); + PatternParameter colParam = new PatternParameter(COLUMN_SEPARATOR_ID, DEFAULT_SEPARATOR); if(config.grab(colParam)) { colSep = colParam.getValue(); } - StringParameter quoteParam = new StringParameter(QUOTE_ID, new StringLengthConstraint(1, 1), QUOTE_CHAR); + StringParameter quoteParam = new StringParameter(QUOTE_ID, new StringLengthConstraint(1, 1), ""+QUOTE_CHAR); if(config.grab(quoteParam)) { quoteChar = quoteParam.getValue().charAt(0); } diff --git a/src/de/lmu/ifi/dbs/elki/datasource/parser/DoubleVectorLabelParser.java b/src/de/lmu/ifi/dbs/elki/datasource/parser/DoubleVectorLabelParser.java index 97c3a8c8..7bc52a29 100644 --- a/src/de/lmu/ifi/dbs/elki/datasource/parser/DoubleVectorLabelParser.java +++ b/src/de/lmu/ifi/dbs/elki/datasource/parser/DoubleVectorLabelParser.java @@ -48,7 +48,10 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameteriz * @author Arthur Zimek
*
* @apiviz.has DoubleVector
+ *
+ * @deprecated Use NumberVectorLabelParser instead, which defaults to DoubleVector.
*/
+@Deprecated
public class DoubleVectorLabelParser extends NumberVectorLabelParser<DoubleVector> {
/**
* Class logger
@@ -70,7 +73,7 @@ public class DoubleVectorLabelParser extends NumberVectorLabelParser<DoubleVecto * Constructor with default values.
*/
public DoubleVectorLabelParser() {
- this(Pattern.compile(WHITESPACE_PATTERN), QUOTE_CHAR.charAt(0), new BitSet());
+ this(Pattern.compile(DEFAULT_SEPARATOR), QUOTE_CHAR, new BitSet());
}
@Override
diff --git a/src/de/lmu/ifi/dbs/elki/datasource/parser/FloatVectorLabelParser.java b/src/de/lmu/ifi/dbs/elki/datasource/parser/FloatVectorLabelParser.java index 3ea2ac80..6465f89a 100644 --- a/src/de/lmu/ifi/dbs/elki/datasource/parser/FloatVectorLabelParser.java +++ b/src/de/lmu/ifi/dbs/elki/datasource/parser/FloatVectorLabelParser.java @@ -51,7 +51,10 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameteriz * @author Arthur Zimek
*
* @apiviz.has FloatVector
+ *
+ * @deprecated Use NumberVectorLabelParser instead, and use vector type FloatVector.
*/
+@Deprecated
public class FloatVectorLabelParser extends NumberVectorLabelParser<FloatVector> {
/**
* Class logger
diff --git a/src/de/lmu/ifi/dbs/elki/datasource/parser/NumberVectorLabelParser.java b/src/de/lmu/ifi/dbs/elki/datasource/parser/NumberVectorLabelParser.java index 01606e77..53ce44bc 100644 --- a/src/de/lmu/ifi/dbs/elki/datasource/parser/NumberVectorLabelParser.java +++ b/src/de/lmu/ifi/dbs/elki/datasource/parser/NumberVectorLabelParser.java @@ -138,6 +138,16 @@ public class NumberVectorLabelParser<V extends NumberVector<V, ?>> extends Abstr protected BundleMeta meta = null; /** + * Column names + */ + protected List<String> columnnames = null; + + /** + * Bitset to indicate which columns are numeric + */ + protected BitSet labelcolumns = null; + + /** * Current vector */ protected V curvec = null; @@ -153,6 +163,15 @@ public class NumberVectorLabelParser<V extends NumberVector<V, ?>> extends Abstr Event nextevent = null; /** + * Constructor with defaults + * + * @param factory Vector factory + */ + public NumberVectorLabelParser(V factory) { + this(Pattern.compile(DEFAULT_SEPARATOR), QUOTE_CHAR, null, factory); + } + + /** * Constructor * * @param colSep @@ -171,6 +190,8 @@ public class NumberVectorLabelParser<V extends NumberVector<V, ?>> extends Abstr reader = new BufferedReader(new InputStreamReader(in)); lineNumber = 1; dimensionality = DIMENSIONALITY_UNKNOWN; + columnnames = null; + labelcolumns = new BitSet(); } @Override @@ -189,6 +210,10 @@ public class NumberVectorLabelParser<V extends NumberVector<V, ?>> extends Abstr for(String line; (line = reader.readLine()) != null; lineNumber++) { if(!line.startsWith(COMMENT) && line.length() > 0) { parseLineInternal(line); + // Maybe a header column? + if(curvec == null) { + continue; + } if(dimensionality == DIMENSIONALITY_UNKNOWN) { dimensionality = curvec.getDimensionality(); buildMeta(); @@ -202,6 +227,10 @@ public class NumberVectorLabelParser<V extends NumberVector<V, ?>> extends Abstr nextevent = Event.NEXT_OBJECT; return Event.META_CHANGED; } + } else if (curlbl != null && meta != null && meta.size() == 1) { + buildMeta(); + nextevent = Event.NEXT_OBJECT; + return Event.META_CHANGED; } return Event.NEXT_OBJECT; } @@ -219,9 +248,15 @@ public class NumberVectorLabelParser<V extends NumberVector<V, ?>> extends Abstr * Update the meta element. */ protected void buildMeta() { - meta = new BundleMeta(2); - meta.add(getTypeInformation(dimensionality)); - meta.add(TypeUtil.LABELLIST); + if(labelcolumns.cardinality() > 0) { + meta = new BundleMeta(2); + meta.add(getTypeInformation(dimensionality)); + meta.add(TypeUtil.LABELLIST); + } + else { + meta = new BundleMeta(1); + meta.add(getTypeInformation(dimensionality)); + } } @Override @@ -259,6 +294,7 @@ public class NumberVectorLabelParser<V extends NumberVector<V, ?>> extends Abstr } catch(NumberFormatException e) { // Ignore attempt, add to labels below. + labelcolumns.set(i); } } if(labels == null) { @@ -266,7 +302,15 @@ public class NumberVectorLabelParser<V extends NumberVector<V, ?>> extends Abstr } labels.add(ent); } - + // Maybe a label row? + if(lineNumber == 1 && attributes.size() == 0) { + columnnames = labels; + labelcolumns.clear(); + curvec = null; + curlbl = null; + return; + } + // Pass outside via class variables curvec = createDBObject(attributes, ArrayLikeUtil.TDOUBLELISTADAPTER); curlbl = labels; } @@ -293,14 +337,26 @@ public class NumberVectorLabelParser<V extends NumberVector<V, ?>> extends Abstr @SuppressWarnings("unchecked") Class<V> cls = (Class<V>) factory.getClass(); if(dimensionality > 0) { + String[] colnames = null; + if(columnnames != null) { + if(columnnames.size() - labelcolumns.cardinality() == dimensionality) { + colnames = new String[dimensionality]; + for(int i = 0, j = 0; i < columnnames.size(); i++) { + if(labelcolumns.get(i) == false) { + colnames[j] = columnnames.get(i); + j++; + } + } + } + } V f = factory.newNumberVector(new double[dimensionality]); if(f instanceof ByteBufferSerializer) { // TODO: Remove, once we have serializers for all types @SuppressWarnings("unchecked") final ByteBufferSerializer<V> ser = (ByteBufferSerializer<V>) f; - return new VectorFieldTypeInformation<V>(cls, ser, dimensionality, f); + return new VectorFieldTypeInformation<V>(cls, ser, dimensionality, colnames, f); } - return new VectorFieldTypeInformation<V>(cls, dimensionality, f); + return new VectorFieldTypeInformation<V>(cls, dimensionality, colnames, f); } // Variable dimensionality - return non-vector field type if(dimensionality == DIMENSIONALITY_VARIABLE) { diff --git a/src/de/lmu/ifi/dbs/elki/datasource/parser/ParameterizationFunctionLabelParser.java b/src/de/lmu/ifi/dbs/elki/datasource/parser/ParameterizationFunctionLabelParser.java deleted file mode 100644 index 8e240ae9..00000000 --- a/src/de/lmu/ifi/dbs/elki/datasource/parser/ParameterizationFunctionLabelParser.java +++ /dev/null @@ -1,141 +0,0 @@ -package de.lmu.ifi.dbs.elki.datasource.parser; - -/* - This file is part of ELKI: - Environment for Developing KDD-Applications Supported by Index-Structures - - Copyright (C) 2012 - Ludwig-Maximilians-Universität München - Lehr- und Forschungseinheit für Datenbanksysteme - ELKI Development Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -import gnu.trove.list.TDoubleList; -import gnu.trove.list.array.TDoubleArrayList; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Pattern; - -import de.lmu.ifi.dbs.elki.data.LabelList; -import de.lmu.ifi.dbs.elki.data.ParameterizationFunction; -import de.lmu.ifi.dbs.elki.data.type.TypeUtil; -import de.lmu.ifi.dbs.elki.data.type.VectorFieldTypeInformation; -import de.lmu.ifi.dbs.elki.datasource.bundle.MultipleObjectsBundle; -import de.lmu.ifi.dbs.elki.logging.Logging; -import de.lmu.ifi.dbs.elki.utilities.datastructures.arraylike.ArrayLikeUtil; -import de.lmu.ifi.dbs.elki.utilities.documentation.Description; -import de.lmu.ifi.dbs.elki.utilities.documentation.Title; - -/** - * Provides a parser for parsing one point per line, attributes separated by - * whitespace. The parser transforms each point into a parameterization function. - * Several labels may be given per point. A label must not be parseable as - * double (or float). Lines starting with "#" will be ignored. - * - * @author Arthur Zimek - * - * @apiviz.has ParameterizationFunction - */ -@Title("Parameterization Function Label Parser") -@Description("Parser for the following line format:\n" + "A single line provides a single point. Attributes are separated by whitespace. The real values will be parsed as as doubles. Any substring not containing whitespace is tried to be read as double. If this fails, it will be appended to a label. (Thus, any label must not be parseable " + "as double.) Empty lines and lines beginning with \"#\" will be ignored. If any point differs in its dimensionality from other points, the parse method will fail with an Exception.") -public class ParameterizationFunctionLabelParser extends AbstractParser implements Parser { - /** - * Class logger - */ - private static final Logging logger = Logging.getLogger(ParameterizationFunctionLabelParser.class); - - /** - * Constructor. - * - * @param colSep - * @param quoteChar - */ - public ParameterizationFunctionLabelParser(Pattern colSep, char quoteChar) { - super(colSep, quoteChar); - } - - @Override - public MultipleObjectsBundle parse(InputStream in) { - BufferedReader reader = new BufferedReader(new InputStreamReader(in)); - int lineNumber = 1; - int dimensionality = -1; - List<ParameterizationFunction> vectors = new ArrayList<ParameterizationFunction>(); - List<LabelList> labels = new ArrayList<LabelList>(); - try { - for(String line; (line = reader.readLine()) != null; lineNumber++) { - if(!line.startsWith(COMMENT) && line.length() > 0) { - List<String> entries = tokenize(line); - TDoubleList attributes = new TDoubleArrayList(entries.size()); - LabelList labellist = null; - for(String entry : entries) { - try { - double attribute = Double.parseDouble(entry); - attributes.add(attribute); - } - catch(NumberFormatException e) { - if(labellist == null) { - labellist = new LabelList(1); - } - labellist.add(entry); - } - } - - if(dimensionality < 0) { - dimensionality = attributes.size(); - } - else if(dimensionality != attributes.size()) { - throw new IllegalArgumentException("Differing dimensionality in line " + lineNumber + ":" + attributes.size() + " != " + dimensionality); - } - vectors.add(ParameterizationFunction.STATIC.newNumberVector(attributes, ArrayLikeUtil.TDOUBLELISTADAPTER)); - labels.add(labellist); - } - } - } - catch(IOException e) { - throw new IllegalArgumentException("Error while parsing line " + lineNumber + "."); - } - - return MultipleObjectsBundle.makeSimple(getTypeInformation(dimensionality), vectors, TypeUtil.LABELLIST, labels); - } - - protected VectorFieldTypeInformation<ParameterizationFunction> getTypeInformation(int dimensionality) { - return new VectorFieldTypeInformation<ParameterizationFunction>(ParameterizationFunction.class, dimensionality, new ParameterizationFunction(new double[dimensionality])); - } - - @Override - protected Logging getLogger() { - return logger; - } - - /** - * Parameterization class. - * - * @author Erich Schubert - * - * @apiviz.exclude - */ - public static class Parameterizer extends AbstractParser.Parameterizer { - @Override - protected ParameterizationFunctionLabelParser makeInstance() { - return new ParameterizationFunctionLabelParser(colSep, quoteChar); - } - } -}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/datasource/parser/SparseFloatVectorLabelParser.java b/src/de/lmu/ifi/dbs/elki/datasource/parser/SparseFloatVectorLabelParser.java index 83d6ed13..d0ab7a85 100644 --- a/src/de/lmu/ifi/dbs/elki/datasource/parser/SparseFloatVectorLabelParser.java +++ b/src/de/lmu/ifi/dbs/elki/datasource/parser/SparseFloatVectorLabelParser.java @@ -23,20 +23,12 @@ package de.lmu.ifi.dbs.elki.datasource.parser; along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-import gnu.trove.map.hash.TIntFloatHashMap;
-
import java.util.BitSet;
-import java.util.List;
import java.util.regex.Pattern;
-import de.lmu.ifi.dbs.elki.data.LabelList;
import de.lmu.ifi.dbs.elki.data.SparseFloatVector;
-import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation;
-import de.lmu.ifi.dbs.elki.data.type.VectorFieldTypeInformation;
-import de.lmu.ifi.dbs.elki.logging.Logging;
import de.lmu.ifi.dbs.elki.utilities.documentation.Description;
import de.lmu.ifi.dbs.elki.utilities.documentation.Title;
-import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException;
/**
* <p>
@@ -50,9 +42,17 @@ import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; * <p>
* A line is expected in the following format: The first entry of each line is
* the number of attributes with coordinate value not zero. Subsequent entries
- * are of the form (index, value), where index is the number of the
- * corresponding dimension, and value is the value of the corresponding
- * attribute.
+ * are of the form <code>index value </code> each, where index is the number of
+ * the corresponding dimension, and value is the value of the corresponding
+ * attribute. A complet line then could look like this:
+ *
+ * <pre>
+ * 3 7 12.34 8 56.78 11 1.234 objectlabel
+ * </pre>
+ *
+ * where <code>3</code> indicates there are three attributes set,
+ * <code>7,8,11</code> are the attributes indexes and there is a non-numerical
+ * object label.
* </p>
* <p>
* An index can be specified to identify an entry to be treated as class label.
@@ -62,87 +62,25 @@ import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; * @author Arthur Zimek
*
* @apiviz.has SparseFloatVector
+ *
+ * @deprecated Use {@link SparseNumberVectorLabelParser} instead!
*/
// FIXME: Maxdim!
@Title("Sparse Float Vector Label Parser")
-@Description("Parser for the following line format:\n" + "A single line provides a single point. Entries are separated by whitespace. " + "The values will be parsed as floats (resulting in a set of SparseFloatVectors). A line is expected in the following format: The first entry of each line is the number of attributes with coordinate value not zero. Subsequent entries are of the form (index, value), where index is the number of the corresponding dimension, and value is the value of the corresponding attribute." + "Any pair of two subsequent substrings not containing whitespace is tried to be read as int and float. If this fails for the first of the pair (interpreted ans index), it will be appended to a label. (Thus, any label must not be parseable as Integer.) If the float component is not parseable, an exception will be thrown. Empty lines and lines beginning with \"#\" will be ignored. Having the file parsed completely, the maximum occuring dimensionality is set as dimensionality to all created SparseFloatvectors.")
-public class SparseFloatVectorLabelParser extends NumberVectorLabelParser<SparseFloatVector> {
- /**
- * Class logger
- */
- private static final Logging logger = Logging.getLogger(SparseFloatVectorLabelParser.class);
-
- /**
- * Holds the dimensionality of the parsed data which is the maximum occurring
- * index of any attribute.
- */
- private int maxdim = -1;
-
+@Description("Parser for the following line format:\n" + "A single line provides a single point. Entries are separated by whitespace. " + "The values will be parsed as floats (resulting in a set of SparseFloatVectors). A line is expected in the following format: The first entry of each line is the number of attributes with coordinate value not zero. Subsequent entries are of the form (index, value), where index is the number of the corresponding dimension, and value is the value of the corresponding attribute." + "Any pair of two subsequent substrings not containing whitespace is tried to be read as int and float. If this fails for the first of the pair (interpreted ans index), it will be appended to a label. (Thus, any label must not be parseable as Integer.) If the float component is not parseable, an exception will be thrown. Empty lines and lines beginning with \"#\" will be ignored.")
+@Deprecated
+public class SparseFloatVectorLabelParser extends SparseNumberVectorLabelParser<SparseFloatVector> {
/**
* Constructor.
*
- * @param colSep
- * @param quoteChar
- * @param labelIndices
+ * @param colSep Column separator
+ * @param quoteChar Quotation character
+ * @param labelIndices Label indexes
*/
public SparseFloatVectorLabelParser(Pattern colSep, char quoteChar, BitSet labelIndices) {
super(colSep, quoteChar, labelIndices, SparseFloatVector.STATIC);
}
- @Override
- protected void parseLineInternal(String line) {
- List<String> entries = tokenize(line);
- int cardinality = Integer.parseInt(entries.get(0));
-
- TIntFloatHashMap values = new TIntFloatHashMap(cardinality, 1);
- LabelList labels = null;
-
- for(int i = 1; i < entries.size() - 1; i++) {
- if(!labelIndices.get(i)) {
- try {
- int index = Integer.valueOf(entries.get(i));
- if(index > maxdim) {
- maxdim = index;
- }
- float attribute = Float.valueOf(entries.get(i));
- values.put(index, attribute);
- i++;
- }
- catch(NumberFormatException e) {
- if(labels == null) {
- labels = new LabelList(1);
- }
- labels.add(entries.get(i));
- continue;
- }
- }
- else {
- if(labels == null) {
- labels = new LabelList(1);
- }
- labels.add(entries.get(i));
- }
- }
- curvec = new SparseFloatVector(values, maxdim);
- curlbl = labels;
- }
-
- @Override
- protected SimpleTypeInformation<SparseFloatVector> getTypeInformation(int dimensionality) {
- if(dimensionality > 0) {
- return new VectorFieldTypeInformation<SparseFloatVector>(SparseFloatVector.class, dimensionality, new SparseFloatVector(SparseFloatVector.EMPTYMAP, dimensionality));
- }
- if(dimensionality == DIMENSIONALITY_VARIABLE) {
- return new SimpleTypeInformation<SparseFloatVector>(SparseFloatVector.class);
- }
- throw new AbortException("No vectors were read from the input file - cannot determine vector data type.");
- }
-
- @Override
- protected Logging getLogger() {
- return logger;
- }
-
/**
* Parameterization class.
*
@@ -150,7 +88,7 @@ public class SparseFloatVectorLabelParser extends NumberVectorLabelParser<Sparse *
* @apiviz.exclude
*/
- public static class Parameterizer extends NumberVectorLabelParser.Parameterizer<SparseFloatVector> {
+ public static class Parameterizer extends SparseNumberVectorLabelParser.Parameterizer<SparseFloatVector> {
@Override
protected SparseFloatVectorLabelParser makeInstance() {
return new SparseFloatVectorLabelParser(colSep, quoteChar, labelIndices);
diff --git a/src/de/lmu/ifi/dbs/elki/datasource/parser/SparseNumberVectorLabelParser.java b/src/de/lmu/ifi/dbs/elki/datasource/parser/SparseNumberVectorLabelParser.java new file mode 100644 index 00000000..917dd2aa --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/datasource/parser/SparseNumberVectorLabelParser.java @@ -0,0 +1,185 @@ +package de.lmu.ifi.dbs.elki.datasource.parser;
+
+/*
+ This file is part of ELKI:
+ Environment for Developing KDD-Applications Supported by Index-Structures +
+ Copyright (C) 2012
+ Ludwig-Maximilians-Universität München
+ Lehr- und Forschungseinheit für Datenbanksysteme
+ ELKI Development Team +
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version. +
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details. +
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import gnu.trove.map.hash.TIntDoubleHashMap;
+
+import java.util.BitSet;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import de.lmu.ifi.dbs.elki.data.LabelList;
+import de.lmu.ifi.dbs.elki.data.SparseDoubleVector;
+import de.lmu.ifi.dbs.elki.data.SparseFloatVector;
+import de.lmu.ifi.dbs.elki.data.SparseNumberVector;
+import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation;
+import de.lmu.ifi.dbs.elki.data.type.VectorFieldTypeInformation;
+import de.lmu.ifi.dbs.elki.logging.Logging;
+import de.lmu.ifi.dbs.elki.utilities.documentation.Description;
+import de.lmu.ifi.dbs.elki.utilities.documentation.Title;
+import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter;
+
+/**
+ * <p>
+ * Provides a parser for parsing one point per line, attributes separated by
+ * whitespace.
+ * </p>
+ * <p>
+ * Several labels may be given per point. A label must not be parseable as
+ * double. Lines starting with "#" will be ignored.
+ * </p>
+ * <p>
+ * A line is expected in the following format: The first entry of each line is
+ * the number of attributes with coordinate value not zero. Subsequent entries
+ * are of the form <code>index value </code> each, where index is the number of
+ * the corresponding dimension, and value is the value of the corresponding
+ * attribute. A complet line then could look like this:
+ *
+ * <pre>
+ * 3 7 12.34 8 56.78 11 1.234 objectlabel
+ * </pre>
+ *
+ * where <code>3</code> indicates there are three attributes set,
+ * <code>7,8,11</code> are the attributes indexes and there is a non-numerical
+ * object label.
+ * </p>
+ * <p>
+ * An index can be specified to identify an entry to be treated as class label.
+ * This index counts all entries (numeric and labels as well) starting with 0.
+ * </p>
+ *
+ * @author Arthur Zimek
+ *
+ * @apiviz.has SparseNumberVector
+ */
+// FIXME: Maxdim!
+@Title("Sparse Vector Label Parser")
+@Description("Parser for the following line format:\n" + "A single line provides a single point. Entries are separated by whitespace. " + "The values will be parsed as floats (resulting in a set of SparseFloatVectors). A line is expected in the following format: The first entry of each line is the number of attributes with coordinate value not zero. Subsequent entries are of the form (index, value), where index is the number of the corresponding dimension, and value is the value of the corresponding attribute." + "Any pair of two subsequent substrings not containing whitespace is tried to be read as int and float. If this fails for the first of the pair (interpreted ans index), it will be appended to a label. (Thus, any label must not be parseable as Integer.) If the float component is not parseable, an exception will be thrown. Empty lines and lines beginning with \"#\" will be ignored.")
+public class SparseNumberVectorLabelParser<V extends SparseNumberVector<V, ?>> extends NumberVectorLabelParser<V> {
+ /**
+ * Class logger
+ */
+ private static final Logging logger = Logging.getLogger(SparseNumberVectorLabelParser.class);
+
+ /**
+ * Holds the dimensionality of the parsed data which is the maximum occurring
+ * index of any attribute.
+ */
+ private int maxdim = -1;
+
+ /**
+ * Constructor.
+ *
+ * @param colSep Column separator
+ * @param quoteChar Quotation character
+ * @param labelIndices Label indexes
+ * @param factory Vector factory
+ */
+ public SparseNumberVectorLabelParser(Pattern colSep, char quoteChar, BitSet labelIndices, V factory) {
+ super(colSep, quoteChar, labelIndices, factory);
+ }
+
+ @Override
+ protected void parseLineInternal(String line) {
+ List<String> entries = tokenize(line);
+ int cardinality = Integer.parseInt(entries.get(0));
+
+ TIntDoubleHashMap values = new TIntDoubleHashMap(cardinality, 1);
+ LabelList labels = null;
+
+ for(int i = 1; i < entries.size() - 1; i++) {
+ if(!labelIndices.get(i)) {
+ try {
+ int index = Integer.valueOf(entries.get(i));
+ if(index >= maxdim) {
+ maxdim = index + 1;
+ }
+ double attribute = Double.valueOf(entries.get(i));
+ values.put(index, attribute);
+ i++;
+ }
+ catch(NumberFormatException e) {
+ if(labels == null) {
+ labels = new LabelList(1);
+ }
+ labels.add(entries.get(i));
+ continue;
+ }
+ }
+ else {
+ if(labels == null) {
+ labels = new LabelList(1);
+ }
+ labels.add(entries.get(i));
+ }
+ }
+ if(values.size() > maxdim) {
+ throw new AbortException("Invalid sparse vector seen: " + line);
+ }
+ curvec = factory.newNumberVector(values, maxdim);
+ curlbl = labels;
+ }
+
+ @Override
+ protected SimpleTypeInformation<V> getTypeInformation(int dimensionality) {
+ @SuppressWarnings("unchecked")
+ Class<V> cls = (Class<V>) factory.getClass();
+ if(dimensionality > 0) {
+ return new VectorFieldTypeInformation<V>(cls, dimensionality, factory.newNumberVector(SparseDoubleVector.EMPTYMAP, dimensionality));
+ }
+ if(dimensionality == DIMENSIONALITY_VARIABLE) {
+ return new SimpleTypeInformation<V>(cls);
+ }
+ throw new AbortException("No vectors were read from the input file - cannot determine vector data type.");
+ }
+
+ @Override
+ protected Logging getLogger() {
+ return logger;
+ }
+
+ /**
+ * Parameterization class.
+ *
+ * @author Erich Schubert
+ *
+ * @apiviz.exclude
+ */
+ public static class Parameterizer<V extends SparseNumberVector<V, ?>> extends NumberVectorLabelParser.Parameterizer<V> {
+ @Override
+ protected void getFactory(Parameterization config) {
+ ObjectParameter<V> factoryP = new ObjectParameter<V>(VECTOR_TYPE_ID, SparseNumberVector.class, SparseFloatVector.class);
+ if(config.grab(factoryP)) {
+ factory = factoryP.instantiateClass(config);
+ }
+ }
+
+ @Override
+ protected SparseNumberVectorLabelParser<V> makeInstance() {
+ return new SparseNumberVectorLabelParser<V>(colSep, quoteChar, labelIndices, factory);
+ }
+ }
+}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/AbstractDBIDDistanceFunction.java b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/AbstractDBIDDistanceFunction.java index f1a53803..9216c4ac 100644 --- a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/AbstractDBIDDistanceFunction.java +++ b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/AbstractDBIDDistanceFunction.java @@ -26,6 +26,7 @@ package de.lmu.ifi.dbs.elki.distance.distancefunction; import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.distance.DBIDDistanceQuery; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.relation.Relation; @@ -47,7 +48,7 @@ public abstract class AbstractDBIDDistanceFunction<D extends Distance<D>> implem } @Override - abstract public D distance(DBID o1, DBID o2); + abstract public D distance(DBIDRef o1, DBIDRef o2); @Override abstract public D getDistanceFactory(); diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/DBIDDistanceFunction.java b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/DBIDDistanceFunction.java index c4e6a891..857b96e1 100644 --- a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/DBIDDistanceFunction.java +++ b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/DBIDDistanceFunction.java @@ -24,6 +24,7 @@ package de.lmu.ifi.dbs.elki.distance.distancefunction; */ import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; /** @@ -49,5 +50,5 @@ public interface DBIDDistanceFunction<D extends Distance<?>> extends DistanceFun * @param id2 second object id * @return the distance between the two objects specified by their object ids */ - D distance(DBID id1, DBID id2); + D distance(DBIDRef id1, DBIDRef id2); }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/LocallyWeightedDistanceFunction.java b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/LocallyWeightedDistanceFunction.java index ee3cc438..90c3a94a 100644 --- a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/LocallyWeightedDistanceFunction.java +++ b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/LocallyWeightedDistanceFunction.java @@ -25,7 +25,7 @@ package de.lmu.ifi.dbs.elki.distance.distancefunction; import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.data.spatial.SpatialComparable; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancevalue.DoubleDistance; import de.lmu.ifi.dbs.elki.index.preprocessed.LocalProjectionIndex; @@ -122,7 +122,7 @@ public class LocallyWeightedDistanceFunction<V extends NumberVector<?, ?>> exten * distance function */ @Override - public DoubleDistance distance(DBID id1, DBID id2) { + public DoubleDistance distance(DBIDRef id1, DBIDRef id2) { Matrix m1 = index.getLocalProjection(id1).similarityMatrix(); Matrix m2 = index.getLocalProjection(id2).similarityMatrix(); diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/MinKDistance.java b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/MinKDistance.java index ef805c6d..8f19af60 100644 --- a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/MinKDistance.java +++ b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/MinKDistance.java @@ -25,7 +25,7 @@ package de.lmu.ifi.dbs.elki.distance.distancefunction; import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.database.QueryUtil; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.DatabaseQuery; import de.lmu.ifi.dbs.elki.database.query.distance.AbstractDatabaseDistanceQuery; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; @@ -155,7 +155,7 @@ public class MinKDistance<O, D extends Distance<D>> extends AbstractDatabaseDist } @Override - public D distance(DBID id1, DBID id2) { + public D distance(DBIDRef id1, DBIDRef id2) { KNNResult<D> neighborhood = knnQuery.getKNNForDBID(id1, k); D truedist = parentDistanceQuery.distance(id1, id2); return computeReachdist(neighborhood, truedist); diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/ProxyDistanceFunction.java b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/ProxyDistanceFunction.java index 9afdbecd..48d49a0c 100644 --- a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/ProxyDistanceFunction.java +++ b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/ProxyDistanceFunction.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.distance.distancefunction; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; @@ -65,7 +65,7 @@ public class ProxyDistanceFunction<O, D extends Distance<D>> extends AbstractDBI } @Override - public D distance(DBID o1, DBID o2) { + public D distance(DBIDRef o1, DBIDRef o2) { return inner.distance(o1, o2); } diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/RandomStableDistanceFunction.java b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/RandomStableDistanceFunction.java index 0632d475..b54f4e74 100644 --- a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/RandomStableDistanceFunction.java +++ b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/RandomStableDistanceFunction.java @@ -25,7 +25,7 @@ package de.lmu.ifi.dbs.elki.distance.distancefunction; import java.util.Random; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.distance.distancevalue.DoubleDistance; import de.lmu.ifi.dbs.elki.utilities.Util; import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; @@ -64,8 +64,8 @@ public class RandomStableDistanceFunction extends AbstractDBIDDistanceFunction<D } @Override - public DoubleDistance distance(DBID o1, DBID o2) { - int c = o1.compareTo(o2); + public DoubleDistance distance(DBIDRef o1, DBIDRef o2) { + int c = o1.compareDBID(o2); if(c == 0) { return DoubleDistance.FACTORY.nullDistance(); } @@ -73,7 +73,7 @@ public class RandomStableDistanceFunction extends AbstractDBIDDistanceFunction<D if(c > 0) { return distance(o2, o1); } - return new DoubleDistance(pseudoRandom(seed, Util.mixHashCodes(o1.hashCode(), o2.hashCode(), (int) seed))); + return new DoubleDistance(pseudoRandom(seed, Util.mixHashCodes(o1.getDBID().hashCode(), o2.getDBID().hashCode(), (int) seed))); } /** diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/SharedNearestNeighborJaccardDistanceFunction.java b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/SharedNearestNeighborJaccardDistanceFunction.java index cfc4217e..4ccf4163 100644 --- a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/SharedNearestNeighborJaccardDistanceFunction.java +++ b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/SharedNearestNeighborJaccardDistanceFunction.java @@ -23,9 +23,8 @@ package de.lmu.ifi.dbs.elki.distance.distancefunction; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.Iterator; - -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancevalue.DoubleDistance; @@ -39,8 +38,11 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameteriz * * @author Erich Schubert * - * @apiviz.composedOf SharedNearestNeighborIndex.Factory - * @apiviz.has SharedNearestNeighborJaccardDistanceFunction.Instance oneway - - «create» + * @apiviz.uses + * de.lmu.ifi.dbs.elki.index.preprocessed.snn.SharedNearestNeighborIndex + * .Factory + * @apiviz.has SharedNearestNeighborJaccardDistanceFunction.Instance oneway - - + * «create» * * @param <O> object type */ @@ -92,39 +94,36 @@ public class SharedNearestNeighborJaccardDistanceFunction<O> extends AbstractInd static protected double jaccardCoefficient(DBIDs neighbors1, DBIDs neighbors2) { int intersection = 0; int union = 0; - Iterator<DBID> iter1 = neighbors1.iterator(); - Iterator<DBID> iter2 = neighbors2.iterator(); - DBID neighbors1ID = iter1.hasNext() ? iter1.next() : null; - DBID neighbors2ID = iter2.hasNext() ? iter2.next() : null; - while(neighbors1ID != null && neighbors2ID != null) { + DBIDIter iter1 = neighbors1.iter(); + DBIDIter iter2 = neighbors2.iter(); + while(iter1.valid() && iter2.valid()) { + final int comp = iter1.compareDBID(iter2); union++; - if(neighbors1ID.equals(neighbors2ID)) { + if(comp == 0) { intersection++; - neighbors1ID = iter1.hasNext() ? iter1.next() : null; - neighbors2ID = iter2.hasNext() ? iter2.next() : null; + iter1.advance(); + iter2.advance(); } - else if(neighbors2ID.compareTo(neighbors1ID) > 0) { - neighbors1ID = iter1.hasNext() ? iter1.next() : null; + else if(comp < 0) { + iter1.advance(); } - else // neighbors1ID > neighbors2ID + else // iter2 < iter1 { - neighbors2ID = iter2.hasNext() ? iter2.next() : null; + iter2.advance(); } } // Count remaining objects - while(neighbors1ID != null) { + for(; iter1.valid(); iter1.advance()) { union++; - neighbors1ID = iter1.hasNext() ? iter1.next() : null; } - while(neighbors2ID != null) { + for(; iter2.valid(); iter2.advance()) { union++; - neighbors2ID = iter2.hasNext() ? iter2.next() : null; } return ((double) intersection) / union; } @Override - public DoubleDistance distance(DBID id1, DBID id2) { + public DoubleDistance distance(DBIDRef id1, DBIDRef id2) { DBIDs neighbors1 = index.getNearestNeighborSet(id1); DBIDs neighbors2 = index.getNearestNeighborSet(id2); return new DoubleDistance(1.0 - jaccardCoefficient(neighbors1, neighbors2)); diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/SparseEuclideanDistanceFunction.java b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/SparseEuclideanDistanceFunction.java new file mode 100644 index 00000000..f98ed234 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/SparseEuclideanDistanceFunction.java @@ -0,0 +1,109 @@ +package de.lmu.ifi.dbs.elki.distance.distancefunction; +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import java.util.BitSet; + +import de.lmu.ifi.dbs.elki.data.SparseNumberVector; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; + +/** + * Euclidean distance function. Optimized for sparse vectors. + * + * @author Erich Schubert + */ +public class SparseEuclideanDistanceFunction extends SparseLPNormDistanceFunction { + /** + * Static instance + */ + public static final SparseEuclideanDistanceFunction STATIC = new SparseEuclideanDistanceFunction(); + + /** + * Constructor. + * + * @deprecated Use static instance instead. + */ + @Deprecated + public SparseEuclideanDistanceFunction() { + super(2.0); + } + + @Override + public double doubleDistance(SparseNumberVector<?, ?> v1, SparseNumberVector<?, ?> v2) { + // Get the bit masks + BitSet b1 = v1.getNotNullMask(); + BitSet b2 = v2.getNotNullMask(); + double sqrDist = 0; + int i1 = b1.nextSetBit(0); + int i2 = b2.nextSetBit(0); + while(i1 >= 0 && i2 >= 0) { + if(i1 == i2) { + // Set in both + double manhattanI = v1.doubleValue(i1) - v2.doubleValue(i2); + sqrDist += manhattanI * manhattanI; + i1 = b1.nextSetBit(i1 + 1); + i2 = b2.nextSetBit(i2 + 1); + } + else if(i1 < i2 && i1 >= 0) { + // In first only + double manhattanI = v1.doubleValue(i1); + sqrDist += manhattanI * manhattanI; + i1 = b1.nextSetBit(i1 + 1); + } + else { + // In second only + double manhattanI = v2.doubleValue(i2); + sqrDist += manhattanI * manhattanI; + i2 = b1.nextSetBit(i2 + 1); + } + } + return Math.sqrt(sqrDist); + } + + @Override + public double doubleNorm(SparseNumberVector<?, ?> v1) { + double sqrDist = 0; + // Get the bit masks + BitSet b1 = v1.getNotNullMask(); + // Set in first only + for(int i = b1.nextSetBit(0); i >= 0; i = b1.nextSetBit(i + 1)) { + double manhattanI = v1.doubleValue(i); + sqrDist += manhattanI * manhattanI; + } + return Math.sqrt(sqrDist); + } + + /** + * Parameterizer + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer extends AbstractParameterizer { + @Override + protected SparseEuclideanDistanceFunction makeInstance() { + return SparseEuclideanDistanceFunction.STATIC; + } + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/SparseLPNormDistanceFunction.java b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/SparseLPNormDistanceFunction.java new file mode 100644 index 00000000..e552d1e5 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/SparseLPNormDistanceFunction.java @@ -0,0 +1,162 @@ +package de.lmu.ifi.dbs.elki.distance.distancefunction; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import java.util.BitSet; + +import de.lmu.ifi.dbs.elki.data.SparseNumberVector; +import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; +import de.lmu.ifi.dbs.elki.data.type.TypeUtil; +import de.lmu.ifi.dbs.elki.distance.distancevalue.DoubleDistance; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.GreaterConstraint; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.DoubleParameter; + +/** + * Provides a LP-Norm for FeatureVectors. + * + * @author Erich Schubert + */ +// TODO: implement SpatialDistanceFunction +public class SparseLPNormDistanceFunction extends AbstractPrimitiveDistanceFunction<SparseNumberVector<?, ?>, DoubleDistance> implements DoubleNorm<SparseNumberVector<?, ?>> { + /** + * Keeps the currently set p. + */ + private double p; + + /** + * Provides a LP-Norm for FeatureVectors. + */ + public SparseLPNormDistanceFunction(double p) { + super(); + } + + @Override + public double doubleDistance(SparseNumberVector<?, ?> v1, SparseNumberVector<?, ?> v2) { + // Get the bit masks + BitSet b1 = v1.getNotNullMask(); + BitSet b2 = v2.getNotNullMask(); + double sqrDist = 0; + int i1 = b1.nextSetBit(0); + int i2 = b2.nextSetBit(0); + while(i1 >= 0 && i2 >= 0) { + if(i1 == i2) { + // Set in both + double manhattanI = Math.abs(v1.doubleValue(i1) - v2.doubleValue(i2)); + sqrDist += Math.pow(manhattanI, p); + i1 = b1.nextSetBit(i1 + 1); + i2 = b2.nextSetBit(i2 + 1); + } + else if(i1 < i2 && i1 >= 0) { + // In first only + double manhattanI = Math.abs(v1.doubleValue(i1)); + sqrDist += Math.pow(manhattanI, p); + i1 = b1.nextSetBit(i1 + 1); + } + else { + // In second only + double manhattanI = Math.abs(v2.doubleValue(i2)); + sqrDist += Math.pow(manhattanI, p); + i2 = b1.nextSetBit(i2 + 1); + } + } + return Math.pow(sqrDist, 1.0 / p); + } + + @Override + public double doubleNorm(SparseNumberVector<?, ?> v1) { + double sqrDist = 0; + // Get the bit masks + BitSet b1 = v1.getNotNullMask(); + // Set in first only + for(int i = b1.nextSetBit(0); i >= 0; i = b1.nextSetBit(i + 1)) { + double manhattanI = Math.abs(v1.doubleValue(i)); + sqrDist += Math.pow(manhattanI, p); + } + return Math.pow(sqrDist, 1.0 / p); + } + + @Override + public DoubleDistance norm(SparseNumberVector<?, ?> obj) { + return new DoubleDistance(doubleNorm(obj)); + } + + @Override + public DoubleDistance distance(SparseNumberVector<?, ?> v1, SparseNumberVector<?, ?> v2) { + return new DoubleDistance(doubleDistance(v1, v2)); + } + + @Override + public DoubleDistance getDistanceFactory() { + return DoubleDistance.FACTORY; + } + + @Override + public SimpleTypeInformation<? super SparseNumberVector<?, ?>> getInputTypeRestriction() { + return TypeUtil.SPARSE_VECTOR_VARIABLE_LENGTH; + } + + @Override + public boolean isMetric() { + return (p >= 1); + } + + /** + * Parameterizer + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer extends AbstractParameterizer { + /** + * Value for p + */ + double p = 2.0; + + @Override + protected void makeOptions(Parameterization config) { + super.makeOptions(config); + DoubleParameter pP = new DoubleParameter(LPNormDistanceFunction.P_ID, new GreaterConstraint(0)); + if(config.grab(pP)) { + p = pP.getValue(); + } + } + + @Override + protected SparseLPNormDistanceFunction makeInstance() { + if(p == 2.0) { + return SparseEuclideanDistanceFunction.STATIC; + } + if(p == 1.0) { + return SparseManhattanDistanceFunction.STATIC; + } + if(p == Double.POSITIVE_INFINITY) { + return SparseMaximumDistanceFunction.STATIC; + } + return new SparseLPNormDistanceFunction(p); + } + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/SparseManhattanDistanceFunction.java b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/SparseManhattanDistanceFunction.java new file mode 100644 index 00000000..9b8399bf --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/SparseManhattanDistanceFunction.java @@ -0,0 +1,106 @@ +package de.lmu.ifi.dbs.elki.distance.distancefunction; +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import java.util.BitSet; + +import de.lmu.ifi.dbs.elki.data.SparseNumberVector; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; + +/** + * Manhattan distance function. Optimized for sparse vectors. + * + * @author Erich Schubert + */ +public class SparseManhattanDistanceFunction extends SparseLPNormDistanceFunction { + /** + * Static instance + */ + public static final SparseManhattanDistanceFunction STATIC = new SparseManhattanDistanceFunction(); + + /** + * Constructor. + * + * @deprecated Use static instance instead. + */ + @Deprecated + public SparseManhattanDistanceFunction() { + super(1.0); + } + + @Override + public double doubleDistance(SparseNumberVector<?, ?> v1, SparseNumberVector<?, ?> v2) { + // Get the bit masks + BitSet b1 = v1.getNotNullMask(); + BitSet b2 = v2.getNotNullMask(); + double sqrDist = 0; + int i1 = b1.nextSetBit(0); + int i2 = b2.nextSetBit(0); + while(i1 >= 0 && i2 >= 0) { + if(i1 == i2) { + // Set in both + sqrDist += Math.abs(v1.doubleValue(i1) - v2.doubleValue(i2)); + i1 = b1.nextSetBit(i1 + 1); + i2 = b2.nextSetBit(i2 + 1); + } + else if(i1 < i2 && i1 >= 0) { + // In first only + sqrDist += Math.abs(v1.doubleValue(i1)); + i1 = b1.nextSetBit(i1 + 1); + } + else { + // In second only + double manhattanI = Math.abs(v2.doubleValue(i2)); + sqrDist += manhattanI; + i2 = b1.nextSetBit(i2 + 1); + } + } + return sqrDist; + } + + @Override + public double doubleNorm(SparseNumberVector<?, ?> v1) { + double sqrDist = 0; + // Get the bit masks + BitSet b1 = v1.getNotNullMask(); + // Set in first only + for(int i = b1.nextSetBit(0); i >= 0; i = b1.nextSetBit(i + 1)) { + sqrDist += Math.abs(v1.doubleValue(i)); + } + return sqrDist; + } + + /** + * Parameterizer + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer extends AbstractParameterizer { + @Override + protected SparseManhattanDistanceFunction makeInstance() { + return SparseManhattanDistanceFunction.STATIC; + } + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/SparseMaximumDistanceFunction.java b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/SparseMaximumDistanceFunction.java new file mode 100644 index 00000000..1b399e83 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/SparseMaximumDistanceFunction.java @@ -0,0 +1,106 @@ +package de.lmu.ifi.dbs.elki.distance.distancefunction; + +import java.util.BitSet; + +import de.lmu.ifi.dbs.elki.data.SparseNumberVector; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +/** + * Maximum distance function. Optimized for sparse vectors. + * + * @author Erich Schubert + */ +public class SparseMaximumDistanceFunction extends SparseLPNormDistanceFunction { + /** + * Static instance + */ + public static final SparseMaximumDistanceFunction STATIC = new SparseMaximumDistanceFunction(); + + /** + * Constructor. + * + * @deprecated Use static instance + */ + @Deprecated + public SparseMaximumDistanceFunction() { + super(Double.POSITIVE_INFINITY); + } + + @Override + public double doubleDistance(SparseNumberVector<?, ?> v1, SparseNumberVector<?, ?> v2) { + // Get the bit masks + BitSet b1 = v1.getNotNullMask(); + BitSet b2 = v2.getNotNullMask(); + double sqrDist = 0; + int i1 = b1.nextSetBit(0); + int i2 = b2.nextSetBit(0); + while(i1 >= 0 && i2 >= 0) { + if(i1 == i2) { + // Set in both + sqrDist = Math.max(sqrDist, Math.abs(v1.doubleValue(i1) - v2.doubleValue(i2))); + i1 = b1.nextSetBit(i1 + 1); + i2 = b2.nextSetBit(i2 + 1); + } + else if(i1 < i2 && i1 >= 0) { + // In first only + sqrDist = Math.max(sqrDist, Math.abs(v1.doubleValue(i1))); + i1 = b1.nextSetBit(i1 + 1); + } + else { + // In second only + double manhattanI = Math.abs(v2.doubleValue(i2)); + sqrDist = Math.max(sqrDist, manhattanI); + i2 = b1.nextSetBit(i2 + 1); + } + } + return sqrDist; + } + + @Override + public double doubleNorm(SparseNumberVector<?, ?> v1) { + double sqrDist = 0; + // Get the bit masks + BitSet b1 = v1.getNotNullMask(); + // Set in first only + for(int i = b1.nextSetBit(0); i >= 0; i = b1.nextSetBit(i + 1)) { + sqrDist = Math.max(sqrDist, Math.abs(v1.doubleValue(i))); + } + return sqrDist; + } + + /** + * Parameterizer + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer extends AbstractParameterizer { + @Override + protected SparseMaximumDistanceFunction makeInstance() { + return SparseMaximumDistanceFunction.STATIC; + } + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/adapter/AbstractSimilarityAdapter.java b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/adapter/AbstractSimilarityAdapter.java index 206a32e7..19e80724 100644 --- a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/adapter/AbstractSimilarityAdapter.java +++ b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/adapter/AbstractSimilarityAdapter.java @@ -24,7 +24,7 @@ package de.lmu.ifi.dbs.elki.distance.distancefunction.adapter; */ import de.lmu.ifi.dbs.elki.data.type.TypeInformation; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.query.similarity.SimilarityQuery; import de.lmu.ifi.dbs.elki.database.relation.Relation; @@ -148,7 +148,7 @@ public abstract class AbstractSimilarityAdapter<O> extends AbstractDatabaseDista public abstract double transform(double similarity); @Override - public DoubleDistance distance(DBID id1, DBID id2) { + public DoubleDistance distance(DBIDRef id1, DBIDRef id2) { final NumberDistance<?, ?> sim = similarityQuery.similarity(id1, id2); return new DoubleDistance(transform(sim.doubleValue())); } diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/correlation/ERiCDistanceFunction.java b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/correlation/ERiCDistanceFunction.java index d59851b9..25766a91 100644 --- a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/correlation/ERiCDistanceFunction.java +++ b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/correlation/ERiCDistanceFunction.java @@ -24,7 +24,7 @@ package de.lmu.ifi.dbs.elki.distance.distancefunction.correlation; */ import de.lmu.ifi.dbs.elki.data.NumberVector; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.AbstractIndexBasedDistanceFunction; import de.lmu.ifi.dbs.elki.distance.distancefunction.FilteredLocalPCABasedDistanceFunction; @@ -248,7 +248,7 @@ public class ERiCDistanceFunction extends AbstractIndexBasedDistanceFunction<Num * than the pca of o2. */ @Override - public BitDistance distance(DBID id1, DBID id2) { + public BitDistance distance(DBIDRef id1, DBIDRef id2) { PCAFilteredResult pca1 = index.getLocalProjection(id1); PCAFilteredResult pca2 = index.getLocalProjection(id2); V v1 = relation.get(id1); diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/correlation/PCABasedCorrelationDistanceFunction.java b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/correlation/PCABasedCorrelationDistanceFunction.java index 05ae0e13..083fc013 100644 --- a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/correlation/PCABasedCorrelationDistanceFunction.java +++ b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/correlation/PCABasedCorrelationDistanceFunction.java @@ -24,7 +24,7 @@ package de.lmu.ifi.dbs.elki.distance.distancefunction.correlation; */ import de.lmu.ifi.dbs.elki.data.NumberVector; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.AbstractIndexBasedDistanceFunction; import de.lmu.ifi.dbs.elki.distance.distancefunction.FilteredLocalPCABasedDistanceFunction; @@ -134,7 +134,7 @@ public class PCABasedCorrelationDistanceFunction extends AbstractIndexBasedDista } @Override - public PCACorrelationDistance distance(DBID id1, DBID id2) { + public PCACorrelationDistance distance(DBIDRef id1, DBIDRef id2) { PCAFilteredResult pca1 = index.getLocalProjection(id1); PCAFilteredResult pca2 = index.getLocalProjection(id2); V dv1 = relation.get(id1); @@ -159,15 +159,15 @@ public class PCABasedCorrelationDistanceFunction extends AbstractIndexBasedDista public int correlationDistance(PCAFilteredResult pca1, PCAFilteredResult pca2, int dimensionality) { // TODO nur in eine Richtung? // pca of rv1 - Matrix v1 = pca1.getEigenvectors(); - Matrix v1_strong = pca1.adapatedStrongEigenvectors(); - Matrix e1_czech = pca1.selectionMatrixOfStrongEigenvectors(); + Matrix v1 = pca1.getEigenvectors().copy(); + Matrix v1_strong = pca1.adapatedStrongEigenvectors().copy(); + Matrix e1_czech = pca1.selectionMatrixOfStrongEigenvectors().copy(); int lambda1 = pca1.getCorrelationDimension(); // pca of rv2 - Matrix v2 = pca2.getEigenvectors(); - Matrix v2_strong = pca2.adapatedStrongEigenvectors(); - Matrix e2_czech = pca2.selectionMatrixOfStrongEigenvectors(); + Matrix v2 = pca2.getEigenvectors().copy(); + Matrix v2_strong = pca2.adapatedStrongEigenvectors().copy(); + Matrix e2_czech = pca2.selectionMatrixOfStrongEigenvectors().copy(); int lambda2 = pca2.getCorrelationDimension(); // for all strong eigenvectors of rv2 diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/external/DiskCacheBasedDoubleDistanceFunction.java b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/external/DiskCacheBasedDoubleDistanceFunction.java index 9d92d727..9e865039 100644 --- a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/external/DiskCacheBasedDoubleDistanceFunction.java +++ b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/external/DiskCacheBasedDoubleDistanceFunction.java @@ -26,7 +26,7 @@ package de.lmu.ifi.dbs.elki.distance.distancefunction.external; import java.io.File; import java.io.IOException; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.distance.distancefunction.AbstractDBIDDistanceFunction; import de.lmu.ifi.dbs.elki.distance.distancevalue.DoubleDistance; import de.lmu.ifi.dbs.elki.persistent.OnDiskUpperTriangleMatrix; @@ -94,7 +94,7 @@ public class DiskCacheBasedDoubleDistanceFunction extends AbstractDBIDDistanceFu * @return the distance between the two objects specified by their objects ids */ @Override - public DoubleDistance distance(DBID id1, DBID id2) { + public DoubleDistance distance(DBIDRef id1, DBIDRef id2) { if(id1 == null) { return getDistanceFactory().undefinedDistance(); } diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/external/DiskCacheBasedFloatDistanceFunction.java b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/external/DiskCacheBasedFloatDistanceFunction.java index ff6b5e3d..df29c280 100644 --- a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/external/DiskCacheBasedFloatDistanceFunction.java +++ b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/external/DiskCacheBasedFloatDistanceFunction.java @@ -26,7 +26,7 @@ package de.lmu.ifi.dbs.elki.distance.distancefunction.external; import java.io.File; import java.io.IOException; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.distance.distancefunction.AbstractDBIDDistanceFunction; import de.lmu.ifi.dbs.elki.distance.distancevalue.FloatDistance; import de.lmu.ifi.dbs.elki.persistent.OnDiskUpperTriangleMatrix; @@ -94,7 +94,7 @@ public class DiskCacheBasedFloatDistanceFunction extends AbstractDBIDDistanceFun * @return the distance between the two objects specified by their objects ids */ @Override - public FloatDistance distance(DBID id1, DBID id2) { + public FloatDistance distance(DBIDRef id1, DBIDRef id2) { if(id1 == null) { return getDistanceFactory().undefinedDistance(); } diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/external/FileBasedDoubleDistanceFunction.java b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/external/FileBasedDoubleDistanceFunction.java index 50344460..e8f379cd 100644 --- a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/external/FileBasedDoubleDistanceFunction.java +++ b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/external/FileBasedDoubleDistanceFunction.java @@ -29,8 +29,8 @@ import java.io.IOException; import java.io.InputStream; import java.util.Map; -import de.lmu.ifi.dbs.elki.database.ids.DBID; import de.lmu.ifi.dbs.elki.database.ids.DBIDPair; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; @@ -107,7 +107,7 @@ public class FileBasedDoubleDistanceFunction extends AbstractDBIDDistanceFunctio * @return the distance between the two objects specified by their objects ids */ @Override - public DoubleDistance distance(DBID id1, DBID id2) { + public DoubleDistance distance(DBIDRef id1, DBIDRef id2) { if(id1 == null) { return getDistanceFactory().undefinedDistance(); } diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/external/FileBasedFloatDistanceFunction.java b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/external/FileBasedFloatDistanceFunction.java index 10a4f206..675a46f3 100644 --- a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/external/FileBasedFloatDistanceFunction.java +++ b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/external/FileBasedFloatDistanceFunction.java @@ -29,8 +29,9 @@ import java.io.IOException; import java.io.InputStream; import java.util.Map; -import de.lmu.ifi.dbs.elki.database.ids.DBID; import de.lmu.ifi.dbs.elki.database.ids.DBIDPair; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; +import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.distance.distancefunction.AbstractDBIDDistanceFunction; import de.lmu.ifi.dbs.elki.distance.distancevalue.FloatDistance; import de.lmu.ifi.dbs.elki.utilities.FileUtil; @@ -44,7 +45,6 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.ListParamet import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.FileParameter; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter; -import de.lmu.ifi.dbs.elki.utilities.pairs.Pair; /** * Provides a DistanceFunction that is based on float distances given by a @@ -105,7 +105,7 @@ public class FileBasedFloatDistanceFunction extends AbstractDBIDDistanceFunction * @return the distance between the two objects specified by their objects ids */ @Override - public FloatDistance distance(DBID id1, DBID id2) { + public FloatDistance distance(DBIDRef id1, DBIDRef id2) { if(id1 == null) { return getDistanceFactory().undefinedDistance(); } @@ -117,7 +117,7 @@ public class FileBasedFloatDistanceFunction extends AbstractDBIDDistanceFunction return distance(id2, id1); } - return cache.get(new Pair<DBID, DBID>(id1, id2)); + return cache.get(DBIDUtil.newPair(id1, id2)); } private void loadCache(DistanceParser<FloatDistance> parser, File matrixfile) throws IOException { diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/external/NumberDistanceParser.java b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/external/NumberDistanceParser.java index c7a09e1a..3af02ac3 100644 --- a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/external/NumberDistanceParser.java +++ b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/external/NumberDistanceParser.java @@ -35,7 +35,9 @@ import java.util.regex.Pattern; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDPair; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; import de.lmu.ifi.dbs.elki.datasource.bundle.MultipleObjectsBundle; @@ -141,13 +143,13 @@ public class NumberDistanceParser<D extends NumberDistance<D, ?>> extends Abstra } // check if all distance values are specified - for(DBID id1 : ids) { - for(DBID id2 : ids) { - if(id2.getIntegerID() < id1.getIntegerID()) { + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + for (DBIDIter iter2 = ids.iter(); iter2.valid(); iter2.advance()) { + if(iter2.getIntegerID() < iter.getIntegerID()) { continue; } - if(!containsKey(id1, id2, distanceCache)) { - throw new IllegalArgumentException("Distance value for " + id1 + " - " + id2 + " is missing!"); + if(!containsKey(iter, iter2, distanceCache)) { + throw new IllegalArgumentException("Distance value for " + iter.getDBID() + " - " + iter2.getDBID() + " is missing!"); } } } @@ -159,8 +161,8 @@ public class NumberDistanceParser<D extends NumberDistance<D, ?>> extends Abstra // This is unusual for ELKI, but here we really need an ArrayList, not a // DBIDs object. So convert. List<DBID> objects = new ArrayList<DBID>(ids.size()); - for(DBID id : ids) { - objects.add(id); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + objects.add(iter.getDBID()); } return new DistanceParsingResult<D>(MultipleObjectsBundle.makeSimple(TypeUtil.DBID, objects), distanceCache); } @@ -197,7 +199,7 @@ public class NumberDistanceParser<D extends NumberDistance<D, ?>> extends Abstra * @return <tt>true</tt> if this cache contains a distance value for the * specified ids, false otherwise */ - public boolean containsKey(DBID id1, DBID id2, Map<DBIDPair, D> cache) { + public boolean containsKey(DBIDRef id1, DBIDRef id2, Map<DBIDPair, D> cache) { if(id1.getIntegerID() > id2.getIntegerID()) { return containsKey(id2, id1, cache); } diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/subspace/AbstractPreferenceVectorBasedCorrelationDistanceFunction.java b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/subspace/AbstractPreferenceVectorBasedCorrelationDistanceFunction.java index 8b5f526e..8164cffa 100644 --- a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/subspace/AbstractPreferenceVectorBasedCorrelationDistanceFunction.java +++ b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/subspace/AbstractPreferenceVectorBasedCorrelationDistanceFunction.java @@ -27,6 +27,7 @@ import java.util.BitSet; import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.AbstractIndexBasedDistanceFunction; import de.lmu.ifi.dbs.elki.distance.distancevalue.PreferenceVectorBasedCorrelationDistance; @@ -125,7 +126,7 @@ public abstract class AbstractPreferenceVectorBasedCorrelationDistanceFunction<V } @Override - public PreferenceVectorBasedCorrelationDistance distance(DBID id1, DBID id2) { + public PreferenceVectorBasedCorrelationDistance distance(DBIDRef id1, DBIDRef id2) { BitSet preferenceVector1 = index.getPreferenceVector(id1); BitSet preferenceVector2 = index.getPreferenceVector(id2); V v1 = relation.get(id1); diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/subspace/LocalSubspaceDistanceFunction.java b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/subspace/LocalSubspaceDistanceFunction.java index b7d36264..8bb912e0 100644 --- a/src/de/lmu/ifi/dbs/elki/distance/distancefunction/subspace/LocalSubspaceDistanceFunction.java +++ b/src/de/lmu/ifi/dbs/elki/distance/distancefunction/subspace/LocalSubspaceDistanceFunction.java @@ -24,7 +24,7 @@ package de.lmu.ifi.dbs.elki.distance.distancefunction.subspace; */ import de.lmu.ifi.dbs.elki.data.NumberVector; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.AbstractIndexBasedDistanceFunction; import de.lmu.ifi.dbs.elki.distance.distancefunction.FilteredLocalPCABasedDistanceFunction; @@ -91,7 +91,7 @@ public class LocalSubspaceDistanceFunction extends AbstractIndexBasedDistanceFun * */ @Override - public SubspaceDistance distance(DBID id1, DBID id2) { + public SubspaceDistance distance(DBIDRef id1, DBIDRef id2) { PCAFilteredResult pca1 = index.getLocalProjection(id1); PCAFilteredResult pca2 = index.getLocalProjection(id2); V o1 = relation.get(id1); diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancevalue/DoubleDistance.java b/src/de/lmu/ifi/dbs/elki/distance/distancevalue/DoubleDistance.java index ce63b4e1..ca84fbfd 100644 --- a/src/de/lmu/ifi/dbs/elki/distance/distancevalue/DoubleDistance.java +++ b/src/de/lmu/ifi/dbs/elki/distance/distancevalue/DoubleDistance.java @@ -157,6 +157,18 @@ public class DoubleDistance extends NumberDistance<DoubleDistance, Double> { return Double.compare(this.value, other.value); } + @Override + public boolean equals(Object o) { + if(this == o) { + return true; + } + if(o == null || getClass() != o.getClass()) { + return false; + } + double delta = Math.abs(value - ((DoubleDistance) o).value); + return delta < Double.MIN_NORMAL; + } + /** * An infinite DoubleDistance is based on {@link Double#POSITIVE_INFINITY * Double.POSITIVE_INFINITY}. diff --git a/src/de/lmu/ifi/dbs/elki/distance/distancevalue/FloatDistance.java b/src/de/lmu/ifi/dbs/elki/distance/distancevalue/FloatDistance.java index 5aaaab91..adb951ae 100644 --- a/src/de/lmu/ifi/dbs/elki/distance/distancevalue/FloatDistance.java +++ b/src/de/lmu/ifi/dbs/elki/distance/distancevalue/FloatDistance.java @@ -162,6 +162,18 @@ public class FloatDistance extends NumberDistance<FloatDistance, Float> { return Float.compare(this.value, other.value); } + @Override + public boolean equals(Object o) { + if(this == o) { + return true; + } + if(o == null || getClass() != o.getClass()) { + return false; + } + float delta = Math.abs(value - ((FloatDistance) o).value); + return delta < Float.MIN_NORMAL; + } + /** * An infinite FloatDistance is based on {@link Float#POSITIVE_INFINITY * Float.POSITIVE_INFINITY}. diff --git a/src/de/lmu/ifi/dbs/elki/distance/similarityfunction/FractionalSharedNearestNeighborSimilarityFunction.java b/src/de/lmu/ifi/dbs/elki/distance/similarityfunction/FractionalSharedNearestNeighborSimilarityFunction.java index ec4460d8..32653173 100644 --- a/src/de/lmu/ifi/dbs/elki/distance/similarityfunction/FractionalSharedNearestNeighborSimilarityFunction.java +++ b/src/de/lmu/ifi/dbs/elki/distance/similarityfunction/FractionalSharedNearestNeighborSimilarityFunction.java @@ -23,10 +23,9 @@ package de.lmu.ifi.dbs.elki.distance.similarityfunction; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.Iterator; - import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancevalue.DoubleDistance; @@ -94,29 +93,28 @@ public class FractionalSharedNearestNeighborSimilarityFunction<O> extends Abstra */ static protected int countSharedNeighbors(DBIDs neighbors1, DBIDs neighbors2) { int intersection = 0; - Iterator<DBID> iter1 = neighbors1.iterator(); - Iterator<DBID> iter2 = neighbors2.iterator(); - DBID neighbors1ID = iter1.hasNext() ? iter1.next() : null; - DBID neighbors2ID = iter2.hasNext() ? iter2.next() : null; - while(neighbors1ID != null && neighbors2ID != null) { - if(neighbors1ID.equals(neighbors2ID)) { + DBIDIter iter1 = neighbors1.iter(); + DBIDIter iter2 = neighbors2.iter(); + while(iter1.valid() && iter2.valid()) { + final int comp = iter1.compareDBID(iter2); + if(comp == 0) { intersection++; - neighbors1ID = iter1.hasNext() ? iter1.next() : null; - neighbors2ID = iter2.hasNext() ? iter2.next() : null; + iter1.advance(); + iter2.advance(); } - else if(neighbors2ID.compareTo(neighbors1ID) > 0) { - neighbors1ID = iter1.hasNext() ? iter1.next() : null; + else if(comp < 0) { + iter1.advance(); } - else // neighbors1ID > neighbors2ID + else // iter2 < iter1 { - neighbors2ID = iter2.hasNext() ? iter2.next() : null; + iter2.advance(); } } return intersection; } @Override - public DoubleDistance similarity(DBID id1, DBID id2) { + public DoubleDistance similarity(DBIDRef id1, DBIDRef id2) { DBIDs neighbors1 = index.getNearestNeighborSet(id1); DBIDs neighbors2 = index.getNearestNeighborSet(id2); int intersection = countSharedNeighbors(neighbors1, neighbors2); diff --git a/src/de/lmu/ifi/dbs/elki/distance/similarityfunction/SharedNearestNeighborSimilarityFunction.java b/src/de/lmu/ifi/dbs/elki/distance/similarityfunction/SharedNearestNeighborSimilarityFunction.java index 0025ee75..98117008 100644 --- a/src/de/lmu/ifi/dbs/elki/distance/similarityfunction/SharedNearestNeighborSimilarityFunction.java +++ b/src/de/lmu/ifi/dbs/elki/distance/similarityfunction/SharedNearestNeighborSimilarityFunction.java @@ -23,9 +23,8 @@ package de.lmu.ifi.dbs.elki.distance.similarityfunction; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.Iterator; - -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.SetDBIDs; import de.lmu.ifi.dbs.elki.database.relation.Relation; @@ -70,22 +69,21 @@ public class SharedNearestNeighborSimilarityFunction<O> extends AbstractIndexBas */ static protected int countSharedNeighbors(DBIDs neighbors1, DBIDs neighbors2) { int intersection = 0; - Iterator<DBID> iter1 = neighbors1.iterator(); - Iterator<DBID> iter2 = neighbors2.iterator(); - DBID neighbors1ID = iter1.hasNext() ? iter1.next() : null; - DBID neighbors2ID = iter2.hasNext() ? iter2.next() : null; - while((iter1.hasNext() || iter2.hasNext()) && neighbors1ID != null && neighbors2ID != null) { - if(neighbors1ID.equals(neighbors2ID)) { + DBIDIter iter1 = neighbors1.iter(); + DBIDIter iter2 = neighbors2.iter(); + while(iter1.valid() && iter2.valid()) { + final int comp = iter1.compareDBID(iter2); + if(comp == 0) { intersection++; - neighbors1ID = iter1.hasNext() ? iter1.next() : null; - neighbors2ID = iter2.hasNext() ? iter2.next() : null; + iter1.advance(); + iter2.advance(); } - else if(neighbors2ID.compareTo(neighbors1ID) > 0) { - neighbors1ID = iter1.hasNext() ? iter1.next() : null; + else if(comp < 0) { + iter1.advance(); } - else // neighbors1ID > neighbors2ID + else // iter2 < iter1 { - neighbors2ID = iter2.hasNext() ? iter2.next() : null; + iter2.advance(); } } return intersection; @@ -119,7 +117,7 @@ public class SharedNearestNeighborSimilarityFunction<O> extends AbstractIndexBas } @Override - public IntegerDistance similarity(DBID id1, DBID id2) { + public IntegerDistance similarity(DBIDRef id1, DBIDRef id2) { DBIDs neighbors1 = index.getNearestNeighborSet(id1); DBIDs neighbors2 = index.getNearestNeighborSet(id2); return new IntegerDistance(countSharedNeighbors(neighbors1, neighbors2)); diff --git a/src/de/lmu/ifi/dbs/elki/evaluation/AutomaticEvaluation.java b/src/de/lmu/ifi/dbs/elki/evaluation/AutomaticEvaluation.java new file mode 100644 index 00000000..888777a5 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/evaluation/AutomaticEvaluation.java @@ -0,0 +1,159 @@ +package de.lmu.ifi.dbs.elki.evaluation; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import java.util.Collection; +import java.util.Iterator; +import java.util.regex.Pattern; + +import de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.ByLabelOrAllInOneClustering; +import de.lmu.ifi.dbs.elki.data.Cluster; +import de.lmu.ifi.dbs.elki.data.Clustering; +import de.lmu.ifi.dbs.elki.evaluation.clustering.EvaluateClustering; +import de.lmu.ifi.dbs.elki.evaluation.histogram.ComputeOutlierHistogram; +import de.lmu.ifi.dbs.elki.evaluation.outlier.OutlierPrecisionAtKCurve; +import de.lmu.ifi.dbs.elki.evaluation.outlier.OutlierPrecisionRecallCurve; +import de.lmu.ifi.dbs.elki.evaluation.outlier.OutlierROCCurve; +import de.lmu.ifi.dbs.elki.logging.Logging; +import de.lmu.ifi.dbs.elki.result.HierarchicalResult; +import de.lmu.ifi.dbs.elki.result.Result; +import de.lmu.ifi.dbs.elki.result.ResultUtil; +import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; +import de.lmu.ifi.dbs.elki.utilities.scaling.LinearScaling; + +/** + * Evaluator that tries to auto-run a number of evaluation methods. + * + * @author Erich Schubert + * + * @apiviz.uses OutlierResult + * @apiviz.uses Clustering + * @apiviz.composedOf OutlierROCCurve + * @apiviz.composedOf OutlierPrecisionAtKCurve + * @apiviz.composedOf OutlierPrecisionRecallCurve + * @apiviz.composedOf ComputeOutlierHistogram + * @apiviz.composedOf EvaluateClustering + */ +public class AutomaticEvaluation implements Evaluator { + /** + * Class logger + */ + private static final Logging logger = Logging.getLogger(AutomaticEvaluation.class); + + @Override + public void processNewResult(HierarchicalResult baseResult, Result newResult) { + autoEvaluateClusterings(baseResult, newResult); + autoEvaluateOutliers(baseResult, newResult); + } + + protected void autoEvaluateOutliers(HierarchicalResult baseResult, Result newResult) { + Collection<OutlierResult> outliers = ResultUtil.filterResults(newResult, OutlierResult.class); + if(logger.isDebugging()) { + logger.debug("Number of new outlier results: " + outliers.size()); + } + if(outliers.size() > 0) { + ResultUtil.ensureClusteringResult(ResultUtil.findDatabase(baseResult), baseResult); + Collection<Clustering<?>> clusterings = ResultUtil.filterResults(baseResult, Clustering.class); + if(clusterings.size() == 0) { + logger.warning("Could not find a clustering result, even after running 'ensureClusteringResult'?!?"); + return; + } + Clustering<?> basec = clusterings.iterator().next(); + // Find minority class label + int min = Integer.MAX_VALUE; + int total = 0; + String label = null; + if(basec.getAllClusters().size() > 1) { + for(Cluster<?> c : basec.getAllClusters()) { + final int csize = c.getIDs().size(); + total += csize; + if(csize < min) { + min = csize; + label = c.getName(); + } + } + } + if(label == null) { + logger.warning("Could not evaluate outlier results, as I could not find a minority label."); + return; + } + if(min == 1) { + logger.warning("The minority class label had a single object. Try using 'ClassLabelFilter' to identify the class label column."); + } + if(min > 0.05 * total) { + logger.warning("The minority class I discovered (labeled '" + label + "') has " + (min * 100. / total) + "% of objects. Outlier classes should be more rare!"); + } + logger.verbose("Evaluating using minority class: " + label); + Pattern pat = Pattern.compile("^" + Pattern.quote(label) + "$"); + // Compute ROC curve + new OutlierROCCurve(pat).processNewResult(baseResult, newResult); + // Compute Precision at k + new OutlierPrecisionAtKCurve(pat, min * 2).processNewResult(baseResult, newResult); + // Compute ROC curve + new OutlierPrecisionRecallCurve(pat).processNewResult(baseResult, newResult); + // Compute outlier histogram + new ComputeOutlierHistogram(pat, 50, new LinearScaling(), false).processNewResult(baseResult, newResult); + } + } + + protected void autoEvaluateClusterings(HierarchicalResult baseResult, Result newResult) { + Collection<Clustering<?>> clusterings = ResultUtil.filterResults(newResult, Clustering.class); + if(logger.isDebugging()) { + logger.warning("Number of new clustering results: " + clusterings.size()); + } + for (Iterator<Clustering<?>> c = clusterings.iterator(); c.hasNext();) { + Clustering<?> test = c.next(); + if ("allinone-clustering".equals(test.getShortName())) { + c.remove(); + } + else if ("allinnoise-clustering".equals(test.getShortName())) { + c.remove(); + } + else if ("bylabel-clustering".equals(test.getShortName())) { + c.remove(); + } + else if ("bymodel-clustering".equals(test.getShortName())) { + c.remove(); + } + } + if (clusterings.size() > 0) { + new EvaluateClustering(new ByLabelOrAllInOneClustering(), false, true).processNewResult(baseResult, newResult); + } + } + + /** + * Parameterization class + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer extends AbstractParameterizer { + @Override + protected AutomaticEvaluation makeInstance() { + return new AutomaticEvaluation(); + } + } +} diff --git a/src/de/lmu/ifi/dbs/elki/evaluation/NoAutomaticEvaluation.java b/src/de/lmu/ifi/dbs/elki/evaluation/NoAutomaticEvaluation.java new file mode 100644 index 00000000..453a337f --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/evaluation/NoAutomaticEvaluation.java @@ -0,0 +1,56 @@ +package de.lmu.ifi.dbs.elki.evaluation; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import de.lmu.ifi.dbs.elki.result.HierarchicalResult; +import de.lmu.ifi.dbs.elki.result.Result; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; + +/** + * No-operation evaluator, that only serves the purpose of explicitely disabling + * the default value of {@link AutomaticEvaluation}, if you do not want + * evaluation to run. + * + * @author Erich Schubert + */ +public class NoAutomaticEvaluation implements Evaluator { + @Override + public void processNewResult(HierarchicalResult baseResult, Result newResult) { + return; + } + + /** + * Parameterization class + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer extends AbstractParameterizer { + @Override + protected NoAutomaticEvaluation makeInstance() { + return new NoAutomaticEvaluation(); + } + } +} diff --git a/src/de/lmu/ifi/dbs/elki/evaluation/clustering/BCubed.java b/src/de/lmu/ifi/dbs/elki/evaluation/clustering/BCubed.java index d8087cf6..fdfaaaa9 100644 --- a/src/de/lmu/ifi/dbs/elki/evaluation/clustering/BCubed.java +++ b/src/de/lmu/ifi/dbs/elki/evaluation/clustering/BCubed.java @@ -69,7 +69,6 @@ public class BCubed { bCubedRecall += (recall * table.contingency[i1][i2]); } } - bCubedPrecision = bCubedPrecision / table.contingency[table.size1][table.size2]; bCubedRecall = bCubedRecall / table.contingency[table.size1][table.size2]; } diff --git a/src/de/lmu/ifi/dbs/elki/evaluation/clustering/ClusterContingencyTable.java b/src/de/lmu/ifi/dbs/elki/evaluation/clustering/ClusterContingencyTable.java index ac156a4e..70f6e6ce 100644 --- a/src/de/lmu/ifi/dbs/elki/evaluation/clustering/ClusterContingencyTable.java +++ b/src/de/lmu/ifi/dbs/elki/evaluation/clustering/ClusterContingencyTable.java @@ -28,7 +28,7 @@ import java.util.List; import de.lmu.ifi.dbs.elki.data.Cluster; import de.lmu.ifi.dbs.elki.data.Clustering; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.math.MeanVariance; @@ -37,6 +37,15 @@ import de.lmu.ifi.dbs.elki.math.MeanVariance; * Class storing the contingency table and related data on two clusterings. * * @author Erich Schubert + * + * @apiviz.landmark + * + * @apiviz.uses Clustering + * @apiviz.composedOf PairCounting + * @apiviz.composedOf Entropy + * @apiviz.composedOf EditDistance + * @apiviz.composedOf BCubed + * @apiviz.composedOf SetMatchingPurity */ public class ClusterContingencyTable { /** @@ -156,8 +165,8 @@ public class ClusterContingencyTable { for(int i2 = 0; it2.hasNext(); i2++) { final Cluster<?> c2 = it2.next(); int count = 0; - for(DBID id : c2.getIDs()) { - if(ids.contains(id)) { + for(DBIDIter iter = c2.getIDs().iter(); iter.valid(); iter.advance()) { + if(ids.contains(iter.getDBID())) { count++; } } diff --git a/src/de/lmu/ifi/dbs/elki/evaluation/clustering/Entropy.java b/src/de/lmu/ifi/dbs/elki/evaluation/clustering/Entropy.java index ded70327..b174f3ed 100644 --- a/src/de/lmu/ifi/dbs/elki/evaluation/clustering/Entropy.java +++ b/src/de/lmu/ifi/dbs/elki/evaluation/clustering/Entropy.java @@ -168,6 +168,9 @@ public class Entropy { * @return Joint Normalized Mutual information */ public double entropyNMIJoint() { + if (entropyJoint() == 0) { + return 0; + } return (entropyMutualInformation() / entropyJoint()); } @@ -204,6 +207,9 @@ public class Entropy { * @return Sqrt Normalized Mutual information */ public double entropyNMISqrt() { + if (entropyFirst() * entropySecond() <= 0) { + return entropyMutualInformation(); + } return (entropyMutualInformation() / Math.sqrt(entropyFirst() * entropySecond())); } diff --git a/src/de/lmu/ifi/dbs/elki/evaluation/clustering/EvaluateClustering.java b/src/de/lmu/ifi/dbs/elki/evaluation/clustering/EvaluateClustering.java index c82f5cf5..a57fa188 100644 --- a/src/de/lmu/ifi/dbs/elki/evaluation/clustering/EvaluateClustering.java +++ b/src/de/lmu/ifi/dbs/elki/evaluation/clustering/EvaluateClustering.java @@ -23,11 +23,11 @@ package de.lmu.ifi.dbs.elki.evaluation.clustering; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.Iterator; +import java.util.Collection; import java.util.List; import de.lmu.ifi.dbs.elki.algorithm.clustering.ClusteringAlgorithm; -import de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.ByLabelClustering; +import de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.ByLabelOrAllInOneClustering; import de.lmu.ifi.dbs.elki.data.Clustering; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.evaluation.Evaluator; @@ -115,9 +115,8 @@ public class EvaluateClustering implements Evaluator { Clustering<?> refc = null; // Try to find an existing reference clustering (globally) if(refc == null) { - Iterator<Clustering<?>> cs = ResultUtil.filteredResults(baseResult, Clustering.class); - while(cs.hasNext()) { - Clustering<?> test = cs.next(); + Collection<Clustering<?>> cs = ResultUtil.filterResults(baseResult, Clustering.class); + for(Clustering<?> test : cs) { if(isReferenceResult(test)) { refc = test; break; @@ -126,9 +125,8 @@ public class EvaluateClustering implements Evaluator { } // Try to find an existing reference clustering (locally) if(refc == null) { - Iterator<Clustering<?>> cs = ResultUtil.filteredResults(result, Clustering.class); - while(cs.hasNext()) { - Clustering<?> test = cs.next(); + Collection<Clustering<?>> cs = ResultUtil.filterResults(result, Clustering.class); + for(Clustering<?> test : cs) { if(isReferenceResult(test)) { refc = test; break; @@ -164,10 +162,16 @@ public class EvaluateClustering implements Evaluator { private boolean isReferenceResult(Clustering<?> t) { // FIXME: don't hard-code strings - if(t.getShortName().startsWith("bylabel-")) { + if("bylabel-clustering".equals(t.getShortName())) { return true; } - if(t.getShortName().startsWith("bymodel-")) { + if("bymodel-clustering".equals(t.getShortName())) { + return true; + } + if("allinone-clustering".equals(t.getShortName())) { + return true; + } + if("allinnoise-clustering".equals(t.getShortName())) { return true; } return false; @@ -264,7 +268,7 @@ public class EvaluateClustering implements Evaluator { @Override protected void makeOptions(Parameterization config) { super.makeOptions(config); - ObjectParameter<ClusteringAlgorithm<?>> referencealgP = new ObjectParameter<ClusteringAlgorithm<?>>(REFERENCE_ID, ClusteringAlgorithm.class, ByLabelClustering.class); + ObjectParameter<ClusteringAlgorithm<?>> referencealgP = new ObjectParameter<ClusteringAlgorithm<?>>(REFERENCE_ID, ClusteringAlgorithm.class, ByLabelOrAllInOneClustering.class); if(config.grab(referencealgP)) { referencealg = referencealgP.instantiateClass(config); } diff --git a/src/de/lmu/ifi/dbs/elki/evaluation/clustering/pairsegments/ClusterPairSegmentAnalysis.java b/src/de/lmu/ifi/dbs/elki/evaluation/clustering/pairsegments/ClusterPairSegmentAnalysis.java index 2471b135..123cd4a1 100644 --- a/src/de/lmu/ifi/dbs/elki/evaluation/clustering/pairsegments/ClusterPairSegmentAnalysis.java +++ b/src/de/lmu/ifi/dbs/elki/evaluation/clustering/pairsegments/ClusterPairSegmentAnalysis.java @@ -1,4 +1,26 @@ package de.lmu.ifi.dbs.elki.evaluation.clustering.pairsegments; +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ import java.util.List; diff --git a/src/de/lmu/ifi/dbs/elki/evaluation/clustering/pairsegments/Segment.java b/src/de/lmu/ifi/dbs/elki/evaluation/clustering/pairsegments/Segment.java index 8ce8fd4a..00700385 100644 --- a/src/de/lmu/ifi/dbs/elki/evaluation/clustering/pairsegments/Segment.java +++ b/src/de/lmu/ifi/dbs/elki/evaluation/clustering/pairsegments/Segment.java @@ -1,4 +1,26 @@ package de.lmu.ifi.dbs.elki.evaluation.clustering.pairsegments; +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ import java.util.Arrays; diff --git a/src/de/lmu/ifi/dbs/elki/evaluation/clustering/pairsegments/Segments.java b/src/de/lmu/ifi/dbs/elki/evaluation/clustering/pairsegments/Segments.java index 97c79dfb..ca352367 100644 --- a/src/de/lmu/ifi/dbs/elki/evaluation/clustering/pairsegments/Segments.java +++ b/src/de/lmu/ifi/dbs/elki/evaluation/clustering/pairsegments/Segments.java @@ -1,5 +1,28 @@ package de.lmu.ifi.dbs.elki.evaluation.clustering.pairsegments; +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -9,6 +32,7 @@ import java.util.TreeMap; import de.lmu.ifi.dbs.elki.data.Cluster; import de.lmu.ifi.dbs.elki.data.Clustering; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.HashSetModifiableDBIDs; @@ -47,6 +71,8 @@ import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; * * @author Sascha Goldhofer * @author Erich Schubert + * + * @apiviz.composedOf Segment */ @Reference(title = "Evaluation of Clusterings – Metrics and Visual Support", authors = "Elke Achtert, Sascha Goldhofer, Hans-Peter Kriegel, Erich Schubert, Arthur Zimek", booktitle = "Proc. 28th International Conference on Data Engineering (ICDE) 2012", url = "http://elki.dbs.ifi.lmu.de/wiki/PairSegments") public class Segments extends BasicResult implements Iterable<Segment> { @@ -154,7 +180,8 @@ public class Segments extends BasicResult implements Iterable<Segment> { HashSetModifiableDBIDs ndelta1 = DBIDUtil.newHashSet(first); HashSetModifiableDBIDs ndelta2 = DBIDUtil.newHashSet(); HashSetModifiableDBIDs nsecond = DBIDUtil.newHashSet(second.size()); - for(DBID id : clust.getIDs()) { + for(DBIDIter iter2 = clust.getIDs().iter(); iter2.valid(); iter2.advance()) { + DBID id = iter2.getDBID(); if(ndelta1.remove(id)) { nfirstp.add(id); } diff --git a/src/de/lmu/ifi/dbs/elki/evaluation/histogram/ComputeOutlierHistogram.java b/src/de/lmu/ifi/dbs/elki/evaluation/histogram/ComputeOutlierHistogram.java index 124abf9e..2a2bb069 100644 --- a/src/de/lmu/ifi/dbs/elki/evaluation/histogram/ComputeOutlierHistogram.java +++ b/src/de/lmu/ifi/dbs/elki/evaluation/histogram/ComputeOutlierHistogram.java @@ -1,26 +1,27 @@ package de.lmu.ifi.dbs.elki.evaluation.histogram;
-/* -This file is part of ELKI: -Environment for Developing KDD-Applications Supported by Index-Structures - -Copyright (C) 2012 -Ludwig-Maximilians-Universität München -Lehr- und Forschungseinheit für Datenbanksysteme -ELKI Development Team - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ +
+/*
+ This file is part of ELKI:
+ Environment for Developing KDD-Applications Supported by Index-Structures +
+ Copyright (C) 2012
+ Ludwig-Maximilians-Universität München
+ Lehr- und Forschungseinheit für Datenbanksysteme
+ ELKI Development Team +
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version. +
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details. +
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
import java.util.ArrayList;
import java.util.Collection;
@@ -29,7 +30,7 @@ import java.util.regex.Pattern; import de.lmu.ifi.dbs.elki.data.DoubleVector;
import de.lmu.ifi.dbs.elki.database.Database;
-import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil;
import de.lmu.ifi.dbs.elki.database.ids.DBIDs;
import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs;
@@ -68,6 +69,7 @@ import de.lmu.ifi.dbs.elki.utilities.scaling.outlier.OutlierScalingFunction; * @author Erich Schubert
*
* @apiviz.landmark
+ * @apiviz.uses OutlierResult
* @apiviz.has ScalingFunction
* @apiviz.has HistogramResult oneway - - «create»
*/
@@ -186,13 +188,13 @@ public class ComputeOutlierHistogram implements Evaluator { }
ids.removeDBIDs(outlierIds);
// fill histogram with values of each object
- for(DBID id : ids) {
- double result = or.getScores().get(id);
+ for(DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) {
+ double result = or.getScores().get(iter);
result = scaling.getScaled(result);
hist.aggregate(result, negative);
}
- for(DBID id : outlierIds) {
- double result = or.getScores().get(id);
+ for(DBIDIter iter = outlierIds.iter(); iter.valid(); iter.advance()) {
+ double result = or.getScores().get(iter);
result = scaling.getScaled(result);
hist.aggregate(result, positive);
}
@@ -223,11 +225,11 @@ public class ComputeOutlierHistogram implements Evaluator { }
/**
- * Parameterization class. - * - * @author Erich Schubert - * - * @apiviz.exclude + * Parameterization class.
+ *
+ * @author Erich Schubert
+ *
+ * @apiviz.exclude
*/
public static class Parameterizer extends AbstractParameterizer {
/**
diff --git a/src/de/lmu/ifi/dbs/elki/evaluation/outlier/JudgeOutlierScores.java b/src/de/lmu/ifi/dbs/elki/evaluation/outlier/JudgeOutlierScores.java index b92e5bde..49775b1a 100644 --- a/src/de/lmu/ifi/dbs/elki/evaluation/outlier/JudgeOutlierScores.java +++ b/src/de/lmu/ifi/dbs/elki/evaluation/outlier/JudgeOutlierScores.java @@ -28,7 +28,7 @@ import java.util.List; import java.util.regex.Pattern;
import de.lmu.ifi.dbs.elki.database.Database;
-import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil;
import de.lmu.ifi.dbs.elki.database.ids.DBIDs;
import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs;
@@ -147,13 +147,13 @@ public class JudgeOutlierScores implements Evaluator { double posscore = 0.0;
double negscore = 0.0;
// fill histogram with values of each object
- for(DBID id : ids) {
- double result = or.getScores().get(id);
+ for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) {
+ double result = or.getScores().get(iter);
result = innerScaling.getScaled(scaling.getScaled(result));
posscore += (1.0 - result);
}
- for(DBID id : outlierIds) {
- double result = or.getScores().get(id);
+ for (DBIDIter iter = outlierIds.iter(); iter.valid(); iter.advance()) {
+ double result = or.getScores().get(iter);
result = innerScaling.getScaled(scaling.getScaled(result));
negscore += result;
}
diff --git a/src/de/lmu/ifi/dbs/elki/evaluation/outlier/OutlierPrecisionAtKCurve.java b/src/de/lmu/ifi/dbs/elki/evaluation/outlier/OutlierPrecisionAtKCurve.java new file mode 100644 index 00000000..6d65999e --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/evaluation/outlier/OutlierPrecisionAtKCurve.java @@ -0,0 +1,227 @@ +package de.lmu.ifi.dbs.elki.evaluation.outlier; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import java.util.List; +import java.util.regex.Pattern; + +import de.lmu.ifi.dbs.elki.database.Database; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; +import de.lmu.ifi.dbs.elki.database.ids.DBIDs; +import de.lmu.ifi.dbs.elki.database.ids.SetDBIDs; +import de.lmu.ifi.dbs.elki.evaluation.Evaluator; +import de.lmu.ifi.dbs.elki.logging.Logging; +import de.lmu.ifi.dbs.elki.math.geometry.XYCurve; +import de.lmu.ifi.dbs.elki.result.HierarchicalResult; +import de.lmu.ifi.dbs.elki.result.OrderingResult; +import de.lmu.ifi.dbs.elki.result.Result; +import de.lmu.ifi.dbs.elki.result.ResultUtil; +import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; +import de.lmu.ifi.dbs.elki.result.textwriter.TextWriterStream; +import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.IntParameter; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.PatternParameter; + +/** + * Compute a curve containing the precision values for an outlier detection + * method. + * + * @author Erich Schubert + * + * @apiviz.has PrecisionAtKCurve + */ +public class OutlierPrecisionAtKCurve implements Evaluator { + /** + * The logger. + */ + private static final Logging logger = Logging.getLogger(OutlierPrecisionAtKCurve.class); + + /** + * The pattern to identify positive classes. + * + * <p> + * Key: {@code -precision.positive} + * </p> + */ + public static final OptionID POSITIVE_CLASS_NAME_ID = OptionID.getOrCreateOptionID("precision.positive", "Class label for the 'positive' class."); + + /** + * Maximum value for k + * + * <p> + * Key: {@code -precision.k} + * </p> + */ + public static final OptionID MAX_K_ID = OptionID.getOrCreateOptionID("precision.maxk", "Maximum value of 'k' to compute the curve up to."); + + /** + * Stores the "positive" class. + */ + private Pattern positiveClassName; + + /** + * Maximum value for k + */ + int maxk = Integer.MAX_VALUE; + + /** + * Constructor. + * + * @param positiveClassName Pattern to recognize outliers + * @param maxk Maximum value for k + */ + public OutlierPrecisionAtKCurve(Pattern positiveClassName, int maxk) { + super(); + this.positiveClassName = positiveClassName; + this.maxk = maxk; + } + + @Override + public void processNewResult(HierarchicalResult baseResult, Result result) { + Database db = ResultUtil.findDatabase(baseResult); + // Prepare + SetDBIDs positiveids = DBIDUtil.ensureSet(DatabaseUtil.getObjectsByLabelMatch(db, positiveClassName)); + + if(positiveids.size() == 0) { + logger.warning("Computing a ROC curve failed - no objects matched."); + return; + } + + List<OutlierResult> oresults = ResultUtil.getOutlierResults(result); + List<OrderingResult> orderings = ResultUtil.getOrderingResults(result); + // Outlier results are the main use case. + for(OutlierResult o : oresults) { + DBIDs sorted = o.getOrdering().iter(o.getOrdering().getDBIDs()); + db.getHierarchy().add(o, computePrecisionResult(o.getScores().size(), positiveids, sorted)); + // Process them only once. + orderings.remove(o.getOrdering()); + } + + // FIXME: find appropriate place to add the derived result + // otherwise apply an ordering to the database IDs. + for(OrderingResult or : orderings) { + DBIDs sorted = or.iter(or.getDBIDs()); + db.getHierarchy().add(or, computePrecisionResult(or.getDBIDs().size(), positiveids, sorted)); + } + } + + private XYCurve computePrecisionResult(int size, SetDBIDs positiveids, DBIDs order) { + if(order.size() != size) { + throw new IllegalStateException("Iterable result doesn't match database size - incomplete ordering?"); + } + int lastk = Math.min(size, maxk); + XYCurve curve = new PrecisionAtKCurve("k", "Precision", lastk); + + int pos = 0; + DBIDIter i = order.iter(); + for(int k = 1; k <= lastk; k++, i.advance()) { + if(positiveids.contains(i.getDBID())) { + pos++; + } + curve.addAndSimplify(k, pos / (double) k); + } + if(logger.isVerbose()) { + logger.verbose("Precision @ " + lastk + " " + ((pos * 1.0) / lastk)); + } + return curve; + } + + /** + * Precision at K curve. + * + * @author Erich Schubert + */ + public static class PrecisionAtKCurve extends XYCurve { + /** + * Constructor. + * + * @param size Size estimation + */ + public PrecisionAtKCurve(String labelx, String labely, int size) { + super("k", "Precision", size); + } + + @Override + public String getLongName() { + return "Precision @ k Curve"; + } + + @Override + public String getShortName() { + return "precision-at-k"; + } + + @Override + public void writeToText(TextWriterStream out, String label) { + final int last = size() - 1; + out.commentPrintLn("Precision @ " + ((int) getX(last)) + ": " + getY(last)); + out.commentPrintSeparator(); + out.flush(); + out.commentPrint(labelx); + out.commentPrint(" "); + out.commentPrint(labely); + out.flush(); + for(int pos = 0; pos < data.size(); pos+=2) { + out.inlinePrint((int)data.get(pos)); + out.inlinePrint(data.get(pos + 1)); + out.flush(); + } + } + } + + /** + * Parameterization class. + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer extends AbstractParameterizer { + protected Pattern positiveClassName = null; + + protected int maxk = Integer.MAX_VALUE; + + @Override + protected void makeOptions(Parameterization config) { + super.makeOptions(config); + PatternParameter positiveClassNameP = new PatternParameter(POSITIVE_CLASS_NAME_ID); + if(config.grab(positiveClassNameP)) { + positiveClassName = positiveClassNameP.getValue(); + } + IntParameter maxkP = new IntParameter(MAX_K_ID, true); + if(config.grab(maxkP)) { + maxk = maxkP.getValue(); + } + } + + @Override + protected OutlierPrecisionAtKCurve makeInstance() { + return new OutlierPrecisionAtKCurve(positiveClassName, maxk); + } + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/evaluation/outlier/OutlierPrecisionRecallCurve.java b/src/de/lmu/ifi/dbs/elki/evaluation/outlier/OutlierPrecisionRecallCurve.java new file mode 100644 index 00000000..c839016c --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/evaluation/outlier/OutlierPrecisionRecallCurve.java @@ -0,0 +1,238 @@ +package de.lmu.ifi.dbs.elki.evaluation.outlier; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import java.util.List; +import java.util.regex.Pattern; + +import de.lmu.ifi.dbs.elki.database.Database; +import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; +import de.lmu.ifi.dbs.elki.database.ids.DBIDs; +import de.lmu.ifi.dbs.elki.database.ids.SetDBIDs; +import de.lmu.ifi.dbs.elki.database.relation.Relation; +import de.lmu.ifi.dbs.elki.evaluation.Evaluator; +import de.lmu.ifi.dbs.elki.logging.Logging; +import de.lmu.ifi.dbs.elki.math.geometry.XYCurve; +import de.lmu.ifi.dbs.elki.result.HierarchicalResult; +import de.lmu.ifi.dbs.elki.result.OrderingResult; +import de.lmu.ifi.dbs.elki.result.Result; +import de.lmu.ifi.dbs.elki.result.ResultUtil; +import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; +import de.lmu.ifi.dbs.elki.result.textwriter.TextWriterStream; +import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.PatternParameter; + +/** + * Compute a curve containing the precision values for an outlier detection + * method. + * + * @author Erich Schubert + * + * @apiviz.has PRCurve + */ +public class OutlierPrecisionRecallCurve implements Evaluator { + /** + * The logger. + */ + private static final Logging logger = Logging.getLogger(OutlierPrecisionRecallCurve.class); + + /** + * The pattern to identify positive classes. + * + * <p> + * Key: {@code -precision.positive} + * </p> + */ + public static final OptionID POSITIVE_CLASS_NAME_ID = OptionID.getOrCreateOptionID("precision.positive", "Class label for the 'positive' class."); + + /** + * Stores the "positive" class. + */ + private Pattern positiveClassName; + + /** + * Constructor. + * + * @param positiveClassName Pattern to recognize outliers + */ + public OutlierPrecisionRecallCurve(Pattern positiveClassName) { + super(); + this.positiveClassName = positiveClassName; + } + + @Override + public void processNewResult(HierarchicalResult baseResult, Result result) { + Database db = ResultUtil.findDatabase(baseResult); + // Prepare + SetDBIDs positiveids = DBIDUtil.ensureSet(DatabaseUtil.getObjectsByLabelMatch(db, positiveClassName)); + + if(positiveids.size() == 0) { + logger.warning("Computing a ROC curve failed - no objects matched."); + return; + } + + List<OutlierResult> oresults = ResultUtil.getOutlierResults(result); + List<OrderingResult> orderings = ResultUtil.getOrderingResults(result); + // Outlier results are the main use case. + for(OutlierResult o : oresults) { + DBIDs sorted = o.getOrdering().iter(o.getOrdering().getDBIDs()); + XYCurve curve = computePrecisionResult(o.getScores().size(), positiveids, sorted.iter(), o.getScores()); + db.getHierarchy().add(o, curve); + // Process them only once. + orderings.remove(o.getOrdering()); + } + + // FIXME: find appropriate place to add the derived result + // otherwise apply an ordering to the database IDs. + for(OrderingResult or : orderings) { + DBIDs sorted = or.iter(or.getDBIDs()); + XYCurve curve = computePrecisionResult(or.getDBIDs().size(), positiveids, sorted.iter(), null); + db.getHierarchy().add(or, curve); + } + } + + private XYCurve computePrecisionResult(int size, SetDBIDs ids, DBIDIter iter, Relation<Double> scores) { + final int postot = ids.size(); + int poscnt = 0, total = 0; + XYCurve curve = new PRCurve(postot + 2); + + double prevscore = Double.NaN; + for(; iter.valid(); iter.advance()) { + // Previous precision rate - y axis + final double curprec = ((double) poscnt) / total; + // Previous recall rate - x axis + final double curreca = ((double) poscnt) / postot; + + // Analyze next point + DBID cur = iter.getDBID(); + // positive or negative match? + if(ids.contains(cur)) { + poscnt += 1; + } + total += 1; + // First iteration ends here + if(total == 1) { + continue; + } + // defer calculation for ties + if(scores != null) { + double curscore = scores.get(cur); + if(Double.compare(prevscore, curscore) == 0) { + continue; + } + prevscore = curscore; + } + // Add a new point (for the previous entry - because of tie handling!) + curve.addAndSimplify(curreca, curprec); + } + // End curve - always at all positives found. + curve.addAndSimplify(1.0, postot / total); + return curve; + } + + /** + * P/R Curve + * + * @author Erich Schubert + */ + public static class PRCurve extends XYCurve { + /** + * AUC value for PR curve + */ + public static final String PRAUC_LABEL = "PR-AUC"; + + /** + * Area under curve + */ + double auc = Double.NaN; + + /** + * Constructor. + * + * @param size Size estimation + */ + public PRCurve(int size) { + super("Recall", "Precision", size); + } + + @Override + public String getLongName() { + return "Precision-Recall-Curve"; + } + + @Override + public String getShortName() { + return "pr-curve"; + } + + /** + * Get AUC value + * + * @return AUC value + */ + public double getAUC() { + if(Double.isNaN(auc)) { + auc = areaUnderCurve(this); + } + return auc; + } + + @Override + public void writeToText(TextWriterStream out, String label) { + out.commentPrintLn(PRAUC_LABEL + ": " + getAUC()); + out.flush(); + super.writeToText(out, label); + } + } + + /** + * Parameterization class. + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer extends AbstractParameterizer { + protected Pattern positiveClassName = null; + + @Override + protected void makeOptions(Parameterization config) { + super.makeOptions(config); + PatternParameter positiveClassNameP = new PatternParameter(POSITIVE_CLASS_NAME_ID); + if(config.grab(positiveClassNameP)) { + positiveClassName = positiveClassNameP.getValue(); + } + } + + @Override + protected OutlierPrecisionRecallCurve makeInstance() { + return new OutlierPrecisionRecallCurve(positiveClassName); + } + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/evaluation/outlier/OutlierROCCurve.java b/src/de/lmu/ifi/dbs/elki/evaluation/outlier/OutlierROCCurve.java new file mode 100644 index 00000000..ec73d67f --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/evaluation/outlier/OutlierROCCurve.java @@ -0,0 +1,241 @@ +package de.lmu.ifi.dbs.elki.evaluation.outlier; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import java.util.List; +import java.util.regex.Pattern; + +import de.lmu.ifi.dbs.elki.database.Database; +import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; +import de.lmu.ifi.dbs.elki.database.ids.DBIDs; +import de.lmu.ifi.dbs.elki.database.ids.SetDBIDs; +import de.lmu.ifi.dbs.elki.evaluation.Evaluator; +import de.lmu.ifi.dbs.elki.evaluation.roc.ROC; +import de.lmu.ifi.dbs.elki.logging.Logging; +import de.lmu.ifi.dbs.elki.math.geometry.XYCurve; +import de.lmu.ifi.dbs.elki.result.HierarchicalResult; +import de.lmu.ifi.dbs.elki.result.OrderingResult; +import de.lmu.ifi.dbs.elki.result.Result; +import de.lmu.ifi.dbs.elki.result.ResultUtil; +import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; +import de.lmu.ifi.dbs.elki.result.textwriter.TextWriterStream; +import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.PatternParameter; + +/** + * Compute a ROC curve to evaluate a ranking algorithm and compute the + * corresponding ROCAUC value. + * + * The parameter {@code -rocauc.positive} specifies the class label of + * "positive" hits. + * + * The nested algorithm {@code -algorithm} will be run, the result will be + * searched for an iterable or ordering result, which then is compared with the + * clustering obtained via the given class label. + * + * @author Erich Schubert + * + * @apiviz.landmark + * + * @apiviz.uses OutlierResult + * @apiviz.uses ROC + * @apiviz.has ROCResult oneway - - «create» + */ +// TODO: maybe add a way to process clustering results as well? +public class OutlierROCCurve implements Evaluator { + /** + * The label we use for marking ROCAUC values. + */ + public static final String ROCAUC_LABEL = "ROCAUC"; + + /** + * The logger. + */ + private static final Logging logger = Logging.getLogger(OutlierROCCurve.class); + + /** + * The pattern to identify positive classes. + * + * <p> + * Key: {@code -rocauc.positive} + * </p> + */ + public static final OptionID POSITIVE_CLASS_NAME_ID = OptionID.getOrCreateOptionID("rocauc.positive", "Class label for the 'positive' class."); + + /** + * Stores the "positive" class. + */ + private Pattern positiveClassName; + + /** + * Constructor. + * + * @param positive_class_name Positive class name pattern + */ + public OutlierROCCurve(Pattern positive_class_name) { + super(); + this.positiveClassName = positive_class_name; + } + + private ROCResult computeROCResult(int size, SetDBIDs positiveids, DBIDs order) { + if(order.size() != size) { + throw new IllegalStateException("Iterable result doesn't match database size - incomplete ordering?"); + } + XYCurve roccurve = ROC.materializeROC(size, positiveids, new ROC.SimpleAdapter(order.iter())); + double rocauc = XYCurve.areaUnderCurve(roccurve); + if(logger.isVerbose()) { + logger.verbose(ROCAUC_LABEL + ": " + rocauc); + } + + final ROCResult rocresult = new ROCResult(roccurve, rocauc); + + return rocresult; + } + + private ROCResult computeROCResult(int size, SetDBIDs positiveids, OutlierResult or) { + XYCurve roccurve = ROC.materializeROC(size, positiveids, new ROC.OutlierScoreAdapter(or)); + double rocauc = XYCurve.areaUnderCurve(roccurve); + if(logger.isVerbose()) { + logger.verbose(ROCAUC_LABEL + ": " + rocauc); + } + + final ROCResult rocresult = new ROCResult(roccurve, rocauc); + + return rocresult; + } + + @Override + public void processNewResult(HierarchicalResult baseResult, Result result) { + Database db = ResultUtil.findDatabase(baseResult); + // Prepare + SetDBIDs positiveids = DBIDUtil.ensureSet(DatabaseUtil.getObjectsByLabelMatch(db, positiveClassName)); + + if(positiveids.size() == 0) { + logger.warning("Computing a ROC curve failed - no objects matched."); + return; + } + + boolean nonefound = true; + List<OutlierResult> oresults = ResultUtil.getOutlierResults(result); + List<OrderingResult> orderings = ResultUtil.getOrderingResults(result); + // Outlier results are the main use case. + for(OutlierResult o : oresults) { + db.getHierarchy().add(o, computeROCResult(o.getScores().size(), positiveids, o)); + // Process them only once. + orderings.remove(o.getOrdering()); + nonefound = false; + } + + // FIXME: find appropriate place to add the derived result + // otherwise apply an ordering to the database IDs. + for(OrderingResult or : orderings) { + DBIDs sorted = or.iter(or.getDBIDs()); + db.getHierarchy().add(or, computeROCResult(or.getDBIDs().size(), positiveids, sorted)); + nonefound = false; + } + + if(nonefound) { + return; + // logger.warning("No results found to process with ROC curve analyzer. Got "+iterables.size()+" iterables, "+orderings.size()+" orderings."); + } + } + + /** + * Result object for ROC curves. + * + * @author Erich Schubert + */ + public static class ROCResult extends XYCurve { + /** + * AUC value + */ + private double auc; + + /** + * Constructor. + * + * @param col roc curve + * @param rocauc ROC AUC value + */ + public ROCResult(XYCurve col, double rocauc) { + super(col); + this.auc = rocauc; + } + + /** + * @return the area under curve + */ + public double getAUC() { + return auc; + } + + @Override + public String getLongName() { + return "ROC Curve"; + } + + @Override + public String getShortName() { + return "roc-curve"; + } + + @Override + public void writeToText(TextWriterStream out, String label) { + out.commentPrintLn(ROCAUC_LABEL + ": " + auc); + out.flush(); + super.writeToText(out, label); + } + } + + /** + * Parameterization class. + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer extends AbstractParameterizer { + /** + * Pattern for positive class. + */ + protected Pattern positiveClassName = null; + + @Override + protected void makeOptions(Parameterization config) { + super.makeOptions(config); + PatternParameter positiveClassNameP = new PatternParameter(POSITIVE_CLASS_NAME_ID); + if(config.grab(positiveClassNameP)) { + positiveClassName = positiveClassNameP.getValue(); + } + } + + @Override + protected OutlierROCCurve makeInstance() { + return new OutlierROCCurve(positiveClassName); + } + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/evaluation/outlier/OutlierSmROCCurve.java b/src/de/lmu/ifi/dbs/elki/evaluation/outlier/OutlierSmROCCurve.java new file mode 100644 index 00000000..928ac842 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/evaluation/outlier/OutlierSmROCCurve.java @@ -0,0 +1,269 @@ +package de.lmu.ifi.dbs.elki.evaluation.outlier; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import java.util.List; +import java.util.regex.Pattern; + +import de.lmu.ifi.dbs.elki.database.Database; +import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; +import de.lmu.ifi.dbs.elki.database.ids.SetDBIDs; +import de.lmu.ifi.dbs.elki.database.relation.Relation; +import de.lmu.ifi.dbs.elki.evaluation.Evaluator; +import de.lmu.ifi.dbs.elki.logging.Logging; +import de.lmu.ifi.dbs.elki.math.geometry.XYCurve; +import de.lmu.ifi.dbs.elki.result.HierarchicalResult; +import de.lmu.ifi.dbs.elki.result.OrderingResult; +import de.lmu.ifi.dbs.elki.result.Result; +import de.lmu.ifi.dbs.elki.result.ResultUtil; +import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; +import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; +import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.PatternParameter; + +/** + * Smooth ROC curves are a variation of classic ROC curves that takes the scores + * into account. + * + * Reference: + * <p> + * W. Klement, P. A. Flach, N. Japkowicz, S. Matwin<br /> + * Smooth Receiver Operating Characteristics (smROC) Curves.<br /> + * In: European Conference on Machine Learning and Principles and Practice of + * Knowledge Discovery in Databases (ECML-PKDD'11) + * </p> + * + * However, this method has some deficiencies when the mean score is not 0.5, as + * discussed in: + * <p> + * E. Schubert, R. Wojdanowski, A. Zimek, H.-P. Kriegel<br> + * On Evaluation of Outlier Rankings and Outlier Scores<br> + * In Proceedings of the 12th SIAM International Conference on Data Mining + * (SDM), Anaheim, CA, 2012. + * </p> + * + * @author Erich Schubert + * + * @apiviz.landmark + * + * @apiviz.uses OutlierResult + * @apiviz.has SmROCResult oneway - - «create» + */ +@Reference(authors = "W. Klement, P. A. Flach, N. Japkowicz, S. Matwin", title = "Smooth Receiver Operating Characteristics (smROC) Curves", booktitle = "In: European Conference on Machine Learning and Principles and Practice of Knowledge Discovery in Databases (ECML-PKDD'11)", url = "http://dx.doi.org/10.1007/978-3-642-23783-6_13") +public class OutlierSmROCCurve implements Evaluator { + /** + * The label we use for marking ROCAUC values. + */ + public static final String SMROCAUC_LABEL = "ROCAUC"; + + /** + * The logger. + */ + private static final Logging logger = Logging.getLogger(OutlierSmROCCurve.class); + + /** + * Stores the "positive" class. + */ + private Pattern positiveClassName; + + /** + * Constructor. + * + * @param positive_class_name Positive class name pattern + */ + public OutlierSmROCCurve(Pattern positive_class_name) { + super(); + this.positiveClassName = positive_class_name; + } + + private SmROCResult computeSmROCResult(SetDBIDs positiveids, OutlierResult or) { + Relation<Double> scores = or.getScores(); + final int size = scores.size(); + + // Compute mean, for inversion + double mean = 0.0; + for(DBIDIter iditer = scores.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); + mean += scores.get(id) / size; + } + + SmROCResult curve = new SmROCResult(positiveids.size() + 2); + + // start in bottom left + curve.add(0.0, 0.0); + + int poscnt = 0, negcnt = 0; + double prevscore = Double.NaN; + double x = 0, y = 0; + for (DBIDIter nei = or.getOrdering().iter(or.getOrdering().getDBIDs()).iter(); nei.valid(); nei.advance()) { + // Analyze next point + final DBID curid = nei.getDBID(); + final double curscore = scores.get(curid); + // defer calculation for ties + if(!Double.isNaN(prevscore) && (Double.compare(prevscore, curscore) == 0)) { + // positive or negative match? + if(positiveids.contains(curid)) { + poscnt += 1; + } + else { + negcnt += 1; + } + continue; + } + else { + // Add point for *previous* result (since we are no longer tied with it) + if(prevscore > mean) { + y += poscnt * prevscore + negcnt * (1.0 - prevscore); + x += poscnt * (1.0 - prevscore) + negcnt * prevscore; + } + else if(prevscore < mean) { + y += poscnt * (1.0 - prevscore) + negcnt * prevscore; + x += poscnt * prevscore + negcnt * (1.0 - prevscore); + } + curve.addAndSimplify(x, y); + // positive or negative match? + if(positiveids.contains(curid)) { + poscnt = 1; + negcnt = 0; + } + else { + poscnt = 0; + negcnt = 1; + } + prevscore = curscore; + } + } + // Last point + { + if(prevscore > mean) { + y += poscnt * prevscore + negcnt * (1.0 - prevscore); + x += poscnt * (1.0 - prevscore) + negcnt * prevscore; + } + else if(prevscore < mean) { + y += poscnt * (1.0 - prevscore) + negcnt * prevscore; + x += poscnt * prevscore + negcnt * (1.0 - prevscore); + } + curve.addAndSimplify(x, y); + } + + double rocauc = XYCurve.areaUnderCurve(curve) / (x * y); + if(logger.isVerbose()) { + logger.verbose(SMROCAUC_LABEL + ": " + rocauc); + } + curve.rocauc = rocauc; + + return curve; + } + + @Override + public void processNewResult(HierarchicalResult baseResult, Result result) { + Database db = ResultUtil.findDatabase(baseResult); + // Prepare + SetDBIDs positiveids = DBIDUtil.ensureSet(DatabaseUtil.getObjectsByLabelMatch(db, positiveClassName)); + + if(positiveids.size() == 0) { + logger.warning("Computing a ROC curve failed - no objects matched."); + return; + } + + List<OutlierResult> oresults = ResultUtil.getOutlierResults(result); + List<OrderingResult> orderings = ResultUtil.getOrderingResults(result); + for(OutlierResult o : oresults) { + db.getHierarchy().add(o, computeSmROCResult(positiveids, o)); + orderings.remove(o.getOrdering()); + } + } + + /** + * Result object for Smooth ROC curves. + * + * @author Erich Schubert + */ + public static class SmROCResult extends XYCurve { + /** + * ROC AUC score + */ + double rocauc = Double.NaN; + + /** + * Constructor. + * + * @param size Size estimate + */ + public SmROCResult(int size) { + super("SmROC Negative", "SmROC Positive", size); + } + + @Override + public String getLongName() { + return "SmROC Curve"; + } + + @Override + public String getShortName() { + return "smroc-curve"; + } + + /** + * SmROC AUC value + * + * @return SmROC auc value + */ + public double getAUC() { + return rocauc; + } + } + + /** + * Parameterization class. + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer extends AbstractParameterizer { + /** + * Pattern for positive class. + */ + protected Pattern positiveClassName = null; + + @Override + protected void makeOptions(Parameterization config) { + super.makeOptions(config); + PatternParameter positiveClassNameP = new PatternParameter(OutlierROCCurve.POSITIVE_CLASS_NAME_ID); + if(config.grab(positiveClassNameP)) { + positiveClassName = positiveClassNameP.getValue(); + } + } + + @Override + protected OutlierSmROCCurve makeInstance() { + return new OutlierSmROCCurve(positiveClassName); + } + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/evaluation/outlier/OutlierThresholdClustering.java b/src/de/lmu/ifi/dbs/elki/evaluation/outlier/OutlierThresholdClustering.java new file mode 100644 index 00000000..e5d35424 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/evaluation/outlier/OutlierThresholdClustering.java @@ -0,0 +1,175 @@ +package de.lmu.ifi.dbs.elki.evaluation.outlier; +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import de.lmu.ifi.dbs.elki.data.Cluster; +import de.lmu.ifi.dbs.elki.data.Clustering; +import de.lmu.ifi.dbs.elki.data.model.Model; +import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; +import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; +import de.lmu.ifi.dbs.elki.database.relation.Relation; +import de.lmu.ifi.dbs.elki.evaluation.Evaluator; +import de.lmu.ifi.dbs.elki.result.HierarchicalResult; +import de.lmu.ifi.dbs.elki.result.Result; +import de.lmu.ifi.dbs.elki.result.ResultUtil; +import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; +import de.lmu.ifi.dbs.elki.utilities.datastructures.arraylike.ArrayLikeUtil; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.DoubleListParameter; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter; +import de.lmu.ifi.dbs.elki.utilities.scaling.IdentityScaling; +import de.lmu.ifi.dbs.elki.utilities.scaling.ScalingFunction; +import de.lmu.ifi.dbs.elki.utilities.scaling.outlier.OutlierScalingFunction; + +/** + * Pseudo clustering algorithm that builds clusters based on their outlier + * score. Useful for transforming a numeric outlier score into a 2-class + * dataset. + * + * @author Erich Schubert + */ +public class OutlierThresholdClustering implements Evaluator { + /** + * Scaling function to use + */ + ScalingFunction scaling = null; + + /** + * Thresholds to use + */ + double[] threshold; + + /** + * Constructor. + * + * @param scaling Scaling function + * @param threshold Threshold + */ + public OutlierThresholdClustering(ScalingFunction scaling, double[] threshold) { + super(); + this.scaling = scaling; + this.threshold = threshold; + Arrays.sort(this.threshold); + } + + @Override + public void processNewResult(HierarchicalResult baseResult, Result newResult) { + List<OutlierResult> ors = ResultUtil.filterResults(newResult, OutlierResult.class); + for(OutlierResult or : ors) { + baseResult.getHierarchy().add(or, split(or)); + } + } + + private Clustering<Model> split(OutlierResult or) { + Relation<Double> scores = or.getScores(); + if(scaling instanceof OutlierScalingFunction) { + ((OutlierScalingFunction) scaling).prepare(or); + } + ArrayList<ModifiableDBIDs> idlists = new ArrayList<ModifiableDBIDs>(threshold.length + 1); + for(int i = 0; i <= threshold.length; i++) { + idlists.add(DBIDUtil.newHashSet()); + } + for(DBIDIter iter = scores.getDBIDs().iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); + double score = scores.get(id); + if(scaling != null) { + score = scaling.getScaled(score); + } + int i = 0; + for(; i < threshold.length; i++) { + if(score < threshold[i]) { + break; + } + } + idlists.get(i).add(id); + } + Clustering<Model> c = new Clustering<Model>("Outlier threshold clustering", "threshold-clustering"); + for(int i = 0; i <= threshold.length; i++) { + String name = (i == 0) ? "Inlier" : "Outlier_" + threshold[i - 1]; + c.addCluster(new Cluster<Model>(name, idlists.get(i), (i > 0))); + } + return c; + } + + /** + * Parameterization helper + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer extends AbstractParameterizer { + /** + * Parameter to specify a scaling function to use. + * <p> + * Key: {@code -thresholdclust.scaling} + * </p> + */ + public static final OptionID SCALING_ID = OptionID.getOrCreateOptionID("thresholdclust.scaling", "Class to use as scaling function."); + + /** + * Parameter to specify the threshold + * <p> + * Key: {@code -thresholdclust.threshold} + * </p> + */ + public static final OptionID THRESHOLD_ID = OptionID.getOrCreateOptionID("thresholdclust.threshold", "Threshold(s) to apply."); + + /** + * Scaling function to use + */ + ScalingFunction scaling = null; + + /** + * Threshold to use + */ + double[] threshold; + + @Override + protected void makeOptions(Parameterization config) { + super.makeOptions(config); + ObjectParameter<ScalingFunction> scalingP = new ObjectParameter<ScalingFunction>(SCALING_ID, ScalingFunction.class, IdentityScaling.class); + if(config.grab(scalingP)) { + scaling = scalingP.instantiateClass(config); + } + + DoubleListParameter thresholdP = new DoubleListParameter(THRESHOLD_ID); + if(config.grab(thresholdP)) { + threshold = ArrayLikeUtil.toPrimitiveDoubleArray(thresholdP.getValue()); + } + } + + @Override + protected OutlierThresholdClustering makeInstance() { + return new OutlierThresholdClustering(scaling, threshold); + } + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/evaluation/roc/ComputeROCCurve.java b/src/de/lmu/ifi/dbs/elki/evaluation/roc/ComputeROCCurve.java index 232089ed..fae03d24 100644 --- a/src/de/lmu/ifi/dbs/elki/evaluation/roc/ComputeROCCurve.java +++ b/src/de/lmu/ifi/dbs/elki/evaluation/roc/ComputeROCCurve.java @@ -23,191 +23,31 @@ package de.lmu.ifi.dbs.elki.evaluation.roc; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; import java.util.regex.Pattern; -import de.lmu.ifi.dbs.elki.database.Database; -import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; -import de.lmu.ifi.dbs.elki.database.ids.DBID; -import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; -import de.lmu.ifi.dbs.elki.database.ids.SetDBIDs; -import de.lmu.ifi.dbs.elki.evaluation.Evaluator; -import de.lmu.ifi.dbs.elki.logging.Logging; -import de.lmu.ifi.dbs.elki.result.CollectionResult; -import de.lmu.ifi.dbs.elki.result.HierarchicalResult; -import de.lmu.ifi.dbs.elki.result.OrderingResult; -import de.lmu.ifi.dbs.elki.result.Result; -import de.lmu.ifi.dbs.elki.result.ResultUtil; -import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; -import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; +import de.lmu.ifi.dbs.elki.evaluation.outlier.OutlierROCCurve; import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; -import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.PatternParameter; -import de.lmu.ifi.dbs.elki.utilities.pairs.DoubleDoublePair; /** - * Compute a ROC curve to evaluate a ranking algorithm and compute the - * corresponding ROCAUC value. + * Compatibility stub. This class has been renamed. * - * The parameter {@code -rocauc.positive} specifies the class label of - * "positive" hits. - * - * The nested algorithm {@code -algorithm} will be run, the result will be - * searched for an iterable or ordering result, which then is compared with the - * clustering obtained via the given class label. + * TODO: remove at 0.6.0 release. * * @author Erich Schubert * - * @apiviz.landmark - * @apiviz.uses ROC - * @apiviz.has ROCResult oneway - - «create» + * @deprecated Renamed to {@link OutlierROCCurve}. */ -// TODO: maybe add a way to process clustering results as well? -public class ComputeROCCurve implements Evaluator { - /** - * The label we use for marking ROCAUC values. - */ - public static final String ROCAUC_LABEL = "ROCAUC"; - - /** - * The logger. - */ - static final Logging logger = Logging.getLogger(ComputeROCCurve.class); - - /** - * The pattern to identify positive classes. - * - * <p> - * Key: {@code -rocauc.positive} - * </p> - */ - public static final OptionID POSITIVE_CLASS_NAME_ID = OptionID.getOrCreateOptionID("rocauc.positive", "Class label for the 'positive' class."); - - /** - * Stores the "positive" class. - */ - private Pattern positiveClassName; - +@Deprecated +public class ComputeROCCurve extends OutlierROCCurve { /** * Constructor. * - * @param positive_class_name Positive class name pattern + * @param positive_class_name Pattern */ public ComputeROCCurve(Pattern positive_class_name) { - super(); - this.positiveClassName = positive_class_name; - } - - private ROCResult computeROCResult(int size, SetDBIDs positiveids, Iterator<DBID> iter) { - ArrayModifiableDBIDs order = DBIDUtil.newArray(size); - while(iter.hasNext()) { - Object o = iter.next(); - if(!(o instanceof DBID)) { - throw new IllegalStateException("Iterable result contained non-DBID - result didn't satisfy requirements"); - } - else { - order.add((DBID) o); - } - } - if(order.size() != size) { - throw new IllegalStateException("Iterable result doesn't match database size - incomplete ordering?"); - } - List<DoubleDoublePair> roccurve = ROC.materializeROC(size, positiveids, new ROC.SimpleAdapter(order.iterator())); - double rocauc = ROC.computeAUC(roccurve); - if(logger.isVerbose()) { - logger.verbose(ROCAUC_LABEL +": " + rocauc); - } - - List<String> header = new ArrayList<String>(1); - header.add(ROCAUC_LABEL+": " + rocauc); - final ROCResult rocresult = new ROCResult(roccurve, header, rocauc); - - return rocresult; - } - - private ROCResult computeROCResult(int size, SetDBIDs positiveids, OutlierResult or) { - List<DoubleDoublePair> roccurve = ROC.materializeROC(size, positiveids, new ROC.OutlierScoreAdapter(or)); - double rocauc = ROC.computeAUC(roccurve); - if(logger.isVerbose()) { - logger.verbose(ROCAUC_LABEL+": " + rocauc); - } - - List<String> header = new ArrayList<String>(1); - header.add(ROCAUC_LABEL+": " + rocauc); - final ROCResult rocresult = new ROCResult(roccurve, header, rocauc); - - return rocresult; - } - - @Override - public void processNewResult(HierarchicalResult baseResult, Result result) { - Database db = ResultUtil.findDatabase(baseResult); - // Prepare - SetDBIDs positiveids = DBIDUtil.ensureSet(DatabaseUtil.getObjectsByLabelMatch(db, positiveClassName)); - - if (positiveids.size() == 0) { - logger.warning("Computing a ROC curve failed - no objects matched."); - return; - } - - boolean nonefound = true; - List<OutlierResult> oresults = ResultUtil.getOutlierResults(result); - List<OrderingResult> orderings = ResultUtil.getOrderingResults(result); - // Outlier results are the main use case. - for(OutlierResult o : oresults) { - db.getHierarchy().add(o, computeROCResult(o.getScores().size(), positiveids, o)); - // Process them only once. - orderings.remove(o.getOrdering()); - nonefound = false; - } - - // FIXME: find appropriate place to add the derived result - // otherwise apply an ordering to the database IDs. - for(OrderingResult or : orderings) { - Iterator<DBID> iter = or.iter(or.getDBIDs()); - db.getHierarchy().add(or, computeROCResult(or.getDBIDs().size(), positiveids, iter)); - nonefound = false; - } - - if(nonefound) { - return; - // logger.warning("No results found to process with ROC curve analyzer. Got "+iterables.size()+" iterables, "+orderings.size()+" orderings."); - } - } - - /** - * Result object for ROC curves. - * - * @author Erich Schubert - */ - public static class ROCResult extends CollectionResult<DoubleDoublePair> { - /** - * AUC value - */ - private double auc; - - /** - * Constructor. - * - * @param col roc curve - * @param header header - * @param rocauc ROC AUC value - */ - public ROCResult(Collection<DoubleDoublePair> col, Collection<String> header, double rocauc) { - super("ROC Curve", "roc", col, header); - this.auc = rocauc; - } - - /** - * @return the area under curve - */ - public double getAUC() { - return auc; - } + super(positive_class_name); } /** @@ -218,12 +58,15 @@ public class ComputeROCCurve implements Evaluator { * @apiviz.exclude */ public static class Parameterizer extends AbstractParameterizer { + /** + * Pattern for positive class. + */ protected Pattern positiveClassName = null; @Override protected void makeOptions(Parameterization config) { super.makeOptions(config); - PatternParameter positiveClassNameP = new PatternParameter(POSITIVE_CLASS_NAME_ID); + PatternParameter positiveClassNameP = new PatternParameter(OutlierROCCurve.POSITIVE_CLASS_NAME_ID); if(config.grab(positiveClassNameP)) { positiveClassName = positiveClassNameP.getValue(); } @@ -234,4 +77,4 @@ public class ComputeROCCurve implements Evaluator { return new ComputeROCCurve(positiveClassName); } } -}
\ No newline at end of file +} diff --git a/src/de/lmu/ifi/dbs/elki/evaluation/roc/ROC.java b/src/de/lmu/ifi/dbs/elki/evaluation/roc/ROC.java index f7fdd254..d85cb137 100644 --- a/src/de/lmu/ifi/dbs/elki/evaluation/roc/ROC.java +++ b/src/de/lmu/ifi/dbs/elki/evaluation/roc/ROC.java @@ -23,13 +23,12 @@ package de.lmu.ifi.dbs.elki.evaluation.roc; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.ArrayList; import java.util.Iterator; -import java.util.List; import java.util.Set; import de.lmu.ifi.dbs.elki.data.Cluster; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDPair; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; @@ -37,8 +36,8 @@ import de.lmu.ifi.dbs.elki.database.ids.SetDBIDs; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; +import de.lmu.ifi.dbs.elki.math.geometry.XYCurve; import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; -import de.lmu.ifi.dbs.elki.utilities.pairs.DoubleDoublePair; import de.lmu.ifi.dbs.elki.utilities.pairs.DoubleObjPair; import de.lmu.ifi.dbs.elki.utilities.pairs.Pair; import de.lmu.ifi.dbs.elki.utilities.pairs.PairInterface; @@ -81,23 +80,16 @@ public class ROC { * 'same positions'. * @return area under curve */ - public static <C extends Comparable<? super C>, T> List<DoubleDoublePair> materializeROC(int size, Set<? super T> ids, Iterator<? extends PairInterface<C, T>> nei) { - int postot = ids.size(); - int negtot = size - postot; - int poscnt = 0; - int negcnt = 0; - ArrayList<DoubleDoublePair> res = new ArrayList<DoubleDoublePair>(postot + 2); + public static <C extends Comparable<? super C>, T> XYCurve materializeROC(int size, Set<? super T> ids, Iterator<? extends PairInterface<C, T>> nei) { + final int postot = ids.size(), negtot = size - postot; + int poscnt = 0, negcnt = 0; + XYCurve curve = new XYCurve("True Negative Rate", "True Positive Rate", postot + 2); // start in bottom left - res.add(new DoubleDoublePair(0.0, 0.0)); + curve.add(0.0, 0.0); - PairInterface<C, T> prev = null; + C prevval = null; while(nei.hasNext()) { - // Previous positive rate - y axis - double curpos = ((double) poscnt) / postot; - // Previous negative rate - x axis - double curneg = ((double) negcnt) / negtot; - // Analyze next point PairInterface<C, T> cur = nei.next(); // positive or negative match? @@ -108,46 +100,17 @@ public class ROC { negcnt += 1; } // defer calculation for ties - if((prev != null) && (prev.getFirst().compareTo(cur.getFirst()) == 0)) { + if((prevval != null) && (prevval.compareTo(cur.getFirst()) == 0)) { continue; } - // simplify curve when possible: - if(res.size() >= 2) { - DoubleDoublePair last1 = res.get(res.size() - 2); - DoubleDoublePair last2 = res.get(res.size() - 1); - final double ldx = last2.first - last1.first; - final double cdx = curneg - last2.first; - final double ldy = last2.second - last1.second; - final double cdy = curpos - last2.second; - // vertical simplification - if((ldx == 0) && (cdx == 0)) { - res.remove(res.size() - 1); - } - // horizontal simplification - else if((ldy == 0) && (cdy == 0)) { - res.remove(res.size() - 1); - } - // diagonal simplification - else if(ldy > 0 && cdy > 0) { - if(Math.abs((ldx / ldy) - (cdx / cdy)) < 1E-10) { - res.remove(res.size() - 1); - } - } - } - // Add a new point (for the previous entry!) - res.add(new DoubleDoublePair(curneg, curpos)); - prev = cur; + // Add a new point. + curve.addAndSimplify(negcnt / (double) negtot, poscnt / (double) postot); + prevval = cur.getFirst(); } - // ensure we end up in the top right corner. - // Since we didn't add a point for the last entry yet, this likely is - // needed. - { - DoubleDoublePair last = res.get(res.size() - 1); - if(last.first < 1.0 || last.second < 1.0) { - res.add(new DoubleDoublePair(1.0, 1.0)); - } - } - return res; + // Ensure we end up in the top right corner. + // Simplification will skip this if we already were. + curve.addAndSimplify(1.0, 1.0); + return curve; } /** @@ -162,23 +125,19 @@ public class ROC { * 'same positions'. * @return area under curve */ - public static <C extends Comparable<? super C>> List<DoubleDoublePair> materializeROC(int size, SetDBIDs ids, Iterator<? extends PairInterface<C, DBID>> nei) { - int postot = ids.size(); - int negtot = size - postot; - int poscnt = 0; - int negcnt = 0; - ArrayList<DoubleDoublePair> res = new ArrayList<DoubleDoublePair>(postot + 2); + public static <C extends Comparable<? super C>> XYCurve materializeROC(int size, SetDBIDs ids, Iterator<? extends PairInterface<C, DBID>> nei) { + final int postot = ids.size(), negtot = size - postot; + int poscnt = 0, negcnt = 0; + XYCurve curve = new XYCurve("True Negative Rate", "True Positive Rate", postot + 2); // start in bottom left - res.add(new DoubleDoublePair(0.0, 0.0)); + curve.add(0.0, 0.0); - PairInterface<C, DBID> prev = null; + C prevval = null; while(nei.hasNext()) { - // Previous positive rate - y axis - double curpos = ((double) poscnt) / postot; - // Previous negative rate - x axis - double curneg = ((double) negcnt) / negtot; - + // Rates at *previous* data point. Because of tie handling strategy! + final double trueneg = negcnt / (double) negtot; + final double truepos = poscnt / (double) postot; // Analyze next point PairInterface<C, DBID> cur = nei.next(); // positive or negative match? @@ -189,46 +148,17 @@ public class ROC { negcnt += 1; } // defer calculation for ties - if((prev != null) && (prev.getFirst().compareTo(cur.getFirst()) == 0)) { + if((prevval != null) && (prevval.compareTo(cur.getFirst()) == 0)) { continue; } - // simplify curve when possible: - if(res.size() >= 2) { - DoubleDoublePair last1 = res.get(res.size() - 2); - DoubleDoublePair last2 = res.get(res.size() - 1); - final double ldx = last2.first - last1.first; - final double cdx = curneg - last2.first; - final double ldy = last2.second - last1.second; - final double cdy = curpos - last2.second; - // vertical simplification - if((ldx == 0) && (cdx == 0)) { - res.remove(res.size() - 1); - } - // horizontal simplification - else if((ldy == 0) && (cdy == 0)) { - res.remove(res.size() - 1); - } - // diagonal simplification - else if(ldy > 0 && cdy > 0) { - if(Math.abs((ldx / ldy) - (cdx / cdy)) < 1E-10) { - res.remove(res.size() - 1); - } - } - } - // Add a new point (for the previous entry!) - res.add(new DoubleDoublePair(curneg, curpos)); - prev = cur; - } - // ensure we end up in the top right corner. - // Since we didn't add a point for the last entry yet, this likely is - // needed. - { - DoubleDoublePair last = res.get(res.size() - 1); - if(last.first < 1.0 || last.second < 1.0) { - res.add(new DoubleDoublePair(1.0, 1.0)); - } + // Add point for *previous* result (since we are no longer tied with it) + curve.addAndSimplify(trueneg, truepos); + prevval = cur.getFirst(); } - return res; + // Ensure we end up in the top right corner. + // Simplification will skip this if we already were. + curve.addAndSimplify(1.0, 1.0); + return curve; } /** @@ -245,27 +175,28 @@ public class ROC { /** * Original Iterator */ - private Iterator<DBID> iter; + private DBIDIter iter; /** * Constructor * * @param iter Iterator for object IDs */ - public SimpleAdapter(Iterator<DBID> iter) { + public SimpleAdapter(DBIDIter iter) { super(); this.iter = iter; } @Override public boolean hasNext() { - return this.iter.hasNext(); + return this.iter.valid(); } @Override public DBIDPair next() { - DBID id = this.iter.next(); - return DBIDUtil.newPair(id, id); + DBIDPair pair = DBIDUtil.newPair(iter, iter); + this.iter.advance(); + return pair; } @Override @@ -332,7 +263,7 @@ public class ROC { /** * Original Iterator */ - private Iterator<DBID> iter; + private DBIDIter iter; /** * Outlier score @@ -346,18 +277,19 @@ public class ROC { */ public OutlierScoreAdapter(OutlierResult o) { super(); - this.iter = o.getOrdering().iter(o.getScores().getDBIDs()); + this.iter = o.getOrdering().iter(o.getScores().getDBIDs()).iter(); this.scores = o.getScores(); } @Override public boolean hasNext() { - return this.iter.hasNext(); + return this.iter.valid(); } @Override public DoubleObjPair<DBID> next() { - DBID id = this.iter.next(); + DBID id = this.iter.getDBID(); + iter.advance(); return new DoubleObjPair<DBID>(scores.get(id), id); } @@ -368,38 +300,6 @@ public class ROC { } /** - * compute the Area Under Curve (difference to y axis) for an arbitrary - * polygon - * - * @param curve Iterable list of points (x,y) - * @return area und curve - */ - public static double computeAUC(Iterable<DoubleDoublePair> curve) { - double result = 0.0; - Iterator<DoubleDoublePair> iter = curve.iterator(); - // it doesn't make sense to speak about the "area under a curve" when there - // is no curve. - if(!iter.hasNext()) { - return Double.NaN; - } - // starting point - DoubleDoublePair prev = iter.next(); - // check there is at least a second point - if(!iter.hasNext()) { - return Double.NaN; - } - while(iter.hasNext()) { - DoubleDoublePair next = iter.next(); - // width * height at half way. - double width = next.first - prev.first; - double meanheight = (next.second + prev.second) / 2; - result += width * meanheight; - prev = next; - } - return result; - } - - /** * Compute a ROC curves Area-under-curve for a QueryResult and a Cluster. * * @param <D> Distance type @@ -424,8 +324,8 @@ public class ROC { */ public static <D extends Distance<D>> double computeROCAUCDistanceResult(int size, DBIDs ids, Iterable<? extends DistanceResultPair<D>> nei) { // TODO: do not materialize the ROC, but introduce an iterator interface - List<DoubleDoublePair> roc = materializeROC(size, DBIDUtil.ensureSet(ids), new DistanceResultAdapter<D>(nei.iterator())); - return computeAUC(roc); + XYCurve roc = materializeROC(size, DBIDUtil.ensureSet(ids), new DistanceResultAdapter<D>(nei.iterator())); + return XYCurve.areaUnderCurve(roc); } /** @@ -438,7 +338,7 @@ public class ROC { */ public static double computeROCAUCSimple(int size, DBIDs ids, DBIDs nei) { // TODO: do not materialize the ROC, but introduce an iterator interface - List<DoubleDoublePair> roc = materializeROC(size, DBIDUtil.ensureSet(ids), new SimpleAdapter(nei.iterator())); - return computeAUC(roc); + XYCurve roc = materializeROC(size, DBIDUtil.ensureSet(ids), new SimpleAdapter(nei.iter())); + return XYCurve.areaUnderCurve(roc); } -} +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/evaluation/similaritymatrix/ComputeSimilarityMatrixImage.java b/src/de/lmu/ifi/dbs/elki/evaluation/similaritymatrix/ComputeSimilarityMatrixImage.java index 5344da15..0e5c7a02 100644 --- a/src/de/lmu/ifi/dbs/elki/evaluation/similaritymatrix/ComputeSimilarityMatrixImage.java +++ b/src/de/lmu/ifi/dbs/elki/evaluation/similaritymatrix/ComputeSimilarityMatrixImage.java @@ -27,7 +27,6 @@ import java.awt.image.BufferedImage; import java.awt.image.RenderedImage; import java.io.File; import java.io.IOException; -import java.util.Iterator; import java.util.List; import javax.imageio.ImageIO; @@ -37,6 +36,7 @@ import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.relation.Relation; @@ -49,7 +49,6 @@ import de.lmu.ifi.dbs.elki.logging.LoggingUtil; import de.lmu.ifi.dbs.elki.logging.progress.FiniteProgress; import de.lmu.ifi.dbs.elki.math.DoubleMinMax; import de.lmu.ifi.dbs.elki.result.HierarchicalResult; -import de.lmu.ifi.dbs.elki.result.IterableResult; import de.lmu.ifi.dbs.elki.result.OrderingResult; import de.lmu.ifi.dbs.elki.result.PixmapResult; import de.lmu.ifi.dbs.elki.result.Result; @@ -124,10 +123,10 @@ public class ComputeSimilarityMatrixImage<O> implements Evaluator { * @param iter DBID iterator * @return result object */ - private SimilarityMatrix computeSimilarityMatrixImage(Relation<O> relation, Iterator<DBID> iter) { + private SimilarityMatrix computeSimilarityMatrixImage(Relation<O> relation, DBIDIter iter) { ArrayModifiableDBIDs order = DBIDUtil.newArray(relation.size()); - while(iter.hasNext()) { - Object o = iter.next(); + for(; iter.valid(); iter.advance()) { + Object o = iter.getDBID(); if(!(o instanceof DBID)) { throw new IllegalStateException("Iterable result contained non-DBID - result didn't satisfy requirements"); } @@ -199,54 +198,27 @@ public class ComputeSimilarityMatrixImage<O> implements Evaluator { return new SimilarityMatrix(img, relation, order); } - /** - * Wrap the uncheckable cast with the manual check. - * - * @param ir Interable result - * @return Iterator if Integer iterable, null otherwise. - */ - @SuppressWarnings("unchecked") - private Iterator<DBID> getDBIDIterator(IterableResult<?> ir) { - Iterator<?> testit = ir.iterator(); - if(testit.hasNext() && (testit.next() instanceof DBID)) { - // note: we DO want a fresh iterator here! - return (Iterator<DBID>) ir.iterator(); - } - return null; - } - @Override public void processNewResult(HierarchicalResult baseResult, Result result) { Database db = ResultUtil.findDatabase(baseResult); boolean nonefound = true; List<OutlierResult> oresults = ResultUtil.getOutlierResults(result); - List<IterableResult<?>> iterables = ResultUtil.getIterableResults(result); List<OrderingResult> orderings = ResultUtil.getOrderingResults(result); // Outlier results are the main use case. for(OutlierResult o : oresults) { final OrderingResult or = o.getOrdering(); Relation<O> relation = db.getRelation(distanceFunction.getInputTypeRestriction()); - db.getHierarchy().add(or, computeSimilarityMatrixImage(relation, or.iter(relation.getDBIDs()))); + db.getHierarchy().add(or, computeSimilarityMatrixImage(relation, or.iter(relation.getDBIDs()).iter())); // Process them only once. orderings.remove(or); nonefound = false; } - // try iterable results first - // FIXME: find the appropriate place to call addDerivedResult - for(IterableResult<?> ir : iterables) { - Iterator<DBID> iter = getDBIDIterator(ir); - if(iter != null) { - Relation<O> relation = db.getRelation(distanceFunction.getInputTypeRestriction()); - db.getHierarchy().add(ir, computeSimilarityMatrixImage(relation, iter)); - nonefound = false; - } - } // FIXME: find appropriate place to add the derived result // otherwise apply an ordering to the database IDs. for(OrderingResult or : orderings) { Relation<O> relation = db.getRelation(distanceFunction.getInputTypeRestriction()); - Iterator<DBID> iter = or.iter(relation.getDBIDs()); + DBIDIter iter = or.iter(relation.getDBIDs()).iter(); db.getHierarchy().add(or, computeSimilarityMatrixImage(relation, iter)); nonefound = false; } diff --git a/src/de/lmu/ifi/dbs/elki/gui/configurator/ClassParameterConfigurator.java b/src/de/lmu/ifi/dbs/elki/gui/configurator/ClassParameterConfigurator.java index 272513bd..91913755 100644 --- a/src/de/lmu/ifi/dbs/elki/gui/configurator/ClassParameterConfigurator.java +++ b/src/de/lmu/ifi/dbs/elki/gui/configurator/ClassParameterConfigurator.java @@ -75,7 +75,7 @@ public class ClassParameterConfigurator extends AbstractSingleParameterConfigura // For parameters with a default value, offer using the default // For optional parameters, offer not specifying them. if(cp.hasDefaultValue()) { - value.addItem(DynamicParameters.STRING_USE_DEFAULT + " " + cp.getDefaultValueAsString()); + value.addItem(DynamicParameters.STRING_USE_DEFAULT + cp.getDefaultValueAsString()); } else if(cp.isOptional()) { value.addItem(DynamicParameters.STRING_OPTIONAL); diff --git a/src/de/lmu/ifi/dbs/elki/gui/configurator/EnumParameterConfigurator.java b/src/de/lmu/ifi/dbs/elki/gui/configurator/EnumParameterConfigurator.java index 62184bf4..40748dcc 100644 --- a/src/de/lmu/ifi/dbs/elki/gui/configurator/EnumParameterConfigurator.java +++ b/src/de/lmu/ifi/dbs/elki/gui/configurator/EnumParameterConfigurator.java @@ -68,7 +68,7 @@ public class EnumParameterConfigurator extends AbstractSingleParameterConfigurat // For parameters with a default value, offer using the default // For optional parameters, offer not specifying them. if(cp.hasDefaultValue()) { - value.addItem(DynamicParameters.STRING_USE_DEFAULT + " " + cp.getDefaultValueAsString()); + value.addItem(DynamicParameters.STRING_USE_DEFAULT + cp.getDefaultValueAsString()); } else if(cp.isOptional()) { value.addItem(DynamicParameters.STRING_OPTIONAL); diff --git a/src/de/lmu/ifi/dbs/elki/gui/configurator/TextParameterConfigurator.java b/src/de/lmu/ifi/dbs/elki/gui/configurator/TextParameterConfigurator.java index 68b977ff..8c011091 100644 --- a/src/de/lmu/ifi/dbs/elki/gui/configurator/TextParameterConfigurator.java +++ b/src/de/lmu/ifi/dbs/elki/gui/configurator/TextParameterConfigurator.java @@ -39,7 +39,7 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.Parameter; * * @author Erich Schubert * - * @apiviz.uses ClassParameter + * @apiviz.uses Parameter */ // FIXME: update on focus loss? // FIXME: restrictions for number input? diff --git a/src/de/lmu/ifi/dbs/elki/gui/minigui/MiniGUI.java b/src/de/lmu/ifi/dbs/elki/gui/minigui/MiniGUI.java index ed59e8a4..d9bf30c2 100644 --- a/src/de/lmu/ifi/dbs/elki/gui/minigui/MiniGUI.java +++ b/src/de/lmu/ifi/dbs/elki/gui/minigui/MiniGUI.java @@ -379,7 +379,7 @@ public class MiniGUI extends JPanel { logger.debug("Task completed successfully."); } catch(Throwable e) { - logger.exception(e); + logger.exception("Task failed", e); } return null; } diff --git a/src/de/lmu/ifi/dbs/elki/gui/util/DynamicParameters.java b/src/de/lmu/ifi/dbs/elki/gui/util/DynamicParameters.java index e59965b1..dc442b07 100644 --- a/src/de/lmu/ifi/dbs/elki/gui/util/DynamicParameters.java +++ b/src/de/lmu/ifi/dbs/elki/gui/util/DynamicParameters.java @@ -72,7 +72,7 @@ public class DynamicParameters { /** * Pseudo-value used in dropdowns for options that have a default value */ - public static final String STRING_USE_DEFAULT = "Default:"; + public static final String STRING_USE_DEFAULT = "Default: "; /** * Pseudo-value used in options that are optional, to unset. @@ -146,7 +146,7 @@ public class DynamicParameters { String value = null; if(option.isDefined()) { if(option.tookDefaultValue()) { - value = DynamicParameters.STRING_USE_DEFAULT + " " + option.getDefaultValueAsString(); + value = DynamicParameters.STRING_USE_DEFAULT + option.getDefaultValueAsString(); } else { value = option.getValueAsString(); diff --git a/src/de/lmu/ifi/dbs/elki/gui/util/LogPane.java b/src/de/lmu/ifi/dbs/elki/gui/util/LogPane.java index e1c9bc53..0c73dc98 100644 --- a/src/de/lmu/ifi/dbs/elki/gui/util/LogPane.java +++ b/src/de/lmu/ifi/dbs/elki/gui/util/LogPane.java @@ -29,12 +29,12 @@ import java.util.logging.Formatter; import java.util.logging.Handler; import java.util.logging.Level; import java.util.logging.LogRecord; -import java.util.logging.SimpleFormatter; import javax.swing.JTextPane; import javax.swing.text.Style; import javax.swing.text.StyleConstants; +import de.lmu.ifi.dbs.elki.logging.ErrorFormatter; import de.lmu.ifi.dbs.elki.logging.LoggingConfiguration; import de.lmu.ifi.dbs.elki.logging.MessageFormatter; import de.lmu.ifi.dbs.elki.logging.OutputStreamLogger; @@ -84,12 +84,12 @@ public class LogPane extends JTextPane { /** * Formatter for debugging messages */ - private Formatter debugformat = new SimpleFormatter(); + private Formatter debugformat = new ErrorFormatter(); /** * Formatter for error messages */ - private Formatter errformat = new SimpleFormatter(); + private Formatter errformat = new ErrorFormatter(); /** * Last newline position diff --git a/src/de/lmu/ifi/dbs/elki/gui/util/ParameterTable.java b/src/de/lmu/ifi/dbs/elki/gui/util/ParameterTable.java index dd674041..e1f75bcf 100644 --- a/src/de/lmu/ifi/dbs/elki/gui/util/ParameterTable.java +++ b/src/de/lmu/ifi/dbs/elki/gui/util/ParameterTable.java @@ -237,7 +237,7 @@ public class ParameterTable extends JTable { // For parameters with a default value, offer using the default // For optional parameters, offer not specifying them. if(cp.hasDefaultValue()) { - comboBox.addItem(DynamicParameters.STRING_USE_DEFAULT + " " + cp.getDefaultValueAsString()); + comboBox.addItem(DynamicParameters.STRING_USE_DEFAULT + cp.getDefaultValueAsString()); } else if(cp.isOptional()) { comboBox.addItem(DynamicParameters.STRING_OPTIONAL); @@ -257,14 +257,15 @@ public class ParameterTable extends JTable { } } // and for Enum parameters. - else if (option instanceof EnumParameter<?>) { + else if(option instanceof EnumParameter<?>) { EnumParameter<?> ep = (EnumParameter<?>) option; - for (String s : ep.getPossibleValues()) { - if (ep.hasDefaultValue() && ep.getDefaultValueAsString().equals(s)) { - if (!(DynamicParameters.STRING_USE_DEFAULT + " " + ep.getDefaultValueAsString()).equals(val)) { - comboBox.addItem(DynamicParameters.STRING_USE_DEFAULT + " " + s); + for(String s : ep.getPossibleValues()) { + if(ep.hasDefaultValue() && ep.getDefaultValueAsString().equals(s)) { + if(!(DynamicParameters.STRING_USE_DEFAULT + ep.getDefaultValueAsString()).equals(val)) { + comboBox.addItem(DynamicParameters.STRING_USE_DEFAULT + s); } - } else if (!s.equals(val)) { + } + else if(!s.equals(val)) { comboBox.addItem(s); } } @@ -534,7 +535,7 @@ public class ParameterTable extends JTable { } if(option.isDefined()) { if(option.tookDefaultValue()) { - textfield.setText(DynamicParameters.STRING_USE_DEFAULT + " " + option.getDefaultValueAsString()); + textfield.setText(DynamicParameters.STRING_USE_DEFAULT + option.getDefaultValueAsString()); } else { textfield.setText(option.getValueAsString()); @@ -612,6 +613,12 @@ public class ParameterTable extends JTable { @Override public Component getTableCellEditorComponent(JTable table, Object value, boolean isSelected, int row, int column) { + if(value instanceof String) { + String s = (String) value; + if(s.startsWith(DynamicParameters.STRING_USE_DEFAULT)) { + value = s.substring(DynamicParameters.STRING_USE_DEFAULT.length()); + } + } if(row < parameters.size()) { Parameter<?, ?> option = parameters.getNode(row).param; if(option instanceof Flag) { diff --git a/src/de/lmu/ifi/dbs/elki/gui/util/ParametersModel.java b/src/de/lmu/ifi/dbs/elki/gui/util/ParametersModel.java index e9fba0fd..bb899701 100644 --- a/src/de/lmu/ifi/dbs/elki/gui/util/ParametersModel.java +++ b/src/de/lmu/ifi/dbs/elki/gui/util/ParametersModel.java @@ -43,7 +43,7 @@ public class ParametersModel extends AbstractTableModel { * Logger */ private static final Logging logger = Logging.getLogger(ParametersModel.class); - + /** * Parameter storage */ @@ -112,12 +112,20 @@ public class ParametersModel extends AbstractTableModel { public boolean isCellEditable(int rowIndex, int columnIndex) { return (columnIndex == 1) || (rowIndex > parameters.size()); } - + @Override public void setValueAt(Object value, int rowIndex, int columnIndex) { if(value instanceof String) { String s = (String) value; if(columnIndex == 1) { + // Unchanged? + if(s.equals(parameters.getNode(rowIndex).value)) { + return; + } + // Default value: + if((DynamicParameters.STRING_USE_DEFAULT + s).equals(parameters.getNode(rowIndex).value)) { + return; + } parameters.getNode(rowIndex).value = s; fireTableCellUpdated(rowIndex, columnIndex); } diff --git a/src/de/lmu/ifi/dbs/elki/index/AbstractRefiningIndex.java b/src/de/lmu/ifi/dbs/elki/index/AbstractRefiningIndex.java index 30abb4a4..1d42b7b3 100644 --- a/src/de/lmu/ifi/dbs/elki/index/AbstractRefiningIndex.java +++ b/src/de/lmu/ifi/dbs/elki/index/AbstractRefiningIndex.java @@ -1,15 +1,37 @@ package de.lmu.ifi.dbs.elki.index; +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ import java.util.List; import java.util.Map; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; -import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; +import de.lmu.ifi.dbs.elki.database.query.DistanceDBIDResult; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.query.knn.AbstractDistanceKNNQuery; -import de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery; import de.lmu.ifi.dbs.elki.database.query.knn.KNNResult; import de.lmu.ifi.dbs.elki.database.query.range.AbstractDistanceRangeQuery; import de.lmu.ifi.dbs.elki.database.relation.Relation; @@ -24,6 +46,9 @@ import de.lmu.ifi.dbs.elki.utilities.datastructures.heap.KNNHeap; * * @author Erich Schubert * + * @apiviz.has AbstractRangeQuery + * @apiviz.has AbstractKNNQuery + * * @param <O> Object type */ public abstract class AbstractRefiningIndex<O> extends AbstractIndex<O> implements PageFileStatistics { @@ -112,7 +137,7 @@ public abstract class AbstractRefiningIndex<O> extends AbstractIndex<O> implemen } @Override - public List<DistanceResultPair<D>> getRangeForDBID(DBID id, D range) { + public DistanceDBIDResult<D> getRangeForDBID(DBIDRef id, D range) { return getRangeForObject(relation.get(id), range); } @@ -143,7 +168,7 @@ public abstract class AbstractRefiningIndex<O> extends AbstractIndex<O> implemen * * @author Erich Schubert */ - abstract public class AbstractKNNQuery<D extends Distance<D>> extends AbstractDistanceKNNQuery<O, D> implements KNNQuery<O, D> { + abstract public class AbstractKNNQuery<D extends Distance<D>> extends AbstractDistanceKNNQuery<O, D> { /** * Constructor. * @@ -164,7 +189,7 @@ public abstract class AbstractRefiningIndex<O> extends AbstractIndex<O> implemen } @Override - public KNNResult<D> getKNNForDBID(DBID id, int k) { + public KNNResult<D> getKNNForDBID(DBIDRef id, int k) { return getKNNForObject(relation.get(id), k); } diff --git a/src/de/lmu/ifi/dbs/elki/index/preprocessed/LocalProjectionIndex.java b/src/de/lmu/ifi/dbs/elki/index/preprocessed/LocalProjectionIndex.java index cef2fbdc..4e187a9c 100644 --- a/src/de/lmu/ifi/dbs/elki/index/preprocessed/LocalProjectionIndex.java +++ b/src/de/lmu/ifi/dbs/elki/index/preprocessed/LocalProjectionIndex.java @@ -24,7 +24,7 @@ package de.lmu.ifi.dbs.elki.index.preprocessed; */ import de.lmu.ifi.dbs.elki.data.NumberVector; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.index.Index; import de.lmu.ifi.dbs.elki.index.IndexFactory; @@ -44,10 +44,10 @@ public interface LocalProjectionIndex<V extends NumberVector<?, ?>, P extends Pr /** * Get the precomputed local projection for a particular object ID. * - * @param objid Object ID + * @param id Object ID * @return local projection */ - public P getLocalProjection(DBID objid); + public P getLocalProjection(DBIDRef id); /** * Factory diff --git a/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/AbstractMaterializeKNNPreprocessor.java b/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/AbstractMaterializeKNNPreprocessor.java index b9b72d87..21e0abf7 100644 --- a/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/AbstractMaterializeKNNPreprocessor.java +++ b/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/AbstractMaterializeKNNPreprocessor.java @@ -27,7 +27,7 @@ import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery; @@ -120,17 +120,17 @@ public abstract class AbstractMaterializeKNNPreprocessor<O, D extends Distance<D /** * Get the k nearest neighbors. * - * @param objid Object ID + * @param id Object ID * @return Neighbors */ - public KNNResult<D> get(DBID objid) { + public KNNResult<D> get(DBIDRef id) { if(storage == null) { if(getLogger().isDebugging()) { getLogger().debug("Running kNN preprocessor: " + this.getClass()); } preprocess(); } - return storage.get(objid); + return storage.get(id); } /** diff --git a/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/MaterializeKNNAndRKNNPreprocessor.java b/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/MaterializeKNNAndRKNNPreprocessor.java index 1436efe2..c200472b 100644 --- a/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/MaterializeKNNAndRKNNPreprocessor.java +++ b/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/MaterializeKNNAndRKNNPreprocessor.java @@ -36,6 +36,8 @@ import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.SetDBIDs; @@ -104,9 +106,9 @@ public class MaterializeKNNAndRKNNPreprocessor<O, D extends Distance<D>> extends */ private void materializeKNNAndRKNNs(ArrayDBIDs ids, FiniteProgress progress) { // add an empty list to each rknn - for(DBID id : ids) { - if(materialized_RkNN.get(id) == null) { - materialized_RkNN.put(id, new TreeSet<DistanceResultPair<D>>()); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + if(materialized_RkNN.get(iter) == null) { + materialized_RkNN.put(iter, new TreeSet<DistanceResultPair<D>>()); } } @@ -117,7 +119,7 @@ public class MaterializeKNNAndRKNNPreprocessor<O, D extends Distance<D>> extends KNNResult<D> kNNs = kNNList.get(i); storage.put(id, kNNs); for(DistanceResultPair<D> kNN : kNNs) { - Set<DistanceResultPair<D>> rknns = materialized_RkNN.get(kNN.getDBID()); + Set<DistanceResultPair<D>> rknns = materialized_RkNN.get(kNN); rknns.add(new GenericDistanceResultPair<D>(kNN.getDistance(), id)); } if(progress != null) { @@ -169,24 +171,24 @@ public class MaterializeKNNAndRKNNPreprocessor<O, D extends Distance<D>> extends private ArrayDBIDs updateKNNsAndRkNNs(DBIDs ids) { ArrayModifiableDBIDs rkNN_ids = DBIDUtil.newArray(); DBIDs oldids = DBIDUtil.difference(relation.getDBIDs(), ids); - for(DBID id1 : oldids) { - KNNResult<D> kNNs = storage.get(id1); + for (DBIDIter iter = oldids.iter(); iter.valid(); iter.advance()) { + KNNResult<D> kNNs = storage.get(iter); D knnDist = kNNs.getKNNDistance(); // look for new kNNs KNNHeap<D> heap = null; - for(DBID id2 : ids) { - D dist = distanceQuery.distance(id1, id2); + for (DBIDIter iter2 = ids.iter(); iter2.valid(); iter2.advance()) { + D dist = distanceQuery.distance(iter, iter2); if(dist.compareTo(knnDist) <= 0) { if(heap == null) { heap = new KNNHeap<D>(k); heap.addAll(kNNs); } - heap.add(dist, id2); + heap.add(dist, iter2); } } if(heap != null) { KNNList<D> newKNNs = heap.toKNNList(); - storage.put(id1, newKNNs); + storage.put(iter, newKNNs); // get the difference int i = 0; @@ -196,7 +198,7 @@ public class MaterializeKNNAndRKNNPreprocessor<O, D extends Distance<D>> extends while(i < kNNs.size() && j < newKNNs.size()) { DistanceResultPair<D> drp1 = kNNs.get(i); DistanceResultPair<D> drp2 = newKNNs.get(j); - if(!drp1.equals(drp2)) { + if(!drp1.sameDBID(drp2)) { added.add(drp2); j++; } @@ -211,16 +213,16 @@ public class MaterializeKNNAndRKNNPreprocessor<O, D extends Distance<D>> extends } // add new RkNN for(DistanceResultPair<D> drp : added) { - Set<DistanceResultPair<D>> rknns = materialized_RkNN.get(drp.getDBID()); - rknns.add(new GenericDistanceResultPair<D>(drp.getDistance(), id1)); + Set<DistanceResultPair<D>> rknns = materialized_RkNN.get(drp); + rknns.add(new GenericDistanceResultPair<D>(drp.getDistance(), iter.getDBID())); } // remove old RkNN for(DistanceResultPair<D> drp : removed) { - Set<DistanceResultPair<D>> rknns = materialized_RkNN.get(drp.getDBID()); - rknns.remove(new GenericDistanceResultPair<D>(drp.getDistance(), id1)); + Set<DistanceResultPair<D>> rknns = materialized_RkNN.get(drp); + rknns.remove(new GenericDistanceResultPair<D>(drp.getDistance(), iter.getDBID())); } - rkNN_ids.add(id1); + rkNN_ids.add(iter); } } return rkNN_ids; @@ -237,11 +239,11 @@ public class MaterializeKNNAndRKNNPreprocessor<O, D extends Distance<D>> extends } List<KNNResult<D>> kNNs = new ArrayList<KNNResult<D>>(ids.size()); List<List<DistanceResultPair<D>>> rkNNs = new ArrayList<List<DistanceResultPair<D>>>(ids.size()); - for(DBID id : aids) { - kNNs.add(storage.get(id)); - storage.delete(id); - rkNNs.add(new ArrayList<DistanceResultPair<D>>(materialized_RkNN.get(id))); - materialized_RkNN.delete(id); + for (DBIDIter iter = aids.iter(); iter.valid(); iter.advance()) { + kNNs.add(storage.get(iter)); + storage.delete(iter); + rkNNs.add(new ArrayList<DistanceResultPair<D>>(materialized_RkNN.get(iter))); + materialized_RkNN.delete(iter); } ArrayDBIDs kNN_ids = extractAndRemoveIDs(kNNs, aids); ArrayDBIDs rkNN_ids = extractAndRemoveIDs(rkNNs, aids); @@ -256,8 +258,7 @@ public class MaterializeKNNAndRKNNPreprocessor<O, D extends Distance<D>> extends DBID id = rkNN_ids.get(i); storage.put(id, kNNList.get(i)); for(DistanceResultPair<D> kNN : kNNList.get(i)) { - Set<DistanceResultPair<D>> rknns = materialized_RkNN.get(kNN.getDBID()); - rknns.add(new GenericDistanceResultPair<D>(kNN.getDistance(), id)); + materialized_RkNN.get(kNN).add(new GenericDistanceResultPair<D>(kNN.getDistance(), id)); } } // update the RkNNs of the kNNs @@ -267,7 +268,7 @@ public class MaterializeKNNAndRKNNPreprocessor<O, D extends Distance<D>> extends SortedSet<DistanceResultPair<D>> rkNN = materialized_RkNN.get(id); for(Iterator<DistanceResultPair<D>> it = rkNN.iterator(); it.hasNext();) { DistanceResultPair<D> drp = it.next(); - if(idsSet.contains(drp.getDBID())) { + if(idsSet.contains(drp)) { it.remove(); } } @@ -300,7 +301,7 @@ public class MaterializeKNNAndRKNNPreprocessor<O, D extends Distance<D>> extends * @param id the query id * @return the RkNNs */ - public List<DistanceResultPair<D>> getRKNN(DBID id) { + public List<DistanceResultPair<D>> getRKNN(DBIDRef id) { SortedSet<DistanceResultPair<D>> rKNN = materialized_RkNN.get(id); if(rKNN == null) return null; diff --git a/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/MaterializeKNNPreprocessor.java b/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/MaterializeKNNPreprocessor.java index fcd6fad1..cdc3fce4 100644 --- a/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/MaterializeKNNPreprocessor.java +++ b/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/MaterializeKNNPreprocessor.java @@ -31,6 +31,7 @@ import javax.swing.event.EventListenerList; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.HashSetModifiableDBIDs; @@ -127,9 +128,9 @@ public class MaterializeKNNPreprocessor<O, D extends Distance<D>> extends Abstra } } else { - for(DBID id : ids) { - KNNResult<D> knn = knnQuery.getKNNForDBID(id, k); - storage.put(id, knn); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + KNNResult<D> knn = knnQuery.getKNNForDBID(iter, k); + storage.put(iter, knn); if(progress != null) { progress.incrementProcessed(getLogger()); } @@ -150,8 +151,9 @@ public class MaterializeKNNPreprocessor<O, D extends Distance<D>> extends Abstra public void insertAll(DBIDs ids) { if(storage == null && ids.size() > 0) { preprocess(); + } else { + objectsInserted(ids); } - objectsInserted(ids); } @Override @@ -213,25 +215,25 @@ public class MaterializeKNNPreprocessor<O, D extends Distance<D>> extends Abstra private ArrayDBIDs updateKNNsAfterInsertion(DBIDs ids) { ArrayModifiableDBIDs rkNN_ids = DBIDUtil.newArray(); DBIDs oldids = DBIDUtil.difference(relation.getDBIDs(), ids); - for(DBID id1 : oldids) { - KNNResult<D> kNNs = storage.get(id1); + for (DBIDIter iter = oldids.iter(); iter.valid(); iter.advance()) { + KNNResult<D> kNNs = storage.get(iter); D knnDist = kNNs.get(kNNs.size() - 1).getDistance(); // look for new kNNs KNNHeap<D> heap = null; - for(DBID id2 : ids) { - D dist = distanceQuery.distance(id1, id2); + for (DBIDIter iter2 = ids.iter(); iter2.valid(); iter2.advance()) { + D dist = distanceQuery.distance(iter, iter2); if(dist.compareTo(knnDist) <= 0) { if(heap == null) { heap = new KNNHeap<D>(k); heap.addAll(kNNs); } - heap.add(dist, id2); + heap.add(dist, iter2); } } if(heap != null) { kNNs = heap.toKNNList(); - storage.put(id1, kNNs); - rkNN_ids.add(id1); + storage.put(iter, kNNs); + rkNN_ids.add(iter); } } return rkNN_ids; @@ -247,11 +249,11 @@ public class MaterializeKNNPreprocessor<O, D extends Distance<D>> extends Abstra private ArrayDBIDs updateKNNsAfterDeletion(DBIDs ids) { SetDBIDs idsSet = DBIDUtil.ensureSet(ids); ArrayModifiableDBIDs rkNN_ids = DBIDUtil.newArray(); - for(DBID id1 : relation.iterDBIDs()) { - KNNResult<D> kNNs = storage.get(id1); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + KNNResult<D> kNNs = storage.get(iditer); for(DistanceResultPair<D> kNN : kNNs) { - if(idsSet.contains(kNN.getDBID())) { - rkNN_ids.add(id1); + if(idsSet.contains(kNN)) { + rkNN_ids.add(iditer); break; } } @@ -280,8 +282,8 @@ public class MaterializeKNNPreprocessor<O, D extends Distance<D>> extends Abstra if(stepprog != null) { stepprog.beginStep(1, "New deletions ocurred, remove their materialized kNNs.", getLogger()); } - for(DBID id : ids) { - storage.delete(id); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + storage.delete(iter); } // update the affected kNNs @@ -349,11 +351,11 @@ public class MaterializeKNNPreprocessor<O, D extends Distance<D>> extends Abstra HashSetModifiableDBIDs ids = DBIDUtil.newHashSet(); for(Collection<DistanceResultPair<D>> drps : extraxt) { for(DistanceResultPair<D> drp : drps) { - ids.add(drp.getDBID()); + ids.add(drp); } } - for(DBID id : remove) { - ids.remove(id); + for (DBIDIter iter = remove.iter(); iter.valid(); iter.advance()) { + ids.remove(iter); } // Convert back to array return DBIDUtil.newArray(ids); diff --git a/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/MetricalIndexApproximationMaterializeKNNPreprocessor.java b/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/MetricalIndexApproximationMaterializeKNNPreprocessor.java index 5ac7d2d2..d3df7855 100644 --- a/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/MetricalIndexApproximationMaterializeKNNPreprocessor.java +++ b/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/MetricalIndexApproximationMaterializeKNNPreprocessor.java @@ -28,7 +28,8 @@ import java.util.HashMap; import java.util.List; import de.lmu.ifi.dbs.elki.data.NumberVector; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDPair; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; @@ -108,14 +109,14 @@ public class MetricalIndexApproximationMaterializeKNNPreprocessor<O extends Numb getLogger().debugFinest("NumEntires = " + size); } // Collect the ids in this node. - DBID[] ids = new DBID[size]; + ArrayModifiableDBIDs ids = DBIDUtil.newArray(size); for(int i = 0; i < size; i++) { - ids[i] = ((LeafEntry) node.getEntry(i)).getDBID(); + ids.add(((LeafEntry) node.getEntry(i)).getDBID()); } HashMap<DBIDPair, D> cache = new HashMap<DBIDPair, D>(size * size * 3 / 8); - for(DBID id : ids) { + for(DBIDIter id = ids.iter(); id.valid(); id.advance()) { KNNHeap<D> kNN = new KNNHeap<D>(k, distanceQuery.infiniteDistance()); - for(DBID id2 : ids) { + for(DBIDIter id2 = ids.iter(); id2.valid(); id2.advance()) { DBIDPair key = DBIDUtil.newPair(id, id2); D d = cache.remove(key); if(d != null) { diff --git a/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/PartitionApproximationMaterializeKNNPreprocessor.java b/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/PartitionApproximationMaterializeKNNPreprocessor.java index 90813b92..79c70642 100644 --- a/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/PartitionApproximationMaterializeKNNPreprocessor.java +++ b/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/PartitionApproximationMaterializeKNNPreprocessor.java @@ -29,7 +29,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDPair; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; @@ -109,26 +109,26 @@ public class PartitionApproximationMaterializeKNNPreprocessor<O, D extends Dista ids.add(aids.get(i * partitions + part)); } HashMap<DBIDPair, D> cache = new HashMap<DBIDPair, D>(size * size * 3 / 8); - for(DBID id : ids) { + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { KNNHeap<D> kNN = new KNNHeap<D>(k, distanceQuery.infiniteDistance()); - for(DBID id2 : ids) { - DBIDPair key = DBIDUtil.newPair(id, id2); + for (DBIDIter iter2 = ids.iter(); iter2.valid(); iter2.advance()) { + DBIDPair key = DBIDUtil.newPair(iter, iter2); D d = cache.remove(key); if(d != null) { // consume the previous result. - kNN.add(d, id2); + kNN.add(d, iter2); } else { // compute new and store the previous result. - d = distanceQuery.distance(id, id2); - kNN.add(d, id2); + d = distanceQuery.distance(iter, iter2); + kNN.add(d, iter2); // put it into the cache, but with the keys reversed - key = DBIDUtil.newPair(id2, id); + key = DBIDUtil.newPair(iter2, iter); cache.put(key, d); } } ksize.put(kNN.size()); - storage.put(id, kNN.toKNNList()); + storage.put(iter, kNN.toKNNList()); } if(logger.isDebugging()) { if(cache.size() > 0) { diff --git a/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/RandomSampleKNNPreprocessor.java b/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/RandomSampleKNNPreprocessor.java index 924c7c3c..fa868109 100644 --- a/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/RandomSampleKNNPreprocessor.java +++ b/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/RandomSampleKNNPreprocessor.java @@ -28,10 +28,9 @@ import java.util.Random; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; -import de.lmu.ifi.dbs.elki.database.query.GenericDistanceResultPair; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.query.knn.KNNResult; import de.lmu.ifi.dbs.elki.database.relation.Relation; @@ -97,17 +96,17 @@ public class RandomSampleKNNPreprocessor<O, D extends Distance<D>> extends Abstr final long iseed = (seed != null) ? seed : (new Random()).nextLong(); int i = 0; - for(DBID id : ids) { + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { KNNHeap<D> kNN = new KNNHeap<D>(k, distanceQuery.infiniteDistance()); long rseed = i * 0x7FFFFFFFFFFFFFE7L + iseed; DBIDs rsamp = DBIDUtil.randomSample(ids, samplesize, rseed); - for(DBID oid : rsamp) { - D dist = distanceQuery.distance(id, oid); - kNN.add(new GenericDistanceResultPair<D>(dist, oid)); + for (DBIDIter iter2 = rsamp.iter(); iter2.valid(); iter2.advance()) { + D dist = distanceQuery.distance(iter, iter2); + kNN.add(dist, iter2); } - storage.put(id, kNN.toKNNList()); + storage.put(iter, kNN.toKNNList()); if(progress != null) { progress.incrementProcessed(getLogger()); } diff --git a/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/SpatialApproximationMaterializeKNNPreprocessor.java b/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/SpatialApproximationMaterializeKNNPreprocessor.java index e78f5e89..b206194b 100644 --- a/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/SpatialApproximationMaterializeKNNPreprocessor.java +++ b/src/de/lmu/ifi/dbs/elki/index/preprocessed/knn/SpatialApproximationMaterializeKNNPreprocessor.java @@ -30,7 +30,8 @@ import java.util.List; import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDPair; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; @@ -113,14 +114,14 @@ public class SpatialApproximationMaterializeKNNPreprocessor<O extends NumberVect getLogger().debugFinest("NumEntires = " + size); } // Collect the ids in this node. - DBID[] ids = new DBID[size]; + ArrayModifiableDBIDs ids = DBIDUtil.newArray(size); for(int i = 0; i < size; i++) { - ids[i] = ((LeafEntry) node.getEntry(i)).getDBID(); + ids.add(((LeafEntry) node.getEntry(i)).getDBID()); } HashMap<DBIDPair, D> cache = new HashMap<DBIDPair, D>(size * size * 3 / 8); - for(DBID id : ids) { + for(DBIDIter id = ids.iter(); id.valid(); id.advance()) { KNNHeap<D> kNN = new KNNHeap<D>(k, distanceQuery.infiniteDistance()); - for(DBID id2 : ids) { + for(DBIDIter id2 = ids.iter(); id2.valid(); id2.advance()) { DBIDPair key = DBIDUtil.newPair(id, id2); D d = cache.remove(key); if(d != null) { diff --git a/src/de/lmu/ifi/dbs/elki/index/preprocessed/localpca/AbstractFilteredPCAIndex.java b/src/de/lmu/ifi/dbs/elki/index/preprocessed/localpca/AbstractFilteredPCAIndex.java index 9cb2b997..99e956c8 100644 --- a/src/de/lmu/ifi/dbs/elki/index/preprocessed/localpca/AbstractFilteredPCAIndex.java +++ b/src/de/lmu/ifi/dbs/elki/index/preprocessed/localpca/AbstractFilteredPCAIndex.java @@ -30,6 +30,8 @@ import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction; @@ -99,7 +101,8 @@ public abstract class AbstractFilteredPCAIndex<NV extends NumberVector<? extends FiniteProgress progress = getLogger().isVerbose() ? new FiniteProgress("Performing local PCA", relation.size(), getLogger()) : null; // TODO: use a bulk operation? - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); Collection<DistanceResultPair<DoubleDistance>> objects = objectsForPCA(id); PCAFilteredResult pcares = pca.processQueryResult(objects, relation); @@ -122,7 +125,7 @@ public abstract class AbstractFilteredPCAIndex<NV extends NumberVector<? extends } @Override - public PCAFilteredResult getLocalProjection(DBID objid) { + public PCAFilteredResult getLocalProjection(DBIDRef objid) { if(storage == null) { preprocess(); } diff --git a/src/de/lmu/ifi/dbs/elki/index/preprocessed/localpca/FilteredLocalPCAIndex.java b/src/de/lmu/ifi/dbs/elki/index/preprocessed/localpca/FilteredLocalPCAIndex.java index 8130f6b3..bc5d08c1 100644 --- a/src/de/lmu/ifi/dbs/elki/index/preprocessed/localpca/FilteredLocalPCAIndex.java +++ b/src/de/lmu/ifi/dbs/elki/index/preprocessed/localpca/FilteredLocalPCAIndex.java @@ -24,7 +24,7 @@ package de.lmu.ifi.dbs.elki.index.preprocessed.localpca; */ import de.lmu.ifi.dbs.elki.data.NumberVector; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.index.preprocessed.LocalProjectionIndex; import de.lmu.ifi.dbs.elki.math.linearalgebra.pca.PCAFilteredResult; @@ -44,7 +44,7 @@ public interface FilteredLocalPCAIndex<NV extends NumberVector<?, ?>> extends Lo * @return Matrix */ @Override - public PCAFilteredResult getLocalProjection(DBID objid); + public PCAFilteredResult getLocalProjection(DBIDRef objid); /** * Factory interface diff --git a/src/de/lmu/ifi/dbs/elki/index/preprocessed/localpca/RangeQueryFilteredPCAIndex.java b/src/de/lmu/ifi/dbs/elki/index/preprocessed/localpca/RangeQueryFilteredPCAIndex.java index 7d5da770..99937dbf 100644 --- a/src/de/lmu/ifi/dbs/elki/index/preprocessed/localpca/RangeQueryFilteredPCAIndex.java +++ b/src/de/lmu/ifi/dbs/elki/index/preprocessed/localpca/RangeQueryFilteredPCAIndex.java @@ -23,12 +23,10 @@ package de.lmu.ifi.dbs.elki.index.preprocessed.localpca; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.List; - import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.database.QueryUtil; import de.lmu.ifi.dbs.elki.database.ids.DBID; -import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; +import de.lmu.ifi.dbs.elki.database.query.DistanceDBIDResult; import de.lmu.ifi.dbs.elki.database.query.range.RangeQuery; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction; @@ -86,7 +84,7 @@ public class RangeQueryFilteredPCAIndex<NV extends NumberVector<? extends NV, ?> } @Override - protected List<DistanceResultPair<DoubleDistance>> objectsForPCA(DBID id) { + protected DistanceDBIDResult<DoubleDistance> objectsForPCA(DBID id) { return rangeQuery.getRangeForDBID(id, epsilon); } diff --git a/src/de/lmu/ifi/dbs/elki/index/preprocessed/preference/AbstractPreferenceVectorIndex.java b/src/de/lmu/ifi/dbs/elki/index/preprocessed/preference/AbstractPreferenceVectorIndex.java index 95c698ee..387985ab 100644 --- a/src/de/lmu/ifi/dbs/elki/index/preprocessed/preference/AbstractPreferenceVectorIndex.java +++ b/src/de/lmu/ifi/dbs/elki/index/preprocessed/preference/AbstractPreferenceVectorIndex.java @@ -28,7 +28,7 @@ import java.util.BitSet; import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.data.type.TypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.index.preprocessed.AbstractPreprocessorIndex; import de.lmu.ifi.dbs.elki.utilities.optionhandling.Parameterizable; @@ -56,7 +56,7 @@ public abstract class AbstractPreferenceVectorIndex<NV extends NumberVector<?, ? abstract protected void preprocess(); @Override - public BitSet getPreferenceVector(DBID objid) { + public BitSet getPreferenceVector(DBIDRef objid) { if(storage == null) { preprocess(); } diff --git a/src/de/lmu/ifi/dbs/elki/index/preprocessed/preference/DiSHPreferenceVectorIndex.java b/src/de/lmu/ifi/dbs/elki/index/preprocessed/preference/DiSHPreferenceVectorIndex.java index ade6c114..416a5ffb 100644 --- a/src/de/lmu/ifi/dbs/elki/index/preprocessed/preference/DiSHPreferenceVectorIndex.java +++ b/src/de/lmu/ifi/dbs/elki/index/preprocessed/preference/DiSHPreferenceVectorIndex.java @@ -27,7 +27,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; @@ -42,9 +41,11 @@ import de.lmu.ifi.dbs.elki.database.UpdatableDatabase; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; +import de.lmu.ifi.dbs.elki.database.query.DistanceDBIDResult; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.database.query.distance.PrimitiveDistanceQuery; import de.lmu.ifi.dbs.elki.database.query.range.RangeQuery; @@ -161,9 +162,9 @@ public class DiSHPreferenceVectorIndex<V extends NumberVector<?, ?>> extends Abs // epsilons as string RangeQuery<V, DoubleDistance>[] rangeQueries = initRangeQueries(relation, dim); - for(Iterator<DBID> it = relation.iterDBIDs(); it.hasNext();) { + for(DBIDIter it = relation.iterDBIDs(); it.valid(); it.advance()) { StringBuffer msg = new StringBuffer(); - final DBID id = it.next(); + final DBID id = it.getDBID(); if(logger.isDebugging()) { msg.append("\nid = ").append(id); @@ -174,7 +175,7 @@ public class DiSHPreferenceVectorIndex<V extends NumberVector<?, ?>> extends Abs // determine neighbors in each dimension ModifiableDBIDs[] allNeighbors = ClassGenericsUtil.newArrayOfNull(dim, ModifiableDBIDs.class); for(int d = 0; d < dim; d++) { - List<DistanceResultPair<DoubleDistance>> qrList = rangeQueries[d].getRangeForDBID(id, epsilon[d]); + DistanceDBIDResult<DoubleDistance> qrList = rangeQueries[d].getRangeForDBID(id, epsilon[d]); allNeighbors[d] = DBIDUtil.newHashSet(qrList.size()); for(DistanceResultPair<DoubleDistance> qr : qrList) { allNeighbors[d].add(qr.getDBID()); @@ -260,8 +261,8 @@ public class DiSHPreferenceVectorIndex<V extends NumberVector<?, ?>> extends Abs // database for apriori UpdatableDatabase apriori_db = new HashmapDatabase(); SimpleTypeInformation<?> bitmeta = VectorFieldTypeInformation.get(BitVector.class, dimensionality); - for(Iterator<DBID> it = relation.iterDBIDs(); it.hasNext();) { - DBID id = it.next(); + for(DBIDIter it = relation.iterDBIDs(); it.valid(); it.advance()) { + DBID id = it.getDBID(); Bit[] bits = new Bit[dimensionality]; boolean allFalse = true; for(int d = 0; d < dimensionality; d++) { diff --git a/src/de/lmu/ifi/dbs/elki/index/preprocessed/preference/HiSCPreferenceVectorIndex.java b/src/de/lmu/ifi/dbs/elki/index/preprocessed/preference/HiSCPreferenceVectorIndex.java index 44ddb17c..65f5f61e 100644 --- a/src/de/lmu/ifi/dbs/elki/index/preprocessed/preference/HiSCPreferenceVectorIndex.java +++ b/src/de/lmu/ifi/dbs/elki/index/preprocessed/preference/HiSCPreferenceVectorIndex.java @@ -24,7 +24,6 @@ package de.lmu.ifi.dbs.elki.index.preprocessed.preference; */ import java.util.BitSet; -import java.util.Iterator; import de.lmu.ifi.dbs.elki.algorithm.clustering.subspace.HiSC; import de.lmu.ifi.dbs.elki.data.NumberVector; @@ -32,6 +31,7 @@ import de.lmu.ifi.dbs.elki.database.QueryUtil; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery; import de.lmu.ifi.dbs.elki.database.query.knn.KNNResult; @@ -107,9 +107,8 @@ public class HiSCPreferenceVectorIndex<V extends NumberVector<?, ?>> extends Abs KNNQuery<V, DoubleDistance> knnQuery = QueryUtil.getKNNQuery(relation, EuclideanDistanceFunction.STATIC, k); - Iterator<DBID> it = relation.iterDBIDs(); - while(it.hasNext()) { - DBID id = it.next(); + for (DBIDIter it = relation.iterDBIDs(); it.valid(); it.advance()) { + DBID id = it.getDBID(); if(logger.isDebugging()) { msg.append("\n\nid = ").append(id); diff --git a/src/de/lmu/ifi/dbs/elki/index/preprocessed/preference/PreferenceVectorIndex.java b/src/de/lmu/ifi/dbs/elki/index/preprocessed/preference/PreferenceVectorIndex.java index c01e9ddb..a0fba8f3 100644 --- a/src/de/lmu/ifi/dbs/elki/index/preprocessed/preference/PreferenceVectorIndex.java +++ b/src/de/lmu/ifi/dbs/elki/index/preprocessed/preference/PreferenceVectorIndex.java @@ -26,7 +26,7 @@ package de.lmu.ifi.dbs.elki.index.preprocessed.preference; import java.util.BitSet; import de.lmu.ifi.dbs.elki.data.NumberVector; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.index.Index; import de.lmu.ifi.dbs.elki.index.IndexFactory; @@ -42,10 +42,10 @@ public interface PreferenceVectorIndex<NV extends NumberVector<?, ?>> extends In /** * Get the precomputed preference vector for a particular object ID. * - * @param objid Object ID + * @param id Object ID * @return Matrix */ - public BitSet getPreferenceVector(DBID objid); + public BitSet getPreferenceVector(DBIDRef id); /** * Factory interface diff --git a/src/de/lmu/ifi/dbs/elki/index/preprocessed/snn/SharedNearestNeighborIndex.java b/src/de/lmu/ifi/dbs/elki/index/preprocessed/snn/SharedNearestNeighborIndex.java index 7efe26e0..3aa0c523 100644 --- a/src/de/lmu/ifi/dbs/elki/index/preprocessed/snn/SharedNearestNeighborIndex.java +++ b/src/de/lmu/ifi/dbs/elki/index/preprocessed/snn/SharedNearestNeighborIndex.java @@ -24,7 +24,7 @@ package de.lmu.ifi.dbs.elki.index.preprocessed.snn; */ import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.index.Index; import de.lmu.ifi.dbs.elki.index.IndexFactory; @@ -38,10 +38,10 @@ public interface SharedNearestNeighborIndex<O> extends Index { /** * Get the precomputed nearest neighbors * - * @param objid Object ID + * @param id Object ID * @return Neighbor DBIDs */ - public ArrayDBIDs getNearestNeighborSet(DBID objid); + public ArrayDBIDs getNearestNeighborSet(DBIDRef id); /** * Get the number of neighbors diff --git a/src/de/lmu/ifi/dbs/elki/index/preprocessed/snn/SharedNearestNeighborPreprocessor.java b/src/de/lmu/ifi/dbs/elki/index/preprocessed/snn/SharedNearestNeighborPreprocessor.java index 46f47a33..e4d96028 100644 --- a/src/de/lmu/ifi/dbs/elki/index/preprocessed/snn/SharedNearestNeighborPreprocessor.java +++ b/src/de/lmu/ifi/dbs/elki/index/preprocessed/snn/SharedNearestNeighborPreprocessor.java @@ -29,8 +29,10 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; +import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery; import de.lmu.ifi.dbs.elki.database.query.knn.KNNResult; import de.lmu.ifi.dbs.elki.database.relation.Relation; @@ -112,13 +114,12 @@ public class SharedNearestNeighborPreprocessor<O, D extends Distance<D>> extends KNNQuery<O, D> knnquery = QueryUtil.getKNNQuery(relation, distanceFunction, numberOfNeighbors); FiniteProgress progress = getLogger().isVerbose() ? new FiniteProgress("assigning nearest neighbor lists", relation.size(), getLogger()) : null; - for(DBID id : relation.iterDBIDs()) { + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { ArrayModifiableDBIDs neighbors = DBIDUtil.newArray(numberOfNeighbors); - KNNResult<D> kNN = knnquery.getKNNForDBID(id, numberOfNeighbors); - for(int i = 0; i < kNN.size(); i++) { - final DBID nid = kNN.get(i).getDBID(); + KNNResult<D> kNN = knnquery.getKNNForDBID(iditer, numberOfNeighbors); + for(DistanceResultPair<D> pair : kNN) { // if(!id.equals(nid)) { - neighbors.add(nid); + neighbors.add(pair); // } // Size limitation to exactly numberOfNeighbors if(neighbors.size() >= numberOfNeighbors) { @@ -126,7 +127,7 @@ public class SharedNearestNeighborPreprocessor<O, D extends Distance<D>> extends } } neighbors.sort(); - storage.put(id, neighbors); + storage.put(iditer, neighbors); if(progress != null) { progress.incrementProcessed(getLogger()); } @@ -137,7 +138,7 @@ public class SharedNearestNeighborPreprocessor<O, D extends Distance<D>> extends } @Override - public ArrayDBIDs getNearestNeighborSet(DBID objid) { + public ArrayDBIDs getNearestNeighborSet(DBIDRef objid) { if(storage == null) { preprocess(); } diff --git a/src/de/lmu/ifi/dbs/elki/index/preprocessed/subspaceproj/AbstractSubspaceProjectionIndex.java b/src/de/lmu/ifi/dbs/elki/index/preprocessed/subspaceproj/AbstractSubspaceProjectionIndex.java index 4a265fc9..da16dd08 100644 --- a/src/de/lmu/ifi/dbs/elki/index/preprocessed/subspaceproj/AbstractSubspaceProjectionIndex.java +++ b/src/de/lmu/ifi/dbs/elki/index/preprocessed/subspaceproj/AbstractSubspaceProjectionIndex.java @@ -23,9 +23,6 @@ package de.lmu.ifi.dbs.elki.index.preprocessed.subspaceproj; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.ArrayList; -import java.util.List; - import de.lmu.ifi.dbs.elki.algorithm.clustering.AbstractProjectedDBSCAN; import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.data.type.TypeInformation; @@ -33,8 +30,11 @@ import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.QueryUtil; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; +import de.lmu.ifi.dbs.elki.database.query.DistanceDBIDResult; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; +import de.lmu.ifi.dbs.elki.database.query.GenericDistanceDBIDList; import de.lmu.ifi.dbs.elki.database.query.range.RangeQuery; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction; @@ -114,20 +114,20 @@ public abstract class AbstractSubspaceProjectionIndex<NV extends NumberVector<?, RangeQuery<NV, D> rangeQuery = QueryUtil.getRangeQuery(relation, rangeQueryDistanceFunction); FiniteProgress progress = getLogger().isVerbose() ? new FiniteProgress(this.getClass().getName(), relation.size(), getLogger()) : null; - for(DBID id : relation.iterDBIDs()) { - List<DistanceResultPair<D>> neighbors = rangeQuery.getRangeForDBID(id, epsilon); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DistanceDBIDResult<D> neighbors = rangeQuery.getRangeForDBID(iditer, epsilon); final P pres; if(neighbors.size() >= minpts) { - pres = computeProjection(id, neighbors, relation); + pres = computeProjection(iditer, neighbors, relation); } else { - DistanceResultPair<D> firstQR = neighbors.get(0); - neighbors = new ArrayList<DistanceResultPair<D>>(); + DistanceResultPair<D> firstQR = neighbors.iterator().next(); + neighbors = new GenericDistanceDBIDList<D>(); neighbors.add(firstQR); - pres = computeProjection(id, neighbors, relation); + pres = computeProjection(iditer, neighbors, relation); } - storage.put(id, pres); + storage.put(iditer, pres); if(progress != null) { progress.incrementProcessed(getLogger()); @@ -146,7 +146,7 @@ public abstract class AbstractSubspaceProjectionIndex<NV extends NumberVector<?, } @Override - public P getLocalProjection(DBID objid) { + public P getLocalProjection(DBIDRef objid) { if(storage == null) { preprocess(); } @@ -167,7 +167,7 @@ public abstract class AbstractSubspaceProjectionIndex<NV extends NumberVector<?, * * @return local subspace projection */ - protected abstract P computeProjection(DBID id, List<DistanceResultPair<D>> neighbors, Relation<NV> relation); + protected abstract P computeProjection(DBIDRef id, DistanceDBIDResult<D> neighbors, Relation<NV> relation); /** * Factory class diff --git a/src/de/lmu/ifi/dbs/elki/index/preprocessed/subspaceproj/FourCSubspaceIndex.java b/src/de/lmu/ifi/dbs/elki/index/preprocessed/subspaceproj/FourCSubspaceIndex.java index 83a9469c..e61b9144 100644 --- a/src/de/lmu/ifi/dbs/elki/index/preprocessed/subspaceproj/FourCSubspaceIndex.java +++ b/src/de/lmu/ifi/dbs/elki/index/preprocessed/subspaceproj/FourCSubspaceIndex.java @@ -24,12 +24,12 @@ package de.lmu.ifi.dbs.elki.index.preprocessed.subspaceproj; */ import java.util.ArrayList; -import java.util.List; import de.lmu.ifi.dbs.elki.data.NumberVector; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; +import de.lmu.ifi.dbs.elki.database.query.DistanceDBIDResult; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction; @@ -93,7 +93,7 @@ public class FourCSubspaceIndex<V extends NumberVector<V, ?>, D extends Distance } @Override - protected PCAFilteredResult computeProjection(DBID id, List<DistanceResultPair<D>> neighbors, Relation<V> database) { + protected PCAFilteredResult computeProjection(DBIDRef id, DistanceDBIDResult<D> neighbors, Relation<V> database) { ModifiableDBIDs ids = DBIDUtil.newArray(neighbors.size()); for(DistanceResultPair<D> neighbor : neighbors) { ids.add(neighbor.getDBID()); diff --git a/src/de/lmu/ifi/dbs/elki/index/preprocessed/subspaceproj/PreDeConSubspaceIndex.java b/src/de/lmu/ifi/dbs/elki/index/preprocessed/subspaceproj/PreDeConSubspaceIndex.java index 1e83ae80..34bfb5e9 100644 --- a/src/de/lmu/ifi/dbs/elki/index/preprocessed/subspaceproj/PreDeConSubspaceIndex.java +++ b/src/de/lmu/ifi/dbs/elki/index/preprocessed/subspaceproj/PreDeConSubspaceIndex.java @@ -23,10 +23,9 @@ package de.lmu.ifi.dbs.elki.index.preprocessed.subspaceproj; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.List; - import de.lmu.ifi.dbs.elki.data.NumberVector; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; +import de.lmu.ifi.dbs.elki.database.query.DistanceDBIDResult; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction; @@ -87,7 +86,7 @@ public class PreDeConSubspaceIndex<V extends NumberVector<? extends V, ?>, D ext } @Override - protected SubspaceProjectionResult computeProjection(DBID id, List<DistanceResultPair<D>> neighbors, Relation<V> database) { + protected SubspaceProjectionResult computeProjection(DBIDRef id, DistanceDBIDResult<D> neighbors, Relation<V> database) { StringBuffer msg = null; int referenceSetSize = neighbors.size(); diff --git a/src/de/lmu/ifi/dbs/elki/index/preprocessed/subspaceproj/SubspaceProjectionIndex.java b/src/de/lmu/ifi/dbs/elki/index/preprocessed/subspaceproj/SubspaceProjectionIndex.java index d5070986..210db8f6 100644 --- a/src/de/lmu/ifi/dbs/elki/index/preprocessed/subspaceproj/SubspaceProjectionIndex.java +++ b/src/de/lmu/ifi/dbs/elki/index/preprocessed/subspaceproj/SubspaceProjectionIndex.java @@ -24,7 +24,7 @@ package de.lmu.ifi.dbs.elki.index.preprocessed.subspaceproj; */ import de.lmu.ifi.dbs.elki.data.NumberVector; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.index.preprocessed.LocalProjectionIndex; import de.lmu.ifi.dbs.elki.math.linearalgebra.ProjectionResult; @@ -46,7 +46,7 @@ public interface SubspaceProjectionIndex<NV extends NumberVector<?, ?>, P extend * @return Matrix */ @Override - public P getLocalProjection(DBID objid); + public P getLocalProjection(DBIDRef objid); /** * Factory interface diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/AbstractMTree.java b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/AbstractMTree.java index 5e949e0b..a35a4057 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/AbstractMTree.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/AbstractMTree.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.distance.DistanceUtil; @@ -378,7 +379,8 @@ public abstract class AbstractMTree<O, D extends Distance<D>, N extends Abstract if(node.isLeaf()) { for(int i = 0; i < node.getNumEntries(); i++) { E p = node.getEntry(i); - for(DBID q : ids) { + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + DBID q = iter.getDBID(); KNNHeap<D> knns_q = knnLists.get(q); D knn_q_maxDist = knns_q.getKNNDistance(); @@ -393,7 +395,8 @@ public abstract class AbstractMTree<O, D extends Distance<D>, N extends Abstract List<DistanceEntry<D, E>> entries = getSortedEntries(node, ids); for(DistanceEntry<D, E> distEntry : entries) { D minDist = distEntry.getDistance(); - for(DBID q : ids) { + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + DBID q = iter.getDBID(); KNNHeap<D> knns_q = knnLists.get(q); D knn_q_maxDist = knns_q.getKNNDistance(); @@ -446,8 +449,8 @@ public abstract class AbstractMTree<O, D extends Distance<D>, N extends Abstract E entry = node.getEntry(i); D minMinDist = getDistanceFactory().infiniteDistance(); - for(DBID q : ids) { - D distance = distanceQuery.distance(entry.getRoutingObjectID(), q); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + D distance = distanceQuery.distance(entry.getRoutingObjectID(), iter.getDBID()); D minDist = entry.getCoveringRadius().compareTo(distance) > 0 ? getDistanceFactory().nullDistance() : distance.minus(entry.getCoveringRadius()); minMinDist = DistanceUtil.max(minMinDist, minDist); } diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/AbstractMkTree.java b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/AbstractMkTree.java index ffd38a74..9391a2fa 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/AbstractMkTree.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/AbstractMkTree.java @@ -25,7 +25,7 @@ package de.lmu.ifi.dbs.elki.index.tree.metrical.mtreevariants.mktrees; import java.util.List; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction; @@ -66,5 +66,5 @@ public abstract class AbstractMkTree<O, D extends Distance<D>, N extends Abstrac * @param k the number of nearest neighbors to be returned * @return a List of the query results */ - public abstract List<DistanceResultPair<D>> reverseKNNQuery(final DBID id, int k); + public abstract List<DistanceResultPair<D>> reverseKNNQuery(final DBIDRef id, int k); }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkapp/MkAppTree.java b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkapp/MkAppTree.java index 5f92ba70..0f15a4a9 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkapp/MkAppTree.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkapp/MkAppTree.java @@ -32,6 +32,8 @@ import java.util.Map.Entry; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; @@ -176,7 +178,7 @@ public class MkAppTree<O, D extends NumberDistance<D, ?>> extends AbstractMkTree * @return a List of the query results */ @Override - public List<DistanceResultPair<D>> reverseKNNQuery(DBID id, int k) { + public List<DistanceResultPair<D>> reverseKNNQuery(DBIDRef id, int k) { List<DistanceResultPair<D>> result = doReverseKNNQuery(k, id); Collections.sort(result); return result; @@ -243,7 +245,7 @@ public class MkAppTree<O, D extends NumberDistance<D, ?>> extends AbstractMkTree * @param q the id of the query object * @return the result of the reverse knn query */ - private List<DistanceResultPair<D>> doReverseKNNQuery(int k, DBID q) { + private List<DistanceResultPair<D>> doReverseKNNQuery(int k, DBIDRef q) { List<DistanceResultPair<D>> result = new ArrayList<DistanceResultPair<D>>(); final Heap<GenericMTreeDistanceSearchCandidate<D>> pq = new UpdatableHeap<GenericMTreeDistanceSearchCandidate<D>>(); @@ -296,7 +298,8 @@ public class MkAppTree<O, D extends NumberDistance<D, ?>> extends AbstractMkTree private List<D> getMeanKNNList(DBIDs ids, Map<DBID, KNNList<D>> knnLists) { double[] means = new double[k_max]; - for(DBID id : ids) { + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); KNNList<D> knns = knnLists.get(id); List<D> knnDists = knns.asDistanceList(); for(int k = 0; k < k_max; k++) { diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkapp/MkAppTreeIndex.java b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkapp/MkAppTreeIndex.java index eedc52ed..90c31676 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkapp/MkAppTreeIndex.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkapp/MkAppTreeIndex.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.List; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.query.DatabaseQuery; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; @@ -99,7 +100,8 @@ public class MkAppTreeIndex<O, D extends NumberDistance<D, ?>> extends MkAppTree @Override public void insertAll(DBIDs ids) { List<MkAppEntry<D>> objs = new ArrayList<MkAppEntry<D>>(ids.size()); - for(DBID id : ids) { + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); final O object = relation.get(id); objs.add(createNewLeafEntry(id, object, getDistanceFactory().undefinedDistance())); } diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkapp/PolynomialApproximation.java b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkapp/PolynomialApproximation.java index 83e60e0e..2ad3558e 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkapp/PolynomialApproximation.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkapp/PolynomialApproximation.java @@ -34,7 +34,7 @@ import de.lmu.ifi.dbs.elki.utilities.FormatUtil; * Provides an polynomial approximation bo + b1*k + b2*k^2 + ... + bp*k^p * for knn-distances consisting of parameters b0, ..., bp. * - * @author Elke Achtert + * @author Elke Achtert */ public class PolynomialApproximation implements Externalizable { private static final long serialVersionUID = 1; diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkcop/MkCoPTree.java b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkcop/MkCoPTree.java index 7fe67a03..26ae17db 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkcop/MkCoPTree.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkcop/MkCoPTree.java @@ -31,6 +31,8 @@ import java.util.Map; import java.util.Map.Entry; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; @@ -171,7 +173,7 @@ public class MkCoPTree<O, D extends NumberDistance<D, ?>> extends AbstractMkTree * @return a List of the query results */ @Override - public List<DistanceResultPair<D>> reverseKNNQuery(DBID id, int k) { + public List<DistanceResultPair<D>> reverseKNNQuery(DBIDRef id, int k) { if(k > this.k_max) { throw new IllegalArgumentException("Parameter k has to be less or equal than " + "parameter kmax of the MCop-Tree!"); } @@ -182,8 +184,8 @@ public class MkCoPTree<O, D extends NumberDistance<D, ?>> extends AbstractMkTree // refinement of candidates Map<DBID, KNNHeap<D>> knnLists = new HashMap<DBID, KNNHeap<D>>(); - for(DBID cid : candidates) { - knnLists.put(cid, new KNNHeap<D>(k, getDistanceQuery().infiniteDistance())); + for (DBIDIter iter = candidates.iter(); iter.valid(); iter.advance()) { + knnLists.put(iter.getDBID(), new KNNHeap<D>(k, getDistanceQuery().infiniteDistance())); } batchNN(getRoot(), candidates, knnLists); @@ -193,7 +195,8 @@ public class MkCoPTree<O, D extends NumberDistance<D, ?>> extends AbstractMkTree rkNNStatistics.addCandidates(candidates.size()); rkNNStatistics.addTrueHits(result.size()); - for(DBID cid : candidates) { + for (DBIDIter iter = candidates.iter(); iter.valid(); iter.advance()) { + DBID cid = iter.getDBID(); for(DistanceResultPair<D> qr : knnLists.get(id)) { if(qr.getDBID().equals(id)) { result.add(new GenericDistanceResultPair<D>(qr.getDistance(), cid)); @@ -285,7 +288,7 @@ public class MkCoPTree<O, D extends NumberDistance<D, ?>> extends AbstractMkTree * @param candidates holds possible candidates for the result (they need a * refinement) */ - private void doReverseKNNQuery(int k, DBID q, List<DistanceResultPair<D>> result, ModifiableDBIDs candidates) { + private void doReverseKNNQuery(int k, DBIDRef q, List<DistanceResultPair<D>> result, ModifiableDBIDs candidates) { final Heap<GenericMTreeDistanceSearchCandidate<D>> pq = new UpdatableHeap<GenericMTreeDistanceSearchCandidate<D>>(); // push root diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkcop/MkCoPTreeIndex.java b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkcop/MkCoPTreeIndex.java index 2c4ffb06..460e15b7 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkcop/MkCoPTreeIndex.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkcop/MkCoPTreeIndex.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.List; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.query.DatabaseQuery; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; @@ -98,7 +99,8 @@ public class MkCoPTreeIndex<O, D extends NumberDistance<D, ?>> extends MkCoPTree @Override public void insertAll(DBIDs ids) { List<MkCoPEntry<D>> objs = new ArrayList<MkCoPEntry<D>>(ids.size()); - for(DBID id : ids) { + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); final O object = relation.get(id); objs.add(createNewLeafEntry(id, object, getDistanceFactory().undefinedDistance())); } diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkmax/MkMaxTree.java b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkmax/MkMaxTree.java index 5b7dc8fc..f9f77723 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkmax/MkMaxTree.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkmax/MkMaxTree.java @@ -30,6 +30,8 @@ import java.util.List; import java.util.Map; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; @@ -88,7 +90,7 @@ public class MkMaxTree<O, D extends Distance<D>> extends AbstractMkTreeUnified<O * in a second step. */ @Override - public List<DistanceResultPair<D>> reverseKNNQuery(DBID id, int k) { + public List<DistanceResultPair<D>> reverseKNNQuery(DBIDRef id, int k) { if(k > this.getKmax()) { throw new IllegalArgumentException("Parameter k has to be equal or less than " + "parameter k of the MkMax-Tree!"); } @@ -115,7 +117,8 @@ public class MkMaxTree<O, D extends Distance<D>> extends AbstractMkTreeUnified<O batchNN(getRoot(), candidateIDs, knnLists); List<DistanceResultPair<D>> result = new ArrayList<DistanceResultPair<D>>(); - for(DBID cid : candidateIDs) { + for (DBIDIter iter = candidateIDs.iter(); iter.valid(); iter.advance()) { + DBID cid = iter.getDBID(); for(DistanceResultPair<D> qr : knnLists.get(cid)) { if(id.equals(qr.getDBID())) { result.add(new GenericDistanceResultPair<D>(qr.getDistance(), cid)); @@ -191,7 +194,7 @@ public class MkMaxTree<O, D extends Distance<D>> extends AbstractMkTreeUnified<O * @param node_entry the entry representing the node * @param result the list for the query result */ - private void doReverseKNNQuery(DBID q, MkMaxTreeNode<O, D> node, MkMaxEntry<D> node_entry, List<DistanceResultPair<D>> result) { + private void doReverseKNNQuery(DBIDRef q, MkMaxTreeNode<O, D> node, MkMaxEntry<D> node_entry, List<DistanceResultPair<D>> result) { // data node if(node.isLeaf()) { for(int i = 0; i < node.getNumEntries(); i++) { diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkmax/MkMaxTreeIndex.java b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkmax/MkMaxTreeIndex.java index 9e1b6d6b..d1fd2b0f 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkmax/MkMaxTreeIndex.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mkmax/MkMaxTreeIndex.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.List; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.query.DatabaseQuery; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; @@ -84,7 +85,8 @@ public class MkMaxTreeIndex<O, D extends Distance<D>> extends MkMaxTree<O, D> im @Override public void insertAll(DBIDs ids) { List<MkMaxEntry<D>> objs = new ArrayList<MkMaxEntry<D>>(ids.size()); - for(DBID id : ids) { + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); final O object = relation.get(id); objs.add(createNewLeafEntry(id, object, getDistanceFactory().undefinedDistance())); } diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mktab/MkTabEntry.java b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mktab/MkTabEntry.java index 88000f5d..ba938fd3 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mktab/MkTabEntry.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mktab/MkTabEntry.java @@ -33,7 +33,7 @@ import de.lmu.ifi.dbs.elki.index.tree.metrical.mtreevariants.MTreeEntry; * Additionally to an entry in an M-Tree an MkTabEntry holds a list of knn distances * for for parameters k <= k_max of the underlying data object or MkTab-Tree node. * - * @author Elke Achtert + * @author Elke Achtert */ interface MkTabEntry<D extends Distance<D>> extends MTreeEntry<D> { /** diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mktab/MkTabTree.java b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mktab/MkTabTree.java index 750dfb72..433a01fa 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mktab/MkTabTree.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mktab/MkTabTree.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.database.query.GenericDistanceResultPair; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; @@ -90,7 +91,7 @@ public class MkTabTree<O, D extends Distance<D>> extends AbstractMkTreeUnified<O } @Override - public List<DistanceResultPair<D>> reverseKNNQuery(DBID id, int k) { + public List<DistanceResultPair<D>> reverseKNNQuery(DBIDRef id, int k) { if(k > this.getKmax()) { throw new IllegalArgumentException("Parameter k has to be less or equal than " + "parameter kmax of the MkTab-Tree!"); } @@ -209,7 +210,7 @@ public class MkTabTree<O, D extends Distance<D>> extends AbstractMkTreeUnified<O * @param node the root of the subtree * @param result the list holding the query result */ - private void doReverseKNNQuery(int k, DBID q, MkTabEntry<D> node_entry, MkTabTreeNode<O, D> node, List<DistanceResultPair<D>> result) { + private void doReverseKNNQuery(int k, DBIDRef q, MkTabEntry<D> node_entry, MkTabTreeNode<O, D> node, List<DistanceResultPair<D>> result) { // data node if(node.isLeaf()) { for(int i = 0; i < node.getNumEntries(); i++) { diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mktab/MkTabTreeIndex.java b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mktab/MkTabTreeIndex.java index 0eb54bd1..f1d23bfd 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mktab/MkTabTreeIndex.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mktrees/mktab/MkTabTreeIndex.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.List; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.query.DatabaseQuery; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; @@ -115,7 +116,8 @@ public class MkTabTreeIndex<O, D extends Distance<D>> extends MkTabTree<O, D> im @Override public void insertAll(DBIDs ids) { List<MkTabEntry<D>> objs = new ArrayList<MkTabEntry<D>>(ids.size()); - for(DBID id : ids) { + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); final O object = relation.get(id); objs.add(createNewLeafEntry(id, object, getDistanceFactory().undefinedDistance())); } diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mtree/MTreeIndex.java b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mtree/MTreeIndex.java index 9bcd1b02..fe60c04d 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mtree/MTreeIndex.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/mtree/MTreeIndex.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.List; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.query.DatabaseQuery; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; @@ -88,7 +89,8 @@ public class MTreeIndex<O, D extends Distance<D>> extends MTree<O, D> implements @Override public void insertAll(DBIDs ids) { List<MTreeEntry<D>> objs = new ArrayList<MTreeEntry<D>>(ids.size()); - for(DBID id : ids) { + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); final O object = relation.get(id); objs.add(createNewLeafEntry(id, object, getDistanceFactory().undefinedDistance())); } diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/query/MetricalIndexKNNQuery.java b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/query/MetricalIndexKNNQuery.java index b591f700..c2450988 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/query/MetricalIndexKNNQuery.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/query/MetricalIndexKNNQuery.java @@ -28,6 +28,7 @@ import java.util.Map; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.query.knn.AbstractDistanceKNNQuery; import de.lmu.ifi.dbs.elki.database.query.knn.KNNResult; @@ -155,7 +156,7 @@ public class MetricalIndexKNNQuery<O, D extends Distance<D>> extends AbstractDis } @Override - public KNNResult<D> getKNNForDBID(DBID id, int k) { + public KNNResult<D> getKNNForDBID(DBIDRef id, int k) { return getKNNForObject(relation.get(id), k); } diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/query/MetricalIndexRangeQuery.java b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/query/MetricalIndexRangeQuery.java index 8536cc52..e2df2dc7 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/query/MetricalIndexRangeQuery.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/query/MetricalIndexRangeQuery.java @@ -23,12 +23,14 @@ package de.lmu.ifi.dbs.elki.index.tree.metrical.mtreevariants.query; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.ArrayList; import java.util.Collections; import java.util.List; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; +import de.lmu.ifi.dbs.elki.database.query.DistanceDBIDResult; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; +import de.lmu.ifi.dbs.elki.database.query.GenericDistanceDBIDList; import de.lmu.ifi.dbs.elki.database.query.GenericDistanceResultPair; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.query.range.AbstractDistanceRangeQuery; @@ -177,8 +179,8 @@ public class MetricalIndexRangeQuery<O, D extends Distance<D>> extends AbstractD } @Override - public List<DistanceResultPair<D>> getRangeForObject(O obj, D range) { - final List<DistanceResultPair<D>> result = new ArrayList<DistanceResultPair<D>>(); + public DistanceDBIDResult<D> getRangeForObject(O obj, D range) { + final GenericDistanceDBIDList<D> result = new GenericDistanceDBIDList<D>(); doRangeQuery(null, index.getRoot(), obj, range, result); @@ -188,7 +190,7 @@ public class MetricalIndexRangeQuery<O, D extends Distance<D>> extends AbstractD } @Override - public List<DistanceResultPair<D>> getRangeForDBID(DBID id, D range) { + public DistanceDBIDResult<D> getRangeForDBID(DBIDRef id, D range) { return getRangeForObject(relation.get(id), range); } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/query/MkTreeRKNNQuery.java b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/query/MkTreeRKNNQuery.java index 370f26ad..067c3abe 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/query/MkTreeRKNNQuery.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/metrical/mtreevariants/query/MkTreeRKNNQuery.java @@ -26,7 +26,7 @@ package de.lmu.ifi.dbs.elki.index.tree.metrical.mtreevariants.query; import java.util.List; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.query.rknn.AbstractRKNNQuery; @@ -65,7 +65,7 @@ public class MkTreeRKNNQuery<O, D extends Distance<D>> extends AbstractRKNNQuery } @Override - public List<DistanceResultPair<D>> getRKNNForDBID(DBID id, int k) { + public List<DistanceResultPair<D>> getRKNNForDBID(DBIDRef id, int k) { return index.reverseKNNQuery(id, k); } diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/AbstractRStarTree.java b/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/AbstractRStarTree.java index e50e0cb4..3bac751c 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/AbstractRStarTree.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/AbstractRStarTree.java @@ -35,7 +35,7 @@ import de.lmu.ifi.dbs.elki.data.HyperBoundingBox; import de.lmu.ifi.dbs.elki.data.ModifiableHyperBoundingBox; import de.lmu.ifi.dbs.elki.data.spatial.SpatialComparable; import de.lmu.ifi.dbs.elki.data.spatial.SpatialUtil; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.index.tree.BreadthFirstEnumeration; import de.lmu.ifi.dbs.elki.index.tree.IndexTreePath; @@ -68,10 +68,10 @@ import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; * * @apiviz.landmark * @apiviz.has AbstractRStarTreeNode oneway - - contains - * @apiviz.uses Enlargement * @apiviz.composedOf BulkSplit * @apiviz.composedOf SplitStrategy * @apiviz.composedOf InsertionStrategy + * @apiviz.composedOf OverflowTreatment * * @param <N> Node type * @param <E> Entry type @@ -196,11 +196,11 @@ public abstract class AbstractRStarTree<N extends AbstractRStarTreeNode<N, E>, E * @return the path to the leaf entry of the specified subtree that represents * the data object with the specified mbr and id */ - protected IndexTreePath<E> findPathToObject(IndexTreePath<E> subtree, SpatialComparable mbr, DBID id) { + protected IndexTreePath<E> findPathToObject(IndexTreePath<E> subtree, SpatialComparable mbr, DBIDRef id) { N node = getNode(subtree.getLastPathComponent().getEntry()); if(node.isLeaf()) { for(int i = 0; i < node.getNumEntries(); i++) { - if(((LeafEntry) node.getEntry(i)).getDBID().equals(id)) { + if(((LeafEntry) node.getEntry(i)).getDBID().sameDBID(id)) { return subtree.pathByAddingChild(new TreeIndexPathComponent<E>(node.getEntry(i), i)); } } diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/deliclu/DeLiCluEntry.java b/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/deliclu/DeLiCluEntry.java index 19cf8a32..8b279268 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/deliclu/DeLiCluEntry.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/deliclu/DeLiCluEntry.java @@ -30,7 +30,7 @@ import de.lmu.ifi.dbs.elki.index.tree.spatial.SpatialEntry; * Additionally to an entry in an R*-Tree two boolean flags that indicate whether this entry's node * contains handled or unhandled data objects. * - * @author Elke Achtert + * @author Elke Achtert */ public interface DeLiCluEntry extends SpatialEntry { /** diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/deliclu/DeLiCluLeafEntry.java b/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/deliclu/DeLiCluLeafEntry.java index c7cfc493..9848f121 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/deliclu/DeLiCluLeafEntry.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/deliclu/DeLiCluLeafEntry.java @@ -32,7 +32,7 @@ import de.lmu.ifi.dbs.elki.index.tree.spatial.SpatialPointLeafEntry; * Additionally to a leaf entry in an R*-Tree two boolean flags that indicate whether this entry's node * contains handled or unhandled data objects. * - * @author Elke Achtert + * @author Elke Achtert */ public class DeLiCluLeafEntry extends SpatialPointLeafEntry implements DeLiCluEntry { private static final long serialVersionUID = 1; diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/deliclu/DeLiCluTreeIndex.java b/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/deliclu/DeLiCluTreeIndex.java index dd523ef8..dd3b4d6b 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/deliclu/DeLiCluTreeIndex.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/deliclu/DeLiCluTreeIndex.java @@ -28,6 +28,7 @@ import java.util.List; import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.query.distance.SpatialDistanceQuery; @@ -150,14 +151,14 @@ public class DeLiCluTreeIndex<O extends NumberVector<?, ?>> extends DeLiCluTree // Make an example leaf if(canBulkLoad()) { List<DeLiCluEntry> leafs = new ArrayList<DeLiCluEntry>(ids.size()); - for(DBID id : ids) { - leafs.add(createNewLeafEntry(id)); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + leafs.add(createNewLeafEntry(iter.getDBID())); } bulkLoad(leafs); } else { - for(DBID id : ids) { - insert(id); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + insert(iter.getDBID()); } } @@ -184,8 +185,8 @@ public class DeLiCluTreeIndex<O extends NumberVector<?, ?>> extends DeLiCluTree @Override public void deleteAll(DBIDs ids) { - for(DBID id : ids) { - delete(id); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + delete(iter.getDBID()); } } diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/query/DoubleDistanceRStarTreeKNNQuery.java b/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/query/DoubleDistanceRStarTreeKNNQuery.java index 9174dc94..f0acc233 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/query/DoubleDistanceRStarTreeKNNQuery.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/query/DoubleDistanceRStarTreeKNNQuery.java @@ -33,6 +33,8 @@ import java.util.Map.Entry; import de.lmu.ifi.dbs.elki.data.spatial.SpatialComparable; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; @@ -214,8 +216,8 @@ public class DoubleDistanceRStarTreeKNNQuery<O extends SpatialComparable> extend for(int i = 0; i < node.getNumEntries(); i++) { SpatialEntry entry = node.getEntry(i); double minMinDist = Double.MAX_VALUE; - for(DBID id : ids) { - double minDist = distanceFunction.doubleMinDist(entry, relation.get(id)); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + double minDist = distanceFunction.doubleMinDist(entry, relation.get(iter)); tree.distanceCalcs++; minMinDist = Math.min(minDist, minMinDist); } @@ -273,7 +275,7 @@ public class DoubleDistanceRStarTreeKNNQuery<O extends SpatialComparable> extend } @Override - public KNNResult<DoubleDistance> getKNNForDBID(DBID id, int k) { + public KNNResult<DoubleDistance> getKNNForDBID(DBIDRef id, int k) { return getKNNForObject(relation.get(id), k); } @@ -285,14 +287,16 @@ public class DoubleDistanceRStarTreeKNNQuery<O extends SpatialComparable> extend // While this works, it seems to be slow at least for large sets! final Map<DBID, KNNHeap<DoubleDistance>> knnLists = new HashMap<DBID, KNNHeap<DoubleDistance>>(ids.size()); - for(DBID id : ids) { + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); knnLists.put(id, new KNNHeap<DoubleDistance>(k, distanceFunction.getDistanceFactory().infiniteDistance())); } batchNN(tree.getRoot(), knnLists); List<KNNResult<DoubleDistance>> result = new ArrayList<KNNResult<DoubleDistance>>(); - for(DBID id : ids) { + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); result.add(knnLists.get(id).toKNNList()); } return result; diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/query/DoubleDistanceRStarTreeRangeQuery.java b/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/query/DoubleDistanceRStarTreeRangeQuery.java index 069db6d4..1a861c3e 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/query/DoubleDistanceRStarTreeRangeQuery.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/query/DoubleDistanceRStarTreeRangeQuery.java @@ -23,14 +23,13 @@ package de.lmu.ifi.dbs.elki.index.tree.spatial.rstarvariants.query; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.ArrayList; import java.util.Collections; -import java.util.List; import de.lmu.ifi.dbs.elki.data.spatial.SpatialComparable; -import de.lmu.ifi.dbs.elki.database.ids.DBID; -import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; +import de.lmu.ifi.dbs.elki.database.query.DistanceDBIDResult; import de.lmu.ifi.dbs.elki.database.query.DoubleDistanceResultPair; +import de.lmu.ifi.dbs.elki.database.query.GenericDistanceDBIDList; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.query.range.AbstractDistanceRangeQuery; import de.lmu.ifi.dbs.elki.distance.distancefunction.SpatialPrimitiveDoubleDistanceFunction; @@ -91,8 +90,8 @@ public class DoubleDistanceRStarTreeRangeQuery<O extends SpatialComparable> exte * @param epsilon Query range * @return Objects contained in query range. */ - protected List<DistanceResultPair<DoubleDistance>> doRangeQuery(O object, double epsilon) { - final List<DistanceResultPair<DoubleDistance>> result = new ArrayList<DistanceResultPair<DoubleDistance>>(); + protected GenericDistanceDBIDList<DoubleDistance> doRangeQuery(O object, double epsilon) { + final GenericDistanceDBIDList<DoubleDistance> result = new GenericDistanceDBIDList<DoubleDistance>(); final Heap<DoubleDistanceSearchCandidate> pq = new Heap<DoubleDistanceSearchCandidate>(); // push root @@ -130,12 +129,12 @@ public class DoubleDistanceRStarTreeRangeQuery<O extends SpatialComparable> exte } @Override - public List<DistanceResultPair<DoubleDistance>> getRangeForObject(O obj, DoubleDistance range) { + public DistanceDBIDResult<DoubleDistance> getRangeForObject(O obj, DoubleDistance range) { return doRangeQuery(obj, range.doubleValue()); } @Override - public List<DistanceResultPair<DoubleDistance>> getRangeForDBID(DBID id, DoubleDistance range) { + public DistanceDBIDResult<DoubleDistance> getRangeForDBID(DBIDRef id, DoubleDistance range) { return getRangeForObject(relation.get(id), range); } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/query/GenericRStarTreeKNNQuery.java b/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/query/GenericRStarTreeKNNQuery.java index 5129f5ca..09ebb61a 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/query/GenericRStarTreeKNNQuery.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/query/GenericRStarTreeKNNQuery.java @@ -33,6 +33,8 @@ import java.util.Map.Entry; import de.lmu.ifi.dbs.elki.data.spatial.SpatialComparable; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; @@ -219,8 +221,8 @@ public class GenericRStarTreeKNNQuery<O extends SpatialComparable, D extends Dis for(int i = 0; i < node.getNumEntries(); i++) { SpatialEntry entry = node.getEntry(i); D minMinDist = distanceQuery.getDistanceFactory().infiniteDistance(); - for(DBID id : ids) { - D minDist = distanceFunction.minDist(entry, relation.get(id)); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + D minDist = distanceFunction.minDist(entry, relation.get(iter)); minMinDist = DistanceUtil.min(minDist, minMinDist); } result.add(new DistanceEntry<D, SpatialEntry>(entry, minMinDist, i)); @@ -242,7 +244,7 @@ public class GenericRStarTreeKNNQuery<O extends SpatialComparable, D extends Dis } @Override - public KNNResult<D> getKNNForDBID(DBID id, int k) { + public KNNResult<D> getKNNForDBID(DBIDRef id, int k) { return getKNNForObject(relation.get(id), k); } @@ -253,15 +255,15 @@ public class GenericRStarTreeKNNQuery<O extends SpatialComparable, D extends Dis } // While this works, it seems to be slow at least for large sets! final Map<DBID, KNNHeap<D>> knnLists = new HashMap<DBID, KNNHeap<D>>(ids.size()); - for(DBID id : ids) { - knnLists.put(id, new KNNHeap<D>(k, distanceFunction.getDistanceFactory().infiniteDistance())); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + knnLists.put(iter.getDBID(), new KNNHeap<D>(k, distanceFunction.getDistanceFactory().infiniteDistance())); } batchNN(tree.getRoot(), knnLists); List<KNNResult<D>> result = new ArrayList<KNNResult<D>>(); - for(DBID id : ids) { - result.add(knnLists.get(id).toKNNList()); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + result.add(knnLists.get(iter.getDBID()).toKNNList()); } return result; } diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/query/GenericRStarTreeRangeQuery.java b/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/query/GenericRStarTreeRangeQuery.java index d2086cb1..3b312ed7 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/query/GenericRStarTreeRangeQuery.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/query/GenericRStarTreeRangeQuery.java @@ -23,13 +23,12 @@ package de.lmu.ifi.dbs.elki.index.tree.spatial.rstarvariants.query; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.ArrayList; import java.util.Collections; -import java.util.List; import de.lmu.ifi.dbs.elki.data.spatial.SpatialComparable; -import de.lmu.ifi.dbs.elki.database.ids.DBID; -import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; +import de.lmu.ifi.dbs.elki.database.query.DistanceDBIDResult; +import de.lmu.ifi.dbs.elki.database.query.GenericDistanceDBIDList; import de.lmu.ifi.dbs.elki.database.query.GenericDistanceResultPair; import de.lmu.ifi.dbs.elki.database.query.distance.SpatialDistanceQuery; import de.lmu.ifi.dbs.elki.database.query.range.AbstractDistanceRangeQuery; @@ -90,8 +89,8 @@ public class GenericRStarTreeRangeQuery<O extends SpatialComparable, D extends D * @param epsilon Query range * @return Objects contained in query range. */ - protected List<DistanceResultPair<D>> doRangeQuery(O object, D epsilon) { - final List<DistanceResultPair<D>> result = new ArrayList<DistanceResultPair<D>>(); + protected GenericDistanceDBIDList<D> doRangeQuery(O object, D epsilon) { + final GenericDistanceDBIDList<D> result = new GenericDistanceDBIDList<D>(); final Heap<GenericDistanceSearchCandidate<D>> pq = new Heap<GenericDistanceSearchCandidate<D>>(); // push root @@ -128,12 +127,12 @@ public class GenericRStarTreeRangeQuery<O extends SpatialComparable, D extends D } @Override - public List<DistanceResultPair<D>> getRangeForObject(O obj, D range) { + public DistanceDBIDResult<D> getRangeForObject(O obj, D range) { return doRangeQuery(obj, range); } @Override - public List<DistanceResultPair<D>> getRangeForDBID(DBID id, D range) { + public DistanceDBIDResult<D> getRangeForDBID(DBIDRef id, D range) { return getRangeForObject(relation.get(id), range); } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/rstar/RStarTreeIndex.java b/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/rstar/RStarTreeIndex.java index 46ef2628..ab136926 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/rstar/RStarTreeIndex.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/rstar/RStarTreeIndex.java @@ -28,6 +28,7 @@ import java.util.List; import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.query.distance.SpatialDistanceQuery; @@ -109,14 +110,14 @@ public class RStarTreeIndex<O extends NumberVector<?, ?>> extends RStarTree impl // Make an example leaf if(canBulkLoad()) { List<SpatialEntry> leafs = new ArrayList<SpatialEntry>(ids.size()); - for(DBID id : ids) { - leafs.add(createNewLeafEntry(id)); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + leafs.add(createNewLeafEntry(iter.getDBID())); } bulkLoad(leafs); } else { - for(DBID id : ids) { - insert(id); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + insert(iter.getDBID()); } } @@ -143,8 +144,8 @@ public class RStarTreeIndex<O extends NumberVector<?, ?>> extends RStarTree impl @Override public void deleteAll(DBIDs ids) { - for(DBID id : ids) { - delete(id); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + delete(iter.getDBID()); } } diff --git a/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/strategies/insert/ApproximativeLeastOverlapInsertionStrategy.java b/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/strategies/insert/ApproximativeLeastOverlapInsertionStrategy.java index 418e92c5..a1dfbdb0 100644 --- a/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/strategies/insert/ApproximativeLeastOverlapInsertionStrategy.java +++ b/src/de/lmu/ifi/dbs/elki/index/tree/spatial/rstarvariants/strategies/insert/ApproximativeLeastOverlapInsertionStrategy.java @@ -50,7 +50,7 @@ import de.lmu.ifi.dbs.elki.utilities.pairs.DoubleIntPair; * * @author Erich Schubert * @author Franz Graf - * @author Marisa Petri + * @author Marisa Thoma */ @Reference(authors = "N. Beckmann, H.-P. Kriegel, R. Schneider, B. Seeger", title = "The R*-tree: an efficient and robust access method for points and rectangles", booktitle = "Proceedings of the 1990 ACM SIGMOD International Conference on Management of Data, Atlantic City, NJ, May 23-25, 1990", url = "http://dx.doi.org/10.1145/93597.98741") public class ApproximativeLeastOverlapInsertionStrategy extends LeastOverlapInsertionStrategy { diff --git a/src/de/lmu/ifi/dbs/elki/index/vafile/DAFile.java b/src/de/lmu/ifi/dbs/elki/index/vafile/DAFile.java new file mode 100644 index 00000000..f99f5918 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/index/vafile/DAFile.java @@ -0,0 +1,111 @@ +package de.lmu.ifi.dbs.elki.index.vafile;
+
+/*
+ This file is part of ELKI:
+ Environment for Developing KDD-Applications Supported by Index-Structures +
+ Copyright (C) 2011
+ Ludwig-Maximilians-Universität München
+ Lehr- und Forschungseinheit für Datenbanksysteme
+ ELKI Development Team +
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version. +
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details. +
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import java.util.Arrays;
+
+import de.lmu.ifi.dbs.elki.data.NumberVector;
+import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
+import de.lmu.ifi.dbs.elki.database.relation.Relation;
+import de.lmu.ifi.dbs.elki.utilities.documentation.Reference;
+
+/**
+ * Dimension approximation file, a one-dimensional part of the
+ * {@link PartialVAFile}.
+ *
+ * Reference:
+ * <p>
+ * Hans-Peter Kriegel, Peer Kröger, Matthias Schubert, Ziyue Zhu:<br />
+ * Efficient Query Processing in Arbitrary Subspaces Using Vector Approximations
+ * <br />
+ * in Proc. 18th Int. Conf. on Scientific and Statistical Database Management
+ * (SSDBM 06), Wien, Austria, 2006.
+ * </p>
+ *
+ * @author Thomas Bernecker
+ * @author Erich Schubert
+ */
+@Reference(authors = "Hans-Peter Kriegel, Peer Kröger, Matthias Schubert, Ziyue Zhu", title = "Efficient Query Processing in Arbitrary Subspaces Using Vector Approximations", booktitle = "Proc. 18th Int. Conf. on Scientific and Statistical Database Management (SSDBM 06), Wien, Austria, 2006", url = "http://dx.doi.org/10.1109/SSDBM.2006.23")
+public class DAFile {
+ /**
+ * Dimension of this approximation file
+ */
+ final private int dimension;
+
+ /**
+ * Splitting grid
+ */
+ final private double[] splitPositions;
+
+ /**
+ * Constructor.
+ *
+ * @param dimension Dimension of this file
+ */
+ public DAFile(Relation<? extends NumberVector<?, ?>> relation, int dimension, int partitions) {
+ final int size = relation.size();
+ this.dimension = dimension;
+ this.splitPositions = new double[partitions + 1];
+
+ double[] tempdata = new double[size];
+ int j = 0;
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID();
+ tempdata[j] = relation.get(id).doubleValue(dimension + 1);
+ j += 1;
+ }
+ Arrays.sort(tempdata);
+
+ for(int b = 0; b < partitions; b++) {
+ int start = (int) (b * size / (double) partitions);
+ splitPositions[b] = tempdata[start];
+ }
+ // make sure that last object will be included
+ splitPositions[partitions] = tempdata[size - 1] + 0.000001;
+ }
+
+ /**
+ * @return the split positions
+ */
+ public double[] getSplitPositions() {
+ return splitPositions;
+ }
+
+ /**
+ * @return the dimension
+ */
+ public int getDimension() {
+ return dimension;
+ }
+
+ /**
+ * Estimate the IO costs for this index.
+ *
+ * @return IO costs
+ */
+ public int getIOCosts() {
+ return splitPositions.length * 8 + 4;
+ }
+}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/index/vafile/PartialVAFile.java b/src/de/lmu/ifi/dbs/elki/index/vafile/PartialVAFile.java new file mode 100644 index 00000000..848597d6 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/index/vafile/PartialVAFile.java @@ -0,0 +1,838 @@ +package de.lmu.ifi.dbs.elki.index.vafile;
+
+/*
+ This file is part of ELKI:
+ Environment for Developing KDD-Applications Supported by Index-Structures +
+ Copyright (C) 2011
+ Ludwig-Maximilians-Universität München
+ Lehr- und Forschungseinheit für Datenbanksysteme
+ ELKI Development Team +
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version. +
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details. +
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+import de.lmu.ifi.dbs.elki.data.NumberVector;
+import de.lmu.ifi.dbs.elki.data.type.TypeInformation;
+import de.lmu.ifi.dbs.elki.data.type.TypeUtil;
+import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDs;
+import de.lmu.ifi.dbs.elki.database.query.DatabaseQuery;
+import de.lmu.ifi.dbs.elki.database.query.DistanceDBIDResult;
+import de.lmu.ifi.dbs.elki.database.query.DoubleDistanceResultPair;
+import de.lmu.ifi.dbs.elki.database.query.GenericDistanceDBIDList;
+import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery;
+import de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery;
+import de.lmu.ifi.dbs.elki.database.query.knn.KNNResult;
+import de.lmu.ifi.dbs.elki.database.query.range.RangeQuery;
+import de.lmu.ifi.dbs.elki.database.relation.Relation;
+import de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction;
+import de.lmu.ifi.dbs.elki.distance.distancefunction.LPNormDistanceFunction;
+import de.lmu.ifi.dbs.elki.distance.distancefunction.subspace.SubspaceLPNormDistanceFunction;
+import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance;
+import de.lmu.ifi.dbs.elki.distance.distancevalue.DoubleDistance;
+import de.lmu.ifi.dbs.elki.index.AbstractRefiningIndex;
+import de.lmu.ifi.dbs.elki.index.IndexFactory;
+import de.lmu.ifi.dbs.elki.index.KNNIndex;
+import de.lmu.ifi.dbs.elki.index.RangeIndex;
+import de.lmu.ifi.dbs.elki.index.tree.TreeIndexFactory;
+import de.lmu.ifi.dbs.elki.logging.Logging;
+import de.lmu.ifi.dbs.elki.math.MathUtil;
+import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector;
+import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil;
+import de.lmu.ifi.dbs.elki.utilities.datastructures.heap.Heap;
+import de.lmu.ifi.dbs.elki.utilities.datastructures.heap.KNNHeap;
+import de.lmu.ifi.dbs.elki.utilities.datastructures.heap.KNNList;
+import de.lmu.ifi.dbs.elki.utilities.datastructures.heap.TopBoundedHeap;
+import de.lmu.ifi.dbs.elki.utilities.documentation.Reference;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.GreaterConstraint;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.IntParameter;
+import de.lmu.ifi.dbs.elki.utilities.pairs.DoubleObjPair;
+
+/**
+ * PartialVAFile. In-memory only implementation.
+ *
+ * Reference:
+ * <p>
+ * Hans-Peter Kriegel, Peer Kröger, Matthias Schubert, Ziyue Zhu:<br />
+ * Efficient Query Processing in Arbitrary Subspaces Using Vector Approximations
+ * <br />
+ * in Proc. 18th Int. Conf. on Scientific and Statistical Database Management
+ * (SSDBM 06), Wien, Austria, 2006.
+ * </p>
+ *
+ * @author Thomas Bernecker
+ * @author Erich Schubert
+ *
+ * @apiviz.landmark
+ *
+ * @apiviz.composedOf DAFile
+ * @apiviz.uses PartialVACandidate
+ * @apiviz.has PartialVAFileRangeQuery
+ * @apiviz.has PartialVAFileKNNQuery
+ */
+@Reference(authors = "Hans-Peter Kriegel, Peer Kröger, Matthias Schubert, Ziyue Zhu", title = "Efficient Query Processing in Arbitrary Subspaces Using Vector Approximations", booktitle = "Proc. 18th Int. Conf. on Scientific and Statistical Database Management (SSDBM 06), Wien, Austria, 2006", url = "http://dx.doi.org/10.1109/SSDBM.2006.23")
+public class PartialVAFile<V extends NumberVector<?, ?>> extends AbstractRefiningIndex<V> implements KNNIndex<V>, RangeIndex<V> {
+ /**
+ * Class logger
+ */
+ private static final Logging logger = Logging.getLogger(PartialVAFile.class);
+
+ /**
+ * Partial VA files
+ */
+ List<DAFile> daFiles;
+
+ /**
+ * Number of partitions
+ */
+ private final int partitions;
+
+ /**
+ * Page size
+ */
+ private final int pageSize;
+
+ /**
+ * Splitting grid
+ */
+ private double[][] splitPartitions;
+
+ /**
+ * Statistics
+ */
+ protected Statistics stats;
+
+ /**
+ * The (full - we are in-memory only right now) vector approximations.
+ */
+ private ArrayList<VectorApproximation> vectorApprox;
+
+ /**
+ * Constructor.
+ *
+ * @param pageSize Page size
+ * @param relation Data relation
+ * @param partitions Number of partitions
+ */
+ public PartialVAFile(int pageSize, Relation<V> relation, int partitions) {
+ super(relation);
+ this.pageSize = pageSize;
+ this.partitions = partitions;
+ this.stats = new Statistics();
+ }
+
+ @Override
+ public void initialize(Relation<V> relation, DBIDs ids) throws IllegalStateException {
+ if(splitPartitions != null) {
+ throw new IllegalStateException("Data already inserted.");
+ }
+
+ if((Math.log(partitions) / MathUtil.LOG2) != (int) (Math.log(partitions) / MathUtil.LOG2)) {
+ throw new IllegalArgumentException("Number of partitions must be a power of 2!");
+ }
+
+ final int dimensions = DatabaseUtil.dimensionality(relation);
+
+ splitPartitions = new double[dimensions][];
+ daFiles = new ArrayList<DAFile>(dimensions);
+ for(int d = 0; d < dimensions; d++) {
+ final DAFile f = new DAFile(relation, d, partitions);
+ splitPartitions[d] = f.getSplitPositions();
+ daFiles.add(f);
+ }
+
+ vectorApprox = new ArrayList<VectorApproximation>();
+ for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) {
+ DBID id = iter.getDBID();
+ V dv = relation.get(id);
+ VectorApproximation va = calculateFullApproximation(id, dv);
+ vectorApprox.add(va);
+ }
+ }
+
+ @Override
+ public String getShortName() {
+ return "pva-file";
+ }
+
+ @Override
+ public String getLongName() {
+ return "partial va-file";
+ }
+
+ /**
+ * Fake subspace (full-dimensional).
+ *
+ * @param relation Relation with full dimensionality
+ * @return Bit set with all bits set.
+ */
+ protected static BitSet fakeSubspace(Relation<? extends NumberVector<?, ?>> relation) {
+ int dim = DatabaseUtil.dimensionality(relation);
+ BitSet bits = new BitSet();
+ for(int i = 0; i < dim; i++) {
+ bits.set(i);
+ }
+ return bits;
+ }
+
+ /**
+ * Calculate the VA file position given the existing borders.
+ *
+ * @param id Object ID
+ * @param dv Data vector
+ * @return Vector approximation
+ */
+ protected VectorApproximation calculateFullApproximation(DBID id, V dv) {
+ int approximation[] = new int[dv.getDimensionality()];
+ for(int d = 0; d < splitPartitions.length; d++) {
+ double[] split = daFiles.get(d).getSplitPositions();
+ final double val = dv.doubleValue(d + 1);
+ final int lastBorderIndex = split.length - 1;
+
+ // Value is below data grid
+ if(val < split[0]) {
+ approximation[d] = 0;
+ if(id != null) {
+ logger.warning("Vector outside of VAFile grid!");
+ }
+ } // Value is above data grid
+ else if(val > split[lastBorderIndex]) {
+ approximation[d] = lastBorderIndex - 1;
+ if(id != null) {
+ logger.warning("Vector outside of VAFile grid!");
+ }
+ } // normal case
+ else {
+ // Search grid position
+ int pos = Arrays.binarySearch(split, val);
+ pos = (pos >= 0) ? pos : ((-pos) - 2);
+ approximation[d] = pos;
+ }
+ }
+ return new VectorApproximation(id, approximation);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <D extends Distance<D>> KNNQuery<V, D> getKNNQuery(DistanceQuery<V, D> distanceQuery, Object... hints) {
+ for(Object hint : hints) {
+ if(hint == DatabaseQuery.HINT_BULK) {
+ // FIXME: support bulk?
+ return null;
+ }
+ }
+ DistanceFunction<? super V, ?> df = distanceQuery.getDistanceFunction();
+ if(df instanceof SubspaceLPNormDistanceFunction) {
+ double p = ((SubspaceLPNormDistanceFunction) df).getP();
+ BitSet bits = ((SubspaceLPNormDistanceFunction) df).getSelectedDimensions();
+ DistanceQuery<V, ?> ddq = (DistanceQuery<V, ?>) distanceQuery;
+ KNNQuery<V, ?> dq = new PartialVAFileKNNQuery((DistanceQuery<V, DoubleDistance>) ddq, p, bits);
+ return (KNNQuery<V, D>) dq;
+ }
+ if(df instanceof LPNormDistanceFunction) {
+ double p = ((LPNormDistanceFunction) df).getP();
+ BitSet bits = fakeSubspace(distanceQuery.getRelation());
+ DistanceQuery<V, ?> ddq = (DistanceQuery<V, ?>) distanceQuery;
+ KNNQuery<V, ?> dq = new PartialVAFileKNNQuery((DistanceQuery<V, DoubleDistance>) ddq, p, bits);
+ return (KNNQuery<V, D>) dq;
+ }
+ // Not supported.
+ return null;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <D extends Distance<D>> RangeQuery<V, D> getRangeQuery(DistanceQuery<V, D> distanceQuery, Object... hints) {
+ DistanceFunction<? super V, ?> df = distanceQuery.getDistanceFunction();
+ if(df instanceof SubspaceLPNormDistanceFunction) {
+ double p = ((SubspaceLPNormDistanceFunction) df).getP();
+ BitSet bits = ((SubspaceLPNormDistanceFunction) df).getSelectedDimensions();
+ DistanceQuery<V, ?> ddq = (DistanceQuery<V, ?>) distanceQuery;
+ RangeQuery<V, ?> dq = new PartialVAFileRangeQuery((DistanceQuery<V, DoubleDistance>) ddq, p, bits);
+ return (RangeQuery<V, D>) dq;
+ }
+ if(df instanceof LPNormDistanceFunction) {
+ double p = ((LPNormDistanceFunction) df).getP();
+ BitSet bits = fakeSubspace(distanceQuery.getRelation());
+ DistanceQuery<V, ?> ddq = (DistanceQuery<V, ?>) distanceQuery;
+ RangeQuery<V, ?> dq = new PartialVAFileRangeQuery((DistanceQuery<V, DoubleDistance>) ddq, p, bits);
+ return (RangeQuery<V, D>) dq;
+ }
+ // Not supported.
+ return null;
+ }
+
+ /**
+ * Calculate selectivity coefficients.
+ *
+ * @param daFiles List of files to use
+ * @param query Query vector
+ * @param epsilon Epsilon radius
+ */
+ protected static void calculateSelectivityCoeffs(List<DoubleObjPair<DAFile>> daFiles, NumberVector<?, ?> query, double epsilon) {
+ final int dimensions = query.getDimensionality();
+ double[] lowerVals = new double[dimensions];
+ double[] upperVals = new double[dimensions];
+
+ VectorApproximation queryApprox = calculatePartialApproximation(null, query, daFiles);
+
+ for(int i = 0; i < dimensions; i++) {
+ lowerVals[i] = query.doubleValue(i + 1) - epsilon;
+ upperVals[i] = query.doubleValue(i + 1) + epsilon;
+ }
+
+ Vector lowerEpsilon = new Vector(lowerVals);
+ VectorApproximation lowerEpsilonPartitions = calculatePartialApproximation(null, lowerEpsilon, daFiles);
+
+ Vector upperEpsilon = new Vector(upperVals);
+ VectorApproximation upperEpsilonPartitions = calculatePartialApproximation(null, upperEpsilon, daFiles);
+
+ for(int i = 0; i < daFiles.size(); i++) {
+ int coeff = (queryApprox.getApproximation(i) - lowerEpsilonPartitions.getApproximation(i)) + (upperEpsilonPartitions.getApproximation(i) - queryApprox.getApproximation(i)) + 1;
+ daFiles.get(i).first = coeff;
+ }
+ }
+
+ /**
+ * Calculate partial vector approximation
+ *
+ * @param id Object ID
+ * @param dv Object vector
+ * @param daFiles List of approximations to use
+ * @return Vector approximation
+ */
+ protected static VectorApproximation calculatePartialApproximation(DBID id, NumberVector<?, ?> dv, List<DoubleObjPair<DAFile>> daFiles) {
+ int[] approximation = new int[dv.getDimensionality()];
+ for(int i = 0; i < daFiles.size(); i++) {
+ double val = dv.doubleValue(i + 1);
+ double[] borders = daFiles.get(i).second.getSplitPositions();
+ assert borders != null : "borders are null";
+ int lastBorderIndex = borders.length - 1;
+
+ // value is lower outlier
+ if(val < borders[0]) {
+ approximation[i] = 0;
+ } // value is upper outlier
+ else if(val > borders[lastBorderIndex]) {
+ approximation[i] = lastBorderIndex - 1;
+ } // normal case
+ else {
+ for(int s = 0; s < lastBorderIndex; s++) {
+ if(val >= borders[s] && val < borders[s + 1] && approximation[i] != -1) {
+ approximation[i] = s;
+ }
+ }
+ }
+ }
+ return new VectorApproximation(id, approximation);
+ }
+
+ /**
+ * Class for tracking Partial VA file statistics
+ *
+ * TODO: refactor into a common statistics API
+ *
+ * @apiviz.exclude
+ */
+ public static class Statistics {
+ public long scannedBytes = 0;
+
+ public long queryTime = 0;
+
+ public int issuedQueries = 0;
+
+ public int refinements = 0;
+ }
+
+ /**
+ * Object in a VA approximation.
+ *
+ * @author Thomas Bernecker
+ * @author Erich Schubert
+ */
+ protected static class PartialVACandidate implements Comparable<PartialVACandidate> {
+ /**
+ * (Current) maximum distance of this candidate.
+ */
+ protected double maxDistP = 0.0;
+
+ /**
+ * (Current) minimum distance of this candidate.
+ */
+ protected double minDistP = 0.0;
+
+ /**
+ * The actual approximation
+ */
+ final private VectorApproximation approx;
+
+ /**
+ *
+ * Constructor.
+ *
+ * @param approx The actual approximation
+ */
+ public PartialVACandidate(VectorApproximation approx) {
+ super();
+ this.approx = approx;
+ }
+
+ public int getApproximation(int dimension) {
+ return approx.getApproximation(dimension);
+ }
+
+ public DBID getId() {
+ return approx.getId();
+ }
+
+ @Override
+ public String toString() {
+ return approx.toString() + ", bounds^p: [" + minDistP + ", " + maxDistP + "]";
+ }
+
+ @Override
+ public int compareTo(PartialVACandidate o) {
+ return Double.compare(this.minDistP, o.minDistP);
+ }
+ }
+
+ /**
+ * Range query for this index.
+ *
+ * @author Erich Schubert
+ * @author Thomas Bernecker
+ */
+ public class PartialVAFileRangeQuery extends AbstractRefiningIndex<V>.AbstractRangeQuery<DoubleDistance> {
+ /**
+ * Lp-Norm p
+ */
+ private double p;
+
+ /**
+ * Subspace
+ */
+ private BitSet subspace;
+
+ /**
+ * Constructor.
+ *
+ * @param ddq Distance query
+ * @param p LP Norm p
+ * @param subspace Subspace
+ */
+ public PartialVAFileRangeQuery(DistanceQuery<V, DoubleDistance> ddq, double p, BitSet subspace) {
+ super(ddq);
+ this.p = p;
+ this.subspace = subspace;
+ }
+
+ @Override
+ public DistanceDBIDResult<DoubleDistance> getRangeForObject(V query, DoubleDistance range) {
+ stats.issuedQueries++;
+ long t = System.nanoTime();
+
+ final double epsilonP = Math.pow(range.doubleValue(), p);
+
+ // generate query approximation and lookup table
+ final VectorApproximation queryApprox = calculateFullApproximation(null, query);
+ final VALPNormDistance dist = new VALPNormDistance(p, splitPartitions, query, queryApprox);
+
+ // perform multi-step range query
+
+ // filter step
+
+ // calculate selectivity coefficients
+ List<DoubleObjPair<DAFile>> subspaceDAFiles = new ArrayList<DoubleObjPair<DAFile>>(subspace.cardinality());
+ for(int d = subspace.nextSetBit(0); d >= 0; d = subspace.nextSetBit(d + 1)) {
+ DAFile daFile = daFiles.get(d);
+ subspaceDAFiles.add(new DoubleObjPair<DAFile>(-1, daFile));
+ }
+ calculateSelectivityCoeffs(subspaceDAFiles, query, range.doubleValue());
+ // sort DA files by selectivity
+ // TODO: validate that this is the correct order
+ Collections.sort(subspaceDAFiles, Collections.reverseOrder());
+
+ // create candidate list (all objects) and prune candidates w.r.t.
+ // mindist (i.e. remove them from the list)
+ // important: this structure contains the maxDist values for refinement!
+ DistanceDBIDResult<DoubleDistance> result = new GenericDistanceDBIDList<DoubleDistance>();
+ int candidates = 0;
+ for(VectorApproximation va : vectorApprox) {
+ DBID id = va.getId();
+ PartialVACandidate pva = new PartialVACandidate(va);
+
+ boolean pruned = false;
+ for(DoubleObjPair<DAFile> da : subspaceDAFiles) {
+ int dimension = da.second.getDimension();
+ int objectCell = va.getApproximation(dimension);
+ pva.minDistP += dist.getPartialMinDist(dimension, objectCell);
+ pva.maxDistP += dist.getPartialMaxDist(dimension, objectCell);
+ if(pva.minDistP > epsilonP) {
+ pruned = true;
+ break;
+ }
+ }
+ if(!pruned) {
+ candidates++;
+ if(pva.maxDistP <= epsilonP) {
+ // candidate cannot be dropped
+ // TODO: actually: no refinement needed - need API that allows
+ // reporting maxdists only.
+ result.add(new DoubleDistanceResultPair(refine(id, query).doubleValue(), id));
+ }
+ else { // refine candidate - true refinement
+ DoubleDistance dis = refine(id, query);
+ stats.refinements += 1;
+ if(dis.doubleValue() <= range.doubleValue()) {
+ result.add(new DoubleDistanceResultPair(dis.doubleValue(), id));
+ }
+ }
+ }
+ }
+ Collections.sort(result);
+
+ stats.scannedBytes += relation.size() * VectorApproximation.byteOnDisk(subspace.cardinality(), partitions);
+
+ stats.queryTime += System.nanoTime() - t;
+
+ if(logger.isDebuggingFine()) {
+ logger.fine("query = " + query);
+ logger.fine("database: " + relation.size() + ", candidates: " + candidates + ", results: " + result.size());
+ }
+
+ return result;
+ }
+ }
+
+ /**
+ * KNN query for this index.
+ *
+ * @author Erich Schubert
+ * @author Thomas Bernecker
+ */
+ public class PartialVAFileKNNQuery extends AbstractRefiningIndex<V>.AbstractKNNQuery<DoubleDistance> {
+ /**
+ * Lp-Norm p
+ */
+ private double p;
+
+ /**
+ * Subspace
+ */
+ private BitSet subspace;
+
+ /**
+ * Constructor.
+ *
+ * @param ddq Distance query
+ * @param p LP-norm p
+ * @param subspace Subspace to query
+ */
+ public PartialVAFileKNNQuery(DistanceQuery<V, DoubleDistance> ddq, double p, BitSet subspace) {
+ super(ddq);
+ this.p = p;
+ this.subspace = subspace;
+ }
+
+ @Override
+ public KNNResult<DoubleDistance> getKNNForObject(V query, int k) {
+ stats.issuedQueries++;
+ long t = System.nanoTime();
+
+ // generate query approximation and lookup table
+ VectorApproximation queryApprox = calculateFullApproximation(null, query);
+ final VALPNormDistance dist = new VALPNormDistance(p, splitPartitions, query, queryApprox);
+
+ // sort DA files by worst case distance
+ List<DAFile> daFiles = getWorstCaseDistOrder(dist, subspace);
+
+ final int currentSubspaceDims = subspace.cardinality();
+ int reducedDims = (2 * currentSubspaceDims) / 3;
+ reducedDims = Math.max(1, reducedDims);
+ if(logger.isDebuggingFine()) {
+ logger.fine("subspaceDims=" + currentSubspaceDims + ", reducedDims=" + reducedDims);
+ }
+
+ // filter 1
+ LinkedList<PartialVACandidate> candidates1 = filter1(k, reducedDims, daFiles, queryApprox, currentSubspaceDims, dist);
+ if(logger.isDebuggingFine()) {
+ logger.fine("candidate set after filter 1: " + candidates1.size());
+ }
+
+ // filters 2+
+ LinkedList<PartialVACandidate> candidates2 = null;
+ int addition = reducedDims;
+ int filterStep = 2;
+
+ if(currentSubspaceDims <= reducedDims) {
+ candidates2 = candidates1;
+ }
+ else {
+ // continue filtering until I/O costs of refining candidates < I/O
+ // costs of loading new DA files
+ while(candidates2 == null || (getIOCosts(candidates2.size(), currentSubspaceDims) >= getIOCosts(daFiles.get(0), currentSubspaceDims - addition)) && addition < currentSubspaceDims) {
+ if(candidates2 != null && logger.isDebuggingFine()) {
+ logger.fine("filter " + filterStep + ": refining costs " + getIOCosts(candidates2.size(), currentSubspaceDims) + " (" + candidates2.size() + "/" + currentSubspaceDims + "), DA file costs " + getIOCosts(daFiles.get(0), currentSubspaceDims - addition) + " (dim " + (addition + 1) + " of " + currentSubspaceDims + ")");
+ }
+ if(candidates2 != null) {
+ candidates1 = candidates2;
+ }
+ candidates2 = new LinkedList<PartialVACandidate>();
+
+ Heap<Double> kMinMaxDists = new TopBoundedHeap<Double>(k, Collections.reverseOrder());
+ for(PartialVACandidate va : candidates1) {
+ int dimension = daFiles.get(addition).getDimension();
+ int objectCell = va.getApproximation(dimension);
+
+ va.minDistP += dist.getPartialMinDist(dimension, objectCell);
+ va.maxDistP += dist.getPartialMaxDist(dimension, objectCell) - dist.getPartialMaxMaxDist(dimension);
+
+ if(kMinMaxDists.size() < k || va.minDistP <= kMinMaxDists.peek()) {
+ candidates2.add(va);
+ kMinMaxDists.add(va.maxDistP);
+ }
+ }
+
+ if(logger.isDebuggingFine()) {
+ logger.fine("candidate set after filter " + filterStep + ": " + candidates2.size());
+ }
+
+ addition++;
+ filterStep++;
+ }
+ }
+
+ stats.scannedBytes += relation.size() * VectorApproximation.byteOnDisk(addition, partitions);
+
+ // refinement step
+ ArrayList<PartialVACandidate> sortedCandidates = new ArrayList<PartialVACandidate>(candidates2);
+ // sort candidates by lower bound (minDist)
+ Collections.sort(sortedCandidates);
+ KNNList<DoubleDistance> result = retrieveAccurateDistances(sortedCandidates, k, subspace, query);
+
+ stats.queryTime += System.nanoTime() - t;
+ return result;
+ }
+
+ private LinkedList<PartialVACandidate> filter1(int k, int reducedDims, List<DAFile> daFiles, VectorApproximation queryApprox, int subspaceDims, VALPNormDistance dist) {
+ LinkedList<PartialVACandidate> candidates1 = new LinkedList<PartialVACandidate>();
+ Heap<Double> minmaxdist = new TopBoundedHeap<Double>(k, Collections.reverseOrder());
+
+ for(VectorApproximation va : vectorApprox) {
+ PartialVACandidate pva = new PartialVACandidate(va);
+ for(int d = 0; d < reducedDims; d++) {
+ int dimension = daFiles.get(d).getDimension();
+ int objectCell = pva.getApproximation(dimension);
+ pva.minDistP += dist.getPartialMinDist(dimension, objectCell);
+ pva.maxDistP += dist.getPartialMaxDist(dimension, objectCell);
+ }
+ for(int d = reducedDims; d < subspaceDims; d++) {
+ pva.maxDistP += dist.getPartialMaxMaxDist(daFiles.get(d).getDimension());
+ }
+ if(minmaxdist.size() < k || pva.minDistP <= minmaxdist.peek()) {
+ candidates1.add(pva);
+ minmaxdist.add(pva.maxDistP);
+ }
+ }
+ // Drop candidates that don't satisfy the latest minmaxdist
+ final double minmax = minmaxdist.peek();
+ Iterator<PartialVACandidate> it = candidates1.iterator();
+ while(it.hasNext()) {
+ PartialVACandidate pva = it.next();
+ if(pva.minDistP > minmax) {
+ it.remove();
+ }
+ }
+
+ return candidates1;
+ }
+
+ /**
+ * Computes IO costs (in bytes) needed for refining the candidates.
+ *
+ * @param size The nuber of candidates
+ * @param subspaceDims the required subspace dimensions
+ * @return the cost value (in bytes)
+ */
+ private int getIOCosts(int size, int subspaceDims) {
+ return size * (subspaceDims * 8 + 4);
+ }
+
+ /**
+ * Computes IO costs (in bytes) needed for reading several DA-files.
+ *
+ * @param sample the DA-file specific costs
+ * @param numberOfDAFiles the number of DA-files that have to be read
+ * @return the cost value (in bytes)
+ */
+ private int getIOCosts(DAFile sample, int numberOfDAFiles) {
+ return sample.getIOCosts() * numberOfDAFiles;
+ }
+
+ /**
+ * Order subspaces by their worst case distance.
+ *
+ * @param dist Distance function
+ * @param subspace Subspace
+ * @return Ordered list of dimension files
+ */
+ public List<DAFile> getWorstCaseDistOrder(VALPNormDistance dist, BitSet subspace) {
+ int subspaceLength = subspace.cardinality();
+ List<DAFile> result = new ArrayList<DAFile>(subspaceLength);
+ for(int i = subspace.nextSetBit(0); i >= 0; i = subspace.nextSetBit(i + 1)) {
+ result.add(daFiles.get(i));
+ }
+ Collections.sort(result, new WorstCaseDistComparator(dist));
+ return result;
+ }
+
+ protected KNNList<DoubleDistance> retrieveAccurateDistances(List<PartialVACandidate> sortedCandidates, int k, BitSet subspace, V query) {
+ KNNHeap<DoubleDistance> result = new KNNHeap<DoubleDistance>(k, DoubleDistance.FACTORY.infiniteDistance());
+ for(PartialVACandidate va : sortedCandidates) {
+ double stopdist = result.getKNNDistance().doubleValue();
+ DBID currentID = va.getId();
+ if(result.size() < k || va.minDistP < stopdist) {
+ DoubleDistance dist = refine(currentID, query);
+ stats.refinements += 1;
+ if(dist.doubleValue() < stopdist) {
+ result.add(new DoubleDistanceResultPair(dist.doubleValue(), currentID));
+ }
+ }
+ }
+ return result.toKNNList();
+ }
+ }
+
+ /**
+ * Compare DAfiles by their worst case distance.
+ *
+ * @apiviz.exclude
+ */
+ protected static class WorstCaseDistComparator implements Comparator<DAFile> {
+ private VALPNormDistance dist;
+
+ public WorstCaseDistComparator(VALPNormDistance dist) {
+ this.dist = dist;
+ }
+
+ @Override
+ public int compare(DAFile a, DAFile b) {
+ return Double.compare(dist.getPartialMaxMaxDist(a.getDimension()), dist.getPartialMaxMaxDist(b.getDimension()));
+ }
+ }
+
+ /**
+ * Index factory class
+ *
+ * @author Erich Schubert
+ *
+ * @apiviz.stereotype «factory»
+ * @apiviz.has PartialVAFile
+ *
+ * @param <V> Vector type
+ */
+ public static class Factory<V extends NumberVector<?, ?>> implements IndexFactory<V, PartialVAFile<V>> {
+ /**
+ * Number of partitions to use in each dimension.
+ *
+ * <pre>
+ * -vafile.partitions 8
+ * </pre>
+ */
+ public static final OptionID PARTITIONS_ID = OptionID.getOrCreateOptionID("vafile.partitions", "Number of partitions to use in each dimension.");
+
+ /**
+ * Page size
+ */
+ int pagesize = 1;
+
+ /**
+ * Number of partitions
+ */
+ int numpart = 2;
+
+ /**
+ * Constructor.
+ *
+ * @param pagesize Page size
+ * @param numpart Number of partitions
+ */
+ public Factory(int pagesize, int numpart) {
+ super();
+ this.pagesize = pagesize;
+ this.numpart = numpart;
+ }
+
+ @Override
+ public PartialVAFile<V> instantiate(Relation<V> relation) {
+ return new PartialVAFile<V>(pagesize, relation, numpart);
+ }
+
+ @Override
+ public TypeInformation getInputTypeRestriction() {
+ return TypeUtil.NUMBER_VECTOR_FIELD;
+ }
+
+ /**
+ * Parameterization class
+ *
+ * @author Erich Schubert
+ *
+ * @apiviz.exclude
+ */
+ public static class Parameterizer extends AbstractParameterizer {
+ /**
+ * Page size
+ */
+ int pagesize = 1;
+
+ /**
+ * Number of partitions
+ */
+ int numpart = 2;
+
+ @Override
+ protected void makeOptions(Parameterization config) {
+ super.makeOptions(config);
+ IntParameter pagesizeP = new IntParameter(TreeIndexFactory.PAGE_SIZE_ID, new GreaterConstraint(0), 1024);
+ if(config.grab(pagesizeP)) {
+ pagesize = pagesizeP.getValue();
+ }
+ IntParameter partitionsP = new IntParameter(Factory.PARTITIONS_ID, new GreaterConstraint(2));
+ if(config.grab(partitionsP)) {
+ numpart = partitionsP.getValue();
+ }
+ }
+
+ @Override
+ protected Factory<?> makeInstance() {
+ return new Factory<NumberVector<?, ?>>(pagesize, numpart);
+ }
+ }
+ }
+}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/index/vafile/VAFile.java b/src/de/lmu/ifi/dbs/elki/index/vafile/VAFile.java index c426c1d6..1d4f8f6d 100644 --- a/src/de/lmu/ifi/dbs/elki/index/vafile/VAFile.java +++ b/src/de/lmu/ifi/dbs/elki/index/vafile/VAFile.java @@ -32,10 +32,12 @@ import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.data.type.TypeInformation;
import de.lmu.ifi.dbs.elki.data.type.TypeUtil;
import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDs;
import de.lmu.ifi.dbs.elki.database.query.DatabaseQuery;
-import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair;
+import de.lmu.ifi.dbs.elki.database.query.DistanceDBIDResult;
import de.lmu.ifi.dbs.elki.database.query.DoubleDistanceResultPair;
+import de.lmu.ifi.dbs.elki.database.query.GenericDistanceDBIDList;
import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery;
import de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery;
import de.lmu.ifi.dbs.elki.database.query.knn.KNNResult;
@@ -76,6 +78,13 @@ import de.lmu.ifi.dbs.elki.utilities.pairs.DoubleObjPair; *
* @author Thomas Bernecker
* @author Erich Schubert
+ *
+ * @apiviz.landmark
+ *
+ * @apiviz.composedOf VectorApproximation
+ * @apiviz.has VAFileRangeQuery
+ * @apiviz.has VAFileKNNQuery
+ * @apiviz.uses VALPNormDistance
*/
@Title("An approximation based data structure for similarity search")
@Reference(authors = "Weber, R. and Blott, S.", title = "An approximation based data structure for similarity search", booktitle = "Report TR1997b, ETH Zentrum, Zurich, Switzerland", url = "http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.40.480&rep=rep1&type=pdf")
@@ -128,7 +137,8 @@ public class VAFile<V extends NumberVector<?, ?>> extends AbstractRefiningIndex< @Override
protected void initialize(Relation<V> relation, DBIDs ids) {
setPartitions(relation);
- for(DBID id : relation.getDBIDs()) {
+ for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) {
+ DBID id = iter.getDBID();
vectorApprox.add(calculateApproximation(id, relation.get(id)));
}
}
@@ -151,7 +161,8 @@ public class VAFile<V extends NumberVector<?, ?>> extends AbstractRefiningIndex< for(int d = 0; d < dimensions; d++) {
double[] tempdata = new double[size];
int j = 0;
- for(DBID id : relation.iterDBIDs()) {
+ for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID();
tempdata[j] = relation.get(id).doubleValue(d + 1);
j += 1;
}
@@ -288,7 +299,7 @@ public class VAFile<V extends NumberVector<?, ?>> extends AbstractRefiningIndex< *
* @author Erich Schubert
*/
- class VAFileRangeQuery extends AbstractRefiningIndex<V>.AbstractRangeQuery<DoubleDistance> {
+ public class VAFileRangeQuery extends AbstractRefiningIndex<V>.AbstractRangeQuery<DoubleDistance> {
/**
* LP Norm p parameter.
*/
@@ -307,7 +318,7 @@ public class VAFile<V extends NumberVector<?, ?>> extends AbstractRefiningIndex< }
@Override
- public List<DistanceResultPair<DoubleDistance>> getRangeForObject(V query, DoubleDistance range) {
+ public DistanceDBIDResult<DoubleDistance> getRangeForObject(V query, DoubleDistance range) {
final double eps = range.doubleValue();
// generate query approximation and lookup table
VectorApproximation queryApprox = calculateApproximation(null, query);
@@ -318,7 +329,7 @@ public class VAFile<V extends NumberVector<?, ?>> extends AbstractRefiningIndex< // Count a VA file scan
scans += 1;
- List<DistanceResultPair<DoubleDistance>> result = new ArrayList<DistanceResultPair<DoubleDistance>>();
+ GenericDistanceDBIDList<DoubleDistance> result = new GenericDistanceDBIDList<DoubleDistance>();
// Approximation step
for(int i = 0; i < vectorApprox.size(); i++) {
VectorApproximation va = vectorApprox.get(i);
@@ -347,7 +358,7 @@ public class VAFile<V extends NumberVector<?, ?>> extends AbstractRefiningIndex< *
* @author Erich Schubert
*/
- class VAFileKNNQuery extends AbstractRefiningIndex<V>.AbstractKNNQuery<DoubleDistance> {
+ public class VAFileKNNQuery extends AbstractRefiningIndex<V>.AbstractKNNQuery<DoubleDistance> {
/**
* LP Norm p parameter.
*/
@@ -434,6 +445,9 @@ public class VAFile<V extends NumberVector<?, ?>> extends AbstractRefiningIndex< *
* @author Erich Schubert
*
+ * @apiviz.stereotype «factory»
+ * @apiviz.has VAFile
+ *
* @param <V> Vector type
*/
public static class Factory<V extends NumberVector<?, ?>> implements IndexFactory<V, VAFile<V>> {
diff --git a/src/de/lmu/ifi/dbs/elki/logging/CLISmartHandler.java b/src/de/lmu/ifi/dbs/elki/logging/CLISmartHandler.java index 4aa6f89b..f51e6e7e 100644 --- a/src/de/lmu/ifi/dbs/elki/logging/CLISmartHandler.java +++ b/src/de/lmu/ifi/dbs/elki/logging/CLISmartHandler.java @@ -31,7 +31,6 @@ import java.util.logging.Formatter; import java.util.logging.Handler; import java.util.logging.Level; import java.util.logging.LogRecord; -import java.util.logging.SimpleFormatter; import de.lmu.ifi.dbs.elki.logging.progress.Progress; import de.lmu.ifi.dbs.elki.logging.progress.ProgressLogRecord; @@ -66,12 +65,12 @@ public class CLISmartHandler extends Handler { /** * Formatter for debugging messages */ - private Formatter debugformat = new SimpleFormatter(); + private Formatter debugformat = new ErrorFormatter(); /** * Formatter for error messages */ - private Formatter errformat = new SimpleFormatter(); + private Formatter errformat = new ErrorFormatter(); /** * Tracker for progress messages diff --git a/src/de/lmu/ifi/dbs/elki/logging/ErrorFormatter.java b/src/de/lmu/ifi/dbs/elki/logging/ErrorFormatter.java new file mode 100644 index 00000000..cd307275 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/logging/ErrorFormatter.java @@ -0,0 +1,90 @@ +package de.lmu.ifi.dbs.elki.logging; + +import java.util.logging.Formatter; +import java.util.logging.LogRecord; + +import de.lmu.ifi.dbs.elki.logging.progress.ProgressLogRecord; + +/** + * Format a log record for error output, including a stack trace if available. + * + * @author Erich Schubert + */ +// TODO: make more configurable and handle suppressed exceptions +public class ErrorFormatter extends Formatter { + /** + * List of packages to prune from stack traces at the end. + * + * TODO: make configurable via logging.properties + */ + public static String[] PRUNE = {// + "de.lmu.ifi.dbs.elki.gui.minigui.MiniGUI", // + "de.lmu.ifi.dbs.elki.KDDTask", // + "java.awt.event.", // + "java.awt.EventDispatchThread",// + "java.awt.EventQueue",// + "java.security.",// + "java.lang.Thread",// + "java.util.concurrent.",// + "javax.swing.SwingWorker", // + "java.util.concurrent.FutureTask", // + "org.apache.batik.", // + }; + + /** + * Constructor. + */ + public ErrorFormatter() { + super(); + } + + @Override + public String format(LogRecord record) { + if(record instanceof ProgressLogRecord) { + return record.getMessage(); + } + String msg = record.getMessage(); + StringBuffer buf = new StringBuffer(msg); + if(!msg.endsWith(OutputStreamLogger.NEWLINE)) { + buf.append(OutputStreamLogger.NEWLINE); + } + if(record.getThrown() != null) { + appendCauses(buf, record.getThrown()); + } + return buf.toString(); + } + + /** + * Append (pruned) stack traces for associated exceptions. + * + * @param buf Buffer to append to + * @param thrown Throwable to format. + */ + private void appendCauses(StringBuffer buf, Throwable thrown) { + buf.append(thrown.toString()).append(OutputStreamLogger.NEWLINE); + StackTraceElement[] stack = thrown.getStackTrace(); + int end = stack.length - 1; + prune: for(; end >= 0; end--) { + String cn = stack[end].getClassName(); + for(String pat : PRUNE) { + if(cn.startsWith(pat)) { + continue prune; + } + } + break; + } + if(end <= 0) { + end = stack.length - 1; + } + for(int i = 0; i <= end; i++) { + buf.append("\tat ").append(stack[i]).append(OutputStreamLogger.NEWLINE); + } + if(end < stack.length - 1) { + buf.append("\tat [...]").append(OutputStreamLogger.NEWLINE); + } + if(thrown.getCause() != null) { + buf.append("Caused by: "); + appendCauses(buf, thrown.getCause()); + } + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/logging/LoggingConfiguration.java b/src/de/lmu/ifi/dbs/elki/logging/LoggingConfiguration.java index e6a61bef..a5f507a7 100644 --- a/src/de/lmu/ifi/dbs/elki/logging/LoggingConfiguration.java +++ b/src/de/lmu/ifi/dbs/elki/logging/LoggingConfiguration.java @@ -160,6 +160,27 @@ public final class LoggingConfiguration { } /** + * Enable runtime performance logging. + * + * @param time Flag + */ + public static void setTime(boolean time) { + Logger logger1 = Logger.getLogger("de.lmu.ifi.dbs.elki.workflow.AlgorithmStep"); + if(time) { + // decrease to INFO if it was higher + if(logger1.getLevel() == null || logger1.getLevel().intValue() > Level.INFO.intValue()) { + logger1.setLevel(Level.INFO); + } + } + else { + // increase to warning level if it was INFO. + if(logger1.getLevel() != null || logger1.getLevel() == Level.INFO) { + logger1.setLevel(Level.WARNING); + } + } + } + + /** * Add a handler to the root logger. * * @param handler Handler diff --git a/src/de/lmu/ifi/dbs/elki/math/MathUtil.java b/src/de/lmu/ifi/dbs/elki/math/MathUtil.java index 7eea5ed7..c44a0203 100644 --- a/src/de/lmu/ifi/dbs/elki/math/MathUtil.java +++ b/src/de/lmu/ifi/dbs/elki/math/MathUtil.java @@ -33,7 +33,7 @@ import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; /** * A collection of math related utility functions. * - * @author Arthur Zimekt + * @author Arthur Zimek * @author Erich Schubert * * @apiviz.landmark @@ -45,7 +45,7 @@ public final class MathUtil { public static final double TWOPI = 2 * Math.PI; /** - * Squre root of two times Pi. + * Square root of two times Pi. */ public static final double SQRTTWOPI = Math.sqrt(TWOPI); @@ -60,7 +60,7 @@ public final class MathUtil { public static final double SQRT5 = Math.sqrt(5); /** - * Square root of 0.5 == 1 / Sqrt(2) + * Square root of 0.5 == 1 / sqrt(2) */ public static final double SQRTHALF = Math.sqrt(.5); @@ -75,11 +75,26 @@ public final class MathUtil { public static final double LOG2 = Math.log(2); /** + * Natural logarithm of 10 + */ + public static final double LOG10 = Math.log(10); + + /** * Math.log(Math.PI) */ public static final double LOGPI = Math.log(Math.PI); /** + * Math.log(Math.PI) / 2 + */ + public static final double LOGPIHALF = LOGPI / 2.; + + /** + * Math.log(Math.sqrt(2*Math.PI)) + */ + public static final double LOGSQRTTWOPI = Math.log(SQRTTWOPI); + + /** * Fake constructor for static class. */ private MathUtil() { diff --git a/src/de/lmu/ifi/dbs/elki/math/geometry/AlphaShape.java b/src/de/lmu/ifi/dbs/elki/math/geometry/AlphaShape.java index 8d2eb0f6..35858c67 100644 --- a/src/de/lmu/ifi/dbs/elki/math/geometry/AlphaShape.java +++ b/src/de/lmu/ifi/dbs/elki/math/geometry/AlphaShape.java @@ -35,6 +35,9 @@ import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; * Compute the alpha-Shape of a point set, using Delaunay triangulation. * * @author Erich Schubert + * + * @apiviz.uses SweepHullDelaunay2D + * @apiviz.has Polygon */ public class AlphaShape { /** diff --git a/src/de/lmu/ifi/dbs/elki/math/geometry/GrahamScanConvexHull2D.java b/src/de/lmu/ifi/dbs/elki/math/geometry/GrahamScanConvexHull2D.java index 6d71bbf1..def48c52 100644 --- a/src/de/lmu/ifi/dbs/elki/math/geometry/GrahamScanConvexHull2D.java +++ b/src/de/lmu/ifi/dbs/elki/math/geometry/GrahamScanConvexHull2D.java @@ -40,6 +40,8 @@ import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; * classic Grahams scan. Also computes a bounding box.
*
* @author Erich Schubert
+ *
+ * @apiviz.has Polygon
*/
@Reference(authors = "Paul Graham", title = "An Efficient Algorithm for Determining the Convex Hull of a Finite Planar Set", booktitle = "Information Processing Letters 1")
public class GrahamScanConvexHull2D {
diff --git a/src/de/lmu/ifi/dbs/elki/math/geometry/SweepHullDelaunay2D.java b/src/de/lmu/ifi/dbs/elki/math/geometry/SweepHullDelaunay2D.java index 2533a2b0..4f78cb76 100644 --- a/src/de/lmu/ifi/dbs/elki/math/geometry/SweepHullDelaunay2D.java +++ b/src/de/lmu/ifi/dbs/elki/math/geometry/SweepHullDelaunay2D.java @@ -46,6 +46,8 @@ import de.lmu.ifi.dbs.elki.utilities.pairs.IntIntPair; * Note: This implementation does not check or handle duplicate points! * * @author Erich Schubert + * + * @apiviz.has Polygon */ @Reference(authors = "David Sinclair", title = "S-hull: a fast sweep-hull routine for Delaunay triangulation", booktitle = "Online: http://s-hull.org/") public class SweepHullDelaunay2D { @@ -680,6 +682,8 @@ public class SweepHullDelaunay2D { * Class representing a triangle, by referencing points in a list. * * @author Erich Schubert + * + * @apiviz.exclude */ public static class Triangle { /** diff --git a/src/de/lmu/ifi/dbs/elki/math/geometry/XYCurve.java b/src/de/lmu/ifi/dbs/elki/math/geometry/XYCurve.java new file mode 100644 index 00000000..5d39de16 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/math/geometry/XYCurve.java @@ -0,0 +1,418 @@ +package de.lmu.ifi.dbs.elki.math.geometry; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import gnu.trove.list.array.TDoubleArrayList; +import de.lmu.ifi.dbs.elki.result.Result; +import de.lmu.ifi.dbs.elki.result.textwriter.TextWriteable; +import de.lmu.ifi.dbs.elki.result.textwriter.TextWriterStream; + +/** + * An XYCurve is an ordered collection of 2d points, meant for chart generation. + * Of key interest is the method {@link #addAndSimplify} which tries to simplify + * the curve while adding points. + * + * @author Erich Schubert + */ +public class XYCurve implements Result, TextWriteable { + /** + * Simplification threshold + */ + protected static final double THRESHOLD = 1E-13; + + /** + * X and Y positions + */ + protected TDoubleArrayList data; + + /** + * Label of X axis + */ + protected String labelx; + + /** + * Label of Y axis + */ + protected String labely; + + /** + * Minimum and maximum for X axis + */ + protected double minx = Double.POSITIVE_INFINITY, + maxx = Double.NEGATIVE_INFINITY; + + /** + * Minimum and maximum for Y axis + */ + protected double miny = Double.POSITIVE_INFINITY, + maxy = Double.NEGATIVE_INFINITY; + + /** + * Constructor with labels + * + * @param labelx Label for X axis + * @param labely Label for Y axis + */ + public XYCurve(String labelx, String labely) { + super(); + this.data = new TDoubleArrayList(); + this.labelx = labelx; + this.labely = labely; + } + + /** + * Constructor with size estimate and labels. + * + * @param labelx Label for X axis + * @param labely Label for Y axis + * @param size Estimated size (initial allocation size) + */ + public XYCurve(String labelx, String labely, int size) { + super(); + this.data = new TDoubleArrayList(size * 2); + this.labelx = labelx; + this.labely = labely; + } + + /** + * Constructor. + */ + public XYCurve() { + this("X", "Y"); + } + + /** + * Constructor with size estimate + * + * @param size Estimated size (initial allocation size) + */ + public XYCurve(int size) { + this("X", "Y", size); + } + + /** + * Constructor, cloning an existing curve. + * + * @param curve Curve to clone. + */ + public XYCurve(XYCurve curve) { + super(); + this.data = new TDoubleArrayList(curve.data); + this.labelx = curve.labelx; + this.labely = curve.labely; + this.minx = curve.minx; + this.maxx = curve.maxx; + this.miny = curve.miny; + this.maxy = curve.maxy; + } + + /** + * Add a coordinate pair, but don't simplify + * + * @param x X coordinate + * @param y Y coordinate + */ + public void add(double x, double y) { + data.add(x); + data.add(y); + minx = Math.min(minx, x); + maxx = Math.max(maxx, x); + miny = Math.min(miny, y); + maxy = Math.max(maxy, y); + } + + /** + * Add a coordinate pair, performing curve simplification if possible. + * + * @param x X coordinate + * @param y Y coordinate + */ + public void addAndSimplify(double x, double y) { + // simplify curve when possible: + final int len = data.size(); + if(len >= 4) { + // Look at the previous 2 points + final double l1x = data.get(len - 4); + final double l1y = data.get(len - 3); + final double l2x = data.get(len - 2); + final double l2y = data.get(len - 1); + // Differences: + final double ldx = l2x - l1x; + final double ldy = l2y - l1y; + final double cdx = x - l2x; + final double cdy = y - l2y; + // X simplification + if((ldx == 0) && (cdx == 0)) { + data.remove(len - 2, 2); + } + // horizontal simplification + else if((ldy == 0) && (cdy == 0)) { + data.remove(len - 2, 2); + } + // diagonal simplification + else if(ldy > 0 && cdy > 0) { + if(Math.abs((ldx / ldy) - (cdx / cdy)) < THRESHOLD) { + data.remove(len - 2, 2); + } + } + } + add(x, y); + } + + /** + * Get label of x axis + * + * @return label of x axis + */ + public String getLabelx() { + return labelx; + } + + /** + * Get label of y axis + * + * @return label of y axis + */ + public String getLabely() { + return labely; + } + + /** + * Minimum on x axis. + * + * @return Minimum on X + */ + public double getMinx() { + return minx; + } + + /** + * Maximum on x axis. + * + * @return Maximum on X + */ + public double getMaxx() { + return maxx; + } + + /** + * Minimum on y axis. + * + * @return Minimum on Y + */ + public double getMiny() { + return miny; + } + + /** + * Maximum on y axis. + * + * @return Maximum on Y + */ + public double getMaxy() { + return maxy; + } + + /** + * Curve X value at given position + * + * @param off Offset + * @return X value + */ + public double getX(int off) { + return data.get(off << 1); + } + + /** + * Curve Y value at given position + * + * @param off Offset + * @return Y value + */ + public double getY(int off) { + return data.get((off << 1) + 1); + } + + /** + * Size of curve. + * + * @return curve length + */ + public int size() { + return data.size() >> 1; + } + + /** + * Get an iterator for the curve. + * + * Note: this is <em>not</em> a Java style iterator, since the only way to get + * positions is using "next" in Java style. Here, we can have two getters for + * current values! + * + * Instead, use this style of iteration: <blockquote> + * + * <pre> + * {@code + * for (XYCurve.Itr it = curve.iterator(); it.valid(); it.advance()) { + * doSomethingWith(it.getX(), it.getY()); + * } + * } + * </pre> + * + * </blockquote> + * + * @return Iterator + */ + public Itr iterator() { + return new Itr(); + } + + @Override + public void writeToText(TextWriterStream out, String label) { + out.commentPrint(labelx); + out.commentPrint(" "); + out.commentPrint(labely); + out.flush(); + for(int pos = 0; pos < data.size(); pos+=2) { + out.inlinePrint(data.get(pos)); + out.inlinePrint(data.get(pos + 1)); + out.flush(); + } + } + + @Override + public String toString() { + StringBuffer buf = new StringBuffer(); + buf.append("XYCurve["); + buf.append(labelx).append(",").append(labely).append(":"); + for(int pos = 0; pos < data.size(); pos += 2) { + buf.append(" ").append(data.get(pos)).append(",").append(data.get(pos + 1)); + } + buf.append("]"); + return buf.toString(); + } + + @Override + public String getLongName() { + return labelx + "-" + labely + "-Curve"; + } + + @Override + public String getShortName() { + return (labelx + "-" + labely + "-curve").toLowerCase(); + } + + /** + * Compute the area under curve for a curve + * <em>monotonously increasing in X</em>. You might need to relate this to the + * total area of the chart. + * + * @param curve Curve + * @return Area under curve. + */ + public static double areaUnderCurve(XYCurve curve) { + TDoubleArrayList data = curve.data; + double prevx = data.get(0), prevy = data.get(1); + if(prevx > curve.minx) { + throw new UnsupportedOperationException("Curves must be monotone on X for areaUnderCurve to be valid."); + } + double area = 0.0; + for(int pos = 2; pos < data.size(); pos += 2) { + final double curx = data.get(pos), cury = data.get(pos + 1); + if(prevx > curx) { + throw new UnsupportedOperationException("Curves must be monotone on X for areaUnderCurve to be valid."); + } + area += (curx - prevx) * (prevy + cury) * .5; // .5 = mean Y + prevx = curx; + prevy = cury; + } + if(prevx < curve.maxx) { + throw new UnsupportedOperationException("Curves must be monotone on X for areaUnderCurve to be valid."); + } + return area; + } + + /** + * Iterator for the curve. 2D, does not follow Java collections style. The + * reason is that we want to have {@code #getX()} and {@code #getY()} + * operations, which does not work consistently with Java's + * <code>next()</code> style of iterations. + * + * Instead, use this style of iteration: <blockquote> + * + * <pre> + * {@code + * for (XYCurve.Itr it = curve.iterator(); it.valid(); it.advance()) { + * doSomethingWith(it.getX(), it.getY()); + * } + * } + * </pre> + * + * </blockquote> + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public class Itr { + /** + * Iterator position + */ + protected int pos = 0; + + /** + * Get x value of current element. + * + * @return X value of current element + */ + public double getX() { + return data.get(pos); + } + + /** + * Get y value of current element. + * + * @return Y value of current element + */ + public double getY() { + return data.get(pos + 1); + } + + /** + * Advance the iterator to the next position. + */ + public void advance() { + pos += 2; + } + + /** + * Test if the iterator can advance. + * + * @return True when the iterator can be advanced. + */ + public boolean valid() { + return pos < data.size(); + } + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/math/histograms/AggregatingHistogram.java b/src/de/lmu/ifi/dbs/elki/math/histograms/AggregatingHistogram.java index baa9eb54..87b93910 100644 --- a/src/de/lmu/ifi/dbs/elki/math/histograms/AggregatingHistogram.java +++ b/src/de/lmu/ifi/dbs/elki/math/histograms/AggregatingHistogram.java @@ -34,7 +34,7 @@ import de.lmu.ifi.dbs.elki.utilities.pairs.Pair; * * @author Erich Schubert * - * @apiviz.composedOf de.lmu.ifi.dbs.elki.math.AggregatingHistogram.Adapter + * @apiviz.composedOf de.lmu.ifi.dbs.elki.math.histograms.AggregatingHistogram.Adapter * * @param <T> Type of data in histogram * @param <D> Type of input data diff --git a/src/de/lmu/ifi/dbs/elki/math/histograms/FlexiHistogram.java b/src/de/lmu/ifi/dbs/elki/math/histograms/FlexiHistogram.java index cee151ed..07d20dd2 100644 --- a/src/de/lmu/ifi/dbs/elki/math/histograms/FlexiHistogram.java +++ b/src/de/lmu/ifi/dbs/elki/math/histograms/FlexiHistogram.java @@ -38,7 +38,7 @@ import de.lmu.ifi.dbs.elki.utilities.pairs.Pair; * * @author Erich Schubert * - * @apiviz.composedOf de.lmu.ifi.dbs.elki.math.FlexiHistogram.Adapter + * @apiviz.composedOf de.lmu.ifi.dbs.elki.math.histograms.FlexiHistogram.Adapter * * @param <T> Type of data in histogram * @param <D> Type of input data diff --git a/src/de/lmu/ifi/dbs/elki/math/histograms/ReplacingHistogram.java b/src/de/lmu/ifi/dbs/elki/math/histograms/ReplacingHistogram.java index 304a434b..515fa016 100644 --- a/src/de/lmu/ifi/dbs/elki/math/histograms/ReplacingHistogram.java +++ b/src/de/lmu/ifi/dbs/elki/math/histograms/ReplacingHistogram.java @@ -37,7 +37,7 @@ import de.lmu.ifi.dbs.elki.utilities.pairs.IntIntPair; * * @author Erich Schubert * - * @apiviz.composedOf de.lmu.ifi.dbs.elki.math.ReplacingHistogram.Adapter + * @apiviz.composedOf de.lmu.ifi.dbs.elki.math.histograms.ReplacingHistogram.Adapter * * @param <T> Histogram data type. */ diff --git a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/Centroid.java b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/Centroid.java index cf3bb25b..a9939a56 100644 --- a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/Centroid.java +++ b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/Centroid.java @@ -24,7 +24,8 @@ package de.lmu.ifi.dbs.elki.math.linearalgebra; */ import de.lmu.ifi.dbs.elki.data.NumberVector; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; @@ -168,8 +169,8 @@ public class Centroid extends Vector { */ public static Centroid make(Relation<? extends NumberVector<?, ?>> relation) { Centroid c = new Centroid(DatabaseUtil.dimensionality(relation)); - for(DBID id : relation.iterDBIDs()) { - c.put(relation.get(id)); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + c.put(relation.get(iditer)); } return c; } @@ -180,10 +181,10 @@ public class Centroid extends Vector { * @param relation Relation to use * @param ids IDs to use */ - public static Centroid make(Relation<? extends NumberVector<?, ?>> relation, Iterable<DBID> ids) { + public static Centroid make(Relation<? extends NumberVector<?, ?>> relation, DBIDs ids) { Centroid c = new Centroid(DatabaseUtil.dimensionality(relation)); - for(DBID id : ids) { - c.put(relation.get(id)); + for(DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + c.put(relation.get(iter)); } return c; } diff --git a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/CovarianceMatrix.java b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/CovarianceMatrix.java index 5d9d27a1..a416b917 100644 --- a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/CovarianceMatrix.java +++ b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/CovarianceMatrix.java @@ -24,7 +24,8 @@ package de.lmu.ifi.dbs.elki.math.linearalgebra; */ import de.lmu.ifi.dbs.elki.data.NumberVector; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; @@ -344,8 +345,8 @@ public class CovarianceMatrix { */ public static CovarianceMatrix make(Relation<? extends NumberVector<?, ?>> relation) { CovarianceMatrix c = new CovarianceMatrix(DatabaseUtil.dimensionality(relation)); - for(DBID id : relation.iterDBIDs()) { - c.put(relation.get(id)); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + c.put(relation.get(iditer)); } return c; } @@ -356,10 +357,10 @@ public class CovarianceMatrix { * @param relation Relation to use. * @param ids IDs to add */ - public static CovarianceMatrix make(Relation<? extends NumberVector<?, ?>> relation, Iterable<DBID> ids) { + public static CovarianceMatrix make(Relation<? extends NumberVector<?, ?>> relation, DBIDs ids) { CovarianceMatrix c = new CovarianceMatrix(DatabaseUtil.dimensionality(relation)); - for(DBID id : ids) { - c.put(relation.get(id)); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + c.put(relation.get(iter)); } return c; } diff --git a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/LinearEquationSystem.java b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/LinearEquationSystem.java index b6a53aa2..80cbe1e6 100644 --- a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/LinearEquationSystem.java +++ b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/LinearEquationSystem.java @@ -733,7 +733,7 @@ public class LinearEquationSystem { if(value < 10) { return 1; } - return (int) (Math.log(value) / Math.log(10) + 1); + return (int) Math.log10(value) + 1; } /** diff --git a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/Matrix.java b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/Matrix.java index f64b1129..eec21404 100644 --- a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/Matrix.java +++ b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/Matrix.java @@ -45,7 +45,6 @@ import de.lmu.ifi.dbs.elki.utilities.FormatUtil; * @author Elke Achtert * @author Erich Schubert * - * @apiviz.uses MatrixLike oneway - - reads * @apiviz.uses Vector * @apiviz.landmark */ diff --git a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/ProjectedCentroid.java b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/ProjectedCentroid.java index 02b5b424..435ac3d7 100644 --- a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/ProjectedCentroid.java +++ b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/ProjectedCentroid.java @@ -26,7 +26,8 @@ package de.lmu.ifi.dbs.elki.math.linearalgebra; import java.util.BitSet; import de.lmu.ifi.dbs.elki.data.NumberVector; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; @@ -134,8 +135,8 @@ public class ProjectedCentroid extends Centroid { public static ProjectedCentroid make(BitSet dims, Relation<? extends NumberVector<?, ?>> relation) { ProjectedCentroid c = new ProjectedCentroid(dims, DatabaseUtil.dimensionality(relation)); assert (dims.size() <= DatabaseUtil.dimensionality(relation)); - for(DBID id : relation.iterDBIDs()) { - c.put(relation.get(id)); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + c.put(relation.get(iditer)); } return c; } @@ -147,11 +148,11 @@ public class ProjectedCentroid extends Centroid { * @param relation Relation to process * @param ids IDs to process */ - public static ProjectedCentroid make(BitSet dims, Relation<? extends NumberVector<?, ?>> relation, Iterable<DBID> ids) { + public static ProjectedCentroid make(BitSet dims, Relation<? extends NumberVector<?, ?>> relation, DBIDs ids) { ProjectedCentroid c = new ProjectedCentroid(dims, DatabaseUtil.dimensionality(relation)); assert (dims.size() <= DatabaseUtil.dimensionality(relation)); - for(DBID id : ids) { - c.put(relation.get(id)); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + c.put(relation.get(iter)); } return c; } diff --git a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/DropEigenPairFilter.java b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/DropEigenPairFilter.java new file mode 100644 index 00000000..1657a096 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/DropEigenPairFilter.java @@ -0,0 +1,146 @@ +package de.lmu.ifi.dbs.elki.math.linearalgebra.pca; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import java.util.ArrayList; +import java.util.List; + +import de.lmu.ifi.dbs.elki.math.linearalgebra.EigenPair; +import de.lmu.ifi.dbs.elki.math.linearalgebra.SortedEigenPairs; +import de.lmu.ifi.dbs.elki.utilities.documentation.Title; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.GreaterEqualConstraint; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.DoubleParameter; + +/** + * The "drop" filter looks for the largest drop in normalized relative + * eigenvalues. + * + * Let s_1 .. s_n be the eigenvalues. + * + * Let a_k := 1/(n-k) sum_{i=k..n} s_i + * + * Then r_k := s_k / a_k is the relative eigenvalue. + * + * The drop filter searches for argmax_k r_k / r_{k+1} + * + * @author Erich Schubert + */ +@Title("Drop EigenPair Filter") +public class DropEigenPairFilter implements EigenPairFilter { + /** + * The default value for walpha. Not used by default, we're going for maximum + * contrast only. + */ + public static final double DEFAULT_WALPHA = 0.0; + + /** + * The noise tolerance level for weak eigenvectors + */ + private double walpha = 0.0; + + /** + * Constructor. + * + * @param walpha + */ + public DropEigenPairFilter(double walpha) { + super(); + this.walpha = walpha; + } + + @Override + public FilteredEigenPairs filter(SortedEigenPairs eigenPairs) { + // init strong and weak eigenpairs + List<EigenPair> strongEigenPairs = new ArrayList<EigenPair>(); + List<EigenPair> weakEigenPairs = new ArrayList<EigenPair>(); + + // default value is "all strong". + int contrastMaximum = eigenPairs.size() - 1; + double maxContrast = 0.0; + + double[] ev = eigenPairs.eigenValues(); + // calc the eigenvalue sum. + double eigenValueSum = 0.0; + for(int i = 0; i < ev.length; i++) { + eigenValueSum += ev[i]; + } + // Minimum value + final double weakEigenvalue = walpha * eigenValueSum / ev.length; + // Now find the maximum contrast, scanning backwards. + double prev_sum = ev[ev.length - 1]; + double prev_rel = 1.0; + for(int i = 2; i <= ev.length; i++) { + double curr_sum = prev_sum + ev[ev.length - i]; + double curr_rel = ev[ev.length - i] / curr_sum * i; + // not too weak? + if(ev[ev.length - i] >= weakEigenvalue) { + double contrast = curr_rel - prev_rel; + if(contrast > maxContrast) { + maxContrast = contrast; + contrastMaximum = ev.length - i; + } + } + prev_sum = curr_sum; + prev_rel = curr_rel; + } + + for(int i = 0; i <= contrastMaximum; i++) { + EigenPair eigenPair = eigenPairs.getEigenPair(i); + strongEigenPairs.add(eigenPair); + } + for(int i = contrastMaximum + 1; i < eigenPairs.size(); i++) { + EigenPair eigenPair = eigenPairs.getEigenPair(i); + weakEigenPairs.add(eigenPair); + } + + return new FilteredEigenPairs(weakEigenPairs, strongEigenPairs); + } + + /** + * Parameterization class. + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer extends AbstractParameterizer { + private double walpha; + + @Override + protected void makeOptions(Parameterization config) { + super.makeOptions(config); + DoubleParameter walphaP = new DoubleParameter(WeakEigenPairFilter.EIGENPAIR_FILTER_WALPHA, new GreaterEqualConstraint(0.0), DEFAULT_WALPHA); + if(config.grab(walphaP)) { + walpha = walphaP.getValue(); + } + } + + @Override + protected DropEigenPairFilter makeInstance() { + return new DropEigenPairFilter(walpha); + } + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/FilteredEigenPairs.java b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/FilteredEigenPairs.java index a2c83249..94c00a7e 100644 --- a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/FilteredEigenPairs.java +++ b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/FilteredEigenPairs.java @@ -31,9 +31,9 @@ import java.util.List; * Encapsulates weak and strong eigenpairs that have been filtered out * by an eigenpair filter. * - * @author Elke Achtert + * @author Elke Achtert * - * @apiviz.has de.lmu.ifi.dbs.elki.math.linearalgebra.EigenPair + * @apiviz.has EigenPair */ public class FilteredEigenPairs { /** diff --git a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/PCAFilteredAutotuningRunner.java b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/PCAFilteredAutotuningRunner.java index 8b53dc43..157dfedc 100644 --- a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/PCAFilteredAutotuningRunner.java +++ b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/PCAFilteredAutotuningRunner.java @@ -32,6 +32,7 @@ import java.util.List; import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.database.query.DoubleDistanceResultPair; @@ -76,7 +77,8 @@ public class PCAFilteredAutotuningRunner<V extends NumberVector<? extends V, ?>> // be L2-spherical to be unbiased. V center = DatabaseUtil.centroid(database, ids); List<DoubleDistanceResultPair> dres = new ArrayList<DoubleDistanceResultPair>(ids.size()); - for(DBID id : ids) { + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); final double dist = EuclideanDistanceFunction.STATIC.doubleDistance(center, database.get(id)); dres.add(new DoubleDistanceResultPair(dist, id)); } diff --git a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/PCAFilteredResult.java b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/PCAFilteredResult.java index 4aa626a9..3e79e7b6 100644 --- a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/PCAFilteredResult.java +++ b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/PCAFilteredResult.java @@ -30,7 +30,6 @@ import de.lmu.ifi.dbs.elki.math.linearalgebra.EigenPair; import de.lmu.ifi.dbs.elki.math.linearalgebra.Matrix; import de.lmu.ifi.dbs.elki.math.linearalgebra.ProjectionResult; import de.lmu.ifi.dbs.elki.math.linearalgebra.SortedEigenPairs; -import de.lmu.ifi.dbs.elki.utilities.Util; /** * Result class for a filtered PCA. This differs from regular PCA by having the @@ -89,10 +88,9 @@ public class PCAFilteredResult extends PCAResult implements ProjectionResult { private Matrix m_czech; /** - * The diagonal matrix of adapted strong eigenvalues: eigenvectors * - * e_czech. + * The diagonal matrix of adapted strong eigenvalues: eigenvectors * e_czech. */ - private Matrix adapatedStrongEigenvectors; + private Matrix adapatedStrongEigenvectors = null; /** * Construct a result object for the filtered PCA result. @@ -152,51 +150,49 @@ public class PCAFilteredResult extends PCAResult implements ProjectionResult { } } - // TODO: unnecessary copy. Matrix V = getEigenvectors(); - adapatedStrongEigenvectors = V.times(e_czech).times(Matrix.identity(dim, localdim)); m_hat = V.times(e_hat).timesTranspose(V); m_czech = V.times(e_czech).timesTranspose(V); } /** - * Returns a copy of the matrix of strong eigenvectors after passing the eigen - * pair filter. + * Returns the matrix of strong eigenvectors after passing the eigen pair + * filter. * * @return the matrix of eigenvectors */ public final Matrix getStrongEigenvectors() { - return strongEigenvectors.copy(); + return strongEigenvectors; } /** - * Returns a copy of the strong eigenvalues of the object after passing the - * eigen pair filter. + * Returns the strong eigenvalues of the object after passing the eigen pair + * filter. * * @return the eigenvalues */ public final double[] getStrongEigenvalues() { - return Util.copy(strongEigenvalues); + return strongEigenvalues; } /** - * Returns a copy of the matrix of weak eigenvectors after passing the eigen - * pair filter. + * Returns the matrix of weak eigenvectors after passing the eigen pair + * filter. * * @return the matrix of eigenvectors */ public final Matrix getWeakEigenvectors() { - return weakEigenvectors.copy(); + return weakEigenvectors; } /** - * Returns a copy of the weak eigenvalues of the object after passing the - * eigen pair filter. + * Returns the weak eigenvalues of the object after passing the eigen pair + * filter. * * @return the eigenvalues */ public final double[] getWeakEigenvalues() { - return Util.copy(weakEigenvalues); + return weakEigenvalues; } /** @@ -219,50 +215,54 @@ public class PCAFilteredResult extends PCAResult implements ProjectionResult { } /** - * Returns a copy of the selection matrix of the weak eigenvectors (E_hat) of + * Returns the selection matrix of the weak eigenvectors (E_hat) of * the object to which this PCA belongs to. * * @return the selection matrix of the weak eigenvectors E_hat */ public Matrix selectionMatrixOfWeakEigenvectors() { - return e_hat.copy(); + return e_hat; } /** - * Returns a copy of the selection matrix of the strong eigenvectors (E_czech) + * Returns the selection matrix of the strong eigenvectors (E_czech) * of this LocalPCA. * * @return the selection matrix of the weak eigenvectors E_czech */ public Matrix selectionMatrixOfStrongEigenvectors() { - return e_czech.copy(); + return e_czech; } /** - * Returns a copy of the similarity matrix (M_hat) of this LocalPCA. + * Returns the similarity matrix (M_hat) of this LocalPCA. * * @return the similarity matrix M_hat */ @Override public Matrix similarityMatrix() { - return m_hat.copy(); + return m_hat; } /** - * Returns a copy of the dissimilarity matrix (M_czech) of this LocalPCA. + * Returns the dissimilarity matrix (M_czech) of this LocalPCA. * * @return the dissimilarity matrix M_hat */ public Matrix dissimilarityMatrix() { - return m_czech.copy(); + return m_czech; } - + /** - * Returns a copy of the adapted strong eigenvectors. - * + * Returns the adapted strong eigenvectors. + * * @return the adapted strong eigenvectors */ public Matrix adapatedStrongEigenvectors() { - return adapatedStrongEigenvectors.copy(); + if (adapatedStrongEigenvectors == null) { + final Matrix ev = getEigenvectors(); + adapatedStrongEigenvectors = ev.times(e_czech).times(Matrix.identity(ev.getRowDimensionality(), strongEigenvalues.length)); + } + return adapatedStrongEigenvectors; } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/PCAResult.java b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/PCAResult.java index 6969a3a3..87518b1d 100644 --- a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/PCAResult.java +++ b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/PCAResult.java @@ -25,7 +25,6 @@ package de.lmu.ifi.dbs.elki.math.linearalgebra.pca; import de.lmu.ifi.dbs.elki.math.linearalgebra.Matrix; import de.lmu.ifi.dbs.elki.math.linearalgebra.SortedEigenPairs; -import de.lmu.ifi.dbs.elki.utilities.Util; /** * Result class for Principal Component Analysis with some convenience methods @@ -33,14 +32,14 @@ import de.lmu.ifi.dbs.elki.utilities.Util; * @author Erich Schubert * * @apiviz.landmark - * @apiviz.has de.lmu.ifi.dbs.elki.math.linearalgebra.SortedEigenPairs + * @apiviz.has SortedEigenPairs */ public class PCAResult { /** * The eigenpairs in decreasing order. */ private SortedEigenPairs eigenPairs; - + /** * The eigenvalues in decreasing order. */ @@ -73,44 +72,46 @@ public class PCAResult { */ public PCAResult(SortedEigenPairs eigenPairs) { super(); - // TODO: we might want to postpone the instantiation of eigenvalue and eigenvectors. + // TODO: we might want to postpone the instantiation of eigenvalue and + // eigenvectors. this.eigenPairs = eigenPairs; this.eigenvalues = eigenPairs.eigenValues(); this.eigenvectors = eigenPairs.eigenVectors(); } /** - * Returns a copy of the matrix of eigenvectors of the object to which this - * PCA belongs to. + * Returns the matrix of eigenvectors of the object to which this PCA belongs + * to. * * @return the matrix of eigenvectors */ public final Matrix getEigenvectors() { - return eigenvectors.copy(); + return eigenvectors; } /** - * Returns a copy of the eigenvalues of the object to which this PCA belongs - * to in decreasing order. + * Returns the eigenvalues of the object to which this PCA belongs to in + * decreasing order. * * @return the eigenvalues */ public final double[] getEigenvalues() { - return Util.copy(eigenvalues); + return eigenvalues; } /** - * Returns a copy of the eigenpairs of the object to which this PCA belongs to + * Returns the eigenpairs of the object to which this PCA belongs to * in decreasing order. * * @return the eigenpairs */ public final SortedEigenPairs getEigenPairs() { - return eigenPairs.copy(); + return eigenPairs; } /** * Returns the number of eigenvectors stored + * * @return length */ public final int length() { diff --git a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/SignificantEigenPairFilter.java b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/SignificantEigenPairFilter.java index 94636553..ab04cbb5 100644 --- a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/SignificantEigenPairFilter.java +++ b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/SignificantEigenPairFilter.java @@ -76,9 +76,6 @@ public class SignificantEigenPairFilter implements EigenPairFilter { this.walpha = walpha; } - /** - * Filter eigenpairs - */ @Override public FilteredEigenPairs filter(SortedEigenPairs eigenPairs) { // init strong and weak eigenpairs diff --git a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/WeightedCovarianceMatrixBuilder.java b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/WeightedCovarianceMatrixBuilder.java index db5e8702..fb7f60c3 100644 --- a/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/WeightedCovarianceMatrixBuilder.java +++ b/src/de/lmu/ifi/dbs/elki/math/linearalgebra/pca/WeightedCovarianceMatrixBuilder.java @@ -27,7 +27,7 @@ import java.util.Collection; import java.util.Iterator; import de.lmu.ifi.dbs.elki.data.NumberVector; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.database.query.DoubleDistanceResultPair; @@ -123,8 +123,8 @@ public class WeightedCovarianceMatrixBuilder<V extends NumberVector<? extends V, double maxdist = 0.0; double stddev = 0.0; { - for(Iterator<DBID> it = ids.iterator(); it.hasNext();) { - V obj = database.get(it.next()); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + V obj = database.get(iter); double distance = weightDistance.distance(centroid, obj).doubleValue(); stddev += distance * distance; if(distance > maxdist) { @@ -138,8 +138,8 @@ public class WeightedCovarianceMatrixBuilder<V extends NumberVector<? extends V, stddev = Math.sqrt(stddev / ids.size()); } - for(Iterator<DBID> it = ids.iterator(); it.hasNext();) { - V obj = database.get(it.next()); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + V obj = database.get(iter); double distance = weightDistance.distance(centroid, obj).doubleValue(); double weight = weightfunction.getWeight(distance, maxdist, stddev); cmat.put(obj, weight); @@ -204,7 +204,7 @@ public class WeightedCovarianceMatrixBuilder<V extends NumberVector<? extends V, dist = res.getDistance().doubleValue(); } - V obj = database.get(res.getDBID()); + V obj = database.get(res); double weight = weightfunction.getWeight(dist, maxdist, stddev); cmat.put(obj, weight); } diff --git a/src/de/lmu/ifi/dbs/elki/math/scales/Scales.java b/src/de/lmu/ifi/dbs/elki/math/scales/Scales.java index ffaaffc7..edad211d 100644 --- a/src/de/lmu/ifi/dbs/elki/math/scales/Scales.java +++ b/src/de/lmu/ifi/dbs/elki/math/scales/Scales.java @@ -24,7 +24,7 @@ package de.lmu.ifi.dbs.elki.math.scales; */ import de.lmu.ifi.dbs.elki.data.NumberVector; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.math.DoubleMinMax; import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; @@ -57,8 +57,8 @@ public class Scales { LinearScale scales[] = new LinearScale[dim]; // analyze data - for(DBID objId : db.iterDBIDs()) { - O v = db.get(objId); + for(DBIDIter iditer = db.iterDBIDs(); iditer.valid(); iditer.advance()) { + O v = db.get(iditer); for(int d = 0; d < dim; d++) { minmax[d].put(v.doubleValue(d+1)); } diff --git a/src/de/lmu/ifi/dbs/elki/math/spacefillingcurves/HilbertSpatialSorter.java b/src/de/lmu/ifi/dbs/elki/math/spacefillingcurves/HilbertSpatialSorter.java index 82e41337..9b4af341 100644 --- a/src/de/lmu/ifi/dbs/elki/math/spacefillingcurves/HilbertSpatialSorter.java +++ b/src/de/lmu/ifi/dbs/elki/math/spacefillingcurves/HilbertSpatialSorter.java @@ -45,7 +45,7 @@ import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; * * @author Erich Schubert * - * @apiviz.uses HilbertRef + * @apiviz.composedOf HilbertRef */ @Reference(authors = "D. Hilbert", title = "Über die stetige Abbildung einer Linie auf ein Flächenstück", booktitle = "Mathematische Annalen, 38(3)") public class HilbertSpatialSorter extends AbstractSpatialSorter { diff --git a/src/de/lmu/ifi/dbs/elki/math/spacefillingcurves/ZCurve.java b/src/de/lmu/ifi/dbs/elki/math/spacefillingcurves/ZCurve.java deleted file mode 100644 index 38f45aef..00000000 --- a/src/de/lmu/ifi/dbs/elki/math/spacefillingcurves/ZCurve.java +++ /dev/null @@ -1,264 +0,0 @@ -package de.lmu.ifi.dbs.elki.math.spacefillingcurves; - -/* - This file is part of ELKI: - Environment for Developing KDD-Applications Supported by Index-Structures - - Copyright (C) 2012 - Ludwig-Maximilians-Universität München - Lehr- und Forschungseinheit für Datenbanksysteme - ELKI Development Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import de.lmu.ifi.dbs.elki.data.NumberVector; -import de.lmu.ifi.dbs.elki.database.ids.DBID; -import de.lmu.ifi.dbs.elki.database.ids.DBIDs; -import de.lmu.ifi.dbs.elki.database.relation.Relation; -import de.lmu.ifi.dbs.elki.logging.Logging; -import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; -import de.lmu.ifi.dbs.elki.utilities.FormatUtil; - -/** - * Computes the z-values for specified double values. - * - * @author Elke Achtert - */ -public class ZCurve { - /** - * The logger of this class. - */ - private static final Logging logger = Logging.getLogger(ZCurve.class); - - /** - * Fake Constructor - use the static methods instead! - */ - private ZCurve() { - // nothing to do. - } - - /** - * Computes the z-values for the specified double values. - * - * @param valuesList the list of double values - * @return the z-values for the specified double values - */ - public static List<byte[]> zValues(List<double[]> valuesList) { - if(valuesList.isEmpty()) { - return new ArrayList<byte[]>(); - } - - // determine min and max value in each dimension and the scaling factor - int dimensionality = valuesList.get(0).length; - double[] minValues = new double[dimensionality]; - double[] maxValues = new double[dimensionality]; - Arrays.fill(minValues, Double.POSITIVE_INFINITY); - Arrays.fill(maxValues, Double.NEGATIVE_INFINITY); - for(double[] values : valuesList) { - for(int d = 0; d < dimensionality; d++) { - maxValues[d] = Math.max(values[d], maxValues[d]); - minValues[d] = Math.min(values[d], minValues[d]); - } - } - - double[] normalizationFactors = new double[dimensionality]; - for(int d = 0; d < dimensionality; d++) { - // has to be > 0!!! - normalizationFactors[d] = (maxValues[d] - minValues[d]); - } - - if(logger.isDebugging()) { - StringBuffer msg = new StringBuffer(); - msg.append("min ").append(FormatUtil.format(minValues)); - msg.append("\nmax ").append(FormatUtil.format(maxValues)); - msg.append("\nscale ").append(FormatUtil.format(normalizationFactors)); - msg.append("\nLong.MAX_VALUE " + Long.MAX_VALUE); - msg.append("\nLong.MIN_VALUE " + Long.MIN_VALUE); - logger.debugFine(msg.toString()); - } - - // discretize the double value over the whole domain - final List<byte[]> zValues = new ArrayList<byte[]>(); - for(double[] values : valuesList) { - // convert the double values to long values - long[] longValues = new long[values.length]; - for(int d = 0; d < values.length; d++) { - // normalize to 0:1 - final double normval = ((values[d] - minValues[d]) / normalizationFactors[d]); - longValues[d] = (long) (normval * Long.MAX_VALUE); - } - - if(logger.isDebugging()) { - StringBuffer msg = new StringBuffer(); - msg.append("double values ").append(FormatUtil.format(values)); - msg.append("\nlong values ").append(FormatUtil.format(longValues)); - logger.debugFine(msg.toString()); - } - byte[] zValue = zValue(longValues); - zValues.add(zValue); - } - - return zValues; - } - - /** - * Computes the z-value for the specified long values - * - * @param longValues the array of the discretized double values - * @return the z-value for the specified long values - */ - private static byte[] zValue(long[] longValues) { - final int numdim = longValues.length; - final int numbits = numdim * 64; - byte[] zValues = new byte[numbits / 8]; - - // convert longValues into zValues - for(int bitnum = 0; bitnum < numbits; bitnum ++) { - // split into in-byte and in-array indexes - final int lowpos = bitnum & 7; - final int higpos = bitnum >> 3; - - final int dim = bitnum % numdim; - final int shift = 63 - (bitnum / numdim); - final byte val = (byte) ((longValues[dim] >> shift) & 1); - - zValues[higpos] |= val << (7 - lowpos); - } - /*for(int shift = 0; shift < 64; shift++) { - for(int dim = 0; dim < longValues.length; dim++) { - // destination bit position - final int bitpos = shift * longValues.length + dim; - // split into in-byte and in-array indexes - final int lowpos = bitpos & 7; - final int higpos = bitpos >> 3; - zValues[higpos] |= ((longValues[dim] >> (shift)) & 1) << lowpos; - } - }*/ - - if(logger.isDebugging()) { - // convert zValues to longValues - long[] loutput = new long[longValues.length]; - for(int shift = 0; shift < 64; shift++) { - for(int dim = 0; dim < longValues.length; dim++) { - // destination bit position - final int bitpos = shift * longValues.length + dim; - // split into in-byte and in-array indexes - final int lowpos = bitpos & 7; - final int higpos = bitpos >> 3; - loutput[dim] |= ((long) (((zValues[higpos] >> (lowpos)) & 1))) << (shift); - } - } - StringBuffer msg = new StringBuffer(); - msg.append("reconstructed values: ").append(FormatUtil.format(loutput)); - logger.debugFine(msg.toString()); - } - - return zValues; - } - - /** - * Class to transform a relation to its Z coordinates. - * - * @author Erich Schubert - */ - public static class Transformer { - /** - * Maximum values in each dimension - */ - private final double[] maxValues; - - /** - * Minimum values in each dimension - */ - private final double[] minValues; - - /** - * Dimensionality - */ - private final int dimensionality; - - /** - * Constructor. - * - * @param relation Relation to transform - * @param ids IDs subset to process - */ - public Transformer(Relation<? extends NumberVector<?, ?>> relation, DBIDs ids) { - this.dimensionality = DatabaseUtil.dimensionality(relation); - this.minValues = new double[dimensionality]; - this.maxValues = new double[dimensionality]; - - // Compute scaling of vector space - Arrays.fill(minValues, Double.POSITIVE_INFINITY); - Arrays.fill(maxValues, Double.NEGATIVE_INFINITY); - for(DBID id : ids) { - NumberVector<?, ?> vector = relation.get(id); - for(int dim = 0; dim < dimensionality; ++dim) { - double dimValue = vector.doubleValue(dim + 1); - minValues[dim] = Math.min(minValues[dim], dimValue); - maxValues[dim] = Math.max(maxValues[dim], dimValue); - } - } - } - - /** - * Transform a single vector. - * - * @param vector Vector to transform - * @return Z curve value as bigint - */ - public BigInteger asBigInteger(NumberVector<?, ?> vector) { - return new BigInteger(asByteArray(vector)); - } - - /** - * Transform a single vector. - * - * @param vector Vector to transform - * @return Z curve value as byte array - */ - public byte[] asByteArray(NumberVector<?, ?> vector) { - final long[] longValueList = new long[dimensionality]; - - for(int dim = 0; dim < dimensionality; ++dim) { - final double minValue = minValues[dim]; - final double maxValue = maxValues[dim]; - double dimValue = vector.doubleValue(dim + 1); - - dimValue = (dimValue - minValue) / (maxValue - minValue); - longValueList[dim] = (long) (dimValue * (Long.MAX_VALUE)); - } - - final byte[] bytes = new byte[Long.SIZE * dimensionality * (Long.SIZE / Byte.SIZE)]; - int shiftCounter = 0; - for(int i = 0; i < Long.SIZE; ++i) { - for(int dim = 0; dim < dimensionality; ++dim) { - long byteValue = longValueList[dim]; - - int localShift = shiftCounter % Byte.SIZE; - bytes[(bytes.length - 1) - (shiftCounter / Byte.SIZE)] |= ((byteValue >> i) & 0x01) << localShift; - - shiftCounter++; - } - } - return bytes; - } - } -}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/math/spacefillingcurves/ZCurveTransformer.java b/src/de/lmu/ifi/dbs/elki/math/spacefillingcurves/ZCurveTransformer.java new file mode 100644 index 00000000..108721eb --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/math/spacefillingcurves/ZCurveTransformer.java @@ -0,0 +1,124 @@ +package de.lmu.ifi.dbs.elki.math.spacefillingcurves; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import java.math.BigInteger; +import java.util.Arrays; + +import de.lmu.ifi.dbs.elki.data.NumberVector; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDs; +import de.lmu.ifi.dbs.elki.database.relation.Relation; +import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; + +/** + * Class to transform a relation to its Z coordinates. + * + * @author Erich Schubert + */ +public class ZCurveTransformer { + /** + * Maximum values in each dimension + */ + private final double[] maxValues; + + /** + * Minimum values in each dimension + */ + private final double[] minValues; + + /** + * Dimensionality + */ + private final int dimensionality; + + /** + * Constructor. + * + * @param relation Relation to transform + * @param ids IDs subset to process + */ + public ZCurveTransformer(Relation<? extends NumberVector<?, ?>> relation, DBIDs ids) { + this.dimensionality = DatabaseUtil.dimensionality(relation); + this.minValues = new double[dimensionality]; + this.maxValues = new double[dimensionality]; + + // Compute scaling of vector space + Arrays.fill(minValues, Double.POSITIVE_INFINITY); + Arrays.fill(maxValues, Double.NEGATIVE_INFINITY); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + NumberVector<?, ?> vector = relation.get(iter); + for(int dim = 0; dim < dimensionality; ++dim) { + double dimValue = vector.doubleValue(dim + 1); + minValues[dim] = Math.min(minValues[dim], dimValue); + maxValues[dim] = Math.max(maxValues[dim], dimValue); + } + } + } + + /** + * Transform a single vector. + * + * @param vector Vector to transform + * @return Z curve value as bigint + */ + @Deprecated + public BigInteger asBigInteger(NumberVector<?, ?> vector) { + return new BigInteger(asByteArray(vector)); + } + + /** + * Transform a single vector. + * + * @param vector Vector to transform + * @return Z curve value as byte array + */ + public byte[] asByteArray(NumberVector<?, ?> vector) { + final long[] longValueList = new long[dimensionality]; + + for(int dim = 0; dim < dimensionality; ++dim) { + final double minValue = minValues[dim]; + final double maxValue = maxValues[dim]; + double dimValue = vector.doubleValue(dim + 1); + + dimValue = (dimValue - minValue) / (maxValue - minValue); + longValueList[dim] = (long) (dimValue * (Long.MAX_VALUE)); + } + + final byte[] bytes = new byte[Long.SIZE * dimensionality * (Long.SIZE / Byte.SIZE)]; + int shiftCounter = 0; + for(int i = 0; i < Long.SIZE; ++i) { + for(int dim = 0; dim < dimensionality; ++dim) { + long byteValue = longValueList[dim]; + + int localShift = shiftCounter % Byte.SIZE; + bytes[(bytes.length - 1) - (shiftCounter / Byte.SIZE)] |= ((byteValue >> i) & 0x01) << localShift; + + shiftCounter++; + } + } + return bytes; + } + +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/BetaDistribution.java b/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/BetaDistribution.java new file mode 100644 index 00000000..1efe19dd --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/BetaDistribution.java @@ -0,0 +1,499 @@ +package de.lmu.ifi.dbs.elki.math.statistics.distribution;
+
+import java.util.Random;
+
+import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException;
+
+/*
+ This file is part of ELKI:
+ Environment for Developing KDD-Applications Supported by Index-Structures
+
+ Copyright (C) 2012
+ Ludwig-Maximilians-Universität München
+ Lehr- und Forschungseinheit für Datenbanksysteme
+ ELKI Development Team
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * Beta Distribution with implementation of the regularized incomplete beta
+ * function
+ *
+ * @author Jan Brusis
+ * @author Erich Schubert
+ */
+public class BetaDistribution implements DistributionWithRandom {
+ /**
+ * Numerical precision to use
+ */
+ static final double NUM_PRECISION = 1E-15;
+
+ /**
+ * Limit of when to switch to quadrature method
+ */
+ static final double SWITCH = 3000;
+
+ /**
+ * Abscissas for Gauss-Legendre quadrature
+ */
+ static final double[] GAUSSLEGENDRE_Y = { 0.0021695375159141994, 0.011413521097787704, 0.027972308950302116, 0.051727015600492421, 0.082502225484340941, 0.12007019910960293, 0.16415283300752470, 0.21442376986779355, 0.27051082840644336, 0.33199876341447887, 0.39843234186401943, 0.46931971407375483, 0.54413605556657973, 0.62232745288031077, 0.70331500465597174, 0.78649910768313447, 0.87126389619061517, 0.95698180152629142 };
+
+ /**
+ * Weights for Gauss-Legendre quadrature
+ */
+ static final double[] GAUSSLEGENDRE_W = { 0.0055657196642445571, 0.012915947284065419, 0.020181515297735382, 0.027298621498568734, 0.034213810770299537, 0.040875750923643261, 0.047235083490265582, 0.053244713977759692, 0.058860144245324798, 0.064039797355015485, 0.068745323835736408, 0.072941885005653087, 0.076598410645870640, 0.079687828912071670, 0.082187266704339706, 0.084078218979661945, 0.085346685739338721, 0.085983275670394821 };
+
+ /**
+ * Shape parameter of beta distribution
+ */
+ private final double alpha;
+
+ /**
+ * Shape parameter of beta distribution
+ */
+ private final double beta;
+
+ /**
+ * For random number generation
+ */
+ private Random random;
+
+ /**
+ * Log beta(a, b) cache
+ */
+ private double logbab;
+
+ /**
+ * Constructor.
+ *
+ * @param a shape Parameter a
+ * @param b shape Parameter b
+ */
+ public BetaDistribution(double a, double b) {
+ this(a, b, new Random());
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param a shape Parameter a
+ * @param b shape Parameter b
+ * @param random Random generator
+ */
+ public BetaDistribution(double a, double b, Random random) {
+ super();
+ if(a <= 0.0 || b <= 0.0) {
+ throw new IllegalArgumentException("Invalid parameters for Beta distribution.");
+ }
+
+ this.alpha = a;
+ this.beta = b;
+ this.logbab = logBeta(a, b);
+ this.random = random;
+ }
+
+ @Override
+ public double pdf(double val) {
+ if(val < 0. || val > 1.) {
+ return 0.;
+ }
+ if(val == 0.) {
+ if(alpha > 1.) {
+ return 0.;
+ }
+ if(alpha < 1.) {
+ return Double.POSITIVE_INFINITY;
+ }
+ return beta;
+ }
+ if(val == 1.) {
+ if(beta > 1.) {
+ return 0.;
+ }
+ if(beta < 1.) {
+ return Double.POSITIVE_INFINITY;
+ }
+ return alpha;
+ }
+ return Math.exp(-logbab + Math.log(val) * (alpha - 1) + Math.log1p(-val) * (beta - 1));
+ }
+
+ @Override
+ public double cdf(double x) {
+ if(alpha <= 0.0 || beta <= 0.0 || Double.isNaN(alpha) || Double.isNaN(beta) || Double.isNaN(x)) {
+ return Double.NaN;
+ }
+ if(x <= 0.0) {
+ return 0.0;
+ }
+ if(x >= 1.0) {
+ return 1.0;
+ }
+ if(alpha > SWITCH && beta > SWITCH) {
+ return regularizedIncBetaQuadrature(alpha, beta, x);
+ }
+ double bt = Math.exp(-logbab + alpha * Math.log(x) + beta * Math.log1p(-x));
+ if(x < (alpha + 1.0) / (alpha + beta + 2.0)) {
+ return bt * regularizedIncBetaCF(alpha, beta, x) / alpha;
+ }
+ else {
+ return 1.0 - bt * regularizedIncBetaCF(beta, alpha, 1.0 - x) / beta;
+ }
+ }
+
+ @Override
+ public double quantile(double x) {
+ // Valid parameters
+ if(x < 0 || x > 1 || Double.isNaN(x)) {
+ return Double.NaN;
+ }
+ if(x == 0) {
+ return 0.0;
+ }
+ if(x == 1) {
+ return 1.0;
+ }
+ // Simpler to compute inverse?
+ if(x > 0.5) {
+ return 1 - rawQuantile(1 - x, beta, alpha, logbab);
+ }
+ else {
+ return rawQuantile(x, alpha, beta, logbab);
+ }
+ }
+
+ @Override
+ public double nextRandom() {
+ double x = GammaDistribution.nextRandom(alpha, 1, random);
+ double y = GammaDistribution.nextRandom(beta, 1, random);
+ return x / (x + y);
+ }
+
+ @Override
+ public String toString() {
+ return "BetaDistribution(alpha=" + alpha + ", beta=" + beta + ")";
+ }
+
+ /**
+ * Static version of the CDF of the beta distribution
+ *
+ * @param val Value
+ * @param alpha Shape parameter a
+ * @param beta Shape parameter b
+ * @return cumulative density
+ */
+ public static double cdf(double val, double alpha, double beta) {
+ return regularizedIncBeta(val, alpha, beta);
+ }
+
+ /**
+ * Static version of the PDF of the beta distribution
+ *
+ * @param val Value
+ * @param alpha Shape parameter a
+ * @param beta Shape parameter b
+ * @return probability density
+ */
+ public static double pdf(double val, double alpha, double beta) {
+ if(alpha <= 0. || beta <= 0. || Double.isNaN(alpha) || Double.isNaN(beta) || Double.isNaN(val)) {
+ return Double.NaN;
+ }
+ if(val < 0. || val > 1.) {
+ return 0.;
+ }
+ if(val == 0.) {
+ if(alpha > 1.) {
+ return 0.;
+ }
+ if(alpha < 1.) {
+ return Double.POSITIVE_INFINITY;
+ }
+ return beta;
+ }
+ if(val == 1.) {
+ if(beta > 1.) {
+ return 0.;
+ }
+ if(beta < 1.) {
+ return Double.POSITIVE_INFINITY;
+ }
+ return alpha;
+ }
+ return Math.exp(-logBeta(alpha, beta) + Math.log(val) * (alpha - 1) + Math.log1p(-val) * (beta - 1));
+ }
+
+ /**
+ * Compute log beta(a,b)
+ *
+ * @param alpha Shape parameter a
+ * @param beta Shape parameter b
+ * @return Logarithm of result
+ */
+ public static double logBeta(double alpha, double beta) {
+ return GammaDistribution.logGamma(alpha) + GammaDistribution.logGamma(beta) - GammaDistribution.logGamma(alpha + beta);
+ }
+
+ /**
+ * Computes the regularized incomplete beta function I_x(a, b) which is also
+ * the CDF of the beta distribution. Based on the book "Numerical Recipes"
+ *
+ * @param alpha Parameter a
+ * @param beta Parameter b
+ * @param x Parameter x
+ * @return Value of the regularized incomplete beta function
+ */
+ public static double regularizedIncBeta(double x, double alpha, double beta) {
+ if(alpha <= 0.0 || beta <= 0.0 || Double.isNaN(alpha) || Double.isNaN(beta) || Double.isNaN(x)) {
+ return Double.NaN;
+ }
+ if(x <= 0.0) {
+ return 0.0;
+ }
+ if(x >= 1.0) {
+ return 1.0;
+ }
+ if(alpha > SWITCH && beta > SWITCH) {
+ return regularizedIncBetaQuadrature(alpha, beta, x);
+ }
+ double bt = Math.exp(-logBeta(alpha, beta) + alpha * Math.log(x) + beta * Math.log1p(-x));
+ if(x < (alpha + 1.0) / (alpha + beta + 2.0)) {
+ return bt * regularizedIncBetaCF(alpha, beta, x) / alpha;
+ }
+ else {
+ return 1.0 - bt * regularizedIncBetaCF(beta, alpha, 1.0 - x) / beta;
+ }
+ }
+
+ /**
+ * Returns the regularized incomplete beta function I_x(a, b) Includes the
+ * continued fraction way of computing, based on the book "Numerical Recipes".
+ *
+ * @param alpha Parameter a
+ * @param beta Parameter b
+ * @param x Parameter x
+ * @return result
+ */
+ protected static double regularizedIncBetaCF(double alpha, double beta, double x) {
+ final double FPMIN = Double.MIN_VALUE / NUM_PRECISION;
+ double qab = alpha + beta;
+ double qap = alpha + 1.0;
+ double qam = alpha - 1.0;
+ double c = 1.0;
+ double d = 1.0 - qab * x / qap;
+ if(Math.abs(d) < FPMIN) {
+ d = FPMIN;
+ }
+ d = 1.0 / d;
+ double h = d;
+ for(int m = 1; m < 10000; m++) {
+ int m2 = 2 * m;
+ double aa = m * (beta - m) * x / ((qam + m2) * (alpha + m2));
+ d = 1.0 + aa * d;
+ if(Math.abs(d) < FPMIN) {
+ d = FPMIN;
+ }
+ c = 1.0 + aa / c;
+ if(Math.abs(c) < FPMIN) {
+ c = FPMIN;
+ }
+ d = 1.0 / d;
+ h *= d * c;
+ aa = -(alpha + m) * (qab + m) * x / ((alpha + m2) * (qap + m2));
+ d = 1.0 + aa * d;
+ if(Math.abs(d) < FPMIN) {
+ d = FPMIN;
+ }
+ c = 1.0 + aa / c;
+ if(Math.abs(c) < FPMIN) {
+ c = FPMIN;
+ }
+ d = 1.0 / d;
+ double del = d * c;
+ h *= del;
+ if(Math.abs(del - 1.0) <= NUM_PRECISION) {
+ break;
+ }
+ }
+ return h;
+ }
+
+ /**
+ * Returns the regularized incomplete beta function I_x(a, b) by quadrature,
+ * based on the book "Numerical Recipes".
+ *
+ * @param alpha Parameter a
+ * @param beta Parameter b
+ * @param x Parameter x
+ * @return result
+ */
+ protected static double regularizedIncBetaQuadrature(double alpha, double beta, double x) {
+ double a1 = alpha - 1.0;
+ double b1 = beta - 1.0;
+ double mu = alpha / (alpha + beta);
+ double lnmu = Math.log(mu);
+ double lnmuc = Math.log1p(-mu);
+ double t = Math.sqrt(alpha * beta / ((alpha + beta) * (alpha + beta) * (alpha + beta + 1.0)));
+ double xu;
+ if(x > alpha / (alpha + beta)) {
+ if(x >= 1.0) {
+ return 1.0;
+ }
+ xu = Math.min(1.0, Math.max(mu + 10.0 * t, x + 5.0 * t));
+ }
+ else {
+ if(x <= 0.0) {
+ return 0.0;
+ }
+ xu = Math.max(0.0, Math.min(mu - 10.0 * t, x - 5.0 * t));
+ }
+ double sum = 0.0;
+ for(int i = 0; i < GAUSSLEGENDRE_Y.length; i++) {
+ t = x + (xu - x) * GAUSSLEGENDRE_Y[i];
+ sum += GAUSSLEGENDRE_W[i] * Math.exp(a1 * (Math.log(t) - lnmu) + b1 * (Math.log1p(-t) - lnmuc));
+ }
+ double ans = sum * (xu - x) * Math.exp(a1 * lnmu - GammaDistribution.logGamma(alpha) + b1 * lnmuc - GammaDistribution.logGamma(b1) + GammaDistribution.logGamma(alpha + beta));
+ return ans > 0 ? 1.0 - ans : -ans;
+ }
+
+ /**
+ * Compute quantile (inverse cdf) for Beta distributions.
+ *
+ * @param p Probability
+ * @param alpha Shape parameter a
+ * @param beta Shape parameter b
+ * @return Probit for Beta distribution
+ */
+ public static double quantile(double p, double alpha, double beta) {
+ // Valid parameters
+ if(Double.isNaN(alpha) || Double.isNaN(beta) || Double.isNaN(p) || alpha < 0. || beta < 0.) {
+ return Double.NaN;
+ }
+ if(p < 0 || p > 1) {
+ return Double.NaN;
+ }
+ if(p == 0) {
+ return 0.0;
+ }
+ if(p == 1) {
+ return 1.0;
+ }
+ // Simpler to compute inverse?
+ if(p > 0.5) {
+ return 1 - rawQuantile(1 - p, beta, alpha, logBeta(beta, alpha));
+ }
+ else {
+ return rawQuantile(p, alpha, beta, logBeta(alpha, beta));
+ }
+ }
+
+ /**
+ * Raw quantile function
+ *
+ * @param p P, must be 0 < p <= .5
+ * @param alpha Alpha
+ * @param beta Beta
+ * @param logbeta log Beta(alpha, beta)
+ * @return Position
+ */
+ protected static double rawQuantile(double p, double alpha, double beta, final double logbeta) {
+ // Initial estimate for x
+ double x;
+ {
+ // Very fast approximation of y.
+ double tmp = Math.sqrt(-2 * Math.log(p));
+ double y = tmp - (2.30753 + 0.27061 * tmp) / (1. + (0.99229 + 0.04481 * tmp) * tmp);
+
+ if(alpha > 1 && beta > 1) {
+ double r = (y * y - 3.) / 6.;
+ double s = 1. / (alpha + alpha - 1.);
+ double t = 1. / (beta + beta - 1.);
+ double h = 2. / (s + t);
+ double w = y * Math.sqrt(h + r) / h - (t - s) * (r + 5. / 6. - 2. / (3. * h));
+ x = alpha / (alpha + beta * Math.exp(w + w));
+ }
+ else {
+ double r = beta + beta;
+ double t = 1. / (9. * beta);
+ t = r * Math.pow(1. - t + y * Math.sqrt(t), 3.0);
+ if(t <= 0.) {
+ x = 1. - Math.exp((Math.log1p(-p) + Math.log(beta) + logbeta) / beta);
+ }
+ else {
+ t = (4. * alpha + r - 2.) / t;
+ if(t <= 1.) {
+ x = Math.exp((Math.log(p * alpha) + logbeta) / alpha);
+ }
+ else {
+ x = 1. - 2. / (t + 1.);
+ }
+ }
+ }
+ // Degenerate initial approximations
+ if(x < 3e-308 || x > 1 - 2.22e-16) {
+ x = 0.5;
+ }
+ }
+
+ // Newon-Raphson method using the CDF
+ {
+ final double ialpha = 1 - alpha;
+ final double ibeta = 1 - beta;
+
+ // Desired accuracy, as from GNU R adoption of AS 109
+ final double acu = Math.max(1e-300, Math.pow(10., -13 - 2.5 / (alpha * alpha) - .5 / (p * p)));
+ double prevstep = 0., y = 0., stepsize = 1;
+
+ for(int outer = 0; outer < 1000; outer++) {
+ // Current CDF value
+ double ynew = cdf(x, alpha, beta);
+ if(Double.isInfinite(ynew)) { // Degenerated.
+ return Double.NaN;
+ }
+ // Error gradient
+ ynew = (ynew - p) * Math.exp(logbeta + ialpha * Math.log(x) + ibeta * Math.log1p(-x));
+ if(ynew * y <= 0.) {
+ prevstep = Math.max(Math.abs(stepsize), 3e-308);
+ }
+ // Inner loop: try different step sizes: y * 3^-i
+ double g = 1, xnew = 0.;
+ for(int inner = 0; inner < 1000; inner++) {
+ stepsize = g * ynew;
+ if(Math.abs(stepsize) < prevstep) {
+ xnew = x - stepsize; // Candidate x
+ if(xnew >= 0. && xnew <= 1.) {
+ // Close enough
+ if(prevstep <= acu || Math.abs(ynew) <= acu) {
+ return x;
+ }
+ if(xnew != 0. && xnew != 1.) {
+ break;
+ }
+ }
+ }
+ g /= 3.;
+ }
+ // Convergence
+ if(Math.abs(xnew - x) < 1e-15 * x) {
+ return x;
+ }
+ // Iterate with new values
+ x = xnew;
+ y = ynew;
+ }
+ }
+ // Not converged in Newton-Raphson
+ throw new AbortException("Beta quantile computation did not converge.");
+ }
+}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/ChiDistribution.java b/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/ChiDistribution.java index c561c4cd..84c86e98 100644 --- a/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/ChiDistribution.java +++ b/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/ChiDistribution.java @@ -1,5 +1,7 @@ package de.lmu.ifi.dbs.elki.math.statistics.distribution; +import de.lmu.ifi.dbs.elki.utilities.exceptions.ExceptionMessages; + /* This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures @@ -27,8 +29,10 @@ package de.lmu.ifi.dbs.elki.math.statistics.distribution; * Chi distribution. * * @author Erich Schubert + * + * @apiviz.composedOf ChiSquaredDistribution */ -public class ChiDistribution implements Distribution { +public class ChiDistribution implements DistributionWithRandom { /** * Degrees of freedom. Usually integer. */ @@ -89,4 +93,15 @@ public class ChiDistribution implements Distribution { public static double cdf(double val, double dof) { return GammaDistribution.regularizedGammaP(dof / 2, val * val / 2); } + + // FIXME: implement! + @Override + public double quantile(double val) { + throw new UnsupportedOperationException(ExceptionMessages.UNSUPPORTED_NOT_YET); + } + + @Override + public String toString() { + return "ChiDistribution(dof=" + dof + ")"; + } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/ChiSquaredDistribution.java b/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/ChiSquaredDistribution.java index 0ab39c78..8555afd3 100644 --- a/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/ChiSquaredDistribution.java +++ b/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/ChiSquaredDistribution.java @@ -1,5 +1,7 @@ package de.lmu.ifi.dbs.elki.math.statistics.distribution; +import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; + /* This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures @@ -35,7 +37,7 @@ public class ChiSquaredDistribution extends GammaDistribution { * @param dof Degrees of freedom. */ public ChiSquaredDistribution(double dof) { - super(.5 * dof, 2.0); + super(.5 * dof, .5); } /** @@ -69,4 +71,23 @@ public class ChiSquaredDistribution extends GammaDistribution { } return Math.exp((k - 1.0) * Math.log(x * 2.0) - x * 2.0 - logGamma(k)) * 2.0; } + + /** + * Return the quantile function for this distribution + * + * Reference: + * <p> + * Algorithm AS 91: The percentage points of the $\chi$^2 distribution<br /> + * D.J. Best, D. E. Roberts<br /> + * Journal of the Royal Statistical Society. Series C (Applied Statistics) + * </p> + * + * @param x Quantile + * @param dof Degrees of freedom + * @return quantile position + */ + @Reference(title = "Algorithm AS 91: The percentage points of the $\\chi^2$ distribution", authors = "D.J. Best, D. E. Roberts", booktitle = "Journal of the Royal Statistical Society. Series C (Applied Statistics)") + public static double quantile(double x, double dof) { + return GammaDistribution.quantile(x, .5 * dof, .5); + } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/ConstantDistribution.java b/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/ConstantDistribution.java index 1f36dd4a..b046f0ef 100644 --- a/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/ConstantDistribution.java +++ b/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/ConstantDistribution.java @@ -28,7 +28,7 @@ package de.lmu.ifi.dbs.elki.math.statistics.distribution; * * @author Erich Schubert */ -public class ConstantDistribution implements Distribution { +public class ConstantDistribution implements DistributionWithRandom { /** * The constant */ @@ -58,4 +58,9 @@ public class ConstantDistribution implements Distribution { public double cdf(double val) { return (val >= c) ? 1.0 : 0.0; } + + @Override + public double quantile(double val) { + return c; + } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/Distribution.java b/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/Distribution.java index 290e6434..5b6cd286 100644 --- a/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/Distribution.java +++ b/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/Distribution.java @@ -24,20 +24,14 @@ package de.lmu.ifi.dbs.elki.math.statistics.distribution; */ /** - * Interface for a simple distribution generator with a PDF, i.e. it can also - * compute a density + * Statistical distributions, with their common functions. See + * {@link DistributionWithRandom} for distributions that also have a random + * generator included. * * @author Erich Schubert */ public interface Distribution { /** - * Generate a new random value - * - * @return new random value - */ - public double nextRandom(); - - /** * Return the density of an existing value * * @param val existing value @@ -54,6 +48,14 @@ public interface Distribution { public double cdf(double val); /** + * Quantile aka probit (for normal) aka inverse CDF (invcdf, cdf^-1) function. + * + * @param val Quantile to find + * @return Quantile position + */ + public double quantile(double val); + + /** * Describe the distribution * * @return description diff --git a/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/DistributionWithRandom.java b/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/DistributionWithRandom.java new file mode 100644 index 00000000..af272528 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/DistributionWithRandom.java @@ -0,0 +1,37 @@ +package de.lmu.ifi.dbs.elki.math.statistics.distribution; +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +/** + * Distribution that also has support for generating random numbers. + * + * @author Erich Schubert + */ +public interface DistributionWithRandom extends Distribution { + /** + * Generate a new random value + * + * @return new random value + */ + public double nextRandom(); +} diff --git a/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/GammaDistribution.java b/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/GammaDistribution.java index 6830f25a..21eebc51 100644 --- a/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/GammaDistribution.java +++ b/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/GammaDistribution.java @@ -25,14 +25,21 @@ package de.lmu.ifi.dbs.elki.math.statistics.distribution; import java.util.Random; +import de.lmu.ifi.dbs.elki.logging.LoggingUtil; import de.lmu.ifi.dbs.elki.math.MathUtil; +import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; /** * Gamma Distribution, with random generation and density functions. * * @author Erich Schubert */ -public class GammaDistribution implements Distribution { +public class GammaDistribution implements DistributionWithRandom { + /** + * Euler–Mascheroni constant + */ + public static final double EULERS_CONST = 0.5772156649015328606065120900824024; + /** * LANCZOS-Coefficients for Gamma approximation. * @@ -102,6 +109,11 @@ public class GammaDistribution implements Distribution { } @Override + public double quantile(double val) { + return quantile(val, k, theta); + } + + @Override public double nextRandom() { return nextRandom(k, theta, random); } @@ -113,7 +125,7 @@ public class GammaDistribution implements Distribution { */ @Override public String toString() { - return "Gamma Distribution (k=" + k + ", theta=" + theta + ")"; + return "GammaDistribution(k=" + k + ", theta=" + theta + ")"; } /** @@ -139,7 +151,19 @@ public class GammaDistribution implements Distribution { * @return cdf value */ public static double cdf(double val, double k, double theta) { - return regularizedGammaP(k, val / theta); + return regularizedGammaP(k, val * theta); + } + + /** + * The log CDF, static version. + * + * @param val Value + * @param k Shape k + * @param theta Theta = 1.0/Beta aka. "scaling" parameter + * @return cdf value + */ + public static double logcdf(double val, double k, double theta) { + return logregularizedGammaP(k, val * theta); } /** @@ -170,6 +194,33 @@ public class GammaDistribution implements Distribution { } /** + * Gamma distribution PDF (with 0.0 for x < 0) + * + * @param x query value + * @param k Alpha + * @param theta Theta = 1 / Beta + * @return probability density + */ + public static double logpdf(double x, double k, double theta) { + if(x < 0) { + return Double.NEGATIVE_INFINITY; + } + if(x == 0) { + if(k == 1.0) { + return Math.log(theta); + } + else { + return Double.NEGATIVE_INFINITY; + } + } + if(k == 1.0) { + return Math.log(theta) - x * theta; + } + + return Math.log(theta) + (k - 1.0) * Math.log(x * theta) - x * theta - logGamma(k); + } + + /** * Compute logGamma. * * Based loosely on "Numerical Recpies" and the work of Paul Godfrey at @@ -195,6 +246,22 @@ public class GammaDistribution implements Distribution { } /** + * Compute the regular Gamma function. + * + * Note: for numerical reasons, it is preferrable to use {@link #logGamma} + * when possible! In particular, this method just computes + * {@code Math.exp(logGamma(x))} anyway. + * + * Try to postpone the {@code Math.exp} call to preserve numeric range! + * + * @param x Position + * @return Gamma at this position + */ + public static double gamma(double x) { + return Math.exp(logGamma(x)); + } + + /** * Returns the regularized gamma function P(a, x). * * Includes the quadrature way of computing. @@ -236,6 +303,49 @@ public class GammaDistribution implements Distribution { } /** + * Returns the regularized gamma function log P(a, x). + * + * Includes the quadrature way of computing. + * + * TODO: find "the" most accurate version of this. We seem to agree with + * others for the first 10+ digits, but diverge a bit later than that. + * + * @param a Parameter a + * @param x Parameter x + * @return Gamma value + */ + public static double logregularizedGammaP(final double a, final double x) { + // Special cases + if(Double.isNaN(a) || Double.isNaN(x) || (a <= 0.0) || (x < 0.0)) { + return Double.NaN; + } + if(x == 0.0) { + return Double.NEGATIVE_INFINITY; + } + if(x >= a + 1) { + // Expected to converge faster + // FIXME: and in log? + return Math.log(1.0 - regularizedGammaQ(a, x)); + } + // Loosely following "Numerical Recipes" + double del = 1.0 / a; + double sum = del; + for(int n = 1; n < Integer.MAX_VALUE; n++) { + // compute next element in the series + del *= x / (a + n); + sum = sum + del; + if(Math.abs(del / sum) < NUM_PRECISION || sum >= Double.POSITIVE_INFINITY) { + break; + } + } + if(Double.isInfinite(sum)) { + return 0; + } + // TODO: reread numerical recipes, can we replace log(sum)? + return -x + (a * Math.log(x)) - logGamma(a) + Math.log(sum); + } + + /** * Returns the regularized gamma function Q(a, x) = 1 - P(a, x). * * Includes the continued fraction way of computing, based loosely on the book @@ -313,7 +423,7 @@ public class GammaDistribution implements Distribution { final double e1 = 1.000000000, e2 = 0.499999994, e3 = 0.166666848; final double e4 = 0.041664508, e5 = 0.008345522, e6 = 0.001353826; final double e7 = 0.000247453; - + if(k < 1.0) { // Base case, for small k final double b = 1.0 + 0.36788794412 * k; // Step 1 while(true) { @@ -360,11 +470,11 @@ public class GammaDistribution implements Distribution { /* v2 = tv2; */ v12 = tv12; } - + // double b = 0.0, c = 0.0; // double si = 0.0, q0 = 0.0; final double b, c, si, q0; - + // Simpler accept cases & parameter computation { final double t = v1 * Math.sqrt(-2.0 * Math.log(v12) / v12); @@ -373,14 +483,14 @@ public class GammaDistribution implements Distribution { if(t >= 0.0) { return (gds / theta); // Immediate acceptance } - + // Random uniform final double un = random.nextDouble(); // Squeeze acceptance if(d * un <= t * t * t) { return (gds / theta); } - + if(k != -1.0) { // Step 4. Set-up for hat case final double r = 1.0 / k; q0 = ((((((((q9 * r + q8) * r + q7) * r + q6) * r + q5) * r + q4) * r + q3) * r + q2) * r + q1) * r; @@ -425,7 +535,7 @@ public class GammaDistribution implements Distribution { } } } - + // Double exponential deviate t while(true) { double e, u, sign_u, t; @@ -438,7 +548,7 @@ public class GammaDistribution implements Distribution { t = b + (e * si) * sign_u; } while(t <= -0.71874483771719); - + // New v(t) and q(t) final double v = t / (s + s); final double q; @@ -467,4 +577,370 @@ public class GammaDistribution implements Distribution { } } } + + /** + * Approximate probit for chi squared distribution + * + * Based on first half of algorithm AS 91 + * + * Reference: + * <p> + * Algorithm AS 91: The percentage points of the $\chi$ 2 distribution<br /> + * D.J. Best, D. E. Roberts<br /> + * Journal of the Royal Statistical Society. Series C (Applied Statistics) + * </p> + * + * @param p Probit value + * @param nu Shape parameter for Chi, nu = 2 * k + * @param g log(nu) + * @return Probit for chi squared + */ + @Reference(title = "Algorithm AS 91: The percentage points of the $\\chi^2$ distribution", authors = "D.J. Best, D. E. Roberts", booktitle = "Journal of the Royal Statistical Society. Series C (Applied Statistics)") + protected static double chisquaredProbitApproximation(final double p, double nu, double g) { + final double EPS1 = 1e-2; // Approximation quality + // Sanity checks + if(Double.isNaN(p) || Double.isNaN(nu)) { + return Double.NaN; + } + // Range check + if(p <= 0) { + return 0; + } + if(p >= 1) { + return Double.POSITIVE_INFINITY; + } + // Invalid parameters + if(nu <= 0) { + return Double.NaN; + } + // Shape of gamma distribution, "XX" in AS 91 + final double k = 0.5 * nu; + + // For small chi squared values - AS 91 + final double logp = Math.log(p); + if(nu < -1.24 * logp) { + // FIXME: implement and use logGammap1 instead - more stable? + // + // final double lgam1pa = (alpha < 0.5) ? logGammap1(alpha) : + // (Math.log(alpha) + g); + // return Math.exp((lgam1pa + logp) / alpha + MathUtil.LOG2); + // This is literal AS 91, above is the GNU R variant. + return Math.pow(p * k * Math.exp(g + k * MathUtil.LOG2), 1 / k); + } + else if(nu > 0.32) { + // Wilson and Hilferty estimate: - AS 91 at 3 + final double x = NormalDistribution.quantile(p, 0, 1); + final double p1 = 2. / (9. * nu); + double ch = nu * Math.pow(x * Math.sqrt(p1) + 1 - p1, 3); + + // Better approximation for p tending to 1: + if(ch > 2.2 * nu + 6) { + ch = -2 * (Math.log1p(-p) - (k - 1) * Math.log(0.5 * ch) + g); + } + return ch; + } + else { + // nu <= 0.32, AS 91 at 1 + final double C7 = 4.67, C8 = 6.66, C9 = 6.73, C10 = 13.32; + final double ag = Math.log1p(-p) + g + (k - 1) * MathUtil.LOG2; + double ch = 0.4; + while(true) { + final double p1 = 1 + ch * (C7 + ch); + final double p2 = ch * (C9 + ch * (C8 + ch)); + final double t = -0.5 + (C7 + 2 * ch) / p1 - (C9 + ch * (C10 + 3 * ch)) / p2; + final double delta = (1 - Math.exp(ag + 0.5 * ch) * p2 / p1) / t; + ch -= delta; + if(Math.abs(delta) > EPS1 * Math.abs(ch)) { + return ch; + } + } + } + } + + /** + * Compute probit (inverse cdf) for Gamma distributions. + * + * Based on algorithm AS 91: + * + * Reference: + * <p> + * Algorithm AS 91: The percentage points of the $\chi$^2 distribution<br /> + * D.J. Best, D. E. Roberts<br /> + * Journal of the Royal Statistical Society. Series C (Applied Statistics) + * </p> + * + * @param p Probability + * @param k k, alpha aka. "shape" parameter + * @param theta Theta = 1.0/Beta aka. "scaling" parameter + * @return Probit for Gamma distribution + */ + @Reference(title = "Algorithm AS 91: The percentage points of the $\\chi$^2 distribution", authors = "D.J. Best, D. E. Roberts", booktitle = "Journal of the Royal Statistical Society. Series C (Applied Statistics)") + public static double quantile(double p, double k, double theta) { + final double EPS2 = 5e-7; // final precision of AS 91 + final int MAXIT = 1000; + + // Avoid degenerates + if(Double.isNaN(p) || Double.isNaN(k) || Double.isNaN(theta)) { + return Double.NaN; + } + // Range check + if(p <= 0) { + return 0; + } + if(p >= 1) { + return Double.POSITIVE_INFINITY; + } + // Shape parameter check + if(k < 0 || theta <= 0) { + return Double.NaN; + } + // Corner case - all at 0 + if(k == 0) { + return 0.; + } + + int max_newton_iterations = 1; + // For small values, ensure some refinement iterations + if(k < 1e-10) { + max_newton_iterations = 7; + } + + final double g = logGamma(k); // == logGamma(v/2) + + // Phase I, an initial rough approximation + // First half of AS 91 + double ch = chisquaredProbitApproximation(p, 2 * k, g); + // Second hald of AS 91 follows: + // Refine ChiSquared approximation + chisq: { + if(Double.isInfinite(ch)) { + // Cannot refine infinity + max_newton_iterations = 0; + break chisq; + } + if(ch < EPS2) { + // Do not iterate, but refine with newton method + max_newton_iterations = 20; + break chisq; + } + if(p > 1 - 1e-14 || p < 1e-100) { + // Not in appropriate value range for AS 91 + max_newton_iterations = 20; + break chisq; + } + + // Phase II: Iteration + final double c = k - 1; + final double ch0 = ch; // backup initial approximation + for(int i = 1; i <= MAXIT; i++) { + final double q = ch; // previous approximation + final double p1 = 0.5 * ch; + final double p2 = p - regularizedGammaP(k, p1); + if(Double.isInfinite(p2) || ch <= 0) { + ch = ch0; + max_newton_iterations = 27; + break chisq; + } + { // Taylor series of AS 91: iteration via "goto 4" + final double t = p2 * Math.exp(k * MathUtil.LOG2 + g + p1 - c * Math.log(ch)); + final double b = t / ch; + final double a = 0.5 * t - b * c; + final double s1 = (210. + a * (140. + a * (105. + a * (84. + a * (70. + 60. * a))))) / 420.; + final double s2 = (420. + a * (735. + a * (966. + a * (1141. + 1278 * a)))) / 2520.; + final double s3 = (210. + a * (462. + a * (707. + 932. * a))) / 2520.; + final double s4 = (252. + a * (672. + 1182. * a) + c * (294. + a * (889. + 1740. * a))) / 5040.; + final double s5 = (84. + 2264. * a + c * (1175. + 606. * a)) / 2520.; + final double s6 = (120. + c * (346. + 127. * c)) / 5040.; + ch += t * (1 + 0.5 * t * s1 - b * c * (s1 - b * (s2 - b * (s3 - b * (s4 - b * (s5 - b * s6)))))); + } + if(Math.abs(q - ch) < EPS2 * ch) { + break chisq; + } + // Divergence treatment, from GNU R + if(Math.abs(q - ch) > 0.1 * Math.abs(ch)) { + ch = ((ch < q) ? 0.9 : 1.1) * q; + } + } + LoggingUtil.warning("No convergence in AS 91 Gamma probit."); + // no convergence in MAXIT iterations -- but we add Newton now... + } + double x = 0.5 * ch / theta; + if(max_newton_iterations > 0) { + // Refine result using final Newton steps. + // TODO: add unit tests that show an improvement! Maybe in logscale only? + x = gammaQuantileNewtonRefinement(Math.log(p), k, theta, max_newton_iterations, x); + } + return x; + } + + /** + * Refinement of ChiSquared probit using Newton iterations. + * + * A trick used by GNU R to improve precision. + * + * @param logpt Target value of log p + * @param k Alpha + * @param theta Theta = 1 / Beta + * @param maxit Maximum number of iterations to do + * @param x Initial estimate + * @return Refined value + */ + protected static double gammaQuantileNewtonRefinement(final double logpt, final double k, final double theta, final int maxit, double x) { + final double EPS_N = 1e-15; // Precision threshold + // 0 is not possible, try MIN_NORMAL instead + if(x <= 0) { + x = Double.MIN_NORMAL; + } + // Current estimation + double logpc = logcdf(x, k, theta); + if(x == Double.MIN_NORMAL && logpc > logpt * (1. + 1e-7)) { + return 0.; + } + if(logpc == Double.NEGATIVE_INFINITY) { + return 0.; + } + // Refine by newton iterations + for(int i = 0; i < maxit; i++) { + // Error of current approximation + final double logpe = logpc - logpt; + if(Math.abs(logpe) < Math.abs(EPS_N * logpt)) { + break; + } + // Step size is controlled by PDF: + final double g = logpdf(x, k, theta); + if(g == Double.NEGATIVE_INFINITY) { + break; + } + final double newx = x - logpe * Math.exp(logpc - g); + // New estimate: + logpc = logcdf(newx, k, theta); + if(Math.abs(logpc - logpt) > Math.abs(logpe) || (i > 0 && Math.abs(logpc - logpt) == Math.abs(logpe))) { + // no further improvement + break; + } + x = newx; + } + return x; + } + + /** + * Compute the Psi / Digamma function + * + * Reference: + * <p> + * J. M. Bernando<br /> + * Algorithm AS 103: Psi (Digamma) Function<br /> + * Statistical Algorithms + * </p> + * + * TODO: is there a more accurate version maybe in R? + * + * @param x Position + * @return digamma value + */ + @Reference(authors = "J. M. Bernando", title = "Algorithm AS 103: Psi (Digamma) Function", booktitle = "Statistical Algorithms") + public static double digamma(double x) { + if(!(x > 0)) { + return Double.NaN; + } + // Method of equation 5: + if(x <= 1e-5) { + return -EULERS_CONST - 1. / x; + } + // Method of equation 4: + else if(x > 49.) { + final double ix2 = 1. / (x * x); + // Partial series expansion + return Math.log(x) - 0.5 / x - ix2 * ((1.0 / 12.) + ix2 * (1.0 / 120. - ix2 / 252.)); + // + O(x^8) error + } + else { + // Stirling expansion + return digamma(x + 1.) - 1. / x; + } + } + + /** + * Compute the Trigamma function. Based on digamma. + * + * TODO: is there a more accurate version maybe in R? + * + * @param x Position + * @return trigamma value + */ + public static double trigamma(double x) { + if(!(x > 0)) { + return Double.NaN; + } + // Method of equation 5: + if(x <= 1e-5) { + return 1. / (x * x); + } + // Method of equation 4: + else if(x > 49.) { + final double ix2 = 1. / (x * x); + // Partial series expansion + return 1 / x - ix2 / 2. + ix2 / x * (1.0 / 6. - ix2 * (1.0 / 30. + ix2 / 42.)); + // + O(x^8) error + } + else { + // Stirling expansion + return trigamma(x + 1.) - 1. / (x * x); + } + } + + /** + * Mean least squares estimation of Gamma distribution to a set of + * observations. + * + * @param data Data + * @return Assumed distribution + */ + public static GammaDistribution estimate(double[] data) { + return estimate(data, data.length); + } + + /** + * Mean least squares estimation of Gamma distribution to a set of + * observations. + * + * Reference: + * <p> + * Maximum likelihood estimation of the parameters of the gamma distribution + * and their bias<br /> + * S. C. Choi, R. Wette<br /> + * in: Technometrics + * </p> + * + * @param data Data + * @param len Length of array + * @return Assumed distribution + */ + @Reference(title = "Maximum likelihood estimation of the parameters of the gamma distribution and their bias", authors = "S. C. Choi, R. Wette", booktitle = "Technometrics", url = "http://www.jstor.org/stable/10.2307/1266892") + public static GammaDistribution estimate(double[] data, int len) { + double meanx = 0, meanlogx = 0; + for(int i = 0; i < len; i++) { + final double logx = Math.log(data[i]); + final double deltax = data[i] - meanx; + final double deltalogx = logx - meanlogx; + meanx += deltax / (i + 1.); + meanlogx += deltalogx / (i + 1.); + } + // Initial approximation + final double logmeanx = Math.log(meanx); + final double diff = logmeanx - meanlogx; + double k = (3 - diff + Math.sqrt((diff - 3) * (diff - 3) + 24 * diff)) / (12 * diff); + + // Refine via newton iteration, based on Choi and Wette equation + while(true) { + double kdelta = (Math.log(k) - digamma(k) - diff) / (1 / k - trigamma(k)); + if(Math.abs(kdelta) < 1E-8 || Double.isNaN(kdelta)) { + break; + } + k += kdelta; + } + // Estimate theta: + final double theta = k / meanx; + return new GammaDistribution(k, theta); + } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/NormalDistribution.java b/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/NormalDistribution.java index 919cc2e3..9180b59e 100644 --- a/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/NormalDistribution.java +++ b/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/NormalDistribution.java @@ -32,7 +32,7 @@ import de.lmu.ifi.dbs.elki.math.MathUtil; * * @author Erich Schubert */ -public class NormalDistribution implements Distribution { +public class NormalDistribution implements DistributionWithRandom { /** * Coefficients for erf approximation. * @@ -148,20 +148,25 @@ public class NormalDistribution implements Distribution { public double pdf(double val) { return pdf(val, mean, stddev); } - + @Override public double cdf(double val) { return cdf(val, mean, stddev); } @Override + public double quantile(double q) { + return quantile(q, mean, stddev); + } + + @Override public double nextRandom() { return mean + random.nextGaussian() * stddev; } @Override public String toString() { - return "Normal Distribution (mean="+mean+", stddev="+stddev+")"; + return "NormalDistribution(mean=" + mean + ", stddev=" + stddev + ")"; } /** @@ -195,7 +200,7 @@ public class NormalDistribution implements Distribution { if(Double.isInfinite(x)) { return (x < 0.0) ? 2 : 0; } - + double result = Double.NaN; double absx = Math.abs(x); // First approximation interval @@ -249,7 +254,7 @@ public class NormalDistribution implements Distribution { * @return erfinv(x) */ public static double erfinv(double x) { - return standardNormalProbit(0.5 * (x + 1)) / MathUtil.SQRT2; + return standardNormalQuantile(0.5 * (x + 1)) / MathUtil.SQRT2; } /** @@ -261,10 +266,13 @@ public class NormalDistribution implements Distribution { * by Peter John Acklam * </p> * + * FIXME: precision of this seems to be rather low, compared to our other + * functions. Only about 8-9 digits agree with SciPy/GNU R. + * * @param d Quantile. Must be in [0:1], obviously. * @return Inverse erf. */ - public static double standardNormalProbit(double d) { + public static double standardNormalQuantile(double d) { if(d == 0) { return Double.NEGATIVE_INFINITY; } @@ -319,7 +327,7 @@ public class NormalDistribution implements Distribution { * @return The CDF of the normal given distribution at x. */ public static double cdf(double x, double mu, double sigma) { - return (1 + erf(x / Math.sqrt(2))) / 2; + return .5 * (1 + erf((x - mu) / (MathUtil.SQRT2 * sigma))); } /** @@ -331,7 +339,7 @@ public class NormalDistribution implements Distribution { * @param sigma Standard deviation. * @return The probit of the normal given distribution at x. */ - public static double probit(double x, double mu, double sigma) { - return mu + sigma * standardNormalProbit(x); + public static double quantile(double x, double mu, double sigma) { + return mu + sigma * standardNormalQuantile(x); } } diff --git a/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/PoissonDistribution.java b/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/PoissonDistribution.java new file mode 100644 index 00000000..53fb0dc8 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/PoissonDistribution.java @@ -0,0 +1,411 @@ +package de.lmu.ifi.dbs.elki.math.statistics.distribution; + +import de.lmu.ifi.dbs.elki.math.MathUtil; +import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; +import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; +import de.lmu.ifi.dbs.elki.utilities.exceptions.ExceptionMessages; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +/** + * INCOMPLETE implementation of the poisson distribution. + * + * TODO: continue implementing, CDF, invcdf and nextRandom are missing + * + * References: + * <p> + * Catherine Loader<br /> + * Fast and Accurate Computation of Binomial Probabilities. + * </p> + * + * @author Erich Schubert + */ +public class PoissonDistribution implements Distribution { + /** + * Number of tries + */ + private int n; + + /** + * Success probability + */ + private double p; + + /** Stirling error constants: 1./12 */ + private final static double S0 = 0.08333333333333333333333d; + + /** Stirling error constants: 1./360 */ + private final static double S1 = 0.0027777777777777777777777777778d; + + /** Stirling error constants: 1./1260 */ + private final static double S2 = 0.00079365079365079365079365d; + + /** Stirling error constants: 1./1680 */ + private final static double S3 = 0.000595238095238095238095238095d; + + /** Stirling error constants: 1./1188 */ + private final static double S4 = 0.00084175084175084175084175084175d; + + /** + * Exact table values for n <= 15 in steps of 0.5 + * + * sfe[n] = ln( (n!*e^n)/((n^n)*sqrt(2*pi*n)) ) + */ + private final static double STIRLING_EXACT_ERROR[] = {// + 0.0, // 0.0 + 0.1534264097200273452913848, // 0.5 + 0.0810614667953272582196702, // 1.0 + 0.0548141210519176538961390, // 1.5 + 0.0413406959554092940938221, // 2.0 + 0.03316287351993628748511048, // 2.5 + 0.02767792568499833914878929, // 3.0 + 0.02374616365629749597132920, // 3.5 + 0.02079067210376509311152277, // 4.0 + 0.01848845053267318523077934, // 4.5 + 0.01664469118982119216319487, // 5.0 + 0.01513497322191737887351255, // 5.5 + 0.01387612882307074799874573, // 6.0 + 0.01281046524292022692424986, // 6.5 + 0.01189670994589177009505572, // 7.0 + 0.01110455975820691732662991, // 7.5 + 0.010411265261972096497478567, // 8.0 + 0.009799416126158803298389475, // 8.5 + 0.009255462182712732917728637, // 9.0 + 0.008768700134139385462952823, // 9.5 + 0.008330563433362871256469318, // 10.0 + 0.007934114564314020547248100, // 10.5 + 0.007573675487951840794972024, // 11.0 + 0.007244554301320383179543912, // 11.5 + 0.006942840107209529865664152, // 12.0 + 0.006665247032707682442354394, // 12.5 + 0.006408994188004207068439631, // 13.0 + 0.006171712263039457647532867, // 13.5 + 0.005951370112758847735624416, // 14.0 + 0.005746216513010115682023589, // 14.5 + 0.0055547335519628013710386900 // 15.0 + }; + + /** + * Constructor. + * + * Private: API not yet completely implemented! + * + * @param n Number of tries + * @param p Success probability + */ + public PoissonDistribution(int n, double p) { + super(); + this.n = n; + this.p = p; + } + + /** + * Poisson PMF for integer values. + * + * @param x integer values + * @return Probability + */ + @Reference(title = "Fast and accurate computation of binomial probabilities", authors = "C. Loader", booktitle = "", url = "http://projects.scipy.org/scipy/raw-attachment/ticket/620/loader2000Fast.pdf") + public double pmf(int x) { + // Invalid values + if(x < 0 || x > n) { + return 0.0; + } + // Extreme probabilities + if(p <= 0d) { + return x == 0 ? 1.0 : 0.0; + } + if(p >= 1d) { + return x == n ? 1.0 : 0.0; + } + // Extreme values of x + if(x == 0) { + if(p < 0.1) { + return Math.exp(-devianceTerm(n, n * (1.0 - p)) - n * p); + } + else { + return Math.exp(n * Math.log(1.0 - p)); + } + } + if(x == n) { + if(p > 0.9) { + return Math.exp(-devianceTerm(n, n * p) - n * (1 - p)); + } + else { + return Math.exp(n * Math.log(p)); + } + } + + final double lc = stirlingError(n) - stirlingError(x) - stirlingError(n - x) - devianceTerm(x, n * p) - devianceTerm(n - x, n * (1.0 - p)); + final double f = (MathUtil.TWOPI * x * (n - x)) / n; + return Math.exp(lc) / Math.sqrt(f); + } + + @Override + @Reference(title = "Fast and accurate computation of binomial probabilities", authors = "C. Loader", booktitle = "", url = "http://projects.scipy.org/scipy/raw-attachment/ticket/620/loader2000Fast.pdf") + public double pdf(double x) { + // Invalid values + if(x < 0 || x > n) { + return 0.0; + } + // Extreme probabilities + if(p <= 0d) { + return x == 0 ? 1.0 : 0.0; + } + if(p >= 1d) { + return x == n ? 1.0 : 0.0; + } + final double q = 1 - p; + // FIXME: check for x to be integer, return 0 otherwise? + + // Extreme values of x + if(x == 0) { + if(p < 0.1) { + return Math.exp(-devianceTerm(n, n * q) - n * p); + } + else { + return Math.exp(n * Math.log(q)); + } + } + if(x == n) { + if(p > 0.9) { + return Math.exp(-devianceTerm(n, n * p) - n * q); + } + else { + return Math.exp(n * Math.log(p)); + } + } + final double lc = stirlingError(n) - stirlingError(x) - stirlingError(n - x) - devianceTerm(x, n * p) - devianceTerm(n - x, n * q); + final double f = (MathUtil.TWOPI * x * (n - x)) / n; + return Math.exp(lc) / Math.sqrt(f); + } + + // FIXME: implement! + @Override + public double cdf(double val) { + throw new AbortException(ExceptionMessages.UNSUPPORTED_NOT_YET); + } + + // FIXME: implement! + @Override + public double quantile(double val) { + throw new AbortException(ExceptionMessages.UNSUPPORTED_NOT_YET); + } + + /** + * Compute the poisson distribution PDF with an offset of + 1 + * + * pdf(x_plus_1 - 1, lambda) + * + * @param x_plus_1 x+1 + * @param lambda Lambda + * @return pdf + */ + public static double poissonPDFm1(double x_plus_1, double lambda) { + if(Double.isInfinite(lambda)) { + return 0.; + } + if(x_plus_1 > 1) { + return rawProbability(x_plus_1 - 1, lambda); + } + if(lambda > Math.abs(x_plus_1 - 1) * MathUtil.LOG2 * Double.MAX_EXPONENT / 1e-14) { + return Math.exp(-lambda - GammaDistribution.logGamma(x_plus_1)); + } + else { + return rawProbability(x_plus_1, lambda) * (x_plus_1 / lambda); + } + } + + /** + * Compute the poisson distribution PDF with an offset of + 1 + * + * log pdf(x_plus_1 - 1, lambda) + * + * @param x_plus_1 x+1 + * @param lambda Lambda + * @return pdf + */ + public static double logpoissonPDFm1(double x_plus_1, double lambda) { + if(Double.isInfinite(lambda)) { + return Double.NEGATIVE_INFINITY; + } + if(x_plus_1 > 1) { + return rawLogProbability(x_plus_1 - 1, lambda); + } + if(lambda > Math.abs(x_plus_1 - 1) * MathUtil.LOG2 * Double.MAX_EXPONENT / 1e-14) { + return -lambda - GammaDistribution.logGamma(x_plus_1); + } + else { + return rawLogProbability(x_plus_1, lambda) + Math.log(x_plus_1 / lambda); + } + } + + /** + * Calculates the Striling Error + * + * stirlerr(n) = ln(n!) - ln(sqrt(2*pi*n)*(n/e)^n) + * + * @param n Parameter n + * @return Stirling error + */ + @Reference(title = "Fast and accurate computation of binomial probabilities", authors = "C. Loader", booktitle = "", url = "http://projects.scipy.org/scipy/raw-attachment/ticket/620/loader2000Fast.pdf") + private static double stirlingError(int n) { + // Try to use a table value: + if(n < 16) { + return STIRLING_EXACT_ERROR[n * 2]; + } + final double nn = n * n; + // Use the appropriate number of terms + if(n > 500) { + return (S0 - S1 / nn) / n; + } + if(n > 80) { + return ((S0 - (S1 - S2 / nn)) / nn) / n; + } + if(n > 35) { + return ((S0 - (S1 - (S2 - S3 / nn) / nn) / nn) / n); + } + return ((S0 - (S1 - (S2 - (S3 - S4 / nn) / nn) / nn) / nn) / n); + } + + /** + * Calculates the Striling Error + * + * stirlerr(n) = ln(n!) - ln(sqrt(2*pi*n)*(n/e)^n) + * + * @param n Parameter n + * @return Stirling error + */ + @Reference(title = "Fast and accurate computation of binomial probabilities", authors = "C. Loader", booktitle = "", url = "http://projects.scipy.org/scipy/raw-attachment/ticket/620/loader2000Fast.pdf") + private static double stirlingError(double n) { + if(n < 16.0) { + // Our table has a step size of 0.5 + final double n2 = 2.0 * n; + if(Math.floor(n2) == n2) { // Exact match + return STIRLING_EXACT_ERROR[(int) n2]; + } + else { + return GammaDistribution.logGamma(n + 1.0) - (n + 0.5) * Math.log(n) + n - MathUtil.LOGSQRTTWOPI; + } + } + final double nn = n * n; + if(n > 500.0) { + return (S0 - S1 / nn) / n; + } + if(n > 80.0) { + return ((S0 - (S1 - S2 / nn)) / nn) / n; + } + if(n > 35.0) { + return ((S0 - (S1 - (S2 - S3 / nn) / nn) / nn) / n); + } + return ((S0 - (S1 - (S2 - (S3 - S4 / nn) / nn) / nn) / nn) / n); + } + + /** + * Evaluate the deviance term of the saddle point approximation. + * + * bd0(x,np) = x*ln(x/np)+np-x + * + * @param x probability density function position + * @param np product of trials and success probability: n*p + * @return Deviance term + */ + @Reference(title = "Fast and accurate computation of binomial probabilities", authors = "C. Loader", booktitle = "", url = "http://projects.scipy.org/scipy/raw-attachment/ticket/620/loader2000Fast.pdf") + private static double devianceTerm(double x, double np) { + if(Math.abs(x - np) < 0.1 * (x + np)) { + final double v = (x - np) / (x + np); + + double s = (x - np) * v; + double ej = 2.0d * x * v; + for(int j = 1;; j++) { + ej *= v * v; + final double s1 = s + ej / (2 * j + 1); + if(s1 == s) { + return s1; + } + s = s1; + } + } + return x * Math.log(x / np) + np - x; + } + + /** + * Poisson distribution probability, but also for non-integer arguments. + * + * lb^x exp(-lb) / x! + * + * @param x X + * @param lambda lambda + * @return Poisson distribution probability + */ + public static double rawProbability(double x, double lambda) { + // Extreme lambda + if(lambda == 0) { + return ((x == 0) ? 1. : 0.); + } + // Extreme values + if(Double.isInfinite(lambda) || x < 0) { + return 0.; + } + if(x <= lambda * Double.MIN_NORMAL) { + return Math.exp(-lambda); + } + if(lambda < x * Double.MIN_NORMAL) { + double r = -lambda + x * Math.log(lambda) - GammaDistribution.logGamma(x + 1); + return Math.exp(r); + } + final double f = MathUtil.TWOPI * x; + final double y = -stirlingError(x) - devianceTerm(x, lambda); + return Math.exp(y) / Math.sqrt(f); + } + + /** + * Poisson distribution probability, but also for non-integer arguments. + * + * lb^x exp(-lb) / x! + * + * @param x X + * @param lambda lambda + * @return Poisson distribution probability + */ + public static double rawLogProbability(double x, double lambda) { + // Extreme lambda + if(lambda == 0) { + return ((x == 0) ? 1. : Double.NEGATIVE_INFINITY); + } + // Extreme values + if(Double.isInfinite(lambda) || x < 0) { + return Double.NEGATIVE_INFINITY; + } + if(x <= lambda * Double.MIN_NORMAL) { + return -lambda; + } + if(lambda < x * Double.MIN_NORMAL) { + return -lambda + x * Math.log(lambda) - GammaDistribution.logGamma(x + 1); + } + final double f = MathUtil.TWOPI * x; + final double y = -stirlingError(x) - devianceTerm(x, lambda); + return -0.5 * Math.log(f) + y; + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/StudentsTDistribution.java b/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/StudentsTDistribution.java new file mode 100644 index 00000000..2e9e0d15 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/StudentsTDistribution.java @@ -0,0 +1,90 @@ +package de.lmu.ifi.dbs.elki.math.statistics.distribution;
+
+/*
+ This file is part of ELKI:
+ Environment for Developing KDD-Applications Supported by Index-Structures
+
+ Copyright (C) 2012
+ Ludwig-Maximilians-Universität München
+ Lehr- und Forschungseinheit für Datenbanksysteme
+ ELKI Development Team
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException;
+import de.lmu.ifi.dbs.elki.utilities.exceptions.ExceptionMessages;
+
+/**
+ * Student's t distribution.
+ *
+ * FIXME: add quantile function!
+ *
+ * @author Jan Brusis
+ */
+public class StudentsTDistribution implements Distribution {
+ /**
+ * Degrees of freedom
+ */
+ private final int v;
+
+ /**
+ * Constructor.
+ *
+ * @param v Degrees of freedom
+ */
+ public StudentsTDistribution(int v) {
+ this.v = v;
+ }
+
+ @Override
+ public double pdf(double val) {
+ return pdf(val, v);
+ }
+
+ @Override
+ public double cdf(double val) {
+ return cdf(val, v);
+ }
+
+ // FIXME: implement!
+ @Override
+ public double quantile(double val) {
+ throw new AbortException(ExceptionMessages.UNSUPPORTED_NOT_YET);
+ }
+
+ /**
+ * Static version of the t distribution's PDF.
+ *
+ * @param val value to evaluate
+ * @param v degrees of freedom
+ * @return f(val,v)
+ */
+ public static double pdf(double val, int v) {
+ // TODO: improve precision by computing "exp" last?
+ return Math.exp(GammaDistribution.logGamma((v + 1) / 2) - GammaDistribution.logGamma(v / 2)) * (1 / Math.sqrt(v * Math.PI)) * Math.pow(1 + (val * val) / v, -((v + 1) / 2));
+ }
+
+ /**
+ * Static version of the CDF of the t-distribution for t > 0
+ *
+ * @param val value to evaluate
+ * @param v degrees of freedom
+ * @return F(val, v)
+ */
+ public static double cdf(double val, int v) {
+ double x = v / (val * val + v);
+ return 1 - (0.5 * BetaDistribution.regularizedIncBeta(x, v / 2, 0.5));
+ }
+}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/UniformDistribution.java b/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/UniformDistribution.java index 9571cfd3..4f54fbf9 100644 --- a/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/UniformDistribution.java +++ b/src/de/lmu/ifi/dbs/elki/math/statistics/distribution/UniformDistribution.java @@ -30,7 +30,7 @@ import java.util.Random; * * @author Erich Schubert */ -public class UniformDistribution implements Distribution { +public class UniformDistribution implements DistributionWithRandom { /** * Minimum */ @@ -100,20 +100,20 @@ public class UniformDistribution implements Distribution { } return (val - min) / len; } + + @Override + public double quantile(double val) { + return min + len * val; + } @Override public double nextRandom() { return min + random.nextDouble() * len; } - /** - * Simple toString explaining the distribution parameters. - * - * Used in describing cluster models. - */ @Override public String toString() { - return "Uniform Distribution (min=" + min + ", max=" + max + ")"; + return "UniformDistribution(min=" + min + ", max=" + max + ")"; } /** diff --git a/src/de/lmu/ifi/dbs/elki/math/statistics/tests/GoodnessOfFitTest.java b/src/de/lmu/ifi/dbs/elki/math/statistics/tests/GoodnessOfFitTest.java new file mode 100644 index 00000000..f50f469f --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/math/statistics/tests/GoodnessOfFitTest.java @@ -0,0 +1,49 @@ +package de.lmu.ifi.dbs.elki.math.statistics.tests;
+
+/*
+ This file is part of ELKI:
+ Environment for Developing KDD-Applications Supported by Index-Structures
+
+ Copyright (C) 2012
+ Ludwig-Maximilians-Universität München
+ Lehr- und Forschungseinheit für Datenbanksysteme
+ ELKI Development Team
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.Parameterizable;
+
+/**
+ * Interface for the statistical test used by HiCS.
+ *
+ * Provides a single method that calculates the deviation between two data
+ * samples, given as arrays of double values
+ *
+ * @author Jan Brusis
+ * @author Erich Schubert
+ */
+public interface GoodnessOfFitTest extends Parameterizable {
+ /**
+ * Measure the deviation of a full sample from a conditional sample.
+ *
+ * Sample arrays *may* be modified, e.g. sorted, by the test.
+ *
+ * @param fullSample Full sample
+ * @param conditionalSample Conditional sample
+ *
+ * @return Deviation
+ */
+ public double deviation(double[] fullSample, double[] conditionalSample);
+}
diff --git a/src/de/lmu/ifi/dbs/elki/math/statistics/tests/KolmogorovSmirnovTest.java b/src/de/lmu/ifi/dbs/elki/math/statistics/tests/KolmogorovSmirnovTest.java new file mode 100644 index 00000000..236f26b8 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/math/statistics/tests/KolmogorovSmirnovTest.java @@ -0,0 +1,117 @@ +package de.lmu.ifi.dbs.elki.math.statistics.tests;
+
+/*
+ This file is part of ELKI:
+ Environment for Developing KDD-Applications Supported by Index-Structures
+
+ Copyright (C) 2012
+ Ludwig-Maximilians-Universität München
+ Lehr- und Forschungseinheit für Datenbanksysteme
+ ELKI Development Team
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import java.util.Arrays;
+
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer;
+
+/**
+ * Kolmogorov-Smirnov test.
+ *
+ * Class that tests two given real-valued data samples on whether they might
+ * have originated from the same underlying distribution using the
+ * Kolmogorov-Smirnov test statistic that compares the two empirical cumulative
+ * distribution functions. The KS statistic is defined as the maximum absolute
+ * difference of the empirical CDFs.
+ *
+ * @author Erich Schubert
+ */
+public class KolmogorovSmirnovTest implements GoodnessOfFitTest {
+ /**
+ * Static instance
+ */
+ public static KolmogorovSmirnovTest STATIC = new KolmogorovSmirnovTest();
+
+ /**
+ * Constructor.
+ */
+ public KolmogorovSmirnovTest() {
+ super();
+ }
+
+ @Override
+ public double deviation(double[] fullSample, double[] conditionalSample) {
+ Arrays.sort(fullSample);
+ Arrays.sort(conditionalSample);
+ return calculateTestStatistic(fullSample, conditionalSample);
+ }
+
+ /**
+ * Calculates the maximum distance between the two empirical CDFs of two data
+ * samples. The sample positions and CDFs must be synchronized!
+ *
+ * @param sample1 first data sample positions
+ * @param sample2 second data sample positions
+ * @return the largest distance between both functions
+ */
+ public static double calculateTestStatistic(double[] sample1, double[] sample2) {
+ double maximum = 0.0;
+
+ int index1 = 0, index2 = 0;
+ double cdf1 = 0, cdf2 = 0;
+
+ // Parallel iteration over both curves. We can stop if we reach either end,
+ // As the difference can then only decrease!
+ while(index1 < sample1.length && index2 < sample2.length) {
+ // Next (!) positions
+ final double x1 = sample1[index1], x2 = sample2[index2];
+ // Advance on first curve
+ if(x1 <= x2) {
+ index1++;
+ // Handle multiple points with same x:
+ while(index1 < sample1.length && sample1[index1] == x1) {
+ index1++;
+ }
+ cdf1 = ((double) index1) / sample1.length;
+ }
+ // Advance on second curve
+ if(x1 >= x2) {
+ index2++;
+ // Handle multiple points with same x:
+ while(index2 < sample2.length && sample2[index2] == x2) {
+ index2++;
+ }
+ cdf2 = ((double) index2) / sample2.length;
+ }
+ maximum = Math.max(maximum, Math.abs(cdf1 - cdf2));
+ }
+
+ return maximum;
+ }
+
+ /**
+ * Parameterizer, to use the static instance.
+ *
+ * @author Erich Schubert
+ *
+ * @apiviz.exclude
+ */
+ public static class Parameterizer extends AbstractParameterizer {
+ @Override
+ protected KolmogorovSmirnovTest makeInstance() {
+ return STATIC;
+ }
+ }
+}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/math/statistics/tests/WelchTTest.java b/src/de/lmu/ifi/dbs/elki/math/statistics/tests/WelchTTest.java new file mode 100644 index 00000000..554a22db --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/math/statistics/tests/WelchTTest.java @@ -0,0 +1,124 @@ +package de.lmu.ifi.dbs.elki.math.statistics.tests;
+
+/*
+ This file is part of ELKI:
+ Environment for Developing KDD-Applications Supported by Index-Structures
+
+ Copyright (C) 2012
+ Ludwig-Maximilians-Universität München
+ Lehr- und Forschungseinheit für Datenbanksysteme
+ ELKI Development Team
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import de.lmu.ifi.dbs.elki.math.MeanVariance;
+import de.lmu.ifi.dbs.elki.math.statistics.distribution.StudentsTDistribution;
+import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer;
+
+/**
+ * Calculates a test statistic according to Welch's t test for two samples
+ * Supplies methods for calculating the degrees of freedom according to the
+ * Welch-Satterthwaite Equation. Also directly calculates a two-sided p-value
+ * for the underlying t-distribution
+ *
+ * @author Jan Brusis
+ * @author Erich Schubert
+ *
+ * @apiviz.uses StudentsTDistribution
+ */
+public class WelchTTest implements GoodnessOfFitTest {
+ /**
+ * Static instance.
+ */
+ public static final WelchTTest STATIC = new WelchTTest();
+
+ /**
+ * Constructor.
+ */
+ public WelchTTest() {
+ super();
+ }
+
+ @Override
+ public double deviation(double[] sample1, double[] sample2) {
+ MeanVariance mv1 = new MeanVariance(), mv2 = new MeanVariance();
+ for(double d : sample1) {
+ mv1.put(d);
+ }
+ for(double d : sample2) {
+ mv2.put(d);
+ }
+
+ final double t = calculateTestStatistic(mv1, mv2);
+ final int v = calculateDOF(mv1, mv2);
+ return 1 - calculatePValue(t, v);
+ }
+
+ /**
+ * Calculate the statistic of Welch's t test using statistical moments of the
+ * provided data samples
+ *
+ * @param mv1 Mean and variance of first sample
+ * @param mv2 Mean and variance of second sample
+ * @return Welch's t statistic
+ */
+ public static double calculateTestStatistic(MeanVariance mv1, MeanVariance mv2) {
+ final double delta = mv1.getMean() - mv2.getMean();
+ final double relvar1 = mv1.getSampleVariance() / mv1.getCount();
+ final double relvar2 = mv2.getSampleVariance() / mv2.getCount();
+ return delta / Math.sqrt(relvar1 + relvar2);
+ }
+
+ /**
+ * Calculates the degree of freedom according to Welch-Satterthwaite
+ *
+ * @param mv1 Mean and variance of first sample
+ * @param mv2 Mean and variance of second sample
+ * @return Estimated degrees of freedom.
+ */
+ public static int calculateDOF(MeanVariance mv1, MeanVariance mv2) {
+ final double relvar1 = mv1.getSampleVariance() / mv1.getCount();
+ final double relvar2 = mv2.getSampleVariance() / mv2.getCount();
+ final double wvariance = relvar1 + relvar2;
+ final double div = relvar1 * relvar1 / (mv1.getCount() - 1) + relvar2 * relvar2 / (mv2.getCount() - 1);
+ return (int) (wvariance * wvariance / div);
+ }
+
+ /**
+ * Calculates the two-sided p-Value of the underlying t-Distribution with v
+ * degrees of freedom
+ *
+ * @param t Integration limit
+ * @param v Degrees of freedom
+ * @return p-Value
+ */
+ public static double calculatePValue(double t, int v) {
+ return 2 * (1 - StudentsTDistribution.cdf(Math.abs(t), v));
+ }
+
+ /**
+ * Parameterizer, to use the static instance.
+ *
+ * @author Erich Schubert
+ *
+ * @apiviz.exclude
+ */
+ public static class Parameterizer extends AbstractParameterizer {
+ @Override
+ protected WelchTTest makeInstance() {
+ return STATIC;
+ }
+ }
+}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/math/statistics/tests/package-info.java b/src/de/lmu/ifi/dbs/elki/math/statistics/tests/package-info.java new file mode 100644 index 00000000..3e39a519 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/math/statistics/tests/package-info.java @@ -0,0 +1,26 @@ +/** + * <p>Statistical tests.</p> + */ +/* +This file is part of ELKI: +Environment for Developing KDD-Applications Supported by Index-Structures + +Copyright (C) 2012 +Ludwig-Maximilians-Universität München +Lehr- und Forschungseinheit für Datenbanksysteme +ELKI Development Team + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ +package de.lmu.ifi.dbs.elki.math.statistics.tests;
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/result/KMLOutputHandler.java b/src/de/lmu/ifi/dbs/elki/result/KMLOutputHandler.java index 7939d0a2..25734bdc 100644 --- a/src/de/lmu/ifi/dbs/elki/result/KMLOutputHandler.java +++ b/src/de/lmu/ifi/dbs/elki/result/KMLOutputHandler.java @@ -45,6 +45,7 @@ import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.logging.Logging; @@ -227,7 +228,8 @@ public class KMLOutputHandler implements ResultHandler, Parameterizable { writeNewlineOnDebug(out); } } - for(DBID id : outlierResult.getOrdering().iter(ids)) { + for (DBIDIter iter = outlierResult.getOrdering().iter(ids).iter(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); Double score = scores.get(id); PolygonsObject poly = polys.get(id); String label = labels.get(id); diff --git a/src/de/lmu/ifi/dbs/elki/result/OrderingFromDataStore.java b/src/de/lmu/ifi/dbs/elki/result/OrderingFromDataStore.java index 27178fe5..7fa1a224 100644 --- a/src/de/lmu/ifi/dbs/elki/result/OrderingFromDataStore.java +++ b/src/de/lmu/ifi/dbs/elki/result/OrderingFromDataStore.java @@ -30,8 +30,6 @@ import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableUtil; /** * Result class providing an ordering backed by a hashmap. @@ -118,7 +116,7 @@ public class OrderingFromDataStore<T extends Comparable<T>> extends BasicResult } @Override - public IterableIterator<DBID> iter(DBIDs ids) { + public ArrayModifiableDBIDs iter(DBIDs ids) { ArrayModifiableDBIDs sorted = DBIDUtil.newArray(ids); if(comparator != null) { sorted.sort(new DerivedComparator()); @@ -126,7 +124,7 @@ public class OrderingFromDataStore<T extends Comparable<T>> extends BasicResult else { sorted.sort(new ImpliedComparator()); } - return IterableUtil.fromIterable(sorted); + return sorted; } /** diff --git a/src/de/lmu/ifi/dbs/elki/result/OrderingResult.java b/src/de/lmu/ifi/dbs/elki/result/OrderingResult.java index 4cd01721..8e4e9e5f 100644 --- a/src/de/lmu/ifi/dbs/elki/result/OrderingResult.java +++ b/src/de/lmu/ifi/dbs/elki/result/OrderingResult.java @@ -23,9 +23,8 @@ package de.lmu.ifi.dbs.elki.result; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; /** * Interface for a result providing an object ordering. @@ -46,5 +45,5 @@ public interface OrderingResult extends Result { * @param ids Collection of ids. * @return iterator for sorted array of ids */ - public IterableIterator<DBID> iter(DBIDs ids); + public ArrayModifiableDBIDs iter(DBIDs ids); } diff --git a/src/de/lmu/ifi/dbs/elki/result/ResultUtil.java b/src/de/lmu/ifi/dbs/elki/result/ResultUtil.java index 1666d6fc..921e7dd2 100644 --- a/src/de/lmu/ifi/dbs/elki/result/ResultUtil.java +++ b/src/de/lmu/ifi/dbs/elki/result/ResultUtil.java @@ -30,21 +30,14 @@ import java.util.Iterator; import java.util.List; import de.lmu.ifi.dbs.elki.algorithm.clustering.ClusteringAlgorithm; -import de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.ByLabelHierarchicalClustering; -import de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.TrivialAllInOne; +import de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.ByLabelOrAllInOneClustering; import de.lmu.ifi.dbs.elki.data.Clustering; import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.data.model.Model; -import de.lmu.ifi.dbs.elki.data.type.NoSupportedDataTypeException; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; import de.lmu.ifi.dbs.elki.utilities.ClassGenericsUtil; -import de.lmu.ifi.dbs.elki.utilities.iterator.EmptyIterator; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; -import de.lmu.ifi.dbs.elki.utilities.iterator.MergedIterator; -import de.lmu.ifi.dbs.elki.utilities.iterator.OneItemIterator; -import de.lmu.ifi.dbs.elki.utilities.iterator.TypeFilterIterator; /** * Utilities for handling result objects @@ -195,7 +188,8 @@ public class ResultUtil { res.add((C) restrictionClass.cast(r)); } if(r instanceof HierarchicalResult) { - for(Result result : ((HierarchicalResult) r).getHierarchy().iterDescendants(r)) { + for(Iterator<Result> iter = ((HierarchicalResult) r).getHierarchy().iterDescendants(r); iter.hasNext();) { + Result result = iter.next(); if(restrictionClass.isInstance(result)) { res.add((C) restrictionClass.cast(result)); } @@ -204,38 +198,6 @@ public class ResultUtil { return res; } - @SuppressWarnings("unchecked") - public static <C extends Result> IterableIterator<C> filteredResults(Result r, Class<?> restrictionClass) { - final Class<C> rc = (Class<C>) restrictionClass; - // Include the current item - IterableIterator<C> curIter; - if(rc.isInstance(r)) { - curIter = new OneItemIterator<C>(rc.cast(r)); - } - else { - curIter = null; - } - if(r instanceof HierarchicalResult) { - ResultHierarchy hier = ((HierarchicalResult) r).getHierarchy(); - final Iterable<Result> iterDescendants = hier.iterDescendants(r); - final IterableIterator<C> others = new TypeFilterIterator<Result, C>(rc, iterDescendants); - if(curIter != null) { - return new MergedIterator<C>(curIter, others); - } - else { - return others; - } - } - else { - if(curIter != null) { - return curIter; - } - else { - return EmptyIterator.STATIC(); - } - } - } - /** * Ensure that the result contains at least one Clustering. * @@ -246,15 +208,9 @@ public class ResultUtil { public static <O> void ensureClusteringResult(final Database db, final Result result) { Collection<Clustering<?>> clusterings = ResultUtil.filterResults(result, Clustering.class); if(clusterings.size() == 0) { - try { - ClusteringAlgorithm<Clustering<Model>> split = new ByLabelHierarchicalClustering(); - Clustering<Model> c = split.run(db); - addChildResult(db, c); - } - catch(NoSupportedDataTypeException e) { - Clustering<Model> c = (new TrivialAllInOne()).run(db); - addChildResult(db, c); - } + ClusteringAlgorithm<Clustering<Model>> split = new ByLabelOrAllInOneClustering(); + Clustering<Model> c = split.run(db); + addChildResult(db, c); } } @@ -265,9 +221,9 @@ public class ResultUtil { * @return selection result */ public static SelectionResult ensureSelectionResult(final Database db) { - Iterator<SelectionResult> selections = ResultUtil.filteredResults(db, SelectionResult.class); - if(selections.hasNext()) { - return selections.next(); + List<SelectionResult> selections = ResultUtil.filterResults(db, SelectionResult.class); + if(!selections.isEmpty()) { + return selections.get(0); } SelectionResult sel = new SelectionResult(); addChildResult(db, sel); @@ -323,9 +279,9 @@ public class ResultUtil { * @return Database */ public static Database findDatabase(Result baseResult) { - final IterableIterator<Database> iter = filteredResults(baseResult, Database.class); - if(iter.hasNext()) { - return iter.next(); + final List<Database> dbs = filterResults(baseResult, Database.class); + if(!dbs.isEmpty()) { + return dbs.get(0); } else { return null; diff --git a/src/de/lmu/ifi/dbs/elki/result/ScalesResult.java b/src/de/lmu/ifi/dbs/elki/result/ScalesResult.java index 89cd4af2..7def7b53 100644 --- a/src/de/lmu/ifi/dbs/elki/result/ScalesResult.java +++ b/src/de/lmu/ifi/dbs/elki/result/ScalesResult.java @@ -44,11 +44,20 @@ public class ScalesResult extends BasicResult { /** * Constructor. * - * @param relation + * @param relation Relation to use */ public ScalesResult(Relation<? extends NumberVector<?, ?>> relation) { + this(Scales.calcScales(relation)); + } + + /** + * Constructor. + * + * @param scales Relation scales to use + */ + public ScalesResult(LinearScale[] scales) { super("scales", "scales"); - scales = Scales.calcScales(relation); + this.scales = scales; } /** diff --git a/src/de/lmu/ifi/dbs/elki/result/optics/ClusterOrderResult.java b/src/de/lmu/ifi/dbs/elki/result/optics/ClusterOrderResult.java index 6eea3415..66b53dd2 100644 --- a/src/de/lmu/ifi/dbs/elki/result/optics/ClusterOrderResult.java +++ b/src/de/lmu/ifi/dbs/elki/result/optics/ClusterOrderResult.java @@ -24,15 +24,20 @@ package de.lmu.ifi.dbs.elki.result.optics; */ import java.util.ArrayList; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; +import de.lmu.ifi.dbs.elki.database.datastore.DataStore; +import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; +import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; +import de.lmu.ifi.dbs.elki.database.datastore.WritableDataStore; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; @@ -43,9 +48,6 @@ import de.lmu.ifi.dbs.elki.result.IterableResult; import de.lmu.ifi.dbs.elki.result.OrderingResult; import de.lmu.ifi.dbs.elki.result.ResultAdapter; import de.lmu.ifi.dbs.elki.result.ResultHierarchy; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIteratorAdapter; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableUtil; /** * Class to store the result of an ordering clustering algorithm such as OPTICS. @@ -70,7 +72,7 @@ public class ClusterOrderResult<D extends Distance<D>> extends BasicResult imple /** * Map of object IDs to their cluster order entry */ - private HashMap<DBID, ClusterOrderEntry<D>> map; + private WritableDataStore<ClusterOrderEntry<D>> map; /** * The DBIDs we are defined for @@ -86,8 +88,8 @@ public class ClusterOrderResult<D extends Distance<D>> extends BasicResult imple public ClusterOrderResult(String name, String shortname) { super(name, shortname); clusterOrder = new ArrayList<ClusterOrderEntry<D>>(); - map = new HashMap<DBID, ClusterOrderEntry<D>>(); dbids = DBIDUtil.newHashSet(); + map = DataStoreUtil.makeStorage(dbids, DataStoreFactory.HINT_DB, ClusterOrderEntry.class); addChildResult(new ClusterOrderAdapter(clusterOrder)); addChildResult(new ReachabilityDistanceAdapter(map, dbids)); @@ -174,23 +176,21 @@ public class ClusterOrderResult<D extends Distance<D>> extends BasicResult imple public DBIDs getDBIDs() { return dbids; } - + /** * Use the cluster order to sort the given collection ids. * * Implementation of the {@link OrderingResult} interface. */ @Override - public IterableIterator<DBID> iter(DBIDs ids) { + public ArrayModifiableDBIDs iter(DBIDs ids) { ArrayModifiableDBIDs res = DBIDUtil.newArray(ids.size()); for(ClusterOrderEntry<D> e : clusterOrder) { if(ids.contains(e.getID())) { res.add(e.getID()); } } - - // TODO: elements in ids that are not in clusterOrder are lost! - return new IterableIteratorAdapter<DBID>(res); + return res; } @Override @@ -213,7 +213,7 @@ public class ClusterOrderResult<D extends Distance<D>> extends BasicResult imple /** * Access reference. */ - private HashMap<DBID, ClusterOrderEntry<D>> map; + private DataStore<ClusterOrderEntry<D>> map; /** * DBIDs @@ -226,14 +226,14 @@ public class ClusterOrderResult<D extends Distance<D>> extends BasicResult imple * @param map Map that stores the results. * @param dbids DBIDs we are defined for. */ - public ReachabilityDistanceAdapter(HashMap<DBID, ClusterOrderEntry<D>> map, DBIDs dbids) { + public ReachabilityDistanceAdapter(DataStore<ClusterOrderEntry<D>> map, DBIDs dbids) { super(); this.map = map; this.dbids = dbids; } @Override - public D get(DBID objID) { + public D get(DBIDRef objID) { return map.get(objID).getReachability(); } @@ -253,8 +253,8 @@ public class ClusterOrderResult<D extends Distance<D>> extends BasicResult imple } @Override - public IterableIterator<DBID> iterDBIDs() { - return IterableUtil.fromIterator(dbids.iterator()); + public DBIDIter iterDBIDs() { + return dbids.iter(); } @Override @@ -268,12 +268,12 @@ public class ClusterOrderResult<D extends Distance<D>> extends BasicResult imple } @Override - public void set(DBID id, D val) { + public void set(DBIDRef id, D val) { throw new UnsupportedOperationException(); } @Override - public void delete(DBID id) { + public void delete(DBIDRef id) { throw new UnsupportedOperationException(); } @@ -302,7 +302,7 @@ public class ClusterOrderResult<D extends Distance<D>> extends BasicResult imple /** * Access reference. */ - private HashMap<DBID, ClusterOrderEntry<D>> map; + private DataStore<ClusterOrderEntry<D>> map; /** * Database IDs @@ -315,14 +315,14 @@ public class ClusterOrderResult<D extends Distance<D>> extends BasicResult imple * @param map Map that stores the results. * @param dbids DBIDs we are defined for */ - public PredecessorAdapter(HashMap<DBID, ClusterOrderEntry<D>> map, DBIDs dbids) { + public PredecessorAdapter(DataStore<ClusterOrderEntry<D>> map, DBIDs dbids) { super(); this.map = map; this.dbids = dbids; } @Override - public DBID get(DBID objID) { + public DBID get(DBIDRef objID) { return map.get(objID).getPredecessorID(); } @@ -342,8 +342,8 @@ public class ClusterOrderResult<D extends Distance<D>> extends BasicResult imple } @Override - public IterableIterator<DBID> iterDBIDs() { - return IterableUtil.fromIterator(dbids.iterator()); + public DBIDIter iterDBIDs() { + return dbids.iter(); } @Override @@ -357,12 +357,12 @@ public class ClusterOrderResult<D extends Distance<D>> extends BasicResult imple } @Override - public void set(DBID id, DBID val) { + public void set(DBIDRef id, DBID val) { throw new UnsupportedOperationException(); } @Override - public void delete(DBID id) { + public void delete(DBIDRef id) { throw new UnsupportedOperationException(); } diff --git a/src/de/lmu/ifi/dbs/elki/result/optics/DoubleDistanceClusterOrderEntry.java b/src/de/lmu/ifi/dbs/elki/result/optics/DoubleDistanceClusterOrderEntry.java index 8f42e1eb..533384aa 100644 --- a/src/de/lmu/ifi/dbs/elki/result/optics/DoubleDistanceClusterOrderEntry.java +++ b/src/de/lmu/ifi/dbs/elki/result/optics/DoubleDistanceClusterOrderEntry.java @@ -80,7 +80,7 @@ public class DoubleDistanceClusterOrderEntry implements Comparable<ClusterOrderE final ClusterOrderEntry<?> that = (ClusterOrderEntry<?>) o; // Compare by ID only, for UpdatableHeap! - return objectID.equals(that.getID()); + return objectID.sameDBID(that.getID()); } /** diff --git a/src/de/lmu/ifi/dbs/elki/result/optics/GenericClusterOrderEntry.java b/src/de/lmu/ifi/dbs/elki/result/optics/GenericClusterOrderEntry.java index c9ff0564..47ee16e5 100644 --- a/src/de/lmu/ifi/dbs/elki/result/optics/GenericClusterOrderEntry.java +++ b/src/de/lmu/ifi/dbs/elki/result/optics/GenericClusterOrderEntry.java @@ -81,7 +81,7 @@ public class GenericClusterOrderEntry<D extends Distance<D>> implements Comparab final ClusterOrderEntry<?> that = (ClusterOrderEntry<?>) o; // Compare by ID only, for UpdatableHeap! - return objectID.equals(that.getID()); + return objectID.sameDBID(that.getID()); } /** diff --git a/src/de/lmu/ifi/dbs/elki/result/outlier/OrderingFromRelation.java b/src/de/lmu/ifi/dbs/elki/result/outlier/OrderingFromRelation.java index 5d35c74a..b272bd56 100644 --- a/src/de/lmu/ifi/dbs/elki/result/outlier/OrderingFromRelation.java +++ b/src/de/lmu/ifi/dbs/elki/result/outlier/OrderingFromRelation.java @@ -31,8 +31,6 @@ import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.result.OrderingResult; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIteratorAdapter; /** * Ordering obtained from an outlier score. @@ -79,10 +77,10 @@ public class OrderingFromRelation implements OrderingResult { } @Override - public IterableIterator<DBID> iter(DBIDs ids) { + public ArrayModifiableDBIDs iter(DBIDs ids) { ArrayModifiableDBIDs sorted = DBIDUtil.newArray(ids); sorted.sort(new ImpliedComparator()); - return new IterableIteratorAdapter<DBID>(sorted); + return sorted; } @Override diff --git a/src/de/lmu/ifi/dbs/elki/result/textwriter/TextWriter.java b/src/de/lmu/ifi/dbs/elki/result/textwriter/TextWriter.java index d947547f..85afbc8e 100644 --- a/src/de/lmu/ifi/dbs/elki/result/textwriter/TextWriter.java +++ b/src/de/lmu/ifi/dbs/elki/result/textwriter/TextWriter.java @@ -45,6 +45,7 @@ import de.lmu.ifi.dbs.elki.data.model.Model; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.datasource.bundle.SingleObjectBundle; @@ -233,7 +234,8 @@ public class TextWriter { // collect other results { - for(Result res : ResultUtil.filteredResults(r, Result.class)) { + List<Result> results = ResultUtil.filterResults(r, Result.class); + for(Result res : results) { if(res instanceof Database) { continue; } @@ -376,10 +378,8 @@ public class TextWriter { // print ids. DBIDs ids = clus.getIDs(); - Iterator<DBID> iter = ids.iterator(); - - while(iter.hasNext()) { - printObject(out, db, iter.next(), ra); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + printObject(out, db, iter.getDBID(), ra); } out.commentPrintSeparator(); out.flush(); @@ -418,9 +418,8 @@ public class TextWriter { TextWriterStream out = new TextWriterStream(outStream, writers); printSettings(out, sr); - Iterator<DBID> i = or.iter(or.getDBIDs()); - while(i.hasNext()) { - printObject(out, db, i.next(), ra); + for (DBIDIter i = or.iter(or.getDBIDs()).iter(); i.valid(); i.advance()) { + printObject(out, db, i.getDBID(), ra); } out.commentPrintSeparator(); out.flush(); diff --git a/src/de/lmu/ifi/dbs/elki/utilities/Base64.java b/src/de/lmu/ifi/dbs/elki/utilities/Base64.java index da3b52a7..9ea9a024 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/Base64.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/Base64.java @@ -1,4 +1,26 @@ package de.lmu.ifi.dbs.elki.utilities; +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ import java.lang.reflect.Constructor; import java.lang.reflect.Method; diff --git a/src/de/lmu/ifi/dbs/elki/utilities/DatabaseUtil.java b/src/de/lmu/ifi/dbs/elki/utilities/DatabaseUtil.java index 23eba080..365e3847 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/DatabaseUtil.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/DatabaseUtil.java @@ -45,6 +45,7 @@ import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.relation.ConvertToStringView; @@ -181,8 +182,8 @@ public final class DatabaseUtil { for(int d = 1; d <= centroid.getDimensionality(); d++) { double mu = centroid.doubleValue(d); - for(Iterator<DBID> it = database.iterDBIDs(); it.hasNext();) { - NumberVector<?, ?> o = database.get(it.next()); + for(DBIDIter it = database.iterDBIDs(); it.valid(); it.advance()) { + NumberVector<?, ?> o = database.get(it); double diff = o.doubleValue(d) - mu; variances[d - 1] += diff * diff; } @@ -220,8 +221,8 @@ public final class DatabaseUtil { for(int d = 1; d <= centroid.getDimensionality(); d++) { double mu = centroid.doubleValue(d); - for(DBID id : ids) { - NumberVector<?, ?> o = database.get(id); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + NumberVector<?, ?> o = database.get(iter); double diff = o.doubleValue(d) - mu; variances[d - 1] += diff * diff; } @@ -247,10 +248,10 @@ public final class DatabaseUtil { mins[i] = Double.MAX_VALUE; maxs[i] = -Double.MAX_VALUE; } - for(DBID it : database.iterDBIDs()) { - NV o = database.get(it); + for(DBIDIter iditer = database.iterDBIDs(); iditer.valid(); iditer.advance()) { + final NV o = database.get(iditer); for(int d = 0; d < dim; d++) { - double v = o.doubleValue(d + 1); + final double v = o.doubleValue(d + 1); mins[d] = Math.min(mins[d], v); maxs[d] = Math.max(maxs[d], v); } @@ -300,8 +301,8 @@ public final class DatabaseUtil { public static <V extends NumberVector<?, ?>> double exactMedian(Relation<V> relation, DBIDs ids, int dimension) { final double[] vals = new double[ids.size()]; int i = 0; - for(DBID id : ids) { - vals[i] = relation.get(id).doubleValue(dimension); + for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) { + vals[i] = relation.get(iter).doubleValue(dimension); i++; } Arrays.sort(vals); @@ -398,8 +399,8 @@ public final class DatabaseUtil { */ public static SortedSet<ClassLabel> getClassLabels(Relation<? extends ClassLabel> database) { SortedSet<ClassLabel> labels = new TreeSet<ClassLabel>(); - for(Iterator<DBID> iter = database.iterDBIDs(); iter.hasNext();) { - labels.add(database.get(iter.next())); + for(DBIDIter it = database.iterDBIDs(); it.valid(); it.advance()) { + labels.add(database.get(it)); } return labels; } @@ -425,10 +426,7 @@ public final class DatabaseUtil { */ @SuppressWarnings("unchecked") public static <O> Class<? extends O> guessObjectClass(Relation<O> database) { - for(DBID id : database.iterDBIDs()) { - return (Class<? extends O>) database.get(id).getClass(); - } - return null; + return (Class<? extends O>) database.get(database.iterDBIDs()).getClass(); } /** @@ -445,16 +443,17 @@ public final class DatabaseUtil { */ public static <O> Class<?> getBaseObjectClassExpensive(Relation<O> database) { List<Class<?>> candidates = new ArrayList<Class<?>>(); - Iterator<DBID> iditer = database.iterDBIDs(); + DBIDIter iditer = database.iterDBIDs(); // empty database?! - if(!iditer.hasNext()) { + if(!iditer.valid()) { return null; } // put first class into result set. - candidates.add(database.get(iditer.next()).getClass()); + candidates.add(database.get(iditer).getClass()); + iditer.advance(); // other objects - while(iditer.hasNext()) { - Class<?> newcls = database.get(iditer.next()).getClass(); + for(; iditer.valid(); iditer.advance()) { + Class<?> newcls = database.get(iditer).getClass(); // validate all candidates Iterator<Class<?>> ci = candidates.iterator(); while(ci.hasNext()) { @@ -509,9 +508,9 @@ public final class DatabaseUtil { return DBIDUtil.newArray(); } ArrayModifiableDBIDs ret = DBIDUtil.newArray(); - for(DBID objid : relation.iterDBIDs()) { - if(name_pattern.matcher(relation.get(objid)).matches()) { - ret.add(objid); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + if(name_pattern.matcher(relation.get(iditer)).find()) { + ret.add(iditer); } } return ret; @@ -556,7 +555,7 @@ public final class DatabaseUtil { /** * The real iterator. */ - final Iterator<DBID> iter; + final DBIDIter iter; /** * The database we use @@ -569,7 +568,7 @@ public final class DatabaseUtil { * @param iter Original iterator. * @param database Database */ - public RelationObjectIterator(Iterator<DBID> iter, Relation<? extends O> database) { + public RelationObjectIterator(DBIDIter iter, Relation<? extends O> database) { super(); this.iter = iter; this.database = database; @@ -588,18 +587,19 @@ public final class DatabaseUtil { @Override public boolean hasNext() { - return iter.hasNext(); + return iter.valid(); } @Override public O next() { - DBID id = iter.next(); - return database.get(id); + O ret = database.get(iter); + iter.advance(); + return ret; } @Override public void remove() { - iter.remove(); + throw new UnsupportedOperationException(); } } diff --git a/src/de/lmu/ifi/dbs/elki/utilities/InspectionUtil.java b/src/de/lmu/ifi/dbs/elki/utilities/InspectionUtil.java index 66ecde64..f7bccc1a 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/InspectionUtil.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/InspectionUtil.java @@ -41,7 +41,6 @@ import java.util.jar.JarFile; import de.lmu.ifi.dbs.elki.logging.Logging; import de.lmu.ifi.dbs.elki.logging.LoggingUtil; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ClassParameter; import de.lmu.ifi.dbs.elki.utilities.pairs.Pair; @@ -253,7 +252,7 @@ public class InspectionUtil { * * @apiviz.exclude */ - static class JarClassIterator implements IterableIterator<String> { + static class JarClassIterator implements Iterator<String> { private Enumeration<JarEntry> jarentries; private String ne; @@ -321,11 +320,6 @@ public class InspectionUtil { public void remove() { throw new UnsupportedOperationException(); } - - @Override - public Iterator<String> iterator() { - return this; - } } /** @@ -335,7 +329,7 @@ public class InspectionUtil { * * @apiviz.exclude */ - static class DirClassIterator implements IterableIterator<String> { + static class DirClassIterator implements Iterator<String> { private static final String CLASS_EXT = ".class"; private static final String FACTORY_FILE_EXT = ClassParameter.FACTORY_POSTFIX + CLASS_EXT; @@ -427,11 +421,6 @@ public class InspectionUtil { public void remove() { throw new UnsupportedOperationException(); } - - @Override - public Iterator<String> iterator() { - return this; - } } /** diff --git a/src/de/lmu/ifi/dbs/elki/utilities/Util.java b/src/de/lmu/ifi/dbs/elki/utilities/Util.java index 6f79c1fa..5faf69cc 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/Util.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/Util.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.utilities; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import gnu.trove.map.hash.TIntFloatHashMap; +import gnu.trove.map.hash.TIntDoubleHashMap; import java.io.PrintStream; import java.util.ArrayList; @@ -35,7 +35,7 @@ import java.util.Random; import java.util.StringTokenizer; import de.lmu.ifi.dbs.elki.data.DoubleVector; -import de.lmu.ifi.dbs.elki.data.SparseFloatVector; +import de.lmu.ifi.dbs.elki.data.SparseNumberVector; import de.lmu.ifi.dbs.elki.utilities.datastructures.arraylike.ArrayLikeUtil; /** @@ -174,14 +174,14 @@ public final class Util { * an attribute as selected which is out of range for the given * SparseFloatVector. */ - public static SparseFloatVector project(SparseFloatVector v, BitSet selectedAttributes) { - TIntFloatHashMap values = new TIntFloatHashMap(selectedAttributes.cardinality(), 1); + public static <V extends SparseNumberVector<V, ?>> V project(V v, BitSet selectedAttributes) { + TIntDoubleHashMap values = new TIntDoubleHashMap(selectedAttributes.cardinality(), 1); for(int d = selectedAttributes.nextSetBit(0); d >= 0; d = selectedAttributes.nextSetBit(d + 1)) { - if(v.getValue(d + 1) != 0.0f) { - values.put(d, v.getValue(d + 1)); + if(v.doubleValue(d + 1) != 0.0) { + values.put(d, v.doubleValue(d + 1)); } } - SparseFloatVector projectedVector = new SparseFloatVector(values, selectedAttributes.cardinality()); + V projectedVector = v.newNumberVector(values, selectedAttributes.cardinality()); return projectedVector; } diff --git a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/QuickSelect.java b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/QuickSelect.java index 4498cf07..a191dde2 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/QuickSelect.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/QuickSelect.java @@ -3,6 +3,9 @@ package de.lmu.ifi.dbs.elki.utilities.datastructures; import java.util.Comparator; import java.util.List; +import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; +import de.lmu.ifi.dbs.elki.database.ids.DBID; + /* This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures @@ -35,6 +38,10 @@ import java.util.List; * Charles Antony Richard Hoare * * @author Erich Schubert + * + * @apiviz.uses ArrayModifiableDBIDs + * @apiviz.uses List + * @apiviz.uses Comparator */ public class QuickSelect { /** @@ -777,4 +784,175 @@ public class QuickSelect { } } } + + /** + * QuickSelect is essentially quicksort, except that we only "sort" that half + * of the array that we are interested in. + * + * Note: the array is <b>modified</b> by this. + * + * @param data Data to process + * @param comparator Comparator to use + * @param rank Rank position that we are interested in (integer!) + * @return Value at the given rank + */ + public static DBID quickSelect(ArrayModifiableDBIDs data, Comparator<? super DBID> comparator, int rank) { + quickSelect(data, comparator, 0, data.size(), rank); + return data.get(rank); + } + + /** + * Compute the median of an array efficiently using the QuickSelect method. + * + * Note: the array is <b>modified</b> by this. + * + * @param data Data to process + * @param comparator Comparator to use + * @return Median value + */ + public static DBID median(ArrayModifiableDBIDs data, Comparator<? super DBID> comparator) { + return median(data, comparator, 0, data.size()); + } + + /** + * Compute the median of an array efficiently using the QuickSelect method. + * + * On an odd length, it will return the lower element. + * + * Note: the array is <b>modified</b> by this. + * + * @param data Data to process + * @param comparator Comparator to use + * @param begin Begin of valid values + * @param end End of valid values (exclusive!) + * @return Median value + */ + public static DBID median(ArrayModifiableDBIDs data, Comparator<? super DBID> comparator, int begin, int end) { + final int length = end - begin; + assert (length > 0); + // Integer division is "floor" since we are non-negative. + final int left = begin + (length - 1) / 2; + quickSelect(data, comparator, begin, end, left); + return data.get(left); + } + + /** + * Compute the median of an array efficiently using the QuickSelect method. + * + * Note: the array is <b>modified</b> by this. + * + * @param data Data to process + * @param comparator Comparator to use + * @param quant Quantile to compute + * @return Value at quantile + */ + public static DBID quantile(ArrayModifiableDBIDs data, Comparator<? super DBID> comparator, double quant) { + return quantile(data, comparator, 0, data.size(), quant); + } + + /** + * Compute the median of an array efficiently using the QuickSelect method. + * + * It will prefer the lower element. + * + * Note: the array is <b>modified</b> by this. + * + * @param data Data to process + * @param comparator Comparator to use + * @param begin Begin of valid values + * @param end End of valid values (inclusive!) + * @param quant Quantile to compute + * @return Value at quantile + */ + public static DBID quantile(ArrayModifiableDBIDs data, Comparator<? super DBID> comparator, int begin, int end, double quant) { + final int length = end - begin; + assert (length > 0) : "Quantile on empty set?"; + // Integer division is "floor" since we are non-negative. + final double dleft = begin + (length - 1) * quant; + final int ileft = (int) Math.floor(dleft); + + quickSelect(data, comparator, begin, end, ileft); + return data.get(ileft); + } + + /** + * QuickSelect is essentially quicksort, except that we only "sort" that half + * of the array that we are interested in. + * + * @param data Data to process + * @param comparator Comparator to use + * @param start Interval start + * @param end Interval end (inclusive) + * @param rank rank position we are interested in (starting at 0) + */ + public static void quickSelect(ArrayModifiableDBIDs data, Comparator<? super DBID> comparator, int start, int end, int rank) { + // Optimization for small arrays + // This also ensures a minimum size below + if(start + SMALL > end) { + insertionSort(data, comparator, start, end); + return; + } + + // Pick pivot from three candidates: start, middle, end + // Since we compare them, we can also just "bubble sort" them. + final int middle = (start + end) / 2; + if(comparator.compare(data.get(start), data.get(middle)) > 0) { + data.swap(start, middle); + } + if(comparator.compare(data.get(start), data.get(end - 1)) > 0) { + data.swap(start, end - 1); + } + if(comparator.compare(data.get(middle), data.get(end - 1)) > 0) { + data.swap(middle, end - 1); + } + // TODO: use more candidates for larger arrays? + + final DBID pivot = data.get(middle); + // Move middle element out of the way, just before end + // (Since we already know that "end" is bigger) + data.swap(middle, end - 2); + + // Begin partitioning + int i = start + 1, j = end - 3; + // This is classic quicksort stuff + while(true) { + while(comparator.compare(data.get(i), pivot) <= 0 && i <= j) { + i++; + } + while(comparator.compare(data.get(j), pivot) >= 0 && j >= i) { + j--; + } + if(i >= j) { + break; + } + data.swap(i, j); + } + + // Move pivot (former middle element) back into the appropriate place + data.swap(i, end - 2); + + // In contrast to quicksort, we only need to recurse into the half we are + // interested in. + if(rank < i) { + quickSelect(data, comparator, start, i, rank); + } + else if(rank > i) { + quickSelect(data, comparator, i + 1, end, rank); + } + } + + /** + * Sort a small array using repetitive insertion sort. + * + * @param data Data to sort + * @param start Interval start + * @param end Interval end + */ + private static void insertionSort(ArrayModifiableDBIDs data, Comparator<? super DBID> comparator, int start, int end) { + for(int i = start + 1; i < end; i++) { + for(int j = i; j > start && comparator.compare(data.get(j - 1), data.get(j)) > 0; j--) { + data.swap(j, j - 1); + } + } + } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/arraylike/ArrayDBIDsAdapter.java b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/arraylike/ArrayDBIDsAdapter.java index 6e6e8122..5b1b92b5 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/arraylike/ArrayDBIDsAdapter.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/arraylike/ArrayDBIDsAdapter.java @@ -30,6 +30,8 @@ import de.lmu.ifi.dbs.elki.database.ids.DBID; * Use a DBID array in a generic array-like context. * * @author Erich Schubert + * + * @apiviz.uses ArrayDBIDs */ public class ArrayDBIDsAdapter implements ArrayAdapter<DBID, ArrayDBIDs> { /** diff --git a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/arraylike/ArrayLikeUtil.java b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/arraylike/ArrayLikeUtil.java index 2d29f40c..f485e214 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/arraylike/ArrayLikeUtil.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/arraylike/ArrayLikeUtil.java @@ -34,6 +34,9 @@ import de.lmu.ifi.dbs.elki.data.NumberVector; * reorganizing the objects into a real array. * * @author Erich Schubert + * + * @apiviz.landmark + * @apiviz.composedOf ArrayAdapter */ public final class ArrayLikeUtil { /** diff --git a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/arraylike/ListArrayAdapter.java b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/arraylike/ListArrayAdapter.java index 37875ab7..792fad0b 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/arraylike/ListArrayAdapter.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/arraylike/ListArrayAdapter.java @@ -1,28 +1,6 @@ package de.lmu.ifi.dbs.elki.utilities.datastructures.arraylike; import java.util.List; -/* - This file is part of ELKI: - Environment for Developing KDD-Applications Supported by Index-Structures - - Copyright (C) 2012 - Ludwig-Maximilians-Universität München - Lehr- und Forschungseinheit für Datenbanksysteme - ELKI Development Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. - */ /** * Static adapter class to use a {@link java.util.List} in an array API. diff --git a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/arraylike/SubsetNumberArrayAdapter.java b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/arraylike/SubsetNumberArrayAdapter.java new file mode 100644 index 00000000..6719b60e --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/arraylike/SubsetNumberArrayAdapter.java @@ -0,0 +1,95 @@ +package de.lmu.ifi.dbs.elki.utilities.datastructures.arraylike; +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +/** + * Subset array adapter (allows reordering and projection) + * + * @author Erich Schubert + * + * @param <T> Entry type + * @param <A> Array type + */ +public class SubsetNumberArrayAdapter<T extends Number, A> implements NumberArrayAdapter<T, A> { + /** + * Wrapped adapter + */ + NumberArrayAdapter<T, ? super A> wrapped; + + /** + * Offsets to return + */ + int[] offs; + + /** + * Constructor. + * + * @param wrapped Wrapped adapter + * @param offs Offsets + */ + public SubsetNumberArrayAdapter(NumberArrayAdapter<T, ? super A> wrapped, int[] offs) { + super(); + this.wrapped = wrapped; + this.offs = offs; + } + + @Override + public int size(A array) { + return offs.length; + } + + @Override + public T get(A array, int off) throws IndexOutOfBoundsException { + return wrapped.get(array, offs[off]); + } + + @Override + public double getDouble(A array, int off) throws IndexOutOfBoundsException { + return wrapped.getDouble(array, offs[off]); + } + + @Override + public float getFloat(A array, int off) throws IndexOutOfBoundsException { + return wrapped.getFloat(array, offs[off]); + } + + @Override + public int getInteger(A array, int off) throws IndexOutOfBoundsException { + return wrapped.getInteger(array, offs[off]); + } + + @Override + public short getShort(A array, int off) throws IndexOutOfBoundsException { + return wrapped.getShort(array, offs[off]); + } + + @Override + public long getLong(A array, int off) throws IndexOutOfBoundsException { + return wrapped.getLong(array, offs[off]); + } + + @Override + public byte getByte(A array, int off) throws IndexOutOfBoundsException { + return wrapped.getByte(array, offs[off]); + } +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/heap/KNNHeap.java b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/heap/KNNHeap.java index a1706c84..6c59123b 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/heap/KNNHeap.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/heap/KNNHeap.java @@ -27,7 +27,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.database.query.GenericDistanceResultPair; import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; @@ -130,9 +130,9 @@ public class KNNHeap<D extends Distance<D>> extends TiedTopBoundedHeap<DistanceR * @param id ID number * @return success code */ - public boolean add(D distance, DBID id) { + public boolean add(D distance, DBIDRef id) { if(size() < maxsize || peek().getDistance().compareTo(distance) >= 0) { - return super.add(new GenericDistanceResultPair<D>(distance, id)); + return super.add(new GenericDistanceResultPair<D>(distance, id.getDBID())); } return true; /* "success" */ } diff --git a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/heap/KNNList.java b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/heap/KNNList.java index b19612f2..3e6ecc9d 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/heap/KNNList.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/heap/KNNList.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.utilities.datastructures.heap; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.AbstractCollection; +import java.util.AbstractList; import java.util.Iterator; import java.util.List; import java.util.Queue; @@ -41,7 +41,7 @@ import de.lmu.ifi.dbs.elki.distance.distancevalue.Distance; * * @param <D> Distance type */ -public class KNNList<D extends Distance<D>> extends AbstractCollection<DistanceResultPair<D>> implements KNNResult<D> { +public class KNNList<D extends Distance<D>> extends AbstractList<DistanceResultPair<D>> implements KNNResult<D> { /** * The value of k this was materialized for. */ @@ -96,47 +96,26 @@ public class KNNList<D extends Distance<D>> extends AbstractCollection<DistanceR assert (heap.size() == 0); } - /** - * Get the K parameter. - * - * @return K - */ + @Override public int getK() { return k; } - /** - * Get the distance to the k nearest neighbor, or maxdist otherwise. - * - * @return Maximum distance - */ @Override public D getKNNDistance() { return get(getK() - 1).getDistance(); } - /** - * View as ArrayDBIDs - * - * @return Static DBIDs - */ @Override public ArrayDBIDs asDBIDs() { return KNNUtil.asDBIDs(this); } - /** - * View as list of distances - * - * @return List of distances view - */ @Override public List<D> asDistanceList() { return KNNUtil.asDistanceList(this); } - /* Make the list unmodifiable! */ - @Override public String toString() { StringBuffer buf = new StringBuffer(); diff --git a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/heap/UpdatableHeap.java b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/heap/UpdatableHeap.java index 745d82cc..05858981 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/heap/UpdatableHeap.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/heap/UpdatableHeap.java @@ -223,6 +223,7 @@ public class UpdatableHeap<O> extends Heap<O> { * @param pos insertion position * @param elem Element to insert */ + @Override @SuppressWarnings("unchecked") protected void heapifyUpComparable(int pos, Object elem) { final Comparable<Object> cur = (Comparable<Object>) elem; // queue[pos]; @@ -247,6 +248,7 @@ public class UpdatableHeap<O> extends Heap<O> { * @param pos insertion position * @param cur Element to insert */ + @Override protected void heapifyUpComparator(int pos, Object cur) { while(pos > 0) { final int parent = (pos - 1) >>> 1; diff --git a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/hierarchy/Hierarchical.java b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/hierarchy/Hierarchical.java index 697ac640..29909069 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/hierarchy/Hierarchical.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/hierarchy/Hierarchical.java @@ -23,10 +23,9 @@ package de.lmu.ifi.dbs.elki.utilities.datastructures.hierarchy; along with this program. If not, see <http://www.gnu.org/licenses/>. */ +import java.util.Iterator; import java.util.List; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; - /** * Interface for objects with an <b>internal</b> hierarchy interface. @@ -65,7 +64,7 @@ public interface Hierarchical<O> { * * @return iterator for descendants */ - public IterableIterator<O> iterDescendants(); + public Iterator<O> iterDescendants(); /** * Get number of parents @@ -87,5 +86,5 @@ public interface Hierarchical<O> { * * @return iterator for ancestors */ - public IterableIterator<O> iterAncestors(); + public Iterator<O> iterAncestors(); }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/hierarchy/Hierarchy.java b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/hierarchy/Hierarchy.java index 3a3c4d45..0a16e9b7 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/hierarchy/Hierarchy.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/hierarchy/Hierarchy.java @@ -23,10 +23,9 @@ package de.lmu.ifi.dbs.elki.utilities.datastructures.hierarchy; along with this program. If not, see <http://www.gnu.org/licenses/>. */ +import java.util.Iterator; import java.util.List; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; - /** * This interface represents an (external) hierarchy of objects. It can contain * arbitrary objects, BUT the hierarchy has to be accessed using the hierarchy @@ -63,7 +62,7 @@ public interface Hierarchy<O> { * @param self object to get descendants for * @return iterator for descendants */ - public IterableIterator<O> iterDescendants(O self); + public Iterator<O> iterDescendants(O self); /** * Get number of (direct) parents @@ -88,5 +87,5 @@ public interface Hierarchy<O> { * @param self object to get ancestors for * @return iterator for ancestors */ - public IterableIterator<O> iterAncestors(O self); + public Iterator<O> iterAncestors(O self); }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/hierarchy/HierarchyHashmapList.java b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/hierarchy/HierarchyHashmapList.java index d09e6d94..76bee0f9 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/hierarchy/HierarchyHashmapList.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/hierarchy/HierarchyHashmapList.java @@ -31,7 +31,6 @@ import java.util.List; import de.lmu.ifi.dbs.elki.logging.LoggingUtil; import de.lmu.ifi.dbs.elki.utilities.iterator.EmptyIterator; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; /** * Centralized hierarchy implementation, using a HashMap of Lists. @@ -149,7 +148,7 @@ public class HierarchyHashmapList<O> implements ModifiableHierarchy<O> { } @Override - public IterableIterator<O> iterDescendants(O obj) { + public Iterator<O> iterDescendants(O obj) { return new ItrDesc(obj); } @@ -172,7 +171,7 @@ public class HierarchyHashmapList<O> implements ModifiableHierarchy<O> { } @Override - public IterableIterator<O> iterAncestors(O obj) { + public Iterator<O> iterAncestors(O obj) { return new ItrAnc(obj); } @@ -183,7 +182,7 @@ public class HierarchyHashmapList<O> implements ModifiableHierarchy<O> { * * @apiviz.exclude */ - private class ItrDesc implements IterableIterator<O> { + private class ItrDesc implements Iterator<O> { /** * Starting object (for cloning); */ @@ -235,11 +234,6 @@ public class HierarchyHashmapList<O> implements ModifiableHierarchy<O> { public void remove() { throw new UnsupportedOperationException(); } - - @Override - public Iterator<O> iterator() { - return new ItrDesc(start); - } } /** @@ -249,7 +243,7 @@ public class HierarchyHashmapList<O> implements ModifiableHierarchy<O> { * * @apiviz.exclude */ - private class ItrAnc implements IterableIterator<O> { + private class ItrAnc implements Iterator<O> { /** * Starting object (for cloning); */ @@ -301,10 +295,5 @@ public class HierarchyHashmapList<O> implements ModifiableHierarchy<O> { public void remove() { throw new UnsupportedOperationException(); } - - @Override - public Iterator<O> iterator() { - return new ItrAnc(start); - } } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/hierarchy/HierarchyReferenceLists.java b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/hierarchy/HierarchyReferenceLists.java index f6c527ab..76091298 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/datastructures/hierarchy/HierarchyReferenceLists.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/datastructures/hierarchy/HierarchyReferenceLists.java @@ -27,7 +27,6 @@ import java.util.Iterator; import java.util.List; import de.lmu.ifi.dbs.elki.utilities.iterator.EmptyIterator; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; /** * Hierarchy implementation with a per-object representation. @@ -88,7 +87,7 @@ public class HierarchyReferenceLists<O extends Hierarchical<O>> implements Hiera } @Override - public IterableIterator<O> iterDescendants(O self) { + public Iterator<O> iterDescendants(O self) { if(owner != self) { return EmptyIterator.STATIC(); } @@ -121,7 +120,7 @@ public class HierarchyReferenceLists<O extends Hierarchical<O>> implements Hiera } @Override - public IterableIterator<O> iterAncestors(O self) { + public Iterator<O> iterAncestors(O self) { if(owner != self) { throw new UnsupportedOperationException("Decentral hierarchy queried for wrong object!"); } @@ -138,7 +137,7 @@ public class HierarchyReferenceLists<O extends Hierarchical<O>> implements Hiera * * @apiviz.exclude */ - private class ItrDesc implements IterableIterator<O> { + private class ItrDesc implements Iterator<O> { /** * Iterator over children */ @@ -179,11 +178,6 @@ public class HierarchyReferenceLists<O extends Hierarchical<O>> implements Hiera public void remove() { throw new UnsupportedOperationException(); } - - @Override - public Iterator<O> iterator() { - return new ItrDesc(owner); - } } /** @@ -193,7 +187,7 @@ public class HierarchyReferenceLists<O extends Hierarchical<O>> implements Hiera * * @apiviz.exclude */ - private class ItrAnc implements IterableIterator<O> { + private class ItrAnc implements Iterator<O> { /** * Iterator over parents */ @@ -234,10 +228,5 @@ public class HierarchyReferenceLists<O extends Hierarchical<O>> implements Hiera public void remove() { throw new UnsupportedOperationException(); } - - @Override - public Iterator<O> iterator() { - return new ItrAnc(owner); - } } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/utilities/iterator/EmptyIterator.java b/src/de/lmu/ifi/dbs/elki/utilities/iterator/EmptyIterator.java index 98a3539e..7ec63d62 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/iterator/EmptyIterator.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/iterator/EmptyIterator.java @@ -25,15 +25,14 @@ package de.lmu.ifi.dbs.elki.utilities.iterator; import java.util.Iterator; - /** - * Empty iterator, that never returns any data. + * Empty iterator/iterable, that never returns any data. * * @author Erich Schubert * * @param <T> Data type */ -public final class EmptyIterator<T> implements IterableIterator<T>, Iterator<T>, Iterable<T> { +public final class EmptyIterator<T> implements Iterator<T>, Iterable<T> { @Override public boolean hasNext() { return false; @@ -50,7 +49,7 @@ public final class EmptyIterator<T> implements IterableIterator<T>, Iterator<T>, } @Override - public IterableIterator<T> iterator() { + public Iterator<T> iterator() { return STATIC(); } @@ -66,7 +65,7 @@ public final class EmptyIterator<T> implements IterableIterator<T>, Iterator<T>, * @return Cast static instance. */ @SuppressWarnings("unchecked") - public static <T> IterableIterator<T> STATIC() { - return (IterableIterator<T>) STATIC_INSTANCE; + public static <T> EmptyIterator<T> STATIC() { + return (EmptyIterator<T>) STATIC_INSTANCE; } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/utilities/iterator/Iter.java b/src/de/lmu/ifi/dbs/elki/utilities/iterator/Iter.java new file mode 100644 index 00000000..11fead71 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/utilities/iterator/Iter.java @@ -0,0 +1,71 @@ +package de.lmu.ifi.dbs.elki.utilities.iterator; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +/** + * Iterator interface for more than one return value. + * + * The Java standard {@link java.util.Iterator} interface has some drawbacks: + * <ul> + * <li>the only way to get the current value is to advance the iterator</li> + * <li>the iterator can only point to a single value</li> + * <li>the iterator can only return objects, not primitives</li> + * </ul> + * + * This iterator interface is a bit more flexible. For example on a distance + * list, we can have a single type of iterator that allows access to the + * distance, the object ID or the combination of both. + * + * In some situations, this can save the creation of many small objects, which + * put load on the garbage collector. This super interface does not have a "get" + * operation, which is to come from specialized interfaces instead. + * + * Usage example: + * + * <pre> + * {@code + * for (Iter iter = ids.iter(); iter.valid(); iter.advance()) { + * iter.doSomething(); + * } + * } + * </pre> + * + * @author Erich Schubert + */ +public interface Iter { + /** + * Returns true if the iterator currently points to a valid object. + * + * @return a <code>boolean</code> value + */ + public boolean valid(); + + /** + * Moves the iterator forward to the next entry. + * + * @throws java.util.NoSuchElementException if the iterator is already + * exhausted + */ + public void advance(); +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/utilities/iterator/IterableIteratorAdapter.java b/src/de/lmu/ifi/dbs/elki/utilities/iterator/IterableIteratorAdapter.java deleted file mode 100644 index 4a8cb512..00000000 --- a/src/de/lmu/ifi/dbs/elki/utilities/iterator/IterableIteratorAdapter.java +++ /dev/null @@ -1,110 +0,0 @@ -package de.lmu.ifi.dbs.elki.utilities.iterator; - -/* - This file is part of ELKI: - Environment for Developing KDD-Applications Supported by Index-Structures - - Copyright (C) 2012 - Ludwig-Maximilians-Universität München - Lehr- und Forschungseinheit für Datenbanksysteme - ELKI Development Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -import java.util.Iterator; - -/** - * This interface can convert an {@link Iterable} to an {@link Iterator} or the - * other way round. Note that {@link Iterator} to {@link Iterable} is for - * single-shot use only. This allows for using an Iterator in for: - * - * <blockquote><pre>{@code - * for (Type var : new IterableIterator<Type>(iterator)) { - * // ... - * } - * }</pre></blockquote> - * - * @apiviz.stereotype decorator - * @apiviz.uses Iterable - * @apiviz.uses Iterator - * - * @author Erich Schubert - * @param <T> object type - */ -public final class IterableIteratorAdapter<T> implements IterableIterator<T> { - /** - * Parent Iterable - */ - Iterable<T> parent = null; - - /** - * Parent Iterator - */ - Iterator<T> iter = null; - - /** - * Constructor from an Iterable (preferred). - * - * @param parent Iterable parent - */ - public IterableIteratorAdapter(Iterable<T> parent) { - this.parent = parent; - assert (parent != null); - } - - /** - * Constructor from an Iterator. - * - * If possible, wrap an Iterable object. - * - * @param iter Iterator - */ - public IterableIteratorAdapter(Iterator<T> iter) { - this.iter = iter; - assert (iter != null); - } - - @Override - public Iterator<T> iterator() { - if(parent == null) { - return this; - } - return parent.iterator(); - } - - @Override - public boolean hasNext() { - if(iter == null) { - iter = parent.iterator(); - } - return iter.hasNext(); - } - - @Override - public T next() { - if(iter == null) { - iter = parent.iterator(); - } - return iter.next(); - } - - @Override - public void remove() { - if(iter == null) { - iter = parent.iterator(); - } - iter.remove(); - } -}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/utilities/iterator/MergedIterator.java b/src/de/lmu/ifi/dbs/elki/utilities/iterator/MergedIterator.java index 9208ab14..b5d2fd02 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/iterator/MergedIterator.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/iterator/MergedIterator.java @@ -37,7 +37,7 @@ import java.util.Iterator; * * @param <E> Entry type */ -public class MergedIterator<E> implements IterableIterator<E> { +public class MergedIterator<E> implements Iterator<E> { /** * All the iterators we process */ @@ -112,9 +112,4 @@ public class MergedIterator<E> implements IterableIterator<E> { } last.remove(); } - - @Override - public Iterator<E> iterator() { - return this; - } -} +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/utilities/iterator/OneItemIterator.java b/src/de/lmu/ifi/dbs/elki/utilities/iterator/OneItemIterator.java index a81dcc18..f6b00242 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/iterator/OneItemIterator.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/iterator/OneItemIterator.java @@ -34,7 +34,7 @@ import java.util.Iterator; * * @param <T> Object type to return */ -public class OneItemIterator<T> implements IterableIterator<T> { +public class OneItemIterator<T> implements Iterator<T> { /** * Object to return. */ @@ -66,9 +66,4 @@ public class OneItemIterator<T> implements IterableIterator<T> { public void remove() { throw new UnsupportedOperationException(); } - - @Override - public Iterator<T> iterator() { - return this; - } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/utilities/iterator/TypeFilterIterator.java b/src/de/lmu/ifi/dbs/elki/utilities/iterator/TypeFilterIterator.java index cd687542..5aa908c8 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/iterator/TypeFilterIterator.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/iterator/TypeFilterIterator.java @@ -33,7 +33,7 @@ import java.util.Iterator; * @param <IN> Input datatype * @param <OUT> Output datatype */ -public class TypeFilterIterator<IN, OUT extends IN> extends AbstractFilteredIterator<IN, OUT> implements IterableIterator<OUT> { +public class TypeFilterIterator<IN, OUT extends IN> extends AbstractFilteredIterator<IN, OUT> implements Iterator<OUT> { /** * Class restriction */ @@ -83,9 +83,4 @@ public class TypeFilterIterator<IN, OUT extends IN> extends AbstractFilteredIter return null; } } - - @Override - public Iterator<OUT> iterator() { - return this; - } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/utilities/optionhandling/OptionID.java b/src/de/lmu/ifi/dbs/elki/utilities/optionhandling/OptionID.java index 0d685e2b..bea6bb4f 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/optionhandling/OptionID.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/optionhandling/OptionID.java @@ -105,6 +105,14 @@ public final class OptionID extends ConstantObject<OptionID> { public static final OptionID VERBOSE_FLAG = new OptionID("verbose", "Enable verbose messages."); /** + * Flag to allow verbose messages while running the application. + * <p> + * Key: {@code -time} + * </p> + */ + public static final OptionID TIME_FLAG = new OptionID("time", "Enable logging of runtime data. Do not combine with more verbose logging, since verbose logging can significantly impact performance."); + + /** * The description of the OptionID. */ private String description; diff --git a/src/de/lmu/ifi/dbs/elki/utilities/package-info.java b/src/de/lmu/ifi/dbs/elki/utilities/package-info.java index a1bbc579..6a8126e1 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/package-info.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/package-info.java @@ -35,7 +35,6 @@ * </ul></li> * <li>Specialized:<ul> * <li>{@link de.lmu.ifi.dbs.elki.data.images.ImageUtil}: image handling.</li> - * <li>{@link de.lmu.ifi.dbs.elki.utilities.iterator.IterableUtil}: Converting iterators and iterables.</li> * <li>{@link de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionUtil}: Managing parameter settings</li> * <li>{@link de.lmu.ifi.dbs.elki.utilities.InspectionUtil}: class and classpath inspection.</li> * <li>{@link de.lmu.ifi.dbs.elki.utilities.documentation.DocumentationUtil}: documentation extraction from annotations.</li> diff --git a/src/de/lmu/ifi/dbs/elki/utilities/pairs/CTriple.java b/src/de/lmu/ifi/dbs/elki/utilities/pairs/CTriple.java index 5a4b8cbd..225fb0b9 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/pairs/CTriple.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/pairs/CTriple.java @@ -34,7 +34,7 @@ import de.lmu.ifi.dbs.elki.utilities.ClassGenericsUtil; * @param <SECOND> second type * @param <THIRD> second type */ -public final class CTriple<FIRST extends Comparable<FIRST>, SECOND extends Comparable<SECOND>, THIRD extends Comparable<THIRD>> extends Triple<FIRST, SECOND, THIRD> implements Comparable<CTriple<FIRST, SECOND, THIRD>> { +public final class CTriple<FIRST extends Comparable<? super FIRST>, SECOND extends Comparable<? super SECOND>, THIRD extends Comparable<? super THIRD>> extends Triple<FIRST, SECOND, THIRD> implements Comparable<CTriple<FIRST, SECOND, THIRD>> { /** * Constructor with fields * diff --git a/src/de/lmu/ifi/dbs/elki/utilities/referencepoints/RandomSampleReferencePoints.java b/src/de/lmu/ifi/dbs/elki/utilities/referencepoints/RandomSampleReferencePoints.java index 48125019..8f6df4db 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/referencepoints/RandomSampleReferencePoints.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/referencepoints/RandomSampleReferencePoints.java @@ -31,6 +31,7 @@ import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.logging.LoggingUtil; @@ -84,7 +85,8 @@ public class RandomSampleReferencePoints<V extends NumberVector<? extends V, ?>> LoggingUtil.warning("Sample size is larger than database size!"); ArrayList<V> selection = new ArrayList<V>(db.size()); - for(DBID id : db.iterDBIDs()) { + for(DBIDIter iditer = db.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); selection.add(db.get(id)); } return selection; diff --git a/src/de/lmu/ifi/dbs/elki/utilities/referencepoints/StarBasedReferencePoints.java b/src/de/lmu/ifi/dbs/elki/utilities/referencepoints/StarBasedReferencePoints.java index 819fc29e..47c7298b 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/referencepoints/StarBasedReferencePoints.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/referencepoints/StarBasedReferencePoints.java @@ -28,6 +28,7 @@ import java.util.Collection; import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; @@ -100,7 +101,8 @@ public class StarBasedReferencePoints<V extends NumberVector<V, ?>> implements R min[d] = Double.MAX_VALUE; max[d] = -Double.MAX_VALUE; } - for(DBID objID : database.iterDBIDs()) { + for(DBIDIter iditer = database.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID objID = iditer.getDBID(); V obj = database.get(objID); for(int d = 0; d < dim; d++) { double val = obj.doubleValue(d + 1); diff --git a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/HeDESNormalizationOutlierScaling.java b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/HeDESNormalizationOutlierScaling.java index e7e6ee90..8a152cff 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/HeDESNormalizationOutlierScaling.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/HeDESNormalizationOutlierScaling.java @@ -24,6 +24,7 @@ package de.lmu.ifi.dbs.elki.utilities.scaling.outlier; */ import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.math.DoubleMinMax; import de.lmu.ifi.dbs.elki.math.MeanVariance; import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; @@ -61,7 +62,8 @@ public class HeDESNormalizationOutlierScaling implements OutlierScalingFunction MeanVariance mv = new MeanVariance(); DoubleMinMax minmax = new DoubleMinMax(); - for(DBID id : or.getScores().iterDBIDs()) { + for(DBIDIter iditer = or.getScores().iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); double val = or.getScores().get(id); if(!Double.isNaN(val) && !Double.isInfinite(val)) { mv.put(val); diff --git a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/MinusLogGammaScaling.java b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/MinusLogGammaScaling.java index 1485204c..860c77d6 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/MinusLogGammaScaling.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/MinusLogGammaScaling.java @@ -24,6 +24,7 @@ package de.lmu.ifi.dbs.elki.utilities.scaling.outlier; */ import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.math.DoubleMinMax; import de.lmu.ifi.dbs.elki.math.MeanVariance; import de.lmu.ifi.dbs.elki.math.statistics.distribution.GammaDistribution; @@ -67,7 +68,8 @@ public class MinusLogGammaScaling extends OutlierGammaScaling { meta = or.getOutlierMeta(); // Determine Minimum and Maximum. DoubleMinMax mm = new DoubleMinMax(); - for(DBID id : or.getScores().iterDBIDs()) { + for(DBIDIter iditer = or.getScores().iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); double score = or.getScores().get(id); if(!Double.isNaN(score) && !Double.isInfinite(score)) { mm.put(score); @@ -77,7 +79,8 @@ public class MinusLogGammaScaling extends OutlierGammaScaling { mlogmax = -Math.log(mm.getMin() / max); // with the prescaling, do Gamma Scaling. MeanVariance mv = new MeanVariance(); - for(DBID id : or.getScores().iterDBIDs()) { + for(DBIDIter iditer = or.getScores().iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); double score = or.getScores().get(id); score = preScale(score); if(!Double.isNaN(score) && !Double.isInfinite(score)) { diff --git a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/MinusLogStandardDeviationScaling.java b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/MinusLogStandardDeviationScaling.java index 477fbdd2..4bc10feb 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/MinusLogStandardDeviationScaling.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/MinusLogStandardDeviationScaling.java @@ -24,6 +24,7 @@ package de.lmu.ifi.dbs.elki.utilities.scaling.outlier; */ import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.math.MathUtil; import de.lmu.ifi.dbs.elki.math.MeanVariance; import de.lmu.ifi.dbs.elki.math.statistics.distribution.NormalDistribution; @@ -67,7 +68,8 @@ public class MinusLogStandardDeviationScaling extends StandardDeviationScaling { public void prepare(OutlierResult or) { if(fixedmean == null) { MeanVariance mv = new MeanVariance(); - for(DBID id : or.getScores().iterDBIDs()) { + for(DBIDIter iditer = or.getScores().iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); double val = -Math.log(or.getScores().get(id)); if(!Double.isNaN(val) && !Double.isInfinite(val)) { mv.put(val); @@ -80,7 +82,8 @@ public class MinusLogStandardDeviationScaling extends StandardDeviationScaling { mean = fixedmean; double sqsum = 0; int cnt = 0; - for(DBID id : or.getScores().iterDBIDs()) { + for(DBIDIter iditer = or.getScores().iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); double val = -Math.log(or.getScores().get(id)); if(!Double.isNaN(val) && !Double.isInfinite(val)) { sqsum += (val - mean) * (val - mean); diff --git a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/MixtureModelOutlierScalingFunction.java b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/MixtureModelOutlierScalingFunction.java index 26abb453..52a7f449 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/MixtureModelOutlierScalingFunction.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/MixtureModelOutlierScalingFunction.java @@ -25,6 +25,7 @@ package de.lmu.ifi.dbs.elki.utilities.scaling.outlier; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.logging.Logging; import de.lmu.ifi.dbs.elki.math.MathUtil; @@ -120,7 +121,8 @@ public class MixtureModelOutlierScalingFunction implements OutlierScalingFunctio public void prepare(OutlierResult or) { // Initial parameters - are these defaults sounds? MeanVariance mv = new MeanVariance(); - for(DBID id : or.getScores().iterDBIDs()) { + for(DBIDIter iditer = or.getScores().iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); double val = or.getScores().get(id); if(!Double.isNaN(val) && !Double.isInfinite(val)) { mv.put(val); diff --git a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/MultiplicativeInverseScaling.java b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/MultiplicativeInverseScaling.java index 9602b9b6..1ebf24bb 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/MultiplicativeInverseScaling.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/MultiplicativeInverseScaling.java @@ -24,6 +24,7 @@ package de.lmu.ifi.dbs.elki.utilities.scaling.outlier; */ import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; @@ -78,7 +79,8 @@ public class MultiplicativeInverseScaling implements OutlierScalingFunction { */ private static double getScaleValue(OutlierResult or) { double max = Double.MIN_VALUE; - for(DBID id : or.getScores().iterDBIDs()) { + for(DBIDIter iditer = or.getScores().iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); double val = or.getScores().get(id); double inv = Math.abs(1.0 / val); if(!Double.isInfinite(inv) && !Double.isNaN(inv)) { diff --git a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/OutlierGammaScaling.java b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/OutlierGammaScaling.java index a3eecdb0..9facaa11 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/OutlierGammaScaling.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/OutlierGammaScaling.java @@ -24,6 +24,7 @@ package de.lmu.ifi.dbs.elki.utilities.scaling.outlier; */ import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.math.MeanVariance; import de.lmu.ifi.dbs.elki.math.statistics.distribution.GammaDistribution; import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; @@ -100,7 +101,8 @@ public class OutlierGammaScaling implements OutlierScalingFunction { public void prepare(OutlierResult or) { meta = or.getOutlierMeta(); MeanVariance mv = new MeanVariance(); - for(DBID id : or.getScores().iterDBIDs()) { + for(DBIDIter iditer = or.getScores().iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); double score = or.getScores().get(id); score = preScale(score); if(!Double.isNaN(score) && !Double.isInfinite(score)) { diff --git a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/OutlierLinearScaling.java b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/OutlierLinearScaling.java index 8f215ec3..e7222b96 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/OutlierLinearScaling.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/OutlierLinearScaling.java @@ -26,6 +26,7 @@ package de.lmu.ifi.dbs.elki.utilities.scaling.outlier; import java.util.ArrayList; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.math.DoubleMinMax; import de.lmu.ifi.dbs.elki.math.MeanVariance; import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; @@ -39,7 +40,7 @@ import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.Flag; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.Parameter; /** - * Scaling that can map arbitrary values to a probability in the range of [0:1]. + * Scaling that can map arbitrary values to a value in the range of [0:1]. * * Transformation is done by linear mapping onto 0:1 using the minimum and * maximum values. @@ -53,10 +54,10 @@ public class OutlierLinearScaling implements OutlierScalingFunction { * Key: {@code -linearscale.min} * </p> */ - public static final OptionID MIN_ID = OptionID.getOrCreateOptionID("linearscale.min", "Fixed minimum to use in lienar scaling."); + public static final OptionID MIN_ID = OptionID.getOrCreateOptionID("linearscale.min", "Fixed minimum to use in linear scaling."); /** - * Parameter to specify the maximum value + * Parameter to specify the maximum value. * <p> * Key: {@code -linearscale.max} * </p> @@ -147,7 +148,8 @@ public class OutlierLinearScaling implements OutlierScalingFunction { MeanVariance mv = new MeanVariance(); DoubleMinMax mm = (max == null) ? new DoubleMinMax() : null; boolean skippedzeros = false; - for(DBID id : or.getScores().iterDBIDs()) { + for(DBIDIter iditer = or.getScores().iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); double val = or.getScores().get(id); if(nozeros && val == 0.0) { skippedzeros = true; @@ -173,7 +175,8 @@ public class OutlierLinearScaling implements OutlierScalingFunction { if(min == null || max == null) { boolean skippedzeros = false; DoubleMinMax mm = new DoubleMinMax(); - for(DBID id : or.getScores().iterDBIDs()) { + for(DBIDIter iditer = or.getScores().iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); double val = or.getScores().get(id); if(nozeros && val == 0.0) { skippedzeros = true; diff --git a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/OutlierMinusLogScaling.java b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/OutlierMinusLogScaling.java index 0839f231..09d84eb4 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/OutlierMinusLogScaling.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/OutlierMinusLogScaling.java @@ -24,6 +24,7 @@ package de.lmu.ifi.dbs.elki.utilities.scaling.outlier; */ import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.math.DoubleMinMax; import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; @@ -77,7 +78,8 @@ public class OutlierMinusLogScaling implements OutlierScalingFunction { @Override public void prepare(OutlierResult or) { DoubleMinMax mm = new DoubleMinMax(); - for(DBID id : or.getScores().iterDBIDs()) { + for(DBIDIter iditer = or.getScores().iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); double val = or.getScores().get(id); if(!Double.isNaN(val) && !Double.isInfinite(val)) { mm.put(val); diff --git a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/OutlierSqrtScaling.java b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/OutlierSqrtScaling.java index d39fed97..3f564fd1 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/OutlierSqrtScaling.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/OutlierSqrtScaling.java @@ -24,6 +24,7 @@ package de.lmu.ifi.dbs.elki.utilities.scaling.outlier; */ import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.math.DoubleMinMax; import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; @@ -97,7 +98,8 @@ public class OutlierSqrtScaling implements OutlierScalingFunction { public void prepare(OutlierResult or) { if(min == null || max == null) { DoubleMinMax mm = new DoubleMinMax(); - for(DBID id : or.getScores().iterDBIDs()) { + for(DBIDIter iditer = or.getScores().iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); double val = or.getScores().get(id); if(!Double.isNaN(val) && !Double.isInfinite(val)) { mm.put(val); diff --git a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/RankingPseudoOutlierScaling.java b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/RankingPseudoOutlierScaling.java index 47eda6aa..be7bf766 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/RankingPseudoOutlierScaling.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/RankingPseudoOutlierScaling.java @@ -26,6 +26,7 @@ package de.lmu.ifi.dbs.elki.utilities.scaling.outlier; import java.util.Arrays; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.result.outlier.InvertedOutlierScoreMeta; import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; @@ -56,7 +57,8 @@ public class RankingPseudoOutlierScaling implements OutlierScalingFunction { if(or.getOutlierMeta() instanceof InvertedOutlierScoreMeta) { inverted = true; } - for(DBID id : or.getScores().iterDBIDs()) { + for(DBIDIter iditer = or.getScores().iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); scores[pos] = or.getScores().get(id); pos++; } diff --git a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/SigmoidOutlierScalingFunction.java b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/SigmoidOutlierScalingFunction.java index 31e78ec8..468079a3 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/SigmoidOutlierScalingFunction.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/SigmoidOutlierScalingFunction.java @@ -27,6 +27,7 @@ import java.util.BitSet; import de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.logging.Logging; @@ -61,7 +62,8 @@ public class SigmoidOutlierScalingFunction implements OutlierScalingFunction { public void prepare(OutlierResult or) { // Initial parameters - are these defaults sounds? MeanVariance mv = new MeanVariance(); - for(DBID id : or.getScores().iterDBIDs()) { + for(DBIDIter iditer = or.getScores().iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); double val = or.getScores().get(id); mv.put(val); } diff --git a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/SqrtStandardDeviationScaling.java b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/SqrtStandardDeviationScaling.java index 24f9e855..d3048b1d 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/SqrtStandardDeviationScaling.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/SqrtStandardDeviationScaling.java @@ -24,6 +24,7 @@ package de.lmu.ifi.dbs.elki.utilities.scaling.outlier; */ import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.math.DoubleMinMax; import de.lmu.ifi.dbs.elki.math.MathUtil; import de.lmu.ifi.dbs.elki.math.MeanVariance; @@ -126,7 +127,8 @@ public class SqrtStandardDeviationScaling implements OutlierScalingFunction { public void prepare(OutlierResult or) { if(min == null) { DoubleMinMax mm = new DoubleMinMax(); - for(DBID id : or.getScores().iterDBIDs()) { + for(DBIDIter iditer = or.getScores().iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); double val = or.getScores().get(id); if(!Double.isNaN(val) && !Double.isInfinite(val)) { mm.put(val); @@ -136,7 +138,8 @@ public class SqrtStandardDeviationScaling implements OutlierScalingFunction { } if(mean == null) { MeanVariance mv = new MeanVariance(); - for(DBID id : or.getScores().iterDBIDs()) { + for(DBIDIter iditer = or.getScores().iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); double val = or.getScores().get(id); val = (val <= min) ? 0 : Math.sqrt(val - min); mv.put(val); @@ -147,7 +150,8 @@ public class SqrtStandardDeviationScaling implements OutlierScalingFunction { else { double sqsum = 0; int cnt = 0; - for(DBID id : or.getScores().iterDBIDs()) { + for(DBIDIter iditer = or.getScores().iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); double val = or.getScores().get(id); val = (val <= min) ? 0 : Math.sqrt(val - min); sqsum += (val - mean) * (val - mean); diff --git a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/StandardDeviationScaling.java b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/StandardDeviationScaling.java index 6ef5c006..7d76a69f 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/StandardDeviationScaling.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/StandardDeviationScaling.java @@ -24,6 +24,7 @@ package de.lmu.ifi.dbs.elki.utilities.scaling.outlier; */ import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.math.MathUtil; import de.lmu.ifi.dbs.elki.math.MeanVariance; import de.lmu.ifi.dbs.elki.math.statistics.distribution.NormalDistribution; @@ -115,7 +116,8 @@ public class StandardDeviationScaling implements OutlierScalingFunction { public void prepare(OutlierResult or) { if(fixedmean == null) { MeanVariance mv = new MeanVariance(); - for(DBID id : or.getScores().iterDBIDs()) { + for(DBIDIter iditer = or.getScores().iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); double val = or.getScores().get(id); if(!Double.isNaN(val) && !Double.isInfinite(val)) { mv.put(val); @@ -131,7 +133,8 @@ public class StandardDeviationScaling implements OutlierScalingFunction { mean = fixedmean; double sqsum = 0; int cnt = 0; - for(DBID id : or.getScores().iterDBIDs()) { + for(DBIDIter iditer = or.getScores().iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); double val = or.getScores().get(id); if(!Double.isNaN(val) && !Double.isInfinite(val)) { sqsum += (val - mean) * (val - mean); diff --git a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/TopKOutlierScaling.java b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/TopKOutlierScaling.java index 8044fce9..c922ab34 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/TopKOutlierScaling.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/scaling/outlier/TopKOutlierScaling.java @@ -23,10 +23,9 @@ package de.lmu.ifi.dbs.elki.utilities.scaling.outlier; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.logging.LoggingUtil; import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.GreaterConstraint; @@ -98,14 +97,9 @@ public class TopKOutlierScaling implements OutlierScalingFunction { if(k <= 0) { LoggingUtil.warning("No k configured for Top-k outlier scaling!"); } - IterableIterator<DBID> order = or.getOrdering().iter(or.getOrdering().getDBIDs()); - for(int i = 0; i < k; i++) { - // stop if no more results. - if(!order.hasNext()) { - return; - } - DBID cur = order.next(); - cutoff = or.getScores().get(cur); + DBIDIter order = or.getOrdering().iter(or.getOrdering().getDBIDs()).iter(); + for(int i = 0; i < k && order.valid(); i++, order.advance()) { + cutoff = or.getScores().get(order.getDBID()); } max = or.getOutlierMeta().getActualMaximum(); ground = or.getOutlierMeta().getTheoreticalBaseline(); diff --git a/src/de/lmu/ifi/dbs/elki/utilities/xml/XMLNodeIterator.java b/src/de/lmu/ifi/dbs/elki/utilities/xml/XMLNodeIterator.java index e0602790..a5b7cfa6 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/xml/XMLNodeIterator.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/xml/XMLNodeIterator.java @@ -28,15 +28,13 @@ import java.util.Iterator; import org.w3c.dom.Node; import de.lmu.ifi.dbs.elki.utilities.exceptions.ExceptionMessages; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; /** * Simple adapter class to iterate over a DOM tree nodes children. * * @author Erich Schubert - * */ -public final class XMLNodeIterator implements IterableIterator<Node> { +public final class XMLNodeIterator implements Iterator<Node> { /** * Store the next node */ @@ -77,12 +75,4 @@ public final class XMLNodeIterator implements IterableIterator<Node> { public void remove() { throw new UnsupportedOperationException(this.getClass().getSimpleName()+": "+ExceptionMessages.UNSUPPORTED_REMOVE); } - - /** - * Iterator interface adapter - clone. - */ - @Override - public Iterator<Node> iterator() { - return new XMLNodeIterator(this.next); - } -} +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/utilities/xml/XMLNodeListIterator.java b/src/de/lmu/ifi/dbs/elki/utilities/xml/XMLNodeListIterator.java index 8157d4ba..b413d1df 100644 --- a/src/de/lmu/ifi/dbs/elki/utilities/xml/XMLNodeListIterator.java +++ b/src/de/lmu/ifi/dbs/elki/utilities/xml/XMLNodeListIterator.java @@ -29,15 +29,13 @@ import org.w3c.dom.Node; import org.w3c.dom.NodeList; import de.lmu.ifi.dbs.elki.utilities.exceptions.ExceptionMessages; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; /** * Simple adapter class to iterate over a DOM tree nodes children. * * @author Erich Schubert - * */ -public final class XMLNodeListIterator implements IterableIterator<Node> { +public final class XMLNodeListIterator implements Iterator<Node> { /** * Store the next node */ @@ -84,12 +82,4 @@ public final class XMLNodeListIterator implements IterableIterator<Node> { public void remove() { throw new UnsupportedOperationException(this.getClass().getSimpleName()+": "+ExceptionMessages.UNSUPPORTED_REMOVE); } - - /** - * Iterable interface adapter - clone. - */ - @Override - public Iterator<Node> iterator() { - return new XMLNodeListIterator(this.nodelist); - } -} +}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/visualization/VisualizationTask.java b/src/de/lmu/ifi/dbs/elki/visualization/VisualizationTask.java index 187b2473..39d6e359 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/VisualizationTask.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/VisualizationTask.java @@ -40,7 +40,6 @@ import de.lmu.ifi.dbs.elki.visualization.visualizers.VisFactory; * @apiviz.has SVGPlot * @apiviz.has VisFactory * @apiviz.has Projection oneway - 0:1 - * @apiviz.has Visualization oneway */ public class VisualizationTask extends AnyMap<String> implements Cloneable, Result, Comparable<VisualizationTask> { /** diff --git a/src/de/lmu/ifi/dbs/elki/visualization/VisualizerContext.java b/src/de/lmu/ifi/dbs/elki/visualization/VisualizerContext.java index 663670e0..c4e6c839 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/VisualizerContext.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/VisualizerContext.java @@ -64,11 +64,13 @@ import de.lmu.ifi.dbs.elki.visualization.visualizers.VisualizerUtil; * TODO: remove this class * * @apiviz.landmark - * @apiviz.uses ContextChangedEvent oneway - - «emit» * @apiviz.composedOf StyleLibrary * @apiviz.composedOf SelectionResult * @apiviz.composedOf ResultHierarchy * @apiviz.composedOf EventListenerList + * @apiviz.composedOf StyleResult + * @apiviz.composedOf ProjectorFactory + * @apiviz.composedOf VisFactory */ public class VisualizerContext implements DataStoreListener, Result { /** diff --git a/src/de/lmu/ifi/dbs/elki/visualization/VisualizerParameterizer.java b/src/de/lmu/ifi/dbs/elki/visualization/VisualizerParameterizer.java index 8e0be062..36bbb823 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/VisualizerParameterizer.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/VisualizerParameterizer.java @@ -26,7 +26,6 @@ package de.lmu.ifi.dbs.elki.visualization; import java.io.File; import java.util.ArrayList; import java.util.Collection; -import java.util.Iterator; import java.util.List; import java.util.Random; import java.util.regex.Pattern; @@ -165,12 +164,10 @@ public class VisualizerParameterizer implements Parameterizable { * @return New context */ public VisualizerContext newContext(HierarchicalResult result) { - VisualizerContext context = new VisualizerContext(result, stylelib, projectors, factories); if(samplesize > 0) { - Iterator<Relation<?>> iter = ResultUtil.filteredResults(result, Relation.class); - while(iter.hasNext()) { - Relation<?> rel = iter.next(); - if(ResultUtil.filteredResults(rel, SamplingResult.class).hasNext()) { + Collection<Relation<?>> rels = ResultUtil.filterResults(result, Relation.class); + for(Relation<?> rel : rels) { + if(!ResultUtil.filterResults(rel, SamplingResult.class).isEmpty()) { continue; } int size = rel.size(); @@ -181,6 +178,7 @@ public class VisualizerParameterizer implements Parameterizable { } } } + VisualizerContext context = new VisualizerContext(result, stylelib, projectors, factories); return context; } diff --git a/src/de/lmu/ifi/dbs/elki/visualization/batikutil/CloneInlineImages.java b/src/de/lmu/ifi/dbs/elki/visualization/batikutil/CloneInlineImages.java index a1dd3ba1..30456ba9 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/batikutil/CloneInlineImages.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/batikutil/CloneInlineImages.java @@ -46,7 +46,6 @@ import de.lmu.ifi.dbs.elki.visualization.svg.SVGCloneVisible; * Clone an SVG document, inlining temporary and in-memory linked images. * * @author Erich Schubert - * */ public class CloneInlineImages extends SVGCloneVisible { @Override diff --git a/src/de/lmu/ifi/dbs/elki/visualization/batikutil/NodeAppendChild.java b/src/de/lmu/ifi/dbs/elki/visualization/batikutil/NodeAppendChild.java index 380d2cbf..a5396732 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/batikutil/NodeAppendChild.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/batikutil/NodeAppendChild.java @@ -30,8 +30,7 @@ import de.lmu.ifi.dbs.elki.visualization.svg.SVGPlot; /** * Runnable wrapper for appending XML-Elements to existing Elements. * - * @author Remigius Wojdanowski. - * + * @author Remigius Wojdanowski */ public class NodeAppendChild implements Runnable { /** @@ -45,7 +44,7 @@ public class NodeAppendChild implements Runnable { protected Element child; /** - * The plot (for ID updates). May be {code null}. + * The plot (for ID updates). May be {@code null}. */ protected SVGPlot plot; diff --git a/src/de/lmu/ifi/dbs/elki/visualization/gui/SelectionTableWindow.java b/src/de/lmu/ifi/dbs/elki/visualization/gui/SelectionTableWindow.java index 0c8e7500..754779fe 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/gui/SelectionTableWindow.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/gui/SelectionTableWindow.java @@ -44,6 +44,7 @@ import de.lmu.ifi.dbs.elki.database.datastore.DataStoreEvent; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreListener;
import de.lmu.ifi.dbs.elki.database.ids.ArrayModifiableDBIDs;
import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil;
import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs;
import de.lmu.ifi.dbs.elki.database.relation.Relation;
@@ -227,8 +228,8 @@ public class SelectionTableWindow extends JFrame implements DataStoreListener, R // Unselect first ...
context.setSelection(new DBIDSelection(remain));
// Now delete them.
- for(DBID id : todel) {
- upd.delete(id);
+ for(DBIDIter iter = todel.iter(); iter.valid(); iter.advance()) {
+ upd.delete(iter.getDBID());
}
}
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/gui/detail/DetailView.java b/src/de/lmu/ifi/dbs/elki/visualization/gui/detail/DetailView.java index 8b6f4892..3e1348ac 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/gui/detail/DetailView.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/gui/detail/DetailView.java @@ -149,6 +149,9 @@ public class DetailView extends SVGPlot implements ResultListener { if(VisualizerUtil.isVisible(task)) { try { Visualization v = task.getFactory().makeVisualization(task.clone(this, context, visi.proj, width, height)); + if (VisualizerUtil.isNoExport(task)) { + v.getLayer().setAttribute(NO_EXPORT_ATTRIBUTE, NO_EXPORT_ATTRIBUTE); + } layers.add(v); layermap.put(task, v); } @@ -291,10 +294,14 @@ public class DetailView extends SVGPlot implements ResultListener { if(VisualizerUtil.isVisible(task)) { // LoggingUtil.warning("Need to recreate a missing layer for " + v); vis = task.getFactory().makeVisualization(task.clone(this, context, visi.proj, width, height)); + if (VisualizerUtil.isNoExport(task)) { + vis.getLayer().setAttribute(NO_EXPORT_ATTRIBUTE, NO_EXPORT_ATTRIBUTE); + } layermap.put(task, vis); this.scheduleUpdate(new InsertVisualization(vis)); } } + // FIXME: ensure layers are ordered correctly! } @Override diff --git a/src/de/lmu/ifi/dbs/elki/visualization/gui/overview/LayerMap.java b/src/de/lmu/ifi/dbs/elki/visualization/gui/overview/LayerMap.java index c3575e43..36851b85 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/gui/overview/LayerMap.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/gui/overview/LayerMap.java @@ -33,6 +33,9 @@ import de.lmu.ifi.dbs.elki.visualization.visualizers.Visualization; * Class to help keeping track of the materialized layers of the different visualizations. * * @author Erich Schubert + * + * @apiviz.has PlotItem + * @apiviz.has VisualizationTask */ public class LayerMap { /** diff --git a/src/de/lmu/ifi/dbs/elki/visualization/gui/overview/OverviewPlot.java b/src/de/lmu/ifi/dbs/elki/visualization/gui/overview/OverviewPlot.java index 56d6ce32..92ddb0b8 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/gui/overview/OverviewPlot.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/gui/overview/OverviewPlot.java @@ -66,9 +66,9 @@ import de.lmu.ifi.dbs.elki.visualization.visualizers.VisualizerUtil; * @apiviz.landmark * @apiviz.has VisualizerContext * @apiviz.composedOf RectangleArranger + * @apiviz.composedOf LayerMap * @apiviz.has DetailViewSelectedEvent * @apiviz.uses DetailView - * @apiviz.uses Projector */ // FIXME: there still is a synchronization issue, that causes the initialization // to be run twice in parallel. @@ -322,6 +322,9 @@ public class OverviewPlot extends SVGPlot implements ResultListener { LoggingUtil.warning("Visualization returned empty layer: " + vis); } else { + if (VisualizerUtil.isNoExport(task)) { + vis.getLayer().setAttribute(NO_EXPORT_ATTRIBUTE, NO_EXPORT_ATTRIBUTE); + } parent.appendChild(vis.getLayer()); } return vis; diff --git a/src/de/lmu/ifi/dbs/elki/visualization/gui/overview/PlotItem.java b/src/de/lmu/ifi/dbs/elki/visualization/gui/overview/PlotItem.java index cbe5a4be..73ee797e 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/gui/overview/PlotItem.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/gui/overview/PlotItem.java @@ -40,6 +40,8 @@ import de.lmu.ifi.dbs.elki.visualization.projections.Projection; * @author Erich Schubert * * @apiviz.composedOf Projection + * @apiviz.composedOf VisualizationTask + * @apiviz.composedOf PlotItem */ public class PlotItem { /** diff --git a/src/de/lmu/ifi/dbs/elki/visualization/gui/overview/RectangleArranger.java b/src/de/lmu/ifi/dbs/elki/visualization/gui/overview/RectangleArranger.java index 77825a5f..a70393a5 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/gui/overview/RectangleArranger.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/gui/overview/RectangleArranger.java @@ -302,6 +302,10 @@ public class RectangleArranger<T> { } protected void splitRow(int bestey, double besthi) { + assert(bestey < heights.size()); + if (heights.get(bestey) - besthi <= Double.MIN_NORMAL) { + return; + } logger.debugFine("Split row " + bestey); heights.add(bestey + 1, besthi); heights.set(bestey, heights.get(bestey) - besthi); @@ -310,6 +314,10 @@ public class RectangleArranger<T> { } protected void splitCol(int bestex, double bestwi) { + assert(bestex < widths.size()); + if (widths.get(bestex) - bestwi <= Double.MIN_NORMAL) { + return; + } final int rows = heights.size(); logger.debugFine("Split column " + bestex); widths.add(bestex + 1, bestwi); @@ -371,7 +379,7 @@ public class RectangleArranger<T> { { double wsum = 0.0; for(int x = 0; x < cols; x++) { - assert (widths.get(x) > 0) : "Negative width: "+widths.get(x); + assert (widths.get(x) > 0) : "Non-positive width: "+widths.get(x); wsum += widths.get(x); } assert (Math.abs(wsum - twidth) < 1E-10); @@ -379,7 +387,7 @@ public class RectangleArranger<T> { { double hsum = 0.0; for(int y = 0; y < rows; y++) { - assert (heights.get(y) > 0) : "Negative height: "+heights.get(y); + assert (heights.get(y) > 0) : "Non-positive height: "+heights.get(y); hsum += heights.get(y); } assert (Math.abs(hsum - theight) < 1E-10); diff --git a/src/de/lmu/ifi/dbs/elki/visualization/projector/HistogramProjector.java b/src/de/lmu/ifi/dbs/elki/visualization/projector/HistogramProjector.java index 95105895..030ea954 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/projector/HistogramProjector.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/projector/HistogramProjector.java @@ -45,7 +45,8 @@ import de.lmu.ifi.dbs.elki.visualization.visualizers.visunproj.LabelVisFactory; * * @author Erich Schubert * - * @apiviz.has LinearScale + * @apiviz.uses ScalesResult + * @apiviz.uses Projection1D * * @param <V> Vector type */ diff --git a/src/de/lmu/ifi/dbs/elki/visualization/projector/ParallelPlotFactory.java b/src/de/lmu/ifi/dbs/elki/visualization/projector/ParallelPlotFactory.java index 6dcb6271..8b110967 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/projector/ParallelPlotFactory.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/projector/ParallelPlotFactory.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.visualization.projector; along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-import java.util.Iterator;
+import java.util.Collection;
import de.lmu.ifi.dbs.elki.data.NumberVector;
import de.lmu.ifi.dbs.elki.data.type.TypeUtil;
@@ -49,9 +49,8 @@ public class ParallelPlotFactory implements ProjectorFactory { @Override
public void processNewResult(HierarchicalResult baseResult, Result newResult) {
- Iterator<Relation<?>> rels = ResultUtil.filteredResults(newResult, Relation.class);
- while (rels.hasNext()) {
- Relation<?> rel = rels.next();
+ Collection<Relation<?>> rels = ResultUtil.filterResults(newResult, Relation.class);
+ for(Relation<?> rel : rels) {
// TODO: multi-relational parallel plots
if(TypeUtil.NUMBER_VECTOR_FIELD.isAssignableFromType(rel.getDataTypeInformation())) {
@SuppressWarnings("unchecked")
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/projector/ScatterPlotProjector.java b/src/de/lmu/ifi/dbs/elki/visualization/projector/ScatterPlotProjector.java index 755b6fd2..f251733e 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/projector/ScatterPlotProjector.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/projector/ScatterPlotProjector.java @@ -47,7 +47,8 @@ import de.lmu.ifi.dbs.elki.visualization.visualizers.visunproj.LabelVisFactory; * * @author Erich Schubert * - * @apiviz.has LinearScale + * @apiviz.uses ScalesResult + * @apiviz.uses Projection2D * * @param <V> Vector type */ diff --git a/src/de/lmu/ifi/dbs/elki/visualization/style/ClassStylingPolicy.java b/src/de/lmu/ifi/dbs/elki/visualization/style/ClassStylingPolicy.java index 5468a764..82000e42 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/style/ClassStylingPolicy.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/style/ClassStylingPolicy.java @@ -23,9 +23,8 @@ package de.lmu.ifi.dbs.elki.visualization.style; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.Iterator; - -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; /** * Styling policy that is based on <em>classes</em>, for example clusters or @@ -41,7 +40,7 @@ public interface ClassStylingPolicy extends StylingPolicy { * @param id Object ID * @return Style number */ - public int getStyleForDBID(DBID id); + public int getStyleForDBID(DBIDRef id); /** * Get the minimum style in use. @@ -63,5 +62,5 @@ public interface ClassStylingPolicy extends StylingPolicy { * @param cnum Class number * @return Iterator over object IDs */ - public Iterator<DBID> iterateClass(int cnum); + public DBIDIter iterateClass(int cnum); }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/visualization/style/ClusterStylingPolicy.java b/src/de/lmu/ifi/dbs/elki/visualization/style/ClusterStylingPolicy.java index 9680ec1f..1d329736 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/style/ClusterStylingPolicy.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/style/ClusterStylingPolicy.java @@ -32,7 +32,8 @@ import java.util.List; import de.lmu.ifi.dbs.elki.data.Cluster; import de.lmu.ifi.dbs.elki.data.Clustering; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.logging.LoggingUtil; @@ -92,7 +93,7 @@ public class ClusterStylingPolicy implements ClassStylingPolicy { } @Override - public int getStyleForDBID(DBID id) { + public int getStyleForDBID(DBIDRef id) { for(int i = 0; i < ids.size(); i++) { if(ids.get(i).contains(id)) { return i; @@ -102,7 +103,7 @@ public class ClusterStylingPolicy implements ClassStylingPolicy { } @Override - public int getColorForDBID(DBID id) { + public int getColorForDBID(DBIDRef id) { for(int i = 0; i < ids.size(); i++) { if(ids.get(i).contains(id)) { return colors.get(i); @@ -122,8 +123,8 @@ public class ClusterStylingPolicy implements ClassStylingPolicy { } @Override - public Iterator<DBID> iterateClass(int cnum) { - return ids.get(cnum).iterator(); + public DBIDIter iterateClass(int cnum) { + return ids.get(cnum).iter(); } /** diff --git a/src/de/lmu/ifi/dbs/elki/visualization/style/StyleLibrary.java b/src/de/lmu/ifi/dbs/elki/visualization/style/StyleLibrary.java index 3de935a2..588c0bc8 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/style/StyleLibrary.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/style/StyleLibrary.java @@ -182,6 +182,11 @@ public interface StyleLibrary { final static String OPACITY = "opacity"; /** + * XY curve styling. + */ + static final String XYCURVE = "xycurve"; + + /** * Retrieve a color for an item * * @param name Reference name diff --git a/src/de/lmu/ifi/dbs/elki/visualization/style/StyleResult.java b/src/de/lmu/ifi/dbs/elki/visualization/style/StyleResult.java index c617bea6..fe8a73ce 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/style/StyleResult.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/style/StyleResult.java @@ -28,6 +28,9 @@ import de.lmu.ifi.dbs.elki.result.Result; * Result to encapsulate active styling rules. * * @author Erich Schubert + * + * @apiviz.landmark + * @apiviz.composedOf StylingPolicy */ // TODO: pull style library etc. from VisualizerContext here? public class StyleResult implements Result { diff --git a/src/de/lmu/ifi/dbs/elki/visualization/style/StylingPolicy.java b/src/de/lmu/ifi/dbs/elki/visualization/style/StylingPolicy.java index a46d7c98..d2390ea0 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/style/StylingPolicy.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/style/StylingPolicy.java @@ -1,7 +1,5 @@ package de.lmu.ifi.dbs.elki.visualization.style; -import de.lmu.ifi.dbs.elki.database.ids.DBID; - /* This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures @@ -25,6 +23,8 @@ import de.lmu.ifi.dbs.elki.database.ids.DBID; along with this program. If not, see <http://www.gnu.org/licenses/>. */ +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; + /** * Styling policy. * @@ -43,5 +43,5 @@ public interface StylingPolicy { * @param id Object ID * @return Color value */ - public int getColorForDBID(DBID id); + public int getColorForDBID(DBIDRef id); } diff --git a/src/de/lmu/ifi/dbs/elki/visualization/style/classic.properties b/src/de/lmu/ifi/dbs/elki/visualization/style/classic.properties index ea1758e7..fe9f3e96 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/style/classic.properties +++ b/src/de/lmu/ifi/dbs/elki/visualization/style/classic.properties @@ -68,7 +68,7 @@ plot.polygons.line-width=0.001 plot.polygons.color=grey ## Curve vis (ROC curves) labels: -curve.labels.text-size=0.04 +xycurve.text-size=0.03 # Text size in overview plot overview.labels.text-size=0.08 diff --git a/src/de/lmu/ifi/dbs/elki/visualization/style/default.properties b/src/de/lmu/ifi/dbs/elki/visualization/style/default.properties index 1dd9a9c7..ca892a6e 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/style/default.properties +++ b/src/de/lmu/ifi/dbs/elki/visualization/style/default.properties @@ -64,7 +64,7 @@ plot.polygons.line-width=0.001 plot.polygons.color=grey ## Curve vis (ROC curves) labels: -curve.labels.text-size=0.04 +xycurve.text-size=0.03 # Text size in overview plot overview.labels.text-size=0.08 diff --git a/src/de/lmu/ifi/dbs/elki/visualization/style/greyscale.properties b/src/de/lmu/ifi/dbs/elki/visualization/style/greyscale.properties index d31e6ec4..0e7c96e0 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/style/greyscale.properties +++ b/src/de/lmu/ifi/dbs/elki/visualization/style/greyscale.properties @@ -68,7 +68,7 @@ plot.polygons.line-width=0.001 plot.polygons.color=dimgrey ## Curve vis (ROC curves) labels: -curve.labels.text-size=0.04 +xycurve.text-size=0.03 # Text size in overview plot overview.labels.text-size=0.08 diff --git a/src/de/lmu/ifi/dbs/elki/visualization/style/neon.properties b/src/de/lmu/ifi/dbs/elki/visualization/style/neon.properties index 311f0884..81515031 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/style/neon.properties +++ b/src/de/lmu/ifi/dbs/elki/visualization/style/neon.properties @@ -68,7 +68,7 @@ plot.polygons.line-width=0.001 plot.polygons.color=grey ## Curve vis (ROC curves) labels: -curve.labels.text-size=0.04 +xycurve.text-size=0.03 # Text size in overview plot overview.labels.text-size=0.08 diff --git a/src/de/lmu/ifi/dbs/elki/visualization/style/presentation.properties b/src/de/lmu/ifi/dbs/elki/visualization/style/presentation.properties index 31c2b569..8c597066 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/style/presentation.properties +++ b/src/de/lmu/ifi/dbs/elki/visualization/style/presentation.properties @@ -68,7 +68,7 @@ plot.polygons.line-width=0.001 plot.polygons.color=grey ## Curve vis (ROC curves) labels: -curve.labels.text-size=0.06 +xycurve.text-size=0.04 # Text size in overview plot overview.labels.text-size=0.08 diff --git a/src/de/lmu/ifi/dbs/elki/visualization/style/print.properties b/src/de/lmu/ifi/dbs/elki/visualization/style/print.properties new file mode 100644 index 00000000..930a8d1c --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/visualization/style/print.properties @@ -0,0 +1,85 @@ +## Libraries +lines-library = SolidLineStyleLibrary +marker-library = PrettyMarkers + +## Default foreground color +color=black +## Default font family +font-family='Times New Roman', serif +## Default background color +background-color=white +## Default line width +line-width=0.01 +## Text font size scale +text-size=0.03 +## Text color +text-color=black +## Default margin (relative) +margin.size=0.10 + +## Named color for the page background +page.background-color=white +## Background color for plot area +plot.background-color=none + +## Named color for a typical axis +axis.color=black +## Named color for a typical axis tick mark +axis.tick.color=grey +## Named color for a typical axis tick mark +axis.tick.minor.color=silver +## Named color for a typical axis label +axis.label.color=black +## Axis line width +axis.line-width=0.004 +## Axis tick width +axis.tick.line-width=0.004 +## Axis label font size +axis.label.text-size=0.03 + +## Named color for the background of the key box +key.background-color=none +## Named color for a label in the key part +key.label.color=black + +## A list of color names for data lines +# We stick to primary colors first to have the least issues +# The first two colors are red and blue to help red-green blind people. +# Yellow usually offers bad contrast, therefore comes late. +# Magenta often shows up too similar to red, cyan too similar to blue in print. +colorset=black,#ed420e,#fdca19,#4548a5,#7ebd3a,#a81e51,#00748b,#fa8116,#512d85,#008a7a,#fea918,#019d60,#cfde3d,#015a9c,#7b1760 +## Line width scaling (for graphs) +plot.line-width=0.005 +## For the cluster order +plot.clusterorder.line-width=0.003 +## Bubble sizes (relative) +plot.bubble.size=0.10 +## Marker size (relative) +plot.marker.size=0.01 +## Greyed out color +plot.grey=grey +## Dot size +plot.dot.size=0.004 +## Reference points +plot.referencepoints.size=0.006 +plot.referencepoints.color=red +## Polygons +plot.polygons.line-width=0.001 +plot.polygons.color=grey + +## Curve vis (ROC curves) labels: +xycurve.text-size=0.04 + +# Text size in overview plot +overview.labels.text-size=0.08 + +## Selection colors +plot.selection.color=darkblue +plot.selection.opacity=0.4 +plot.selection.size=0.015 + +## Circle segment colors. Will be interpolated to produce extra classes. +segments.border.color=#FF0073 +segments.hover.color=#73ff00 +segments.cluster.first.color=#404050 +segments.cluster.second.color=#D0D0F0
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/visualization/svg/SVGPlot.java b/src/de/lmu/ifi/dbs/elki/visualization/svg/SVGPlot.java index 81299b10..e45c7b5a 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/svg/SVGPlot.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/svg/SVGPlot.java @@ -50,8 +50,10 @@ import org.apache.batik.transcoder.image.JPEGTranscoder; import org.apache.batik.transcoder.image.PNGTranscoder; import org.apache.batik.util.SVGConstants; import org.w3c.dom.DOMImplementation; +import org.w3c.dom.Document; import org.w3c.dom.DocumentType; import org.w3c.dom.Element; +import org.w3c.dom.Node; import org.w3c.dom.events.Event; import org.w3c.dom.svg.SVGDocument; import org.w3c.dom.svg.SVGPoint; @@ -84,6 +86,11 @@ public class SVGPlot { public static final double DEFAULT_QUALITY = 0.85; /** + * Attribute to block export of element. + */ + public static final String NO_EXPORT_ATTRIBUTE = "noexport"; + + /** * SVG document we plot to. */ private SVGDocument document; @@ -295,7 +302,7 @@ public class SVGPlot { cssman.addClass(cls); } catch(CSSNamingConflict e) { - //de.lmu.ifi.dbs.elki.logging.LoggingUtil.exception(e); + // de.lmu.ifi.dbs.elki.logging.LoggingUtil.exception(e); } } @@ -366,7 +373,7 @@ public class SVGPlot { * @return cloned document */ protected SVGDocument cloneDocument() { - return (SVGDocument) new CloneInlineImages().cloneDocument(SVGDOMImplementation.getDOMImplementation(), document); + return (SVGDocument) new CloneNoExport().cloneDocument(SVGDOMImplementation.getDOMImplementation(), document); } /** @@ -640,4 +647,26 @@ public class SVGPlot { public void setDisableInteractions(boolean disable) { disableInteractions = disable; } + + /** + * Class to skip nodes during cloning that have the "noexport" attribute set. + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + protected class CloneNoExport extends CloneInlineImages { + @Override + public Node cloneNode(Document doc, Node eold) { + // Skip elements with noexport attribute set + if(eold instanceof Element) { + Element eeold = (Element) eold; + String vis = eeold.getAttribute(NO_EXPORT_ATTRIBUTE); + if(vis != null && vis.length() > 0) { + return null; + } + } + return super.cloneNode(doc, eold); + } + } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/visualization/svg/SVGScoreBar.java b/src/de/lmu/ifi/dbs/elki/visualization/svg/SVGScoreBar.java index 185a6061..ac352169 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/svg/SVGScoreBar.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/svg/SVGScoreBar.java @@ -1,5 +1,28 @@ package de.lmu.ifi.dbs.elki.visualization.svg; +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + import java.text.NumberFormat; import org.apache.batik.util.SVGConstants; @@ -89,20 +112,21 @@ public class SVGScoreBar { bar.setAttribute(SVGConstants.SVG_FILL_ATTRIBUTE, "#a0a0a0"); bar.setAttribute(SVGConstants.SVG_STROKE_ATTRIBUTE, "#a0a0a0"); bar.setAttribute(SVGConstants.SVG_STROKE_WIDTH_ATTRIBUTE, "" + height * 0.01); - - double fpos = (fill / size) * (width - (0.04 * height)); - Element chart = svgp.svgRect(x + 0.02 * height, y + 0.02 * height, fpos, height - 0.04 * height); - chart.setAttribute(SVGConstants.SVG_FILL_ATTRIBUTE, "#d4e4f1"); - chart.setAttribute(SVGConstants.SVG_STROKE_ATTRIBUTE, "#a0a0a0"); - chart.setAttribute(SVGConstants.SVG_STROKE_WIDTH_ATTRIBUTE, "" + height * 0.01); - - barchart = svgp.svgElement(SVGConstants.SVG_G_TAG); barchart.appendChild(bar); - barchart.appendChild(chart); + + if(fill >= 0 && fill <= size + 1) { + double fpos = (fill / size) * (width - (0.04 * height)); + Element chart = svgp.svgRect(x + 0.02 * height, y + 0.02 * height, fpos, height - 0.04 * height); + chart.setAttribute(SVGConstants.SVG_FILL_ATTRIBUTE, "#d4e4f1"); + chart.setAttribute(SVGConstants.SVG_STROKE_ATTRIBUTE, "#a0a0a0"); + chart.setAttribute(SVGConstants.SVG_STROKE_WIDTH_ATTRIBUTE, "" + height * 0.01); + barchart.appendChild(chart); + } // Draw the values: if(format != null) { - Element lbl = svgp.svgText(x + 0.05 * width, y + 0.75 * height, FormatUtil.format(fill, format)); + String num = Double.isNaN(fill) ? "NaN" : FormatUtil.format(fill, format); + Element lbl = svgp.svgText(x + 0.05 * width, y + 0.75 * height, num); lbl.setAttribute(SVGConstants.SVG_STYLE_ATTRIBUTE, "font-size: " + 0.75 * height + "; font-weight: bold"); barchart.appendChild(lbl); } diff --git a/src/de/lmu/ifi/dbs/elki/visualization/svg/SVGSimpleLinearAxis.java b/src/de/lmu/ifi/dbs/elki/visualization/svg/SVGSimpleLinearAxis.java index 9a676c38..987915b7 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/svg/SVGSimpleLinearAxis.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/svg/SVGSimpleLinearAxis.java @@ -41,7 +41,6 @@ import de.lmu.ifi.dbs.elki.visualization.style.StyleLibrary; * @apiviz.uses CSSClassManager * @apiviz.uses LinearScale * @apiviz.uses StyleLibrary - * @apiviz.uses STYLE * @apiviz.uses Element oneway - - «create» */ public class SVGSimpleLinearAxis { @@ -51,14 +50,14 @@ public class SVGSimpleLinearAxis { * * @apiviz.exclude */ - private enum ALIGNMENT { + private enum Alignment { LL, RL, LC, RC, LR, RR } /** * Labeling style: left-handed, right-handed, no ticks, labels at ends * - * @author Erich Schubert + * @apiviz.exclude */ public enum LabelStyle { LEFTHAND, RIGHTHAND, NOLABELS, NOTHING, ENDLABEL @@ -128,11 +127,11 @@ public class SVGSimpleLinearAxis { SVGUtil.setCSSClass(line, CSS_AXIS); parent.appendChild(line); - double tx = x2 - x1; - double ty = y2 - y1; + final double tx = x2 - x1; + final double ty = y2 - y1; // ticks are orthogonal - double tw = ty * 0.01; - double th = -tx * 0.01; + final double tw = ty * 0.01; + final double th = -tx * 0.01; // choose where to print labels. final boolean labels, ticks; @@ -152,24 +151,24 @@ public class SVGSimpleLinearAxis { labels = false; ticks = false; } - ALIGNMENT pos = ALIGNMENT.LL; + Alignment pos = Alignment.LL; if(labels) { double angle = Math.atan2(ty, tx); // System.err.println(tx + " " + (-ty) + " " + angle); if(angle > 2.6) { // pi .. 2.6 = 180 .. 150 - pos = labelstyle == LabelStyle.RIGHTHAND ? ALIGNMENT.RC : ALIGNMENT.LC; + pos = labelstyle == LabelStyle.RIGHTHAND ? Alignment.RC : Alignment.LC; } else if(angle > 0.5) { // 2.3 .. 0.7 = 130 .. 40 - pos = labelstyle == LabelStyle.RIGHTHAND ? ALIGNMENT.RR : ALIGNMENT.LL; + pos = labelstyle == LabelStyle.RIGHTHAND ? Alignment.RR : Alignment.LL; } else if(angle > -0.5) { // 0.5 .. -0.5 = 30 .. -30 - pos = labelstyle == LabelStyle.RIGHTHAND ? ALIGNMENT.RC : ALIGNMENT.LC; + pos = labelstyle == LabelStyle.RIGHTHAND ? Alignment.RC : Alignment.LC; } else if(angle > -2.6) { // -0.5 .. -2.6 = -30 .. -150 - pos = labelstyle == LabelStyle.RIGHTHAND ? ALIGNMENT.RL : ALIGNMENT.LR; + pos = labelstyle == LabelStyle.RIGHTHAND ? Alignment.RL : Alignment.LR; } else { // -2.6 .. -pi = -150 .. -180 - pos = labelstyle == LabelStyle.RIGHTHAND ? ALIGNMENT.RC : ALIGNMENT.LC; + pos = labelstyle == LabelStyle.RIGHTHAND ? Alignment.RC : Alignment.LC; } } // vertical text offset; align approximately with middle instead of @@ -178,7 +177,28 @@ public class SVGSimpleLinearAxis { // draw ticks on x axis if(ticks || labels) { - for(double tick = scale.getMin(); tick <= scale.getMax() + scale.getRes() / 10; tick += scale.getRes()) { + int sw = 1; + { // Compute how many ticks to draw + int numticks = (int) ((scale.getMax() - scale.getMin()) / scale.getRes()); + double tlen = Math.sqrt(tx * tx + ty * ty); + double minl = 10 * style.getLineWidth(StyleLibrary.AXIS_TICK); + // Try proper divisors first. + if(sw * tlen / numticks < minl) { + for(int i = 2; i <= (numticks >> 1); i++) { + if(numticks % i == 0) { + if(i * tlen / numticks >= minl) { + sw = i; + break; + } + } + } + } + // Otherwise, also allow non-divisors. + if(sw * tlen / numticks < minl) { + sw = (int)Math.floor(minl * numticks / tlen); + } + } + for(double tick = scale.getMin(); tick <= scale.getMax() + scale.getRes() / 10; tick += sw * scale.getRes()) { double x = x1 + tx * scale.getScaled(tick); double y = y1 + ty * scale.getScaled(tick); if(ticks) { diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/VisualizerUtil.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/VisualizerUtil.java index a3b0e3f4..1bc9b332 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/VisualizerUtil.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/VisualizerUtil.java @@ -24,6 +24,7 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers; */ import java.util.Iterator; +import java.util.List; import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.data.type.SimpleTypeInformation; @@ -34,7 +35,6 @@ import de.lmu.ifi.dbs.elki.result.HierarchicalResult; import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil; import de.lmu.ifi.dbs.elki.utilities.iterator.AbstractFilteredIterator; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; import de.lmu.ifi.dbs.elki.visualization.VisualizationTask; import de.lmu.ifi.dbs.elki.visualization.VisualizerContext; @@ -53,9 +53,9 @@ public final class VisualizerUtil { * @return Visualizer context */ public static VisualizerContext getContext(HierarchicalResult baseResult) { - IterableIterator<VisualizerContext> iter = ResultUtil.filteredResults(baseResult, VisualizerContext.class); - if(iter.hasNext()) { - return iter.next(); + List<VisualizerContext> contexts = ResultUtil.filterResults(baseResult, VisualizerContext.class); + if(!contexts.isEmpty()) { + return contexts.get(0); } else { return null; @@ -106,7 +106,7 @@ public final class VisualizerUtil { public static void setVisible(VisualizerContext context, VisualizationTask task, boolean visibility) { // Hide other tools if(visibility && VisualizerUtil.isTool(task)) { - final Iterable<VisualizationTask> visualizers = ResultUtil.filteredResults(context.getResult(), VisualizationTask.class); + final List<VisualizationTask> visualizers = ResultUtil.filterResults(context.getResult(), VisualizationTask.class); for(VisualizationTask other : visualizers) { if(other != task && VisualizerUtil.isTool(other) && VisualizerUtil.isVisible(other)) { other.put(VisualizationTask.META_VISIBLE, false); @@ -131,6 +131,18 @@ public final class VisualizerUtil { } /** + * Utility function to test for a visualizer being "no export". + * + * @param vis Visualizer to test + * @return true when not to export + */ + public static boolean isNoExport(VisualizationTask vis) { + // Currently enabled? + Boolean noexport = vis.getGenerics(VisualizationTask.META_NOEXPORT, Boolean.class); + return (noexport != null) && noexport; + } + + /** * Utility function to test for a visualizer having options. * * @param vis Visualizer to test @@ -149,9 +161,9 @@ public final class VisualizerUtil { * @return Iterator over suitable representations */ // TODO: move to DatabaseUtil? - public static IterableIterator<Relation<? extends NumberVector<?, ?>>> iterateVectorFieldRepresentations(final Result result) { - Iterator<Relation<?>> parent = ResultUtil.filteredResults(result, Relation.class); - return new VectorspaceIterator(parent); + public static Iterator<Relation<? extends NumberVector<?, ?>>> iterateVectorFieldRepresentations(final Result result) { + List<Relation<?>> parent = ResultUtil.filterResults(result, Relation.class); + return new VectorspaceIterator(parent.iterator()); } /** @@ -161,7 +173,7 @@ public final class VisualizerUtil { * * @apiviz.exclude */ - private static class VectorspaceIterator extends AbstractFilteredIterator<Relation<?>, Relation<? extends NumberVector<?, ?>>> implements IterableIterator<Relation<? extends NumberVector<?, ?>>> { + private static class VectorspaceIterator extends AbstractFilteredIterator<Relation<?>, Relation<? extends NumberVector<?, ?>>> { /** Parent iterator */ private Iterator<Relation<?>> parent; @@ -187,11 +199,6 @@ public final class VisualizerUtil { } return (Relation<? extends NumberVector<?, ?>>) nextobj; } - - @Override - public Iterator<Relation<? extends NumberVector<?, ?>>> iterator() { - return this; - } }; /** diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/histogram/ColoredHistogramVisualizer.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/histogram/ColoredHistogramVisualizer.java index d518d0cf..e300a3c4 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/histogram/ColoredHistogramVisualizer.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/histogram/ColoredHistogramVisualizer.java @@ -23,13 +23,13 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.histogram; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.Iterator; +import java.util.Collection; import org.apache.batik.util.SVGConstants; import org.w3c.dom.Element; import de.lmu.ifi.dbs.elki.data.NumberVector; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.logging.LoggingUtil; import de.lmu.ifi.dbs.elki.math.DoubleMinMax; @@ -39,9 +39,9 @@ import de.lmu.ifi.dbs.elki.math.scales.LinearScale; import de.lmu.ifi.dbs.elki.result.HierarchicalResult; import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil; +import de.lmu.ifi.dbs.elki.result.SamplingResult; import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; import de.lmu.ifi.dbs.elki.utilities.exceptions.ObjectNotFoundException; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.GreaterEqualConstraint; @@ -111,6 +111,11 @@ public class ColoredHistogramVisualizer<NV extends NumberVector<NV, ?>> extends private StyleResult style; /** + * Sampling result + */ + private SamplingResult sample; + + /** * Constructor. * * @param task Visualization task @@ -123,6 +128,7 @@ public class ColoredHistogramVisualizer<NV extends NumberVector<NV, ?>> extends this.bins = bins; this.relation = task.getRelation(); this.style = task.getResult(); + this.sample = ResultUtil.getSamplingResult(relation); context.addResultListener(this); } @@ -180,10 +186,12 @@ public class ColoredHistogramVisualizer<NV extends NumberVector<NV, ?>> extends double[] inc = new double[cols]; inc[0] = frac; inc[snum + 1] = frac; - for(Iterator<DBID> iter = cspol.iterateClass(snum + off); iter.hasNext();) { - DBID id = iter.next(); + for(DBIDIter iter = cspol.iterateClass(snum + off); iter.valid(); iter.advance()) { + if(!sample.getSample().contains(iter)) { + continue; // TODO: can we test more efficiently than this? + } try { - double pos = proj.fastProjectDataToRenderSpace(relation.get(id)) / Projection.SCALE; + double pos = proj.fastProjectDataToRenderSpace(relation.get(iter)) / Projection.SCALE; histogram.aggregate(pos, inc); } catch(ObjectNotFoundException e) { @@ -196,8 +204,8 @@ public class ColoredHistogramVisualizer<NV extends NumberVector<NV, ?>> extends // Actual data distribution. double[] inc = new double[cols]; inc[0] = frac; - for(DBID id : relation.iterDBIDs()) { - double pos = proj.fastProjectDataToRenderSpace(relation.get(id)) / Projection.SCALE; + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + double pos = proj.fastProjectDataToRenderSpace(relation.get(iditer)) / Projection.SCALE; histogram.aggregate(pos, inc); } } @@ -332,7 +340,7 @@ public class ColoredHistogramVisualizer<NV extends NumberVector<NV, ?>> extends * @author Erich Schubert * * @apiviz.stereotype factory - * @apiviz.uses P1DHistogramVisualizer oneway - - «create» + * @apiviz.uses ColoredHistogramVisualizer oneway - - «create» * * @param <NV> Number vector type */ @@ -391,9 +399,9 @@ public class ColoredHistogramVisualizer<NV extends NumberVector<NV, ?>> extends @Override public void processNewResult(HierarchicalResult baseResult, Result result) { // Find a style result to visualize: - IterableIterator<StyleResult> styleres = ResultUtil.filteredResults(result, StyleResult.class); + Collection<StyleResult> styleres = ResultUtil.filterResults(result, StyleResult.class); for(StyleResult c : styleres) { - IterableIterator<HistogramProjector<?>> ps = ResultUtil.filteredResults(baseResult, HistogramProjector.class); + Collection<HistogramProjector<?>> ps = ResultUtil.filterResults(baseResult, HistogramProjector.class); for(HistogramProjector<?> p : ps) { // register self final VisualizationTask task = new VisualizationTask(CNAME, c, p.getRelation(), this); diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/optics/OPTICSClusterVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/optics/OPTICSClusterVisualization.java index f69bdc59..2369588b 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/optics/OPTICSClusterVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/optics/OPTICSClusterVisualization.java @@ -23,8 +23,10 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.optics; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.Iterator; +import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.batik.util.SVGConstants; import org.w3c.dom.Element; @@ -38,8 +40,8 @@ import de.lmu.ifi.dbs.elki.logging.Logging; import de.lmu.ifi.dbs.elki.result.HierarchicalResult; import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; import de.lmu.ifi.dbs.elki.visualization.VisualizationTask; +import de.lmu.ifi.dbs.elki.visualization.colors.ColorLibrary; import de.lmu.ifi.dbs.elki.visualization.css.CSSClass; import de.lmu.ifi.dbs.elki.visualization.projector.OPTICSProjector; import de.lmu.ifi.dbs.elki.visualization.style.StyleLibrary; @@ -103,9 +105,8 @@ public class OPTICSClusterVisualization<D extends Distance<D>> extends AbstractO */ @SuppressWarnings("unchecked") protected static Clustering<OPTICSModel> findOPTICSClustering(Result result) { - Iterator<Clustering<?>> cs = ResultUtil.filteredResults(result, Clustering.class); - while (cs.hasNext()) { - Clustering<?> clus = cs.next(); + Collection<Clustering<?>> cs = ResultUtil.filterResults(result, Clustering.class); + for(Clustering<?> clus : cs) { if(clus.getToplevelClusters().size() == 0) { continue; } @@ -126,7 +127,15 @@ public class OPTICSClusterVisualization<D extends Distance<D>> extends AbstractO protected void redraw() { makeLayerElement(); addCSSClasses(); - drawClusters(clus.getToplevelClusters(), 1); + + ColorLibrary colors = context.getStyleLibrary().getColorSet(StyleLibrary.PLOT); + HashMap<Cluster<?>, String> colormap = new HashMap<Cluster<?>, String>(); + int cnum = 0; + for (Cluster<?> c : clus.getAllClusters()) { + colormap.put(c, colors.getColor(cnum)); + cnum++; + } + drawClusters(clus.getToplevelClusters(), 1, colormap); } /** @@ -134,9 +143,11 @@ public class OPTICSClusterVisualization<D extends Distance<D>> extends AbstractO * * @param clusters Current set of clusters * @param depth Recursion depth + * @param colormap Color mapping */ - private void drawClusters(List<Cluster<OPTICSModel>> clusters, int depth) { + private void drawClusters(List<Cluster<OPTICSModel>> clusters, int depth, Map<Cluster<?>,String> colormap) { final double scale = StyleLibrary.SCALE; + for(Cluster<OPTICSModel> cluster : clusters) { try { OPTICSModel model = cluster.getModel(); @@ -145,6 +156,10 @@ public class OPTICSClusterVisualization<D extends Distance<D>> extends AbstractO final double y = plotheight + depth * scale * 0.01; Element e = svgp.svgLine(x1, y, x2, y); SVGUtil.addCSSClass(e, CSS_BRACKET); + String color = colormap.get(cluster); + if (color != null) { + SVGUtil.setAtt(e, SVGConstants.SVG_STYLE_ATTRIBUTE, SVGConstants.CSS_STROKE_PROPERTY+":"+color); + } layer.appendChild(e); } catch(ClassCastException e) { @@ -153,7 +168,7 @@ public class OPTICSClusterVisualization<D extends Distance<D>> extends AbstractO // Descend final List<Cluster<OPTICSModel>> children = cluster.getChildren(); if(children != null) { - drawClusters(children, depth + 1); + drawClusters(children, depth + 1, colormap); } } } @@ -190,7 +205,7 @@ public class OPTICSClusterVisualization<D extends Distance<D>> extends AbstractO @Override public void processNewResult(HierarchicalResult baseResult, Result result) { - IterableIterator<OPTICSProjector<?>> ops = ResultUtil.filteredResults(result, OPTICSProjector.class); + Collection<OPTICSProjector<?>> ops = ResultUtil.filterResults(result, OPTICSProjector.class); for(OPTICSProjector<?> p : ops) { final Clustering<OPTICSModel> ocl = findOPTICSClustering(baseResult); if(ocl != null) { diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/optics/OPTICSPlotCutVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/optics/OPTICSPlotCutVisualization.java index 58cd9af3..2bd5c63a 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/optics/OPTICSPlotCutVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/optics/OPTICSPlotCutVisualization.java @@ -23,6 +23,8 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.optics; along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
+import java.util.Collection;
+
import org.apache.batik.util.SVG12Constants;
import org.apache.batik.util.SVGConstants;
import org.w3c.dom.Element;
@@ -38,7 +40,6 @@ import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil;
import de.lmu.ifi.dbs.elki.result.optics.ClusterOrderResult;
import de.lmu.ifi.dbs.elki.utilities.FormatUtil;
-import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator;
import de.lmu.ifi.dbs.elki.visualization.VisualizationTask;
import de.lmu.ifi.dbs.elki.visualization.batikutil.DragableArea;
import de.lmu.ifi.dbs.elki.visualization.css.CSSClass;
@@ -275,7 +276,7 @@ public class OPTICSPlotCutVisualization<D extends Distance<D>> extends AbstractO @Override
public void processNewResult(HierarchicalResult baseResult, Result result) {
- IterableIterator<OPTICSProjector<?>> ops = ResultUtil.filteredResults(result, OPTICSProjector.class);
+ Collection<OPTICSProjector<?>> ops = ResultUtil.filterResults(result, OPTICSProjector.class);
for(OPTICSProjector<?> p : ops) {
final VisualizationTask task = new VisualizationTask(NAME, p, null, this);
task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_INTERACTIVE);
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/optics/OPTICSPlotSelectionVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/optics/OPTICSPlotSelectionVisualization.java index f8f30317..87b24df2 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/optics/OPTICSPlotSelectionVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/optics/OPTICSPlotSelectionVisualization.java @@ -23,6 +23,7 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.optics; along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
+import java.util.Collection;
import java.util.List;
import org.apache.batik.dom.events.DOMMouseEvent;
@@ -44,7 +45,6 @@ import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil;
import de.lmu.ifi.dbs.elki.result.SelectionResult;
import de.lmu.ifi.dbs.elki.result.optics.ClusterOrderEntry;
-import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator;
import de.lmu.ifi.dbs.elki.visualization.VisualizationTask;
import de.lmu.ifi.dbs.elki.visualization.batikutil.DragableArea;
import de.lmu.ifi.dbs.elki.visualization.css.CSSClass;
@@ -349,7 +349,7 @@ public class OPTICSPlotSelectionVisualization<D extends Distance<D>> extends Abs @Override
public void processNewResult(HierarchicalResult baseResult, Result result) {
- IterableIterator<OPTICSProjector<?>> ops = ResultUtil.filteredResults(result, OPTICSProjector.class);
+ Collection<OPTICSProjector<?>> ops = ResultUtil.filterResults(result, OPTICSProjector.class);
for(OPTICSProjector<?> p : ops) {
final VisualizationTask task = new VisualizationTask(NAME, p, null, this);
task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_INTERACTIVE);
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/optics/OPTICSPlotVisualizer.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/optics/OPTICSPlotVisualizer.java index da287252..d93c98cb 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/optics/OPTICSPlotVisualizer.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/optics/OPTICSPlotVisualizer.java @@ -23,6 +23,8 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.optics; along with this program. If not, see <http://www.gnu.org/licenses/>. */ +import java.util.Collection; + import org.apache.batik.util.SVGConstants; import org.w3c.dom.Element; @@ -32,7 +34,6 @@ import de.lmu.ifi.dbs.elki.logging.LoggingUtil; import de.lmu.ifi.dbs.elki.result.HierarchicalResult; import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; import de.lmu.ifi.dbs.elki.visualization.VisualizationTask; import de.lmu.ifi.dbs.elki.visualization.css.CSSClassManager.CSSNamingConflict; import de.lmu.ifi.dbs.elki.visualization.opticsplot.OPTICSPlot; @@ -110,7 +111,7 @@ public class OPTICSPlotVisualizer<D extends Distance<D>> extends AbstractOPTICSV @Override public void processNewResult(HierarchicalResult baseResult, Result result) { - IterableIterator<OPTICSProjector<?>> ops = ResultUtil.filteredResults(result, OPTICSProjector.class); + Collection<OPTICSProjector<?>> ops = ResultUtil.filterResults(result, OPTICSProjector.class); for(OPTICSProjector<?> p : ops) { // Add plots, attach visualizer final VisualizationTask task = new VisualizationTask(NAME, p, null, this); diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/optics/OPTICSSteepAreaVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/optics/OPTICSSteepAreaVisualization.java index 95b3d53b..4e557a74 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/optics/OPTICSSteepAreaVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/optics/OPTICSSteepAreaVisualization.java @@ -24,6 +24,7 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.optics; */ import java.awt.Color; +import java.util.Collection; import java.util.List; import org.apache.batik.util.SVGConstants; @@ -40,7 +41,6 @@ import de.lmu.ifi.dbs.elki.result.ResultUtil; import de.lmu.ifi.dbs.elki.result.SelectionResult; import de.lmu.ifi.dbs.elki.result.optics.ClusterOrderEntry; import de.lmu.ifi.dbs.elki.result.optics.ClusterOrderResult; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; import de.lmu.ifi.dbs.elki.visualization.VisualizationTask; import de.lmu.ifi.dbs.elki.visualization.css.CSSClass; import de.lmu.ifi.dbs.elki.visualization.opticsplot.OPTICSDistanceAdapter; @@ -195,7 +195,7 @@ public class OPTICSSteepAreaVisualization<D extends Distance<D>> extends Abstrac @Override public void processNewResult(HierarchicalResult baseResult, Result result) { - IterableIterator<OPTICSProjector<?>> ops = ResultUtil.filteredResults(result, OPTICSProjector.class); + Collection<OPTICSProjector<?>> ops = ResultUtil.filterResults(result, OPTICSProjector.class); for(OPTICSProjector<?> p : ops) { final SteepAreaResult steep = findSteepAreaResult(p.getResult()); if(steep != null) { diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/pairsegments/CircleSegmentsVisualizer.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/pairsegments/CircleSegmentsVisualizer.java index 360197ab..f0655ab6 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/pairsegments/CircleSegmentsVisualizer.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/pairsegments/CircleSegmentsVisualizer.java @@ -58,6 +58,7 @@ import de.lmu.ifi.dbs.elki.visualization.svg.SVGUtil; import de.lmu.ifi.dbs.elki.visualization.visualizers.AbstractVisFactory; import de.lmu.ifi.dbs.elki.visualization.visualizers.AbstractVisualization; import de.lmu.ifi.dbs.elki.visualization.visualizers.Visualization; +import de.lmu.ifi.dbs.elki.visualization.visualizers.thumbs.ThumbnailVisualization; /** * Visualizer to draw circle segments of clusterings and enable interactive @@ -74,6 +75,11 @@ import de.lmu.ifi.dbs.elki.visualization.visualizers.Visualization; * * @author Sascha Goldhofer * @author Erich Schubert + * + * @apiviz.landmark + * + * @apiviz.uses Segments + * @apiviz.has SegmentsStylingPolicy */ @Reference(title = "Evaluation of Clusterings – Metrics and Visual Support", authors = "Elke Achtert, Sascha Goldhofer, Hans-Peter Kriegel, Erich Schubert, Arthur Zimek", booktitle = "Proc. 28th International Conference on Data Engineering (ICDE) 2012", url = "http://elki.dbs.ifi.lmu.de/wiki/PairSegments") public class CircleSegmentsVisualizer extends AbstractVisualization implements ResultListener { @@ -190,8 +196,10 @@ public class CircleSegmentsVisualizer extends AbstractVisualization implements R */ public CircleSegmentsVisualizer(VisualizationTask task) { super(task); - segments = task.getResult(); - policy = new SegmentsStylingPolicy(segments, context.getStyleLibrary()); + policy = task.getResult(); + segments = policy.segments; + // FIXME: handle this more generally. + policy.setStyleLibrary(context.getStyleLibrary()); // Listen for result changes (Selection changed) context.addResultListener(this); } @@ -612,6 +620,8 @@ public class CircleSegmentsVisualizer extends AbstractVisualization implements R * Proxy element to connect signals. * * @author Erich Schubert + * + * @apiviz.exclude */ private class SegmentListenerProxy implements EventListener { /** @@ -684,6 +694,7 @@ public class CircleSegmentsVisualizer extends AbstractVisualization implements R */ public Factory() { super(); + this.thumbmask |= ThumbnailVisualization.ON_STYLE; } @Override @@ -696,8 +707,16 @@ public class CircleSegmentsVisualizer extends AbstractVisualization implements R // If no comparison result found abort List<Segments> segments = ResultUtil.filterResults(result, Segments.class); for(Segments segmentResult : segments) { + SegmentsStylingPolicy policy; + List<SegmentsStylingPolicy> styles = ResultUtil.filterResults(segmentResult, SegmentsStylingPolicy.class); + if (!styles.isEmpty()) { + policy = styles.get(0); + } else { + policy = new SegmentsStylingPolicy(segmentResult); + baseResult.getHierarchy().add(segmentResult, policy); + } // create task for visualization - final VisualizationTask task = new VisualizationTask(NAME, segmentResult, null, this); + final VisualizationTask task = new VisualizationTask(NAME, policy, null, this); task.width = 2.0; task.height = 2.0; task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_INTERACTIVE); diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/pairsegments/SegmentsStylingPolicy.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/pairsegments/SegmentsStylingPolicy.java index 99ecf081..0ea700c8 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/pairsegments/SegmentsStylingPolicy.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/pairsegments/SegmentsStylingPolicy.java @@ -28,12 +28,14 @@ import java.util.Iterator; import java.util.Map.Entry; import java.util.TreeMap; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs; import de.lmu.ifi.dbs.elki.evaluation.clustering.pairsegments.Segment; import de.lmu.ifi.dbs.elki.evaluation.clustering.pairsegments.Segments; +import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.visualization.colors.ColorLibrary; import de.lmu.ifi.dbs.elki.visualization.style.ClassStylingPolicy; import de.lmu.ifi.dbs.elki.visualization.style.StyleLibrary; @@ -45,7 +47,7 @@ import de.lmu.ifi.dbs.elki.visualization.svg.SVGUtil; * @author Sascha Goldhofer * @author Erich Schubert */ -public class SegmentsStylingPolicy implements ClassStylingPolicy { +public class SegmentsStylingPolicy implements ClassStylingPolicy, Result { /** * The segments we use for visualization */ @@ -70,17 +72,16 @@ public class SegmentsStylingPolicy implements ClassStylingPolicy { * Color library (only used in compatibility mode) */ // TODO: move to abstract super class? - ColorLibrary colorset; + ColorLibrary colorset = null; /** * Constructor. * * @param segments Segments */ - public SegmentsStylingPolicy(Segments segments, StyleLibrary style) { + public SegmentsStylingPolicy(Segments segments) { super(); this.segments = segments; - this.colorset = style.getColorSet(StyleLibrary.PLOT); // get all selectable segments for(Segment segment : segments) { @@ -95,6 +96,17 @@ public class SegmentsStylingPolicy implements ClassStylingPolicy { } /** + * Assign the style library, for compatibility color styling. + * + * FIXME: handle this more generally + * + * @param style Style library + */ + public void setStyleLibrary(StyleLibrary style) { + this.colorset = style.getColorSet(StyleLibrary.PLOT); + } + + /** * Test whether a segment is selected. * * @param segment Segment to test @@ -105,7 +117,7 @@ public class SegmentsStylingPolicy implements ClassStylingPolicy { } @Override - public int getStyleForDBID(DBID id) { + public int getStyleForDBID(DBIDRef id) { Iterator<Segment> s = selectedSegments.iterator(); for(int i = 0; s.hasNext(); i++) { Segment seg = s.next(); @@ -118,10 +130,14 @@ public class SegmentsStylingPolicy implements ClassStylingPolicy { } @Override - public int getColorForDBID(DBID id) { + public int getColorForDBID(DBIDRef id) { int style = getStyleForDBID(id); - // FIXME: slow - return SVGUtil.stringToColor(colorset.getColor(style)).getRGB(); + if (colorset != null) { + // FIXME: add caching + return SVGUtil.stringToColor(colorset.getColor(style)).getRGB(); + } else { + return 0; + } } @Override @@ -136,17 +152,17 @@ public class SegmentsStylingPolicy implements ClassStylingPolicy { } @Override - public Iterator<DBID> iterateClass(int cnum) { + public DBIDIter iterateClass(int cnum) { // unselected if(cnum == -2) { - return unselectedObjects.iterator(); + return unselectedObjects.iter(); } else if(cnum == -1) { - return DBIDUtil.EMPTYDBIDS.iterator(); + return DBIDUtil.EMPTYDBIDS.iter(); } // colors DBIDs ids = selectedSegments.get(cnum).getDBIDs(); - return (ids != null) ? ids.iterator() : DBIDUtil.EMPTYDBIDS.iterator(); + return (ids != null) ? ids.iter() : DBIDUtil.EMPTYDBIDS.iter(); } /** @@ -284,4 +300,14 @@ public class SegmentsStylingPolicy implements ClassStylingPolicy { public int indexOfSegment(Segment segment) { return selectedSegments.indexOf(segment); } + + @Override + public String getLongName() { + return "Pair segments styling policy"; + } + + @Override + public String getShortName() { + return "pairsegments-styling"; + } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/LineVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/LineVisualization.java index 3a2c7c1d..8269d8fe 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/LineVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/LineVisualization.java @@ -23,19 +23,18 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.parallel; along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-import java.util.Iterator;
+import java.util.Collection;
import org.apache.batik.util.SVGConstants;
import org.w3c.dom.Element;
import de.lmu.ifi.dbs.elki.data.NumberVector;
import de.lmu.ifi.dbs.elki.database.datastore.DataStoreListener;
-import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.result.HierarchicalResult;
import de.lmu.ifi.dbs.elki.result.Result;
import de.lmu.ifi.dbs.elki.result.ResultUtil;
import de.lmu.ifi.dbs.elki.result.SamplingResult;
-import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator;
import de.lmu.ifi.dbs.elki.visualization.VisualizationTask;
import de.lmu.ifi.dbs.elki.visualization.css.CSSClass;
import de.lmu.ifi.dbs.elki.visualization.projector.ParallelPlotProjector;
@@ -99,21 +98,20 @@ public class LineVisualization extends AbstractParallelVisualization<NumberVecto StylingPolicy sp = context.getStyleResult().getStylingPolicy();
addCSSClasses(svgp, sp);
- Iterator<DBID> ids = sample.getSample().iterator();
- if(ids == null || !ids.hasNext()) {
+ DBIDIter ids = sample.getSample().iter();
+ if(ids == null || !ids.valid()) {
ids = relation.iterDBIDs();
}
if(sp instanceof ClassStylingPolicy) {
ClassStylingPolicy csp = (ClassStylingPolicy) sp;
for(int c = csp.getMinStyle(); c < csp.getMaxStyle(); c++) {
String key = DATALINE + "_" + c;
- for(Iterator<DBID> iter = csp.iterateClass(c); iter.hasNext();) {
- DBID id = iter.next();
- if(!sample.getSample().contains(id)) {
+ for(DBIDIter iter = csp.iterateClass(c); iter.valid(); iter.advance()) {
+ if(!sample.getSample().contains(iter)) {
continue; // TODO: can we test more efficiently than this?
}
SVGPath path = new SVGPath();
- double[] yPos = proj.fastProjectDataToRenderSpace(relation.get(id));
+ double[] yPos = proj.fastProjectDataToRenderSpace(relation.get(iter));
for(int i = 0; i < yPos.length; i++) {
path.drawTo(getVisibleAxisX(i), yPos[i]);
}
@@ -124,17 +122,16 @@ public class LineVisualization extends AbstractParallelVisualization<NumberVecto }
}
else {
- while(ids.hasNext()) {
- DBID id = ids.next();
+ for(; ids.valid(); ids.advance()) {
SVGPath path = new SVGPath();
- double[] yPos = proj.fastProjectDataToRenderSpace(relation.get(id));
+ double[] yPos = proj.fastProjectDataToRenderSpace(relation.get(ids));
for(int i = 0; i < yPos.length; i++) {
path.drawTo(getVisibleAxisX(i), yPos[i]);
}
Element line = path.makeElement(svgp);
SVGUtil.addCSSClass(line, DATALINE);
// assign color
- line.setAttribute(SVGConstants.SVG_STYLE_ATTRIBUTE, SVGConstants.CSS_STROKE_PROPERTY + ":" + SVGUtil.colorToString(sp.getColorForDBID(id)));
+ line.setAttribute(SVGConstants.SVG_STYLE_ATTRIBUTE, SVGConstants.CSS_STROKE_PROPERTY + ":" + SVGUtil.colorToString(sp.getColorForDBID(ids)));
layer.appendChild(line);
}
}
@@ -207,7 +204,7 @@ public class LineVisualization extends AbstractParallelVisualization<NumberVecto @Override
public void processNewResult(HierarchicalResult baseResult, Result result) {
- IterableIterator<ParallelPlotProjector<?>> ps = ResultUtil.filteredResults(result, ParallelPlotProjector.class);
+ Collection<ParallelPlotProjector<?>> ps = ResultUtil.filterResults(result, ParallelPlotProjector.class);
for(ParallelPlotProjector<?> p : ps) {
final VisualizationTask task = new VisualizationTask(NAME, p.getRelation(), p.getRelation(), this);
task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_DATA);
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/ParallelAxisVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/ParallelAxisVisualization.java index a47ea734..6209db6b 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/ParallelAxisVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/ParallelAxisVisualization.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.parallel; along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-import java.util.Iterator;
+import java.util.Collection;
import org.apache.batik.util.SVGConstants;
import org.w3c.dom.Element;
@@ -178,9 +178,8 @@ public class ParallelAxisVisualization extends AbstractParallelVisualization<Num @Override
public void processNewResult(HierarchicalResult baseResult, Result result) {
- Iterator<ParallelPlotProjector<?>> ps = ResultUtil.filteredResults(result, ParallelPlotProjector.class);
- while(ps.hasNext()) {
- ParallelPlotProjector<?> p = ps.next();
+ Collection<ParallelPlotProjector<?>> ps = ResultUtil.filterResults(result, ParallelPlotProjector.class);
+ for(ParallelPlotProjector<?> p : ps) {
final VisualizationTask task = new VisualizationTask(NAME, p, p.getRelation(), this);
task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_BACKGROUND);
baseResult.getHierarchy().add(p, task);
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/cluster/ClusterOutlineVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/cluster/ClusterOutlineVisualization.java index 821ca2a5..f34b7514 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/cluster/ClusterOutlineVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/cluster/ClusterOutlineVisualization.java @@ -23,6 +23,7 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.parallel.cluster; along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
+import java.util.Collection;
import java.util.Iterator;
import org.apache.batik.util.SVGConstants;
@@ -38,7 +39,6 @@ import de.lmu.ifi.dbs.elki.math.DoubleMinMax; import de.lmu.ifi.dbs.elki.result.HierarchicalResult;
import de.lmu.ifi.dbs.elki.result.Result;
import de.lmu.ifi.dbs.elki.result.ResultUtil;
-import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator;
import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID;
import de.lmu.ifi.dbs.elki.visualization.VisualizationTask;
import de.lmu.ifi.dbs.elki.visualization.colors.ColorLibrary;
@@ -234,11 +234,10 @@ public class ClusterOutlineVisualization extends AbstractParallelVisualization<N @Override
public void processNewResult(HierarchicalResult baseResult, Result result) {
// Find clusterings we can visualize:
- Iterator<Clustering<?>> clusterings = ResultUtil.filteredResults(result, Clustering.class);
- while(clusterings.hasNext()) {
- Clustering<?> c = clusterings.next();
+ Collection<Clustering<?>> clusterings = ResultUtil.filterResults(result, Clustering.class);
+ for(Clustering<?> c : clusterings) {
if(c.getAllClusters().size() > 0) {
- IterableIterator<ParallelPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ParallelPlotProjector.class);
+ Collection<ParallelPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ParallelPlotProjector.class);
for(ParallelPlotProjector<?> p : ps) {
final VisualizationTask task = new VisualizationTask(NAME, c, p.getRelation(), this);
task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_DATA - 1);
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/cluster/ClusterParallelMeanVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/cluster/ClusterParallelMeanVisualization.java index df731ac2..1808c241 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/cluster/ClusterParallelMeanVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/cluster/ClusterParallelMeanVisualization.java @@ -23,6 +23,7 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.parallel.cluster; along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
+import java.util.Collection;
import java.util.Iterator;
import org.apache.batik.util.SVGConstants;
@@ -172,16 +173,14 @@ public class ClusterParallelMeanVisualization extends AbstractParallelVisualizat @Override
public void processNewResult(HierarchicalResult baseResult, Result result) {
// Find clusterings we can visualize:
- Iterator<Clustering<?>> clusterings = ResultUtil.filteredResults(result, Clustering.class);
- while(clusterings.hasNext()) {
- Clustering<?> c = clusterings.next();
+ Collection<Clustering<?>> clusterings = ResultUtil.filterResults(result, Clustering.class);
+ for(Clustering<?> c : clusterings) {
if(c.getAllClusters().size() > 0) {
// Does the cluster have a model with cluster means?
Clustering<MeanModel<? extends NumberVector<?, ?>>> mcls = findMeanModel(c);
if(mcls != null) {
- Iterator<ParallelPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ParallelPlotProjector.class);
- while(ps.hasNext()) {
- ParallelPlotProjector<?> p = ps.next();
+ Collection<ParallelPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ParallelPlotProjector.class);
+ for (ParallelPlotProjector<?> p : ps) {
final VisualizationTask task = new VisualizationTask(NAME, c, p.getRelation(), this);
task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_DATA + 1);
baseResult.getHierarchy().add(c, task);
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/index/RTreeParallelVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/index/RTreeParallelVisualization.java index c36dff0c..499d14a5 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/index/RTreeParallelVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/index/RTreeParallelVisualization.java @@ -23,6 +23,7 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.parallel.index; */
import java.util.ArrayList;
+import java.util.Collection;
import org.apache.batik.util.SVGConstants;
import org.w3c.dom.Element;
@@ -37,7 +38,6 @@ import de.lmu.ifi.dbs.elki.index.tree.spatial.rstarvariants.rstar.RStarTreeNode; import de.lmu.ifi.dbs.elki.result.HierarchicalResult;
import de.lmu.ifi.dbs.elki.result.Result;
import de.lmu.ifi.dbs.elki.result.ResultUtil;
-import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator;
import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer;
import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization;
import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.Flag;
@@ -227,7 +227,7 @@ public class RTreeParallelVisualization<N extends AbstractRStarTreeNode<N, E>, E ArrayList<AbstractRStarTree<RStarTreeNode, SpatialEntry>> trees = ResultUtil.filterResults(result, AbstractRStarTree.class);
for(AbstractRStarTree<RStarTreeNode, SpatialEntry> tree : trees) {
if(tree instanceof Result) {
- IterableIterator<ParallelPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ParallelPlotProjector.class);
+ Collection<ParallelPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ParallelPlotProjector.class);
for(ParallelPlotProjector<?> p : ps) {
final VisualizationTask task = new VisualizationTask(NAME, (Result) tree, p.getRelation(), this);
task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_BACKGROUND + 2);
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/selection/SelectionAxisRangeVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/selection/SelectionAxisRangeVisualization.java index cb8ee645..b894a87b 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/selection/SelectionAxisRangeVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/selection/SelectionAxisRangeVisualization.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.parallel.selection; along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-import java.util.ArrayList;
+import java.util.Collection;
import org.apache.batik.util.SVGConstants;
import org.w3c.dom.Element;
@@ -35,7 +35,6 @@ import de.lmu.ifi.dbs.elki.result.RangeSelection; import de.lmu.ifi.dbs.elki.result.Result;
import de.lmu.ifi.dbs.elki.result.ResultUtil;
import de.lmu.ifi.dbs.elki.result.SelectionResult;
-import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator;
import de.lmu.ifi.dbs.elki.utilities.pairs.DoubleDoublePair;
import de.lmu.ifi.dbs.elki.visualization.VisualizationTask;
import de.lmu.ifi.dbs.elki.visualization.css.CSSClass;
@@ -169,9 +168,9 @@ public class SelectionAxisRangeVisualization extends AbstractParallelVisualizati @Override
public void processNewResult(HierarchicalResult baseResult, Result result) {
- final ArrayList<SelectionResult> selectionResults = ResultUtil.filterResults(result, SelectionResult.class);
+ Collection<SelectionResult> selectionResults = ResultUtil.filterResults(result, SelectionResult.class);
for(SelectionResult selres : selectionResults) {
- IterableIterator<ParallelPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ParallelPlotProjector.class);
+ Collection<ParallelPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ParallelPlotProjector.class);
for(ParallelPlotProjector<?> p : ps) {
final VisualizationTask task = new VisualizationTask(NAME, selres, p.getRelation(), this);
task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_DATA - 1);
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/selection/SelectionLineVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/selection/SelectionLineVisualization.java index f6de4133..b9a9ac48 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/selection/SelectionLineVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/selection/SelectionLineVisualization.java @@ -23,21 +23,20 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.parallel.selection; along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-import java.util.ArrayList;
+import java.util.Collection;
import org.apache.batik.util.SVGConstants;
import org.w3c.dom.Element;
import de.lmu.ifi.dbs.elki.data.NumberVector;
import de.lmu.ifi.dbs.elki.database.datastore.DataStoreListener;
-import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDs;
import de.lmu.ifi.dbs.elki.result.DBIDSelection;
import de.lmu.ifi.dbs.elki.result.HierarchicalResult;
import de.lmu.ifi.dbs.elki.result.Result;
import de.lmu.ifi.dbs.elki.result.ResultUtil;
import de.lmu.ifi.dbs.elki.result.SelectionResult;
-import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator;
import de.lmu.ifi.dbs.elki.visualization.VisualizationTask;
import de.lmu.ifi.dbs.elki.visualization.css.CSSClass;
import de.lmu.ifi.dbs.elki.visualization.projector.ParallelPlotProjector;
@@ -94,8 +93,8 @@ public class SelectionLineVisualization extends AbstractParallelVisualization<Nu if(selContext != null) {
DBIDs selection = selContext.getSelectedIds();
- for(DBID objId : selection) {
- double[] yPos = proj.fastProjectDataToRenderSpace(relation.get(objId));
+ for(DBIDIter iter = selection.iter(); iter.valid(); iter.advance()) {
+ double[] yPos = proj.fastProjectDataToRenderSpace(relation.get(iter));
SVGPath path = new SVGPath();
for(int i = 0; i < proj.getVisibleDimensions(); i++) {
@@ -152,9 +151,9 @@ public class SelectionLineVisualization extends AbstractParallelVisualization<Nu @Override
public void processNewResult(HierarchicalResult baseResult, Result result) {
- final ArrayList<SelectionResult> selectionResults = ResultUtil.filterResults(result, SelectionResult.class);
+ Collection<SelectionResult> selectionResults = ResultUtil.filterResults(result, SelectionResult.class);
for(SelectionResult selres : selectionResults) {
- IterableIterator<ParallelPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ParallelPlotProjector.class);
+ Collection<ParallelPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ParallelPlotProjector.class);
for(ParallelPlotProjector<?> p : ps) {
final VisualizationTask task = new VisualizationTask(NAME, selres, p.getRelation(), this);
task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_DATA -1);
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/selection/SelectionToolAxisRangeVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/selection/SelectionToolAxisRangeVisualization.java index ead8e5ce..2f3c6778 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/selection/SelectionToolAxisRangeVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/selection/SelectionToolAxisRangeVisualization.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.parallel.selection; along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-import java.util.ArrayList;
+import java.util.Collection;
import org.apache.batik.util.SVGConstants;
import org.w3c.dom.Element;
@@ -31,7 +31,7 @@ import org.w3c.dom.events.Event; import org.w3c.dom.svg.SVGPoint;
import de.lmu.ifi.dbs.elki.data.NumberVector;
-import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil;
import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs;
import de.lmu.ifi.dbs.elki.logging.Logging;
@@ -41,7 +41,6 @@ import de.lmu.ifi.dbs.elki.result.RangeSelection; import de.lmu.ifi.dbs.elki.result.Result;
import de.lmu.ifi.dbs.elki.result.ResultUtil;
import de.lmu.ifi.dbs.elki.result.SelectionResult;
-import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator;
import de.lmu.ifi.dbs.elki.utilities.pairs.DoubleDoublePair;
import de.lmu.ifi.dbs.elki.visualization.VisualizationTask;
import de.lmu.ifi.dbs.elki.visualization.batikutil.DragableArea;
@@ -167,8 +166,8 @@ public class SelectionToolAxisRangeVisualization extends AbstractParallelVisuali for(int i = minaxis; i < maxaxis; i++) {
double v1 = proj.fastProjectRenderToDataSpace(z1, i);
double v2 = proj.fastProjectRenderToDataSpace(z2, i);
- if (logger.isDebugging()) {
- logger.debug("Axis "+i+" dimension "+proj.getDimForVisibleAxis(i)+" "+v1+" to "+v2);
+ if(logger.isDebugging()) {
+ logger.debug("Axis " + i + " dimension " + proj.getDimForVisibleAxis(i) + " " + v1 + " to " + v2);
}
ranges[proj.getDimForVisibleAxis(i)] = new DoubleDoublePair(Math.min(v1, v2), Math.max(v1, v2));
}
@@ -236,21 +235,17 @@ public class SelectionToolAxisRangeVisualization extends AbstractParallelVisuali updateSelectionRectKoordinates(x1, x2, y1, y2, ranges);
selection.clear();
- boolean idIn = true;
- for(DBID id : relation.iterDBIDs()) {
- NumberVector<?, ?> dbTupel = relation.get(id);
- idIn = true;
+
+ candidates: for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) {
+ NumberVector<?, ?> dbTupel = relation.get(iditer);
for(int i = 0; i < dim; i++) {
if(ranges != null && ranges[i] != null) {
if(dbTupel.doubleValue(i + 1) < ranges[i].first || dbTupel.doubleValue(i + 1) > ranges[i].second) {
- idIn = false;
- break;
+ continue candidates;
}
}
}
- if(idIn == true) {
- selection.add(id);
- }
+ selection.add(iditer);
}
context.setSelection(new RangeSelection(selection, ranges));
}
@@ -297,9 +292,9 @@ public class SelectionToolAxisRangeVisualization extends AbstractParallelVisuali @Override
public void processNewResult(HierarchicalResult baseResult, Result result) {
- final ArrayList<SelectionResult> selectionResults = ResultUtil.filterResults(result, SelectionResult.class);
+ Collection<SelectionResult> selectionResults = ResultUtil.filterResults(result, SelectionResult.class);
for(SelectionResult selres : selectionResults) {
- IterableIterator<ParallelPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ParallelPlotProjector.class);
+ Collection<ParallelPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ParallelPlotProjector.class);
for(ParallelPlotProjector<?> p : ps) {
final VisualizationTask task = new VisualizationTask(NAME, selres, p.getRelation(), this);
task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_INTERACTIVE);
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/selection/SelectionToolLineVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/selection/SelectionToolLineVisualization.java index 966e754a..3e66b4cd 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/selection/SelectionToolLineVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/parallel/selection/SelectionToolLineVisualization.java @@ -24,7 +24,7 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.parallel.selection; */
import java.awt.geom.Line2D;
-import java.util.ArrayList;
+import java.util.Collection;
import org.apache.batik.dom.events.DOMMouseEvent;
import org.apache.batik.util.SVGConstants;
@@ -33,7 +33,7 @@ import org.w3c.dom.events.Event; import org.w3c.dom.svg.SVGPoint;
import de.lmu.ifi.dbs.elki.data.NumberVector;
-import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil;
import de.lmu.ifi.dbs.elki.database.ids.DBIDs;
import de.lmu.ifi.dbs.elki.database.ids.HashSetModifiableDBIDs;
@@ -42,7 +42,6 @@ import de.lmu.ifi.dbs.elki.result.HierarchicalResult; import de.lmu.ifi.dbs.elki.result.Result;
import de.lmu.ifi.dbs.elki.result.ResultUtil;
import de.lmu.ifi.dbs.elki.result.SelectionResult;
-import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator;
import de.lmu.ifi.dbs.elki.visualization.VisualizationTask;
import de.lmu.ifi.dbs.elki.visualization.batikutil.DragableArea;
import de.lmu.ifi.dbs.elki.visualization.css.CSSClass;
@@ -195,23 +194,23 @@ public class SelectionToolLineVisualization extends AbstractParallelVisualizatio selection = DBIDUtil.newHashSet(selContext.getSelectedIds());
}
int[] axisrange = getAxisRange(Math.min(p1.getX(), p2.getX()), Math.max(p1.getX(), p2.getX()));
- DBIDs objIds = ResultUtil.getSamplingResult(relation).getSample();
- for(DBID objId : objIds){
- double[] yPos = proj.fastProjectDataToRenderSpace(relation.get(objId));
+ DBIDs ids = ResultUtil.getSamplingResult(relation).getSample();
+ for(DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) {
+ double[] yPos = proj.fastProjectDataToRenderSpace(relation.get(iter));
if(checkSelected(axisrange, yPos, Math.max(p1.getX(), p2.getX()), Math.min(p1.getX(), p2.getX()), Math.max(p1.getY(), p2.getY()), Math.min(p1.getY(), p2.getY()))) {
if(mode == Mode.INVERT) {
- if(!selection.contains(objId)) {
- selection.add(objId);
+ if(!selection.contains(iter)) {
+ selection.add(iter);
}
else {
- selection.remove(objId);
+ selection.remove(iter);
}
}
else {
// In REPLACE and ADD, add objects.
// The difference was done before by not re-using the selection.
// Since we are using a set, we can just add in any case.
- selection.add(objId);
+ selection.add(iter);
}
}
}
@@ -312,9 +311,9 @@ public class SelectionToolLineVisualization extends AbstractParallelVisualizatio @Override
public void processNewResult(HierarchicalResult baseResult, Result result) {
- final ArrayList<SelectionResult> selectionResults = ResultUtil.filterResults(result, SelectionResult.class);
+ Collection<SelectionResult> selectionResults = ResultUtil.filterResults(result, SelectionResult.class);
for(SelectionResult selres : selectionResults) {
- IterableIterator<ParallelPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ParallelPlotProjector.class);
+ Collection<ParallelPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ParallelPlotProjector.class);
for(ParallelPlotProjector<?> p : ps) {
final VisualizationTask task = new VisualizationTask(NAME, selres, p.getRelation(), this);
task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_INTERACTIVE);
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/AxisVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/AxisVisualization.java index a8f7fdca..4173084d 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/AxisVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/AxisVisualization.java @@ -23,6 +23,8 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot; along with this program. If not, see <http://www.gnu.org/licenses/>. */ +import java.util.Collection; + import org.apache.batik.util.SVGConstants; import org.w3c.dom.Element; @@ -30,7 +32,6 @@ import de.lmu.ifi.dbs.elki.result.HierarchicalResult; import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil; import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; import de.lmu.ifi.dbs.elki.visualization.VisualizationTask; import de.lmu.ifi.dbs.elki.visualization.css.CSSClass; import de.lmu.ifi.dbs.elki.visualization.css.CSSClassManager.CSSNamingConflict; @@ -141,7 +142,7 @@ public class AxisVisualization extends AbstractScatterplotVisualization { @Override public void processNewResult(HierarchicalResult baseResult, Result result) { - IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(result, ScatterPlotProjector.class); + Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(result, ScatterPlotProjector.class); for(ScatterPlotProjector<?> p : ps) { final VisualizationTask task = new VisualizationTask(NAME, p, p.getRelation(), this); task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_BACKGROUND); diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/MarkerVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/MarkerVisualization.java index 7672ee93..c98ca83b 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/MarkerVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/MarkerVisualization.java @@ -23,19 +23,18 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.Iterator; +import java.util.Collection; import org.apache.batik.util.SVGConstants; import org.w3c.dom.Element; import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreListener; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.result.HierarchicalResult; import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil; import de.lmu.ifi.dbs.elki.utilities.exceptions.ObjectNotFoundException; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; import de.lmu.ifi.dbs.elki.visualization.VisualizationTask; import de.lmu.ifi.dbs.elki.visualization.projector.ScatterPlotProjector; import de.lmu.ifi.dbs.elki.visualization.style.ClassStylingPolicy; @@ -97,10 +96,12 @@ public class MarkerVisualization extends AbstractScatterplotVisualization implem if(spol instanceof ClassStylingPolicy) { ClassStylingPolicy cspol = (ClassStylingPolicy) spol; for(int cnum = cspol.getMinStyle(); cnum < cspol.getMaxStyle(); cnum++) { - for(Iterator<DBID> iter = cspol.iterateClass(cnum); iter.hasNext();) { - DBID cur = iter.next(); + for(DBIDIter iter = cspol.iterateClass(cnum); iter.valid(); iter.advance()) { + if(!sample.getSample().contains(iter)) { + continue; // TODO: can we test more efficiently than this? + } try { - final NumberVector<?, ?> vec = rel.get(cur); + final NumberVector<?, ?> vec = rel.get(iter); double[] v = proj.fastProjectDataToRenderSpace(vec); ml.useMarker(svgp, layer, v[0], v[1], cnum, marker_size); } @@ -113,12 +114,12 @@ public class MarkerVisualization extends AbstractScatterplotVisualization implem else { final String FILL = SVGConstants.CSS_FILL_PROPERTY + ":"; // Color-based styling. Fall back to dots - for(DBID id : sample.getSample()) { + for(DBIDIter iter = sample.getSample().iter(); iter.valid(); iter.advance()) { try { - double[] v = proj.fastProjectDataToRenderSpace(rel.get(id)); + double[] v = proj.fastProjectDataToRenderSpace(rel.get(iter)); Element dot = svgp.svgCircle(v[0], v[1], marker_size); SVGUtil.addCSSClass(dot, DOTMARKER); - int col = spol.getColorForDBID(id); + int col = spol.getColorForDBID(iter); SVGUtil.setAtt(dot, SVGConstants.SVG_STYLE_ATTRIBUTE, FILL + SVGUtil.colorToString(col)); layer.appendChild(dot); } @@ -160,9 +161,9 @@ public class MarkerVisualization extends AbstractScatterplotVisualization implem @Override public void processNewResult(HierarchicalResult baseResult, Result result) { // Find a style result to visualize: - IterableIterator<StyleResult> styleres = ResultUtil.filteredResults(result, StyleResult.class); + Collection<StyleResult> styleres = ResultUtil.filterResults(result, StyleResult.class); for(StyleResult c : styleres) { - IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class); + Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class); for(ScatterPlotProjector<?> p : ps) { final VisualizationTask task = new VisualizationTask(NAME, c, p.getRelation(), this); task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_DATA); diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/PolygonVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/PolygonVisualization.java index 4dc78ecf..5b7c13b4 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/PolygonVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/PolygonVisualization.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.ArrayList; +import java.util.Collection; import org.apache.batik.util.SVGConstants; import org.w3c.dom.Element; @@ -32,7 +32,7 @@ import de.lmu.ifi.dbs.elki.data.spatial.Polygon; import de.lmu.ifi.dbs.elki.data.spatial.PolygonsObject; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreListener; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; import de.lmu.ifi.dbs.elki.result.HierarchicalResult; @@ -40,7 +40,6 @@ import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil; import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil; import de.lmu.ifi.dbs.elki.utilities.exceptions.ObjectNotFoundException; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; import de.lmu.ifi.dbs.elki.visualization.VisualizationTask; import de.lmu.ifi.dbs.elki.visualization.css.CSSClass; import de.lmu.ifi.dbs.elki.visualization.projections.Projection2D; @@ -109,9 +108,9 @@ public class PolygonVisualization extends AbstractScatterplotVisualization imple svgp.updateStyleElement(); // draw data - for(DBID id : rep.iterDBIDs()) { + for(DBIDIter iditer = rep.iterDBIDs(); iditer.valid(); iditer.advance()) { try { - PolygonsObject poly = rep.get(id); + PolygonsObject poly = rep.get(iditer); if(poly == null) { continue; } @@ -163,11 +162,11 @@ public class PolygonVisualization extends AbstractScatterplotVisualization imple @Override public void processNewResult(HierarchicalResult baseResult, Result result) { - ArrayList<Relation<?>> results = ResultUtil.filterResults(result, Relation.class); + Collection<Relation<?>> results = ResultUtil.filterResults(result, Relation.class); for(Relation<?> rel : results) { if(TypeUtil.POLYGON_TYPE.isAssignableFromType(rel.getDataTypeInformation())) { // Assume that a 2d projector is using the same coordinates as the polygons. - IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class); + Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class); for(ScatterPlotProjector<?> p : ps) { if(DatabaseUtil.dimensionality(p.getRelation()) == 2) { final VisualizationTask task = new VisualizationTask(NAME, rel, p.getRelation(), this); diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/ReferencePointsVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/ReferencePointsVisualization.java index 644fc96f..056b788b 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/ReferencePointsVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/ReferencePointsVisualization.java @@ -34,7 +34,6 @@ import de.lmu.ifi.dbs.elki.result.HierarchicalResult; import de.lmu.ifi.dbs.elki.result.ReferencePointsResult; import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; import de.lmu.ifi.dbs.elki.visualization.VisualizationTask; import de.lmu.ifi.dbs.elki.visualization.css.CSSClass; import de.lmu.ifi.dbs.elki.visualization.projector.ScatterPlotProjector; @@ -126,7 +125,7 @@ public class ReferencePointsVisualization extends AbstractScatterplotVisualizati public void processNewResult(HierarchicalResult baseResult, Result result) { Collection<ReferencePointsResult<?>> rps = ResultUtil.filterResults(result, ReferencePointsResult.class); for(ReferencePointsResult<?> rp : rps) { - IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class); + Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class); for(ScatterPlotProjector<?> p : ps) { final VisualizationTask task = new VisualizationTask(NAME, rp, p.getRelation(), this); task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_DATA); diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/ToolBox2DVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/ToolBox2DVisualization.java index 7ec5f1ac..d0c05cc7 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/ToolBox2DVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/ToolBox2DVisualization.java @@ -24,6 +24,7 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot; */
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import org.apache.batik.util.SVGConstants;
@@ -37,7 +38,6 @@ import de.lmu.ifi.dbs.elki.result.HierarchicalResult; import de.lmu.ifi.dbs.elki.result.Result;
import de.lmu.ifi.dbs.elki.result.ResultUtil;
import de.lmu.ifi.dbs.elki.utilities.FormatUtil;
-import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator;
import de.lmu.ifi.dbs.elki.visualization.VisualizationTask;
import de.lmu.ifi.dbs.elki.visualization.css.CSSClass;
import de.lmu.ifi.dbs.elki.visualization.projections.CanvasSize;
@@ -126,7 +126,7 @@ public class ToolBox2DVisualization extends AbstractScatterplotVisualization { deleteChildren(container);
ArrayList<VisualizationTask> vis = new ArrayList<VisualizationTask>();
- final Iterable<VisualizationTask> visualizers = ResultUtil.filteredResults(task.getResult(), VisualizationTask.class);
+ Collection<VisualizationTask> visualizers = ResultUtil.filterResults(task.getResult(), VisualizationTask.class);
for(VisualizationTask task : visualizers) {
if(VisualizerUtil.isTool(task) && !vis.contains(task)) {
vis.add(task);
@@ -292,7 +292,7 @@ public class ToolBox2DVisualization extends AbstractScatterplotVisualization { @Override
public void processNewResult(HierarchicalResult baseResult, Result result) {
- IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(result, ScatterPlotProjector.class);
+ Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(result, ScatterPlotProjector.class);
for(ScatterPlotProjector<?> p : ps) {
final VisualizationTask task = new VisualizationTask(NAME, p, p.getRelation(), this);
task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_INTERACTIVE);
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/TooltipScoreVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/TooltipScoreVisualization.java index 838390d0..139fa1ed 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/TooltipScoreVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/TooltipScoreVisualization.java @@ -24,7 +24,7 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot; */ import java.text.NumberFormat; -import java.util.List; +import java.util.Collection; import java.util.Locale; import org.apache.batik.util.SVGConstants; @@ -37,7 +37,6 @@ import de.lmu.ifi.dbs.elki.result.HierarchicalResult; import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil; import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.GreaterEqualConstraint; @@ -67,7 +66,7 @@ public class TooltipScoreVisualization extends AbstractTooltipVisualization { /** * A short name characterizing this Visualizer. */ - public static final String NAME_GEN = "Score Tooltips"; + public static final String NAME_GEN = " Tooltips"; /** * Number format. @@ -137,6 +136,8 @@ public class TooltipScoreVisualization extends AbstractTooltipVisualization { tooltiparea.setStatement(SVGConstants.CSS_FILL_OPACITY_PROPERTY, "0"); tooltiparea.setStatement(SVGConstants.CSS_CURSOR_PROPERTY, SVGConstants.CSS_POINTER_VALUE); svgp.addCSSClassOrLogError(tooltiparea); + + svgp.updateStyleElement(); } /** @@ -186,9 +187,9 @@ public class TooltipScoreVisualization extends AbstractTooltipVisualization { @Override public void processNewResult(HierarchicalResult baseResult, Result result) { // TODO: we can also visualize other scores! - List<OutlierResult> ors = ResultUtil.filterResults(result, OutlierResult.class); + Collection<OutlierResult> ors = ResultUtil.filterResults(result, OutlierResult.class); for(OutlierResult o : ors) { - IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class); + Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class); for(ScatterPlotProjector<?> p : ps) { final VisualizationTask task = new VisualizationTask(NAME, o.getScores(), p.getRelation(), this); task.put(VisualizationTask.META_TOOL, true); @@ -197,9 +198,9 @@ public class TooltipScoreVisualization extends AbstractTooltipVisualization { baseResult.getHierarchy().add(p, task); } } - List<Relation<?>> rrs = ResultUtil.filterResults(result, Relation.class); + Collection<Relation<?>> rrs = ResultUtil.filterResults(result, Relation.class); for(Relation<?> r : rrs) { - if(!TypeUtil.DOUBLE.isAssignableFromType(r.getDataTypeInformation())) { + if(!TypeUtil.DOUBLE.isAssignableFromType(r.getDataTypeInformation()) && !TypeUtil.INTEGER.isAssignableFromType(r.getDataTypeInformation())) { continue; } // Skip if we already considered it above @@ -211,9 +212,9 @@ public class TooltipScoreVisualization extends AbstractTooltipVisualization { } } if(add) { - IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class); + Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class); for(ScatterPlotProjector<?> p : ps) { - final VisualizationTask task = new VisualizationTask(NAME_GEN, r, p.getRelation(), this); + final VisualizationTask task = new VisualizationTask(r.getLongName() + NAME_GEN, r, p.getRelation(), this); task.put(VisualizationTask.META_TOOL, true); task.put(VisualizationTask.META_VISIBLE_DEFAULT, false); baseResult.getHierarchy().add(r, task); diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/TooltipStringVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/TooltipStringVisualization.java index 9baea926..d015793c 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/TooltipStringVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/TooltipStringVisualization.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.ArrayList; +import java.util.Collection; import org.apache.batik.util.SVGConstants; import org.w3c.dom.Element; @@ -36,7 +36,6 @@ import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.result.HierarchicalResult; import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; import de.lmu.ifi.dbs.elki.visualization.VisualizationTask; import de.lmu.ifi.dbs.elki.visualization.css.CSSClass; import de.lmu.ifi.dbs.elki.visualization.projector.ScatterPlotProjector; @@ -174,10 +173,10 @@ public class TooltipStringVisualization extends AbstractTooltipVisualization { @Override public void processNewResult(HierarchicalResult baseResult, Result result) { - ArrayList<Relation<?>> reps = ResultUtil.filterResults(result, Relation.class); + Collection<Relation<?>> reps = ResultUtil.filterResults(result, Relation.class); for(Relation<?> rep : reps) { if(DBID.class.isAssignableFrom(rep.getDataTypeInformation().getRestrictionClass())) { - IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class); + Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class); for(ScatterPlotProjector<?> p : ps) { final VisualizationTask task = new VisualizationTask(NAME_ID, rep, p.getRelation(), this); task.put(VisualizationTask.META_TOOL, true); @@ -187,7 +186,7 @@ public class TooltipStringVisualization extends AbstractTooltipVisualization { } } if(ClassLabel.class.isAssignableFrom(rep.getDataTypeInformation().getRestrictionClass())) { - IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class); + Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class); for(ScatterPlotProjector<?> p : ps) { final VisualizationTask task = new VisualizationTask(NAME_CLASS, rep, p.getRelation(), this); task.put(VisualizationTask.META_TOOL, true); @@ -197,7 +196,7 @@ public class TooltipStringVisualization extends AbstractTooltipVisualization { } } if(LabelList.class.isAssignableFrom(rep.getDataTypeInformation().getRestrictionClass())) { - IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class); + Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class); for(ScatterPlotProjector<?> p : ps) { final VisualizationTask task = new VisualizationTask(NAME_LABEL, rep, p.getRelation(), this); task.put(VisualizationTask.META_TOOL, true); @@ -207,7 +206,7 @@ public class TooltipStringVisualization extends AbstractTooltipVisualization { } } if(ExternalID.class.isAssignableFrom(rep.getDataTypeInformation().getRestrictionClass())) { - IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class); + Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class); for(ScatterPlotProjector<?> p : ps) { final VisualizationTask task = new VisualizationTask(NAME_EID, rep, p.getRelation(), this); task.put(VisualizationTask.META_TOOL, true); diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/cluster/ClusterHullVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/cluster/ClusterHullVisualization.java index 430ab194..e1817b1e 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/cluster/ClusterHullVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/cluster/ClusterHullVisualization.java @@ -36,7 +36,7 @@ import de.lmu.ifi.dbs.elki.data.Clustering; import de.lmu.ifi.dbs.elki.data.model.Model;
import de.lmu.ifi.dbs.elki.data.spatial.Polygon;
import de.lmu.ifi.dbs.elki.data.spatial.SpatialUtil;
-import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDs;
import de.lmu.ifi.dbs.elki.math.geometry.AlphaShape;
import de.lmu.ifi.dbs.elki.math.geometry.GrahamScanConvexHull2D;
@@ -44,7 +44,6 @@ import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector; import de.lmu.ifi.dbs.elki.result.HierarchicalResult;
import de.lmu.ifi.dbs.elki.result.Result;
import de.lmu.ifi.dbs.elki.result.ResultUtil;
-import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator;
import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer;
import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID;
import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization;
@@ -71,7 +70,7 @@ import de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot.AbstractScatter * @author Erich Schubert
*
* @apiviz.has Clustering oneway - - visualizes
- * @apiviz.uses ConvexHull2D
+ * @apiviz.uses GrahamScanConvexHull2D
* @apiviz.uses AlphaShape
*/
public class ClusterHullVisualization extends AbstractScatterplotVisualization {
@@ -125,8 +124,8 @@ public class ClusterHullVisualization extends AbstractScatterplotVisualization { if(alpha >= Double.POSITIVE_INFINITY) {
GrahamScanConvexHull2D hull = new GrahamScanConvexHull2D();
- for(DBID clpnum : ids) {
- double[] projP = proj.fastProjectDataToRenderSpace(rel.get(clpnum));
+ for(DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) {
+ double[] projP = proj.fastProjectDataToRenderSpace(rel.get(iter));
hull.add(new Vector(projP));
}
Polygon chres = hull.getHull();
@@ -148,8 +147,8 @@ public class ClusterHullVisualization extends AbstractScatterplotVisualization { }
else {
ArrayList<Vector> ps = new ArrayList<Vector>(ids.size());
- for(DBID clpnum : ids) {
- double[] projP = proj.fastProjectDataToRenderSpace(rel.get(clpnum));
+ for(DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) {
+ double[] projP = proj.fastProjectDataToRenderSpace(rel.get(iter));
ps.add(new Vector(projP));
}
List<Polygon> polys = (new AlphaShape(ps, alpha * Projection.SCALE)).compute();
@@ -225,7 +224,7 @@ public class ClusterHullVisualization extends AbstractScatterplotVisualization { // Find clusterings we can visualize:
Collection<Clustering<?>> clusterings = ResultUtil.filterResults(result, Clustering.class);
for(Clustering<?> c : clusterings) {
- IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class);
+ Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class);
for(ScatterPlotProjector<?> p : ps) {
final VisualizationTask task = new VisualizationTask(NAME, c, p.getRelation(), this);
task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_DATA - 1);
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/cluster/ClusterMeanVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/cluster/ClusterMeanVisualization.java index c9443a9f..4f14f4ef 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/cluster/ClusterMeanVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/cluster/ClusterMeanVisualization.java @@ -23,6 +23,7 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot.cluster; along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
+import java.util.Collection;
import java.util.Iterator;
import org.apache.batik.util.SVGConstants;
@@ -32,6 +33,8 @@ import de.lmu.ifi.dbs.elki.data.Cluster; import de.lmu.ifi.dbs.elki.data.Clustering;
import de.lmu.ifi.dbs.elki.data.NumberVector;
import de.lmu.ifi.dbs.elki.data.model.MeanModel;
+import de.lmu.ifi.dbs.elki.data.model.MedoidModel;
+import de.lmu.ifi.dbs.elki.data.model.Model;
import de.lmu.ifi.dbs.elki.database.ids.DBID;
import de.lmu.ifi.dbs.elki.result.HierarchicalResult;
import de.lmu.ifi.dbs.elki.result.Result;
@@ -59,6 +62,7 @@ import de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot.AbstractScatter * @author Heidi Kolb
*
* @apiviz.has MeanModel oneway - - visualizes
+ * @apiviz.has MedoidModel oneway - - visualizes
*/
public class ClusterMeanVisualization extends AbstractScatterplotVisualization {
/**
@@ -84,7 +88,7 @@ public class ClusterMeanVisualization extends AbstractScatterplotVisualization { /**
* Clustering to visualize.
*/
- Clustering<MeanModel<? extends NumberVector<?, ?>>> clustering;
+ Clustering<Model> clustering;
/**
* Draw stars
@@ -111,10 +115,23 @@ public class ClusterMeanVisualization extends AbstractScatterplotVisualization { MarkerLibrary ml = context.getStyleLibrary().markers();
double marker_size = context.getStyleLibrary().getSize(StyleLibrary.MARKERPLOT);
- Iterator<Cluster<MeanModel<? extends NumberVector<?, ?>>>> ci = clustering.getAllClusters().iterator();
+ Iterator<Cluster<Model>> ci = clustering.getAllClusters().iterator();
for(int cnum = 0; ci.hasNext(); cnum++) {
- Cluster<MeanModel<? extends NumberVector<?, ?>>> clus = ci.next();
- double[] mean = proj.fastProjectDataToRenderSpace(clus.getModel().getMean());
+ Cluster<Model> clus = ci.next();
+ Model model = clus.getModel();
+ double[] mean;
+ if(model instanceof MeanModel) {
+ @SuppressWarnings("unchecked")
+ MeanModel<? extends NumberVector<?, ?>> mmodel = (MeanModel<? extends NumberVector<?, ?>>) model;
+ mean = proj.fastProjectDataToRenderSpace(mmodel.getMean());
+ }
+ else if(model instanceof MedoidModel) {
+ MedoidModel mmodel = (MedoidModel) model;
+ mean = proj.fastProjectDataToRenderSpace(rel.get(mmodel.getMedoid()));
+ }
+ else {
+ continue;
+ }
// add a greater Marker for the mean
Element meanMarker = ml.useMarker(svgp, layer, mean[0], mean[1], cnum, marker_size * 3);
@@ -163,7 +180,7 @@ public class ClusterMeanVisualization extends AbstractScatterplotVisualization { if(stars) {
ColorLibrary colors = context.getStyleLibrary().getColorSet(StyleLibrary.PLOT);
- Iterator<Cluster<MeanModel<? extends NumberVector<?, ?>>>> ci = clustering.getAllClusters().iterator();
+ Iterator<Cluster<Model>> ci = clustering.getAllClusters().iterator();
for(int cnum = 0; ci.hasNext(); cnum++) {
ci.next();
if(!svgp.getCSSClassManager().contains(CSS_MEAN_STAR + "_" + cnum)) {
@@ -219,16 +236,13 @@ public class ClusterMeanVisualization extends AbstractScatterplotVisualization { @Override
public void processNewResult(HierarchicalResult baseResult, Result result) {
// Find clusterings we can visualize:
- Iterator<Clustering<?>> clusterings = ResultUtil.filteredResults(result, Clustering.class);
- while(clusterings.hasNext()) {
- Clustering<?> c = clusterings.next();
+ Collection<Clustering<?>> clusterings = ResultUtil.filterResults(result, Clustering.class);
+ for(Clustering<?> c : clusterings) {
if(c.getAllClusters().size() > 0) {
// Does the cluster have a model with cluster means?
- Clustering<MeanModel<? extends NumberVector<?, ?>>> mcls = findMeanModel(c);
- if(mcls != null) {
- Iterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class);
- while(ps.hasNext()) {
- ScatterPlotProjector<?> p = ps.next();
+ if(testMeanModel(c)) {
+ Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class);
+ for(ScatterPlotProjector<?> p : ps) {
final VisualizationTask task = new VisualizationTask(NAME, c, p.getRelation(), this);
task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_DATA + 1);
baseResult.getHierarchy().add(c, task);
@@ -243,14 +257,17 @@ public class ClusterMeanVisualization extends AbstractScatterplotVisualization { * Test if the given clustering has a mean model.
*
* @param c Clustering to inspect
- * @return the clustering cast to return a mean model, null otherwise.
+ * @return true when the clustering has a mean or medoid model.
*/
- @SuppressWarnings("unchecked")
- private static Clustering<MeanModel<? extends NumberVector<?, ?>>> findMeanModel(Clustering<?> c) {
- if(c.getAllClusters().get(0).getModel() instanceof MeanModel<?>) {
- return (Clustering<MeanModel<? extends NumberVector<?, ?>>>) c;
+ private static boolean testMeanModel(Clustering<?> c) {
+ Model firstmodel = c.getAllClusters().get(0).getModel();
+ if(firstmodel instanceof MeanModel<?>) {
+ return true;
+ }
+ if(firstmodel instanceof MedoidModel) {
+ return true;
}
- return null;
+ return false;
}
/**
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/cluster/ClusterOrderVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/cluster/ClusterOrderVisualization.java index 1ac55851..0d43875c 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/cluster/ClusterOrderVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/cluster/ClusterOrderVisualization.java @@ -35,7 +35,6 @@ import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil; import de.lmu.ifi.dbs.elki.result.optics.ClusterOrderEntry; import de.lmu.ifi.dbs.elki.result.optics.ClusterOrderResult; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; import de.lmu.ifi.dbs.elki.visualization.VisualizationTask; import de.lmu.ifi.dbs.elki.visualization.css.CSSClass; import de.lmu.ifi.dbs.elki.visualization.projector.ScatterPlotProjector; @@ -135,7 +134,7 @@ public class ClusterOrderVisualization extends AbstractScatterplotVisualization public void processNewResult(HierarchicalResult baseResult, Result result) { Collection<ClusterOrderResult<DoubleDistance>> cos = ResultUtil.filterResults(result, ClusterOrderResult.class); for(ClusterOrderResult<DoubleDistance> co : cos) { - IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class); + Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class); for(ScatterPlotProjector<?> p : ps) { final VisualizationTask task = new VisualizationTask(NAME, co, p.getRelation(), this); task.put(VisualizationTask.META_VISIBLE_DEFAULT, false); diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/cluster/EMClusterVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/cluster/EMClusterVisualization.java index 6b2f43a3..6070361e 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/cluster/EMClusterVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/cluster/EMClusterVisualization.java @@ -50,7 +50,6 @@ import de.lmu.ifi.dbs.elki.result.HierarchicalResult; import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil; import de.lmu.ifi.dbs.elki.utilities.ClassGenericsUtil; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.EmptyParameterization; import de.lmu.ifi.dbs.elki.visualization.VisualizationTask; import de.lmu.ifi.dbs.elki.visualization.colors.ColorLibrary; @@ -71,7 +70,7 @@ import de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot.AbstractScatter * @author Robert Rödler * * @apiviz.has EMModel oneway - - visualizes - * @apiviz.uses ConvexHull2D + * @apiviz.uses GrahamScanConvexHull2D * * @param <NV> Type of the NumberVector being visualized. */ @@ -442,7 +441,7 @@ public class EMClusterVisualization<NV extends NumberVector<NV, ?>> extends Abst // Does the cluster have a model with cluster means? Clustering<MeanModel<NV>> mcls = findMeanModel(c); if(mcls != null) { - IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class); + Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class); for(ScatterPlotProjector<?> p : ps) { final VisualizationTask task = new VisualizationTask(NAME, c, p.getRelation(), this); task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_DATA + 3); diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/cluster/VoronoiVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/cluster/VoronoiVisualization.java index 37f125d4..d6ce810c 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/cluster/VoronoiVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/cluster/VoronoiVisualization.java @@ -1,26 +1,27 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot.cluster;
-/* - This file is part of ELKI: +
+/*
+ This file is part of ELKI:
Environment for Developing KDD-Applications Supported by Index-Structures - - Copyright (C) 2012 - Ludwig-Maximilians-Universität München - Lehr- und Forschungseinheit für Datenbanksysteme +
+ Copyright (C) 2012
+ Ludwig-Maximilians-Universität München
+ Lehr- und Forschungseinheit für Datenbanksysteme
ELKI Development Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or +
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
(at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. - */ +
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
import java.util.ArrayList;
import java.util.Collection;
@@ -32,8 +33,8 @@ import org.w3c.dom.Element; import de.lmu.ifi.dbs.elki.data.Cluster;
import de.lmu.ifi.dbs.elki.data.Clustering;
import de.lmu.ifi.dbs.elki.data.NumberVector;
-import de.lmu.ifi.dbs.elki.data.model.EMModel;
import de.lmu.ifi.dbs.elki.data.model.MeanModel;
+import de.lmu.ifi.dbs.elki.data.model.MedoidModel;
import de.lmu.ifi.dbs.elki.data.model.Model;
import de.lmu.ifi.dbs.elki.math.geometry.SweepHullDelaunay2D;
import de.lmu.ifi.dbs.elki.math.geometry.SweepHullDelaunay2D.Triangle;
@@ -42,7 +43,6 @@ import de.lmu.ifi.dbs.elki.result.HierarchicalResult; import de.lmu.ifi.dbs.elki.result.Result;
import de.lmu.ifi.dbs.elki.result.ResultUtil;
import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil;
-import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator;
import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer;
import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID;
import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization;
@@ -62,13 +62,14 @@ import de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot.AbstractScatter /**
* Visualizer drawing Voronoi cells for k-means clusterings.
*
- * See also: {@link de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMeansLloyd KMeans
- * clustering}
+ * See also: {@link de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMeansLloyd
+ * KMeans clustering}
*
* @author Robert Rödler
* @author Erich Schubert
*
* @apiviz.has MeanModel oneway - - visualizes
+ * @apiviz.has MedoidModel oneway - - visualizes
*/
public class VoronoiVisualization extends AbstractScatterplotVisualization {
/**
@@ -95,7 +96,7 @@ public class VoronoiVisualization extends AbstractScatterplotVisualization { /**
* The result we work on
*/
- Clustering<MeanModel<? extends NumberVector<?, ?>>> clustering;
+ Clustering<Model> clustering;
/**
* The Voronoi diagram
@@ -123,7 +124,7 @@ public class VoronoiVisualization extends AbstractScatterplotVisualization { @Override
protected void redraw() {
addCSSClasses(svgp);
- final List<Cluster<MeanModel<? extends NumberVector<?, ?>>>> clusters = clustering.getAllClusters();
+ final List<Cluster<Model>> clusters = clustering.getAllClusters();
if(clusters.size() < 2) {
return;
@@ -133,8 +134,22 @@ public class VoronoiVisualization extends AbstractScatterplotVisualization { if(clusters.size() == 2) {
ArrayList<double[]> means = new ArrayList<double[]>(clusters.size());
{
- for(Cluster<MeanModel<? extends NumberVector<?, ?>>> clus : clusters) {
- means.add(clus.getModel().getMean().getColumnVector().getArrayRef());
+ for(Cluster<Model> clus : clusters) {
+ Model model = clus.getModel();
+ double[] mean;
+ if(model instanceof MeanModel) {
+ @SuppressWarnings("unchecked")
+ MeanModel<? extends NumberVector<?, ?>> mmodel = (MeanModel<? extends NumberVector<?, ?>>) model;
+ mean = proj.fastProjectDataToRenderSpace(mmodel.getMean());
+ }
+ else if(model instanceof MedoidModel) {
+ MedoidModel mmodel = (MedoidModel) model;
+ mean = proj.fastProjectDataToRenderSpace(rel.get(mmodel.getMedoid()));
+ }
+ else {
+ continue;
+ }
+ means.add(mean);
}
}
if(mode == Mode.VORONOI || mode == Mode.V_AND_D) {
@@ -143,7 +158,7 @@ public class VoronoiVisualization extends AbstractScatterplotVisualization { layer.appendChild(path);
}
if(mode == Mode.DELAUNAY || mode == Mode.V_AND_D) {
- Element path = new SVGPath(proj.fastProjectDataToRenderSpace(means.get(0))).drawTo(proj.fastProjectDataToRenderSpace(means.get(1))).makeElement(svgp);
+ Element path = new SVGPath(means.get(0)).drawTo(means.get(1)).makeElement(svgp);
SVGUtil.addCSSClass(path, KMEANSBORDER);
layer.appendChild(path);
}
@@ -152,10 +167,23 @@ public class VoronoiVisualization extends AbstractScatterplotVisualization { ArrayList<Vector> vmeans = new ArrayList<Vector>(clusters.size());
ArrayList<double[]> means = new ArrayList<double[]>(clusters.size());
{
- for(Cluster<MeanModel<? extends NumberVector<?, ?>>> clus : clusters) {
- Vector v = clus.getModel().getMean().getColumnVector();
- vmeans.add(v);
- means.add(v.getArrayRef());
+ for(Cluster<Model> clus : clusters) {
+ Model model = clus.getModel();
+ Vector mean;
+ if(model instanceof MeanModel) {
+ @SuppressWarnings("unchecked")
+ MeanModel<? extends NumberVector<?, ?>> mmodel = (MeanModel<? extends NumberVector<?, ?>>) model;
+ mean = mmodel.getMean().getColumnVector();
+ }
+ else if(model instanceof MedoidModel) {
+ MedoidModel mmodel = (MedoidModel) model;
+ mean = rel.get(mmodel.getMedoid()).getColumnVector();
+ }
+ else {
+ continue;
+ }
+ vmeans.add(mean);
+ means.add(mean.getArrayRef());
}
}
// Compute Delaunay Triangulation
@@ -200,7 +228,7 @@ public class VoronoiVisualization extends AbstractScatterplotVisualization { * @author Erich Schubert
*
* @apiviz.stereotype factory
- * @apiviz.uses VoronoiVisualisation oneway - - «create»
+ * @apiviz.uses VoronoiVisualization oneway - - «create»
*/
public static class Factory extends AbstractVisFactory {
/**
@@ -239,9 +267,8 @@ public class VoronoiVisualization extends AbstractScatterplotVisualization { for(Clustering<?> c : clusterings) {
if(c.getAllClusters().size() > 0) {
// Does the cluster have a model with cluster means?
- Clustering<MeanModel<? extends NumberVector<?, ?>>> mcls = findMeanModel(c);
- if(mcls != null) {
- IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class);
+ if(testMeanModel(c)) {
+ Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class);
for(ScatterPlotProjector<?> p : ps) {
if(DatabaseUtil.dimensionality(p.getRelation()) == 2) {
final VisualizationTask task = new VisualizationTask(NAME, c, p.getRelation(), this);
@@ -259,15 +286,17 @@ public class VoronoiVisualization extends AbstractScatterplotVisualization { * Test if the given clustering has a mean model.
*
* @param c Clustering to inspect
- * @return the clustering cast to return a mean model, null otherwise.
+ * @return true when the clustering has a mean or medoid model.
*/
- @SuppressWarnings("unchecked")
- private static Clustering<MeanModel<? extends NumberVector<?, ?>>> findMeanModel(Clustering<?> c) {
- final Model firstModel = c.getAllClusters().get(0).getModel();
- if(firstModel instanceof MeanModel<?> && !(firstModel instanceof EMModel<?>)) {
- return (Clustering<MeanModel<? extends NumberVector<?, ?>>>) c;
+ private static boolean testMeanModel(Clustering<?> c) {
+ Model firstmodel = c.getAllClusters().get(0).getModel();
+ if(firstmodel instanceof MeanModel<?>) {
+ return true;
+ }
+ if(firstmodel instanceof MedoidModel) {
+ return true;
}
- return null;
+ return false;
}
/**
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/density/DensityEstimationOverlay.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/density/DensityEstimationOverlay.java index 28e4da32..c07bc571 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/density/DensityEstimationOverlay.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/density/DensityEstimationOverlay.java @@ -24,12 +24,13 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot.density; */ import java.awt.image.BufferedImage; import java.util.Arrays; +import java.util.Collection; import java.util.Comparator; import org.apache.batik.util.SVGConstants; import org.w3c.dom.Element; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.math.MathUtil; import de.lmu.ifi.dbs.elki.math.MeanVariance; import de.lmu.ifi.dbs.elki.result.HierarchicalResult; @@ -37,7 +38,6 @@ import de.lmu.ifi.dbs.elki.result.KMLOutputHandler; import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil; import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; import de.lmu.ifi.dbs.elki.visualization.VisualizationTask; import de.lmu.ifi.dbs.elki.visualization.batikutil.ThumbnailRegistryEntry; import de.lmu.ifi.dbs.elki.visualization.projections.CanvasSize; @@ -124,8 +124,8 @@ public class DensityEstimationOverlay extends AbstractScatterplotVisualization { double[][] data = new double[rel.size()][]; { int i = 0; - for(DBID id : rel.iterDBIDs()) { - data[i] = proj.fastProjectDataToRenderSpace(rel.get(id)); + for(DBIDIter iditer = rel.iterDBIDs(); iditer.valid(); iditer.advance()) { + data[i] = proj.fastProjectDataToRenderSpace(rel.get(iditer)); i++; } } @@ -206,7 +206,7 @@ public class DensityEstimationOverlay extends AbstractScatterplotVisualization { * @author Erich Schubert * * @apiviz.stereotype factory - * @apiviz.uses DensityEstimation2DVisualization oneway - - «create» + * @apiviz.uses DensityEstimationOverlay oneway - - «create» */ public static class Factory extends AbstractVisFactory { /** @@ -224,7 +224,7 @@ public class DensityEstimationOverlay extends AbstractScatterplotVisualization { @Override public void processNewResult(HierarchicalResult baseResult, Result result) { - IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(result, ScatterPlotProjector.class); + Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(result, ScatterPlotProjector.class); for(ScatterPlotProjector<?> p : ps) { final VisualizationTask task = new VisualizationTask(NAME, p.getRelation(), p.getRelation(), this); task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_DATA + 1); diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/index/TreeMBRVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/index/TreeMBRVisualization.java index 0b4bee2f..61635625 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/index/TreeMBRVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/index/TreeMBRVisualization.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot.index; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.ArrayList; +import java.util.Collection; import org.apache.batik.util.SVGConstants; import org.w3c.dom.Element; @@ -38,7 +38,6 @@ import de.lmu.ifi.dbs.elki.index.tree.spatial.rstarvariants.rstar.RStarTreeNode; import de.lmu.ifi.dbs.elki.result.HierarchicalResult; import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; @@ -214,10 +213,10 @@ public class TreeMBRVisualization<N extends AbstractRStarTreeNode<N, E>, E exten @Override public void processNewResult(HierarchicalResult baseResult, Result result) { - ArrayList<AbstractRStarTree<RStarTreeNode, SpatialEntry>> trees = ResultUtil.filterResults(result, AbstractRStarTree.class); + Collection<AbstractRStarTree<RStarTreeNode, SpatialEntry>> trees = ResultUtil.filterResults(result, AbstractRStarTree.class); for(AbstractRStarTree<RStarTreeNode, SpatialEntry> tree : trees) { if(tree instanceof Result) { - IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class); + Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class); for(ScatterPlotProjector<?> p : ps) { final VisualizationTask task = new VisualizationTask(NAME, (Result) tree, p.getRelation(), this); task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_BACKGROUND + 1); diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/index/TreeSphereVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/index/TreeSphereVisualization.java index 85429eb1..daebab02 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/index/TreeSphereVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/index/TreeSphereVisualization.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot.index; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.ArrayList; +import java.util.Collection; import org.apache.batik.util.SVGConstants; import org.w3c.dom.Element; @@ -44,7 +44,6 @@ import de.lmu.ifi.dbs.elki.index.tree.metrical.mtreevariants.mtree.MTreeNode; import de.lmu.ifi.dbs.elki.result.HierarchicalResult; import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.Flag; @@ -271,9 +270,9 @@ public class TreeSphereVisualization<D extends NumberDistance<D, ?>, N extends A @Override public void processNewResult(HierarchicalResult baseResult, Result result) { - IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class); + Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class); for(ScatterPlotProjector<?> p : ps) { - ArrayList<AbstractMTree<?, DoubleDistance, ?, ?>> trees = ResultUtil.filterResults(result, AbstractMTree.class); + Collection<AbstractMTree<?, DoubleDistance, ?, ?>> trees = ResultUtil.filterResults(result, AbstractMTree.class); for(AbstractMTree<?, DoubleDistance, ?, ?> tree : trees) { if(canVisualize(tree) && tree instanceof Result) { final VisualizationTask task = new VisualizationTask(NAME, (Result) tree, p.getRelation(), this); diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/outlier/BubbleVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/outlier/BubbleVisualization.java index 7c40c1cb..769dc53d 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/outlier/BubbleVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/outlier/BubbleVisualization.java @@ -23,20 +23,20 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot.outlier; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.List; +import java.util.Collection; import org.apache.batik.util.SVGConstants; import org.w3c.dom.Element; import de.lmu.ifi.dbs.elki.data.NumberVector; import de.lmu.ifi.dbs.elki.database.datastore.DataStoreListener; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.ids.DBIDRef; import de.lmu.ifi.dbs.elki.result.HierarchicalResult; import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil; import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; import de.lmu.ifi.dbs.elki.utilities.documentation.Reference; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; @@ -56,6 +56,7 @@ import de.lmu.ifi.dbs.elki.visualization.svg.SVGUtil; import de.lmu.ifi.dbs.elki.visualization.visualizers.AbstractVisFactory; import de.lmu.ifi.dbs.elki.visualization.visualizers.Visualization; import de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot.AbstractScatterplotVisualization; +import de.lmu.ifi.dbs.elki.visualization.visualizers.thumbs.ThumbnailVisualization; /** * Generates a SVG-Element containing bubbles. A Bubble is a circle visualizing @@ -99,18 +100,22 @@ public class BubbleVisualization extends AbstractScatterplotVisualization implem * * @param task Visualization task * @param scaling Scaling function + * @param fill Fill flag */ - public BubbleVisualization(VisualizationTask task, ScalingFunction scaling) { + public BubbleVisualization(VisualizationTask task, ScalingFunction scaling, boolean fill) { super(task); this.result = task.getResult(); this.scaling = scaling; + this.fill = fill; context.addDataStoreListener(this); + context.addResultListener(this); incrementalRedraw(); } @Override public void destroy() { super.destroy(); + context.removeResultListener(this); context.removeDataStoreListener(this); } @@ -123,7 +128,7 @@ public class BubbleVisualization extends AbstractScatterplotVisualization implem ClassStylingPolicy colors = (ClassStylingPolicy) stylepolicy; setupCSS(svgp, colors); // draw data - for(DBID objId : sample.getSample()) { + for(DBIDIter objId = sample.getSample().iter(); objId.valid(); objId.advance()) { final Double radius = getScaledForId(objId); if(radius > 0.01 && !Double.isInfinite(radius)) { final NumberVector<?, ?> vec = rel.get(objId); @@ -138,7 +143,7 @@ public class BubbleVisualization extends AbstractScatterplotVisualization implem } else { // draw data - for(DBID objId : sample.getSample()) { + for(DBIDIter objId = sample.getSample().iter(); objId.valid(); objId.advance()) { final Double radius = getScaledForId(objId); if(radius > 0.01 && !Double.isInfinite(radius)) { final NumberVector<?, ?> vec = rel.get(objId); @@ -165,7 +170,8 @@ public class BubbleVisualization extends AbstractScatterplotVisualization implem @Override public void resultChanged(Result current) { - if(sample == current) { + super.resultChanged(current); + if(sample == current || context.getStyleResult() == current) { synchronizedRedraw(); } } @@ -207,7 +213,7 @@ public class BubbleVisualization extends AbstractScatterplotVisualization implem * @return a Double representing a outlierness-score, after it has modified by * the given scales. */ - protected double getScaledForId(DBID id) { + protected double getScaledForId(DBIDRef id) { double d = result.getScores().get(id).doubleValue(); if(Double.isNaN(d) || Double.isInfinite(d)) { return 0.0; @@ -267,6 +273,7 @@ public class BubbleVisualization extends AbstractScatterplotVisualization implem super(); this.fill = fill; this.scaling = scaling; + thumbmask |= ThumbnailVisualization.ON_DATA | ThumbnailVisualization.ON_STYLE; } @Override @@ -275,14 +282,14 @@ public class BubbleVisualization extends AbstractScatterplotVisualization implem final OutlierResult outlierResult = task.getResult(); ((OutlierScalingFunction) this.scaling).prepare(outlierResult); } - return new BubbleVisualization(task, scaling); + return new BubbleVisualization(task, scaling, fill); } @Override public void processNewResult(HierarchicalResult baseResult, Result result) { - List<OutlierResult> ors = ResultUtil.filterResults(result, OutlierResult.class); + Collection<OutlierResult> ors = ResultUtil.filterResults(result, OutlierResult.class); for(OutlierResult o : ors) { - IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class); + Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class); boolean vis = true; // Quick and dirty hack: hide if parent result is also an outlier result // Since that probably is already visible and we're redundant. diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/MoveObjectsToolVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/MoveObjectsToolVisualization.java index fb9de7d5..6b466a88 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/MoveObjectsToolVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/MoveObjectsToolVisualization.java @@ -23,13 +23,14 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot.selection; along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-import java.util.Iterator;
+import java.util.Collection;
import org.apache.batik.util.SVGConstants;
import org.w3c.dom.Element;
import org.w3c.dom.events.Event;
import org.w3c.dom.svg.SVGPoint;
+import de.lmu.ifi.dbs.elki.data.NumberVector;
import de.lmu.ifi.dbs.elki.database.UpdatableDatabase;
import de.lmu.ifi.dbs.elki.database.ids.DBIDs;
import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector;
@@ -37,7 +38,6 @@ import de.lmu.ifi.dbs.elki.result.HierarchicalResult; import de.lmu.ifi.dbs.elki.result.Result;
import de.lmu.ifi.dbs.elki.result.ResultUtil;
import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException;
-import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator;
import de.lmu.ifi.dbs.elki.visualization.VisualizationTask;
import de.lmu.ifi.dbs.elki.visualization.batikutil.DragableArea;
import de.lmu.ifi.dbs.elki.visualization.batikutil.DragableArea.DragListener;
@@ -79,6 +79,11 @@ public class MoveObjectsToolVisualization extends AbstractScatterplotVisualizati */
private Element rtag;
+ /**
+ * Constructor.
+ *
+ * @param task Task
+ */
public MoveObjectsToolVisualization(VisualizationTask task) {
super(task);
incrementalRedraw();
@@ -113,6 +118,7 @@ public class MoveObjectsToolVisualization extends AbstractScatterplotVisualizati */
// TODO: move to DatabaseUtil?
private void updateDB(DBIDs dbids, Vector movingVector) {
+ NumberVector<?, ?> nv = null;
throw new AbortException("FIXME: INCOMPLETE TRANSITION");
/*
* database.accumulateDataStoreEvents();
@@ -208,11 +214,11 @@ public class MoveObjectsToolVisualization extends AbstractScatterplotVisualizati @Override
public void processNewResult(HierarchicalResult baseResult, Result result) {
- Iterator<UpdatableDatabase> dbs = ResultUtil.filteredResults(result, UpdatableDatabase.class);
- if(!dbs.hasNext()) {
+ Collection<UpdatableDatabase> dbs = ResultUtil.filterResults(result, UpdatableDatabase.class);
+ if(dbs.isEmpty()) {
return;
}
- IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class);
+ Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class);
for(ScatterPlotProjector<?> p : ps) {
final VisualizationTask task = new VisualizationTask(NAME, p.getRelation(), p.getRelation(), this);
task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_INTERACTIVE);
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/SelectionConvexHullVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/SelectionConvexHullVisualization.java index 5702800d..8a28ab3e 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/SelectionConvexHullVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/SelectionConvexHullVisualization.java @@ -23,14 +23,14 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot.selection; along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-import java.util.ArrayList;
+import java.util.Collection;
import org.apache.batik.util.SVGConstants;
import org.w3c.dom.Element;
import de.lmu.ifi.dbs.elki.data.spatial.Polygon;
import de.lmu.ifi.dbs.elki.database.datastore.DataStoreListener;
-import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDs;
import de.lmu.ifi.dbs.elki.math.geometry.GrahamScanConvexHull2D;
import de.lmu.ifi.dbs.elki.math.linearalgebra.Vector;
@@ -40,7 +40,6 @@ import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil;
import de.lmu.ifi.dbs.elki.result.SelectionResult;
import de.lmu.ifi.dbs.elki.utilities.exceptions.ObjectNotFoundException;
-import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator;
import de.lmu.ifi.dbs.elki.visualization.VisualizationTask;
import de.lmu.ifi.dbs.elki.visualization.css.CSSClass;
import de.lmu.ifi.dbs.elki.visualization.projector.ScatterPlotProjector;
@@ -61,7 +60,7 @@ import de.lmu.ifi.dbs.elki.visualization.visualizers.thumbs.ThumbnailVisualizati *
* @apiviz.has SelectionResult oneway - - visualizes
* @apiviz.has DBIDSelection oneway - - visualizes
- * @apiviz.uses ConvexHull2D
+ * @apiviz.uses GrahamScanConvexHull2D
*/
public class SelectionConvexHullVisualization extends AbstractScatterplotVisualization implements DataStoreListener {
/**
@@ -93,9 +92,9 @@ public class SelectionConvexHullVisualization extends AbstractScatterplotVisuali if(selContext != null) {
DBIDs selection = selContext.getSelectedIds();
GrahamScanConvexHull2D hull = new GrahamScanConvexHull2D();
- for(DBID i : selection) {
+ for(DBIDIter iter = selection.iter(); iter.valid(); iter.advance()) {
try {
- hull.add(new Vector(proj.fastProjectDataToRenderSpace(rel.get(i))));
+ hull.add(new Vector(proj.fastProjectDataToRenderSpace(rel.get(iter))));
}
catch(ObjectNotFoundException e) {
// ignore
@@ -158,9 +157,9 @@ public class SelectionConvexHullVisualization extends AbstractScatterplotVisuali @Override
public void processNewResult(HierarchicalResult baseResult, Result result) {
- final ArrayList<SelectionResult> selectionResults = ResultUtil.filterResults(result, SelectionResult.class);
+ Collection<SelectionResult> selectionResults = ResultUtil.filterResults(result, SelectionResult.class);
for(SelectionResult selres : selectionResults) {
- IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class);
+ Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class);
for(ScatterPlotProjector<?> p : ps) {
final VisualizationTask task = new VisualizationTask(NAME, selres, p.getRelation(), this);
task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_DATA - 2);
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/SelectionCubeVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/SelectionCubeVisualization.java index 9fd24b43..5151e81e 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/SelectionCubeVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/SelectionCubeVisualization.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot.selection; along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-import java.util.ArrayList;
+import java.util.Collection;
import org.apache.batik.util.SVGConstants;
import org.w3c.dom.Element;
@@ -35,7 +35,6 @@ import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil;
import de.lmu.ifi.dbs.elki.result.SelectionResult;
import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil;
-import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator;
import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer;
import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID;
import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization;
@@ -227,9 +226,9 @@ public class SelectionCubeVisualization extends AbstractScatterplotVisualization @Override
public void processNewResult(HierarchicalResult baseResult, Result result) {
- final ArrayList<SelectionResult> selectionResults = ResultUtil.filterResults(result, SelectionResult.class);
+ Collection<SelectionResult> selectionResults = ResultUtil.filterResults(result, SelectionResult.class);
for(SelectionResult selres : selectionResults) {
- IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class);
+ Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class);
for(ScatterPlotProjector<?> p : ps) {
final VisualizationTask task = new VisualizationTask(NAME, selres, p.getRelation(), this);
task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_DATA - 2);
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/SelectionDotVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/SelectionDotVisualization.java index 79d1afe5..1dc5ce13 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/SelectionDotVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/SelectionDotVisualization.java @@ -23,13 +23,13 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot.selection; along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-import java.util.ArrayList;
+import java.util.Collection;
import org.apache.batik.util.SVGConstants;
import org.w3c.dom.Element;
import de.lmu.ifi.dbs.elki.database.datastore.DataStoreListener;
-import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDs;
import de.lmu.ifi.dbs.elki.result.DBIDSelection;
import de.lmu.ifi.dbs.elki.result.HierarchicalResult;
@@ -37,7 +37,6 @@ import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil;
import de.lmu.ifi.dbs.elki.result.SelectionResult;
import de.lmu.ifi.dbs.elki.utilities.exceptions.ObjectNotFoundException;
-import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator;
import de.lmu.ifi.dbs.elki.visualization.VisualizationTask;
import de.lmu.ifi.dbs.elki.visualization.css.CSSClass;
import de.lmu.ifi.dbs.elki.visualization.projector.ScatterPlotProjector;
@@ -94,9 +93,9 @@ public class SelectionDotVisualization extends AbstractScatterplotVisualization DBIDSelection selContext = context.getSelection();
if(selContext != null) {
DBIDs selection = selContext.getSelectedIds();
- for(DBID i : selection) {
+ for(DBIDIter iter = selection.iter(); iter.valid(); iter.advance()) {
try {
- double[] v = proj.fastProjectDataToRenderSpace(rel.get(i));
+ double[] v = proj.fastProjectDataToRenderSpace(rel.get(iter));
Element dot = svgp.svgCircle(v[0], v[1], size);
SVGUtil.addCSSClass(dot, MARKER);
layer.appendChild(dot);
@@ -149,9 +148,9 @@ public class SelectionDotVisualization extends AbstractScatterplotVisualization @Override
public void processNewResult(HierarchicalResult baseResult, Result result) {
- final ArrayList<SelectionResult> selectionResults = ResultUtil.filterResults(result, SelectionResult.class);
+ Collection<SelectionResult> selectionResults = ResultUtil.filterResults(result, SelectionResult.class);
for(SelectionResult selres : selectionResults) {
- IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class);
+ Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class);
for(ScatterPlotProjector<?> p : ps) {
final VisualizationTask task = new VisualizationTask(NAME, selres, p.getRelation(), this);
task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_DATA - 1);
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/SelectionToolCubeVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/SelectionToolCubeVisualization.java index 8e86f920..bc16fd83 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/SelectionToolCubeVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/SelectionToolCubeVisualization.java @@ -23,8 +23,8 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot.selection; along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-import java.util.ArrayList;
import java.util.BitSet;
+import java.util.Collection;
import org.apache.batik.util.SVGConstants;
import org.w3c.dom.Element;
@@ -32,7 +32,7 @@ import org.w3c.dom.events.Event; import org.w3c.dom.svg.SVGPoint;
import de.lmu.ifi.dbs.elki.data.NumberVector;
-import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil;
import de.lmu.ifi.dbs.elki.database.ids.ModifiableDBIDs;
import de.lmu.ifi.dbs.elki.logging.Logging;
@@ -43,7 +43,6 @@ import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil;
import de.lmu.ifi.dbs.elki.result.SelectionResult;
import de.lmu.ifi.dbs.elki.utilities.DatabaseUtil;
-import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator;
import de.lmu.ifi.dbs.elki.utilities.pairs.DoubleDoublePair;
import de.lmu.ifi.dbs.elki.visualization.VisualizationTask;
import de.lmu.ifi.dbs.elki.visualization.batikutil.DragableArea;
@@ -192,6 +191,11 @@ public class SelectionToolCubeVisualization extends AbstractScatterplotVisualiza * @param p2 Second Point of the selected rectangle
*/
private void updateSelection(Projection proj, SVGPoint p1, SVGPoint p2) {
+ if(p1 == null || p2 == null) {
+ logger.warning("no rect selected: p1: " + p1 + " p2: " + p2);
+ return;
+ }
+
DBIDSelection selContext = context.getSelection();
ModifiableDBIDs selection;
if(selContext != null) {
@@ -202,42 +206,32 @@ public class SelectionToolCubeVisualization extends AbstractScatterplotVisualiza }
DoubleDoublePair[] ranges;
- if(p1 == null || p2 == null) {
- logger.warning("no rect selected: p1: " + p1 + " p2: " + p2);
+ double x1 = Math.min(p1.getX(), p2.getX());
+ double x2 = Math.max(p1.getX(), p2.getX());
+ double y1 = Math.max(p1.getY(), p2.getY());
+ double y2 = Math.min(p1.getY(), p2.getY());
+
+ if(selContext instanceof RangeSelection) {
+ ranges = ((RangeSelection) selContext).getRanges();
}
else {
- double x1 = Math.min(p1.getX(), p2.getX());
- double x2 = Math.max(p1.getX(), p2.getX());
- double y1 = Math.max(p1.getY(), p2.getY());
- double y2 = Math.min(p1.getY(), p2.getY());
-
- if(selContext instanceof RangeSelection) {
- ranges = ((RangeSelection) selContext).getRanges();
- }
- else {
- ranges = new DoubleDoublePair[dim];
- }
- updateSelectionRectKoordinates(x1, x2, y1, y2, ranges);
-
- selection.clear();
- boolean idIn = true;
- for(DBID id : rel.iterDBIDs()) {
- NumberVector<?, ?> dbTupel = rel.get(id);
- idIn = true;
- for(int i = 0; i < dim; i++) {
- if(ranges != null && ranges[i] != null) {
- if(dbTupel.doubleValue(i + 1) < ranges[i].first || dbTupel.doubleValue(i + 1) > ranges[i].second) {
- idIn = false;
- break;
- }
+ ranges = new DoubleDoublePair[dim];
+ }
+ updateSelectionRectKoordinates(x1, x2, y1, y2, ranges);
+
+ selection.clear();
+ candidates: for(DBIDIter iditer = rel.iterDBIDs(); iditer.valid(); iditer.advance()) {
+ NumberVector<?, ?> dbTupel = rel.get(iditer);
+ for(int i = 0; i < dim; i++) {
+ if(ranges != null && ranges[i] != null) {
+ if(dbTupel.doubleValue(i + 1) < ranges[i].first || dbTupel.doubleValue(i + 1) > ranges[i].second) {
+ continue candidates;
}
}
- if(idIn == true) {
- selection.add(id);
- }
}
- context.setSelection(new RangeSelection(selection, ranges));
+ selection.add(iditer);
}
+ context.setSelection(new RangeSelection(selection, ranges));
}
/**
@@ -281,9 +275,9 @@ public class SelectionToolCubeVisualization extends AbstractScatterplotVisualiza @Override
public void processNewResult(HierarchicalResult baseResult, Result result) {
- final ArrayList<SelectionResult> selectionResults = ResultUtil.filterResults(result, SelectionResult.class);
+ Collection<SelectionResult> selectionResults = ResultUtil.filterResults(result, SelectionResult.class);
for(SelectionResult selres : selectionResults) {
- IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class);
+ Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class);
for(ScatterPlotProjector<?> p : ps) {
final VisualizationTask task = new VisualizationTask(NAME, selres, p.getRelation(), this);
task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_INTERACTIVE);
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/SelectionToolDotVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/SelectionToolDotVisualization.java index 877833f7..b52f37c9 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/SelectionToolDotVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/scatterplot/selection/SelectionToolDotVisualization.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.scatterplot.selection; along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-import java.util.ArrayList;
+import java.util.Collection;
import org.apache.batik.dom.events.DOMMouseEvent;
import org.apache.batik.util.SVGConstants;
@@ -31,7 +31,7 @@ import org.w3c.dom.Element; import org.w3c.dom.events.Event;
import org.w3c.dom.svg.SVGPoint;
-import de.lmu.ifi.dbs.elki.database.ids.DBID;
+import de.lmu.ifi.dbs.elki.database.ids.DBIDIter;
import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil;
import de.lmu.ifi.dbs.elki.database.ids.HashSetModifiableDBIDs;
import de.lmu.ifi.dbs.elki.result.DBIDSelection;
@@ -39,7 +39,6 @@ import de.lmu.ifi.dbs.elki.result.HierarchicalResult; import de.lmu.ifi.dbs.elki.result.Result;
import de.lmu.ifi.dbs.elki.result.ResultUtil;
import de.lmu.ifi.dbs.elki.result.SelectionResult;
-import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator;
import de.lmu.ifi.dbs.elki.visualization.VisualizationTask;
import de.lmu.ifi.dbs.elki.visualization.batikutil.DragableArea;
import de.lmu.ifi.dbs.elki.visualization.css.CSSClass;
@@ -194,22 +193,22 @@ public class SelectionToolDotVisualization extends AbstractScatterplotVisualizat else {
selection = DBIDUtil.newHashSet(selContext.getSelectedIds());
}
- for(DBID id : rel.iterDBIDs()) {
- double[] vec = proj.fastProjectDataToRenderSpace(rel.get(id));
+ for(DBIDIter iditer = rel.iterDBIDs(); iditer.valid(); iditer.advance()) { + double[] vec = proj.fastProjectDataToRenderSpace(rel.get(iditer));
if(vec[0] >= Math.min(p1.getX(), p2.getX()) && vec[0] <= Math.max(p1.getX(), p2.getX()) && vec[1] >= Math.min(p1.getY(), p2.getY()) && vec[1] <= Math.max(p1.getY(), p2.getY())) {
if(mode == Mode.INVERT) {
- if(!selection.contains(id)) {
- selection.add(id);
+ if(!selection.contains(iditer)) {
+ selection.add(iditer);
}
else {
- selection.remove(id);
+ selection.remove(iditer);
}
}
else {
// In REPLACE and ADD, add objects.
// The difference was done before by not re-using the selection.
// Since we are using a set, we can just add in any case.
- selection.add(id);
+ selection.add(iditer);
}
}
}
@@ -256,9 +255,9 @@ public class SelectionToolDotVisualization extends AbstractScatterplotVisualizat @Override
public void processNewResult(HierarchicalResult baseResult, Result result) {
- final ArrayList<SelectionResult> selectionResults = ResultUtil.filterResults(result, SelectionResult.class);
+ Collection<SelectionResult> selectionResults = ResultUtil.filterResults(result, SelectionResult.class);
for(SelectionResult selres : selectionResults) {
- IterableIterator<ScatterPlotProjector<?>> ps = ResultUtil.filteredResults(baseResult, ScatterPlotProjector.class);
+ Collection<ScatterPlotProjector<?>> ps = ResultUtil.filterResults(baseResult, ScatterPlotProjector.class);
for(ScatterPlotProjector<?> p : ps) {
final VisualizationTask task = new VisualizationTask(NAME, selres, p.getRelation(), this);
task.put(VisualizationTask.META_LEVEL, VisualizationTask.LEVEL_INTERACTIVE);
diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/thumbs/ThumbnailThread.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/thumbs/ThumbnailThread.java index 2c9425de..8d045138 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/thumbs/ThumbnailThread.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/thumbs/ThumbnailThread.java @@ -31,7 +31,6 @@ import java.util.concurrent.ConcurrentLinkedQueue; * * @author Erich Schubert * - * @apiviz.composedOf Thumbnailer * @apiviz.uses Listener oneway - - signals */ public class ThumbnailThread extends Thread { diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/visunproj/ClusterEvaluationVisFactory.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/visunproj/ClusterEvaluationVisFactory.java index 05d9038c..5b58e842 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/visunproj/ClusterEvaluationVisFactory.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/visunproj/ClusterEvaluationVisFactory.java @@ -59,7 +59,7 @@ import de.lmu.ifi.dbs.elki.visualization.visualizers.Visualization; * * @apiviz.stereotype factory * @apiviz.uses StaticVisualization oneway - - «create» - * @apiviz.has EvaluateClustering.ScoreResult oneway - - visualizes + * @apiviz.has de.lmu.ifi.dbs.elki.evaluation.clustering.EvaluateClustering.ScoreResult oneway - - visualizes */ public class ClusterEvaluationVisFactory extends AbstractVisFactory { /** diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/visunproj/KeyVisualization.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/visunproj/KeyVisualization.java index 15c50a39..62a4fb5c 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/visunproj/KeyVisualization.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/visunproj/KeyVisualization.java @@ -23,7 +23,7 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.visunproj; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.Iterator; +import java.util.Collection; import java.util.List; import org.apache.batik.util.SVGConstants; @@ -98,13 +98,14 @@ public class KeyVisualization extends AbstractVisualization { SVGPlot svgp = task.getPlot(); final List<Cluster<Model>> allcs = clustering.getAllClusters(); - MarkerLibrary ml = context.getStyleLibrary().markers(); + StyleLibrary style = context.getStyleLibrary(); + MarkerLibrary ml = style.markers(); layer = svgp.svgElement(SVGConstants.SVG_G_TAG); // Add a label for the clustering. { Element label = svgp.svgText(0.1, 0.7, clustering.getLongName()); - label.setAttribute(SVGConstants.SVG_STYLE_ATTRIBUTE, "font-size: 0.4"); + label.setAttribute(SVGConstants.SVG_STYLE_ATTRIBUTE, "font-size: 0.4; fill: "+style.getTextColor(StyleLibrary.DEFAULT)); layer.appendChild(label); } @@ -113,7 +114,7 @@ public class KeyVisualization extends AbstractVisualization { for(Cluster<Model> c : allcs) { ml.useMarker(svgp, layer, 0.3, i + 1.5, i, 0.3); Element label = svgp.svgText(0.7, i + 1.7, c.getNameAutomatic()); - label.setAttribute(SVGConstants.SVG_STYLE_ATTRIBUTE, "font-size: 0.6"); + label.setAttribute(SVGConstants.SVG_STYLE_ATTRIBUTE, "font-size: 0.6; fill: "+style.getTextColor(StyleLibrary.DEFAULT)); layer.appendChild(label); i++; } @@ -146,7 +147,7 @@ public class KeyVisualization extends AbstractVisualization { int rows = i + 2; int cols = Math.max(6, (int) (rows * task.getHeight() / task.getWidth())); - final double margin = context.getStyleLibrary().getSize(StyleLibrary.MARGIN); + final double margin = style.getSize(StyleLibrary.MARGIN); final String transform = SVGUtil.makeMarginTransform(task.getWidth(), task.getHeight(), cols, rows, margin / StyleLibrary.SCALE); SVGUtil.setAtt(layer, SVGConstants.SVG_TRANSFORM_ATTRIBUTE, transform); } @@ -171,9 +172,8 @@ public class KeyVisualization extends AbstractVisualization { @Override public void processNewResult(HierarchicalResult baseResult, Result newResult) { // Find clusterings we can visualize: - Iterator<Clustering<?>> clusterings = ResultUtil.filteredResults(newResult, Clustering.class); - while(clusterings.hasNext()) { - Clustering<?> c = clusterings.next(); + Collection<Clustering<?>> clusterings = ResultUtil.filterResults(newResult, Clustering.class); + for (Clustering<?> c : clusterings) { if(c.getAllClusters().size() > 0) { final VisualizationTask task = new VisualizationTask(NAME, c, null, this); task.width = 1.0; diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/visunproj/SettingsVisFactory.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/visunproj/SettingsVisFactory.java index 5bbc698c..2d726257 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/visunproj/SettingsVisFactory.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/visunproj/SettingsVisFactory.java @@ -32,7 +32,6 @@ import de.lmu.ifi.dbs.elki.result.HierarchicalResult; import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil; import de.lmu.ifi.dbs.elki.result.SettingsResult; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ClassParameter; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.Parameter; import de.lmu.ifi.dbs.elki.utilities.pairs.Pair; @@ -138,7 +137,7 @@ public class SettingsVisFactory extends AbstractVisFactory { @Override public void processNewResult(HierarchicalResult baseResult, Result newResult) { - final IterableIterator<SettingsResult> settingsResults = ResultUtil.filteredResults(newResult, SettingsResult.class); + Collection<SettingsResult> settingsResults = ResultUtil.filterResults(newResult, SettingsResult.class); for(SettingsResult sr : settingsResults) { final VisualizationTask task = new VisualizationTask(NAME, sr, null, this); task.width = 1.0; diff --git a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/visunproj/CurveVisFactory.java b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/visunproj/XYCurveVisFactory.java index e12a392a..507de376 100644 --- a/src/de/lmu/ifi/dbs/elki/visualization/visualizers/visunproj/CurveVisFactory.java +++ b/src/de/lmu/ifi/dbs/elki/visualization/visualizers/visunproj/XYCurveVisFactory.java @@ -23,25 +23,21 @@ package de.lmu.ifi.dbs.elki.visualization.visualizers.visunproj; along with this program. If not, see <http://www.gnu.org/licenses/>. */ -import java.util.Collection; -import java.util.Iterator; +import java.util.ArrayList; import org.apache.batik.util.SVGConstants; import org.w3c.dom.Element; -import de.lmu.ifi.dbs.elki.evaluation.roc.ComputeROCCurve; -import de.lmu.ifi.dbs.elki.evaluation.roc.ComputeROCCurve.ROCResult; +import de.lmu.ifi.dbs.elki.evaluation.outlier.OutlierROCCurve; +import de.lmu.ifi.dbs.elki.evaluation.outlier.OutlierROCCurve.ROCResult; +import de.lmu.ifi.dbs.elki.evaluation.outlier.OutlierPrecisionRecallCurve.PRCurve; import de.lmu.ifi.dbs.elki.logging.LoggingUtil; -import de.lmu.ifi.dbs.elki.math.DoubleMinMax; +import de.lmu.ifi.dbs.elki.math.geometry.XYCurve; import de.lmu.ifi.dbs.elki.math.scales.LinearScale; import de.lmu.ifi.dbs.elki.result.HierarchicalResult; -import de.lmu.ifi.dbs.elki.result.IterableResult; import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil; import de.lmu.ifi.dbs.elki.utilities.FormatUtil; -import de.lmu.ifi.dbs.elki.utilities.iterator.AbstractFilteredIterator; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableIterator; -import de.lmu.ifi.dbs.elki.utilities.pairs.DoubleDoublePair; import de.lmu.ifi.dbs.elki.visualization.VisualizationTask; import de.lmu.ifi.dbs.elki.visualization.VisualizerContext; import de.lmu.ifi.dbs.elki.visualization.css.CSSClass; @@ -62,13 +58,13 @@ import de.lmu.ifi.dbs.elki.visualization.visualizers.Visualization; * * @apiviz.stereotype factory * @apiviz.uses StaticVisualization oneway - - «create» - * @apiviz.has IterableResult oneway - - visualizes + * @apiviz.has XYCurve oneway - - visualizes */ -public class CurveVisFactory extends AbstractVisFactory { +public class XYCurveVisFactory extends AbstractVisFactory { /** * Name for this visualizer. */ - private static final String NAME = "Curve"; + private static final String NAME = "XYCurve"; /** * SVG class name for plot line @@ -76,9 +72,14 @@ public class CurveVisFactory extends AbstractVisFactory { private static final String SERIESID = "series"; /** + * Axis labels + */ + private static final String CSS_AXIS_LABEL = "xy-axis-label"; + + /** * Constructor, Parameterizable style - does nothing. */ - public CurveVisFactory() { + public XYCurveVisFactory() { super(); } @@ -86,7 +87,7 @@ public class CurveVisFactory extends AbstractVisFactory { public Visualization makeVisualization(VisualizationTask task) { VisualizerContext context = task.getContext(); SVGPlot svgp = task.getPlot(); - IterableResult<DoubleDoublePair> curve = task.getResult(); + XYCurve curve = task.getResult(); setupCSS(context, svgp); double scale = StyleLibrary.SCALE; @@ -98,19 +99,13 @@ public class CurveVisFactory extends AbstractVisFactory { SVGUtil.setAtt(layer, SVGConstants.SVG_TRANSFORM_ATTRIBUTE, transform); // determine scaling - DoubleMinMax minmaxx = new DoubleMinMax(); - DoubleMinMax minmaxy = new DoubleMinMax(); - for(DoubleDoublePair pair : curve) { - minmaxx.put(pair.first); - minmaxy.put(pair.second); - } - LinearScale scalex = new LinearScale(minmaxx.getMin(), minmaxx.getMax()); - LinearScale scaley = new LinearScale(minmaxy.getMin(), minmaxy.getMax()); + LinearScale scalex = new LinearScale(curve.getMinx(), curve.getMaxx()); + LinearScale scaley = new LinearScale(curve.getMiny(), curve.getMaxy()); // plot the line SVGPath path = new SVGPath(); - for(DoubleDoublePair pair : curve) { - final double x = scalex.getScaled(pair.first); - final double y = 1 - scaley.getScaled(pair.second); + for(XYCurve.Itr iterator = curve.iterator(); iterator.valid(); iterator.advance()) { + final double x = scalex.getScaled(iterator.getX()); + final double y = 1 - scaley.getScaled(iterator.getY()); path.drawTo(sizex * x, sizey * y); } Element line = path.makeElement(svgp); @@ -124,35 +119,44 @@ public class CurveVisFactory extends AbstractVisFactory { catch(CSSNamingConflict e) { LoggingUtil.exception(e); } + // Add axis labels + { + Element labelx = svgp.svgText(sizex / 2, sizey + margin * .9, curve.getLabelx()); + SVGUtil.setCSSClass(labelx, CSS_AXIS_LABEL); + layer.appendChild(labelx); + Element labely = svgp.svgText(margin * -.8, sizey * .5, curve.getLabely()); + SVGUtil.setCSSClass(labely, CSS_AXIS_LABEL); + SVGUtil.setAtt(labely, SVGConstants.SVG_TRANSFORM_ATTRIBUTE, "rotate(-90,"+FormatUtil.NF6.format(margin * -.8)+","+FormatUtil.NF6.format(sizey * .5)+")"); + layer.appendChild(labely); + } // Add AUC value when found if(curve instanceof ROCResult) { - Collection<String> header = ((ROCResult) curve).getHeader(); - for(String str : header) { - String[] parts = str.split(":\\s*"); - if(parts[0].equals(ComputeROCCurve.ROCAUC_LABEL) && parts.length == 2) { - double rocauc = Double.parseDouble(parts[1]); - StyleLibrary style = context.getStyleLibrary(); - CSSClass cls = new CSSClass(svgp, "unmanaged"); - String lt = "ROC AUC: " + FormatUtil.NF8.format(rocauc); - double fontsize = style.getTextSize("curve.labels"); - cls.setStatement(SVGConstants.CSS_FONT_SIZE_PROPERTY, SVGUtil.fmt(fontsize)); - cls.setStatement(SVGConstants.CSS_FILL_PROPERTY, style.getTextColor("curve.labels")); - cls.setStatement(SVGConstants.CSS_FONT_FAMILY_PROPERTY, style.getFontFamily("curve.labels")); - if(rocauc <= 0.5) { - Element auclbl = svgp.svgText(sizex * 0.95, sizey * 0.95, lt); - SVGUtil.setAtt(auclbl, SVGConstants.SVG_STYLE_ATTRIBUTE, cls.inlineCSS()); - // SVGUtil.setAtt(auclbl, SVGConstants.SVG_TEXT_ANCHOR_ATTRIBUTE, - // SVGConstants.SVG_START_VALUE); - layer.appendChild(auclbl); - } - else { - Element auclbl = svgp.svgText(sizex * 0.95, sizey * 0.95, lt); - SVGUtil.setAtt(auclbl, SVGConstants.SVG_STYLE_ATTRIBUTE, cls.inlineCSS()); - SVGUtil.setAtt(auclbl, SVGConstants.SVG_TEXT_ANCHOR_ATTRIBUTE, SVGConstants.SVG_END_VALUE); - layer.appendChild(auclbl); - } - } + double rocauc = ((ROCResult) curve).getAUC(); + String lt = OutlierROCCurve.ROCAUC_LABEL + ": " + FormatUtil.NF8.format(rocauc); + if(rocauc <= 0.5) { + Element auclbl = svgp.svgText(sizex * 0.5, sizey * 0.10, lt); + SVGUtil.setCSSClass(auclbl, CSS_AXIS_LABEL); + layer.appendChild(auclbl); + } + else { + Element auclbl = svgp.svgText(sizex * 0.5, sizey * 0.95, lt); + SVGUtil.setCSSClass(auclbl, CSS_AXIS_LABEL); + layer.appendChild(auclbl); + } + } + if(curve instanceof PRCurve) { + double prauc = ((PRCurve) curve).getAUC(); + String lt = PRCurve.PRAUC_LABEL + ": " + FormatUtil.NF8.format(prauc); + if(prauc <= 0.5) { + Element auclbl = svgp.svgText(sizex * 0.5, sizey * 0.10, lt); + SVGUtil.setCSSClass(auclbl, CSS_AXIS_LABEL); + layer.appendChild(auclbl); + } + else { + Element auclbl = svgp.svgText(sizex * 0.5, sizey * 0.95, lt); + SVGUtil.setCSSClass(auclbl, CSS_AXIS_LABEL); + layer.appendChild(auclbl); } } @@ -166,66 +170,26 @@ public class CurveVisFactory extends AbstractVisFactory { * @param svgp Plot */ private void setupCSS(VisualizerContext context, SVGPlot svgp) { + StyleLibrary style = context.getStyleLibrary(); CSSClass csscls = new CSSClass(this, SERIESID); // csscls.setStatement(SVGConstants.SVG_STROKE_WIDTH_ATTRIBUTE, "0.2%"); csscls.setStatement(SVGConstants.SVG_FILL_ATTRIBUTE, SVGConstants.SVG_NONE_VALUE); - context.getStyleLibrary().lines().formatCSSClass(csscls, 0, context.getStyleLibrary().getLineWidth(StyleLibrary.PLOT)); + style.lines().formatCSSClass(csscls, 0, style.getLineWidth(StyleLibrary.XYCURVE)); svgp.addCSSClassOrLogError(csscls); - } - - /** - * Filter to only retrieve double-double-pair results. - * - * @author Erich Schubert - * - * @apiviz.exclude - */ - class CurveFilter extends AbstractFilteredIterator<IterableResult<?>, IterableResult<DoubleDoublePair>> implements IterableIterator<IterableResult<DoubleDoublePair>> { - /** - * Parent iterator to use - */ - Iterator<IterableResult<?>> parent; - - /** - * Constructor. - * - * @param parent Parent iterator to decorate. - */ - public CurveFilter(Iterator<IterableResult<?>> parent) { - super(); - this.parent = parent; - } - - @Override - protected Iterator<IterableResult<?>> getParentIterator() { - return parent; - } - - @Override - protected IterableResult<DoubleDoublePair> testFilter(IterableResult<?> nextobj) { - Iterator<?> iterator = nextobj.iterator(); - if(iterator.hasNext()) { - Object o = iterator.next(); - if(o instanceof DoubleDoublePair) { - @SuppressWarnings("unchecked") - final IterableResult<DoubleDoublePair> ret = (IterableResult<DoubleDoublePair>) nextobj; - return ret; - } - } - return null; - } - - @Override - public Iterator<IterableResult<DoubleDoublePair>> iterator() { - return this; - } + // Axis label + CSSClass label = new CSSClass(this, CSS_AXIS_LABEL); + label.setStatement(SVGConstants.CSS_FILL_PROPERTY, style.getTextColor(StyleLibrary.XYCURVE)); + label.setStatement(SVGConstants.CSS_FONT_FAMILY_PROPERTY, style.getFontFamily(StyleLibrary.XYCURVE)); + label.setStatement(SVGConstants.CSS_FONT_SIZE_PROPERTY, style.getTextSize(StyleLibrary.XYCURVE)); + label.setStatement(SVGConstants.CSS_TEXT_ANCHOR_PROPERTY, SVGConstants.CSS_MIDDLE_VALUE); + svgp.addCSSClassOrLogError(label); + svgp.updateStyleElement(); } @Override public void processNewResult(HierarchicalResult baseResult, Result result) { - final IterableIterator<IterableResult<?>> iterableResults = ResultUtil.filteredResults(result, IterableResult.class); - final IterableIterator<IterableResult<DoubleDoublePair>> curves = new CurveFilter(iterableResults); - for (IterableResult<DoubleDoublePair> curve : curves) { + final ArrayList<XYCurve> curves = ResultUtil.filterResults(result, XYCurve.class); + for(XYCurve curve : curves) { final VisualizationTask task = new VisualizationTask(NAME, curve, null, this); task.width = 1.0; task.height = 1.0; diff --git a/src/de/lmu/ifi/dbs/elki/workflow/AlgorithmStep.java b/src/de/lmu/ifi/dbs/elki/workflow/AlgorithmStep.java index 807f2684..e6788568 100644 --- a/src/de/lmu/ifi/dbs/elki/workflow/AlgorithmStep.java +++ b/src/de/lmu/ifi/dbs/elki/workflow/AlgorithmStep.java @@ -29,6 +29,7 @@ import de.lmu.ifi.dbs.elki.algorithm.Algorithm; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.index.Index; import de.lmu.ifi.dbs.elki.logging.Logging; +import de.lmu.ifi.dbs.elki.logging.LoggingConfiguration; import de.lmu.ifi.dbs.elki.persistent.PageFileStatistics; import de.lmu.ifi.dbs.elki.persistent.PageFileUtil; import de.lmu.ifi.dbs.elki.result.BasicResult; @@ -38,6 +39,7 @@ import de.lmu.ifi.dbs.elki.utilities.FormatUtil; import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.Flag; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectListParameter; /** @@ -133,6 +135,11 @@ public class AlgorithmStep implements WorkflowStep { */ public static class Parameterizer extends AbstractParameterizer { /** + * Enable logging of performance data + */ + protected boolean time = false; + + /** * Holds the algorithm to run. */ protected List<Algorithm> algorithms; @@ -140,6 +147,11 @@ public class AlgorithmStep implements WorkflowStep { @Override protected void makeOptions(Parameterization config) { super.makeOptions(config); + // Time parameter + final Flag timeF = new Flag(OptionID.TIME_FLAG); + if(config.grab(timeF)) { + time = timeF.getValue(); + } // parameter algorithm final ObjectListParameter<Algorithm> ALGORITHM_PARAM = new ObjectListParameter<Algorithm>(OptionID.ALGORITHM, Algorithm.class); if(config.grab(ALGORITHM_PARAM)) { @@ -149,6 +161,7 @@ public class AlgorithmStep implements WorkflowStep { @Override protected AlgorithmStep makeInstance() { + LoggingConfiguration.setTime(time); return new AlgorithmStep(algorithms); } } diff --git a/src/de/lmu/ifi/dbs/elki/workflow/EvaluationStep.java b/src/de/lmu/ifi/dbs/elki/workflow/EvaluationStep.java index 65ac83a6..fdc1f201 100644 --- a/src/de/lmu/ifi/dbs/elki/workflow/EvaluationStep.java +++ b/src/de/lmu/ifi/dbs/elki/workflow/EvaluationStep.java @@ -23,9 +23,11 @@ package de.lmu.ifi.dbs.elki.workflow; along with this program. If not, see <http://www.gnu.org/licenses/>. */ +import java.util.ArrayList; import java.util.List; import de.lmu.ifi.dbs.elki.database.Database; +import de.lmu.ifi.dbs.elki.evaluation.AutomaticEvaluation; import de.lmu.ifi.dbs.elki.evaluation.Evaluator; import de.lmu.ifi.dbs.elki.result.HierarchicalResult; import de.lmu.ifi.dbs.elki.result.Result; @@ -67,7 +69,7 @@ public class EvaluationStep implements WorkflowStep { public void runEvaluators(HierarchicalResult r, Database db) { // Run evaluation helpers if(evaluators != null) { - new Evaluation(db, evaluators).update(r); + new Evaluation(r, evaluators).update(r); } this.result = r; } @@ -83,7 +85,7 @@ public class EvaluationStep implements WorkflowStep { /** * Database */ - private Database database; + private HierarchicalResult baseResult; /** * Evaluators to run. @@ -93,14 +95,14 @@ public class EvaluationStep implements WorkflowStep { /** * Constructor. * - * @param database Database + * @param baseResult base result * @param evaluators Evaluators */ - public Evaluation(Database database, List<Evaluator> evaluators) { - this.database = database; + public Evaluation(HierarchicalResult baseResult, List<Evaluator> evaluators) { + this.baseResult = baseResult; this.evaluators = evaluators; - database.getHierarchy().addResultListener(this); + baseResult.getHierarchy().addResultListener(this); } /** @@ -113,7 +115,7 @@ public class EvaluationStep implements WorkflowStep { /* * if(normalizationUndo) { evaluator.setNormalization(normalization); } */ - evaluator.processNewResult(database, r); + evaluator.processNewResult(baseResult, r); } } @@ -154,10 +156,13 @@ public class EvaluationStep implements WorkflowStep { @Override protected void makeOptions(Parameterization config) { super.makeOptions(config); + List<Class<? extends Evaluator>> def = new ArrayList<Class<? extends Evaluator>>(1); + def.add(AutomaticEvaluation.class); // evaluator parameter - final ObjectListParameter<Evaluator> ealuatorP = new ObjectListParameter<Evaluator>(OptionID.EVALUATOR, Evaluator.class, true); - if(config.grab(ealuatorP)) { - evaluators = ealuatorP.instantiateClasses(config); + final ObjectListParameter<Evaluator> evaluatorP = new ObjectListParameter<Evaluator>(OptionID.EVALUATOR, Evaluator.class); + evaluatorP.setDefaultValue(def); + if(config.grab(evaluatorP)) { + evaluators = evaluatorP.instantiateClasses(config); } } diff --git a/src/de/lmu/ifi/dbs/elki/workflow/LoggingStep.java b/src/de/lmu/ifi/dbs/elki/workflow/LoggingStep.java index 1e1a144f..bfe9dba2 100644 --- a/src/de/lmu/ifi/dbs/elki/workflow/LoggingStep.java +++ b/src/de/lmu/ifi/dbs/elki/workflow/LoggingStep.java @@ -92,8 +92,14 @@ public class LoggingStep implements WorkflowStep { * @apiviz.exclude */ public static class Parameterizer extends AbstractParameterizer { + /** + * Verbose mode + */ protected boolean verbose = false; - + + /** + * Enable logging levels manually + */ protected String[][] levels = null; @Override diff --git a/src/tutorial/outlier/DistanceStddevOutlier.java b/src/tutorial/outlier/DistanceStddevOutlier.java new file mode 100644 index 00000000..56c9e002 --- /dev/null +++ b/src/tutorial/outlier/DistanceStddevOutlier.java @@ -0,0 +1,148 @@ +package tutorial.outlier; + +import de.lmu.ifi.dbs.elki.algorithm.AbstractDistanceBasedAlgorithm; +import de.lmu.ifi.dbs.elki.algorithm.outlier.OutlierAlgorithm; +import de.lmu.ifi.dbs.elki.data.type.TypeInformation; +import de.lmu.ifi.dbs.elki.data.type.TypeUtil; +import de.lmu.ifi.dbs.elki.database.Database; +import de.lmu.ifi.dbs.elki.database.QueryUtil; +import de.lmu.ifi.dbs.elki.database.datastore.DataStoreFactory; +import de.lmu.ifi.dbs.elki.database.datastore.DataStoreUtil; +import de.lmu.ifi.dbs.elki.database.datastore.WritableDoubleDataStore; +import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; +import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; +import de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery; +import de.lmu.ifi.dbs.elki.database.query.knn.KNNResult; +import de.lmu.ifi.dbs.elki.database.relation.MaterializedRelation; +import de.lmu.ifi.dbs.elki.database.relation.Relation; +import de.lmu.ifi.dbs.elki.distance.distancefunction.DistanceFunction; +import de.lmu.ifi.dbs.elki.distance.distancevalue.NumberDistance; +import de.lmu.ifi.dbs.elki.logging.Logging; +import de.lmu.ifi.dbs.elki.math.DoubleMinMax; +import de.lmu.ifi.dbs.elki.math.MeanVariance; +import de.lmu.ifi.dbs.elki.result.outlier.BasicOutlierScoreMeta; +import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; +import de.lmu.ifi.dbs.elki.result.outlier.OutlierScoreMeta; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.GreaterConstraint; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.IntParameter; + +/** + * A simple outlier detection algorithm that computes the standard deviation of + * the kNN distances. + * + * @author Erich Schubert + * + * @param <O> Object type + * @param <D> Distance type + */ +public class DistanceStddevOutlier<O, D extends NumberDistance<D, ?>> extends AbstractDistanceBasedAlgorithm<O, D, OutlierResult> implements OutlierAlgorithm { + /** + * Class logger + */ + private static final Logging logger = Logging.getLogger(DistanceStddevOutlier.class); + + /** + * Number of neighbors to get. + */ + protected int k; + + /** + * Constructor. + * + * @param distanceFunction Distance function to use + * @param k Number of neighbors to use + */ + public DistanceStddevOutlier(DistanceFunction<? super O, D> distanceFunction, int k) { + super(distanceFunction); + this.k = k; + } + + /** + * Run the outlier detection algorithm + * + * @param database Database to use + * @param relation Relation to analyze + * @return Outlier score result + */ + public OutlierResult run(Database database, Relation<O> relation) { + // Get a nearest neighbor query on the relation. + KNNQuery<O, D> knnq = QueryUtil.getKNNQuery(relation, getDistanceFunction(), k); + // Output data storage + WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_DB); + // Track minimum and maximum scores + DoubleMinMax minmax = new DoubleMinMax(); + + // Iterate over all objects + for(DBIDIter iter = relation.iterDBIDs(); iter.valid(); iter.advance()) { + DBID id = iter.getDBID(); + KNNResult<D> neighbors = knnq.getKNNForDBID(id, k); + // Aggregate distances + MeanVariance mv = new MeanVariance(); + for(DistanceResultPair<D> neighbor : neighbors) { + // Skip the object itself. The 0 is not very informative. + if(id.sameDBID(neighbor.getDBID())) { + continue; + } + mv.put(neighbor.getDistance().doubleValue()); + } + // Store score + scores.putDouble(id, mv.getSampleStddev()); + } + + // Wrap the result in the standard containers + // Actual min-max, theoretical min-max! + OutlierScoreMeta meta = new BasicOutlierScoreMeta(minmax.getMin(), minmax.getMax(), 0, Double.POSITIVE_INFINITY); + Relation<Double> rel = new MaterializedRelation<Double>(database, TypeUtil.DOUBLE, relation.getDBIDs(), "stddev-outlier", scores); + return new OutlierResult(meta, rel); + } + + @Override + public TypeInformation[] getInputTypeRestriction() { + return TypeUtil.array(getDistanceFunction().getInputTypeRestriction()); + } + + @Override + protected Logging getLogger() { + return logger; + } + + /** + * Parameterization class + * + * @author Erich Schubert + * + * @apiviz.exclude + * + * @param <O> Object type + * @param <D> Distance type + */ + public static class Parameterizer<O, D extends NumberDistance<D, ?>> extends AbstractDistanceBasedAlgorithm.Parameterizer<O, D> { + /** + * Option ID for parameterization. + */ + public static final OptionID K_ID = OptionID.getOrCreateOptionID("stddevout.k", "Number of neighbors to get for stddev based outlier detection."); + + /** + * Number of neighbors to get + */ + int k; + + @Override + protected void makeOptions(Parameterization config) { + // The super class has the distance function parameter! + super.makeOptions(config); + IntParameter kParam = new IntParameter(K_ID, new GreaterConstraint(1)); + if(config.grab(kParam)) { + k = kParam.getValue(); + } + } + + @Override + protected DistanceStddevOutlier<O, D> makeInstance() { + return new DistanceStddevOutlier<O, D>(distanceFunction, k); + } + } +}
\ No newline at end of file diff --git a/test/de/lmu/ifi/dbs/elki/algorithm/AbstractSimpleAlgorithmTest.java b/test/de/lmu/ifi/dbs/elki/algorithm/AbstractSimpleAlgorithmTest.java index 7b6fbbfa..3f9171aa 100644 --- a/test/de/lmu/ifi/dbs/elki/algorithm/AbstractSimpleAlgorithmTest.java +++ b/test/de/lmu/ifi/dbs/elki/algorithm/AbstractSimpleAlgorithmTest.java @@ -29,8 +29,8 @@ import static org.junit.Assert.fail; import java.io.File; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; -import java.util.Iterator; import java.util.List; import de.lmu.ifi.dbs.elki.algorithm.clustering.trivial.ByLabelClustering; @@ -46,7 +46,7 @@ import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.datasource.FileBasedDatabaseConnection; import de.lmu.ifi.dbs.elki.datasource.filter.FixedDBIDsFilter; import de.lmu.ifi.dbs.elki.evaluation.clustering.ClusterContingencyTable; -import de.lmu.ifi.dbs.elki.evaluation.roc.ComputeROCCurve; +import de.lmu.ifi.dbs.elki.evaluation.outlier.OutlierROCCurve; import de.lmu.ifi.dbs.elki.logging.Logging; import de.lmu.ifi.dbs.elki.result.Result; import de.lmu.ifi.dbs.elki.result.ResultUtil; @@ -206,8 +206,8 @@ public abstract class AbstractSimpleAlgorithmTest { */ protected void testAUC(Database db, String positive, OutlierResult result, double expected) { ListParameterization params = new ListParameterization(); - params.addParameter(ComputeROCCurve.POSITIVE_CLASS_NAME_ID, positive); - ComputeROCCurve rocCurve = ClassGenericsUtil.parameterizeOrAbort(ComputeROCCurve.class, params); + params.addParameter(OutlierROCCurve.POSITIVE_CLASS_NAME_ID, positive); + OutlierROCCurve rocCurve = ClassGenericsUtil.parameterizeOrAbort(OutlierROCCurve.class, params); // Ensure the result has been added to the hierarchy: if(db.getHierarchy().getParents(result).size() < 1) { @@ -217,10 +217,10 @@ public abstract class AbstractSimpleAlgorithmTest { // Compute ROC and AUC: rocCurve.processNewResult(db, result); // Find the ROC results - Iterator<ComputeROCCurve.ROCResult> iter = ResultUtil.filteredResults(result, ComputeROCCurve.ROCResult.class); - org.junit.Assert.assertTrue("No ROC result found.", iter.hasNext()); - double auc = iter.next().getAUC(); - org.junit.Assert.assertFalse("More than one ROC result found.", iter.hasNext()); + Collection<OutlierROCCurve.ROCResult> rocs = ResultUtil.filterResults(result, OutlierROCCurve.ROCResult.class); + org.junit.Assert.assertTrue("No ROC result found.", !rocs.isEmpty()); + double auc = rocs.iterator().next().getAUC(); + org.junit.Assert.assertFalse("More than one ROC result found.", rocs.size() > 1); org.junit.Assert.assertEquals("ROC value does not match.", expected, auc, 0.0001); } diff --git a/test/de/lmu/ifi/dbs/elki/algorithm/TestKNNJoin.java b/test/de/lmu/ifi/dbs/elki/algorithm/TestKNNJoin.java index c846aa6b..6c3ddc1a 100644 --- a/test/de/lmu/ifi/dbs/elki/algorithm/TestKNNJoin.java +++ b/test/de/lmu/ifi/dbs/elki/algorithm/TestKNNJoin.java @@ -37,6 +37,7 @@ import de.lmu.ifi.dbs.elki.database.QueryUtil; import de.lmu.ifi.dbs.elki.database.StaticArrayDatabase; import de.lmu.ifi.dbs.elki.database.datastore.DataStore; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery; import de.lmu.ifi.dbs.elki.database.query.knn.KNNResult; @@ -101,8 +102,8 @@ public class TestKNNJoin implements JUnit4Test { KNNQuery<NumberVector<?, ?>, DoubleDistance> knnq = QueryUtil.getLinearScanKNNQuery(dq); MeanVariance meansize = new MeanVariance(); - for(DBID id : relation.iterDBIDs()) { - KNNResult<DoubleDistance> knnlist = knnq.getKNNForDBID(id, 2); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + KNNResult<DoubleDistance> knnlist = knnq.getKNNForDBID(iditer, 2); meansize.put(knnlist.size()); } org.junit.Assert.assertEquals("Euclidean mean 2NN", mean2nnEuclid, meansize.getMean(), 0.00001); @@ -114,8 +115,8 @@ public class TestKNNJoin implements JUnit4Test { KNNQuery<NumberVector<?, ?>, DoubleDistance> knnq = QueryUtil.getLinearScanKNNQuery(dq); MeanVariance meansize = new MeanVariance(); - for(DBID id : relation.iterDBIDs()) { - KNNResult<DoubleDistance> knnlist = knnq.getKNNForDBID(id, 2); + for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) { + KNNResult<DoubleDistance> knnlist = knnq.getKNNForDBID(iditer, 2); meansize.put(knnlist.size()); } org.junit.Assert.assertEquals("Manhattan mean 2NN", mean2nnManhattan, meansize.getMean(), 0.00001); diff --git a/test/de/lmu/ifi/dbs/elki/algorithm/clustering/TestDBSCANResults.java b/test/de/lmu/ifi/dbs/elki/algorithm/clustering/TestDBSCANResults.java index 1e95165a..714f345c 100644 --- a/test/de/lmu/ifi/dbs/elki/algorithm/clustering/TestDBSCANResults.java +++ b/test/de/lmu/ifi/dbs/elki/algorithm/clustering/TestDBSCANResults.java @@ -27,6 +27,7 @@ import org.junit.Test; import de.lmu.ifi.dbs.elki.JUnit4Test; import de.lmu.ifi.dbs.elki.algorithm.AbstractSimpleAlgorithmTest; +import de.lmu.ifi.dbs.elki.algorithm.clustering.gdbscan.GeneralizedDBSCAN; import de.lmu.ifi.dbs.elki.data.Clustering; import de.lmu.ifi.dbs.elki.data.DoubleVector; import de.lmu.ifi.dbs.elki.data.model.Model; @@ -93,4 +94,51 @@ public class TestDBSCANResults extends AbstractSimpleAlgorithmTest implements JU testFMeasure(db, result, 0.954382); testClusterSizes(result, new int[] { 11, 200, 203, 224 }); } + + /** + * Run DBSCAN with fixed parameters and compare the result to a golden + * standard. + * + * @throws ParameterException + */ + @Test + public void testGDBSCANResults() { + Database db = makeSimpleDatabase(UNITTEST + "3clusters-and-noise-2d.csv", 330); + + // setup algorithm + ListParameterization params = new ListParameterization(); + params.addParameter(DBSCAN.EPSILON_ID, 0.04); + params.addParameter(DBSCAN.MINPTS_ID, 20); + GeneralizedDBSCAN dbscan = ClassGenericsUtil.parameterizeOrAbort(GeneralizedDBSCAN.class, params); + testParameterizationOk(params); + + // run DBSCAN on database + Clustering<Model> result = dbscan.run(db); + + testClusterSizes(result, new int[] { 29, 50, 101, 150 }); + testFMeasure(db, result, 0.996413); + } + + /** + * Run DBSCAN with fixed parameters and compare the result to a golden + * standard. + * + * @throws ParameterException + */ + @Test + public void testGDBSCANOnSingleLinkDataset() { + Database db = makeSimpleDatabase(UNITTEST + "single-link-effect.ascii", 638); + + // Setup algorithm + ListParameterization params = new ListParameterization(); + params.addParameter(DBSCAN.EPSILON_ID, 11.5); + params.addParameter(DBSCAN.MINPTS_ID, 120); + GeneralizedDBSCAN dbscan = ClassGenericsUtil.parameterizeOrAbort(GeneralizedDBSCAN.class, params); + testParameterizationOk(params); + + // run DBSCAN on database + Clustering<Model> result = dbscan.run(db); + testClusterSizes(result, new int[] { 11, 200, 203, 224 }); + testFMeasure(db, result, 0.954382); + } }
\ No newline at end of file diff --git a/test/de/lmu/ifi/dbs/elki/algorithm/clustering/TestDeLiCluResults.java b/test/de/lmu/ifi/dbs/elki/algorithm/clustering/TestDeLiCluResults.java index 7b62b913..a4fadeb2 100644 --- a/test/de/lmu/ifi/dbs/elki/algorithm/clustering/TestDeLiCluResults.java +++ b/test/de/lmu/ifi/dbs/elki/algorithm/clustering/TestDeLiCluResults.java @@ -23,6 +23,8 @@ package de.lmu.ifi.dbs.elki.algorithm.clustering; along with this program. If not, see <http://www.gnu.org/licenses/>. */ +import static org.junit.Assert.assertEquals; + import org.junit.Test; import de.lmu.ifi.dbs.elki.JUnit4Test; @@ -82,6 +84,6 @@ public class TestDeLiCluResults extends AbstractSimpleAlgorithmTest implements J double score = ct.getPaircount().f1Measure(); // We cannot test exactly - due to Hashing, DeLiClu sequence is not // identical each time, the results will vary slightly. - org.junit.Assert.assertTrue(this.getClass().getSimpleName() + ": Score does not match: " + score, score > 0.85); + assertEquals(this.getClass().getSimpleName() + ": Score does not match: " + score, score, 0.807415, 1E-5); } }
\ No newline at end of file diff --git a/test/de/lmu/ifi/dbs/elki/algorithm/clustering/TestEMResults.java b/test/de/lmu/ifi/dbs/elki/algorithm/clustering/TestEMResults.java index d9f6e7e6..421141a3 100644 --- a/test/de/lmu/ifi/dbs/elki/algorithm/clustering/TestEMResults.java +++ b/test/de/lmu/ifi/dbs/elki/algorithm/clustering/TestEMResults.java @@ -27,7 +27,7 @@ import org.junit.Test; import de.lmu.ifi.dbs.elki.JUnit4Test; import de.lmu.ifi.dbs.elki.algorithm.AbstractSimpleAlgorithmTest; -import de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.AbstractKMeans; +import de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMeans; import de.lmu.ifi.dbs.elki.data.Clustering; import de.lmu.ifi.dbs.elki.data.DoubleVector; import de.lmu.ifi.dbs.elki.data.model.EMModel; @@ -57,7 +57,7 @@ public class TestEMResults extends AbstractSimpleAlgorithmTest implements JUnit4 // Setup algorithm ListParameterization params = new ListParameterization(); - params.addParameter(AbstractKMeans.SEED_ID, 1); + params.addParameter(KMeans.SEED_ID, 1); params.addParameter(EM.K_ID, 5); EM<DoubleVector> em = ClassGenericsUtil.parameterizeOrAbort(EM.class, params); testParameterizationOk(params); diff --git a/test/de/lmu/ifi/dbs/elki/algorithm/clustering/TestSNNClusteringResults.java b/test/de/lmu/ifi/dbs/elki/algorithm/clustering/TestSNNClusteringResults.java index ba13af08..302d5c0b 100644 --- a/test/de/lmu/ifi/dbs/elki/algorithm/clustering/TestSNNClusteringResults.java +++ b/test/de/lmu/ifi/dbs/elki/algorithm/clustering/TestSNNClusteringResults.java @@ -66,7 +66,7 @@ public class TestSNNClusteringResults extends AbstractSimpleAlgorithmTest implem // run SNN on database Clustering<Model> result = snn.run(db); - testFMeasure(db, result, 0.835000); - testClusterSizes(result, new int[] { 76, 213, 219, 225, 231, 236 }); + testFMeasure(db, result, 0.832371422); + testClusterSizes(result, new int[] { 73, 228, 213, 219, 231, 236 }); } }
\ No newline at end of file diff --git a/test/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/TestCASHResults.java b/test/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/TestCASHResults.java index f6d4aba5..2757ab36 100644 --- a/test/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/TestCASHResults.java +++ b/test/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/TestCASHResults.java @@ -28,10 +28,10 @@ import org.junit.Test; import de.lmu.ifi.dbs.elki.JUnit4Test; import de.lmu.ifi.dbs.elki.algorithm.AbstractSimpleAlgorithmTest; import de.lmu.ifi.dbs.elki.data.Clustering; +import de.lmu.ifi.dbs.elki.data.ParameterizationFunction; import de.lmu.ifi.dbs.elki.data.model.Model; import de.lmu.ifi.dbs.elki.database.Database; -import de.lmu.ifi.dbs.elki.datasource.FileBasedDatabaseConnection; -import de.lmu.ifi.dbs.elki.datasource.parser.ParameterizationFunctionLabelParser; +import de.lmu.ifi.dbs.elki.datasource.parser.NumberVectorLabelParser; import de.lmu.ifi.dbs.elki.utilities.ClassGenericsUtil; import de.lmu.ifi.dbs.elki.utilities.optionhandling.ParameterException; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.ListParameterization; @@ -55,7 +55,7 @@ public class TestCASHResults extends AbstractSimpleAlgorithmTest implements JUni public void testCASHResults() { ListParameterization inp = new ListParameterization(); // CASH input - inp.addParameter(FileBasedDatabaseConnection.PARSER_ID, ParameterizationFunctionLabelParser.class); + inp.addParameter(NumberVectorLabelParser.VECTOR_TYPE_ID, ParameterizationFunction.class); // Input Database db = makeSimpleDatabase(UNITTEST + "hierarchical-3d2d1d.csv", 600, inp, null); @@ -86,7 +86,7 @@ public class TestCASHResults extends AbstractSimpleAlgorithmTest implements JUni public void testCASHEmbedded() { // CASH input ListParameterization inp = new ListParameterization(); - inp.addParameter(FileBasedDatabaseConnection.PARSER_ID, ParameterizationFunctionLabelParser.class); + inp.addParameter(NumberVectorLabelParser.VECTOR_TYPE_ID, ParameterizationFunction.class); Database db = makeSimpleDatabase(UNITTEST + "correlation-embedded-2-4d.ascii", 600, inp, null); // CASH parameters diff --git a/test/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/TestORCLUSResults.java b/test/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/TestORCLUSResults.java index 2660c586..ef9ad063 100644 --- a/test/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/TestORCLUSResults.java +++ b/test/de/lmu/ifi/dbs/elki/algorithm/clustering/correlation/TestORCLUSResults.java @@ -56,7 +56,6 @@ public class TestORCLUSResults extends AbstractSimpleAlgorithmTest implements JU Database db = makeSimpleDatabase(UNITTEST + "correlation-hierarchy.csv", 450); ListParameterization params = new ListParameterization(); - // these parameters are not picked too smartly - room for improvement. params.addParameter(ORCLUS.K_ID, 3); params.addParameter(ORCLUS.L_ID, 1); params.addParameter(ORCLUS.SEED_ID, 2); @@ -68,8 +67,8 @@ public class TestORCLUSResults extends AbstractSimpleAlgorithmTest implements JU // run ORCLUS on database Clustering<Model> result = orclus.run(db); - testFMeasure(db, result, 0.640306); // Hierarchical pairs scored: 0.789113 - testClusterSizes(result, new int[] { 22, 27, 401 }); + testFMeasure(db, result, 0.6361108); // Hierarchical pairs scored: 0.789113 + testClusterSizes(result, new int[] { 19, 33, 398 }); } /** diff --git a/test/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/TestKMeansResults.java b/test/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/TestKMeansResults.java index c35a7194..589be8ad 100644 --- a/test/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/TestKMeansResults.java +++ b/test/de/lmu/ifi/dbs/elki/algorithm/clustering/kmeans/TestKMeansResults.java @@ -31,6 +31,7 @@ import de.lmu.ifi.dbs.elki.algorithm.clustering.kmeans.KMeansLloyd; import de.lmu.ifi.dbs.elki.data.Clustering; import de.lmu.ifi.dbs.elki.data.DoubleVector; import de.lmu.ifi.dbs.elki.data.model.MeanModel; +import de.lmu.ifi.dbs.elki.data.model.MedoidModel; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.distance.distancevalue.DoubleDistance; import de.lmu.ifi.dbs.elki.utilities.ClassGenericsUtil; @@ -59,8 +60,8 @@ public class TestKMeansResults extends AbstractSimpleAlgorithmTest implements JU // Setup algorithm ListParameterization params = new ListParameterization(); - params.addParameter(AbstractKMeans.K_ID, 5); - params.addParameter(AbstractKMeans.SEED_ID, 3); + params.addParameter(KMeans.K_ID, 5); + params.addParameter(KMeans.SEED_ID, 3); AbstractKMeans<DoubleVector, DoubleDistance> kmeans = ClassGenericsUtil.parameterizeOrAbort(KMeansLloyd.class, params); testParameterizationOk(params); @@ -82,8 +83,8 @@ public class TestKMeansResults extends AbstractSimpleAlgorithmTest implements JU // Setup algorithm ListParameterization params = new ListParameterization(); - params.addParameter(AbstractKMeans.K_ID, 5); - params.addParameter(AbstractKMeans.SEED_ID, 3); + params.addParameter(KMeans.K_ID, 5); + params.addParameter(KMeans.SEED_ID, 3); AbstractKMeans<DoubleVector, DoubleDistance> kmeans = ClassGenericsUtil.parameterizeOrAbort(KMeansMacQueen.class, params); testParameterizationOk(params); @@ -92,4 +93,71 @@ public class TestKMeansResults extends AbstractSimpleAlgorithmTest implements JU testFMeasure(db, result, 0.998005); testClusterSizes(result, new int[] { 199, 200, 200, 200, 201 }); } + + /** + * Run KMedians with fixed parameters and compare the result to a golden + * standard. + * + * @throws ParameterException + */ + @Test + public void testKMediansLloyd() { + Database db = makeSimpleDatabase(UNITTEST + "different-densities-2d-no-noise.ascii", 1000); + + // Setup algorithm + ListParameterization params = new ListParameterization(); + params.addParameter(KMeans.K_ID, 5); + params.addParameter(KMeans.SEED_ID, 3); + AbstractKMeans<DoubleVector, DoubleDistance> kmedians = ClassGenericsUtil.parameterizeOrAbort(KMediansLloyd.class, params); + testParameterizationOk(params); + + // run KMedians on database + Clustering<MeanModel<DoubleVector>> result = kmedians.run(db); + testFMeasure(db, result, 0.998005); + testClusterSizes(result, new int[] { 199, 200, 200, 200, 201 }); + } + + /** + * Run KMedians PAM with fixed parameters and compare the result to a golden + * standard. + * + * @throws ParameterException + */ + @Test + public void testKMedoidsPAM() { + Database db = makeSimpleDatabase(UNITTEST + "different-densities-2d-no-noise.ascii", 1000); + + // Setup algorithm + ListParameterization params = new ListParameterization(); + params.addParameter(KMeans.K_ID, 5); + KMedoidsPAM<DoubleVector, DoubleDistance> kmedians = ClassGenericsUtil.parameterizeOrAbort(KMedoidsPAM.class, params); + testParameterizationOk(params); + + // run KMedians on database + Clustering<MedoidModel> result = kmedians.run(db); + testFMeasure(db, result, 0.998005); + testClusterSizes(result, new int[] { 199, 200, 200, 200, 201 }); + } + + /** + * Run KMedoidsEM with fixed parameters and compare the result to a golden + * standard. + * + * @throws ParameterException + */ + @Test + public void testKMedoidsEM() { + Database db = makeSimpleDatabase(UNITTEST + "different-densities-2d-no-noise.ascii", 1000); + + // Setup algorithm + ListParameterization params = new ListParameterization(); + params.addParameter(KMeans.K_ID, 5); + KMedoidsEM<DoubleVector, DoubleDistance> kmedians = ClassGenericsUtil.parameterizeOrAbort(KMedoidsEM.class, params); + testParameterizationOk(params); + + // run KMedians on database + Clustering<MedoidModel> result = kmedians.run(db); + testFMeasure(db, result, 0.998005); + testClusterSizes(result, new int[] { 199, 200, 200, 200, 201 }); + } }
\ No newline at end of file diff --git a/test/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/TestPROCLUSResults.java b/test/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/TestPROCLUSResults.java index c5f37c88..82038b8b 100644 --- a/test/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/TestPROCLUSResults.java +++ b/test/de/lmu/ifi/dbs/elki/algorithm/clustering/subspace/TestPROCLUSResults.java @@ -29,7 +29,6 @@ import de.lmu.ifi.dbs.elki.JUnit4Test; import de.lmu.ifi.dbs.elki.algorithm.AbstractSimpleAlgorithmTest; import de.lmu.ifi.dbs.elki.data.Clustering; import de.lmu.ifi.dbs.elki.data.DoubleVector; -import de.lmu.ifi.dbs.elki.data.model.Model; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.utilities.ClassGenericsUtil; import de.lmu.ifi.dbs.elki.utilities.optionhandling.ParameterException; @@ -66,10 +65,10 @@ public class TestPROCLUSResults extends AbstractSimpleAlgorithmTest implements J testParameterizationOk(params); // run PROCLUS on database - Clustering<Model> result = proclus.run(db); + Clustering<?> result = proclus.run(db); - testFMeasure(db, result, 0.68932); - testClusterSizes(result, new int[] { 78, 93, 203, 226 }); + testFMeasure(db, result, 0.7541038); + testClusterSizes(result, new int[] { 292, 308 }); } /** @@ -86,13 +85,13 @@ public class TestPROCLUSResults extends AbstractSimpleAlgorithmTest implements J ListParameterization params = new ListParameterization(); params.addParameter(PROCLUS.L_ID, 2); params.addParameter(PROCLUS.K_ID, 3); - params.addParameter(PROCLUS.SEED_ID, 2); + params.addParameter(PROCLUS.SEED_ID, 4); PROCLUS<DoubleVector> proclus = ClassGenericsUtil.parameterizeOrAbort(PROCLUS.class, params); testParameterizationOk(params); // run PROCLUS on database - Clustering<Model> result = proclus.run(db); - testFMeasure(db, result, 0.9673718); - testClusterSizes(result, new int[] { 150, 289, 411 }); + Clustering<?> result = proclus.run(db); + testFMeasure(db, result, 0.6329819); + testClusterSizes(result, new int[] { 282, 568 }); } }
\ No newline at end of file diff --git a/test/de/lmu/ifi/dbs/elki/algorithm/outlier/TestINFLO.java b/test/de/lmu/ifi/dbs/elki/algorithm/outlier/TestINFLO.java index 39dd9bc0..8a1e84ef 100644 --- a/test/de/lmu/ifi/dbs/elki/algorithm/outlier/TestINFLO.java +++ b/test/de/lmu/ifi/dbs/elki/algorithm/outlier/TestINFLO.java @@ -55,7 +55,7 @@ public class TestINFLO extends AbstractSimpleAlgorithmTest implements JUnit4Test // run INFLO on database OutlierResult result = inflo.run(db); - testSingleScore(result, 945, 2.5711647857619484); - testAUC(db, "Noise", result, 0.935222); + testSingleScore(result, 945, 1.215459716); + testAUC(db, "Noise", result, 0.9389259259259); } }
\ No newline at end of file diff --git a/test/de/lmu/ifi/dbs/elki/algorithm/outlier/TestOnlineLOF.java b/test/de/lmu/ifi/dbs/elki/algorithm/outlier/TestOnlineLOF.java index 10cff737..d9b46aea 100644 --- a/test/de/lmu/ifi/dbs/elki/algorithm/outlier/TestOnlineLOF.java +++ b/test/de/lmu/ifi/dbs/elki/algorithm/outlier/TestOnlineLOF.java @@ -37,7 +37,7 @@ import de.lmu.ifi.dbs.elki.data.VectorUtil; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.HashmapDatabase; import de.lmu.ifi.dbs.elki.database.UpdatableDatabase; -import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.ids.DBIDs; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.datasource.FileBasedDatabaseConnection; @@ -105,10 +105,10 @@ public class TestOnlineLOF implements JUnit4Test { Relation<Double> scores1 = result1.getScores(); Relation<Double> scores2 = result2.getScores(); - for(DBID id : scores1.getDBIDs()) { + for(DBIDIter id = scores1.getDBIDs().iter(); id.valid(); id.advance()) { Double lof1 = scores1.get(id); Double lof2 = scores2.get(id); - assertTrue("lof(" + id + ") != lof(" + id + "): " + lof1 + " != " + lof2, lof1.equals(lof2)); + assertTrue("lof(" + id.getDBID() + ") != lof(" + id.getDBID() + "): " + lof1 + " != " + lof2, lof1.equals(lof2)); } } diff --git a/test/de/lmu/ifi/dbs/elki/algorithm/outlier/meta/TestHiCS.java b/test/de/lmu/ifi/dbs/elki/algorithm/outlier/meta/TestHiCS.java new file mode 100644 index 00000000..40240ec4 --- /dev/null +++ b/test/de/lmu/ifi/dbs/elki/algorithm/outlier/meta/TestHiCS.java @@ -0,0 +1,88 @@ +package de.lmu.ifi.dbs.elki.algorithm.outlier.meta; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import org.junit.Test; + +import de.lmu.ifi.dbs.elki.JUnit4Test; +import de.lmu.ifi.dbs.elki.algorithm.AbstractSimpleAlgorithmTest; +import de.lmu.ifi.dbs.elki.algorithm.outlier.LOF; +import de.lmu.ifi.dbs.elki.data.DoubleVector; +import de.lmu.ifi.dbs.elki.database.Database; +import de.lmu.ifi.dbs.elki.math.statistics.tests.KolmogorovSmirnovTest; +import de.lmu.ifi.dbs.elki.math.statistics.tests.WelchTTest; +import de.lmu.ifi.dbs.elki.result.outlier.OutlierResult; +import de.lmu.ifi.dbs.elki.utilities.ClassGenericsUtil; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.ListParameterization; + +/** + * Tests the HiCS algorithm. + * + * @author Erich Schubert + */ +public class TestHiCS extends AbstractSimpleAlgorithmTest implements JUnit4Test { + @Test + public void testHiCSKS() { + Database db = makeSimpleDatabase(UNITTEST + "outlier-axis-subspaces-6d.ascii", 1345); + + // Parameterization + ListParameterization params = new ListParameterization(); + params.addParameter(LOF.K_ID, 10); + params.addParameter(HiCS.Parameterizer.LIMIT_ID, 10); + params.addParameter(HiCS.Parameterizer.SEED_ID, 0); + params.addParameter(HiCS.Parameterizer.TEST_ID, KolmogorovSmirnovTest.STATIC); + + // setup Algorithm + HiCS<DoubleVector> fb = ClassGenericsUtil.parameterizeOrAbort(HiCS.class, params); + testParameterizationOk(params); + + // run HiCS on database + OutlierResult result = fb.run(db); + + testAUC(db, "Noise", result, 0.9024537815126049); + testSingleScore(result, 1293, 5.0754391836); + } + + @Test + public void testHiCSWelch() { + Database db = makeSimpleDatabase(UNITTEST + "outlier-axis-subspaces-6d.ascii", 1345); + + // Parameterization + ListParameterization params = new ListParameterization(); + params.addParameter(LOF.K_ID, 10); + params.addParameter(HiCS.Parameterizer.LIMIT_ID, 10); + params.addParameter(HiCS.Parameterizer.SEED_ID, 0); + params.addParameter(HiCS.Parameterizer.TEST_ID, WelchTTest.STATIC); + + // setup Algorithm + HiCS<DoubleVector> fb = ClassGenericsUtil.parameterizeOrAbort(HiCS.class, params); + testParameterizationOk(params); + + // run HiCS on database + OutlierResult result = fb.run(db); + + testAUC(db, "Noise", result, 0.6597983193); + testSingleScore(result, 1293, 2.6993476951); + } +}
\ No newline at end of file diff --git a/test/de/lmu/ifi/dbs/elki/algorithm/outlier/TestSOD.java b/test/de/lmu/ifi/dbs/elki/algorithm/outlier/subspace/TestSOD.java index de637957..c43f2f35 100644 --- a/test/de/lmu/ifi/dbs/elki/algorithm/outlier/TestSOD.java +++ b/test/de/lmu/ifi/dbs/elki/algorithm/outlier/subspace/TestSOD.java @@ -1,4 +1,4 @@ -package de.lmu.ifi.dbs.elki.algorithm.outlier; +package de.lmu.ifi.dbs.elki.algorithm.outlier.subspace; /* This file is part of ELKI: @@ -57,7 +57,7 @@ public class TestSOD extends AbstractSimpleAlgorithmTest implements JUnit4Test { // run SOD on database OutlierResult result = sod.run(db); - testSingleScore(result, 1293, 1.7277777); - testAUC(db, "Noise", result, 0.94956862); + testSingleScore(result, 1293, 1.5167500); + testAUC(db, "Noise", result, 0.949131652); } }
\ No newline at end of file diff --git a/test/de/lmu/ifi/dbs/elki/evaluation/roc/TestComputeROC.java b/test/de/lmu/ifi/dbs/elki/evaluation/roc/TestComputeROC.java index 206fbcfd..6a6157a1 100644 --- a/test/de/lmu/ifi/dbs/elki/evaluation/roc/TestComputeROC.java +++ b/test/de/lmu/ifi/dbs/elki/evaluation/roc/TestComputeROC.java @@ -24,7 +24,6 @@ package de.lmu.ifi.dbs.elki.evaluation.roc; */ import java.util.ArrayList; -import java.util.List; import junit.framework.Assert; @@ -34,8 +33,7 @@ import de.lmu.ifi.dbs.elki.JUnit4Test; import de.lmu.ifi.dbs.elki.database.ids.DBID; import de.lmu.ifi.dbs.elki.database.ids.DBIDUtil; import de.lmu.ifi.dbs.elki.database.ids.HashSetModifiableDBIDs; -import de.lmu.ifi.dbs.elki.evaluation.roc.ROC; -import de.lmu.ifi.dbs.elki.utilities.pairs.DoubleDoublePair; +import de.lmu.ifi.dbs.elki.math.geometry.XYCurve; import de.lmu.ifi.dbs.elki.utilities.pairs.Pair; /** @@ -67,11 +65,11 @@ public class TestComputeROC implements JUnit4Test { distances.add(new Pair<Double, DBID>(5.0, DBIDUtil.importInteger(9))); distances.add(new Pair<Double, DBID>(6.0, DBIDUtil.importInteger(5))); - List<DoubleDoublePair> roccurve = ROC.materializeROC(9, positive, distances.iterator()); - // System.out.println(roccurve); + XYCurve roccurve = ROC.materializeROC(9, positive, distances.iterator()); + System.out.println(roccurve); Assert.assertEquals("ROC curve too complex", 6, roccurve.size()); - double auc = ROC.computeAUC(roccurve); + double auc = XYCurve.areaUnderCurve(roccurve); Assert.assertEquals("ROC AUC not right.", 0.6, auc, 0.0001); } } diff --git a/test/de/lmu/ifi/dbs/elki/index/TestIndexStructures.java b/test/de/lmu/ifi/dbs/elki/index/TestIndexStructures.java index 4ce9dc6e..e0b3a479 100644 --- a/test/de/lmu/ifi/dbs/elki/index/TestIndexStructures.java +++ b/test/de/lmu/ifi/dbs/elki/index/TestIndexStructures.java @@ -35,7 +35,6 @@ import de.lmu.ifi.dbs.elki.data.DoubleVector; import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.StaticArrayDatabase; -import de.lmu.ifi.dbs.elki.database.ids.DBID; import de.lmu.ifi.dbs.elki.database.query.DistanceResultPair; import de.lmu.ifi.dbs.elki.database.query.distance.DistanceQuery; import de.lmu.ifi.dbs.elki.database.query.knn.KNNQuery; @@ -58,6 +57,8 @@ import de.lmu.ifi.dbs.elki.index.tree.spatial.rstarvariants.query.DoubleDistance import de.lmu.ifi.dbs.elki.index.tree.spatial.rstarvariants.rstar.RStarTree; import de.lmu.ifi.dbs.elki.index.tree.spatial.rstarvariants.rstar.RStarTreeFactory; import de.lmu.ifi.dbs.elki.index.tree.spatial.rstarvariants.strategies.insert.ApproximativeLeastOverlapInsertionStrategy; +import de.lmu.ifi.dbs.elki.index.vafile.PartialVAFile; +import de.lmu.ifi.dbs.elki.index.vafile.VAFile; import de.lmu.ifi.dbs.elki.utilities.ClassGenericsUtil; import de.lmu.ifi.dbs.elki.utilities.optionhandling.ParameterException; import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.ListParameterization; @@ -131,6 +132,32 @@ public class TestIndexStructures implements JUnit4Test { } /** + * Test {@link VAFile} using a file based database connection. + * + * @throws ParameterException on errors. + */ + @Test + public void testVAFile() { + ListParameterization spatparams = new ListParameterization(); + spatparams.addParameter(StaticArrayDatabase.INDEX_ID, VAFile.Factory.class); + spatparams.addParameter(VAFile.Factory.PARTITIONS_ID, 4); + testFileBasedDatabaseConnection(spatparams, VAFile.VAFileKNNQuery.class, VAFile.VAFileRangeQuery.class); + } + + /** + * Test {@link PartialVAFile} using a file based database connection. + * + * @throws ParameterException on errors. + */ + @Test + public void testPartialVAFile() { + ListParameterization spatparams = new ListParameterization(); + spatparams.addParameter(StaticArrayDatabase.INDEX_ID, PartialVAFile.Factory.class); + spatparams.addParameter(PartialVAFile.Factory.PARTITIONS_ID, 4); + testFileBasedDatabaseConnection(spatparams, PartialVAFile.PartialVAFileKNNQuery.class, PartialVAFile.PartialVAFileRangeQuery.class); + } + + /** * Test {@link RStarTree} using a file based database connection. With "fast" * mode enabled on an extreme level (since this should only reduce * performance, not accuracy!) @@ -195,8 +222,7 @@ public class TestIndexStructures implements JUnit4Test { // Verify distance assertEquals("Expected distance doesn't match.", shouldd[i], res.getDistance().doubleValue()); // verify vector - DBID id = res.getDBID(); - DoubleVector c = rep.get(id); + DoubleVector c = rep.get(res); DoubleVector c2 = new DoubleVector(shouldc[i]); assertEquals("Expected vector doesn't match: " + c.toString(), 0.0, dist.distance(c, c2).doubleValue(), 0.00001); @@ -217,8 +243,7 @@ public class TestIndexStructures implements JUnit4Test { // Verify distance assertEquals("Expected distance doesn't match.", shouldd[i], res.getDistance().doubleValue()); // verify vector - DBID id = res.getDBID(); - DoubleVector c = rep.get(id); + DoubleVector c = rep.get(res); DoubleVector c2 = new DoubleVector(shouldc[i]); assertEquals("Expected vector doesn't match: " + c.toString(), 0.0, dist.distance(c, c2).doubleValue(), 0.00001); diff --git a/test/de/lmu/ifi/dbs/elki/index/preprocessed/TestMaterializedKNNAndRKNNPreprocessor.java b/test/de/lmu/ifi/dbs/elki/index/preprocessed/TestMaterializedKNNAndRKNNPreprocessor.java index f2372d7a..0ef07a69 100644 --- a/test/de/lmu/ifi/dbs/elki/index/preprocessed/TestMaterializedKNNAndRKNNPreprocessor.java +++ b/test/de/lmu/ifi/dbs/elki/index/preprocessed/TestMaterializedKNNAndRKNNPreprocessor.java @@ -167,7 +167,7 @@ public class TestMaterializedKNNAndRKNNPreprocessor implements JUnit4Test { } assertEquals("kNN sizes do not agree.", lin_knn.size(), pre_knn.size()); for(int j = 0; j < lin_knn.size(); j++) { - assertTrue("kNNs of linear scan and preprocessor do not match!", lin_knn.get(j).getDBID().equals(pre_knn.get(j).getDBID())); + assertTrue("kNNs of linear scan and preprocessor do not match!", lin_knn.get(j).sameDBID(pre_knn.get(j))); assertTrue("kNNs of linear scan and preprocessor do not match!", lin_knn.get(j).getDistance().equals(pre_knn.get(j).getDistance())); } } @@ -188,7 +188,7 @@ public class TestMaterializedKNNAndRKNNPreprocessor implements JUnit4Test { } assertEquals("rkNN sizes do not agree for k=" + k, lin_rknn.size(), pre_rknn.size()); for(int j = 0; j < lin_rknn.size(); j++) { - assertTrue("rkNNs of linear scan and preprocessor do not match!", lin_rknn.get(j).getDBID().equals(pre_rknn.get(j).getDBID())); + assertTrue("rkNNs of linear scan and preprocessor do not match!", lin_rknn.get(j).sameDBID(pre_rknn.get(j))); assertTrue("rkNNs of linear scan and preprocessor do not match!", lin_rknn.get(j).getDistance().equals(pre_rknn.get(j).getDistance())); } } diff --git a/test/de/lmu/ifi/dbs/elki/math/TestKernelDensityFitting.java b/test/de/lmu/ifi/dbs/elki/math/TestKernelDensityFitting.java index c7ebfec2..56d4549d 100644 --- a/test/de/lmu/ifi/dbs/elki/math/TestKernelDensityFitting.java +++ b/test/de/lmu/ifi/dbs/elki/math/TestKernelDensityFitting.java @@ -35,6 +35,7 @@ import de.lmu.ifi.dbs.elki.data.type.TypeUtil; import de.lmu.ifi.dbs.elki.database.Database; import de.lmu.ifi.dbs.elki.database.StaticArrayDatabase; import de.lmu.ifi.dbs.elki.database.ids.DBID; +import de.lmu.ifi.dbs.elki.database.ids.DBIDIter; import de.lmu.ifi.dbs.elki.database.relation.Relation; import de.lmu.ifi.dbs.elki.datasource.FileBasedDatabaseConnection; import de.lmu.ifi.dbs.elki.math.linearalgebra.fitting.FittingFunction; @@ -92,7 +93,8 @@ public class TestKernelDensityFitting implements JUnit4Test { // transform into double array { int i = 0; - for(DBID id : rep.iterDBIDs()) { + for(DBIDIter iditer = rep.iterDBIDs(); iditer.valid(); iditer.advance()) { + DBID id = iditer.getDBID(); fulldata[i] = rep.get(id).doubleValue(1); i++; } diff --git a/test/de/lmu/ifi/dbs/elki/math/histograms/TestFlexiHistogram.java b/test/de/lmu/ifi/dbs/elki/math/histograms/TestFlexiHistogram.java index bcc7f330..7eafa270 100644 --- a/test/de/lmu/ifi/dbs/elki/math/histograms/TestFlexiHistogram.java +++ b/test/de/lmu/ifi/dbs/elki/math/histograms/TestFlexiHistogram.java @@ -26,10 +26,11 @@ package de.lmu.ifi.dbs.elki.math.histograms; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import java.util.Iterator; + import org.junit.Test; import de.lmu.ifi.dbs.elki.JUnit4Test; -import de.lmu.ifi.dbs.elki.utilities.iterator.IterableUtil; import de.lmu.ifi.dbs.elki.utilities.pairs.DoubleObjPair; /** @@ -75,7 +76,8 @@ public class TestFlexiHistogram implements JUnit4Test { } // backwards... off--; - for(DoubleObjPair<Double> pair : IterableUtil.fromIterator(hist.reverseIterator())) { + for(Iterator<DoubleObjPair<Double>> iter = hist.reverseIterator(); iter.hasNext(); ) { + DoubleObjPair<Double> pair = iter.next(); assertEquals("Array iterator bin position", -0.1 + 0.2 * off, pair.first, 0.00001); assertEquals("Array iterator bin contents", resized[off], pair.getSecond(), 0.00001); off--; diff --git a/test/de/lmu/ifi/dbs/elki/math/statistics/distribution/AbstractDistributionTest.java b/test/de/lmu/ifi/dbs/elki/math/statistics/distribution/AbstractDistributionTest.java new file mode 100644 index 00000000..fd975c15 --- /dev/null +++ b/test/de/lmu/ifi/dbs/elki/math/statistics/distribution/AbstractDistributionTest.java @@ -0,0 +1,77 @@ +package de.lmu.ifi.dbs.elki.math.statistics.distribution; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Abstract base class for distribution unit testing. + * + * @author Erich Schubert + */ +public class AbstractDistributionTest { + public void checkPDF(Distribution d, double[] x, double[] expected, double err) { + int maxerrlev = Integer.MIN_VALUE; + for(int i = 0; i < x.length; i++) { + double val = d.pdf(x[i]); + if(val == expected[i]) { + continue; + } + double diff = Math.abs(val - expected[i]); + final int errlev = (int) Math.ceil(Math.log10(diff / expected[i])); + maxerrlev = Math.max(errlev, maxerrlev); + if(diff < err || diff / expected[i] < err) { + continue; + } + assertEquals("Error magnitude: 1e" + errlev + " at " + x[i], expected[i], val, err); + } + int given = (int) Math.floor(Math.log10(err * 1.1)); + // if (given > maxerrlev) { + // System.err.println("PDF Error for "+d+" magnitude is not tight: expected "+maxerrlev+" got "+given); + // } + assertTrue("Error magnitude is not tight: expected " + maxerrlev + " got " + given, given <= maxerrlev); + } + + public void checkCDF(Distribution d, double[] x, double[] expected, double err) { + int maxerrlev = Integer.MIN_VALUE; + for(int i = 0; i < x.length; i++) { + double val = d.cdf(x[i]); + if(val == expected[i]) { + continue; + } + double diff = Math.abs(val - expected[i]); + final int errlev = (int) Math.ceil(Math.log10(diff / expected[i])); + maxerrlev = Math.max(errlev, maxerrlev); + if(diff < err || diff / expected[i] < err) { + continue; + } + assertEquals("Error magnitude: 1e" + errlev + " at " + x[i], expected[i], val, err); + } + int given = (int) Math.floor(Math.log10(err * 1.1)); + // if (given > maxerrlev) { + // System.err.println("CDF Error for "+d+" magnitude is not tight: expected "+maxerrlev+" got "+given); + // } + assertTrue("Error magnitude is not tight: expected " + maxerrlev + " got " + given, given <= maxerrlev); + } + + public void checkQuantile(Distribution d, double[] x, double[] expected, double err) { + int maxerrlev = Integer.MIN_VALUE; + for(int i = 0; i < x.length; i++) { + double val = d.quantile(x[i]); + if(val == expected[i]) { + continue; + } + double diff = Math.abs(val - expected[i]); + final int errlev = (int) Math.ceil(Math.log10(diff / expected[i])); + maxerrlev = Math.max(errlev, maxerrlev); + if(diff < err || diff / expected[i] < err) { + continue; + } + assertEquals("Error magnitude: 1e" + errlev + " at " + x[i], expected[i], val, err); + } + int given = (int) Math.floor(Math.log10(err * 1.1)); + // if (given > maxerrlev) { + // System.err.println("Probit Error for "+d+" magnitude is not tight: expected "+maxerrlev+" got "+given); + // } + assertTrue("Error magnitude is not tight: expected " + maxerrlev + " got " + given, given <= maxerrlev); + } +}
\ No newline at end of file diff --git a/test/de/lmu/ifi/dbs/elki/math/statistics/distribution/TestBetaDistribution.java b/test/de/lmu/ifi/dbs/elki/math/statistics/distribution/TestBetaDistribution.java new file mode 100644 index 00000000..f5efdce0 --- /dev/null +++ b/test/de/lmu/ifi/dbs/elki/math/statistics/distribution/TestBetaDistribution.java @@ -0,0 +1,3909 @@ +package de.lmu.ifi.dbs.elki.math.statistics.distribution; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +import org.junit.Test; + +import de.lmu.ifi.dbs.elki.JUnit4Test; + +/** + * Unit test for the Beta distribution in ELKI. + * + * The reference values were computed using GNU R and SciPy. + * + * @author Erich Schubert + */ +public class TestBetaDistribution extends AbstractDistributionTest implements JUnit4Test { + public static final double[] P_CDFPDF = { // + 1e-10, 1e-05, 0.1, 0.1234567, 0.2, 0.271828182846, 0.3, 0.314159265359, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0, 2.71828182846, 3.14159265359 // + }; + + public static final double[] SCIPY_BETA_CDF_01_01 = { // + 5.07237274393781481518495013460778864100575447082520e-02, // 0.000000 + 1.60402641359773445728365004470106214284896850585938e-01, // 0.000010 + 4.06385093936275976211902616341831162571907043457031e-01, // 0.100000 + 4.15927737016427878646851468147360719740390777587891e-01, // 0.123457 + 4.39709190223345591785886199431843124330043792724609e-01, // 0.200000 + 4.56830072616762117299060719233239069581031799316406e-01, // 0.271828 + 4.62804186115552074198831178364343941211700439453125e-01, // 0.300000 + 4.65695944978432807559443062928039580583572387695312e-01, // 0.314159 + 4.82120045609327974034385988488793373107910156250000e-01, // 0.400000 + 4.99999999999999888977697537484345957636833190917969e-01, // 0.500000 + 5.17879954390672025965614011511206626892089843750000e-01, // 0.600000 + 5.37195813884447925801168821635656058788299560546875e-01, // 0.700000 + 5.60290809776654241680660106794675812125205993652344e-01, // 0.800000 + 5.93614906063723468676585071079898625612258911132812e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_01_01 = { // + 5.07237274435283020138740539550781250000000000000000e+07, // 0.000000 + 1.60403953757008639513514935970306396484375000000000e+03, // 0.000010 + 4.42988958666523380269808285447652451694011688232422e-01, // 0.100000 + 3.75276980422167505402342158049577847123146057128906e-01, // 0.123457 + 2.63938762316794750528714530446450226008892059326172e-01, // 0.200000 + 2.17939347824759516170090023479133378714323043823242e-01, // 0.271828 + 2.06639715373252902397283037316810805350542068481445e-01, // 0.300000 + 2.01918135102394286084503960410074796527624130249023e-01, // 0.314159 + 1.83240320487424945827825695232604630291461944580078e-01, // 0.400000 + 1.76630277977873700878674867453810293227434158325195e-01, // 0.500000 + 1.83240320487424945827825695232604630291461944580078e-01, // 0.600000 + 2.06639715373252874641707421687897294759750366210938e-01, // 0.700000 + 2.63938762316794861551016992962104268372058868408203e-01, // 0.800000 + 4.42988958666523491292110747963306494057178497314453e-01, // 0.900000 + Double.POSITIVE_INFINITY, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_01_01 = { // + 5.07237274393781412129555974388495087623596191406250e-02, // 0.000000 + 1.60402641359773390217213773212279193103313446044922e-01, // 0.000010 + 4.06385093936275865189600153826177120208740234375000e-01, // 0.100000 + 4.15927737016427712113397774373879656195640563964844e-01, // 0.123457 + 4.39709190223345480763583736916189081966876983642578e-01, // 0.200000 + 4.56830072616782767447318747144890949130058288574219e-01, // 0.271828 + 4.62804186115551741131923790817381814122200012207031e-01, // 0.300000 + 4.65695944978436748851180482233758084475994110107422e-01, // 0.314159 + 4.82120045609327474434024907168350182473659515380859e-01, // 0.400000 + 4.99999999999999500399638918679556809365749359130859e-01, // 0.500000 + 5.17879954390672470054823861573822796344757080078125e-01, // 0.600000 + 5.37195813884448258868076209182618185877799987792969e-01, // 0.700000 + 5.60290809776654574747567494341637939214706420898438e-01, // 0.800000 + 5.93614906063724134810399846173822879791259765625000e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_01_01 = { // + 5.07237274435282945632934570312500000000000000000000e+07, // 0.000000 + 1.60403953757008684988250024616718292236328125000000e+03, // 0.000010 + 4.42988958666523380269808285447652451694011688232422e-01, // 0.100000 + 3.75276980422167782958098314338712953031063079833984e-01, // 0.123457 + 2.63938762316794861551016992962104268372058868408203e-01, // 0.200000 + 2.17939347824716384005583336147537920624017715454102e-01, // 0.271828 + 2.06639715373252930152858652945724315941333770751953e-01, // 0.300000 + 2.01918135102387819035385518873226828873157501220703e-01, // 0.314159 + 1.83240320487424973583401310861518140882253646850586e-01, // 0.400000 + 1.76630277977873756389826098711637314409017562866211e-01, // 0.500000 + 1.83240320487424973583401310861518140882253646850586e-01, // 0.600000 + 2.06639715373252930152858652945724315941333770751953e-01, // 0.700000 + 2.63938762316794917062168224219931289553642272949219e-01, // 0.800000 + 4.42988958666523602314413210478960536420345306396484e-01, // 0.900000 + Double.POSITIVE_INFINITY, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_01_05 = { // + 8.83151389893383070850418903319223318248987197875977e-02, // 0.000000 + 2.79277118023966042237304918671725317835807800292969e-01, // 0.000010 + 7.04833622051221020399225380970165133476257324218750e-01, // 0.100000 + 7.20681507439501545242421798320719972252845764160156e-01, // 0.123457 + 7.59304219445318007153389316954417154192924499511719e-01, // 0.200000 + 7.86085328767555213680395809205947443842887878417969e-01, // 0.271828 + 7.95176530367367906393383236718364059925079345703125e-01, // 0.300000 + 7.99524278991131209792797562840860337018966674804688e-01, // 0.314159 + 8.23494838536657169925092603079974651336669921875000e-01, // 0.400000 + 8.48001712399770513073349320620764046907424926757812e-01, // 0.500000 + 8.70603436980005307432861627603415399789810180664062e-01, // 0.600000 + 8.92658587836405614801549290859838947653770446777344e-01, // 0.700000 + 9.15640640375003589390701108641223981976509094238281e-01, // 0.800000 + 9.42366288315789946139489074994344264268875122070312e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_01_05 = { // + 8.83151389933526813983917236328125000000000000000000e+07, // 0.000000 + 2.79278387474808914703316986560821533203125000000000e+03, // 0.000010 + 7.39458664410644117559456844901433214545249938964844e-01, // 0.100000 + 6.19848096459823749704298734286567196249961853027344e-01, // 0.123457 + 4.20303655461324865427741315215826034545898437500000e-01, // 0.200000 + 3.34235972869352193637126902103773318231105804443359e-01, // 0.271828 + 3.11944266317603025040483544216840527951717376708984e-01, // 0.300000 + 3.02335144044583059574904382316162809729576110839844e-01, // 0.314159 + 2.60079151351645165846093732398003339767456054687500e-01, // 0.400000 + 2.33065049073819779978933297570620197802782058715820e-01, // 0.500000 + 2.21140881157234348020068637197255156934261322021484e-01, // 0.600000 + 2.22272786172716479979882819861813914030790328979492e-01, // 0.700000 + 2.41401058813832225524720342946238815784454345703125e-01, // 0.800000 + 3.07055512274369113434602240886306390166282653808594e-01, // 0.900000 + Double.POSITIVE_INFINITY, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_01_05 = { // + 8.83151389893382793294662747030088212341070175170898e-02, // 0.000000 + 2.79277118023966042237304918671725317835807800292969e-01, // 0.000010 + 7.04833622051220909376922918454511091113090515136719e-01, // 0.100000 + 7.20681507439501323197816873289411887526512145996094e-01, // 0.123457 + 7.59304219445317896131086854438763111829757690429688e-01, // 0.200000 + 7.86085328767586744014295163651695474982261657714844e-01, // 0.271828 + 7.95176530367367462304173386655747890472412109375000e-01, // 0.300000 + 7.99524278991137093974828076170524582266807556152344e-01, // 0.314159 + 8.23494838536657169925092603079974651336669921875000e-01, // 0.400000 + 8.48001712399770513073349320620764046907424926757812e-01, // 0.500000 + 8.70603436980005307432861627603415399789810180664062e-01, // 0.600000 + 8.92658587836405725823851753375492990016937255859375e-01, // 0.700000 + 9.15640640375003700413003571156878024339675903320312e-01, // 0.800000 + 9.42366288315790390228698925056960433721542358398438e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_01_05 = { // + 8.83151389933526664972305297851562500000000000000000e+07, // 0.000000 + 2.79278387474808823753846809267997741699218750000000e+03, // 0.000010 + 7.39458664410643895514851919870125129818916320800781e-01, // 0.100000 + 6.19848096459823971748903659317875280976295471191406e-01, // 0.123457 + 4.20303655461324865427741315215826034545898437500000e-01, // 0.200000 + 3.34235972869268427309918934042798355221748352050781e-01, // 0.271828 + 3.11944266317603025040483544216840527951717376708984e-01, // 0.300000 + 3.02335144044569681387457649179850704967975616455078e-01, // 0.314159 + 2.60079151351645221357244963655830360949039459228516e-01, // 0.400000 + 2.33065049073819752223357681941706687211990356445312e-01, // 0.500000 + 2.21140881157234375775644252826168667525053024291992e-01, // 0.600000 + 2.22272786172716479979882819861813914030790328979492e-01, // 0.700000 + 2.41401058813832253280295958575152326375246047973633e-01, // 0.800000 + 3.07055512274369168945753472144133411347866058349609e-01, // 0.900000 + Double.POSITIVE_INFINITY, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_01_1 = { // + 1.00000000000000019428902930940239457413554191589355e-01, // 0.000000 + 3.16227766016837996687627310166135430335998535156250e-01, // 0.000010 + 7.94328234724281712075821815233211964368820190429688e-01, // 0.100000 + 8.11243941442105320760447284555993974208831787109375e-01, // 0.123457 + 8.51339922520784830872742077190196141600608825683594e-01, // 0.200000 + 8.77868464423642858029950275522423908114433288574219e-01, // 0.271828 + 8.86568150565213541902664928784361109137535095214844e-01, // 0.300000 + 8.90666232270154401007289379776921123266220092773438e-01, // 0.314159 + 9.12443536555481049532545512192882597446441650390625e-01, // 0.400000 + 9.33032991536807632115824162610806524753570556640625e-01, // 0.500000 + 9.50200216505676653611089932383038103580474853515625e-01, // 0.600000 + 9.64961095119817868770439872605493292212486267089844e-01, // 0.700000 + 9.77932768542928765320709771913243457674980163574219e-01, // 0.800000 + 9.89519258206214669115752258221618831157684326171875e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_01_1 = { // + 1.00000000000000059604644775390625000000000000000000e+08, // 0.000000 + 3.16227766016838040741276927292346954345703125000000e+03, // 0.000010 + 7.94328234724281601053519352717557922005653381347656e-01, // 0.100000 + 6.57108072256997965965297225920949131250381469726562e-01, // 0.123457 + 4.25669961260392415436371038595098070800304412841797e-01, // 0.200000 + 3.22949760114201955829571488720830529928207397460938e-01, // 0.271828 + 2.95522716855071199137938720014062710106372833251953e-01, // 0.300000 + 2.83507867021658532813432884722715243697166442871094e-01, // 0.314159 + 2.28110884138870234627560762419307138770818710327148e-01, // 0.400000 + 1.86606598307361509769819463144813198596239089965820e-01, // 0.500000 + 1.58366702750946081179606039768259506672620773315430e-01, // 0.600000 + 1.37851585017116834430694893853797111660242080688477e-01, // 0.700000 + 1.22241596067866081787300913674698676913976669311523e-01, // 0.800000 + 1.09946584245134940105437237889418611302971839904785e-01, // 0.900000 + 1.00000000000000019428902930940239457413554191589355e-01, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_01_1 = { // + 9.99999999999999777955395074968691915273666381835938e-02, // 0.000000 + 3.16227766016837885665324847650481387972831726074219e-01, // 0.000010 + 7.94328234724281379008914427686249837279319763183594e-01, // 0.100000 + 8.11243941442104987693539897009031847119331359863281e-01, // 0.123457 + 8.51339922520784497805834689643234014511108398437500e-01, // 0.200000 + 8.77868464423673389163127467327285557985305786132812e-01, // 0.271828 + 8.86568150565213208835757541237398982048034667968750e-01, // 0.300000 + 8.90666232270159952122412505559623241424560546875000e-01, // 0.314159 + 9.12443536555480938510243049677228555083274841308594e-01, // 0.400000 + 9.33032991536807410071219237579498440027236938476562e-01, // 0.500000 + 9.50200216505676431566485007351730018854141235351562e-01, // 0.600000 + 9.64961095119817535703532485058531165122985839843750e-01, // 0.700000 + 9.77932768542928543276104846881935372948646545410156e-01, // 0.800000 + 9.89519258206214447071147333190310746431350708007812e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_01_1 = { // + 9.99999999999998211860656738281250000000000000000000e+07, // 0.000000 + 3.16227766016838040741276927292346954345703125000000e+03, // 0.000010 + 7.94328234724281601053519352717557922005653381347656e-01, // 0.100000 + 6.57108072256998076987599688436603173613548278808594e-01, // 0.123457 + 4.25669961260392415436371038595098070800304412841797e-01, // 0.200000 + 3.22949760114099870822457205576938576996326446533203e-01, // 0.271828 + 2.95522716855071199137938720014062710106372833251953e-01, // 0.300000 + 2.83507867021641657423458582343300804495811462402344e-01, // 0.314159 + 2.28110884138870234627560762419307138770818710327148e-01, // 0.400000 + 1.86606598307361537525395078773726709187030792236328e-01, // 0.500000 + 1.58366702750946108935181655397173017263412475585938e-01, // 0.600000 + 1.37851585017116834430694893853797111660242080688477e-01, // 0.700000 + 1.22241596067866095665088721489155432209372520446777e-01, // 0.800000 + 1.09946584245134940105437237889418611302971839904785e-01, // 0.900000 + 1.00000000000000005551115123125782702118158340454102e-01, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_01_2 = { // + 1.09999999998999994921256018187705194577574729919434e-01, // 0.000000 + 3.47850226390755745686789168757968582212924957275391e-01, // 0.000010 + 8.65817775849466841897594804322579875588417053222656e-01, // 0.100000 + 8.82352985595772132398906251182779669761657714843750e-01, // 0.123457 + 9.19447116322447466352230094344122335314750671386719e-01, // 0.200000 + 9.41792371919806692304177886398974806070327758789062e-01, // 0.271828 + 9.48627921104778337735297100152820348739624023437500e-01, // 0.300000 + 9.51751750576165678197071429167408496141433715820312e-01, // 0.314159 + 9.67190148748809930268066636926960200071334838867188e-01, // 0.400000 + 9.79684641113647836085931430716300383210182189941406e-01, // 0.500000 + 9.88208225165903453302007619640789926052093505859375e-01, // 0.600000 + 9.93909927973412155033372528123436495661735534667969e-01, // 0.700000 + 9.97491423913787089716720402066130191087722778320312e-01, // 0.800000 + 9.99414450788276509385354984260629862546920776367188e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_01_2 = { // + 1.09999999989000067114830017089843750000000000000000e+08, // 0.000000 + 3.47847064113095666471053846180438995361328125000000e+03, // 0.000010 + 7.86384952377038759507854592811781913042068481445312e-01, // 0.100000 + 6.33582045924066195041746141214389353990554809570312e-01, // 0.123457 + 3.74589565909145327804452563213999383151531219482422e-01, // 0.200000 + 2.58679205039021475887039969165925867855548858642578e-01, // 0.271828 + 2.27552491978404769490396120090736076235771179199219e-01, // 0.300000 + 2.13885368174107387551075021292490419000387191772461e-01, // 0.314159 + 1.50553183531654338755956246131972875446081161499023e-01, // 0.400000 + 1.02633629069048826210064362385310232639312744140625e-01, // 0.500000 + 6.96813492104162851559223668118647765368223190307617e-02, // 0.600000 + 4.54910230556485536967947780340182362124323844909668e-02, // 0.700000 + 2.68931511349305325808689559607955743558704853057861e-02, // 0.800000 + 1.20941242669648403584847784486555610783398151397705e-02, // 0.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_01_2 = { // + 1.09999999998999994921256018187705194577574729919434e-01, // 0.000000 + 3.47850226390755745686789168757968582212924957275391e-01, // 0.000010 + 8.65817775849466841897594804322579875588417053222656e-01, // 0.100000 + 8.82352985595771910354301326151471585035324096679688e-01, // 0.123457 + 9.19447116322447355329927631828468292951583862304688e-01, // 0.200000 + 9.41792371919831339255324564874172210693359375000000e-01, // 0.271828 + 9.48627921104778337735297100152820348739624023437500e-01, // 0.300000 + 9.51751750576169897044565004762262105941772460937500e-01, // 0.314159 + 9.67190148748809708223461711895652115345001220703125e-01, // 0.400000 + 9.79684641113647725063628968200646340847015380859375e-01, // 0.500000 + 9.88208225165903453302007619640789926052093505859375e-01, // 0.600000 + 9.93909927973412155033372528123436495661735534667969e-01, // 0.700000 + 9.97491423913787089716720402066130191087722778320312e-01, // 0.800000 + 9.99414450788276509385354984260629862546920776367188e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_01_2 = { // + 1.09999999989000082015991210937500000000000000000000e+08, // 0.000000 + 3.47847064113095257198438048362731933593750000000000e+03, // 0.000010 + 7.86384952377038648485552130296127870678901672363281e-01, // 0.100000 + 6.33582045924066306064048603730043396353721618652344e-01, // 0.123457 + 3.74589565909145272293301331956172361969947814941406e-01, // 0.200000 + 2.58679205038905735136722796596586704254150390625000e-01, // 0.271828 + 2.27552491978404825001547351348563097417354583740234e-01, // 0.300000 + 2.13885368174088180692749006084341090172529220581055e-01, // 0.314159 + 1.50553183531654338755956246131972875446081161499023e-01, // 0.400000 + 1.02633629069048826210064362385310232639312744140625e-01, // 0.500000 + 6.96813492104162851559223668118647765368223190307617e-02, // 0.600000 + 4.54910230556485675745825858484749915078282356262207e-02, // 0.700000 + 2.68931511349305360503159079144097631797194480895996e-02, // 0.800000 + 1.20941242669648386237613024718484666664153337478638e-02, // 0.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_01_4 = { // + 1.19349999996744948749238801610772497951984405517578e-01, // 0.000000 + 3.77416809425109245346163788781268522143363952636719e-01, // 0.000010 + 9.23499112106322184168050171138020232319831848144531e-01, // 0.100000 + 9.37669077635512548951624012261163443326950073242188e-01, // 0.123457 + 9.66195894147908407312286271917400881648063659667969e-01, // 0.200000 + 9.80442982290223752350755148654570803046226501464844e-01, // 0.271828 + 9.84228508474799701488677783345337957143783569335938e-01, // 0.300000 + 9.85856268655784573340383758477400988340377807617188e-01, // 0.314159 + 9.92844411222603584477042204525787383317947387695312e-01, // 0.400000 + 9.97004066019049783875516368425451219081878662109375e-01, // 0.500000 + 9.98911280404623402873198756424244493246078491210938e-01, // 0.600000 + 9.99689562452632296540855350031051784753799438476562e-01, // 0.700000 + 9.99944079297292787167350525123765692114830017089844e-01, // 0.800000 + 9.99996782871730882646943427971564233303070068359375e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_01_4 = { // + 1.19349999964195013046264648437500000000000000000000e+08, // 0.000000 + 3.77406516319158845362835563719272613525390625000000e+03, // 0.000010 + 6.91114415396560266557912655116524547338485717773438e-01, // 0.100000 + 5.28176834178893939153454084589611738920211791992188e-01, // 0.123457 + 2.60114994567310409134819337850785814225673675537109e-01, // 0.200000 + 1.48819207734607078519317724385473411530256271362305e-01, // 0.271828 + 1.20978282360318822874489796959096565842628479003906e-01, // 0.300000 + 1.09158451358456268232011154850624734535813331604004e-01, // 0.314159 + 5.88060734874641527492045156577660236507654190063477e-02, // 0.400000 + 2.78393718849794802316921504825586453080177307128906e-02, // 0.500000 + 1.20966822229282636169678255555481882765889167785645e-02, // 0.600000 + 4.44219840138408098711986227158377005252987146377563e-03, // 0.700000 + 1.16716275925598405617533703804156175465323030948639e-03, // 0.800000 + 1.31221248296568387655194887209120224724756553769112e-04, // 0.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_01_4 = { // + 1.19349999996745004260390032868599519133567810058594e-01, // 0.000000 + 3.77416809425109411879617482554749585688114166259766e-01, // 0.000010 + 9.23499112106322628257260021200636401772499084472656e-01, // 0.100000 + 9.37669077635512770996228937292471528053283691406250e-01, // 0.123457 + 9.66195894147908740379193659464363008737564086914062e-01, // 0.200000 + 9.80442982290238074227772813173942267894744873046875e-01, // 0.271828 + 9.84228508474799701488677783345337957143783569335938e-01, // 0.300000 + 9.85856268655786793786433008790481835603713989257812e-01, // 0.314159 + 9.92844411222603584477042204525787383317947387695312e-01, // 0.400000 + 9.97004066019049783875516368425451219081878662109375e-01, // 0.500000 + 9.98911280404623402873198756424244493246078491210938e-01, // 0.600000 + 9.99689562452632296540855350031051784753799438476562e-01, // 0.700000 + 9.99944079297292676145048062608111649751663208007812e-01, // 0.800000 + 9.99996782871730882646943427971564233303070068359375e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_01_4 = { // + 1.19349999964195147156715393066406250000000000000000e+08, // 0.000000 + 3.77406516319158572514425031840801239013671875000000e+03, // 0.000010 + 6.91114415396560155535610192600870504975318908691406e-01, // 0.100000 + 5.28176834178893939153454084589611738920211791992188e-01, // 0.123457 + 2.60114994567310353623668106592958793044090270996094e-01, // 0.200000 + 1.48819207734501468554100256369565613567829132080078e-01, // 0.271828 + 1.20978282360318920019004451660293852910399436950684e-01, // 0.300000 + 1.09158451358439906320185741606110241264104843139648e-01, // 0.314159 + 5.88060734874641596880984195649944012984633445739746e-02, // 0.400000 + 2.78393718849794941094799582970154006034135818481445e-02, // 0.500000 + 1.20966822229282670864147775091623771004378795623779e-02, // 0.600000 + 4.44219840138408185448160025998731725849211215972900e-03, // 0.700000 + 1.16716275925598514037750952354599576210603117942810e-03, // 0.800000 + 1.31221248296568414760249199346731074911076575517654e-04, // 0.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_05_01 = { // + 1.76630277983172680022875850741614911498800211120397e-06, // 0.000000 + 5.58555657830258874343987152144563879119232296943665e-04, // 0.000010 + 5.76337116842095750768315554068976780399680137634277e-02, // 0.100000 + 6.45370774761398202645779065278475172817707061767578e-02, // 0.123457 + 8.43593596249963134647842366575787309557199478149414e-02, // 0.200000 + 1.01039482858121226760417243895062711089849472045898e-01, // 0.271828 + 1.07341412163594357442875093511247541755437850952148e-01, // 0.300000 + 1.10480921737836620954986699416622286662459373474121e-01, // 0.314159 + 1.29396563019994831345016450541152153164148330688477e-01, // 0.400000 + 1.51998287600229570193377526265976484864950180053711e-01, // 0.500000 + 1.76505161463342941097209859435679391026496887207031e-01, // 0.600000 + 2.04823469632632232384494841426203493028879165649414e-01, // 0.700000 + 2.40695780554681937335459451787755824625492095947266e-01, // 0.800000 + 2.95166377948778924089623387772007845342159271240234e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_05_01 = { // + 8.83151389968852345191407948732376098632812500000000e+03, // 0.000000 + 2.79279504596178647091164748417213559150695800781250e+01, // 0.000010 + 3.07055512274369002412299778370652347803115844726562e-01, // 0.100000 + 2.82996971327952206998190831654937937855720520019531e-01, // 0.123457 + 2.41401058813832225524720342946238815784454345703125e-01, // 0.200000 + 2.25360312523454175970272217455203644931316375732422e-01, // 0.271828 + 2.22272786172716479979882819861813914030790328979492e-01, // 0.300000 + 2.21237769900911845910584929697506595402956008911133e-01, // 0.314159 + 2.21140881157234348020068637197255156934261322021484e-01, // 0.400000 + 2.33065049073819779978933297570620197802782058715820e-01, // 0.500000 + 2.60079151351645165846093732398003339767456054687500e-01, // 0.600000 + 3.11944266317603025040483544216840527951717376708984e-01, // 0.700000 + 4.20303655461324976450043777731480076909065246582031e-01, // 0.800000 + 7.39458664410644339604061769932741299271583557128906e-01, // 0.900000 + Double.POSITIVE_INFINITY, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_05_01 = { // + 1.76630277983172658847052169384106434790737694129348e-06, // 0.000000 + 5.58555657830258765923769903594120478373952209949493e-04, // 0.000010 + 5.76337116842095611990437475924409227445721626281738e-02, // 0.100000 + 6.45370774761398063867900987133907619863748550415039e-02, // 0.123457 + 8.43593596249962857092086210286652203649282455444336e-02, // 0.200000 + 1.01039482858142709575943740674119908362627029418945e-01, // 0.271828 + 1.07341412163594315809511670067877275869250297546387e-01, // 0.300000 + 1.10480921737841158991599854743981268256902694702148e-01, // 0.314159 + 1.29396563019994748078289603654411621391773223876953e-01, // 0.400000 + 1.51998287600229459171075063750322442501783370971680e-01, // 0.500000 + 1.76505161463342774563756165662198327481746673583984e-01, // 0.600000 + 2.04823469632632537695826613344252109527587890625000e-01, // 0.700000 + 2.40695780554682103868913145561236888170242309570312e-01, // 0.800000 + 2.95166377948779090623077081545488908886909484863281e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_05_01 = { // + 8.83151389968851799494586884975433349609375000000000e+03, // 0.000000 + 2.79279504596178611564027960412204265594482421875000e+01, // 0.000010 + 3.07055512274369057923451009628479368984699249267578e-01, // 0.100000 + 2.82996971327952262509342062912764959037303924560547e-01, // 0.123457 + 2.41401058813832197769144727317325305193662643432617e-01, // 0.200000 + 2.25360312523441158605308487494767177850008010864258e-01, // 0.271828 + 2.22272786172716479979882819861813914030790328979492e-01, // 0.300000 + 2.21237769900910513642955379509658087044954299926758e-01, // 0.314159 + 2.21140881157234375775644252826168667525053024291992e-01, // 0.400000 + 2.33065049073819752223357681941706687211990356445312e-01, // 0.500000 + 2.60079151351645221357244963655830360949039459228516e-01, // 0.600000 + 3.11944266317603025040483544216840527951717376708984e-01, // 0.700000 + 4.20303655461324976450043777731480076909065246582031e-01, // 0.800000 + 7.39458664410644228581759307417087256908416748046875e-01, // 0.900000 + Double.POSITIVE_INFINITY, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_05_05 = { // + 6.36619772378191858851811252284491615682782139629126e-06, // 0.000000 + 2.01317183947538743890581791617933049565181136131287e-03, // 0.000010 + 2.04832764699133501595795792127319145947694778442383e-01, // 0.100000 + 2.28564106667586530718949688889551907777786254882812e-01, // 0.123457 + 2.95167235300866637182082286017248407006263732910156e-01, // 0.200000 + 3.49158781117712757602333795148297213017940521240234e-01, // 0.271828 + 3.69010119565545413600204938120441511273384094238281e-01, // 0.300000 + 3.78781359553035823495292788720689713954925537109375e-01, // 0.314159 + 4.35905783151025183475013591305469162762165069580078e-01, // 0.400000 + 5.00000000000000111022302462515654042363166809082031e-01, // 0.500000 + 5.64094216848975094080742564983665943145751953125000e-01, // 0.600000 + 6.30989880434454586399795061879558488726615905761719e-01, // 0.700000 + 7.04832764699133362817917713982751592993736267089844e-01, // 0.800000 + 7.95167235300866193092872435954632237553596496582031e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_05_05 = { // + 3.18309886199706197658088058233261108398437500000000e+04, // 0.000000 + 1.00658927504869836866419063881039619445800781250000e+02, // 0.000010 + 1.06103295394596908174378313560737296938896179199219e+00, // 0.100000 + 9.67622833349070621267173919477500021457672119140625e-01, // 0.123457 + 7.95774715459476977841291045479010790586471557617188e-01, // 0.200000 + 7.15460980314251537848235784622374922037124633789062e-01, // 0.271828 + 6.94609118042856854557953738549258559942245483398438e-01, // 0.300000 + 6.85746426302663469876108592870878055691719055175781e-01, // 0.314159 + 6.49747334361396755575412953476188704371452331542969e-01, // 0.400000 + 6.36619772367581604477493328886339440941810607910156e-01, // 0.500000 + 6.49747334361396755575412953476188704371452331542969e-01, // 0.600000 + 6.94609118042856854557953738549258559942245483398438e-01, // 0.700000 + 7.95774715459476977841291045479010790586471557617188e-01, // 0.800000 + 1.06103295394596930378838806063868105411529541015625e+00, // 0.900000 + Double.POSITIVE_INFINITY, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_05_05 = { // + 6.36619772378191689445221801424423802018282003700733e-06, // 0.000000 + 2.01317183947538700522494892197755689267069101333618e-03, // 0.000010 + 2.04832764699133446084644560869492124766111373901367e-01, // 0.100000 + 2.28564106667586419696647226373897865414619445800781e-01, // 0.123457 + 2.95167235300866415137477360985940322279930114746094e-01, // 0.200000 + 3.49158781117780925296045779759879224002361297607422e-01, // 0.271828 + 3.69010119565545247066751244346960447728633880615234e-01, // 0.300000 + 3.78781359553049756794251834435272030532360076904297e-01, // 0.314159 + 4.35905783151024850408106203758507035672664642333984e-01, // 0.400000 + 4.99999999999999555910790149937383830547332763671875e-01, // 0.500000 + 5.64094216848975094080742564983665943145751953125000e-01, // 0.600000 + 6.30989880434454697422097524395212531089782714843750e-01, // 0.700000 + 7.04832764699133584862522639014059677720069885253906e-01, // 0.800000 + 7.95167235300866637182082286017248407006263732910156e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_05_05 = { // + 3.18309886199705724720843136310577392578125000000000e+04, // 0.000000 + 1.00658927504869879498983209487050771713256835937500e+02, // 0.000010 + 1.06103295394596863765457328554475679993629455566406e+00, // 0.100000 + 9.67622833349070399222568994446191936731338500976562e-01, // 0.123457 + 7.95774715459476422729778732900740578770637512207031e-01, // 0.200000 + 7.15460980314172489968882473476696759462356567382812e-01, // 0.271828 + 6.94609118042856410468743888486642390489578247070312e-01, // 0.300000 + 6.85746426302651035378232791117625311017036437988281e-01, // 0.314159 + 6.49747334361396533530808028444880619645118713378906e-01, // 0.400000 + 6.36619772367581160388283478823723271489143371582031e-01, // 0.500000 + 6.49747334361396533530808028444880619645118713378906e-01, // 0.600000 + 6.94609118042856410468743888486642390489578247070312e-01, // 0.700000 + 7.95774715459476533752081195416394621133804321289062e-01, // 0.800000 + 1.06103295394596885969917821057606488466262817382812e+00, // 0.900000 + Double.POSITIVE_INFINITY, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_05_1 = { // + 1.00000000000000008180305391403130954586231382563710e-05, // 0.000000 + 3.16227766016837939441752602931501314742490649223328e-03, // 0.000010 + 3.16227766016837941176476078908308409154415130615234e-01, // 0.100000 + 3.51364056215202502642114268383011221885681152343750e-01, // 0.123457 + 4.47213595499957927703604809721582569181919097900391e-01, // 0.200000 + 5.21371444217943791343827797390986233949661254882812e-01, // 0.271828 + 5.47722557505166074420799304789397865533828735351562e-01, // 0.300000 + 5.60499121639792874383090293122222647070884704589844e-01, // 0.314159 + 6.32455532033675882352952157816616818308830261230469e-01, // 0.400000 + 7.07106781186547572737310929369414225220680236816406e-01, // 0.500000 + 7.74596669241483404277914814883843064308166503906250e-01, // 0.600000 + 8.36660026534075562665293546160683035850524902343750e-01, // 0.700000 + 8.94427190999915855407209619443165138363838195800781e-01, // 0.800000 + 9.48683298050513768018277005467098206281661987304688e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_05_1 = { // + 5.00000000000000072759576141834259033203125000000000e+04, // 0.000000 + 1.58113883008418980580245261080563068389892578125000e+02, // 0.000010 + 1.58113883008418976139353162579936906695365905761719e+00, // 0.100000 + 1.42302546648016092767363716120598837733268737792969e+00, // 0.123457 + 1.11803398874989512457034379622200503945350646972656e+00, // 0.200000 + 9.59009177708225135638997471687616780400276184082031e-01, // 0.271828 + 9.12870929175277012745937099680304527282714843750000e-01, // 0.300000 + 8.92062058076385722138468281627865508198738098144531e-01, // 0.314159 + 7.90569415042094880696765812899684533476829528808594e-01, // 0.400000 + 7.07106781186547683759613391885068267583847045898438e-01, // 0.500000 + 6.45497224367902910913130654080305248498916625976562e-01, // 0.600000 + 5.97614304667197004938827831210801377892494201660156e-01, // 0.700000 + 5.59016994374947562285171898111002519726753234863281e-01, // 0.800000 + 5.27046276694730031486813004448777064681053161621094e-01, // 0.900000 + 5.00000000000000111022302462515654042363166809082031e-01, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_05_1 = { // + 1.00000000000000008180305391403130954586231382563710e-05, // 0.000000 + 3.16227766016837939441752602931501314742490649223328e-03, // 0.000010 + 3.16227766016837941176476078908308409154415130615234e-01, // 0.100000 + 3.51364056215202502642114268383011221885681152343750e-01, // 0.123457 + 4.47213595499957927703604809721582569181919097900391e-01, // 0.200000 + 5.21371444218035384743359372805571183562278747558594e-01, // 0.271828 + 5.47722557505166074420799304789397865533828735351562e-01, // 0.300000 + 5.60499121639811304085299070720793679356575012207031e-01, // 0.314159 + 6.32455532033675882352952157816616818308830261230469e-01, // 0.400000 + 7.07106781186547572737310929369414225220680236816406e-01, // 0.500000 + 7.74596669241483404277914814883843064308166503906250e-01, // 0.600000 + 8.36660026534075562665293546160683035850524902343750e-01, // 0.700000 + 8.94427190999915966429512081958819180727005004882812e-01, // 0.800000 + 9.48683298050513768018277005467098206281661987304688e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_05_1 = { // + 5.00000000000000072759576141834259033203125000000000e+04, // 0.000000 + 1.58113883008418980580245261080563068389892578125000e+02, // 0.000010 + 1.58113883008418953934892670076806098222732543945312e+00, // 0.100000 + 1.42302546648016092767363716120598837733268737792969e+00, // 0.123457 + 1.11803398874989490252573887119069695472717285156250e+00, // 0.200000 + 9.59009177708056492761556910409126430749893188476562e-01, // 0.271828 + 9.12870929175276901723634637164650484919548034667969e-01, // 0.300000 + 8.92062058076356190206013252463890239596366882324219e-01, // 0.314159 + 7.90569415042094769674463350384030491113662719726562e-01, // 0.400000 + 7.07106781186547572737310929369414225220680236816406e-01, // 0.500000 + 6.45497224367902799890828191564651206135749816894531e-01, // 0.600000 + 5.97614304667196893916525368695147335529327392578125e-01, // 0.700000 + 5.59016994374947451262869435595348477363586425781250e-01, // 0.800000 + 5.27046276694729920464510541933123022317886352539062e-01, // 0.900000 + 5.00000000000000000000000000000000000000000000000000e-01, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_05_2 = { // + 1.49999999995000036075122260004199858940410194918513e-05, // 0.000000 + 4.74340067886426866583526873455411987379193305969238e-03, // 0.000010 + 4.58530260724415050788138614734634757041931152343750e-01, // 0.100000 + 5.05356960883332151368563245341647416353225708007812e-01, // 0.123457 + 6.26099033699941132091737472364911809563636779785156e-01, // 0.200000 + 7.11195440192161520975844268832588568329811096191406e-01, // 0.271828 + 7.39425452631974344797072262736037373542785644531250e-01, // 0.300000 + 7.52705686315334188130066195299150422215461730957031e-01, // 0.314159 + 8.22192191643778769183370513928821310400962829589844e-01, // 0.400000 + 8.83883476483184660210667971114162355661392211914062e-01, // 0.500000 + 9.29516003089779996315655807848088443279266357421875e-01, // 0.600000 + 9.62159030514186830451706100575393065810203552246094e-01, // 0.700000 + 9.83869910099907429845700335135916247963905334472656e-01, // 0.800000 + 9.96117462953039489725881594495149329304695129394531e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_05_2 = { // + 7.49999999925000010989606380462646484375000000000000e+04, // 0.000000 + 2.37168452804383292686907225288450717926025390625000e+02, // 0.000010 + 2.13453742061365581150766956852748990058898925781250e+00, // 0.100000 + 1.87101515755883918146196265297476202249526977539062e+00, // 0.123457 + 1.34164078649987406066657058545388281345367431640625e+00, // 0.200000 + 1.04748518339887985995062535948818549513816833496094e+00, // 0.271828 + 9.58514475634040685747550014639273285865783691406250e-01, // 0.300000 + 9.17718745884733677620204161939909681677818298339844e-01, // 0.314159 + 7.11512473537885270502556522842496633529663085937500e-01, // 0.400000 + 5.30330085889910707308558812655974179506301879882812e-01, // 0.500000 + 3.87298334620741757650108638699748553335666656494141e-01, // 0.600000 + 2.68926437100238668875817893422208726406097412109375e-01, // 0.700000 + 1.67705098312484202072170091923908330500125885009766e-01, // 0.800000 + 7.90569415042094714163312119126203469932079315185547e-02, // 0.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_05_2 = { // + 1.49999999994999815846555973886111701176560018211603e-05, // 0.000000 + 4.74340067886426779847353074615057266782969236373901e-03, // 0.000010 + 4.58530260724415050788138614734634757041931152343750e-01, // 0.100000 + 5.05356960883331929323958320310339331626892089843750e-01, // 0.123457 + 6.26099033699941021069435009849257767200469970703125e-01, // 0.200000 + 7.11195440192261552070362995436880737543106079101562e-01, // 0.271828 + 7.39425452631974677863979650282999500632286071777344e-01, // 0.300000 + 7.52705686315353283966089747991645708680152893066406e-01, // 0.314159 + 8.22192191643778880205672976444475352764129638671875e-01, // 0.400000 + 8.83883476483184438166063046082854270935058593750000e-01, // 0.500000 + 9.29516003089779996315655807848088443279266357421875e-01, // 0.600000 + 9.62159030514186941474008563091047108173370361328125e-01, // 0.700000 + 9.83869910099907540868002797651570290327072143554688e-01, // 0.800000 + 9.96117462953039489725881594495149329304695129394531e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_05_2 = { // + 7.49999999924999574432149529457092285156250000000000e+04, // 0.000000 + 2.37168452804383349530326086096465587615966796875000e+02, // 0.000010 + 2.13453742061365536741845971846487373113632202148438e+00, // 0.100000 + 1.87101515755883918146196265297476202249526977539062e+00, // 0.123457 + 1.34164078649987361657736073539126664400100708007812e+00, // 0.200000 + 1.04748518339855811731808898912277072668075561523438e+00, // 0.271828 + 9.58514475634040685747550014639273285865783691406250e-01, // 0.300000 + 9.17718745884675612956016266252845525741577148437500e-01, // 0.314159 + 7.11512473537885270502556522842496633529663085937500e-01, // 0.400000 + 5.30330085889910596286256350140320137143135070800781e-01, // 0.500000 + 3.87298334620741702138957407441921532154083251953125e-01, // 0.600000 + 2.68926437100238557853515430906554684042930603027344e-01, // 0.700000 + 1.67705098312484174316594476294994819909334182739258e-01, // 0.800000 + 7.90569415042094297829677884692500811070203781127930e-02, // 0.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_05_4 = { // + 2.18749999978125033101364338428140854375669732689857e-05, // 0.000000 + 6.91741320720956186923755737439023505430668592453003e-03, // 0.000010 + 6.26625082597740412637676854501478374004364013671875e-01, // 0.100000 + 6.80541225384169945478163299412699416279792785644531e-01, // 0.123457 + 8.04984471899924280968718903750414028763771057128906e-01, // 0.200000 + 8.77770786854125240061819113179808482527732849121094e-01, // 0.271828 + 8.98778484206133598810595231043407693505287170410156e-01, // 0.300000 + 9.08079033661920775877263167785713449120521545410156e-01, // 0.314159 + 9.50264436880598029055988718027947470545768737792969e-01, // 0.400000 + 9.77796095859522762516746752226026728749275207519531e-01, // 0.500000 + 9.91483736629098721948594175046309828758239746093750e-01, // 0.600000 + 9.97455625383593225130596238159341737627983093261719e-01, // 0.700000 + 9.99522385942405944270205964130582287907600402832031e-01, // 0.800000 + 9.99971488851369660899592872738139703869819641113281e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_05_4 = { // + 1.09374999967187526635825634002685546875000000000000e+05, // 0.000000 + 3.45863742961106026996276341378688812255859375000000e+02, // 0.000010 + 2.52142232809988131592149329662788659334182739257812e+00, // 0.100000 + 2.09643237315321817604285570268984884023666381835938e+00, // 0.123457 + 1.25219806739988270827268479479243978857994079589844e+00, // 0.200000 + 8.09976508987270138462122304190415889024734497070312e-01, // 0.271828 + 6.84938469046824982555676797346677631139755249023438e-01, // 0.300000 + 6.29524964821042476437185086979297921061515808105469e-01, // 0.314159 + 3.73544048607389767013842174492310732603073120117188e-01, // 0.400000 + 1.93349510480696623604401906959537882357835769653320e-01, // 0.500000 + 9.03696114115064286220757594492170028388500213623047e-02, // 0.600000 + 3.52965948694063461066328102333500282838940620422363e-02, // 0.700000 + 9.78279740156157671948644605208755820058286190032959e-03, // 0.800000 + 1.15291373026972119794630877720464923186227679252625e-03, // 0.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_05_4 = { // + 2.18749999978125100864000118772167979841469787061214e-05, // 0.000000 + 6.91741320720955839979060542077604623045772314071655e-03, // 0.000010 + 6.26625082597740301615374391985824331641197204589844e-01, // 0.100000 + 6.80541225384170056500465761928353458642959594726562e-01, // 0.123457 + 8.04984471899924280968718903750414028763771057128906e-01, // 0.200000 + 8.77770786854202622606635486590676009654998779296875e-01, // 0.271828 + 8.98778484206133709832897693559061735868453979492188e-01, // 0.300000 + 9.08079033661933876508953744632890447974205017089844e-01, // 0.314159 + 9.50264436880598029055988718027947470545768737792969e-01, // 0.400000 + 9.77796095859522762516746752226026728749275207519531e-01, // 0.500000 + 9.91483736629098721948594175046309828758239746093750e-01, // 0.600000 + 9.97455625383593114108293775643687695264816284179688e-01, // 0.700000 + 9.99522385942406055292508426646236330270767211914062e-01, // 0.800000 + 9.99971488851369771921895335253793746232986450195312e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_05_4 = { // + 1.09374999967187599395401775836944580078125000000000e+05, // 0.000000 + 3.45863742961106026996276341378688812255859375000000e+02, // 0.000010 + 2.52142232809988087183228344656527042388916015625000e+00, // 0.100000 + 2.09643237315321773195364585262723267078399658203125e+00, // 0.123457 + 1.25219806739988204213887001969851553440093994140625e+00, // 0.200000 + 8.09976508986808951817693014163523912429809570312500e-01, // 0.271828 + 6.84938469046825204600281722377985715866088867187500e-01, // 0.300000 + 6.29524964820964538780856400990160182118415832519531e-01, // 0.314159 + 3.73544048607389822524993405750137753784656524658203e-01, // 0.400000 + 1.93349510480696651359977522588451392948627471923828e-01, // 0.500000 + 9.03696114115064008665001438203034922480583190917969e-02, // 0.600000 + 3.52965948694063183510571946044365176931023597717285e-02, // 0.700000 + 9.78279740156157845420992202889465261250734329223633e-03, // 0.800000 + 1.15291373026972098110587428010376243037171661853790e-03, // 0.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_1_01 = { // + 1.00000000004500018040698788008499304624104864203105e-11, // 0.000000 + 1.00000450002850051417838117551184495823690667748451e-06, // 0.000010 + 1.04807417937856101747273740443233691621571779251099e-02, // 0.100000 + 1.30904819719778343217742033743888896424323320388794e-02, // 0.123457 + 2.20672314570714948878116246078207041136920452117920e-02, // 0.200000 + 3.12239657447760353414789591397493495605885982513428e-02, // 0.271828 + 3.50389048801823949075284758691850584000349044799805e-02, // 0.300000 + 3.70087797866922516010212973469606367871165275573730e-02, // 0.314159 + 4.97997834943236170057723199988686246797442436218262e-02, // 0.400000 + 6.69670084631925899287807624205015599727630615234375e-02, // 0.500000 + 8.75564634445191447564837972095119766891002655029297e-02, // 0.600000 + 1.13431849434786694019727804061403730884194374084473e-01, // 0.700000 + 1.48660077479215418927438463470025453716516494750977e-01, // 0.800000 + 2.05671765275718482213207494169182609766721725463867e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_1_01 = { // + 1.00000000009000014622451146806270116940140724182129e-01, // 0.000000 + 1.00000900008550108744032058893935754895210266113281e-01, // 0.000010 + 1.09946584245134940105437237889418611302971839904785e-01, // 0.100000 + 1.12591074283269546030261665237048873677849769592285e-01, // 0.123457 + 1.22241596067866081787300913674698676913976669311523e-01, // 0.200000 + 1.33042231439480740373682010613265447318553924560547e-01, // 0.271828 + 1.37851585017116834430694893853797111660242080688477e-01, // 0.300000 + 1.40410327292290693090848208157694898545742034912109e-01, // 0.314159 + 1.58366702750946081179606039768259506672620773315430e-01, // 0.400000 + 1.86606598307361509769819463144813198596239089965820e-01, // 0.500000 + 2.28110884138870234627560762419307138770818710327148e-01, // 0.600000 + 2.95522716855071143626787488756235688924789428710938e-01, // 0.700000 + 4.25669961260392526458673501110752113163471221923828e-01, // 0.800000 + 7.94328234724281823098124277748866006731986999511719e-01, // 0.900000 + Double.POSITIVE_INFINITY, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_1_01 = { // + 1.00000000004500001884827449082177529791903847211643e-11, // 0.000000 + 1.00000450002850030242014436193676019115628150757402e-06, // 0.000010 + 1.04807417937856067052804220907091803383082151412964e-02, // 0.100000 + 1.30904819719778325870507273975817952305078506469727e-02, // 0.123457 + 2.20672314570714879489177207005923264659941196441650e-02, // 0.200000 + 3.12239657447887335173231093676804448477923870086670e-02, // 0.271828 + 3.50389048801823740908467641474999254569411277770996e-02, // 0.300000 + 3.70087797866951451197792266611941158771514892578125e-02, // 0.314159 + 4.97997834943235961890906082771834917366504669189453e-02, // 0.400000 + 6.69670084631925344176295311626745387911796569824219e-02, // 0.500000 + 8.75564634445190614897569503227714449167251586914062e-02, // 0.600000 + 1.13431849434786791164242458762601017951965332031250e-01, // 0.700000 + 1.48660077479215502194165310356765985488891601562500e-01, // 0.800000 + 2.05671765275718620991085572313750162720680236816406e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_1_01 = { // + 1.00000000009000028500238954620726872235536575317383e-01, // 0.000000 + 1.00000900008550108744032058893935754895210266113281e-01, // 0.000010 + 1.09946584245134940105437237889418611302971839904785e-01, // 0.100000 + 1.12591074283269559908049473051505628973245620727539e-01, // 0.123457 + 1.22241596067866095665088721489155432209372520446777e-01, // 0.200000 + 1.33042231439496422273904840949398931115865707397461e-01, // 0.271828 + 1.37851585017116834430694893853797111660242080688477e-01, // 0.300000 + 1.40410327292294495604707549318845849484205245971680e-01, // 0.314159 + 1.58366702750946108935181655397173017263412475585938e-01, // 0.400000 + 1.86606598307361537525395078773726709187030792236328e-01, // 0.500000 + 2.28110884138870234627560762419307138770818710327148e-01, // 0.600000 + 2.95522716855071199137938720014062710106372833251953e-01, // 0.700000 + 4.25669961260392470947522269852925091981887817382812e-01, // 0.800000 + 7.94328234724281934120426740264520049095153808593750e-01, // 0.900000 + Double.POSITIVE_INFINITY, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_1_05 = { // + 5.00000000012500009741482020021919761496032030834158e-11, // 0.000000 + 5.00001250006250130353616101253066972276428714394569e-06, // 0.000010 + 5.13167019494862042261473789039882831275463104248047e-02, // 0.100000 + 6.37610881831496972838024817065161187201738357543945e-02, // 0.123457 + 1.05572809000084130715002572742378106340765953063965e-01, // 0.200000 + 1.46670159226752749859201685467269271612167358398438e-01, // 0.271828 + 1.63339973465924465090282069468230474740266799926758e-01, // 0.300000 + 1.71844981515525241366759701122646220028400421142578e-01, // 0.314159 + 2.25403330758516595722085185116156935691833496093750e-01, // 0.400000 + 2.92893218813452427262689070630585774779319763183594e-01, // 0.500000 + 3.67544467966324062135896610925556160509586334228516e-01, // 0.600000 + 4.52277442494833814556898232694948092103004455566406e-01, // 0.700000 + 5.52786404500041905762941496504936367273330688476562e-01, // 0.800000 + 6.83772233983161559223162839771248400211334228515625e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_1_05 = { // + 5.00000000025000113090811737492913380265235900878906e-01, // 0.000000 + 5.00002500018750239973996940534561872482299804687500e-01, // 0.000010 + 5.27046276694730031486813004448777064681053161621094e-01, // 0.100000 + 5.34051718732463376326791149040218442678451538085938e-01, // 0.123457 + 5.59016994374947562285171898111002519726753234863281e-01, // 0.200000 + 5.85939898160509287095010222401469945907592773437500e-01, // 0.271828 + 5.97614304667197004938827831210801377892494201660156e-01, // 0.300000 + 6.03751699669707986295463797432603314518928527832031e-01, // 0.314159 + 6.45497224367902910913130654080305248498916625976562e-01, // 0.400000 + 7.07106781186547683759613391885068267583847045898438e-01, // 0.500000 + 7.90569415042094880696765812899684533476829528808594e-01, // 0.600000 + 9.12870929175276901723634637164650484919548034667969e-01, // 0.700000 + 1.11803398874989512457034379622200503945350646972656e+00, // 0.800000 + 1.58113883008418998343813655083067715167999267578125e+00, // 0.900000 + Double.POSITIVE_INFINITY, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_1_05 = { // + 5.00000000012500009741482020021919761496032030834158e-11, // 0.000000 + 5.00001250006250130353616101253066972276428714394569e-06, // 0.000010 + 5.13167019494862042261473789039882831275463104248047e-02, // 0.100000 + 6.37610881831496972838024817065161187201738357543945e-02, // 0.123457 + 1.05572809000084130715002572742378106340765953063965e-01, // 0.200000 + 1.46670159226808705099642793356906622648239135742188e-01, // 0.271828 + 1.63339973465924409579130838210403453558683395385742e-01, // 0.300000 + 1.71844981515537759131362349762639496475458145141602e-01, // 0.314159 + 2.25403330758516567966509569487243425101041793823242e-01, // 0.400000 + 2.92893218813452260729235376857104711234569549560547e-01, // 0.500000 + 3.67544467966324117647047842183383181691169738769531e-01, // 0.600000 + 4.52277442494833814556898232694948092103004455566406e-01, // 0.700000 + 5.52786404500042127807546421536244451999664306640625e-01, // 0.800000 + 6.83772233983162114334675152349518612027168273925781e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_1_05 = { // + 5.00000000025000002068509274977259337902069091796875e-01, // 0.000000 + 5.00002500018750128951694478018907830119132995605469e-01, // 0.000010 + 5.27046276694729920464510541933123022317886352539062e-01, // 0.100000 + 5.34051718732463265304488686524564400315284729003906e-01, // 0.123457 + 5.59016994374947451262869435595348477363586425781250e-01, // 0.200000 + 5.85939898160547589789359790302114561200141906738281e-01, // 0.271828 + 5.97614304667196782894222906179493293166160583496094e-01, // 0.300000 + 6.03751699669717090124265723716234788298606872558594e-01, // 0.314159 + 6.45497224367902799890828191564651206135749816894531e-01, // 0.400000 + 7.07106781186547572737310929369414225220680236816406e-01, // 0.500000 + 7.90569415042094769674463350384030491113662719726562e-01, // 0.600000 + 9.12870929175276790701332174648996442556381225585938e-01, // 0.700000 + 1.11803398874989490252573887119069695472717285156250e+00, // 0.800000 + 1.58113883008418998343813655083067715167999267578125e+00, // 0.900000 + Double.POSITIVE_INFINITY, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_1_1 = { // + 1.00000000000000003643219731549774157916554706559964e-10, // 0.000000 + 1.00000000000000008180305391403130954586231382563710e-05, // 0.000010 + 1.00000000000000005551115123125782702118158340454102e-01, // 0.100000 + 1.23456700000000002548361521803599316626787185668945e-01, // 0.123457 + 2.00000000000000011102230246251565404236316680908203e-01, // 0.200000 + 2.71828182845904486875099337339634075760841369628906e-01, // 0.271828 + 2.99999999999999988897769753748434595763683319091797e-01, // 0.300000 + 3.14159265358979311599796346854418516159057617187500e-01, // 0.314159 + 4.00000000000000022204460492503130808472633361816406e-01, // 0.400000 + 5.00000000000000000000000000000000000000000000000000e-01, // 0.500000 + 5.99999999999999977795539507496869191527366638183594e-01, // 0.600000 + 6.99999999999999955591079014993738383054733276367188e-01, // 0.700000 + 8.00000000000000044408920985006261616945266723632812e-01, // 0.800000 + 9.00000000000000022204460492503130808472633361816406e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_1_1 = { // + 1.00000000000000000000000000000000000000000000000000e+00, // 0.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.000010 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.123457 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.271828 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.314159 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_1_1 = { // + 9.99999999999999648691285181266018983192722657804552e-11, // 0.000000 + 9.99999999999999742989875012311173918533313553780317e-06, // 0.000010 + 1.00000000000000019428902930940239457413554191589355e-01, // 0.100000 + 1.23456699999999974792785906174685806035995483398438e-01, // 0.123457 + 2.00000000000000011102230246251565404236316680908203e-01, // 0.200000 + 2.71828182846000021566368332059937529265880584716797e-01, // 0.271828 + 2.99999999999999988897769753748434595763683319091797e-01, // 0.300000 + 3.14159265359000017259205606023897416889667510986328e-01, // 0.314159 + 4.00000000000000022204460492503130808472633361816406e-01, // 0.400000 + 5.00000000000000000000000000000000000000000000000000e-01, // 0.500000 + 5.99999999999999977795539507496869191527366638183594e-01, // 0.600000 + 6.99999999999999955591079014993738383054733276367188e-01, // 0.700000 + 8.00000000000000044408920985006261616945266723632812e-01, // 0.800000 + 9.00000000000000022204460492503130808472633361816406e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_1_1 = { // + 1.00000000000000000000000000000000000000000000000000e+00, // 0.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.000010 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.123457 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.271828 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.314159 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_1_2 = { // + 1.99999999990000014066132773281109138302502614692457e-10, // 0.000000 + 1.99999000000000014167899492489866020150657277554274e-05, // 0.000010 + 1.90000000000000002220446049250313080847263336181641e-01, // 0.100000 + 2.31671843225110002384425911259313579648733139038086e-01, // 0.123457 + 3.60000000000000042188474935755948536098003387451172e-01, // 0.200000 + 4.69765804702502509648809336795238777995109558105469e-01, // 0.271828 + 5.10000000000000008881784197001252323389053344726562e-01, // 0.300000 + 5.29622486707065021072082799946656450629234313964844e-01, // 0.314159 + 6.40000000000000124344978758017532527446746826171875e-01, // 0.400000 + 7.50000000000000000000000000000000000000000000000000e-01, // 0.500000 + 8.39999999999999968913755310495616868138313293457031e-01, // 0.600000 + 9.09999999999999920063942226988729089498519897460938e-01, // 0.700000 + 9.59999999999999964472863211994990706443786621093750e-01, // 0.800000 + 9.89999999999999991118215802998747676610946655273438e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_1_2 = { // + 1.99999999979999998345192580018192529678344726562500e+00, // 0.000000 + 1.99998000000000009102052445086883381009101867675781e+00, // 0.000010 + 1.80000000000000004440892098500626161694526672363281e+00, // 0.100000 + 1.75308660000000005041442818765062838792800903320312e+00, // 0.123457 + 1.60000000000000008881784197001252323389053344726562e+00, // 0.200000 + 1.45634363430819102624980132532073184847831726074219e+00, // 0.271828 + 1.39999999999999991118215802998747676610946655273438e+00, // 0.300000 + 1.37168146928204137680040730629116296768188476562500e+00, // 0.314159 + 1.19999999999999995559107901499373838305473327636719e+00, // 0.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.500000 + 8.00000000000000044408920985006261616945266723632812e-01, // 0.600000 + 6.00000000000000088817841970012523233890533447265625e-01, // 0.700000 + 3.99999999999999911182158029987476766109466552734375e-01, // 0.800000 + 1.99999999999999955591079014993738383054733276367188e-01, // 0.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_1_2 = { // + 1.99999999989999600475826496767271702598156579711031e-10, // 0.000000 + 1.99998999999999980286581602317852457417757250368595e-05, // 0.000010 + 1.89999999999999502620084967929869890213012695312500e-01, // 0.100000 + 2.31671843225110141162303989403881132602691650390625e-01, // 0.123457 + 3.59999999999999764632718779466813430190086364746094e-01, // 0.200000 + 4.69765804702641842638399793941061943769454956054688e-01, // 0.271828 + 5.10000000000000008881784197001252323389053344726562e-01, // 0.300000 + 5.29622486707093331759210741438437253236770629882812e-01, // 0.314159 + 6.40000000000000013322676295501878485083580017089844e-01, // 0.400000 + 7.50000000000000000000000000000000000000000000000000e-01, // 0.500000 + 8.39999999999999968913755310495616868138313293457031e-01, // 0.600000 + 9.09999999999999920063942226988729089498519897460938e-01, // 0.700000 + 9.59999999999999964472863211994990706443786621093750e-01, // 0.800000 + 9.89999999999999991118215802998747676610946655273438e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_1_2 = { // + 1.99999999979999998345192580018192529678344726562500e+00, // 0.000000 + 1.99998000000000009102052445086883381009101867675781e+00, // 0.000010 + 1.79999999999999982236431605997495353221893310546875e+00, // 0.100000 + 1.75308660000000005041442818765062838792800903320312e+00, // 0.123457 + 1.59999999999999986677323704498121514916419982910156e+00, // 0.200000 + 1.45634363430799984584496087336447089910507202148438e+00, // 0.271828 + 1.40000000000000013322676295501878485083580017089844e+00, // 0.300000 + 1.37168146928199985445928632543655112385749816894531e+00, // 0.314159 + 1.19999999999999995559107901499373838305473327636719e+00, // 0.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.500000 + 8.00000000000000044408920985006261616945266723632812e-01, // 0.600000 + 6.00000000000000088817841970012523233890533447265625e-01, // 0.700000 + 3.99999999999999966693309261245303787291049957275391e-01, // 0.800000 + 1.99999999999999983346654630622651893645524978637695e-01, // 0.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_1_4 = { // + 3.99999999940000003552250502724231887019534781302355e-10, // 0.000000 + 3.99994000039999964652912001206175318657187744975090e-05, // 0.000010 + 3.43900000000000038991032624835497699677944183349609e-01, // 0.100000 + 4.09671843506900057452213559372466988861560821533203e-01, // 0.123457 + 5.90400000000000035882408155885059386491775512695312e-01, // 0.200000 + 7.18851698137215278094913628592621535062789916992188e-01, // 0.271828 + 7.59900000000000019895196601282805204391479492187500e-01, // 0.300000 + 7.78744994988354810594444188609486445784568786621094e-01, // 0.314159 + 8.70400000000000062527760746888816356658935546875000e-01, // 0.400000 + 9.37500000000000000000000000000000000000000000000000e-01, // 0.500000 + 9.74400000000000043876013933186186477541923522949219e-01, // 0.600000 + 9.91900000000000003907985046680551022291183471679688e-01, // 0.700000 + 9.98399999999999954169993543473538011312484741210938e-01, // 0.800000 + 9.99900000000000011013412404281552881002426147460938e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_1_4 = { // + 3.99999999879999990071155480109155178070068359375000e+00, // 0.000000 + 3.99988000119999664860870325355790555477142333984375e+00, // 0.000010 + 2.91600000000000036948222259525209665298461914062500e+00, // 0.100000 + 2.69389159208951811663723674428183585405349731445312e+00, // 0.123457 + 2.04800000000000048672177399566862732172012329101562e+00, // 0.200000 + 1.54440639002807333923783517093397676944732666015625e+00, // 0.271828 + 1.37199999999999966426855735335266217589378356933594e+00, // 0.300000 + 1.29041623710177200123894181160721927881240844726562e+00, // 0.314159 + 8.63999999999999879207734920782968401908874511718750e-01, // 0.400000 + 5.00000000000000000000000000000000000000000000000000e-01, // 0.500000 + 2.56000000000000060840221749458578415215015411376953e-01, // 0.600000 + 1.08000000000000054289905904170154826715588569641113e-01, // 0.700000 + 3.19999999999999798494521030534087913110852241516113e-02, // 0.800000 + 3.99999999999999748118151288167609891388565301895142e-03, // 0.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_1_4 = { // + 3.99999999939999900154673933595772528093448272556998e-10, // 0.000000 + 3.99994000039999896890276220862148193191387690603733e-05, // 0.000010 + 3.43899999999999872457578931062016636133193969726562e-01, // 0.100000 + 4.09671843506900223985667253145948052406311035156250e-01, // 0.123457 + 5.90399999999999813837803230853751301765441894531250e-01, // 0.200000 + 7.18851698137362937757188774412497878074645996093750e-01, // 0.271828 + 7.59900000000000019895196601282805204391479492187500e-01, // 0.300000 + 7.78744994988381455947035192366456612944602966308594e-01, // 0.314159 + 8.70400000000000062527760746888816356658935546875000e-01, // 0.400000 + 9.37500000000000000000000000000000000000000000000000e-01, // 0.500000 + 9.74399999999999932853711470670532435178756713867188e-01, // 0.600000 + 9.91900000000000003907985046680551022291183471679688e-01, // 0.700000 + 9.98399999999999954169993543473538011312484741210938e-01, // 0.800000 + 9.99900000000000011013412404281552881002426147460938e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_1_4 = { // + 3.99999999879999990071155480109155178070068359375000e+00, // 0.000000 + 3.99988000119999531634107370337005704641342163085938e+00, // 0.000010 + 2.91600000000000036948222259525209665298461914062500e+00, // 0.100000 + 2.69389159208951767254802689421921968460083007812500e+00, // 0.123457 + 2.04799999999999959854335429554339498281478881835938e+00, // 0.200000 + 1.54440639002746538110955043521244078874588012695312e+00, // 0.271828 + 1.37200000000000033040237212844658643007278442382812e+00, // 0.300000 + 1.29041623710165476168754139507655054330825805664062e+00, // 0.314159 + 8.63999999999999879207734920782968401908874511718750e-01, // 0.400000 + 5.00000000000000111022302462515654042363166809082031e-01, // 0.500000 + 2.56000000000000060840221749458578415215015411376953e-01, // 0.600000 + 1.08000000000000054289905904170154826715588569641113e-01, // 0.700000 + 3.19999999999999937272399108678655466064810752868652e-02, // 0.800000 + 3.99999999999999834854325087007964611984789371490479e-03, // 0.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_2_01 = { // + 5.50000000033000091061882744625813924005303568789147e-22, // 0.000000 + 5.50003300023512856945298627756354276335110098372638e-12, // 0.000010 + 5.85549211723463618010920850309730667504481971263885e-04, // 0.100000 + 9.06422742544821247179243073333054780960083007812500e-04, // 0.123457 + 2.50857608621292025793958480051060178084298968315125e-03, // 0.200000 + 4.88990284715010506411125845716014737263321876525879e-03, // 0.271828 + 6.09007202658786231386223164463444845750927925109863e-03, // 0.300000 + 6.75551835775626500352197822962807549629360437393188e-03, // 0.314159 + 1.17917748340965571063332362200526404194533824920654e-02, // 0.400000 + 2.03153588863522159557728485879124491475522518157959e-02, // 0.500000 + 3.28098512511902778987504802898911293596029281616211e-02, // 0.600000 + 5.13720788952217247147480350122350500896573066711426e-02, // 0.700000 + 8.05528836775526307922845603570749517530202865600586e-02, // 0.800000 + 1.34182224150533185857980811306333635002374649047852e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_2_01 = { // + 1.10000000009900018879889436430572250639331466359749e-11, // 0.000000 + 1.10000990009405111972099443329309309547170414589345e-06, // 0.000010 + 1.20941242669648438279317304022697499021887779235840e-02, // 0.100000 + 1.52901347285140567361372276877773401793092489242554e-02, // 0.123457 + 2.68931511349305395197628598680239520035684108734131e-02, // 0.200000 + 3.97810908153541434306710300461418228223919868469238e-02, // 0.271828 + 4.54910230556485536967947780340182362124323844909668e-02, // 0.300000 + 4.85223257980558694835160338243440492078661918640137e-02, // 0.314159 + 6.96813492104162851559223668118647765368223190307617e-02, // 0.400000 + 1.02633629069048826210064362385310232639312744140625e-01, // 0.500000 + 1.50553183531654338755956246131972875446081161499023e-01, // 0.600000 + 2.27552491978404769490396120090736076235771179199219e-01, // 0.700000 + 3.74589565909145438826755025729653425514698028564453e-01, // 0.800000 + 7.86384952377038870530157055327435955405235290527344e-01, // 0.900000 + Double.POSITIVE_INFINITY, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_2_01 = { // + 5.50000000033000185101430810408814561504226546568802e-22, // 0.000000 + 5.50003300023513099283368711651180898818125353244568e-12, // 0.000010 + 5.85549211723463618010920850309730667504481971263885e-04, // 0.100000 + 9.06422742544820921918591327681724578724242746829987e-04, // 0.123457 + 2.50857608621292025793958480051060178084298968315125e-03, // 0.200000 + 4.88990284715390410852364766469690948724746704101562e-03, // 0.271828 + 6.09007202658786057913875566782735404558479785919189e-03, // 0.300000 + 6.75551835775727114313804477774283441249281167984009e-03, // 0.314159 + 1.17917748340965536368862842664384515956044197082520e-02, // 0.400000 + 2.03153588863522090168789446806840714998543262481689e-02, // 0.500000 + 3.28098512511902501431748646609776187688112258911133e-02, // 0.600000 + 5.13720788952216692035968037544080289080739021301270e-02, // 0.700000 + 8.05528836775526863034357916149019729346036911010742e-02, // 0.800000 + 1.34182224150533130346829580048506613820791244506836e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_2_01 = { // + 1.10000000009900018879889436430572250639331466359749e-11, // 0.000000 + 1.10000990009405090796275761971800832839107897598296e-06, // 0.000010 + 1.20941242669648507668256343094981275498867034912109e-02, // 0.100000 + 1.52901347285140602055841796413915290031582117080688e-02, // 0.123457 + 2.68931511349305464586567637752523296512663364410400e-02, // 0.200000 + 3.97810908153728159941664443977060727775096893310547e-02, // 0.271828 + 4.54910230556485467579008741267898585647344589233398e-02, // 0.300000 + 4.85223257980603936423413813372462755069136619567871e-02, // 0.314159 + 6.96813492104162851559223668118647765368223190307617e-02, // 0.400000 + 1.02633629069048826210064362385310232639312744140625e-01, // 0.500000 + 1.50553183531654338755956246131972875446081161499023e-01, // 0.600000 + 2.27552491978404769490396120090736076235771179199219e-01, // 0.700000 + 3.74589565909145383315603794471826404333114624023438e-01, // 0.800000 + 7.86384952377038981552459517843089997768402099609375e-01, // 0.900000 + Double.POSITIVE_INFINITY, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_2_05 = { // + 3.75000000012500073511663321748728148736677675747611e-21, // 0.000000 + 3.75001250007031431176997547885672810996648962600375e-11, // 0.000010 + 3.88253704696051200884188148165776510722935199737549e-03, // 0.100000 + 5.96860495090002600437895097229556995443999767303467e-03, // 0.123457 + 1.61300899000925389292770972815560526214540004730225e-02, // 0.200000 + 3.06906092339643381261549137661859276704490184783936e-02, // 0.271828 + 3.78409694858131209760365720740082906559109687805176e-02, // 0.300000 + 4.17586954103080082445487164477526675909757614135742e-02, // 0.314159 + 7.04839969102199759287685765229980461299419403076172e-02, // 0.400000 + 1.16116523516815603467300377360515994951128959655762e-01, // 0.500000 + 1.77807808356221369594507564215746242552995681762695e-01, // 0.600000 + 2.60574547368025710714078968521789647638797760009766e-01, // 0.700000 + 3.73900966300058923419413758892915211617946624755859e-01, // 0.800000 + 5.41469739275584838189558922749711200594902038574219e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_2_05 = { // + 7.50000000037500098835526107000383749107141895251516e-11, // 0.000000 + 7.50003750028125248017121101673509997453948017209768e-06, // 0.000010 + 7.90569415042094991719068275415338575839996337890625e-02, // 0.100000 + 9.88983942360571466334562273914343677461147308349609e-02, // 0.123457 + 1.67705098312484257583321323181735351681709289550781e-01, // 0.200000 + 2.38912466660828326503462903929175809025764465332031e-01, // 0.271828 + 2.68926437100238613364666662164381705224514007568359e-01, // 0.300000 + 2.84511285641205868834902048547519370913505554199219e-01, // 0.314159 + 3.87298334620741757650108638699748553335666656494141e-01, // 0.400000 + 5.30330085889910707308558812655974179506301879882812e-01, // 0.500000 + 7.11512473537885270502556522842496633529663085937500e-01, // 0.600000 + 9.58514475634040685747550014639273285865783691406250e-01, // 0.700000 + 1.34164078649987406066657058545388281345367431640625e+00, // 0.800000 + 2.13453742061365625559687941859010607004165649414062e+00, // 0.900000 + Double.POSITIVE_INFINITY, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_2_05 = { // + 3.75000000012498794573809627099919478751325177944313e-21, // 0.000000 + 3.75001250007031495800482903590959910325453030566223e-11, // 0.000010 + 3.88253704696051200884188148165776510722935199737549e-03, // 0.100000 + 5.96860495090002253493199901868138113059103488922119e-03, // 0.123457 + 1.61300899000925389292770972815560526214540004730225e-02, // 0.200000 + 3.06906092339871497398640087794774444773793220520020e-02, // 0.271828 + 3.78409694858131001593548603523231577128171920776367e-02, // 0.300000 + 4.17586954103138993654731336846452904865145683288574e-02, // 0.314159 + 7.04839969102199342954051530796277802437543869018555e-02, // 0.400000 + 1.16116523516815506322785722659318707883358001708984e-01, // 0.500000 + 1.77807808356221147549902639184438157826662063598633e-01, // 0.600000 + 2.60574547368025322136020349717000499367713928222656e-01, // 0.700000 + 3.73900966300058978930564990150742232799530029296875e-01, // 0.800000 + 5.41469739275584949211861385265365242958068847656250e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_2_05 = { // + 7.50000000037501132811291798284977338368006982705083e-11, // 0.000000 + 7.50003750028124570390763298233238742795947473496199e-06, // 0.000010 + 7.90569415042094991719068275415338575839996337890625e-02, // 0.100000 + 9.88983942360571466334562273914343677461147308349609e-02, // 0.123457 + 1.67705098312484229827745707552821841090917587280273e-01, // 0.200000 + 2.38912466660927941264347396099765319377183914184570e-01, // 0.271828 + 2.68926437100238502342364199648727662861347198486328e-01, // 0.300000 + 2.84511285641228850451511789287906140089035034179688e-01, // 0.314159 + 3.87298334620741702138957407441921532154083251953125e-01, // 0.400000 + 5.30330085889910596286256350140320137143135070800781e-01, // 0.500000 + 7.11512473537885270502556522842496633529663085937500e-01, // 0.600000 + 9.58514475634040463702945089607965201139450073242188e-01, // 0.700000 + 1.34164078649987383862196566042257472872734069824219e+00, // 0.800000 + 2.13453742061365581150766956852748990058898925781250e+00, // 0.900000 + Double.POSITIVE_INFINITY, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_2_1 = { // + 1.00000000000000009561654835946237267172778046723484e-20, // 0.000000 + 1.00000000000000016567916802690831577782315520153134e-10, // 0.000010 + 1.00000000000000019428902930940239457413554191589355e-02, // 0.100000 + 1.52415567748900009775736563710779591929167509078979e-02, // 0.123457 + 4.00000000000000077715611723760957829654216766357422e-02, // 0.200000 + 7.38905609893064779791771456984861288219690322875977e-02, // 0.271828 + 8.99999999999999966693309261245303787291049957275391e-02, // 0.300000 + 9.86960440108935743719342781332670710980892181396484e-02, // 0.314159 + 1.60000000000000031086244689504383131861686706542969e-01, // 0.400000 + 2.50000000000000000000000000000000000000000000000000e-01, // 0.500000 + 3.59999999999999986677323704498121514916419982910156e-01, // 0.600000 + 4.89999999999999935607064571740920655429363250732422e-01, // 0.700000 + 6.40000000000000124344978758017532527446746826171875e-01, // 0.800000 + 8.10000000000000053290705182007513940334320068359375e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_2_1 = { // + 2.00000000000000007286439463099548315833109413119928e-10, // 0.000000 + 2.00000000000000016360610782806261909172462765127420e-05, // 0.000010 + 2.00000000000000011102230246251565404236316680908203e-01, // 0.100000 + 2.46913400000000005096723043607198633253574371337891e-01, // 0.123457 + 4.00000000000000022204460492503130808472633361816406e-01, // 0.200000 + 5.43656365691808973750198674679268151521682739257812e-01, // 0.271828 + 5.99999999999999977795539507496869191527366638183594e-01, // 0.300000 + 6.28318530717958623199592693708837032318115234375000e-01, // 0.314159 + 8.00000000000000044408920985006261616945266723632812e-01, // 0.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.500000 + 1.19999999999999995559107901499373838305473327636719e+00, // 0.600000 + 1.39999999999999991118215802998747676610946655273438e+00, // 0.700000 + 1.60000000000000008881784197001252323389053344726562e+00, // 0.800000 + 1.80000000000000004440892098500626161694526672363281e+00, // 0.900000 + 2.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_2_1 = { // + 1.00000000000000114885948669623197981171571781836697e-20, // 0.000000 + 9.99999999999997968480665932928554400643816890692506e-11, // 0.000010 + 1.00000000000000054123372450476381345652043819427490e-02, // 0.100000 + 1.52415567748899975081267044174637703690677881240845e-02, // 0.123457 + 4.00000000000000077715611723760957829654216766357422e-02, // 0.200000 + 7.38905609893584225389417952101211994886398315429688e-02, // 0.271828 + 8.99999999999999827915431183100736234337091445922852e-02, // 0.300000 + 9.86960440109066056146858159081602934747934341430664e-02, // 0.314159 + 1.60000000000000003330669073875469621270895004272461e-01, // 0.400000 + 2.50000000000000000000000000000000000000000000000000e-01, // 0.500000 + 3.59999999999999986677323704498121514916419982910156e-01, // 0.600000 + 4.89999999999999935607064571740920655429363250732422e-01, // 0.700000 + 6.40000000000000235367281220533186569809913635253906e-01, // 0.800000 + 8.10000000000000053290705182007513940334320068359375e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_2_1 = { // + 1.99999999999999593696133186585710880128763378138501e-10, // 0.000000 + 1.99999999999999982479292892634248346439562737941742e-05, // 0.000010 + 2.00000000000000066613381477509392425417900085449219e-01, // 0.100000 + 2.46913399999999977341147427978285122662782669067383e-01, // 0.123457 + 4.00000000000000022204460492503130808472633361816406e-01, // 0.200000 + 5.43656365692000043132736664119875058531761169433594e-01, // 0.271828 + 5.99999999999999866773237044981215149164199829101562e-01, // 0.300000 + 6.28318530718000034518411212047794833779335021972656e-01, // 0.314159 + 8.00000000000000044408920985006261616945266723632812e-01, // 0.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.500000 + 1.19999999999999995559107901499373838305473327636719e+00, // 0.600000 + 1.39999999999999991118215802998747676610946655273438e+00, // 0.700000 + 1.60000000000000008881784197001252323389053344726562e+00, // 0.800000 + 1.80000000000000004440892098500626161694526672363281e+00, // 0.900000 + 2.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_2_2 = { // + 2.99999999980000028621542970819290774575628845094157e-20, // 0.000000 + 2.99998000000000032546659169620955655288785379752881e-10, // 0.000010 + 2.80000000000000040523140398818213725462555885314941e-02, // 0.100000 + 4.19613257200888781217607004236924694851040840148926e-02, // 0.123457 + 1.04000000000000022981616609740740386769175529479980e-01, // 0.200000 + 1.81500609121544087543043133337050676345825195312500e-01, // 0.271828 + 2.16000000000000025313084961453569121658802032470703e-01, // 0.300000 + 2.34075578672081091591294921272492501884698867797852e-01, // 0.314159 + 3.52000000000000090594198809412773698568344116210938e-01, // 0.400000 + 5.00000000000000000000000000000000000000000000000000e-01, // 0.500000 + 6.47999999999999909405801190587226301431655883789062e-01, // 0.600000 + 7.83999999999999919175763807288603857159614562988281e-01, // 0.700000 + 8.96000000000000018651746813702629879117012023925781e-01, // 0.800000 + 9.71999999999999975131004248396493494510650634765625e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_2_2 = { // + 5.99999999940000036688084108105895042584165821608622e-10, // 0.000000 + 5.99994000000000069806882496692423956119455397129059e-05, // 0.000010 + 5.40000000000000146549439250520663335919380187988281e-01, // 0.100000 + 6.49290859350659999016386336734285578131675720214844e-01, // 0.123457 + 9.60000000000000186517468137026298791170120239257812e-01, // 0.200000 + 1.18762573113958813664225999673362821340560913085938e+00, // 0.271828 + 1.26000000000000000888178419700125232338905334472656e+00, // 0.300000 + 1.29277932808851447887832364358473569154739379882812e+00, // 0.314159 + 1.43999999999999994670929481799248605966567993164062e+00, // 0.400000 + 1.50000000000000000000000000000000000000000000000000e+00, // 0.500000 + 1.43999999999999994670929481799248605966567993164062e+00, // 0.600000 + 1.26000000000000023092638912203256040811538696289062e+00, // 0.700000 + 9.59999999999999853450560749479336664080619812011719e-01, // 0.800000 + 5.39999999999999924504834325489355251193046569824219e-01, // 0.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_2_2 = { // + 2.99999999979999607324367636111447918580453904641306e-20, // 0.000000 + 2.99997999999999722353929462235577578510525853516810e-10, // 0.000010 + 2.80000000000000109912079437890497501939535140991211e-02, // 0.100000 + 4.19613257200888642439728926092357141897082328796387e-02, // 0.123457 + 1.04000000000000022981616609740740386769175529479980e-01, // 0.200000 + 1.81500609121657580091735439964395482093095779418945e-01, // 0.271828 + 2.15999999999999942046358114566828589886426925659180e-01, // 0.300000 + 2.34075578672107875721764003174030221998691558837891e-01, // 0.314159 + 3.52000000000000035083047578154946677386760711669922e-01, // 0.400000 + 5.00000000000000000000000000000000000000000000000000e-01, // 0.500000 + 6.47999999999999909405801190587226301431655883789062e-01, // 0.600000 + 7.84000000000000030198066269804257899522781372070312e-01, // 0.700000 + 8.96000000000000018651746813702629879117012023925781e-01, // 0.800000 + 9.71999999999999975131004248396493494510650634765625e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_2_2 = { // + 5.99999999940000553675966953748191837214598365335405e-10, // 0.000000 + 5.99993999999999459943160473596179826927254907786846e-05, // 0.000010 + 5.40000000000000035527136788005009293556213378906250e-01, // 0.100000 + 6.49290859350659776971781411702977493405342102050781e-01, // 0.123457 + 9.60000000000000075495165674510644748806953430175781e-01, // 0.200000 + 1.18762573113984970518686168361455202102661132812500e+00, // 0.271828 + 1.26000000000000000888178419700125232338905334472656e+00, // 0.300000 + 1.29277932808856044211154312506550922989845275878906e+00, // 0.314159 + 1.44000000000000016875389974302379414439201354980469e+00, // 0.400000 + 1.50000000000000000000000000000000000000000000000000e+00, // 0.500000 + 1.44000000000000016875389974302379414439201354980469e+00, // 0.600000 + 1.26000000000000000888178419700125232338905334472656e+00, // 0.700000 + 9.59999999999999853450560749479336664080619812011719e-01, // 0.800000 + 5.39999999999999813482531862973701208829879760742188e-01, // 0.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_2_4 = { // + 9.99999999800000094982332989268162402300727516471889e-20, // 0.000000 + 9.99980000149999789015279420541053301274558862132835e-10, // 0.000010 + 8.14600000000000185051973744521092157810926437377930e-02, // 0.100000 + 1.18151979036013266011195810278877615928649902343750e-01, // 0.123457 + 2.62720000000000009077183449335279874503612518310547e-01, // 0.200000 + 4.13155570114924819424828683622763492166996002197266e-01, // 0.271828 + 4.71779999999999921644899814054952003061771392822266e-01, // 0.300000 + 5.00707755662531783791280304285464808344841003417969e-01, // 0.314159 + 6.63040000000000073754335971898399293422698974609375e-01, // 0.400000 + 8.12500000000000000000000000000000000000000000000000e-01, // 0.500000 + 9.12959999999999993747223925311118364334106445312500e-01, // 0.600000 + 9.69219999999999970441422192379832267761230468750000e-01, // 0.700000 + 9.93280000000000051763038300123298540711402893066406e-01, // 0.800000 + 9.99539999999999984048315582185750827193260192871094e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_2_4 = { // + 1.99999999939999970416416897342568731454903740996087e-09, // 0.000000 + 1.99994000059999853073644526801899701240472495555878e-04, // 0.000010 + 1.45799999999999996269650637259474024176597595214844e+00, // 0.100000 + 1.66289483058558995587361550860805436968803405761719e+00, // 0.123457 + 2.04800000000000048672177399566862732172012329101562e+00, // 0.200000 + 2.09906591288467181399823857645969837903976440429688e+00, // 0.271828 + 2.05799999999999938538053356751333922147750854492188e+00, // 0.300000 + 2.02698108527595532990517313010059297084808349609375e+00, // 0.314159 + 1.72799999999999975841546984156593680381774902343750e+00, // 0.400000 + 1.25000000000000000000000000000000000000000000000000e+00, // 0.500000 + 7.68000000000000127009514017117908224463462829589844e-01, // 0.600000 + 3.78000000000000113686837721616029739379882812500000e-01, // 0.700000 + 1.27999999999999919397808412213635165244340896606445e-01, // 0.800000 + 1.79999999999999882316359389733406715095043182373047e-02, // 0.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_2_4 = { // + 9.99999999799998650534874698841272610317270577776400e-20, // 0.000000 + 9.99980000149998548244360590999540994161520757188555e-10, // 0.000010 + 8.14600000000000601385607978954794816672801971435547e-02, // 0.100000 + 1.18151979036013293766771425907791126519441604614258e-01, // 0.123457 + 2.62720000000000064588334680593106895685195922851562e-01, // 0.200000 + 4.13155570115125381214227218151791021227836608886719e-01, // 0.271828 + 4.71779999999999810622597351539297960698604583740234e-01, // 0.300000 + 5.00707755662573861243913597718346863985061645507812e-01, // 0.314159 + 6.63040000000000073754335971898399293422698974609375e-01, // 0.400000 + 8.12500000000000000000000000000000000000000000000000e-01, // 0.500000 + 9.12959999999999993747223925311118364334106445312500e-01, // 0.600000 + 9.69219999999999970441422192379832267761230468750000e-01, // 0.700000 + 9.93279999999999940740735837607644498348236083984375e-01, // 0.800000 + 9.99539999999999984048315582185750827193260192871094e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_2_4 = { // + 1.99999999940000135852539407948103705736642154988658e-09, // 0.000000 + 1.99994000059999527812992781150569499004632234573364e-04, // 0.000010 + 1.45799999999999996269650637259474024176597595214844e+00, // 0.100000 + 1.66289483058558951178440565854543820023536682128906e+00, // 0.123457 + 2.04799999999999959854335429554339498281478881835938e+00, // 0.200000 + 2.09906591288458299615626856393646448850631713867188e+00, // 0.271828 + 2.05799999999999938538053356751333922147750854492188e+00, // 0.300000 + 2.02698108527590559191366992308758199214935302734375e+00, // 0.314159 + 1.72799999999999998046007476659724488854408264160156e+00, // 0.400000 + 1.25000000000000000000000000000000000000000000000000e+00, // 0.500000 + 7.67999999999999904964909092086600139737129211425781e-01, // 0.600000 + 3.78000000000000002664535259100375697016716003417969e-01, // 0.700000 + 1.27999999999999947153384027842548675835132598876953e-01, // 0.800000 + 1.79999999999999847621889870197264826856553554534912e-02, // 0.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_4_01 = { // + 2.98375000021482896530143510253419751265415869320405e-42, // 0.000000 + 2.98377148317007480476821711655106704510022866858290e-22, // 0.000010 + 3.21712826910625692496299710132490190517273731529713e-06, // 0.100000 + 7.61463101593263861873502745813979686317907180637121e-06, // 0.123457 + 5.59207027072549877851938282535826374441967345774174e-05, // 0.200000 + 2.03667699680023080383364786349886799143860116600990e-04, // 0.271828 + 3.10437547367714518006809321093442122219130396842957e-04, // 0.300000 + 3.78567315617660054768989219553532166173681616783142e-04, // 0.314159 + 1.08871959537661425719556884672556407167576253414154e-03, // 0.400000 + 2.99593398095022393073927347018070577178150415420532e-03, // 0.500000 + 7.15558877739638343057348990328137006144970655441284e-03, // 0.600000 + 1.57714915252002985113222166546620428562164306640625e-02, // 0.700000 + 3.38041058520913012541697639790072571486234664916992e-02, // 0.800000 + 7.65008878936773301093765553559933323413133621215820e-02, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_4_01 = { // + 1.19350000010741456808122504902637373369716469559713e-31, // 0.000000 + 1.19351074160204525818413908323422939743433979524131e-16, // 0.000010 + 1.31221248296568523180466447897174475656356662511826e-04, // 0.100000 + 2.52854320367995155215828928518817519943695515394211e-04, // 0.123457 + 1.16716275925598492353707502644510896061547100543976e-03, // 0.200000 + 3.18930012206767362487047634544978791382163763046265e-03, // 0.271828 + 4.44219840138407838503464830637312843464314937591553e-03, // 0.300000 + 5.19602333868628314672566403942255419678986072540283e-03, // 0.314159 + 1.20966822229282636169678255555481882765889167785645e-02, // 0.400000 + 2.78393718849794802316921504825586453080177307128906e-02, // 0.500000 + 5.88060734874641527492045156577660236507654190063477e-02, // 0.600000 + 1.20978282360318822874489796959096565842628479003906e-01, // 0.700000 + 2.60114994567310464645970569108612835407257080078125e-01, // 0.800000 + 6.91114415396560488602517580147832632064819335937500e-01, // 0.900000 + Double.POSITIVE_INFINITY, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_4_01 = { // + 2.98375000021482960253819955551510832420631780028878e-42, // 0.000000 + 2.98377148317007715575691876112608298257330311307425e-22, // 0.000010 + 3.21712826910625988957831249137608864430148969404399e-06, // 0.100000 + 7.61463101593263438357029118663810152156656840816140e-06, // 0.123457 + 5.59207027072550148902481403911934876305167563259602e-05, // 0.200000 + 2.03667699680327795403941837371064593753544613718987e-04, // 0.271828 + 3.10437547367714355376483448267777021101210266351700e-04, // 0.300000 + 3.78567315617767824464934278694272506982088088989258e-04, // 0.314159 + 1.08871959537661447403600334382645087316632270812988e-03, // 0.400000 + 2.99593398095022523178188045278602658072486519813538e-03, // 0.500000 + 7.15558877739638603265870386849201167933642864227295e-03, // 0.600000 + 1.57714915252002915724283127474336652085185050964355e-02, // 0.700000 + 3.38041058520913290097453796079207677394151687622070e-02, // 0.800000 + 7.65008878936773439871643631704500876367092132568359e-02, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_4_01 = { // + 1.19350000010741500598699515053170839736265947369592e-31, // 0.000000 + 1.19351074160204328603187603070471586811292658827574e-16, // 0.000010 + 1.31221248296568631600683696447617876401636749505997e-04, // 0.100000 + 2.52854320367994938375394431417930718453135341405869e-04, // 0.123457 + 1.16716275925598514037750952354599576210603117942810e-03, // 0.200000 + 3.18930012207141325500381334734356642002239823341370e-03, // 0.271828 + 4.44219840138407751767291031796958122868090867996216e-03, // 0.300000 + 5.19602333868745148298673441900064062792807817459106e-03, // 0.314159 + 1.20966822229282670864147775091623771004378795623779e-02, // 0.400000 + 2.78393718849794941094799582970154006034135818481445e-02, // 0.500000 + 5.88060734874641596880984195649944012984633445739746e-02, // 0.600000 + 1.20978282360318822874489796959096565842628479003906e-01, // 0.700000 + 2.60114994567310464645970569108612835407257080078125e-01, // 0.800000 + 6.91114415396560599624820042663486674427986145019531e-01, // 0.900000 + Double.POSITIVE_INFINITY, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_4_05 = { // + 2.73437500010937531608195608847323954680264550128516e-41, // 0.000000 + 2.73438593756836093040429464081171992743562950508136e-21, // 0.000010 + 2.85111486302981175650073097926906484644860029220581e-05, // 0.100000 + 6.69233684394836031601727643902677300502546131610870e-05, // 0.123457 + 4.77614057594005748073884287663304348825477063655853e-04, // 0.200000 + 1.68960523493624824656333949235431646229699254035950e-03, // 0.271828 + 2.54437461640680739546893640579128259560093283653259e-03, // 0.300000 + 3.08346021657933443485544167117495817365124821662903e-03, // 0.314159 + 8.51626337090127978612930093049726565368473529815674e-03, // 0.400000 + 2.22039041404772513610410555884300265461206436157227e-02, // 0.500000 + 4.97355631194020056384808015081944176927208900451660e-02, // 0.600000 + 1.01221515793866442822768192399962572380900382995605e-01, // 0.700000 + 1.95015528100075691275705480620672460645437240600586e-01, // 0.800000 + 3.73374917402259476340020682982867583632469177246094e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_4_05 = { // + 1.09375000005468786566718251296557641016341269128500e-30, // 0.000000 + 1.09375546879101630526115027849392190014539904489160e-15, // 0.000010 + 1.15291373026972206530804676560819643782451748847961e-03, // 0.100000 + 2.19824134059588231077109199418373464141041040420532e-03, // 0.123457 + 9.78279740156158365838034995931593584828078746795654e-03, // 0.200000 + 2.57445069421568979717296343778798473067581653594971e-02, // 0.271828 + 3.52965948694063183510571946044365176931023597717285e-02, // 0.300000 + 4.09502017884755334375945778901950689032673835754395e-02, // 0.314159 + 9.03696114115064286220757594492170028388500213623047e-02, // 0.400000 + 1.93349510480696623604401906959537882357835769653320e-01, // 0.500000 + 3.73544048607389767013842174492310732603073120117188e-01, // 0.600000 + 6.84938469046824871533374334831023588776588439941406e-01, // 0.700000 + 1.25219806739988270827268479479243978857994079589844e+00, // 0.800000 + 2.52142232809988220409991299675311893224716186523438e+00, // 0.900000 + Double.POSITIVE_INFINITY, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_4_05 = { // + 2.73437500010936715945137109031758115893500893060072e-41, // 0.000000 + 2.73438593756835415955683390443567402751317510494625e-21, // 0.000010 + 2.85111486302981480581934109475028549240960273891687e-05, // 0.100000 + 6.69233684394835353975369840462406045844545587897301e-05, // 0.123457 + 4.77614057594005693863775663388082648452837020158768e-04, // 0.200000 + 1.68960523493870678340966762220887176226824522018433e-03, // 0.271828 + 2.54437461640680566074546042898418818367645144462585e-03, // 0.300000 + 3.08346021658018184727345634144057839876040816307068e-03, // 0.314159 + 8.51626337090127805140582495369017124176025390625000e-03, // 0.400000 + 2.22039041404772374832532477739732712507247924804688e-02, // 0.500000 + 4.97355631194019570662234741575957741588354110717773e-02, // 0.600000 + 1.01221515793866317922677922069851774722337722778320e-01, // 0.700000 + 1.95015528100075719031281096249585971236228942871094e-01, // 0.800000 + 3.73374917402259698384625608014175668358802795410156e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_4_05 = { // + 1.09375000005468611404410210694423775550143357888984e-30, // 0.000000 + 1.09375546879101808019818702577048407653467093116062e-15, // 0.000010 + 1.15291373026972206530804676560819643782451748847961e-03, // 0.100000 + 2.19824134059588057604761601737664022948592901229858e-03, // 0.123457 + 9.78279740156158712782730191293012467212975025177002e-03, // 0.200000 + 2.57445069421857186675595130509464070200920104980469e-02, // 0.271828 + 3.52965948694062836565876750682946294546127319335938e-02, // 0.300000 + 4.09502017884842486883378853690373944118618965148926e-02, // 0.314159 + 9.03696114115064008665001438203034922480583190917969e-02, // 0.400000 + 1.93349510480696651359977522588451392948627471923828e-01, // 0.500000 + 3.73544048607389822524993405750137753784656524658203e-01, // 0.600000 + 6.84938469046824760511071872315369546413421630859375e-01, // 0.700000 + 1.25219806739988248622807986976113170385360717773438e+00, // 0.800000 + 2.52142232809988176001070314669050276279449462890625e+00, // 0.900000 + Double.POSITIVE_INFINITY, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_4_1 = { // + 1.00000000000000013320864402483403646202733603332908e-40, // 0.000000 + 1.00000000000000039654310216996797471172433399612973e-20, // 0.000010 + 1.00000000000000018344700758454735023406101390719414e-04, // 0.100000 + 2.32305052922195277983682948352850416995352134108543e-04, // 0.123457 + 1.60000000000000029351521213527576037449762225151062e-03, // 0.200000 + 5.45981500331442059364972507751190278213471174240112e-03, // 0.271828 + 8.09999999999999956146190527306316653266549110412598e-03, // 0.300000 + 9.74090910340024221381050040236004861071705818176270e-03, // 0.314159 + 2.56000000000000046962433941644121659919619560241699e-02, // 0.400000 + 6.25000000000000000000000000000000000000000000000000e-02, // 0.500000 + 1.29599999999999992983390484369010664522647857666016e-01, // 0.600000 + 2.40099999999999952349227783088281285017728805541992e-01, // 0.700000 + 4.09600000000000075139894306630594655871391296386719e-01, // 0.800000 + 6.56100000000000016520118606422329321503639221191406e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_4_1 = { // + 4.00000000000000033334568243034394140372534410747462e-30, // 0.000000 + 4.00000000000000109968250472765612236495730768756629e-15, // 0.000010 + 4.00000000000000095062846483529028773773461580276489e-03, // 0.100000 + 7.52668920916224962192053737908281618729233741760254e-03, // 0.123457 + 3.20000000000000076050277186823223019018769264221191e-02, // 0.200000 + 8.03421476927506372778253762589883990585803985595703e-02, // 0.271828 + 1.07999999999999984900966865097871050238609313964844e-01, // 0.300000 + 1.24025106721199263049015826254617422819137573242188e-01, // 0.314159 + 2.56000000000000060840221749458578415215015411376953e-01, // 0.400000 + 5.00000000000000000000000000000000000000000000000000e-01, // 0.500000 + 8.63999999999999879207734920782968401908874511718750e-01, // 0.600000 + 1.37199999999999966426855735335266217589378356933594e+00, // 0.700000 + 2.04800000000000048672177399566862732172012329101562e+00, // 0.800000 + 2.91600000000000036948222259525209665298461914062500e+00, // 0.900000 + 4.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_4_1 = { // + 1.00000000000000217236629027437295105899424517600020e-40, // 0.000000 + 9.99999999999995882644795012383944111776031062706330e-21, // 0.000010 + 1.00000000000000113212390850936372999058221466839314e-04, // 0.100000 + 2.32305052922195169563465699802407016250072047114372e-04, // 0.123457 + 1.60000000000000094403651562657842077896930277347565e-03, // 0.200000 + 5.45981500332209674503092244890467554796487092971802e-03, // 0.271828 + 8.09999999999999782673842929625607212074100971221924e-03, // 0.300000 + 9.74090910340281480872537400728106149472296237945557e-03, // 0.314159 + 2.56000000000000012267964422107979771681129932403564e-02, // 0.400000 + 6.25000000000000000000000000000000000000000000000000e-02, // 0.500000 + 1.29599999999999965227814868740097153931856155395508e-01, // 0.600000 + 2.40099999999999924593652167459367774426937103271484e-01, // 0.700000 + 4.09600000000000130651045537888421677052974700927734e-01, // 0.800000 + 6.56100000000000127542421068937983363866806030273438e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_4_1 = { // + 4.00000000000002415541957595223414710712826003604885e-30, // 0.000000 + 3.99999999999999084449073685450265201248595901134530e-15, // 0.000010 + 4.00000000000000181799020282369383494369685649871826e-03, // 0.100000 + 7.52668920916224701983532341387217456940561532974243e-03, // 0.123457 + 3.20000000000000214828155264967790571972727775573730e-02, // 0.200000 + 8.03421476928353334168164678885659668594598770141602e-02, // 0.271828 + 1.07999999999999957145391249468957539647817611694336e-01, // 0.300000 + 1.24025106721223812855647850028617540374398231506348e-01, // 0.314159 + 2.56000000000000060840221749458578415215015411376953e-01, // 0.400000 + 5.00000000000000111022302462515654042363166809082031e-01, // 0.500000 + 8.63999999999999879207734920782968401908874511718750e-01, // 0.600000 + 1.37199999999999966426855735335266217589378356933594e+00, // 0.700000 + 2.04800000000000004263256414560601115226745605468750e+00, // 0.800000 + 2.91600000000000036948222259525209665298461914062500e+00, // 0.900000 + 4.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_4_2 = { // + 4.99999999960000092956526294536008149911609020135072e-40, // 0.000000 + 4.99996000000000182972062623064444698658055305533981e-20, // 0.000010 + 4.60000000000000122332699525884436297928914427757263e-04, // 0.100000 + 1.04680680370257796554800400201656884746626019477844e-03, // 0.123457 + 6.72000000000000114602771716931783885229378938674927e-03, // 0.200000 + 2.13625486524690426370920448562173987738788127899170e-02, // 0.271828 + 3.07799999999999983335552400376400328241288661956787e-02, // 0.300000 + 3.64637581255899539955933619239658582955598831176758e-02, // 0.314159 + 8.70400000000000062527760746888816356658935546875000e-02, // 0.400000 + 1.87500000000000000000000000000000000000000000000000e-01, // 0.500000 + 3.36959999999999981756815259359427727758884429931641e-01, // 0.600000 + 5.28219999999999911821646492171566933393478393554688e-01, // 0.700000 + 7.37280000000000157456270244438201189041137695312500e-01, // 0.800000 + 9.18540000000000023128166048991261050105094909667969e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_4_2 = { // + 1.99999999980000012361177456368779558183353543188356e-29, // 0.000000 + 1.99998000000000044302004307985331939009172267512904e-14, // 0.000010 + 1.80000000000000021094237467877974268049001693725586e-02, // 0.100000 + 3.29873449873673424415088106798066291958093643188477e-02, // 0.123457 + 1.28000000000000030420110874729289207607507705688477e-01, // 0.200000 + 2.92514438397464715535534196533262729644775390625000e-01, // 0.271828 + 3.77999999999999891642232796584721654653549194335938e-01, // 0.300000 + 4.25307351537991429335505699782515875995159149169922e-01, // 0.314159 + 7.68000000000000127009514017117908224463462829589844e-01, // 0.400000 + 1.25000000000000000000000000000000000000000000000000e+00, // 0.500000 + 1.72799999999999975841546984156593680381774902343750e+00, // 0.600000 + 2.05799999999999982946974341757595539093017578125000e+00, // 0.700000 + 2.04800000000000004263256414560601115226745605468750e+00, // 0.800000 + 1.45799999999999974065190144756343215703964233398438e+00, // 0.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_4_2 = { // + 4.99999999960002458379395944001149082393223625633561e-40, // 0.000000 + 4.99995999999999340377711953648758986667705424628279e-20, // 0.000010 + 4.60000000000000501803459895810988200537394732236862e-04, // 0.100000 + 1.04680680370257774870756950491568204597570002079010e-03, // 0.123457 + 6.72000000000000461547466912293202767614275217056274e-03, // 0.200000 + 2.13625486524969925017369831721225637011229991912842e-02, // 0.271828 + 3.07799999999999948641082880840258440002799034118652e-02, // 0.300000 + 3.64637581255987802686391319184622261673212051391602e-02, // 0.314159 + 8.70400000000000062527760746888816356658935546875000e-02, // 0.400000 + 1.87499999999999972244424384371086489409208297729492e-01, // 0.500000 + 3.36959999999999926245664028101600706577301025390625e-01, // 0.600000 + 5.28220000000000022843948954687220975756645202636719e-01, // 0.700000 + 7.37280000000000046433967781922547146677970886230469e-01, // 0.800000 + 9.18540000000000023128166048991261050105094909667969e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_4_2 = { // + 1.99999999980001413659641781185850481912936833104487e-29, // 0.000000 + 1.99998000000000801608473320156665134268594938987684e-14, // 0.000010 + 1.80000000000000159872115546022541821002960205078125e-02, // 0.100000 + 3.29873449873673146859331950508931186050176620483398e-02, // 0.123457 + 1.28000000000000058175686490358202718198299407958984e-01, // 0.200000 + 2.92514438397734666263971803346066735684871673583984e-01, // 0.271828 + 3.77999999999999669597627871553413569927215576171875e-01, // 0.300000 + 4.25307351538062705653686634832411073148250579833984e-01, // 0.314159 + 7.67999999999999904964909092086600139737129211425781e-01, // 0.400000 + 1.25000000000000000000000000000000000000000000000000e+00, // 0.500000 + 1.72799999999999998046007476659724488854408264160156e+00, // 0.600000 + 2.05799999999999938538053356751333922147750854492188e+00, // 0.700000 + 2.04799999999999959854335429554339498281478881835938e+00, // 0.800000 + 1.45799999999999929656269159750081598758697509765625e+00, // 0.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_CDF_4_4 = { // + 3.49999999916000016318033258661157178322633534961105e-39, // 0.000000 + 3.49991600069999950401327387562812102476143042072235e-19, // 0.000010 + 2.72800000000000038985481509712371916975826025009155e-03, // 0.100000 + 5.96069507924967562778606477991161227691918611526489e-03, // 0.123457 + 3.33440000000000055790927433463366469368338584899902e-02, // 0.200000 + 9.24732206974749615380204659231822006404399871826172e-02, // 0.271828 + 1.26036000000000009135803225035488139837980270385742e-01, // 0.300000 + 1.45131940494089783921793923582299612462520599365234e-01, // 0.314159 + 2.89791999999999994042099160651559941470623016357422e-01, // 0.400000 + 4.99999999999999888977697537484345957636833190917969e-01, // 0.500000 + 7.10207999999999950446749608090613037347793579101562e-01, // 0.600000 + 8.73963999999999963108621159335598349571228027343750e-01, // 0.700000 + 9.66656000000000070748740199633175507187843322753906e-01, // 0.800000 + 9.97272000000000047315040774265071377158164978027344e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] SCIPY_BETA_PDF_4_4 = { // + 1.39999999958000002624274888250361173924268353172324e-28, // 0.000000 + 1.39995800041999896773103113821248488557241404506293e-13, // 0.000010 + 1.02060000000000025699442574023123597726225852966309e-01, // 0.100000 + 1.77415741797289500114587212920014280825853347778320e-01, // 0.123457 + 5.73440000000000171809233506792224943637847900390625e-01, // 0.200000 + 1.08570810499605374666032275854377076029777526855469e+00, // 0.271828 + 1.29653999999999935965888653299771249294281005859375e+00, // 0.300000 + 1.40038510081151179598180078755831345915794372558594e+00, // 0.314159 + 1.93535999999999996923349954158766195178031921386719e+00, // 0.400000 + 2.18750000000000000000000000000000000000000000000000e+00, // 0.500000 + 1.93535999999999996923349954158766195178031921386719e+00, // 0.600000 + 1.29654000000000024783730623312294483184814453125000e+00, // 0.700000 + 5.73439999999999727720023656729608774185180664062500e-01, // 0.800000 + 1.02059999999999942432715727136383065953850746154785e-01, // 0.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_CDF_4_4 = { // + 3.49999999915999951064988578675911911219692442395630e-39, // 0.000000 + 3.49991600069999998549575997243708428875591606695418e-19, // 0.000010 + 2.72800000000000125721655308552726637572050094604492e-03, // 0.100000 + 5.96069507924966868889216087268323462922126054763794e-03, // 0.123457 + 3.33440000000000194568805511607934022322297096252441e-02, // 0.200000 + 9.24732206975786980018838789874280337244272232055664e-02, // 0.271828 + 1.26035999999999953624651993777661118656396865844727e-01, // 0.300000 + 1.45131940494118955031765949570399243384599685668945e-01, // 0.314159 + 2.89792000000000049553250391909386962652206420898438e-01, // 0.400000 + 4.99999999999999944488848768742172978818416595458984e-01, // 0.500000 + 7.10207999999999950446749608090613037347793579101562e-01, // 0.600000 + 8.73963999999999963108621159335598349571228027343750e-01, // 0.700000 + 9.66655999999999959726437737117521464824676513671875e-01, // 0.800000 + 9.97271999999999936292738311749417334794998168945312e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] GNUR_BETA_PDF_4_4 = { // + 1.39999999957999778416520596279629826127535026785743e-28, // 0.000000 + 1.39995800042000149208592784545026220310382294997886e-13, // 0.000010 + 1.02060000000000108966169420909864129498600959777832e-01, // 0.100000 + 1.77415741797289527870162828548927791416645050048828e-01, // 0.123457 + 5.73439999999999949764628581760916858911514282226562e-01, // 0.200000 + 1.08570810499677206095725523482542484998703002929688e+00, // 0.271828 + 1.29653999999999958170349145802902057766914367675781e+00, // 0.300000 + 1.40038510081166234222393995878519490361213684082031e+00, // 0.314159 + 1.93536000000000041332270939165027812123298645019531e+00, // 0.400000 + 2.18750000000000000000000000000000000000000000000000e+00, // 0.500000 + 1.93536000000000041332270939165027812123298645019531e+00, // 0.600000 + 1.29653999999999958170349145802902057766914367675781e+00, // 0.700000 + 5.73439999999999505675418731698300689458847045898438e-01, // 0.800000 + 1.02060000000000011821654766208666842430830001831055e-01, // 0.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 0.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 0.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 0.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + }; + + public static final double[] P_QUANT = { // + 0.0001, 0.001, 0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99, 0.999, 0.9999 // + }; + + public static final double[] SCIPY_BETA_QUANT_01_01 = { // + 8.86928065555026613565596956864396236737615274357892e-38, // 0.000100 + 8.86928065555024393294670900210330809197198311562804e-28, // 0.001000 + 8.86928065555025006242057470083821192914616784639358e-18, // 0.010000 + 8.86928001193459112153072432249700796091929078102112e-08, // 0.100000 + 8.45255532847129752160864057941580540500581264495850e-04, // 0.250000 + 5.00000000000000777156117237609578296542167663574219e-01, // 0.500000 + 9.99154744467152911013840821397025138139724731445312e-01, // 0.750000 + 9.99999911307199895738051509397337213158607482910156e-01, // 0.900000 + 9.99999999999999888977697537484345957636833190917969e-01, // 0.990000 + 9.99999999999999888977697537484345957636833190917969e-01, // 0.999000 + 9.99999999999999888977697537484345957636833190917969e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_01_01 = { // + 8.86928065555028388448412252463067501937612992138827e-38, // 0.000100 + 8.86928065555027263153925837435692060995384889311041e-28, // 0.001000 + 8.86928065555026392911617428893635393218735445787027e-18, // 0.010000 + 8.86928001193460965037644551031692508047399314818904e-08, // 0.100000 + 8.45255532847132245825860774601778757642023265361786e-04, // 0.250000 + 4.99999999999999500399638918679556809365749359130859e-01, // 0.500000 + 9.99154744467152911013840821397025138139724731445312e-01, // 0.750000 + 9.99999911307199895738051509397337213158607482910156e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.990000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.999000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_01_05 = { // + 3.46456275607431455261002811459588941188867309477607e-40, // 0.000100 + 3.46456275607431613825500469117221110902093083941640e-30, // 0.001000 + 3.46456275607430439357088457229084493496058065920175e-20, // 0.010000 + 3.46456275552870422056197107589232790647582049814446e-10, // 0.100000 + 3.30405955584976185706561242760859187228561495430768e-06, // 0.250000 + 3.37816430372528137687782034959127486217767000198364e-03, // 0.500000 + 1.78799793029011389045024316146736964583396911621094e-01, // 0.750000 + 7.32777808689458409752148781990399584174156188964844e-01, // 0.900000 + 9.96800847460084904483323953172657638788223266601562e-01, // 0.990000 + 9.99967947541768209518409094016533344984054565429688e-01, // 0.999000 + 9.99999679469315005952978481218451634049415588378906e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_01_05 = { // + 3.46456275607432760321896411164494283247689160787118e-40, // 0.000100 + 3.46456275607431613825500469117221110902093083941640e-30, // 0.001000 + 3.46456275607431342136749888745890613485718652604856e-20, // 0.010000 + 3.46456275552870783947715099538840546888884830423194e-10, // 0.100000 + 3.30405955584976524519740144480994814557561767287552e-06, // 0.250000 + 3.37816430372529221889954520463561493670567870140076e-03, // 0.500000 + 1.78799793029011944156536628725007176399230957031250e-01, // 0.750000 + 7.32777808689457965662938931927783414721488952636719e-01, // 0.900000 + 9.96800847460084904483323953172657638788223266601562e-01, // 0.990000 + 9.99967947541768209518409094016533344984054565429688e-01, // 0.999000 + 9.99999679469315005952978481218451634049415588378906e-01, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_01_1 = { // + 1.00000000000000298802934877418851689778100883306864e-40, // 0.000100 + 1.00000000000000218528411709481159173652571096174285e-30, // 0.001000 + 1.00000000000000084793293288572637777171916428947207e-20, // 0.010000 + 9.99999999999999131703402335623722188562290114077769e-11, // 0.100000 + 9.53674316406248835329697525337033781056561565492302e-07, // 0.250000 + 9.76562499999998590537175768844235790311358869075775e-04, // 0.500000 + 5.63135147094726076777426726494013564661145210266113e-02, // 0.750000 + 3.48678440099999320533896707274834625422954559326172e-01, // 0.900000 + 9.04382075008804409144147484767017886042594909667969e-01, // 0.990000 + 9.90044880209748234811684142187004908919334411621094e-01, // 0.999000 + 9.99000449880021124116069586307276040315628051757812e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_01_1 = { // + 1.00000000000000665851311202335856317232144528987664e-40, // 0.000100 + 1.00000000000000673950412615046707223864685665397028e-30, // 0.001000 + 1.00000000000000445905157861179360225167780663621080e-20, // 0.010000 + 1.00000000000000313835949438935152234694814232796034e-10, // 0.100000 + 9.53674316406252541098841762901017204967502038925886e-07, // 0.250000 + 9.76562500000002385244779468109754816396161913871765e-04, // 0.500000 + 5.63135147094727672723024625156540423631668090820312e-02, // 0.750000 + 3.48678440100000486268072563689202070236206054687500e-01, // 0.900000 + 9.04382075008804409144147484767017886042594909667969e-01, // 0.990000 + 9.90044880209748234811684142187004908919334411621094e-01, // 0.999000 + 9.99000449880021124116069586307276040315628051757812e-01, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_01_2 = { // + 3.85543289429533451133658448963742592666926086214652e-41, // 0.000100 + 3.85543289429532603687340095433216245505652506485049e-31, // 0.001000 + 3.85543289429532404962034705548650193823057760610265e-21, // 0.010000 + 3.85543289443045237890743375235041599455310112887219e-11, // 0.100000 + 3.67682855892327505770053293690513562808064307319000e-07, // 0.250000 + 3.76636057551797004145505143313243934244383126497269e-04, // 0.500000 + 2.21534346953469116203727651281951693817973136901855e-02, // 0.750000 + 1.54915841004560961291502962922095321118831634521484e-01, // 0.900000 + 6.27148329248184310813485353719443082809448242187500e-01, // 0.990000 + 8.70585822529799657765181564172962680459022521972656e-01, // 0.999000 + 9.57904427593739837831776640086900442838668823242188e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_01_2 = { // + 3.85543289429538701964597541526447679856717128592763e-41, // 0.000100 + 3.85543289429535537655999775518958492064467519746949e-31, // 0.001000 + 3.85543289429532931583503873933453763817026436176329e-21, // 0.010000 + 3.85543289443044591655889818182170606167269433228739e-11, // 0.100000 + 3.67682855892327611649171700478055946348376892274246e-07, // 0.250000 + 3.76636057551797437826374137515017537225503474473953e-04, // 0.500000 + 2.21534346953469185592666690354235470294952392578125e-02, // 0.750000 + 1.54915841004561238847259119211230427026748657226562e-01, // 0.900000 + 6.27148329248184310813485353719443082809448242187500e-01, // 0.990000 + 8.70585822529799657765181564172962680459022521972656e-01, // 0.999000 + 9.57904427593739837831776640086900442838668823242188e-01, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_01_4 = { // + 1.70520173780031775915342203870768816354885853037205e-41, // 0.000100 + 1.70520173780031788553917314540519037060879260315405e-31, // 0.001000 + 1.70520173780031400068231169360407429564207370948071e-21, // 0.010000 + 1.70520173787961264841636790620568427459535776335997e-11, // 0.100000 + 1.62620782287267823496376044703493057852483616443351e-07, // 0.250000 + 1.66599281457928790261754925516868297563632950186729e-04, // 0.500000 + 9.86339324158605999792737151210531010292470455169678e-03, // 0.750000 + 7.19532648151699621985599719664605800062417984008789e-02, // 0.900000 + 3.58513695489701578189567499066470190882682800292969e-01, // 0.990000 + 6.07588693735668705642183340387418866157531738281250e-01, // 0.999000 + 7.70239957250841489333481604262487962841987609863281e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_01_4 = { // + 1.70520173780032387662636078732443195444958595838538e-41, // 0.000100 + 1.70520173780031000323531131830916642462988659737581e-31, // 0.001000 + 1.70520173780030873446762000975603859570238695382007e-21, // 0.010000 + 1.70520173787960327801099132893905487191876790831202e-11, // 0.100000 + 1.62620782287266764705191976828069222449357766890898e-07, // 0.250000 + 1.66599281457928193950560058489429593464592471718788e-04, // 0.500000 + 9.86339324158602703818132795277051627635955810546875e-03, // 0.750000 + 7.19532648151695042315623140893876552581787109375000e-02, // 0.900000 + 3.58513695489701134100357649003854021430015563964844e-01, // 0.990000 + 6.07588693735668594619880877871764823794364929199219e-01, // 0.999000 + 7.70239957250841489333481604262487962841987609863281e-01, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_05_01 = { // + 3.20530684976785390026245365149848431940426962682977e-07, // 0.000100 + 3.20524582318020690016244222952934705972438678145409e-05, // 0.001000 + 3.19915253991507426631346611145545466570183634757996e-03, // 0.010000 + 2.67222191310541978825909836814389564096927642822266e-01, // 0.100000 + 8.21200206970988388910370758821954950690269470214844e-01, // 0.250000 + 9.96621835696274716021036965685198083519935607910156e-01, // 0.500000 + 9.99996695940444157990611984132556244730949401855469e-01, // 0.750000 + 9.99999999653543691913171187479747459292411804199219e-01, // 0.900000 + 9.99999999999999888977697537484345957636833190917969e-01, // 0.990000 + 9.99999999999999888977697537484345957636833190917969e-01, // 0.999000 + 9.99999999999999888977697537484345957636833190917969e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_05_01 = { // + 3.20530684976785442965804568543619623710583255160600e-07, // 0.000100 + 3.20524582318020825541515783640988956904038786888123e-05, // 0.001000 + 3.19915253991507513367520409985900187166407704353333e-03, // 0.010000 + 2.67222191310542089848212299330043606460094451904297e-01, // 0.100000 + 8.21200206970988055843463371274992823600769042968750e-01, // 0.250000 + 9.96621835696274716021036965685198083519935607910156e-01, // 0.500000 + 9.99996695940444157990611984132556244730949401855469e-01, // 0.750000 + 9.99999999653543691913171187479747459292411804199219e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.990000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.999000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_05_05 = { // + 2.46740107997877850811582013451292438865891654131701e-08, // 0.000100 + 2.46739907091694248724746130196905369302839972078800e-06, // 0.001000 + 2.46719817134221513794251068318885700136888772249222e-04, // 0.010000 + 2.44717418524231997145612638178135966882109642028809e-02, // 0.100000 + 1.46446609406726185875768919686379376798868179321289e-01, // 0.250000 + 4.99999999999999666933092612453037872910499572753906e-01, // 0.500000 + 8.53553390593273841879806695942534133791923522949219e-01, // 0.750000 + 9.75528258147576821102120447903871536254882812500000e-01, // 0.900000 + 9.99753280182865800007618872768944129347801208496094e-01, // 0.990000 + 9.99997532600929051049831741693196818232536315917969e-01, // 0.999000 + 9.99999975325989165497730937204323709011077880859375e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_05_05 = { // + 2.46740107997877950073255519814613423434934702527244e-08, // 0.000100 + 2.46739907091694375779688218341956229551215074025095e-06, // 0.001000 + 2.46719817134221622214468316869329100882168859243393e-04, // 0.010000 + 2.44717418524232274701368794467271072790026664733887e-02, // 0.100000 + 1.46446609406726185875768919686379376798868179321289e-01, // 0.250000 + 4.99999999999999167332731531132594682276248931884766e-01, // 0.500000 + 8.53553390593273841879806695942534133791923522949219e-01, // 0.750000 + 9.75528258147576821102120447903871536254882812500000e-01, // 0.900000 + 9.99753280182865800007618872768944129347801208496094e-01, // 0.990000 + 9.99997532600929051049831741693196818232536315917969e-01, // 0.999000 + 9.99999975325989165497730937204323709011077880859375e-01, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_05_1 = { // + 1.00000000000000018635868334073400764960837250328041e-08, // 0.000100 + 1.00000000000000016650634863946134345269456389360130e-06, // 0.001000 + 1.00000000000000004792173602385929598312941379845142e-04, // 0.010000 + 1.00000000000000002081668171172168513294309377670288e-02, // 0.100000 + 6.25000000000000000000000000000000000000000000000000e-02, // 0.250000 + 2.50000000000000000000000000000000000000000000000000e-01, // 0.500000 + 5.62500000000000000000000000000000000000000000000000e-01, // 0.750000 + 8.10000000000000053290705182007513940334320068359375e-01, // 0.900000 + 9.80099999999999971223019201715942472219467163085938e-01, // 0.990000 + 9.98001000000000026979307676811004057526588439941406e-01, // 0.999000 + 9.99800010000000072274417561857262626290321350097656e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_05_1 = { // + 1.00000000000000018635868334073400764960837250328041e-08, // 0.000100 + 1.00000000000000037826458545303642821977518906351179e-06, // 0.001000 + 1.00000000000000086107336538798762148871901445090771e-04, // 0.010000 + 1.00000000000000036776137690708310401532799005508423e-02, // 0.100000 + 6.25000000000000277555756156289135105907917022705078e-02, // 0.250000 + 2.50000000000000000000000000000000000000000000000000e-01, // 0.500000 + 5.62499999999999777955395074968691915273666381835938e-01, // 0.750000 + 8.10000000000000053290705182007513940334320068359375e-01, // 0.900000 + 9.80099999999999971223019201715942472219467163085938e-01, // 0.990000 + 9.98001000000000026979307676811004057526588439941406e-01, // 0.999000 + 9.99800010000000072274417561857262626290321350097656e-01, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_05_2 = { // + 4.44444445761316705817644639965255382030306918750284e-09, // 0.000100 + 4.44444576131755449727476570478956752197063906351104e-07, // 0.001000 + 4.44457613851590344110700780966283218731405213475227e-05, // 0.010000 + 4.45768188762137376746075290157023118808865547180176e-03, // 0.100000 + 2.83095437186913906346674707492638844996690750122070e-02, // 0.250000 + 1.20614758428183241334430420010903617367148399353027e-01, // 0.500000 + 3.11224179038489423554381119174649938941001892089844e-01, // 0.750000 + 5.31877433473580407685687987395795062184333801269531e-01, // 0.900000 + 8.41255287053071842962026494205929338932037353515625e-01, // 0.990000 + 9.48808055284168672116607012867461889982223510742188e-01, // 0.999000 + 9.83714619081483720997027830890147015452384948730469e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_05_2 = { // + 4.44444445761315465046725810423743074917268813806004e-09, // 0.000100 + 4.44444576131755290908798960297643176886595028918236e-07, // 0.001000 + 4.44457613851590750686515463030445971526205539703369e-05, // 0.010000 + 4.45768188762137550218422887837732560001313686370850e-03, // 0.100000 + 2.83095437186913941041144227028780733235180377960205e-02, // 0.250000 + 1.20614758428183130312127957495249575003981590270996e-01, // 0.500000 + 3.11224179038489201509776194143341854214668273925781e-01, // 0.750000 + 5.31877433473580518707990449911449104547500610351562e-01, // 0.900000 + 8.41255287053071842962026494205929338932037353515625e-01, // 0.990000 + 9.48808055284168672116607012867461889982223510742188e-01, // 0.999000 + 9.83714619081483720997027830890147015452384948730469e-01, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_05_4 = { // + 2.08979592710184102207603484900426404902162857979420e-09, // 0.000100 + 2.08979679181727248935671503900224355021464361925609e-07, // 0.001000 + 2.08988326860078756129837640465041204151930287480354e-05, // 0.010000 + 2.09858372142358862166844346575089730322360992431641e-03, // 0.100000 + 1.34159346080160047504081788360963400918990373611450e-02, // 0.250000 + 5.87108013397836248081773646845249459147453308105469e-02, // 0.500000 + 1.61283978400102917527547674581001047044992446899414e-01, // 0.750000 + 3.01792928793774728468690682348096743226051330566406e-01, // 0.900000 + 5.84601685951977589894568154704757034778594970703125e-01, // 0.990000 + 7.60584846730546160209485151426633819937705993652344e-01, // 0.999000 + 8.63700924381154200304422374756541103124618530273438e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_05_4 = { // + 2.08979592710183854053419718992123943479555236990564e-09, // 0.000100 + 2.08979679181727301875230707293995546791620654403232e-07, // 0.001000 + 2.08988326860079027180380761841149706015130504965782e-05, // 0.010000 + 2.09858372142358948903018145415444450918585062026978e-03, // 0.100000 + 1.34159346080160134240255587201318121515214443206787e-02, // 0.250000 + 5.87108013397836039914956529628398129716515541076660e-02, // 0.500000 + 1.61283978400102778749669596436433494091033935546875e-01, // 0.750000 + 3.01792928793774506424085757316788658499717712402344e-01, // 0.900000 + 5.84601685951977145805358304642140865325927734375000e-01, // 0.990000 + 7.60584846730546049187182688910979777574539184570312e-01, // 0.999000 + 8.63700924381154200304422374756541103124618530273438e-01, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_1_01 = { // + 9.99550119979002518744159999641851754859089851379395e-04, // 0.000100 + 9.95511979025178947444452148829441284760832786560059e-03, // 0.001000 + 9.56179249911954937113378605317848268896341323852539e-02, // 0.010000 + 6.51321559899999957821137286373414099216461181640625e-01, // 0.100000 + 9.43686485290527454772302462515654042363166809082031e-01, // 0.250000 + 9.99023437500000000000000000000000000000000000000000e-01, // 0.500000 + 9.99999046325683593750000000000000000000000000000000e-01, // 0.750000 + 9.99999999899999991725962900090962648391723632812500e-01, // 0.900000 + 9.99999999999999888977697537484345957636833190917969e-01, // 0.990000 + 9.99999999999999888977697537484345957636833190917969e-01, // 0.999000 + 9.99999999999999888977697537484345957636833190917969e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_1_01 = { // + 9.99550119979002735584594496742738556349650025367737e-04, // 0.000100 + 9.95511979025179120916799746510150725953280925750732e-03, // 0.001000 + 9.56179249911955353447012839751550927758216857910156e-02, // 0.010000 + 6.51321559899999735776532361342106014490127563476562e-01, // 0.100000 + 9.43686485290527232727697537484345957636833190917969e-01, // 0.250000 + 9.99023437500000000000000000000000000000000000000000e-01, // 0.500000 + 9.99999046325683593750000000000000000000000000000000e-01, // 0.750000 + 9.99999999899999991725962900090962648391723632812500e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.990000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.999000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_1_05 = { // + 1.99989999999999987657234301607900306407827883958817e-04, // 0.000100 + 1.99899999999999990890620082950590585824102163314819e-03, // 0.001000 + 1.99000000000000010214051826551440171897411346435547e-02, // 0.010000 + 1.90000000000000002220446049250313080847263336181641e-01, // 0.100000 + 4.37500000000000055511151231257827021181583404541016e-01, // 0.250000 + 7.50000000000000111022302462515654042363166809082031e-01, // 0.500000 + 9.37500000000000000000000000000000000000000000000000e-01, // 0.750000 + 9.89999999999999991118215802998747676610946655273438e-01, // 0.900000 + 9.99900000000000011013412404281552881002426147460938e-01, // 0.990000 + 9.99998999999999971244335483788745477795600891113281e-01, // 0.999000 + 9.99999989999999949752407246705843135714530944824219e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_1_05 = { // + 1.99989999999999987657234301607900306407827883958817e-04, // 0.000100 + 1.99899999999999990890620082950590585824102163314819e-03, // 0.001000 + 1.99000000000000010214051826551440171897411346435547e-02, // 0.010000 + 1.90000000000000057731597280508140102028846740722656e-01, // 0.100000 + 4.37500000000000166533453693773481063544750213623047e-01, // 0.250000 + 7.50000000000000000000000000000000000000000000000000e-01, // 0.500000 + 9.37500000000000000000000000000000000000000000000000e-01, // 0.750000 + 9.89999999999999991118215802998747676610946655273438e-01, // 0.900000 + 9.99900000000000011013412404281552881002426147460938e-01, // 0.990000 + 9.99998999999999971244335483788745477795600891113281e-01, // 0.999000 + 9.99999989999999949752407246705843135714530944824219e-01, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_1_1 = { // + 1.00000000000000004792173602385929598312941379845142e-04, // 0.000100 + 1.00000000000000002081668171172168513294309377670288e-03, // 0.001000 + 1.00000000000000002081668171172168513294309377670288e-02, // 0.010000 + 1.00000000000000005551115123125782702118158340454102e-01, // 0.100000 + 2.50000000000000000000000000000000000000000000000000e-01, // 0.250000 + 5.00000000000000000000000000000000000000000000000000e-01, // 0.500000 + 7.50000000000000000000000000000000000000000000000000e-01, // 0.750000 + 9.00000000000000022204460492503130808472633361816406e-01, // 0.900000 + 9.89999999999999991118215802998747676610946655273438e-01, // 0.990000 + 9.98999999999999999111821580299874767661094665527344e-01, // 0.999000 + 9.99900000000000011013412404281552881002426147460938e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_1_1 = { // + 1.00000000000000086107336538798762148871901445090771e-04, // 0.000100 + 1.00000000000000023765711620882257193443365395069122e-03, // 0.001000 + 1.00000000000000036776137690708310401532799005508423e-02, // 0.010000 + 1.00000000000000019428902930940239457413554191589355e-01, // 0.100000 + 2.50000000000000000000000000000000000000000000000000e-01, // 0.250000 + 5.00000000000000000000000000000000000000000000000000e-01, // 0.500000 + 7.50000000000000000000000000000000000000000000000000e-01, // 0.750000 + 9.00000000000000022204460492503130808472633361816406e-01, // 0.900000 + 9.89999999999999991118215802998747676610946655273438e-01, // 0.990000 + 9.98999999999999999111821580299874767661094665527344e-01, // 0.999000 + 9.99900000000000011013412404281552881002426147460938e-01, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_1_2 = { // + 5.00012500625039114031909759372496182550094090402126e-05, // 0.000100 + 5.00125062539089927649005318244235240854322910308838e-04, // 0.001000 + 5.01256289338004520916847184253128943964838981628418e-03, // 0.010000 + 5.13167019494862042261473789039882831275463104248047e-02, // 0.100000 + 1.33974596215561375656122322652663569897413253784180e-01, // 0.250000 + 2.92893218813452482773840301888412795960903167724609e-01, // 0.500000 + 5.00000000000000000000000000000000000000000000000000e-01, // 0.750000 + 6.83772233983162003312372689833864569664001464843750e-01, // 0.900000 + 8.99999999999999911182158029987476766109466552734375e-01, // 0.990000 + 9.68377223398316222535697761486517265439033508300781e-01, // 0.999000 + 9.90000000000000546229728115577017888426780700683594e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_1_2 = { // + 5.00012500625039249557181320060550433481694199144840e-05, // 0.000100 + 5.00125062539089710808570821143348439363762736320496e-04, // 0.001000 + 5.01256289338004520916847184253128943964838981628418e-03, // 0.010000 + 5.13167019494862042261473789039882831275463104248047e-02, // 0.100000 + 1.33974596215561375656122322652663569897413253784180e-01, // 0.250000 + 2.92893218813452371751537839372758753597736358642578e-01, // 0.500000 + 5.00000000000000000000000000000000000000000000000000e-01, // 0.750000 + 6.83772233983162114334675152349518612027168273925781e-01, // 0.900000 + 8.99999999999999911182158029987476766109466552734375e-01, // 0.990000 + 9.68377223398316222535697761486517265439033508300781e-01, // 0.999000 + 9.90000000000000546229728115577017888426780700683594e-01, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_1_4 = { // + 2.50009375546912596263108424787802164246386382728815e-05, // 0.000100 + 2.50093804725125897110704631387534391251392662525177e-04, // 0.001000 + 2.50943006631889524785106182491745130391791462898254e-03, // 0.010000 + 2.59962535747032368049769246454161475412547588348389e-02, // 0.100000 + 6.93951408979004008559243743547995109111070632934570e-02, // 0.250000 + 1.59103584746285470208349011045356746762990951538086e-01, // 0.500000 + 2.92893218813452482773840301888412795960903167724609e-01, // 0.750000 + 4.37658674809650927173265699821058660745620727539062e-01, // 0.900000 + 6.83772233983162003312372689833864569664001464843750e-01, // 0.990000 + 8.22172058996107679718079452868551015853881835937500e-01, // 0.999000 + 9.00000000000002797762022055394481867551803588867188e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_1_4 = { // + 2.50009375546912697907062095303842852445086464285851e-05, // 0.000100 + 2.50093804725125951320813255662756091624032706022263e-04, // 0.001000 + 2.50943006631889611521279981332099850988015532493591e-03, // 0.010000 + 2.59962535747032368049769246454161475412547588348389e-02, // 0.100000 + 6.93951408979004286114999899837130215018987655639648e-02, // 0.250000 + 1.59103584746285470208349011045356746762990951538086e-01, // 0.500000 + 2.92893218813452427262689070630585774779319763183594e-01, // 0.750000 + 4.37658674809650927173265699821058660745620727539062e-01, // 0.900000 + 6.83772233983162003312372689833864569664001464843750e-01, // 0.990000 + 8.22172058996107679718079452868551015853881835937500e-01, // 0.999000 + 9.00000000000002686739719592878827825188636779785156e-01, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_2_01 = { // + 4.20955724062624797587872649273776914924383163452148e-02, // 0.000100 + 1.29414177470200258968091588940296787768602371215820e-01, // 0.001000 + 3.72851670751815522653060952507075853645801544189453e-01, // 0.010000 + 8.45084158995438761152740880788769572973251342773438e-01, // 0.100000 + 9.77846565304653081440733330964576452970504760742188e-01, // 0.250000 + 9.99623363942448217578373714786721393465995788574219e-01, // 0.500000 + 9.99999632317144060422720031056087464094161987304688e-01, // 0.750000 + 9.99999999961445618090749576367670670151710510253906e-01, // 0.900000 + 9.99999999999999888977697537484345957636833190917969e-01, // 0.990000 + 9.99999999999999888977697537484345957636833190917969e-01, // 0.999000 + 9.99999999999999888977697537484345957636833190917969e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_2_01 = { // + 4.20955724062624728198933610201493138447403907775879e-02, // 0.000100 + 1.29414177470200231212515973311383277177810668945312e-01, // 0.001000 + 3.72851670751815578164212183764902874827384948730469e-01, // 0.010000 + 8.45084158995438761152740880788769572973251342773438e-01, // 0.100000 + 9.77846565304653081440733330964576452970504760742188e-01, // 0.250000 + 9.99623363942448217578373714786721393465995788574219e-01, // 0.500000 + 9.99999632317144060422720031056087464094161987304688e-01, // 0.750000 + 9.99999999961445618090749576367670670151710510253906e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.990000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.999000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_2_05 = { // + 1.62853809185172157536491965856839669868350028991699e-02, // 0.000100 + 5.11919447158312862500295636891678441315889358520508e-02, // 0.001000 + 1.58744712946928157037973505794070661067962646484375e-01, // 0.010000 + 4.68122566526419536803160781346377916634082794189453e-01, // 0.100000 + 6.88775820961510354401013955794041976332664489746094e-01, // 0.250000 + 8.79385241571816744787781772174639627337455749511719e-01, // 0.500000 + 9.71690456281308567731969105807365849614143371582031e-01, // 0.750000 + 9.95542318112378588068622775608673691749572753906250e-01, // 0.900000 + 9.99955554238614818807207029749406501650810241699219e-01, // 0.990000 + 9.99999555555423857988728286727564409375190734863281e-01, // 0.999000 + 9.99999995555555520887480724923079833388328552246094e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_2_05 = { // + 1.62853809185172088147552926784555893391370773315430e-02, // 0.000100 + 5.11919447158312793111356597819394664838910102844238e-02, // 0.001000 + 1.58744712946928129282397890165157150477170944213867e-01, // 0.010000 + 4.68122566526419647825463243862031958997249603271484e-01, // 0.100000 + 6.88775820961510798490223805856658145785331726074219e-01, // 0.250000 + 8.79385241571817299899294084752909839153289794921875e-01, // 0.500000 + 9.71690456281308567731969105807365849614143371582031e-01, // 0.750000 + 9.95542318112378588068622775608673691749572753906250e-01, // 0.900000 + 9.99955554238614818807207029749406501650810241699219e-01, // 0.990000 + 9.99999555555423857988728286727564409375190734863281e-01, // 0.999000 + 9.99999995555555520887480724923079833388328552246094e-01, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_2_1 = { // + 1.00000000000000002081668171172168513294309377670288e-02, // 0.000100 + 3.16227766016837913420900463279394898563623428344727e-02, // 0.001000 + 9.99999999999999916733273153113259468227624893188477e-02, // 0.010000 + 3.16227766016837941176476078908308409154415130615234e-01, // 0.100000 + 5.00000000000000000000000000000000000000000000000000e-01, // 0.250000 + 7.07106781186547461715008466853760182857513427734375e-01, // 0.500000 + 8.66025403784438596588302061718422919511795043945312e-01, // 0.750000 + 9.48683298050513768018277005467098206281661987304688e-01, // 0.900000 + 9.94987437106619965199172384018311277031898498535156e-01, // 0.990000 + 9.99499874937460952573076156113529577851295471191406e-01, // 0.999000 + 9.99949998749937507547258519480237737298011779785156e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_2_1 = { // + 9.99999999999999500399638918679556809365749359130859e-03, // 0.000100 + 3.16227766016837982809839502351678675040602684020996e-02, // 0.001000 + 1.00000000000000019428902930940239457413554191589355e-01, // 0.010000 + 3.16227766016837885665324847650481387972831726074219e-01, // 0.100000 + 5.00000000000000000000000000000000000000000000000000e-01, // 0.250000 + 7.07106781186547572737310929369414225220680236816406e-01, // 0.500000 + 8.66025403784438596588302061718422919511795043945312e-01, // 0.750000 + 9.48683298050513768018277005467098206281661987304688e-01, // 0.900000 + 9.94987437106619965199172384018311277031898498535156e-01, // 0.990000 + 9.99499874937460952573076156113529577851295471191406e-01, // 0.999000 + 9.99949998749937507547258519480237737298011779785156e-01, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_2_2 = { // + 5.78466759288776954245347994287840265315026044845581e-03, // 0.000100 + 1.83702538588116116857751336510773398913443088531494e-02, // 0.001000 + 5.89031357781952535912495250158826820552349090576172e-02, // 0.010000 + 1.95800105659091727705600760600646026432514190673828e-01, // 0.100000 + 3.26351822333069641057790022387052886188030242919922e-01, // 0.250000 + 5.00000000000000000000000000000000000000000000000000e-01, // 0.500000 + 6.73648177666930303431058746355120092630386352539062e-01, // 0.750000 + 8.04199894340908327805550470657180994749069213867188e-01, // 0.900000 + 9.41096864221804718653174859355203807353973388671875e-01, // 0.990000 + 9.81629746141188364028096202673623338341712951660156e-01, // 0.999000 + 9.94215332407112573065433025476522743701934814453125e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_2_2 = { // + 5.78466759288776954245347994287840265315026044845581e-03, // 0.000100 + 1.83702538588116116857751336510773398913443088531494e-02, // 0.001000 + 5.89031357781952674690373328303394373506307601928711e-02, // 0.010000 + 1.95800105659091699950025144971732515841722488403320e-01, // 0.100000 + 3.26351822333069696568941253644879907369613647460938e-01, // 0.250000 + 5.00000000000000000000000000000000000000000000000000e-01, // 0.500000 + 6.73648177666930303431058746355120092630386352539062e-01, // 0.750000 + 8.04199894340908327805550470657180994749069213867188e-01, // 0.900000 + 9.41096864221804718653174859355203807353973388671875e-01, // 0.990000 + 9.81629746141188364028096202673623338341712951660156e-01, // 0.999000 + 9.94215332407112573065433025476522743701934814453125e-01, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_2_4 = { // + 3.17233337277439495643927003243334183935075998306274e-03, // 0.000100 + 1.01017878837377523409957191802277520764619112014771e-02, // 0.001000 + 3.26821122006028808026911747219855897128582000732422e-02, // 0.010000 + 1.12234958545858548584206459963752422481775283813477e-01, // 0.100000 + 1.93763609553971150045725835298071615397930145263672e-01, // 0.250000 + 3.13810170455697423630425646479125134646892547607422e-01, // 0.500000 + 4.54180564773615791374083983100717887282371520996094e-01, // 0.750000 + 5.83890374619528973454407605458982288837432861328125e-01, // 0.900000 + 7.77927716615001818567520786018576472997665405273438e-01, // 0.990000 + 8.77986170388521558471950356761226430535316467285156e-01, // 0.999000 + 9.32186920845740840668725013529183343052864074707031e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_2_4 = { // + 3.17233337277439452275840103823156823636963963508606e-03, // 0.000100 + 1.01017878837377488715487672266135632526129484176636e-02, // 0.001000 + 3.26821122006028738637972708147572120651602745056152e-02, // 0.010000 + 1.12234958545858534706418652149295667186379432678223e-01, // 0.100000 + 1.93763609553971177801301450926985125988721847534180e-01, // 0.250000 + 3.13810170455697368119274415221298113465309143066406e-01, // 0.500000 + 4.54180564773615680351781520585063844919204711914062e-01, // 0.750000 + 5.83890374619528973454407605458982288837432861328125e-01, // 0.900000 + 7.77927716615001929589823248534230515360832214355469e-01, // 0.990000 + 8.77986170388521558471950356761226430535316467285156e-01, // 0.999000 + 9.32186920845740840668725013529183343052864074707031e-01, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_4_01 = { // + 2.29760042749164533626426987211743835359811782836914e-01, // 0.000100 + 3.92411306264331405380119122128235176205635070800781e-01, // 0.001000 + 6.41486304510298754877339888480491936206817626953125e-01, // 0.010000 + 9.28046735184830051679227835847996175289154052734375e-01, // 0.100000 + 9.90136606758413972961818672047229483723640441894531e-01, // 0.250000 + 9.99833400718542097962426851154305040836334228515625e-01, // 0.500000 + 9.99999837379217715316315207019215449690818786621094e-01, // 0.750000 + 9.99999999982947973542479758179979398846626281738281e-01, // 0.900000 + 9.99999999999999888977697537484345957636833190917969e-01, // 0.990000 + 9.99999999999999888977697537484345957636833190917969e-01, // 0.999000 + 9.99999999999999888977697537484345957636833190917969e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_4_01 = { // + 2.29760042749164533626426987211743835359811782836914e-01, // 0.000100 + 3.92411306264331349868967890870408155024051666259766e-01, // 0.001000 + 6.41486304510298754877339888480491936206817626953125e-01, // 0.010000 + 9.28046735184830495768437685910612344741821289062500e-01, // 0.100000 + 9.90136606758413972961818672047229483723640441894531e-01, // 0.250000 + 9.99833400718542097962426851154305040836334228515625e-01, // 0.500000 + 9.99999837379217715316315207019215449690818786621094e-01, // 0.750000 + 9.99999999982947973542479758179979398846626281738281e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.990000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.999000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_4_05 = { // + 1.36299075618849435675983272631128784269094467163086e-01, // 0.000100 + 2.39415153269453812034939232944452669471502304077148e-01, // 0.001000 + 4.15398314048022576638885539068724028766155242919922e-01, // 0.010000 + 6.98207071206225382553611780167557299137115478515625e-01, // 0.100000 + 8.38716021599897110228027941047912463545799255371094e-01, // 0.250000 + 9.41289198660216319680671404057648032903671264648438e-01, // 0.500000 + 9.86584065391983955350951873697340488433837890625000e-01, // 0.750000 + 9.97901416278576425256119364348705857992172241210938e-01, // 0.900000 + 9.99979101167314032849731120222713798284530639648438e-01, // 0.990000 + 9.99999791020320771828266970260301604866981506347656e-01, // 0.999000 + 9.99999997910204108109155640704557299613952636718750e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_4_05 = { // + 1.36299075618849463431558888260042294859886169433594e-01, // 0.000100 + 2.39415153269453950812817311089020222425460815429688e-01, // 0.001000 + 4.15398314048022687661188001584378071129322052001953e-01, // 0.010000 + 6.98207071206225493575914242683211341500282287597656e-01, // 0.100000 + 8.38716021599897221250330403563566505908966064453125e-01, // 0.250000 + 9.41289198660216541725276329088956117630004882812500e-01, // 0.500000 + 9.86584065391983955350951873697340488433837890625000e-01, // 0.750000 + 9.97901416278576425256119364348705857992172241210938e-01, // 0.900000 + 9.99979101167314032849731120222713798284530639648438e-01, // 0.990000 + 9.99999791020320771828266970260301604866981506347656e-01, // 0.999000 + 9.99999997910204108109155640704557299613952636718750e-01, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_4_1 = { // + 1.00000000000000005551115123125782702118158340454102e-01, // 0.000100 + 1.77827941003892292526344931502535473555326461791992e-01, // 0.001000 + 3.16227766016837941176476078908308409154415130615234e-01, // 0.010000 + 5.62341325190349072826734300178941339254379272460938e-01, // 0.100000 + 7.07106781186547572737310929369414225220680236816406e-01, // 0.250000 + 8.40896415253714613058377835841383785009384155273438e-01, // 0.500000 + 9.30604859102099557510712202201830223202705383300781e-01, // 0.750000 + 9.74003746425296745847788315586512908339500427246094e-01, // 0.900000 + 9.97490569933681126002511518890969455242156982421875e-01, // 0.990000 + 9.99749906195274928855099005886586382985115051269531e-01, // 0.999000 + 9.99974999062445357722594962979201227426528930664062e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_4_1 = { // + 1.00000000000000019428902930940239457413554191589355e-01, // 0.000100 + 1.77827941003892264770769315873621962964534759521484e-01, // 0.001000 + 3.16227766016837941176476078908308409154415130615234e-01, // 0.010000 + 5.62341325190349072826734300178941339254379272460938e-01, // 0.100000 + 7.07106781186547572737310929369414225220680236816406e-01, // 0.250000 + 8.40896415253714502036075373325729742646217346191406e-01, // 0.500000 + 9.30604859102099557510712202201830223202705383300781e-01, // 0.750000 + 9.74003746425296745847788315586512908339500427246094e-01, // 0.900000 + 9.97490569933681126002511518890969455242156982421875e-01, // 0.990000 + 9.99749906195274928855099005886586382985115051269531e-01, // 0.999000 + 9.99974999062445357722594962979201227426528930664062e-01, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_4_2 = { // + 6.78130791542610467104168492369353771209716796875000e-02, // 0.000100 + 1.22013829611478455405837451053230324760079383850098e-01, // 0.001000 + 2.22072283384998070410176751465769484639167785644531e-01, // 0.010000 + 4.16109625380471082056743625798844732344150543212891e-01, // 0.100000 + 5.45819435226384208625916016899282112717628479003906e-01, // 0.250000 + 6.86189829544302631880725584778701886534690856933594e-01, // 0.500000 + 8.06236390446028794443122933444101363420486450195312e-01, // 0.750000 + 8.87765041454141479171369155665161088109016418457031e-01, // 0.900000 + 9.67317887799397091441733209649100899696350097656250e-01, // 0.990000 + 9.89898212116262277149303372425492852926254272460938e-01, // 0.999000 + 9.96827666627225772444376161729451268911361694335938e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_4_2 = { // + 6.78130791542610883437802726803056430071592330932617e-02, // 0.000100 + 1.22013829611478469283625258867687080055475234985352e-01, // 0.001000 + 2.22072283384998042654601135836855974048376083374023e-01, // 0.010000 + 4.16109625380471026545592394541017711162567138671875e-01, // 0.100000 + 5.45819435226384319648218479414936155080795288085938e-01, // 0.250000 + 6.86189829544302631880725584778701886534690856933594e-01, // 0.500000 + 8.06236390446028794443122933444101363420486450195312e-01, // 0.750000 + 8.87765041454141479171369155665161088109016418457031e-01, // 0.900000 + 9.67317887799397091441733209649100899696350097656250e-01, // 0.990000 + 9.89898212116262277149303372425492852926254272460938e-01, // 0.999000 + 9.96827666627225772444376161729451268911361694335938e-01, // 0.999900 + }; + + public static final double[] SCIPY_BETA_QUANT_4_4 = { // + 4.21840560047029092261006155695213237777352333068848e-02, // 0.000100 + 7.66545428268873701993513236629951279610395431518555e-02, // 0.001000 + 1.42270377006857268176176489760109689086675643920898e-01, // 0.010000 + 2.78602048581713057640030228867544792592525482177734e-01, // 0.100000 + 3.78848440641707306930641152575844898819923400878906e-01, // 0.250000 + 5.00000000000000000000000000000000000000000000000000e-01, // 0.500000 + 6.21151559358292693069358847424155101180076599121094e-01, // 0.750000 + 7.21397951418286886848818539874628186225891113281250e-01, // 0.900000 + 8.57729622993142704068247894610976800322532653808594e-01, // 0.990000 + 9.23345457173112560411709637264721095561981201171875e-01, // 0.999000 + 9.57815943995298235691393529123160988092422485351562e-01, // 0.999900 + }; + + public static final double[] GNUR_BETA_QUANT_4_4 = { // + 4.21840560047029092261006155695213237777352333068848e-02, // 0.000100 + 7.66545428268873701993513236629951279610395431518555e-02, // 0.001000 + 1.42270377006857268176176489760109689086675643920898e-01, // 0.010000 + 2.78602048581713057640030228867544792592525482177734e-01, // 0.100000 + 3.78848440641707306930641152575844898819923400878906e-01, // 0.250000 + 5.00000000000000111022302462515654042363166809082031e-01, // 0.500000 + 6.21151559358292693069358847424155101180076599121094e-01, // 0.750000 + 7.21397951418286886848818539874628186225891113281250e-01, // 0.900000 + 8.57729622993142704068247894610976800322532653808594e-01, // 0.990000 + 9.23345457173112560411709637264721095561981201171875e-01, // 0.999000 + 9.57815943995298235691393529123160988092422485351562e-01, // 0.999900 + }; + + @Test + public void testPDF() { + checkPDF(new BetaDistribution(1., 1.), P_CDFPDF, SCIPY_BETA_PDF_1_1, 1e-15); + checkPDF(new BetaDistribution(2., 1.), P_CDFPDF, SCIPY_BETA_PDF_2_1, 1e-12); + checkPDF(new BetaDistribution(4., 1.), P_CDFPDF, SCIPY_BETA_PDF_4_1, 1e-11); + checkPDF(new BetaDistribution(.1, 1.), P_CDFPDF, SCIPY_BETA_PDF_01_1, 1e-12); + checkPDF(new BetaDistribution(.5, 1.), P_CDFPDF, SCIPY_BETA_PDF_05_1, 1e-12); + checkPDF(new BetaDistribution(1., 2.), P_CDFPDF, SCIPY_BETA_PDF_1_2, 1e-12); + checkPDF(new BetaDistribution(2., 2.), P_CDFPDF, SCIPY_BETA_PDF_2_2, 1e-12); + checkPDF(new BetaDistribution(4., 2.), P_CDFPDF, SCIPY_BETA_PDF_4_2, 1e-12); + checkPDF(new BetaDistribution(.1, 2.), P_CDFPDF, SCIPY_BETA_PDF_01_2, 1e-12); + checkPDF(new BetaDistribution(.5, 2.), P_CDFPDF, SCIPY_BETA_PDF_05_2, 1e-12); + checkPDF(new BetaDistribution(1., 4.), P_CDFPDF, SCIPY_BETA_PDF_1_4, 1e-12); + checkPDF(new BetaDistribution(2., 4.), P_CDFPDF, SCIPY_BETA_PDF_2_4, 1e-13); + checkPDF(new BetaDistribution(4., 4.), P_CDFPDF, SCIPY_BETA_PDF_4_4, 1e-12); + checkPDF(new BetaDistribution(.1, 4.), P_CDFPDF, SCIPY_BETA_PDF_01_4, 1e-12); + checkPDF(new BetaDistribution(.5, 4.), P_CDFPDF, SCIPY_BETA_PDF_05_4, 1e-12); + checkPDF(new BetaDistribution(1., .1), P_CDFPDF, SCIPY_BETA_PDF_1_01, 1e-12); + checkPDF(new BetaDistribution(2., .1), P_CDFPDF, SCIPY_BETA_PDF_2_01, 1e-12); + checkPDF(new BetaDistribution(4., .1), P_CDFPDF, SCIPY_BETA_PDF_4_01, 1e-11); + checkPDF(new BetaDistribution(.1, .1), P_CDFPDF, SCIPY_BETA_PDF_01_01, 1e-12); + checkPDF(new BetaDistribution(.5, .1), P_CDFPDF, SCIPY_BETA_PDF_05_01, 1e-13); + checkPDF(new BetaDistribution(1., .5), P_CDFPDF, SCIPY_BETA_PDF_1_05, 1e-13); + checkPDF(new BetaDistribution(2., .5), P_CDFPDF, SCIPY_BETA_PDF_2_05, 1e-12); + checkPDF(new BetaDistribution(4., .5), P_CDFPDF, SCIPY_BETA_PDF_4_05, 1e-11); + checkPDF(new BetaDistribution(.1, .5), P_CDFPDF, SCIPY_BETA_PDF_01_05, 1e-12); + checkPDF(new BetaDistribution(.5, .5), P_CDFPDF, SCIPY_BETA_PDF_05_05, 1e-12); + + checkPDF(new BetaDistribution(1., 1.), P_CDFPDF, GNUR_BETA_PDF_1_1, 1e-15); + checkPDF(new BetaDistribution(2., 1.), P_CDFPDF, GNUR_BETA_PDF_2_1, 1e-14); + checkPDF(new BetaDistribution(4., 1.), P_CDFPDF, GNUR_BETA_PDF_4_1, 1e-14); + checkPDF(new BetaDistribution(.1, 1.), P_CDFPDF, GNUR_BETA_PDF_01_1, 1e-14); + checkPDF(new BetaDistribution(.5, 1.), P_CDFPDF, GNUR_BETA_PDF_05_1, 1e-14); + checkPDF(new BetaDistribution(1., 2.), P_CDFPDF, GNUR_BETA_PDF_1_2, 1e-15); + checkPDF(new BetaDistribution(2., 2.), P_CDFPDF, GNUR_BETA_PDF_2_2, 1e-15); + checkPDF(new BetaDistribution(4., 2.), P_CDFPDF, GNUR_BETA_PDF_4_2, 1e-13); + checkPDF(new BetaDistribution(.1, 2.), P_CDFPDF, GNUR_BETA_PDF_01_2, 1e-15); + checkPDF(new BetaDistribution(.5, 2.), P_CDFPDF, GNUR_BETA_PDF_05_2, 1e-14); + checkPDF(new BetaDistribution(1., 4.), P_CDFPDF, GNUR_BETA_PDF_1_4, 1e-15); + checkPDF(new BetaDistribution(2., 4.), P_CDFPDF, GNUR_BETA_PDF_2_4, 1e-15); + checkPDF(new BetaDistribution(4., 4.), P_CDFPDF, GNUR_BETA_PDF_4_4, 1e-14); + checkPDF(new BetaDistribution(.1, 4.), P_CDFPDF, GNUR_BETA_PDF_01_4, 1e-14); + checkPDF(new BetaDistribution(.5, 4.), P_CDFPDF, GNUR_BETA_PDF_05_4, 1e-15); + checkPDF(new BetaDistribution(1., .1), P_CDFPDF, GNUR_BETA_PDF_1_01, 1e-15); + checkPDF(new BetaDistribution(2., .1), P_CDFPDF, GNUR_BETA_PDF_2_01, 1e-15); + checkPDF(new BetaDistribution(4., .1), P_CDFPDF, GNUR_BETA_PDF_4_01, 1e-14); + checkPDF(new BetaDistribution(.1, .1), P_CDFPDF, GNUR_BETA_PDF_01_01, 1e-14); + checkPDF(new BetaDistribution(.5, .1), P_CDFPDF, GNUR_BETA_PDF_05_01, 1e-15); + checkPDF(new BetaDistribution(1., .5), P_CDFPDF, GNUR_BETA_PDF_1_05, 1e-14); + checkPDF(new BetaDistribution(2., .5), P_CDFPDF, GNUR_BETA_PDF_2_05, 1e-14); + checkPDF(new BetaDistribution(4., .5), P_CDFPDF, GNUR_BETA_PDF_4_05, 1e-15); + checkPDF(new BetaDistribution(.1, .5), P_CDFPDF, GNUR_BETA_PDF_01_05, 1e-15); + checkPDF(new BetaDistribution(.5, .5), P_CDFPDF, GNUR_BETA_PDF_05_05, 1e-14); + } + + @Test + public void testCDF() { + checkCDF(new BetaDistribution(1., 1.), P_CDFPDF, SCIPY_BETA_CDF_1_1, 1e-12); + checkCDF(new BetaDistribution(2., 1.), P_CDFPDF, SCIPY_BETA_CDF_2_1, 1e-12); + checkCDF(new BetaDistribution(4., 1.), P_CDFPDF, SCIPY_BETA_CDF_4_1, 1e-11); + checkCDF(new BetaDistribution(.1, 1.), P_CDFPDF, SCIPY_BETA_CDF_01_1, 1e-13); + checkCDF(new BetaDistribution(.5, 1.), P_CDFPDF, SCIPY_BETA_CDF_05_1, 1e-12); + checkCDF(new BetaDistribution(1., 2.), P_CDFPDF, SCIPY_BETA_CDF_1_2, 1e-12); + checkCDF(new BetaDistribution(2., 2.), P_CDFPDF, SCIPY_BETA_CDF_2_2, 1e-12); + checkCDF(new BetaDistribution(4., 2.), P_CDFPDF, SCIPY_BETA_CDF_4_2, 1e-11); + checkCDF(new BetaDistribution(.1, 2.), P_CDFPDF, SCIPY_BETA_CDF_01_2, 1e-13); + checkCDF(new BetaDistribution(.5, 2.), P_CDFPDF, SCIPY_BETA_CDF_05_2, 1e-12); + checkCDF(new BetaDistribution(1., 4.), P_CDFPDF, SCIPY_BETA_CDF_1_4, 1e-12); + checkCDF(new BetaDistribution(2., 4.), P_CDFPDF, SCIPY_BETA_CDF_2_4, 1e-12); + checkCDF(new BetaDistribution(4., 4.), P_CDFPDF, SCIPY_BETA_CDF_4_4, 1e-11); + checkCDF(new BetaDistribution(.1, 4.), P_CDFPDF, SCIPY_BETA_CDF_01_4, 1e-13); + checkCDF(new BetaDistribution(.5, 4.), P_CDFPDF, SCIPY_BETA_CDF_05_4, 1e-13); + checkCDF(new BetaDistribution(1., .1), P_CDFPDF, SCIPY_BETA_CDF_1_01, 1e-12); + checkCDF(new BetaDistribution(2., .1), P_CDFPDF, SCIPY_BETA_CDF_2_01, 1e-12); + checkCDF(new BetaDistribution(4., .1), P_CDFPDF, SCIPY_BETA_CDF_4_01, 1e-11); + checkCDF(new BetaDistribution(.1, .1), P_CDFPDF, SCIPY_BETA_CDF_01_01, 1e-13); + checkCDF(new BetaDistribution(.5, .1), P_CDFPDF, SCIPY_BETA_CDF_05_01, 1e-12); + checkCDF(new BetaDistribution(1., .5), P_CDFPDF, SCIPY_BETA_CDF_1_05, 1e-12); + checkCDF(new BetaDistribution(2., .5), P_CDFPDF, SCIPY_BETA_CDF_2_05, 1e-12); + checkCDF(new BetaDistribution(4., .5), P_CDFPDF, SCIPY_BETA_CDF_4_05, 1e-11); + checkCDF(new BetaDistribution(.1, .5), P_CDFPDF, SCIPY_BETA_CDF_01_05, 1e-13); + checkCDF(new BetaDistribution(.5, .5), P_CDFPDF, SCIPY_BETA_CDF_05_05, 1e-12); + + checkCDF(new BetaDistribution(1., 1.), P_CDFPDF, GNUR_BETA_CDF_1_1, 1e-14); + checkCDF(new BetaDistribution(2., 1.), P_CDFPDF, GNUR_BETA_CDF_2_1, 1e-14); + checkCDF(new BetaDistribution(4., 1.), P_CDFPDF, GNUR_BETA_CDF_4_1, 1e-13); + checkCDF(new BetaDistribution(.1, 1.), P_CDFPDF, GNUR_BETA_CDF_01_1, 1e-15); + checkCDF(new BetaDistribution(.5, 1.), P_CDFPDF, GNUR_BETA_CDF_05_1, 1e-14); + checkCDF(new BetaDistribution(1., 2.), P_CDFPDF, GNUR_BETA_CDF_1_2, 1e-14); + checkCDF(new BetaDistribution(2., 2.), P_CDFPDF, GNUR_BETA_CDF_2_2, 1e-14); + checkCDF(new BetaDistribution(4., 2.), P_CDFPDF, GNUR_BETA_CDF_4_2, 1e-14); + checkCDF(new BetaDistribution(.1, 2.), P_CDFPDF, GNUR_BETA_CDF_01_2, 1e-15); + checkCDF(new BetaDistribution(.5, 2.), P_CDFPDF, GNUR_BETA_CDF_05_2, 1e-14); + checkCDF(new BetaDistribution(1., 4.), P_CDFPDF, GNUR_BETA_CDF_1_4, 1e-14); + checkCDF(new BetaDistribution(2., 4.), P_CDFPDF, GNUR_BETA_CDF_2_4, 1e-14); + checkCDF(new BetaDistribution(4., 4.), P_CDFPDF, GNUR_BETA_CDF_4_4, 1e-14); + checkCDF(new BetaDistribution(.1, 4.), P_CDFPDF, GNUR_BETA_CDF_01_4, 1e-15); + checkCDF(new BetaDistribution(.5, 4.), P_CDFPDF, GNUR_BETA_CDF_05_4, 1e-15); + checkCDF(new BetaDistribution(1., .1), P_CDFPDF, GNUR_BETA_CDF_1_01, 1e-14); + checkCDF(new BetaDistribution(2., .1), P_CDFPDF, GNUR_BETA_CDF_2_01, 1e-14); + checkCDF(new BetaDistribution(4., .1), P_CDFPDF, GNUR_BETA_CDF_4_01, 1e-14); + checkCDF(new BetaDistribution(.1, .1), P_CDFPDF, GNUR_BETA_CDF_01_01, 1e-14); + checkCDF(new BetaDistribution(.5, .1), P_CDFPDF, GNUR_BETA_CDF_05_01, 1e-14); + checkCDF(new BetaDistribution(1., .5), P_CDFPDF, GNUR_BETA_CDF_1_05, 1e-14); + checkCDF(new BetaDistribution(2., .5), P_CDFPDF, GNUR_BETA_CDF_2_05, 1e-14); + checkCDF(new BetaDistribution(4., .5), P_CDFPDF, GNUR_BETA_CDF_4_05, 1e-13); + checkCDF(new BetaDistribution(.1, .5), P_CDFPDF, GNUR_BETA_CDF_01_05, 1e-14); + checkCDF(new BetaDistribution(.5, .5), P_CDFPDF, GNUR_BETA_CDF_05_05, 1e-14); + } + + @Test + public void testQuantile() { + checkQuantile(new BetaDistribution(1., 1.), P_QUANT, SCIPY_BETA_QUANT_1_1, 1e-15); + checkQuantile(new BetaDistribution(2., 1.), P_QUANT, SCIPY_BETA_QUANT_2_1, 1e-15); + checkQuantile(new BetaDistribution(4., 1.), P_QUANT, SCIPY_BETA_QUANT_4_1, 1e-15); + checkQuantile(new BetaDistribution(.1, 1.), P_QUANT, SCIPY_BETA_QUANT_01_1, 1e-14); + checkQuantile(new BetaDistribution(.5, 1.), P_QUANT, SCIPY_BETA_QUANT_05_1, 1e-14); + checkQuantile(new BetaDistribution(1., 2.), P_QUANT, SCIPY_BETA_QUANT_1_2, 1e-14); + checkQuantile(new BetaDistribution(2., 2.), P_QUANT, SCIPY_BETA_QUANT_2_2, 1e-15); + checkQuantile(new BetaDistribution(4., 2.), P_QUANT, SCIPY_BETA_QUANT_4_2, 1e-15); + checkQuantile(new BetaDistribution(.1, 2.), P_QUANT, SCIPY_BETA_QUANT_01_2, 1e-13); + checkQuantile(new BetaDistribution(.5, 2.), P_QUANT, SCIPY_BETA_QUANT_05_2, 1e-14); + checkQuantile(new BetaDistribution(1., 4.), P_QUANT, SCIPY_BETA_QUANT_1_4, 1e-14); + checkQuantile(new BetaDistribution(2., 4.), P_QUANT, SCIPY_BETA_QUANT_2_4, 1e-15); + checkQuantile(new BetaDistribution(4., 4.), P_QUANT, SCIPY_BETA_QUANT_4_4, 1e-15); + checkQuantile(new BetaDistribution(.1, 4.), P_QUANT, SCIPY_BETA_QUANT_01_4, 1e-13); + checkQuantile(new BetaDistribution(.5, 4.), P_QUANT, SCIPY_BETA_QUANT_05_4, 1e-14); + checkQuantile(new BetaDistribution(1., .1), P_QUANT, SCIPY_BETA_QUANT_1_01, 1e-14); + checkQuantile(new BetaDistribution(2., .1), P_QUANT, SCIPY_BETA_QUANT_2_01, 1e-15); + checkQuantile(new BetaDistribution(4., .1), P_QUANT, SCIPY_BETA_QUANT_4_01, 1e-15); + checkQuantile(new BetaDistribution(.1, .1), P_QUANT, SCIPY_BETA_QUANT_01_01, 1e-13); + checkQuantile(new BetaDistribution(.5, .1), P_QUANT, SCIPY_BETA_QUANT_05_01, 1e-14); + checkQuantile(new BetaDistribution(1., .5), P_QUANT, SCIPY_BETA_QUANT_1_05, 1e-14); + checkQuantile(new BetaDistribution(2., .5), P_QUANT, SCIPY_BETA_QUANT_2_05, 1e-15); + checkQuantile(new BetaDistribution(4., .5), P_QUANT, SCIPY_BETA_QUANT_4_05, 1e-15); + checkQuantile(new BetaDistribution(.1, .5), P_QUANT, SCIPY_BETA_QUANT_01_05, 1e-14); + checkQuantile(new BetaDistribution(.5, .5), P_QUANT, SCIPY_BETA_QUANT_05_05, 1e-14); + + checkQuantile(new BetaDistribution(1., 1.), P_QUANT, GNUR_BETA_QUANT_1_1, 1e-15); + checkQuantile(new BetaDistribution(2., 1.), P_QUANT, GNUR_BETA_QUANT_2_1, 1e-15); + checkQuantile(new BetaDistribution(4., 1.), P_QUANT, GNUR_BETA_QUANT_4_1, 1e-15); + checkQuantile(new BetaDistribution(.1, 1.), P_QUANT, GNUR_BETA_QUANT_01_1, 1e-13); + checkQuantile(new BetaDistribution(.5, 1.), P_QUANT, GNUR_BETA_QUANT_05_1, 1e-14); + checkQuantile(new BetaDistribution(1., 2.), P_QUANT, GNUR_BETA_QUANT_1_2, 1e-14); + checkQuantile(new BetaDistribution(2., 2.), P_QUANT, GNUR_BETA_QUANT_2_2, 1e-15); + checkQuantile(new BetaDistribution(4., 2.), P_QUANT, GNUR_BETA_QUANT_4_2, 1e-15); + checkQuantile(new BetaDistribution(.1, 2.), P_QUANT, GNUR_BETA_QUANT_01_2, 1e-14); + checkQuantile(new BetaDistribution(.5, 2.), P_QUANT, GNUR_BETA_QUANT_05_2, 1e-14); + checkQuantile(new BetaDistribution(1., 4.), P_QUANT, GNUR_BETA_QUANT_1_4, 1e-14); + checkQuantile(new BetaDistribution(2., 4.), P_QUANT, GNUR_BETA_QUANT_2_4, 1e-15); + checkQuantile(new BetaDistribution(4., 4.), P_QUANT, GNUR_BETA_QUANT_4_4, 1e-15); + checkQuantile(new BetaDistribution(.1, 4.), P_QUANT, GNUR_BETA_QUANT_01_4, 1e-13); + checkQuantile(new BetaDistribution(.5, 4.), P_QUANT, GNUR_BETA_QUANT_05_4, 1e-14); + checkQuantile(new BetaDistribution(1., .1), P_QUANT, GNUR_BETA_QUANT_1_01, 1e-14); + checkQuantile(new BetaDistribution(2., .1), P_QUANT, GNUR_BETA_QUANT_2_01, 1e-15); + checkQuantile(new BetaDistribution(4., .1), P_QUANT, GNUR_BETA_QUANT_4_01, 1e-15); + checkQuantile(new BetaDistribution(.1, .1), P_QUANT, GNUR_BETA_QUANT_01_01, 1e-13); + checkQuantile(new BetaDistribution(.5, .1), P_QUANT, GNUR_BETA_QUANT_05_01, 1e-14); + checkQuantile(new BetaDistribution(1., .5), P_QUANT, GNUR_BETA_QUANT_1_05, 1e-14); + checkQuantile(new BetaDistribution(2., .5), P_QUANT, GNUR_BETA_QUANT_2_05, 1e-15); + checkQuantile(new BetaDistribution(4., .5), P_QUANT, GNUR_BETA_QUANT_4_05, 1e-15); + checkQuantile(new BetaDistribution(.1, .5), P_QUANT, GNUR_BETA_QUANT_01_05, 1e-14); + checkQuantile(new BetaDistribution(.5, .5), P_QUANT, GNUR_BETA_QUANT_05_05, 1e-14); + } +}
\ No newline at end of file diff --git a/test/de/lmu/ifi/dbs/elki/math/statistics/distribution/TestChiSquaredDistribution.java b/test/de/lmu/ifi/dbs/elki/math/statistics/distribution/TestChiSquaredDistribution.java new file mode 100644 index 00000000..4bfb52fd --- /dev/null +++ b/test/de/lmu/ifi/dbs/elki/math/statistics/distribution/TestChiSquaredDistribution.java @@ -0,0 +1,826 @@ +package de.lmu.ifi.dbs.elki.math.statistics.distribution; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +import org.junit.Test; + +import de.lmu.ifi.dbs.elki.JUnit4Test; + +/** + * Unit test for the Chi Squared distribution in ELKI. + * + * The reference values were computed using GNU R and SciPy. + * + * @author Erich Schubert + */ +public class TestChiSquaredDistribution extends AbstractDistributionTest implements JUnit4Test { + public static final double[] P_CDFPDF = { // + 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0, 1e-05, 1e-10, 0.1234567, 3.14159265359, 2.71828182846, 0.314159265359, 0.271828182846 // + }; + + public static final double[] SCIPY_CHISQ_CDF_01 = { // + 8.82243563885668047319654760940466076135635375976562e-01, // 0.100000 + 9.11257625237519408578634738660184666514396667480469e-01, // 0.200000 + 9.27832443365792181033668839518213644623756408691406e-01, // 0.300000 + 9.39209088699221039675535394053440541028976440429688e-01, // 0.400000 + 9.47708820173318144952645525336265563964843750000000e-01, // 0.500000 + 9.54381121874345095257297089119674637913703918457031e-01, // 0.600000 + 9.59792062842079185536192653671605512499809265136719e-01, // 0.700000 + 9.64282450147988190458647750347154214978218078613281e-01, // 0.800000 + 9.68073590190356547680039511760696768760681152343750e-01, // 0.900000 + 9.71317371244163729748777313943719491362571716308594e-01, // 1.000000 + 9.74122543962944176243468064058106392621994018554688e-01, // 1.100000 + 9.76569616477515944907850098388735204935073852539062e-01, // 1.200000 + 9.78719820077030289162678400316508486866950988769531e-01, // 1.300000 + 9.80620773380832644328108926856657490134239196777344e-01, // 1.400000 + 9.82310206057613877206335928349290043115615844726562e-01, // 1.500000 + 9.83818489875346147144341557577718049287796020507812e-01, // 1.600000 + 9.85170408569036260004736504924949258565902709960938e-01, // 1.700000 + 9.86386426034563057463344648567726835608482360839844e-01, // 1.800000 + 9.87483614564462985896398095064796507358551025390625e-01, // 1.900000 + 9.88476347051459813819462851824937388300895690917969e-01, // 2.000000 + 5.57969600280356692323380229936446994543075561523438e-01, // 0.000010 + 3.13769439143751660559189531340962275862693786621094e-01, // 0.000000 + 8.91102932913000800319025529461214318871498107910156e-01, // 0.123457 + 9.95185831509797558247498727723723277449607849121094e-01, // 3.141593 + 9.93427663775691249803401206008857116103172302246094e-01, // 2.718282 + 9.29682286188087236844523886247770860791206359863281e-01, // 0.314159 + 9.23848956395416420761534936900716274976730346679688e-01, // 0.271828 + }; + + public static final double[] SCIPY_CHISQ_PDF_01 = { // + 4.20596758859699160204570489440811797976493835449219e-01, // 0.100000 + 2.07096472560618249936226220597745850682258605957031e-01, // 0.200000 + 1.34020514700378817751769133792549837380647659301758e-01, // 0.300000 + 9.69984417734248427489518462607520632445812225341797e-02, // 0.400000 + 7.46423873161204676218716258517815731465816497802734e-02, // 0.500000 + 5.97102120258812332775910647342243464663624763488770e-02, // 0.600000 + 4.90607789226714316455080222567630698904395103454590e-02, // 0.700000 + 4.11080963115560762966005370344646507874131202697754e-02, // 0.800000 + 3.49637289621889876811700048619968583807349205017090e-02, // 0.900000 + 3.00907771823936791288911507535885903052985668182373e-02, // 1.000000 + 2.61454204266986685312179616857974906452000141143799e-02, // 1.100000 + 2.28971681180981344316816006312365061603486537933350e-02, // 1.200000 + 2.01856643669713550770516974353085970506072044372559e-02, // 1.300000 + 1.78958724006224693947597614851474645547568798065186e-02, // 1.400000 + 1.59431116775097907189984169917806866578757762908936e-02, // 1.500000 + 1.42636632388964210843562696595654415432363748550415e-02, // 1.600000 + 1.28086648126372482786594630965737451333552598953247e-02, // 1.700000 + 1.15400245359752948171028208435018314048647880554199e-02, // 1.800000 + 1.04276144940537648930778402700525475665926933288574e-02, // 1.900000 + 9.44729915895063145081511635225979262031614780426025e-03, // 2.000000 + 2.78983471644215933338273316621780395507812500000000e+03, // 0.000010 + 1.56884719564404964447021484375000000000000000000000e+08, // 0.000000 + 3.40277656462092736866509312676498666405677795410156e-01, // 0.123457 + 3.47616265678607179803738524981326918350532650947571e-03, // 3.141593 + 4.92871253406783164929816365429360303096473217010498e-03, // 2.718282 + 1.27370674605358003006472245033364742994308471679688e-01, // 0.314159 + 1.49270555162829721140838046267163008451461791992188e-01, // 0.271828 + }; + + public static final double[] GNUR_CHISQ_CDF_01 = { // + 8.82243563885668047319654760940466076135635375976562e-01, // 0.100000 + 9.11257625237519630623239663691492751240730285644531e-01, // 0.200000 + 9.27832443365792847167483614612137898802757263183594e-01, // 0.300000 + 9.39209088699221039675535394053440541028976440429688e-01, // 0.400000 + 9.47708820173318589041855375398881733417510986328125e-01, // 0.500000 + 9.54381121874345650368809401697944849729537963867188e-01, // 0.600000 + 9.59792062842079629625402503734221681952476501464844e-01, // 0.700000 + 9.64282450147988301480950212862808257341384887695312e-01, // 0.800000 + 9.68073590190356436657737049245042726397514343261719e-01, // 0.900000 + 9.71317371244163951793382238975027576088905334472656e-01, // 1.000000 + 9.74122543962944731354980376636376604437828063964844e-01, // 1.100000 + 9.76569616477516500019362410967005416750907897949219e-01, // 1.200000 + 9.78719820077030733251888250379124656319618225097656e-01, // 1.300000 + 9.80620773380832866372713851887965574860572814941406e-01, // 1.400000 + 9.82310206057613877206335928349290043115615844726562e-01, // 1.500000 + 9.83818489875346147144341557577718049287796020507812e-01, // 1.600000 + 9.85170408569036704093946354987565428018569946289062e-01, // 1.700000 + 9.86386426034563279507949573599034920334815979003906e-01, // 1.800000 + 9.87483614564463429985607945127412676811218261718750e-01, // 1.900000 + 9.88476347051460035864067776856245473027229309082031e-01, // 2.000000 + 5.57969600280356914367985154967755079269409179687500e-01, // 0.000010 + 3.13769439143751827092643225114443339407444000244141e-01, // 0.000000 + 8.91102932913001466452840304555138573050498962402344e-01, // 0.123457 + 9.95185831509798335403615965333301573991775512695312e-01, // 3.141593 + 9.93427663775695912740104631666326895356178283691406e-01, // 2.718282 + 9.29682286188089901379782986623467877507209777832031e-01, // 0.314159 + 9.23848956395430742638552601420087739825248718261719e-01, // 0.271828 + }; + + public static final double[] GNUR_CHISQ_PDF_01 = { // + 4.20596758859699271226872951956465840339660644531250e-01, // 0.100000 + 2.07096472560618277691801836226659361273050308227539e-01, // 0.200000 + 1.34020514700378845507344749421463347971439361572266e-01, // 0.300000 + 9.69984417734248705045274618896655738353729248046875e-02, // 0.400000 + 7.46423873161204814996594336662383284419775009155273e-02, // 0.500000 + 5.97102120258812332775910647342243464663624763488770e-02, // 0.600000 + 4.90607789226714316455080222567630698904395103454590e-02, // 0.700000 + 4.11080963115560832354944409416930284351110458374023e-02, // 0.800000 + 3.49637289621889876811700048619968583807349205017090e-02, // 0.900000 + 3.00907771823936825983381027072027791291475296020508e-02, // 1.000000 + 2.61454204266986720006649136394116794690489768981934e-02, // 1.100000 + 2.28971681180981309622346486776223173364996910095215e-02, // 1.200000 + 2.01856643669713516076047454816944082267582416534424e-02, // 1.300000 + 1.78958724006224659253128095315332757309079170227051e-02, // 1.400000 + 1.59431116775097872495514650381664978340268135070801e-02, // 1.500000 + 1.42636632388964228190797456363725359551608562469482e-02, // 1.600000 + 1.28086648126372430744890351661524618975818157196045e-02, // 1.700000 + 1.15400245359752878782089169362734537571668624877930e-02, // 1.800000 + 1.04276144940537579541839363628241699188947677612305e-02, // 1.900000 + 9.44729915895062624664468842183850938454270362854004e-03, // 2.000000 + 2.78983471644215978813008405268192291259765625000000e+03, // 0.000010 + 1.56884719564405262470245361328125000000000000000000e+08, // 0.000000 + 3.40277656462092847888811775192152708768844604492188e-01, // 0.123457 + 3.47616265678549370143901597884905640967190265655518e-03, // 3.141593 + 4.92871253406383397904777510234453075099736452102661e-03, // 2.718282 + 1.27370674605348788155367856234079226851463317871094e-01, // 0.314159 + 1.49270555162772794455250391365552786737680435180664e-01, // 0.271828 + }; + + public static final double[] SCIPY_CHISQ_CDF_1 = { // + 2.48170365954150762277663488930556923151016235351562e-01, // 0.100000 + 3.45279153981423170272080369613831862807273864746094e-01, // 0.200000 + 4.16117579229634992898922973836306482553482055664062e-01, // 0.300000 + 4.72910743134461963865788902694475837051868438720703e-01, // 0.400000 + 5.20499877813046629704274437244748696684837341308594e-01, // 0.500000 + 5.61421973919000283359537206706590950489044189453125e-01, // 0.600000 + 5.97216305753524356880745926900999620556831359863281e-01, // 0.700000 + 6.28906630477302552861829099128954112529754638671875e-01, // 0.800000 + 6.57218288852088661222694554453482851386070251464844e-01, // 0.900000 + 6.82689492137085962575326902879169210791587829589844e-01, // 1.000000 + 7.05733895695037305451080555940279737114906311035156e-01, // 1.100000 + 7.26678321707701857512518017756519839167594909667969e-01, // 1.200000 + 7.45786776396035855363209066126728430390357971191406e-01, // 1.300000 + 7.63276429362142794232681808352936059236526489257812e-01, // 1.400000 + 7.79328638080153068834476925985654816031455993652344e-01, // 1.500000 + 7.94096789267931812972278748929966241121292114257812e-01, // 1.600000 + 8.07712022888480207782890829548705369234085083007812e-01, // 1.700000 + 8.20287505121000126706576338619925081729888916015625e-01, // 1.800000 + 8.31921680965029608501026814337819814682006835937500e-01, // 1.900000 + 8.42700792949715116186837349232519045472145080566406e-01, // 2.000000 + 2.52312831680559686298170873897106503136456012725830e-03, // 0.000010 + 7.97884560789568197394176080061711786584055516868830e-06, // 0.000000 + 2.74684755560843596455811166379135102033615112304688e-01, // 0.123457 + 9.23680750542945294689900492812739685177803039550781e-01, // 3.141593 + 9.00795249588885127423054655082523822784423828125000e-01, // 2.718282 + 4.24860961379973645080099231563508510589599609375000e-01, // 0.314159 + 3.97891959928843685911914462849381379783153533935547e-01, // 0.271828 + }; + + public static final double[] SCIPY_CHISQ_PDF_1 = { // + 1.20003894843013592108604825625661760568618774414062e+00, // 0.100000 + 8.07171129357680983140710395673522725701332092285156e-01, // 0.200000 + 6.26910099227520833942151057271985337138175964355469e-01, // 0.300000 + 5.16441547467278438965365694457432255148887634277344e-01, // 0.400000 + 4.39391289467722434558538679993944242596626281738281e-01, // 0.500000 + 3.81545289384093044837698016635840758681297302246094e-01, // 0.600000 + 3.36014467726770404976122108564595691859722137451172e-01, // 0.700000 + 2.98983539918204976881810353006585501134395599365234e-01, // 0.800000 + 2.68136721052083026872736581935896538197994232177734e-01, // 0.900000 + 2.41970724519143365327522587904240936040878295898438e-01, // 1.000000 + 2.19458172413343671758312325437145773321390151977539e-01, // 1.100000 + 1.99867763901733275222483143807039596140384674072266e-01, // 1.200000 + 1.82661481795109131853749317997426260262727737426758e-01, // 1.300000 + 1.67432557345083526945117569084686692804098129272461e-01, // 1.400000 + 1.53866322805455257860529627578216604888439178466797e-01, // 1.500000 + 1.41714565306223921892225803276232909411191940307617e-01, // 1.600000 + 1.30778181923888126014787758322199806571006774902344e-01, // 1.700000 + 1.20895122473204871815788408184744184836745262145996e-01, // 1.800000 + 1.11931805086169955387731533846817910671234130859375e-01, // 1.900000 + 1.03776874355148679418547885688894893974065780639648e-01, // 2.000000 + 1.26155995319454547143322997726500034332275390625000e+02, // 0.000010 + 3.98942280381486198166385293006896972656250000000000e+04, // 0.000000 + 1.06744240620741748237776391761144623160362243652344e+00, // 0.123457 + 4.67893435961044684701093387957371305674314498901367e-02, // 3.141593 + 6.21577700803671237550496186941018095239996910095215e-02, // 2.718282 + 6.08297892453185884420463480637408792972564697265625e-01, // 0.314159 + 6.67937897133326075049808423500508069992065429687500e-01, // 0.271828 + }; + + public static final double[] GNUR_CHISQ_CDF_1 = { // + 2.48170365954150706766512257672729901969432830810547e-01, // 0.100000 + 3.45279153981422948227475444582523778080940246582031e-01, // 0.200000 + 4.16117579229634826365469280062825419008731842041016e-01, // 0.300000 + 4.72910743134461908354637671436648815870285034179688e-01, // 0.400000 + 5.20499877813046518681971974729094654321670532226562e-01, // 0.500000 + 5.61421973919000061314932281675282865762710571289062e-01, // 0.600000 + 5.97216305753524356880745926900999620556831359863281e-01, // 0.700000 + 6.28906630477302330817224174097646027803421020507812e-01, // 0.800000 + 6.57218288852088550200392091937828809022903442382812e-01, // 0.900000 + 6.82689492137085962575326902879169210791587829589844e-01, // 1.000000 + 7.05733895695037083406475630908971652388572692871094e-01, // 1.100000 + 7.26678321707701857512518017756519839167594909667969e-01, // 1.200000 + 7.45786776396035633318604141095420345664024353027344e-01, // 1.300000 + 7.63276429362142572188076883321627974510192871093750e-01, // 1.400000 + 7.79328638080153179856779388501308858394622802734375e-01, // 1.500000 + 7.94096789267931590927673823898658156394958496093750e-01, // 1.600000 + 8.07712022888480207782890829548705369234085083007812e-01, // 1.700000 + 8.20287505121000237728878801135579124093055725097656e-01, // 1.800000 + 8.31921680965029608501026814337819814682006835937500e-01, // 1.900000 + 8.42700792949715449253744736779481172561645507812500e-01, // 2.000000 + 2.52312831680559773034344672737461223732680082321167e-03, // 0.000010 + 7.97884560789567350361228825761372718261554837226868e-06, // 0.000000 + 2.74684755560843485433508703863481059670448303222656e-01, // 0.123457 + 9.23680750542954953630214731674641370773315429687500e-01, // 3.141593 + 9.00795249588944302310267175926128402352333068847656e-01, // 2.718282 + 4.24860961379986301622579958348069339990615844726562e-01, // 0.314159 + 3.97891959928907357202376715576974675059318542480469e-01, // 0.271828 + }; + + public static final double[] GNUR_CHISQ_PDF_1 = { // + 1.20003894843013592108604825625661760568618774414062e+00, // 0.100000 + 8.07171129357681094163012858189176768064498901367188e-01, // 0.200000 + 6.26910099227520833942151057271985337138175964355469e-01, // 0.300000 + 5.16441547467278327943063231941778212785720825195312e-01, // 0.400000 + 4.39391289467722434558538679993944242596626281738281e-01, // 0.500000 + 3.81545289384093044837698016635840758681297302246094e-01, // 0.600000 + 3.36014467726770404976122108564595691859722137451172e-01, // 0.700000 + 2.98983539918204921370659121748758479952812194824219e-01, // 0.800000 + 2.68136721052082971361585350678069517016410827636719e-01, // 0.900000 + 2.41970724519143365327522587904240936040878295898438e-01, // 1.000000 + 2.19458172413343699513887941066059283912181854248047e-01, // 1.100000 + 1.99867763901733275222483143807039596140384674072266e-01, // 1.200000 + 1.82661481795109104098173702368512749671936035156250e-01, // 1.300000 + 1.67432557345083526945117569084686692804098129272461e-01, // 1.400000 + 1.53866322805455257860529627578216604888439178466797e-01, // 1.500000 + 1.41714565306223921892225803276232909411191940307617e-01, // 1.600000 + 1.30778181923888126014787758322199806571006774902344e-01, // 1.700000 + 1.20895122473204871815788408184744184836745262145996e-01, // 1.800000 + 1.11931805086169955387731533846817910671234130859375e-01, // 1.900000 + 1.03776874355148693296335693503351649269461631774902e-01, // 2.000000 + 1.26155995319454461878194706514477729797363281250000e+02, // 0.000010 + 3.98942280381485470570623874664306640625000000000000e+04, // 0.000000 + 1.06744240620741726033315899258013814687728881835938e+00, // 0.123457 + 4.67893435960980846877177441456296946853399276733398e-02, // 3.141593 + 6.21577700803265520423934731297777034342288970947266e-02, // 2.718282 + 6.08297892453159572134779864427400752902030944824219e-01, // 0.314159 + 6.67937897133176639030693877430167049169540405273438e-01, // 0.271828 + }; + + public static final double[] SCIPY_CHISQ_CDF_2 = { // + 4.87705754992859910612601481716410489752888679504395e-02, // 0.100000 + 9.51625819640404407850908796717703808099031448364258e-02, // 0.200000 + 1.39292023574942191999070928432047367095947265625000e-01, // 0.300000 + 1.81269246922018151257915974383649881929159164428711e-01, // 0.400000 + 2.21199216928595121522960198490181937813758850097656e-01, // 0.500000 + 2.59181779318282068391710026844521053135395050048828e-01, // 0.600000 + 2.95311910281286504886821830950793810188770294189453e-01, // 0.700000 + 3.29679953964360561435853469447465613484382629394531e-01, // 0.800000 + 3.62371848378226779630040255142375826835632324218750e-01, // 0.900000 + 3.93469340287366520225731392201851122081279754638672e-01, // 1.000000 + 4.23050189619513239414061445131665095686912536621094e-01, // 1.100000 + 4.51188363905973610634703163668746128678321838378906e-01, // 1.200000 + 4.77954223238983844090910224622348323464393615722656e-01, // 1.300000 + 5.03414696208590473069932613725541159510612487792969e-01, // 1.400000 + 5.27633447258985310845957883429946377873420715332031e-01, // 1.500000 + 5.50671035882778325820652298716595396399497985839844e-01, // 1.600000 + 5.72585068051273293754377391451271250844001770019531e-01, // 1.700000 + 5.93430340259400890268182138242991641163825988769531e-01, // 1.800000 + 6.13258976545498768473407835699617862701416015625000e-01, // 1.900000 + 6.32120558828557776998025019565830007195472717285156e-01, // 2.000000 + 4.99998750002083230991115070374064544012071564793587e-06, // 0.000010 + 4.99999999987500220561171362591683115655927238663025e-11, // 0.000000 + 5.98617593408454426451292818001093110069632530212402e-02, // 0.123457 + 7.92120423649238181162957062042551115155220031738281e-01, // 3.141593 + 7.43118634686529722088721428008284419775009155273438e-01, // 2.718282 + 1.45364000846766538987964167972677387297153472900391e-01, // 0.314159 + 1.27082379621731661467620710936898831278085708618164e-01, // 0.271828 + }; + + public static final double[] SCIPY_CHISQ_PDF_2 = { // + 4.75614712250357007938816877867793664336204528808594e-01, // 0.100000 + 4.52418709017979814301924079700256697833538055419922e-01, // 0.200000 + 4.30353988212528904000464535783976316452026367187500e-01, // 0.300000 + 4.09365376538990910493254204993718303740024566650391e-01, // 0.400000 + 3.89400391535702439238519900754909031093120574951172e-01, // 0.500000 + 3.70409110340858993559720602206652984023094177246094e-01, // 0.600000 + 3.52344044859356719801013468895689584314823150634766e-01, // 0.700000 + 3.35160023017819663770922034018440172076225280761719e-01, // 0.800000 + 3.18814075810886610184979872428812086582183837890625e-01, // 0.900000 + 3.03265329856316712131558688270160928368568420410156e-01, // 1.000000 + 2.88474905190243380292969277434167452156543731689453e-01, // 1.100000 + 2.74405818047013250193799649423453956842422485351562e-01, // 1.200000 + 2.61022888380508022443393656430998817086219787597656e-01, // 1.300000 + 2.48292651895704791220609308766142930835485458374023e-01, // 1.400000 + 2.36183276370507344577021058285026811063289642333984e-01, // 1.500000 + 2.24664482058610809334098235012788791209459304809570e-01, // 1.600000 + 2.13707465974363353122811304274364374577999114990234e-01, // 1.700000 + 2.03284829870299554865908930878504179418087005615234e-01, // 1.800000 + 1.93370511727250643518871697779104579240083694458008e-01, // 1.900000 + 1.83939720585721139256563105845998506993055343627930e-01, // 2.000000 + 4.99997500006249984139117259474005550146102905273438e-01, // 0.000010 + 4.99999999974999997931490725022740662097930908203125e-01, // 0.000000 + 4.70069120329577316841351830589701421558856964111328e-01, // 0.123457 + 1.03939788175380951051884892422094708308577537536621e-01, // 3.141593 + 1.28440682656735138955639285995857790112495422363281e-01, // 2.718282 + 4.27317999576616702750442300384747795760631561279297e-01, // 0.314159 + 4.36458810189134183143977452346007339656352996826172e-01, // 0.271828 + }; + + public static final double[] GNUR_CHISQ_CDF_2 = { // + 4.87705754992859910612601481716410489752888679504395e-02, // 0.100000 + 9.51625819640404269073030718573136255145072937011719e-02, // 0.200000 + 1.39292023574942191999070928432047367095947265625000e-01, // 0.300000 + 1.81269246922018151257915974383649881929159164428711e-01, // 0.400000 + 2.21199216928595121522960198490181937813758850097656e-01, // 0.500000 + 2.59181779318282123902861258102348074316978454589844e-01, // 0.600000 + 2.95311910281286560397973062208620831370353698730469e-01, // 0.700000 + 3.29679953964360727969307163220946677029132843017578e-01, // 0.800000 + 3.62371848378226724118889023884548805654048919677734e-01, // 0.900000 + 3.93469340287366575736882623459678143262863159179688e-01, // 1.000000 + 4.23050189619513350436363907647319138050079345703125e-01, // 1.100000 + 4.51188363905973610634703163668746128678321838378906e-01, // 1.200000 + 4.77954223238983955113212687138002365827560424804688e-01, // 1.300000 + 5.03414696208590473069932613725541159510612487792969e-01, // 1.400000 + 5.27633447258985310845957883429946377873420715332031e-01, // 1.500000 + 5.50671035882778325820652298716595396399497985839844e-01, // 1.600000 + 5.72585068051273404776679853966925293207168579101562e-01, // 1.700000 + 5.93430340259400890268182138242991641163825988769531e-01, // 1.800000 + 6.13258976545498657451105373183963820338249206542969e-01, // 1.900000 + 6.32120558828557665975722557050175964832305908203125e-01, // 2.000000 + 4.99998750002083400397704521234132357676571700721979e-06, // 0.000010 + 4.99999999987500026690715295475821817669515034765482e-11, // 0.000000 + 5.98617593408454426451292818001093110069632530212402e-02, // 0.123457 + 7.92120423649259608467332327563781291246414184570312e-01, // 3.141593 + 7.43118634686652401732942507806001231074333190917969e-01, // 2.718282 + 1.45364000846775420772161169225000776350498199462891e-01, // 0.314159 + 1.27082379621773322586619769936078228056430816650391e-01, // 0.271828 + }; + + public static final double[] GNUR_CHISQ_PDF_2 = { // + 4.75614712250357007938816877867793664336204528808594e-01, // 0.100000 + 4.52418709017979758790772848442429676651954650878906e-01, // 0.200000 + 4.30353988212528904000464535783976316452026367187500e-01, // 0.300000 + 4.09365376538990910493254204993718303740024566650391e-01, // 0.400000 + 3.89400391535702439238519900754909031093120574951172e-01, // 0.500000 + 3.70409110340858938048569370948825962841510772705078e-01, // 0.600000 + 3.52344044859356719801013468895689584314823150634766e-01, // 0.700000 + 3.35160023017819663770922034018440172076225280761719e-01, // 0.800000 + 3.18814075810886665696131103686639107763767242431641e-01, // 0.900000 + 3.03265329856316712131558688270160928368568420410156e-01, // 1.000000 + 2.88474905190243324781818046176340430974960327148438e-01, // 1.100000 + 2.74405818047013194682648418165626935660839080810547e-01, // 1.200000 + 2.61022888380508022443393656430998817086219787597656e-01, // 1.300000 + 2.48292651895704763465033693137229420244693756103516e-01, // 1.400000 + 2.36183276370507344577021058285026811063289642333984e-01, // 1.500000 + 2.24664482058610781578522619383875280618667602539062e-01, // 1.600000 + 2.13707465974363353122811304274364374577999114990234e-01, // 1.700000 + 2.03284829870299554865908930878504179418087005615234e-01, // 1.800000 + 1.93370511727250615763296082150191068649291992187500e-01, // 1.900000 + 1.83939720585721167012138721474912017583847045898438e-01, // 2.000000 + 4.99997500006249984139117259474005550146102905273438e-01, // 0.000010 + 4.99999999974999997931490725022740662097930908203125e-01, // 0.000000 + 4.70069120329577261330200599331874400377273559570312e-01, // 0.123457 + 1.03939788175370209644121644032566109672188758850098e-01, // 3.141593 + 1.28440682656673799133528746096999384462833404541016e-01, // 2.718282 + 4.27317999576612317369495031016413122415542602539062e-01, // 0.314159 + 4.36458810189113310951114499403047375380992889404297e-01, // 0.271828 + }; + + public static final double[] SCIPY_CHISQ_CDF_4 = { // + 1.20910427425029100463593767500469766673631966114044e-03, // 0.100000 + 4.67884016044447376136972138738201465457677841186523e-03, // 0.200000 + 1.01858271111835239214338244551072421018034219741821e-02, // 0.300000 + 1.75230963064217717950477037902601296082139015197754e-02, // 0.400000 + 2.64990211607439123120411039735699887387454509735107e-02, // 0.500000 + 3.69363131137667791148437856918462784960865974426270e-02, // 0.600000 + 4.86710788797368384961394838228443404659628868103027e-02, // 0.700000 + 6.15519355501049983403483167876402148976922035217285e-02, // 0.800000 + 7.54391801484287222168134690036822576075792312622070e-02, // 0.900000 + 9.02040104310498636053239351895172148942947387695312e-02, // 1.000000 + 1.05727793910245615460752333092386834323406219482422e-01, // 1.100000 + 1.21901382249557641013204545288317603990435600280762e-01, // 1.200000 + 1.38624468344323514834570687526138499379158020019531e-01, // 1.300000 + 1.55804983554603787565540073956071864813566207885742e-01, // 1.400000 + 1.73358532703224266224850680373492650687694549560547e-01, // 1.500000 + 1.91207864589001103050591723331308458000421524047852e-01, // 1.600000 + 2.09282375894855626752288912939548026770353317260742e-01, // 1.700000 + 2.27517646492861713714006555164814926683902740478516e-01, // 1.800000 + 2.45855004263722698443217495878343470394611358642578e-01, // 1.900000 + 2.64241117657115276440293882842524908483028411865234e-01, // 2.000000 + 1.24999583334114298362906795536861261509270493164081e-11, // 0.000010 + 1.24999999995833552159194802776351939183872839712033e-21, // 0.000000 + 1.82857697305291249344205706250932053080759942531586e-03, // 0.123457 + 4.65583948701781880075145636510569602251052856445312e-01, // 3.141593 + 3.93980660985851938171720121317775920033454895019531e-01, // 2.718282 + 1.11180920251080228577933439737535081803798675537109e-02, // 0.314159 + 8.44057436093374746644002470929990522563457489013672e-03, // 0.271828 + }; + + public static final double[] SCIPY_CHISQ_PDF_4 = { // + 2.37807356125178670502862132707377895712852478027344e-02, // 0.100000 + 4.52418709017979800424136271885799942538142204284668e-02, // 0.200000 + 6.45530982318793383756272419304877985268831253051758e-02, // 0.300000 + 8.18730753077981932008810872503090649843215942382812e-02, // 0.400000 + 9.73500978839256236874177830031840130686759948730469e-02, // 0.500000 + 1.11122733102257703619031303787778597325086593627930e-01, // 0.600000 + 1.23320415700774871359257645053730811923742294311523e-01, // 0.700000 + 1.34064009207127854406138567355810664594173431396484e-01, // 0.800000 + 1.43466334114899007889931681347661651670932769775391e-01, // 0.900000 + 1.51632664928158356065779344135080464184284210205078e-01, // 1.000000 + 1.58661197854633867487805787277466151863336563110352e-01, // 1.100000 + 1.64643490828207922360704174025158863514661788940430e-01, // 1.200000 + 1.69664877447330247894896615434845443814992904663086e-01, // 1.300000 + 1.73804856326993356629984077699191402643918991088867e-01, // 1.400000 + 1.77137457277880522310553601528226863592863082885742e-01, // 1.500000 + 1.79731585646888653018393711136013735085725784301758e-01, // 1.600000 + 1.81651346078208847378832047070318367332220077514648e-01, // 1.700000 + 1.82956346883269588277087791539088357239961624145508e-01, // 1.800000 + 1.83701986140888090526246401168464217334985733032227e-01, // 1.900000 + 1.83939720585721139256563105845998506993055343627930e-01, // 2.000000 + 2.49998750003124934010532293138240333973953966051340e-06, // 0.000010 + 2.49999999987499629841753832369663826905303860570484e-11, // 0.000000 + 2.90165911838962502222738493173892493359744548797607e-02, // 0.123457 + 1.63268237473728039521603250250336714088916778564453e-01, // 3.141593 + 1.74568986850338975225227500231994781643152236938477e-01, // 2.718282 + 6.71229544108292858206610276283754501491785049438477e-02, // 0.314159 + 5.93209026303989431228025352993427077308297157287598e-02, // 0.271828 + }; + + public static final double[] GNUR_CHISQ_CDF_4 = { // + 1.20910427425029057095506868080292406375519931316376e-03, // 0.100000 + 4.67884016044447115928450742217137303669005632400513e-03, // 0.200000 + 1.01858271111835187172633965246859588660299777984619e-02, // 0.300000 + 1.75230963064217613867068479294175631366670131683350e-02, // 0.400000 + 2.64990211607439261898289117880267440341413021087646e-02, // 0.500000 + 3.69363131137667860537376895990746561437845230102539e-02, // 0.600000 + 4.86710788797368662517150994517578510567545890808105e-02, // 0.700000 + 6.15519355501050191570300285093253478407859802246094e-02, // 0.800000 + 7.54391801484287360946012768181390129029750823974609e-02, // 0.900000 + 9.02040104310498774831117430039739701896905899047852e-02, // 1.000000 + 1.05727793910245615460752333092386834323406219482422e-01, // 1.100000 + 1.21901382249557682646567968731687869876623153686523e-01, // 1.200000 + 1.38624468344323542590146303155052009969949722290039e-01, // 1.300000 + 1.55804983554603898587842536471725907176733016967773e-01, // 1.400000 + 1.73358532703224266224850680373492650687694549560547e-01, // 1.500000 + 1.91207864589001158561742954589135479182004928588867e-01, // 1.600000 + 2.09282375894855654507864528568461537361145019531250e-01, // 1.700000 + 2.27517646492861658202855323906987905502319335937500e-01, // 1.800000 + 2.45855004263722753954368727136170491576194763183594e-01, // 1.900000 + 2.64241117657115276440293882842524908483028411865234e-01, // 2.000000 + 1.24999583334114362986392151242148360838074561129929e-11, // 0.000010 + 1.24999999995832724611171823885946329193350635251076e-21, // 0.000000 + 1.82857697305291162608031907410577332484535872936249e-03, // 0.123457 + 4.65583948701815852899699166300706565380096435546875e-01, // 3.141593 + 3.93980660986018471625413894798839464783668518066406e-01, // 2.718282 + 1.11180920251094158407445533498503209557384252548218e-02, // 0.314159 + 8.44057436093940786914213703084897133521735668182373e-03, // 0.271828 + }; + + public static final double[] GNUR_CHISQ_PDF_4 = { // + 2.37807356125178601113923093635094119235873222351074e-02, // 0.100000 + 4.52418709017979661646258193741232389584183692932129e-02, // 0.200000 + 6.45530982318793522534150497449445538222789764404297e-02, // 0.300000 + 8.18730753077982070786688950647658202797174453735352e-02, // 0.400000 + 9.73500978839256236874177830031840130686759948730469e-02, // 0.500000 + 1.11122733102257675863455688158865086734294891357422e-01, // 0.600000 + 1.23320415700774885237045452868187567219138145446777e-01, // 0.700000 + 1.34064009207127882161714182984724175184965133666992e-01, // 0.800000 + 1.43466334114899007889931681347661651670932769775391e-01, // 0.900000 + 1.51632664928158356065779344135080464184284210205078e-01, // 1.000000 + 1.58661197854633839732230171648552641272544860839844e-01, // 1.100000 + 1.64643490828207977871855405282985884696245193481445e-01, // 1.200000 + 1.69664877447330220139320999805931933224201202392578e-01, // 1.300000 + 1.73804856326993328874408462070277892053127288818359e-01, // 1.400000 + 1.77137457277880550066129217157140374183654785156250e-01, // 1.500000 + 1.79731585646888653018393711136013735085725784301758e-01, // 1.600000 + 1.81651346078208847378832047070318367332220077514648e-01, // 1.700000 + 1.82956346883269616032663407168001867830753326416016e-01, // 1.800000 + 1.83701986140888090526246401168464217334985733032227e-01, // 1.900000 + 1.83939720585721167012138721474912017583847045898438e-01, // 2.000000 + 2.49998750003124891658884930423223380557828932069242e-06, // 0.000010 + 2.49999999987500469947063456538396118179756744126507e-11, // 0.000000 + 2.90165911838962536917208012710034381598234176635742e-02, // 0.123457 + 1.63268237473721905539392196260450873523950576782227e-01, // 3.141593 + 1.74568986850316937298188690874667372554540634155273e-01, // 2.718282 + 6.71229544108329911900057140883291140198707580566406e-02, // 0.314159 + 5.93209026304169703691648862786678364500403404235840e-02, // 0.271828 + }; + + public static final double[] SCIPY_CHISQ_CDF_10 = { // + 2.49795133600650746949935939182803334990268240289879e-09, // 0.100000 + 7.66780168618932654114059375288769970779867435339838e-08, // 0.200000 + 5.58580784810275424538130676493929982484587526414543e-07, // 0.300000 + 2.25819055295782344450553069670206696173409000039101e-06, // 0.400000 + 6.61171056103424409242054135549260251991654513403773e-06, // 0.500000 + 1.57850405416599700160560498796158412915247026830912e-05, // 0.600000 + 3.27361747531325168996535479593035233847331255674362e-05, // 0.700000 + 6.12433271023337950488690806238878394651692360639572e-05, // 0.800000 + 1.05904769283482884140341895307102504375507123768330e-04, // 0.900000 + 1.72115629955840721259513470542401591956149786710739e-04, // 1.000000 + 2.66026240432705039958005688305320290965028107166290e-04, // 1.100000 + 3.94486018340255682485678834225950595282483845949173e-04, // 1.200000 + 5.64976348978787402434209585777580286958254873752594e-04, // 1.300000 + 7.85535448953023811638485263131315150531008839607239e-04, // 1.400000 + 1.06467777278579280725301359211698581930249929428101e-03, // 1.500000 + 1.41131014588673502248294955307983400416560471057892e-03, // 1.600000 + 1.83464647201952166591376602866603207075968384742737e-03, // 1.700000 + 2.34412256627762561819738884594244154868647456169128e-03, // 1.800000 + 2.94931240186648664300395061843573785154148936271667e-03, // 1.900000 + 3.65984682734371308959753221756727725733071565628052e-03, // 2.000000 + 2.60415581599546609733287774064553218283559941928103e-29, // 0.000010 + 2.60416666655820244729903965275602508551632679775379e-54, // 0.000000 + 7.09444350487475695446226334845915562077323102130322e-09, // 0.123457 + 2.21073482327962915483521300075153703801333904266357e-02, // 3.141593 + 1.27010572812203369197359137388048111461102962493896e-02, // 2.718282 + 6.99320669493774398455519271389224655877114855684340e-07, // 0.314159 + 3.45169511596339609613224696377242750600089493673295e-07, // 0.271828 + }; + + public static final double[] SCIPY_CHISQ_PDF_10 = { // + 1.23857997981863822630169027876056997428122485871427e-07, // 0.100000 + 1.88507795424158187552754212951144907606249034870416e-06, // 0.200000 + 9.07777943885801461183782112840390254859812557697296e-06, // 0.300000 + 2.72910251025994052959371821698297821967571508139372e-05, // 0.400000 + 6.33789699765140417814235362925501249264925718307495e-05, // 0.500000 + 1.25013074740039885346387649178723222576081752777100e-04, // 0.600000 + 2.20306784298780056327235477020565213024383410811424e-04, // 0.700000 + 3.57504024552340871068167871271725744009017944335938e-04, // 0.800000 + 5.44723737342507358799581496100472577381879091262817e-04, // 0.900000 + 7.89753463167491365562455918336581817129626870155334e-04, // 1.000000 + 1.09988569971103010078361528201185137731954455375671e-03, // 1.100000 + 1.48179141745387061429584107941082038450986146926880e-03, // 1.200000 + 1.94142570704054513025527128888825245667248964309692e-03, // 1.300000 + 2.48396107167327909875931268857129907701164484024048e-03, // 1.400000 + 3.11374436621274317676943965693681093398481607437134e-03, // 1.500000 + 3.83427382713362290000969068159974995069205760955811e-03, // 1.600000 + 4.64819303792833159877373461199567827861756086349487e-03, // 1.700000 + 5.55729903657931241511835906976557453162968158721924e-03, // 1.800000 + 6.56256209864766217043641205464155063964426517486572e-03, // 1.900000 + 7.66415502440504978198809737932606367394328117370605e-03, // 2.000000 + 1.30207682293293247342964777423805818639678689634258e-23, // 0.000010 + 1.30208333326823731527652024476599117068802519713176e-43, // 0.000000 + 2.84373520508469333784316900384792070610728842439130e-07, // 0.123457 + 2.63663549177880519869265896204524324275553226470947e-02, // 3.141593 + 1.82620407865934676616692655670703970827162265777588e-02, // 2.718282 + 1.08397546669860501598013635682349331545992754399776e-05, // 0.314159 + 6.20568843801925594646975298251945218908076640218496e-06, // 0.271828 + }; + + public static final double[] GNUR_CHISQ_CDF_10 = { // + 2.49795133600649837051262130852360976440706963330740e-09, // 0.100000 + 7.66780168618931198276181281960062197100569392205216e-08, // 0.200000 + 5.58580784810275424538130676493929982484587526414543e-07, // 0.300000 + 2.25819055295782302098905706955189742757283966057003e-06, // 0.400000 + 6.61171056103425340978296115279633227146405261009932e-06, // 0.500000 + 1.57850405416599598516606828280117724716546945273876e-05, // 0.600000 + 3.27361747531325507809714381313170861176331527531147e-05, // 0.700000 + 6.12433271023336866286518320734444387198891490697861e-05, // 0.800000 + 1.05904769283483060323194924201573030586587265133858e-04, // 0.900000 + 1.72115629955840395998861724891071389720309525728226e-04, // 1.000000 + 2.66026240432705039958005688305320290965028107166290e-04, // 1.100000 + 3.94486018340255411435135712849842093419283628463745e-04, // 1.200000 + 5.64976348978787619274644082878467088448815047740936e-04, // 1.300000 + 7.85535448953025112681092245736635959474369883537292e-04, // 1.400000 + 1.06467777278579280725301359211698581930249929428101e-03, // 1.500000 + 1.41131014588673415512121156467628679820336401462555e-03, // 1.600000 + 1.83464647201952231643506951996869247523136436939240e-03, // 1.700000 + 2.34412256627762518451651985174066794570535421371460e-03, // 1.800000 + 2.94931240186649314821698553146234189625829458236694e-03, // 1.900000 + 3.65984682734371439064013920017259806627407670021057e-03, // 2.000000 + 2.60415581599546665785226347057236055232743273524748e-29, // 0.000010 + 2.60416666655812826317602590432831045081109584496352e-54, // 0.000000 + 7.09444350487472965750204909854588486428639271252905e-09, // 0.123457 + 2.21073482328017455189606010890202014707028865814209e-02, // 3.141593 + 1.27010572812377743601164326037178398109972476959229e-02, // 2.718282 + 6.99320669493999391582133694916789679041357885580510e-07, // 0.314159 + 3.45169511596932850313657926977217726971502997912467e-07, // 0.271828 + }; + + public static final double[] GNUR_CHISQ_PDF_10 = { // + 1.23857997981864007918626239754256168623669509543106e-07, // 0.100000 + 1.88507795424158484014285751956263581519124272745103e-06, // 0.200000 + 9.07777943885802816436497719720932764175813645124435e-06, // 0.300000 + 2.72910251025994493416504393934474137495271861553192e-05, // 0.400000 + 6.33789699765140824390050044989664002059726044535637e-05, // 0.500000 + 1.25013074740039749821116088490668971644481644034386e-04, // 0.600000 + 2.20306784298780218957561349846230314142303541302681e-04, // 0.700000 + 3.57504024552340437387298877069952141027897596359253e-04, // 0.800000 + 5.44723737342507684060233241751802779617719352245331e-04, // 0.900000 + 7.89753463167491907663542161088798820856027305126190e-04, // 1.000000 + 1.09988569971102879974100829940653056837618350982666e-03, // 1.100000 + 1.48179141745387169849801356491525439196266233921051e-03, // 1.200000 + 1.94142570704054556393614028309002605965360999107361e-03, // 1.300000 + 2.48396107167327953244018168277307267999276518821716e-03, // 1.400000 + 3.11374436621274404413117764534035813994705677032471e-03, // 1.500000 + 3.83427382713362723681838062361748598050326108932495e-03, // 1.600000 + 4.64819303792833680294416254241696151439100503921509e-03, // 1.700000 + 5.55729903657931414984183504657266894355416297912598e-03, // 1.800000 + 6.56256209864766303779815004304509784560650587081909e-03, // 1.900000 + 7.66415502440505238407331134453670529183000326156616e-03, // 2.000000 + 1.30207682293293908558537114960529051053980877147452e-23, // 0.000010 + 1.30208333326823472650216465453104099875737882460008e-43, // 0.000000 + 2.84373520508469280844757696991020878840572549961507e-07, // 0.123457 + 2.63663549177922673649732132616918534040451049804688e-02, // 3.141593 + 1.82620407866104020322417511579260462895035743713379e-02, // 2.718282 + 1.08397546669888199575388851303436865691764978691936e-05, // 0.314159 + 6.20568843802768815945966954239487733957503223791718e-06, // 0.271828 + }; + + public static final double[] P_QUANT = { // + 0.0001, 0.001, 0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99, 0.999, 0.9999 // + }; + + public static final double[] SCIPY_CHISQ_QUANT_01 = { // + 1.16892641156733644781219710813445996321826192936318e-80, // 0.000100 + 1.16892641146821240352815603348024825656653422803765e-60, // 0.001000 + 1.16892641145814869325987350586058419394529211182657e-40, // 0.010000 + 1.16892641145721739436487227654675467673153240504758e-20, // 0.100000 + 1.06313237798343417157777858600350238808297798076552e-12, // 0.250000 + 1.11477568814925462586246096563513674482237547636032e-06, // 0.500000 + 3.71347136769252343657665704768078285269439220428467e-03, // 0.750000 + 1.52634227818377987695441788673633709549903869628906e-01, // 0.900000 + 2.17525480018361871970000720466487109661102294921875e+00, // 0.990000 + 5.47291719745734983604279477731324732303619384765625e+00, // 0.999000 + 9.24820164420099821711573895299807190895080566406250e+00, // 0.999900 + }; + + public static final double[] GNUR_CHISQ_QUANT_01 = { // + 1.16892641145733484384707827779994069422035569294114e-80, // 0.000100 + 1.16892641145731079523768921414073055688308402765104e-60, // 0.001000 + 1.16892641145731997959243769324569198659341653028700e-40, // 0.010000 + 1.16892641145730436213892351266574423573550225567180e-20, // 0.100000 + 1.06313237798339660917691558230537590321561347561641e-12, // 0.250000 + 1.11477568814925102597243513485869570445174758788198e-06, // 0.500000 + 3.71347136769249915044799337238146108575165271759033e-03, // 0.750000 + 1.52634227818377238294900166692968923598527908325195e-01, // 0.900000 + 2.17525480018361827561079735460225492715835571289062e+00, // 0.990000 + 5.47291719745735161239963417756371200084686279296875e+00, // 0.999000 + 9.24820164420099821711573895299807190895080566406250e+00, // 0.999900 + }; + + public static final double[] SCIPY_CHISQ_QUANT_1 = { // + 1.57079633500628175845386844643181145642074625357054e-08, // 0.000100 + 1.57079714926323284399762268692679612058782367967069e-06, // 0.001000 + 1.57087857909697853949632095904576090106274932622910e-04, // 0.010000 + 1.57907740934312285085994176370149943977594375610352e-02, // 0.100000 + 1.01531044267621481380636794256133725866675376892090e-01, // 0.250000 + 4.54936423119572663775755927417776547372341156005859e-01, // 0.500000 + 1.32330369693146554510576606844551861286163330078125e+00, // 0.750000 + 2.70554345409541641132022959936875849962234497070312e+00, // 0.900000 + 6.63489660102121447948775312397629022598266601562500e+00, // 0.990000 + 1.08275661706627310820749698905274271965026855468750e+01, // 0.999000 + 1.51367052266236044033576035872101783752441406250000e+01, // 0.999900 + }; + + public static final double[] GNUR_CHISQ_QUANT_1 = { // + 1.57079633501956727170820513452345612570582034095423e-08, // 0.000100 + 1.57079714926249020786111747910451796883535280358046e-06, // 0.001000 + 1.57087857909702001022941852959036168613238260149956e-04, // 0.010000 + 1.57907740934312285085994176370149943977594375610352e-02, // 0.100000 + 1.01531044267621550769575833328417502343654632568359e-01, // 0.250000 + 4.54936423119572830309209621191257610917091369628906e-01, // 0.500000 + 1.32330369693146598919497591850813478231430053710938e+00, // 0.750000 + 2.70554345409541507905260004918090999126434326171875e+00, // 0.900000 + 6.63489660102121181495249402360059320926666259765625e+00, // 0.990000 + 1.08275661706627310820749698905274271965026855468750e+01, // 0.999000 + 1.51367052266236044033576035872101783752441406250000e+01, // 0.999900 + }; + + public static final double[] SCIPY_CHISQ_QUANT_2 = { // + 2.00010000666460950081992908877737136208452284336090e-04, // 0.000100 + 2.00100066716752279100122180466314603108912706375122e-03, // 0.001000 + 2.01006717070028942395687465705123031511902809143066e-02, // 0.010000 + 2.10721031315652534976479159922746475785970687866211e-01, // 0.100000 + 5.75364144903561913757528145652031525969505310058594e-01, // 0.250000 + 1.38629436111989057245352796599036082625389099121094e+00, // 0.500000 + 2.77258872223978158899626578204333782196044921875000e+00, // 0.750000 + 4.60517018598809180218722758581861853599548339843750e+00, // 0.900000 + 9.21034037197618005166077637113630771636962890625000e+00, // 0.990000 + 1.38155105579642718538480039569549262523651123046875e+01, // 0.999000 + 1.84206807439525839242833171738311648368835449218750e+01, // 0.999900 + }; + + public static final double[] GNUR_CHISQ_QUANT_2 = { // + 2.00010000666716686269427927236108644137857481837273e-04, // 0.000100 + 2.00100066716706699240790889859908929793164134025574e-03, // 0.001000 + 2.01006717070028838312278907096697366796433925628662e-02, // 0.010000 + 2.10721031315652618243206006809487007558345794677734e-01, // 0.100000 + 5.75364144903561802735225683136377483606338500976562e-01, // 0.250000 + 1.38629436111989057245352796599036082625389099121094e+00, // 0.500000 + 2.77258872223978114490705593198072165250778198242188e+00, // 0.750000 + 4.60517018598809180218722758581861853599548339843750e+00, // 0.900000 + 9.21034037197618005166077637113630771636962890625000e+00, // 0.990000 + 1.38155105579642718538480039569549262523651123046875e+01, // 0.999000 + 1.84206807439525839242833171738311648368835449218750e+01, // 0.999900 + }; + + public static final double[] SCIPY_CHISQ_QUANT_4 = { // + 2.84184752435540088910670419863890856504440307617188e-02, // 0.000100 + 9.08040355389811981723369171959348022937774658203125e-02, // 0.001000 + 2.97109480506531686838656014515436254441738128662109e-01, // 0.010000 + 1.06362321677922411211625330906827002763748168945312e+00, // 0.100000 + 1.92255752622955422559414273564470931887626647949219e+00, // 0.250000 + 3.35669398003332153379574265272822231054306030273438e+00, // 0.500000 + 5.38526905777939113306729268515482544898986816406250e+00, // 0.750000 + 7.77944033973485904454037154209800064563751220703125e+00, // 0.900000 + 1.32767041359876234452030985266901552677154541015625e+01, // 0.990000 + 1.84668269529031690012743638362735509872436523437500e+01, // 0.999000 + 2.35127424449910797932261630194261670112609863281250e+01, // 0.999900 + }; + + public static final double[] GNUR_CHISQ_QUANT_4 = { // + 2.84184752435549976834483487664329004473984241485596e-02, // 0.000100 + 9.08040355389791442597413606563350185751914978027344e-02, // 0.001000 + 2.97109480506531964394412170804571360349655151367188e-01, // 0.010000 + 1.06362321677922389007164838403696194291114807128906e+00, // 0.100000 + 1.92255752622955400354953781061340123414993286132812e+00, // 0.250000 + 3.35669398003332108970653280266560614109039306640625e+00, // 0.500000 + 5.38526905777939202124571238528005778789520263671875e+00, // 0.750000 + 7.77944033973485904454037154209800064563751220703125e+00, // 0.900000 + 1.32767041359876216688462591264396905899047851562500e+01, // 0.990000 + 1.84668269529031690012743638362735509872436523437500e+01, // 0.999000 + 2.35127424449910762405124842189252376556396484375000e+01, // 0.999900 + }; + + public static final double[] SCIPY_CHISQ_QUANT_10 = { // + 8.88920357912773129172023800492752343416213989257812e-01, // 0.000100 + 1.47874346383567578655515717400703579187393188476562e+00, // 0.001000 + 2.55821216018720765106309045222587883472442626953125e+00, // 0.010000 + 4.86518205192532882108480407623574137687683105468750e+00, // 0.100000 + 6.73720077195464384089973464142531156539916992187500e+00, // 0.250000 + 9.34181776559196919151872862130403518676757812500000e+00, // 0.500000 + 1.25488613968893769623491607489995658397674560546875e+01, // 0.750000 + 1.59871791721052627366361775784753262996673583984375e+01, // 0.900000 + 2.32092511589543590844186837784945964813232421875000e+01, // 0.990000 + 2.95882984450744146442957571707665920257568359375000e+01, // 0.999000 + 3.55640139419523890751406725030392408370971679687500e+01, // 0.999900 + }; + + public static final double[] GNUR_CHISQ_QUANT_10 = { // + 8.88920357912895919838547342806123197078704833984375e-01, // 0.000100 + 1.47874346383566490636951584747293964028358459472656e+00, // 0.001000 + 2.55821216018720631879546090203803032636642456054688e+00, // 0.010000 + 4.86518205192532882108480407623574137687683105468750e+00, // 0.100000 + 6.73720077195464206454289524117484688758850097656250e+00, // 0.250000 + 9.34181776559196741516188922105357050895690917968750e+00, // 0.500000 + 1.25488613968893769623491607489995658397674560546875e+01, // 0.750000 + 1.59871791721052591839224987779743969440460205078125e+01, // 0.900000 + 2.32092511589543590844186837784945964813232421875000e+01, // 0.990000 + 2.95882984450744181970094359712675213813781738281250e+01, // 0.999000 + 3.55640139419523890751406725030392408370971679687500e+01, // 0.999900 + }; + + @Test + public void testPDF() { + checkPDF(new ChiSquaredDistribution(1.), P_CDFPDF, SCIPY_CHISQ_PDF_1, 1e-12); + checkPDF(new ChiSquaredDistribution(2.), P_CDFPDF, SCIPY_CHISQ_PDF_2, 1e-12); + checkPDF(new ChiSquaredDistribution(4.), P_CDFPDF, SCIPY_CHISQ_PDF_4, 1e-12); + checkPDF(new ChiSquaredDistribution(10), P_CDFPDF, SCIPY_CHISQ_PDF_10, 1e-12); + checkPDF(new ChiSquaredDistribution(.1), P_CDFPDF, SCIPY_CHISQ_PDF_01, 1e-12); + checkPDF(new ChiSquaredDistribution(1.), P_CDFPDF, GNUR_CHISQ_PDF_1, 1e-14); + checkPDF(new ChiSquaredDistribution(2.), P_CDFPDF, GNUR_CHISQ_PDF_2, 1e-15); + checkPDF(new ChiSquaredDistribution(4.), P_CDFPDF, GNUR_CHISQ_PDF_4, 1e-15); + checkPDF(new ChiSquaredDistribution(10), P_CDFPDF, GNUR_CHISQ_PDF_10, 1e-15); + checkPDF(new ChiSquaredDistribution(.1), P_CDFPDF, GNUR_CHISQ_PDF_01, 1e-14); + } + + @Test + public void testCDF() { + checkCDF(new ChiSquaredDistribution(1.), P_CDFPDF, SCIPY_CHISQ_CDF_1, 1e-12); + checkCDF(new ChiSquaredDistribution(2.), P_CDFPDF, SCIPY_CHISQ_CDF_2, 1e-12); + checkCDF(new ChiSquaredDistribution(4.), P_CDFPDF, SCIPY_CHISQ_CDF_4, 1e-12); + checkCDF(new ChiSquaredDistribution(10), P_CDFPDF, SCIPY_CHISQ_CDF_10, 1e-11); + checkCDF(new ChiSquaredDistribution(.1), P_CDFPDF, SCIPY_CHISQ_CDF_01, 1e-13); + checkCDF(new ChiSquaredDistribution(1.), P_CDFPDF, GNUR_CHISQ_CDF_1, 1e-15); + checkCDF(new ChiSquaredDistribution(2.), P_CDFPDF, GNUR_CHISQ_CDF_2, 1e-15); + checkCDF(new ChiSquaredDistribution(4.), P_CDFPDF, GNUR_CHISQ_CDF_4, 1e-15); + checkCDF(new ChiSquaredDistribution(10), P_CDFPDF, GNUR_CHISQ_CDF_10, 1e-15); + checkCDF(new ChiSquaredDistribution(.1), P_CDFPDF, GNUR_CHISQ_CDF_01, 1e-14); + } + + @Test + public void testQuantile() { + checkQuantile(new ChiSquaredDistribution(1.), P_QUANT, SCIPY_CHISQ_QUANT_1, 1e-13); + checkQuantile(new ChiSquaredDistribution(2.), P_QUANT, SCIPY_CHISQ_QUANT_2, 1e-13); + checkQuantile(new ChiSquaredDistribution(4.), P_QUANT, SCIPY_CHISQ_QUANT_4, 1e-13); + checkQuantile(new ChiSquaredDistribution(10), P_QUANT, SCIPY_CHISQ_QUANT_10, 1e-12); + checkQuantile(new ChiSquaredDistribution(.1), P_QUANT, SCIPY_CHISQ_QUANT_01, 1e-13); + checkQuantile(new ChiSquaredDistribution(1.), P_QUANT, GNUR_CHISQ_QUANT_1, 1e-13); + checkQuantile(new ChiSquaredDistribution(2.), P_QUANT, GNUR_CHISQ_QUANT_2, 1e-14); + checkQuantile(new ChiSquaredDistribution(4.), P_QUANT, GNUR_CHISQ_QUANT_4, 1e-13); + checkQuantile(new ChiSquaredDistribution(10), P_QUANT, GNUR_CHISQ_QUANT_10, 1e-13); + checkQuantile(new ChiSquaredDistribution(.1), P_QUANT, GNUR_CHISQ_QUANT_01, 1e-13); + } +}
\ No newline at end of file diff --git a/test/de/lmu/ifi/dbs/elki/math/statistics/distribution/TestGammaDistribution.java b/test/de/lmu/ifi/dbs/elki/math/statistics/distribution/TestGammaDistribution.java new file mode 100644 index 00000000..8ec972f3 --- /dev/null +++ b/test/de/lmu/ifi/dbs/elki/math/statistics/distribution/TestGammaDistribution.java @@ -0,0 +1,1304 @@ +package de.lmu.ifi.dbs.elki.math.statistics.distribution; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +import static org.junit.Assert.assertEquals; + +import java.util.Random; + +import org.junit.Test; + +import de.lmu.ifi.dbs.elki.JUnit4Test; + +/** + * Unit test for the Gamma distribution in ELKI. + * + * The reference values were computed using GNU R and SciPy. + * + * @author Erich Schubert + */ +public class TestGammaDistribution extends AbstractDistributionTest implements JUnit4Test { + public static final double[] P_CDFPDF = { // + 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0, 1e-05, 1e-10, 0.1234567, 3.14159265359, 2.71828182846, 0.314159265359, 0.271828182846 // + }; + + public static final double[] SCIPY_GAMMA_CDF_1_1 = { // + 9.51625819640404407850908796717703808099031448364258e-02, // 0.100000 + 1.81269246922018151257915974383649881929159164428711e-01, // 0.200000 + 2.59181779318282068391710026844521053135395050048828e-01, // 0.300000 + 3.29679953964360561435853469447465613484382629394531e-01, // 0.400000 + 3.93469340287366520225731392201851122081279754638672e-01, // 0.500000 + 4.51188363905973610634703163668746128678321838378906e-01, // 0.600000 + 5.03414696208590473069932613725541159510612487792969e-01, // 0.700000 + 5.50671035882778325820652298716595396399497985839844e-01, // 0.800000 + 5.93430340259400890268182138242991641163825988769531e-01, // 0.900000 + 6.32120558828557776998025019565830007195472717285156e-01, // 1.000000 + 6.67128916301920504849931603530421853065490722656250e-01, // 1.100000 + 6.98805788087797807861534238327294588088989257812500e-01, // 1.200000 + 7.27468206965987351786395720409927889704704284667969e-01, // 1.300000 + 7.53403036058393538176858328370144590735435485839844e-01, // 1.400000 + 7.76869839851570209710018843907164409756660461425781e-01, // 1.500000 + 7.98103482005344533334323386952746659517288208007812e-01, // 1.600000 + 8.17316475947265308477085454796906560659408569335938e-01, // 1.700000 + 8.34701111778413440411839019361650571227073669433594e-01, // 1.800000 + 8.50431380777364909206994525447953492403030395507812e-01, // 1.900000 + 8.64664716763387297682186272140825167298316955566406e-01, // 2.000000 + 9.99995000016666799329441428501041855270159430801868e-06, // 0.000010 + 9.99999999950001362800370980511287678088594432779246e-11, // 0.000000 + 1.16140088450309555967443486679258057847619056701660e-01, // 0.123457 + 9.56786081736227700389463279861956834793090820312500e-01, // 3.141593 + 9.34011964154687457373427150741918012499809265136719e-01, // 2.718282 + 2.69597308951354475148320943844737485051155090332031e-01, // 0.314159 + 2.38014828033141445651210688083665445446968078613281e-01, // 0.271828 + }; + + public static final double[] SCIPY_GAMMA_PDF_1_1 = { // + 9.04837418035959517581545696884859353303909301757812e-01, // 0.100000 + 8.18730753077981820986508409987436607480049133300781e-01, // 0.200000 + 7.40818220681717876097138741897651925683021545410156e-01, // 0.300000 + 6.70320046035639327541844068036880344152450561523438e-01, // 0.400000 + 6.06530659712633424263117376540321856737136840820312e-01, // 0.500000 + 5.48811636094026389365296836331253871321678161621094e-01, // 0.600000 + 4.96585303791409526930067386274458840489387512207031e-01, // 0.700000 + 4.49328964117221563157045238767750561237335205078125e-01, // 0.800000 + 4.06569659740599109731817861757008358836174011230469e-01, // 0.900000 + 3.67879441171442334024277442949824035167694091796875e-01, // 1.000000 + 3.32871083698079550661219627727405168116092681884766e-01, // 1.100000 + 3.01194211912202136627314530414878390729427337646484e-01, // 1.200000 + 2.72531793034012592702453048332245089113712310791016e-01, // 1.300000 + 2.46596963941606489578717287258768919855356216430664e-01, // 1.400000 + 2.23130160148429818045556771721749100834131240844727e-01, // 1.500000 + 2.01896517994655383398949766160512808710336685180664e-01, // 1.600000 + 1.82683524052734663767338929574179928749799728393555e-01, // 1.700000 + 1.65298888221586531832585365009435918182134628295898e-01, // 1.800000 + 1.49568619222635063037429858923132997006177902221680e-01, // 1.900000 + 1.35335283236612702317813727859174832701683044433594e-01, // 2.000000 + 9.99990000049999827602675850357627496123313903808594e-01, // 0.000010 + 9.99999999899999991725962900090962648391723632812500e-01, // 0.000000 + 8.83859911549690457910344321135198697447776794433594e-01, // 0.123457 + 4.32139182637722579771732966946728993207216262817383e-02, // 3.141593 + 6.59880358453125426265728492580819875001907348632812e-02, // 2.718282 + 7.30402691048645635873981518670916557312011718750000e-01, // 0.314159 + 7.61985171966858665371091774431988596916198730468750e-01, // 0.271828 + }; + + public static final double[] GNUR_GAMMA_CDF_1_1 = { // + 9.51625819640404269073030718573136255145072937011719e-02, // 0.100000 + 1.81269246922018151257915974383649881929159164428711e-01, // 0.200000 + 2.59181779318282123902861258102348074316978454589844e-01, // 0.300000 + 3.29679953964360727969307163220946677029132843017578e-01, // 0.400000 + 3.93469340287366575736882623459678143262863159179688e-01, // 0.500000 + 4.51188363905973610634703163668746128678321838378906e-01, // 0.600000 + 5.03414696208590473069932613725541159510612487792969e-01, // 0.700000 + 5.50671035882778325820652298716595396399497985839844e-01, // 0.800000 + 5.93430340259400890268182138242991641163825988769531e-01, // 0.900000 + 6.32120558828557665975722557050175964832305908203125e-01, // 1.000000 + 6.67128916301920504849931603530421853065490722656250e-01, // 1.100000 + 6.98805788087797807861534238327294588088989257812500e-01, // 1.200000 + 7.27468206965987462808698182925581932067871093750000e-01, // 1.300000 + 7.53403036058393538176858328370144590735435485839844e-01, // 1.400000 + 7.76869839851570209710018843907164409756660461425781e-01, // 1.500000 + 7.98103482005344644356625849468400701880455017089844e-01, // 1.600000 + 8.17316475947265308477085454796906560659408569335938e-01, // 1.700000 + 8.34701111778413440411839019361650571227073669433594e-01, // 1.800000 + 8.50431380777364909206994525447953492403030395507812e-01, // 1.900000 + 8.64664716763387297682186272140825167298316955566406e-01, // 2.000000 + 9.99995000016666799329441428501041855270159430801868e-06, // 0.000010 + 9.99999999950000070330663866405545691512513073462287e-11, // 0.000000 + 1.16140088450309583723019102308171568438410758972168e-01, // 0.123457 + 9.56786081736236693195962743629934266209602355957031e-01, // 3.141593 + 9.34011964154750407018923397117760032415390014648438e-01, // 2.718282 + 2.69597308951369518670304614715860225260257720947266e-01, // 0.314159 + 2.38014828033214137503748020208149682730436325073242e-01, // 0.271828 + }; + + public static final double[] GNUR_GAMMA_PDF_1_1 = { // + 9.04837418035959517581545696884859353303909301757812e-01, // 0.100000 + 8.18730753077981820986508409987436607480049133300781e-01, // 0.200000 + 7.40818220681717876097138741897651925683021545410156e-01, // 0.300000 + 6.70320046035639327541844068036880344152450561523438e-01, // 0.400000 + 6.06530659712633424263117376540321856737136840820312e-01, // 0.500000 + 5.48811636094026389365296836331253871321678161621094e-01, // 0.600000 + 4.96585303791409526930067386274458840489387512207031e-01, // 0.700000 + 4.49328964117221563157045238767750561237335205078125e-01, // 0.800000 + 4.06569659740599109731817861757008358836174011230469e-01, // 0.900000 + 3.67879441171442334024277442949824035167694091796875e-01, // 1.000000 + 3.32871083698079550661219627727405168116092681884766e-01, // 1.100000 + 3.01194211912202136627314530414878390729427337646484e-01, // 1.200000 + 2.72531793034012592702453048332245089113712310791016e-01, // 1.300000 + 2.46596963941606489578717287258768919855356216430664e-01, // 1.400000 + 2.23130160148429818045556771721749100834131240844727e-01, // 1.500000 + 2.01896517994655383398949766160512808710336685180664e-01, // 1.600000 + 1.82683524052734663767338929574179928749799728393555e-01, // 1.700000 + 1.65298888221586531832585365009435918182134628295898e-01, // 1.800000 + 1.49568619222635063037429858923132997006177902221680e-01, // 1.900000 + 1.35335283236612702317813727859174832701683044433594e-01, // 2.000000 + 9.99990000049999827602675850357627496123313903808594e-01, // 0.000010 + 9.99999999899999991725962900090962648391723632812500e-01, // 0.000000 + 8.83859911549690457910344321135198697447776794433594e-01, // 0.123457 + 4.32139182637633137429311602772941114380955696105957e-02, // 3.141593 + 6.59880358452495374699253716244129464030265808105469e-02, // 2.718282 + 7.30402691048630536840846616541966795921325683593750e-01, // 0.314159 + 7.61985171966785834740676364162936806678771972656250e-01, // 0.271828 + }; + + public static final double[] SCIPY_GAMMA_CDF_2_1 = { // + 4.67884016044447376136972138738201465457677841186523e-03, // 0.100000 + 1.75230963064217717950477037902601296082139015197754e-02, // 0.200000 + 3.69363131137667791148437856918462784960865974426270e-02, // 0.300000 + 6.15519355501049983403483167876402148976922035217285e-02, // 0.400000 + 9.02040104310498636053239351895172148942947387695312e-02, // 0.500000 + 1.21901382249557641013204545288317603990435600280762e-01, // 0.600000 + 1.55804983554603787565540073956071864813566207885742e-01, // 0.700000 + 1.91207864589001103050591723331308458000421524047852e-01, // 0.800000 + 2.27517646492861713714006555164814926683902740478516e-01, // 0.900000 + 2.64241117657115276440293882842524908483028411865234e-01, // 1.000000 + 3.00970724234032926958093412395101040601730346679688e-01, // 1.100000 + 3.37372733793155299419908033087267540395259857177734e-01, // 1.200000 + 3.73176876021771020131012619458488188683986663818359e-01, // 1.300000 + 4.08167286540144502726690234339912422001361846923828e-01, // 1.400000 + 4.42174599628925357741593415994429960846900939941406e-01, // 1.500000 + 4.75069053213895919896003761095926165580749511718750e-01, // 1.600000 + 5.06754485057616443910433190467301756143569946289062e-01, // 1.700000 + 5.37163112979557655357609746715752407908439636230469e-01, // 1.800000 + 5.66251004254358369927047078817849978804588317871094e-01, // 1.900000 + 5.93994150290161671001953891391167417168617248535156e-01, // 2.000000 + 4.99996666679166626271834837367592264956162040334675e-11, // 0.000010 + 4.99999999966665360199746137423608777641557195129907e-21, // 0.000000 + 7.02166050809290703815790379849204327911138534545898e-03, // 0.123457 + 8.21025553585930989441976635134778916835784912109375e-01, // 3.141593 + 7.54637885420670229130735151557018980383872985839844e-01, // 2.718282 + 4.01345361152903382095935569395805941894650459289551e-02, // 0.314159 + 3.08857833818661116354054030352926929481327533721924e-02, // 0.271828 + }; + + public static final double[] SCIPY_GAMMA_PDF_2_1 = { // + 9.04837418035959739626150621916167438030242919921875e-02, // 0.100000 + 1.63746150615596386401762174500618129968643188476562e-01, // 0.200000 + 2.22245466204515323971335760688816662877798080444336e-01, // 0.300000 + 2.68128018414255708812277134711621329188346862792969e-01, // 0.400000 + 3.03265329856316712131558688270160928368568420410156e-01, // 0.500000 + 3.29286981656415900232559579308144748210906982421875e-01, // 0.600000 + 3.47609712653986657748816924140555784106254577636719e-01, // 0.700000 + 3.59463171293777250525636191014200448989868164062500e-01, // 0.800000 + 3.65912693766539232065326814336003735661506652832031e-01, // 0.900000 + 3.67879441171442334024277442949824035167694091796875e-01, // 1.000000 + 3.66158192067887522380686959877493791282176971435547e-01, // 1.100000 + 3.61433054294642563952777436497854068875312805175781e-01, // 1.200000 + 3.54291330944216387166534332209266722202301025390625e-01, // 1.300000 + 3.45235749518249090961319325288059189915657043457031e-01, // 1.400000 + 3.34695240222644740946122965397080406546592712402344e-01, // 1.500000 + 3.23034428791448668949470857114647515118122100830078e-01, // 1.600000 + 3.10561990889648920077803495587431825697422027587891e-01, // 1.700000 + 2.97537998798855785054229272645898163318634033203125e-01, // 1.800000 + 2.84180376523006650302249909145757555961608886718750e-01, // 1.900000 + 2.70670566473225404635627455718349665403366088867188e-01, // 2.000000 + 9.99990000050000075939183458162418105530377943068743e-06, // 0.000010 + 9.99999999900001396698837531419091790435560440641893e-11, // 0.000000 + 1.09118427942216628112603871159080881625413894653320e-01, // 0.123457 + 1.35760528150296710947486644727177917957305908203125e-01, // 3.141593 + 1.79374078734017200487116383555985521525144577026367e-01, // 2.718282 + 2.29462772836064060610894443925644736737012863159180e-01, // 0.314159 + 2.07129044651275240340737582300789654254913330078125e-01, // 0.271828 + }; + + public static final double[] GNUR_GAMMA_CDF_2_1 = { // + 4.67884016044447115928450742217137303669005632400513e-03, // 0.100000 + 1.75230963064217613867068479294175631366670131683350e-02, // 0.200000 + 3.69363131137667860537376895990746561437845230102539e-02, // 0.300000 + 6.15519355501050191570300285093253478407859802246094e-02, // 0.400000 + 9.02040104310498774831117430039739701896905899047852e-02, // 0.500000 + 1.21901382249557682646567968731687869876623153686523e-01, // 0.600000 + 1.55804983554603898587842536471725907176733016967773e-01, // 0.700000 + 1.91207864589001158561742954589135479182004928588867e-01, // 0.800000 + 2.27517646492861658202855323906987905502319335937500e-01, // 0.900000 + 2.64241117657115276440293882842524908483028411865234e-01, // 1.000000 + 3.00970724234032926958093412395101040601730346679688e-01, // 1.100000 + 3.37372733793155243908756801829440519213676452636719e-01, // 1.200000 + 3.73176876021770964619861388200661167502403259277344e-01, // 1.300000 + 4.08167286540144225170934078050777316093444824218750e-01, // 1.400000 + 4.42174599628925357741593415994429960846900939941406e-01, // 1.500000 + 4.75069053213895919896003761095926165580749511718750e-01, // 1.600000 + 5.06754485057616221865828265435993671417236328125000e-01, // 1.700000 + 5.37163112979557766379912209231406450271606445312500e-01, // 1.800000 + 5.66251004254358258904744616302195936441421508789062e-01, // 1.900000 + 5.93994150290161893046558816422475501894950866699219e-01, // 2.000000 + 4.99996666679166820142290904483453562942574244232219e-11, // 0.000010 + 4.99999999966665585894661495302810307638972341801077e-21, // 0.000000 + 7.02166050809290617079616581008849607314914464950562e-03, // 0.123457 + 8.21025553585959189106802114110905677080154418945312e-01, // 3.141593 + 7.54637885420841425521132350695552304387092590332031e-01, // 2.718282 + 4.01345361152950913519177333910192828625440597534180e-02, // 0.314159 + 3.08857833818859291163949620795392547734081745147705e-02, // 0.271828 + }; + + public static final double[] GNUR_GAMMA_PDF_2_1 = { // + 9.04837418035959323292516387482464779168367385864258e-02, // 0.100000 + 1.63746150615596414157337790129531640559434890747070e-01, // 0.200000 + 2.22245466204515351726911376317730173468589782714844e-01, // 0.300000 + 2.68128018414255764323428365969448350369930267333984e-01, // 0.400000 + 3.03265329856316712131558688270160928368568420410156e-01, // 0.500000 + 3.29286981656415955743710810565971769392490386962891e-01, // 0.600000 + 3.47609712653986657748816924140555784106254577636719e-01, // 0.700000 + 3.59463171293777306036787422272027470171451568603516e-01, // 0.800000 + 3.65912693766539232065326814336003735661506652832031e-01, // 0.900000 + 3.67879441171442334024277442949824035167694091796875e-01, // 1.000000 + 3.66158192067887522380686959877493791282176971435547e-01, // 1.100000 + 3.61433054294642563952777436497854068875312805175781e-01, // 1.200000 + 3.54291330944216442677685563467093743383884429931641e-01, // 1.300000 + 3.45235749518249146472470556545886211097240447998047e-01, // 1.400000 + 3.34695240222644796457274196654907427728176116943359e-01, // 1.500000 + 3.23034428791448668949470857114647515118122100830078e-01, // 1.600000 + 3.10561990889648975588954726845258846879005432128906e-01, // 1.700000 + 2.97537998798855729543078041388071142137050628662109e-01, // 1.800000 + 2.84180376523006705813401140403584577143192291259766e-01, // 1.900000 + 2.70670566473225404635627455718349665403366088867188e-01, // 2.000000 + 9.99990000049999567719415105582214664536877535283566e-06, // 0.000010 + 9.99999999900002689168544645524833777011641799958852e-11, // 0.000000 + 1.09118427942216628112603871159080881625413894653320e-01, // 0.123457 + 1.35760528150277559600311860776855610311031341552734e-01, // 3.141593 + 1.79374078733908981497791046422207728028297424316406e-01, // 2.718282 + 2.29462772836074496707325920397124718874692916870117e-01, // 0.314159 + 2.07129044651328225734587817896681372076272964477539e-01, // 0.271828 + }; + + public static final double[] SCIPY_GAMMA_CDF_4_1 = { // + 3.84683392534506373979050902756071650401281658560038e-06, // 0.100000 + 5.68402407581566679176977852083751940881484188139439e-05, // 0.200000 + 2.65811190021739791530808183495082630543038249015808e-04, // 0.300000 + 7.76251376207015455870041886754506776924245059490204e-04, // 0.400000 + 1.75162255629082394027540292569256052956916391849518e-03, // 0.500000 + 3.35806885324799794106942485427680367138236761093140e-03, // 0.600000 + 5.75345759229957919023146217796238488517701625823975e-03, // 0.700000 + 9.07985780015398494247058636119618313387036323547363e-03, // 0.800000 + 1.34587206394362543515619279332895530387759208679199e-02, // 0.900000 + 1.89881568761538083167650370342016685754060745239258e-02, // 1.000000 + 2.57418165296708391365232415637365193106234073638916e-02, // 1.100000 + 3.37689681856556886407894069179747020825743675231934e-02, // 1.200000 + 4.30954526920760624952855266656115418300032615661621e-02, // 1.300000 + 5.37252503680754125992002911971212597563862800598145e-02, // 1.400000 + 6.56424543784500796883563111805415246635675430297852e-02, // 1.500000 + 7.88134872297189398304695373553840909153223037719727e-02, // 1.600000 + 9.31894338562339608067475182906491681933403015136719e-02, // 1.700000 + 1.08708394709205430350706933495530392974615097045898e-01, // 1.800000 + 1.25297786682826373105825723541784100234508514404297e-01, // 1.900000 + 1.42876539501452959335381365235662087798118591308594e-01, // 2.000000 + 4.16663333347220766666711883927879838582564067271435e-22, // 0.000010 + 4.16666666633332144820066532191263979309384757370912e-42, // 0.000000 + 8.77087751463964086925343549561162603822594974189997e-06, // 0.123457 + 3.84456296955979215379528568519162945449352264404297e-01, // 3.141593 + 2.89942380984430481127844814182026311755180358886719e-01, // 2.718282 + 3.16096723469245968057839135667563823517411947250366e-04, // 0.314159 + 1.83213919706970028346976198108109201712068170309067e-04, // 0.271828 + }; + + public static final double[] SCIPY_GAMMA_PDF_4_1 = { // + 1.50806236339326763494506078444601371302269399166107e-04, // 0.100000 + 1.09164100410397580526167260472902853507548570632935e-03, // 0.200000 + 3.33368199306773070958453963896772620500996708869934e-03, // 0.300000 + 7.15008049104682522761899932106643973384052515029907e-03, // 0.400000 + 1.26360554106798653184462466469994978979229927062988e-02, // 0.500000 + 1.97572188993849519322854035863201715983450412750244e-02, // 0.600000 + 2.83881265334089138985262934511411003768444061279297e-02, // 0.700000 + 3.83427382713362394084377626768400659784674644470215e-02, // 0.800000 + 4.93982136584827924430385337473126128315925598144531e-02, // 0.900000 + 6.13132401952403913170108751273801317438483238220215e-02, // 1.000000 + 7.38419020670239845127724720441619865596294403076172e-02, // 1.100000 + 8.67439330307142181242241463223763275891542434692383e-02, // 1.200000 + 9.97920582159542851607980651351681444793939590454102e-02, // 1.300000 + 1.12777011509294713964557388408138649538159370422363e-01, // 1.400000 + 1.25510715083491819488159535467275418341159820556641e-01, // 1.500000 + 1.37828022951018119846011700246890541166067123413086e-01, // 1.600000 + 1.49587358945180909364225385616009589284658432006836e-01, // 1.700000 + 1.60670519351382129480398930354567710310220718383789e-01, // 1.800000 + 1.70981859874675595767357094700855668634176254272461e-01, // 1.900000 + 1.80447044315483556831125611097377259284257888793945e-01, // 2.000000 + 1.66665000008333050761503298375805932114390142846638e-16, // 0.000010 + 1.66666666650000169406744350520921226284591055366191e-31, // 0.000000 + 2.77189119111339720303438527437833727162797003984451e-04, // 0.123457 + 2.23317117687730742892071589267288800328969955444336e-01, // 3.141593 + 2.20900855076609420324373900257342029362916946411133e-01, // 2.718282 + 3.77451132111497611454353773297043517231941223144531e-03, // 0.314159 + 2.55081355107697037992720012766767467837780714035034e-03, // 0.271828 + }; + + public static final double[] GNUR_GAMMA_CDF_4_1 = { // + 3.84683392534506289275756177326037743569031590595841e-06, // 0.100000 + 5.68402407581567289040699875179996070073684677481651e-05, // 0.200000 + 2.65811190021739466270156437843752428307197988033295e-04, // 0.300000 + 7.76251376207014588508303898350959570962004363536835e-04, // 0.400000 + 1.75162255629082459079670641699522093404084444046021e-03, // 0.500000 + 3.35806885324799880843116284268035087734460830688477e-03, // 0.600000 + 5.75345759229958439440189010838366812095046043395996e-03, // 0.700000 + 9.07985780015399361608796624523165519349277019500732e-03, // 0.800000 + 1.34587206394362543515619279332895530387759208679199e-02, // 0.900000 + 1.89881568761538083167650370342016685754060745239258e-02, // 1.000000 + 2.57418165296708252587354337492797640152275562286377e-02, // 1.100000 + 3.37689681856556678241076951962895691394805908203125e-02, // 1.200000 + 4.30954526920761110675428540162101853638887405395508e-02, // 1.300000 + 5.37252503680754125992002911971212597563862800598145e-02, // 1.400000 + 6.56424543784501351995075424383685458451509475708008e-02, // 1.500000 + 7.88134872297189398304695373553840909153223037719727e-02, // 1.600000 + 9.31894338562340163178987495484761893749237060546875e-02, // 1.700000 + 1.08708394709205402595131317866616882383823394775391e-01, // 1.800000 + 1.25297786682826484128128186057438142597675323486328e-01, // 1.900000 + 1.42876539501453014846532596493489108979701995849609e-01, // 2.000000 + 4.16663333347220531567841719470378244835256622822300e-22, // 0.000010 + 4.16666666633328958636244267286709921548589221947301e-42, // 0.000000 + 8.77087751463965272771469705581637299474095925688744e-06, // 0.123457 + 3.84456296956025345146201743773417547345161437988281e-01, // 3.141593 + 2.89942380984641312480221131409052759408950805664062e-01, // 2.718282 + 3.16096723469323813773823594885925558628514409065247e-04, // 0.314159 + 1.83213919707213540154916442403987275611143559217453e-04, // 0.271828 + }; + + public static final double[] GNUR_GAMMA_PDF_4_1 = { // + 1.50806236339326655074288829894157970556989312171936e-04, // 0.100000 + 1.09164100410397688946384509023346254252828657627106e-03, // 0.200000 + 3.33368199306772810749932567375708458712324500083923e-03, // 0.300000 + 7.15008049104682436025726133266289252787828445434570e-03, // 0.400000 + 1.26360554106798635837227706701924034859985113143921e-02, // 0.500000 + 1.97572188993849727489671153080053045414388179779053e-02, // 0.600000 + 2.83881265334089312457610532192120444960892200469971e-02, // 0.700000 + 3.83427382713362532862255704912968212738633155822754e-02, // 0.800000 + 4.93982136584827855041446298400842351838946342468262e-02, // 0.900000 + 6.13132401952403913170108751273801317438483238220215e-02, // 1.000000 + 7.38419020670239845127724720441619865596294403076172e-02, // 1.100000 + 8.67439330307141764908607228790060617029666900634766e-02, // 1.200000 + 9.97920582159543684275249120219086762517690658569336e-02, // 1.300000 + 1.12777011509294713964557388408138649538159370422363e-01, // 1.400000 + 1.25510715083491819488159535467275418341159820556641e-01, // 1.500000 + 1.37828022951018064334860468989063519984483718872070e-01, // 1.600000 + 1.49587358945180937119801001244923099875450134277344e-01, // 1.700000 + 1.60670519351382129480398930354567710310220718383789e-01, // 1.800000 + 1.70981859874675679034083941587596200406551361083984e-01, // 1.900000 + 1.80447044315483640097852457984117791056632995605469e-01, // 2.000000 + 1.66665000008333075413406586532424851230907807933708e-16, // 0.000010 + 1.66666666649999030851742086607051100754304632309334e-31, // 0.000000 + 2.77189119111339503463004030336946925672236829996109e-04, // 0.123457 + 2.23317117687728605712749185840948484838008880615234e-01, // 3.141593 + 2.20900855076631263962383400212274864315986633300781e-01, // 2.718282 + 3.77451132111564354940091980950001016026362776756287e-03, // 0.314159 + 2.55081355107941764107093440827611630083993077278137e-03, // 0.271828 + }; + + public static final double[] SCIPY_GAMMA_CDF_4_10 = { // + 1.89881568761538083167650370342016685754060745239258e-02, // 0.100000 + 1.42876539501452959335381365235662087798118591308594e-01, // 0.200000 + 3.52768111217768576270970015684724785387516021728516e-01, // 0.300000 + 5.66529879633290933682587819930631667375564575195312e-01, // 0.400000 + 7.34974084702638363886251227086177095770835876464844e-01, // 0.500000 + 8.48796117223352020175752841169014573097229003906250e-01, // 0.600000 + 9.18234583755278332972693533520214259624481201171875e-01, // 0.700000 + 9.57619888008315989225138764595612883567810058593750e-01, // 0.800000 + 9.78773513697091090968172011343995109200477600097656e-01, // 0.900000 + 9.89663949324074310176513336045900359749794006347656e-01, // 1.000000 + 9.95084132734071036274769994633970782160758972167969e-01, // 1.100000 + 9.97708208792208561810355149646056815981864929199219e-01, // 1.200000 + 9.98949700268889517751347284502116963267326354980469e-01, // 1.300000 + 9.99525751453871236229531405115267261862754821777344e-01, // 1.400000 + 9.99788621496533247423599277681205421686172485351562e-01, // 1.500000 + 9.99906858387057395631813960790168493986129760742188e-01, // 1.600000 + 9.99959373411186347446744093758752569556236267089844e-01, // 1.700000 + 9.99982439833354330716019831015728414058685302734375e-01, // 1.800000 + 9.99992471709186814265990506100933998823165893554688e-01, // 1.900000 + 9.99996796280219535368871675018453970551490783691406e-01, // 2.000000 + 4.16633334722182824321055502020601847065999725794631e-18, // 0.000010 + 4.16666666333333208899682539512773183295181978120839e-38, // 0.000000 + 3.68452685531321239009727719349029939621686935424805e-02, // 0.123457 + 9.99999999870692435344210480252513661980628967285156e-01, // 3.141593 + 9.99999994136864356342186965775908902287483215332031e-01, // 2.718282 + 3.84456296955979215379528568519162945449352264404297e-01, // 0.314159 + 2.89942380984430259083239889150718227028846740722656e-01, // 0.271828 + }; + + public static final double[] SCIPY_GAMMA_PDF_4_10 = { // + 6.13132401952403927047896559088258072733879089355469e-01, // 0.100000 + 1.80447044315483551280010487971594557166099548339844e+00, // 0.200000 + 2.24041807655387747999498060380574315786361694335938e+00, // 0.300000 + 1.95366814813164535458156478853197768330574035644531e+00, // 0.400000 + 1.40373895814280502669646466529229655861854553222656e+00, // 0.500000 + 8.92350783599890173825031070009572431445121765136719e-01, // 0.600000 + 5.21292523641998628569638185581425204873085021972656e-01, // 0.700000 + 2.86261442476809790935732280559022910892963409423828e-01, // 0.800000 + 1.49942911965315728295422559313010424375534057617188e-01, // 0.900000 + 7.56665496041414309402739490906242281198501586914062e-02, // 1.000000 + 3.70499395863616506896498492551472736522555351257324e-02, // 1.100000 + 1.76953315775852937363765704503748565912246704101562e-02, // 1.200000 + 8.27657284522896781464584847753940266557037830352783e-03, // 1.300000 + 3.80285800870032181936197801519483618903905153274536e-03, // 1.400000 + 1.72070055282277012062563947836224542697891592979431e-03, // 1.500000 + 7.68240126083475261160871117027681975741870701313019e-04, // 1.600000 + 3.38991900206525934025059143550606677308678627014160e-04, // 1.700000 + 1.48035403118606612247326603792885180155280977487564e-04, // 1.800000 + 6.40493012751137394053535523141817975556477904319763e-05, // 1.900000 + 2.74820482991807852311567234027123163286887574940920e-05, // 2.000000 + 1.66650000833305830210934716499431774874677436582004e-12, // 0.000010 + 1.66666666500001019823879501718183287893096342876298e-27, // 0.000000 + 9.12487883209794348182697376614669337868690490722656e-01, // 0.123457 + 1.17363980155635999975867359517269286617313639453641e-09, // 3.141593 + 5.24063860510259035108958120419836435033289490093011e-08, // 2.718282 + 2.23317117687730748443186712393071502447128295898438e+00, // 0.314159 + 2.20900855076609303750956314615905284881591796875000e+00, // 0.271828 + }; + + public static final double[] GNUR_GAMMA_CDF_4_10 = { // + 1.89881568761538083167650370342016685754060745239258e-02, // 0.100000 + 1.42876539501453014846532596493489108979701995849609e-01, // 0.200000 + 3.52768111217768742804423709458205848932266235351562e-01, // 0.300000 + 5.66529879633290933682587819930631667375564575195312e-01, // 0.400000 + 7.34974084702638252863948764570523053407669067382812e-01, // 0.500000 + 8.48796117223352020175752841169014573097229003906250e-01, // 0.600000 + 9.18234583755278332972693533520214259624481201171875e-01, // 0.700000 + 9.57619888008315989225138764595612883567810058593750e-01, // 0.800000 + 9.78773513697091090968172011343995109200477600097656e-01, // 0.900000 + 9.89663949324074310176513336045900359749794006347656e-01, // 1.000000 + 9.95084132734071036274769994633970782160758972167969e-01, // 1.100000 + 9.97708208792208561810355149646056815981864929199219e-01, // 1.200000 + 9.98949700268889517751347284502116963267326354980469e-01, // 1.300000 + 9.99525751453871236229531405115267261862754821777344e-01, // 1.400000 + 9.99788621496533247423599277681205421686172485351562e-01, // 1.500000 + 9.99906858387057395631813960790168493986129760742188e-01, // 1.600000 + 9.99959373411186347446744093758752569556236267089844e-01, // 1.700000 + 9.99982439833354330716019831015728414058685302734375e-01, // 1.800000 + 9.99992471709186814265990506100933998823165893554688e-01, // 1.900000 + 9.99996796280219535368871675018453970551490783691406e-01, // 2.000000 + 4.16633334722182670246659951041733602587764319000446e-18, // 0.000010 + 4.16666666333330128955973644209196576036362409030393e-38, // 0.000000 + 3.68452685531321239009727719349029939621686935424805e-02, // 0.123457 + 9.99999999870692435344210480252513661980628967285156e-01, // 3.141593 + 9.99999994136864356342186965775908902287483215332031e-01, // 2.718282 + 3.84456296956025345146201743773417547345161437988281e-01, // 0.314159 + 2.89942380984641312480221131409052759408950805664062e-01, // 0.271828 + }; + + public static final double[] GNUR_GAMMA_PDF_4_10 = { // + 6.13132401952403927047896559088258072733879089355469e-01, // 0.100000 + 1.80447044315483640097852457984117791056632995605469e+00, // 0.200000 + 2.24041807655387747999498060380574315786361694335938e+00, // 0.300000 + 1.95366814813164602071537956362590193748474121093750e+00, // 0.400000 + 1.40373895814280613691948929044883698225021362304688e+00, // 0.500000 + 8.92350783599889285646611369884340092539787292480469e-01, // 0.600000 + 5.21292523641998628569638185581425204873085021972656e-01, // 0.700000 + 2.86261442476810124002639668105985037982463836669922e-01, // 0.800000 + 1.49942911965315672784271328055183403193950653076172e-01, // 0.900000 + 7.56665496041414725736373725339944940060377120971680e-02, // 1.000000 + 3.70499395863616229340742336262337630614638328552246e-02, // 1.100000 + 1.76953315775852486335661950533904018811881542205811e-02, // 1.200000 + 8.27657284522894699796413675585426972247660160064697e-03, // 1.300000 + 3.80285800870031921727676404998419457115232944488525e-03, // 1.400000 + 1.72070055282276881958303249575692461803555488586426e-03, // 1.500000 + 7.68240126083476779043912596733889586175791919231415e-04, // 1.600000 + 3.38991900206525934025059143550606677308678627014160e-04, // 1.700000 + 1.48035403118606883297869725168993682018481194972992e-04, // 1.800000 + 6.40493012751137529578807083829872226488078013062477e-05, // 1.900000 + 2.74820482991807886192885124199136726019787602126598e-05, // 2.000000 + 1.66650000833305870600613063815236211955179979060659e-12, // 0.000010 + 1.66666666499998652189994178507260255159592416234002e-27, // 0.000000 + 9.12487883209794348182697376614669337868690490722656e-01, // 0.123457 + 1.17363980155416280125657961541131568683482555570663e-09, // 3.141593 + 5.24063860505807479924442746684748462016045778000262e-08, // 2.718282 + 2.23317117687728572406058447086252272129058837890625e+00, // 0.314159 + 2.20900855076631241757922907709144055843353271484375e+00, // 0.271828 + }; + + public static final double[] SCIPY_GAMMA_CDF_01_10 = { // + 9.75872656273672145488262685830704867839813232421875e-01, // 0.100000 + 9.94326176020188468029914474755059927701950073242188e-01, // 0.200000 + 9.98434728252885594912413580459542572498321533203125e-01, // 0.300000 + 9.99535388712767813856885368295479565858840942382812e-01, // 0.400000 + 9.99856061034153253963552288041682913899421691894531e-01, // 0.500000 + 9.99954127338806841862606233917176723480224609375000e-01, // 0.600000 + 9.99985082831851612716889121657004579901695251464844e-01, // 0.700000 + 9.99995075192051996459952079021604731678962707519531e-01, // 0.800000 + 9.99998354830390345782120675721671432256698608398438e-01, // 0.900000 + 9.99999445201428249774266987515147775411605834960938e-01, // 1.000000 + 9.99999811450770770271390119887655600905418395996094e-01, // 1.100000 + 9.99999935505229697874085559305967763066291809082031e-01, // 1.200000 + 9.99999977817657148548846635094378143548965454101562e-01, // 1.300000 + 9.99999992334555942719021004450041800737380981445312e-01, // 1.400000 + 9.99999997340225710296124361775582656264305114746094e-01, // 1.500000 + 9.99999999073787781966871079930569976568222045898438e-01, // 1.600000 + 9.99999999676440931573040415969444438815116882324219e-01, // 1.700000 + 9.99999999886649670877147855208022519946098327636719e-01, // 1.800000 + 9.99999999960190066872200986836105585098266601562500e-01, // 1.900000 + 9.99999999985986431916273886599810793995857238769531e-01, // 2.000000 + 4.18461375237962951434411706941318698227405548095703e-01, // 0.000010 + 1.32330308815105796016453609809104818850755691528320e-01, // 0.000000 + 9.83240178351675986334612389327958226203918457031250e-01, // 0.123457 + 9.99999999999999888977697537484345957636833190917969e-01, // 3.141593 + 9.99999999999991784349617773841600865125656127929688e-01, // 2.718282 + 9.98686603432971331528733571758493781089782714843750e-01, // 0.314159 + 9.97771302397556292262947863491717725992202758789062e-01, // 0.271828 + }; + + public static final double[] SCIPY_GAMMA_PDF_01_10 = { // + 3.86691694403023700221666558718425221741199493408203e-01, // 0.100000 + 7.62330623530854123437450198252918198704719543457031e-02, // 0.200000 + 1.94700358454044236689028934961243066936731338500977e-02, // 0.300000 + 5.52875577773295160538546966222384071443229913711548e-03, // 0.400000 + 1.66384900992015260373646512448431167285889387130737e-03, // 0.500000 + 5.19465021361826249403359501144450405263341963291168e-04, // 0.600000 + 1.66344986720691771688723781252861044777091592550278e-04, // 0.700000 + 5.42653342633515591237128306012493794696638360619545e-05, // 0.800000 + 1.79552199849926109312056587974382182437693700194359e-05, // 0.900000 + 6.00778672619989704319900733486825572526868199929595e-06, // 1.000000 + 2.02846074898835833691911056941847846246673725545406e-06, // 1.100000 + 6.90021180431038364101552478463430517763299576472491e-07, // 1.200000 + 2.36201169837157485606116071215343499289929241058417e-07, // 1.300000 + 8.12870470963516557411429738419506030311367794638500e-08, // 1.400000 + 2.81034711509505172708305416806248699934656087862095e-08, // 1.500000 + 9.75527757384449392403810659207472055776122488168767e-09, // 1.600000 + 3.39820132151311703127167328079318020517263221336179e-09, // 1.700000 + 1.18744471030516355255038446822732009500178662619874e-09, // 1.800000 + 4.16088707178314669861834363610679499156930205572280e-10, // 1.900000 + 1.46164764735815876829497270010841617621233368140565e-10, // 2.000000 + 4.18423334941553275712067261338233947753906250000000e+03, // 0.000010 + 1.32330308694805443286895751953125000000000000000000e+08, // 0.000000 + 2.53005809030332373676941415396868251264095306396484e-01, // 0.123457 + 1.07265707643817319806681139845227667938280975701293e-15, // 3.141593 + 8.42260257482259737446868026839438676448175435940158e-14, // 2.718282 + 1.62124289852739640604983861749133211560547351837158e-02, // 0.314159 + 2.82006743783194402386271804061834700405597686767578e-02, // 0.271828 + }; + + public static final double[] GNUR_GAMMA_CDF_01_10 = { // + 9.75872656273672256510565148346358910202980041503906e-01, // 0.100000 + 9.94326176020188468029914474755059927701950073242188e-01, // 0.200000 + 9.98434728252885594912413580459542572498321533203125e-01, // 0.300000 + 9.99535388712767813856885368295479565858840942382812e-01, // 0.400000 + 9.99856061034153253963552288041682913899421691894531e-01, // 0.500000 + 9.99954127338806841862606233917176723480224609375000e-01, // 0.600000 + 9.99985082831851612716889121657004579901695251464844e-01, // 0.700000 + 9.99995075192051996459952079021604731678962707519531e-01, // 0.800000 + 9.99998354830390345782120675721671432256698608398438e-01, // 0.900000 + 9.99999445201428249774266987515147775411605834960938e-01, // 1.000000 + 9.99999811450770770271390119887655600905418395996094e-01, // 1.100000 + 9.99999935505229697874085559305967763066291809082031e-01, // 1.200000 + 9.99999977817657148548846635094378143548965454101562e-01, // 1.300000 + 9.99999992334555942719021004450041800737380981445312e-01, // 1.400000 + 9.99999997340225710296124361775582656264305114746094e-01, // 1.500000 + 9.99999999073787781966871079930569976568222045898438e-01, // 1.600000 + 9.99999999676440931573040415969444438815116882324219e-01, // 1.700000 + 9.99999999886649670877147855208022519946098327636719e-01, // 1.800000 + 9.99999999960190066872200986836105585098266601562500e-01, // 1.900000 + 9.99999999985986431916273886599810793995857238769531e-01, // 2.000000 + 4.18461375237963006945562938199145719408988952636719e-01, // 0.000010 + 1.32330308815105768260877994180191308259963989257812e-01, // 0.000000 + 9.83240178351675764290007464296650141477584838867188e-01, // 0.123457 + 9.99999999999999888977697537484345957636833190917969e-01, // 3.141593 + 9.99999999999991784349617773841600865125656127929688e-01, // 2.718282 + 9.98686603432971664595640959305455908179283142089844e-01, // 0.314159 + 9.97771302397558956798206963867414742708206176757812e-01, // 0.271828 + }; + + public static final double[] GNUR_GAMMA_PDF_01_10 = { // + 3.86691694403023811243969021234079264104366302490234e-01, // 0.100000 + 7.62330623530854400993206354542053304612636566162109e-02, // 0.200000 + 1.94700358454044201994559415425101178698241710662842e-02, // 0.300000 + 5.52875577773295420747068362743448233231902122497559e-03, // 0.400000 + 1.66384900992015282057689962158519847434945404529572e-03, // 0.500000 + 5.19465021361826032562925004043563603772781789302826e-04, // 0.600000 + 1.66344986720691988529158278353747846267651766538620e-04, // 0.700000 + 5.42653342633515862287671427388602296559838578104973e-05, // 0.800000 + 1.79552199849926244837328148662436433369293808937073e-05, // 0.900000 + 6.00778672619989450210016557196723852030117996037006e-06, // 1.000000 + 2.02846074898835876043558419656864799662798759527504e-06, // 1.100000 + 6.90021180431039528771854953126396736706738010980189e-07, // 1.200000 + 2.36201169837157459136336469518457903404851094819605e-07, // 1.300000 + 8.12870470963517086807021772357217948012930719414726e-08, // 1.400000 + 2.81034711509505834452795459228388597061609743832378e-08, // 1.500000 + 9.75527757384449888712178191024076978621337730146479e-09, // 1.600000 + 3.39820132151312695743902391712527866207693705291604e-09, // 1.700000 + 1.18744471030516479332130329776883240211482473114302e-09, // 1.800000 + 4.16088707178313997777586664275693666137367898727462e-10, // 1.900000 + 1.46164764735815825130708985446611938158190113767887e-10, // 2.000000 + 4.18423334941553457611007615923881530761718750000000e+03, // 0.000010 + 1.32330308694805443286895751953125000000000000000000e+08, // 0.000000 + 2.53005809030332484699243877912522293627262115478516e-01, // 0.123457 + 1.07265707643589812321615400040546925420053420152655e-15, // 3.141593 + 8.42260257473954736054446049913952863989448707759955e-14, // 2.718282 + 1.62124289852696411295962519716340466402471065521240e-02, // 0.314159 + 2.82006743782835765654848358963135979138314723968506e-02, // 0.271828 + }; + + public static final double[] SCIPY_GAMMA_CDF_01_20 = { // + 9.94326176020188468029914474755059927701950073242188e-01, // 0.100000 + 9.99535388712767813856885368295479565858840942382812e-01, // 0.200000 + 9.99954127338806841862606233917176723480224609375000e-01, // 0.300000 + 9.99995075192051996459952079021604731678962707519531e-01, // 0.400000 + 9.99999445201428249774266987515147775411605834960938e-01, // 0.500000 + 9.99999935505229697874085559305967763066291809082031e-01, // 0.600000 + 9.99999992334555942719021004450041800737380981445312e-01, // 0.700000 + 9.99999999073787781966871079930569976568222045898438e-01, // 0.800000 + 9.99999999886649670877147855208022519946098327636719e-01, // 0.900000 + 9.99999999985986431916273886599810793995857238769531e-01, // 1.000000 + 9.99999999998253064070752316183643415570259094238281e-01, // 1.100000 + 9.99999999999780730952636531583266332745552062988281e-01, // 1.200000 + 9.99999999999972355446686833602143451571464538574219e-01, // 1.300000 + 9.99999999999996447286321199499070644378662109375000e-01, // 1.400000 + 9.99999999999999555910790149937383830547332763671875e-01, // 1.500000 + 9.99999999999999888977697537484345957636833190917969e-01, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 4.48491720125246684514053185921511612832546234130859e-01, // 0.000010 + 1.41828113264369309876045122109644580632448196411133e-01, // 0.000000 + 9.96936964362935218453287689044373109936714172363281e-01, // 0.123457 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 9.99966686426665196663066126347985118627548217773438e-01, // 0.314159 + 9.99912881838271760948089195153443142771720886230469e-01, // 0.271828 + }; + + public static final double[] SCIPY_GAMMA_PDF_01_20 = { // + 1.52466124706170824687490039650583639740943908691406e-01, // 0.100000 + 1.10575115554659032107709393244476814288645982742310e-02, // 0.200000 + 1.03893004272365249880671900228890081052668392658234e-03, // 0.300000 + 1.08530668526703118247425661202498758939327672123909e-04, // 0.400000 + 1.20155734523997940863980146697365114505373639985919e-05, // 0.500000 + 1.38004236086207672820310495692686103552659915294498e-06, // 0.600000 + 1.62574094192703311482285947683901206062273558927700e-07, // 0.700000 + 1.95105551476889878480762131841494411155224497633753e-08, // 0.800000 + 2.37488942061032710510076893645464019000357325239747e-09, // 0.900000 + 2.92329529471631753658994540021683235242466736281131e-10, // 1.000000 + 3.63103392931038151766683113755789609131130468711035e-11, // 1.100000 + 4.54393006390842164952338174282894757080586423114710e-12, // 1.200000 + 5.72211759915234715705708461587397688436276710355344e-13, // 1.300000 + 7.24438629339447104119041928796962738566333345202608e-14, // 1.400000 + 9.21394754227480053810143174892231491926485650173495e-15, // 1.500000 + 1.17660562505859836487690040130500292983677893422084e-15, // 1.600000 + 1.50780754320226546403459477832592082942578975198605e-16, // 1.700000 + 1.93827646502898085931666826628168232690749572821615e-17, // 1.800000 + 2.49858295196862659910572709810793896012441320426695e-18, // 1.900000 + 3.22891084281901708463501316045877085134772951439524e-19, // 2.000000 + 4.48410183235842850990593433380126953125000000000000e+03, // 0.000010 + 1.41828113006499916315078735351562500000000000000000e+08, // 0.000000 + 7.88983383252931935736285140592372044920921325683594e-02, // 0.123457 + 2.61096087098872756423344011324704568131684902965870e-29, // 3.141593 + 1.41319250269510810248688074925111402327654344402472e-25, // 2.718282 + 7.50887254129001474947657790437460789689794182777405e-04, // 0.314159 + 1.99447085860641592364173035889507445972412824630737e-03, // 0.271828 + }; + + public static final double[] GNUR_GAMMA_CDF_01_20 = { // + 9.94326176020188468029914474755059927701950073242188e-01, // 0.100000 + 9.99535388712767813856885368295479565858840942382812e-01, // 0.200000 + 9.99954127338806841862606233917176723480224609375000e-01, // 0.300000 + 9.99995075192051996459952079021604731678962707519531e-01, // 0.400000 + 9.99999445201428249774266987515147775411605834960938e-01, // 0.500000 + 9.99999935505229697874085559305967763066291809082031e-01, // 0.600000 + 9.99999992334555942719021004450041800737380981445312e-01, // 0.700000 + 9.99999999073787781966871079930569976568222045898438e-01, // 0.800000 + 9.99999999886649670877147855208022519946098327636719e-01, // 0.900000 + 9.99999999985986431916273886599810793995857238769531e-01, // 1.000000 + 9.99999999998253064070752316183643415570259094238281e-01, // 1.100000 + 9.99999999999780730952636531583266332745552062988281e-01, // 1.200000 + 9.99999999999972355446686833602143451571464538574219e-01, // 1.300000 + 9.99999999999996447286321199499070644378662109375000e-01, // 1.400000 + 9.99999999999999555910790149937383830547332763671875e-01, // 1.500000 + 9.99999999999999888977697537484345957636833190917969e-01, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 4.48491720125246851047506879694992676377296447753906e-01, // 0.000010 + 1.41828113264369393142771968996385112404823303222656e-01, // 0.000000 + 9.96936964362935329475590151560027152299880981445312e-01, // 0.123457 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 9.99966686426665196663066126347985118627548217773438e-01, // 0.314159 + 9.99912881838271982992694120184751227498054504394531e-01, // 0.271828 + }; + + public static final double[] GNUR_GAMMA_PDF_01_20 = { // + 1.52466124706170880198641270908410660922527313232422e-01, // 0.100000 + 1.10575115554659084149413672548689646646380424499512e-02, // 0.200000 + 1.03893004272365206512585000808712720754556357860565e-03, // 0.300000 + 1.08530668526703172457534285477720459311967715620995e-04, // 0.400000 + 1.20155734523997890042003311439344770406023599207401e-05, // 0.500000 + 1.38004236086207905754370990625279347341347602196038e-06, // 0.600000 + 1.62574094192703417361404354471443589602586143882945e-07, // 0.700000 + 1.95105551476889977742435638204815395724267546029296e-08, // 0.800000 + 2.37488942061032958664260659553766480422964946228603e-09, // 0.900000 + 2.92329529471631650261417970893223876316380227535774e-10, // 1.000000 + 3.63103392931038281013653825166363807788738604642731e-11, // 1.100000 + 4.54393006390842084172981479651285882919581338157400e-12, // 1.200000 + 5.72211759915235624473471276192997522747583916125080e-13, // 1.300000 + 7.24438629339446220594828081263740677430340228482031e-14, // 1.400000 + 9.21394754227482262620677793725286644766468441974938e-15, // 1.500000 + 1.17660562505860112589006867484632187088675742397265e-15, // 1.600000 + 1.50780754320227852954333750133394796118015224813298e-16, // 1.700000 + 1.93827646502899996454171658766134464220868617069515e-17, // 1.800000 + 2.49858295196864470284720433812495768631707350258375e-18, // 1.900000 + 3.22891084281903345503954045196352182716024148627745e-19, // 2.000000 + 4.48410183235842760041123256087303161621093750000000e+03, // 0.000010 + 1.41828113006500124931335449218750000000000000000000e+08, // 0.000000 + 7.88983383252932352069919375026074703782796859741211e-02, // 0.123457 + 2.61096087097776828920364858389876537699185525357919e-29, // 3.141593 + 1.41319250266768533456788338854519414081259096541228e-25, // 2.718282 + 7.50887254128645965055299793533549745916388928890228e-04, // 0.314159 + 1.99447085860197633258583671533870074199512600898743e-03, // 0.271828 + }; + + public static final double[] SCIPY_GAMMA_CDF_01_4 = { // + 9.27573833553704485410662528011016547679901123046875e-01, // 0.100000 + 9.66394653856728291785316287132445722818374633789062e-01, // 0.200000 + 9.82338840734720442071647994453087449073791503906250e-01, // 0.300000 + 9.90161634314574046200618795410264283418655395507812e-01, // 0.400000 + 9.94326176020188468029914474755059927701950073242188e-01, // 0.500000 + 9.96650923670122979025620679749408736824989318847656e-01, // 0.600000 + 9.97989690314743826959897887718398123979568481445312e-01, // 0.700000 + 9.98777830395332189006296630395809188485145568847656e-01, // 0.800000 + 9.99249499182313538625521687208674848079681396484375e-01, // 0.900000 + 9.99535388712767813856885368295479565858840942382812e-01, // 1.000000 + 9.99710441300151630983350514725316315889358520507812e-01, // 1.100000 + 9.99818519608100975126774301315890625119209289550781e-01, // 1.200000 + 9.99885709402319799998792859696550294756889343261719e-01, // 1.300000 + 9.99927723862083506212172778759850189089775085449219e-01, // 1.400000 + 9.99954127338806841862606233917176723480224609375000e-01, // 1.500000 + 9.99970792125264407168572233786107972264289855957031e-01, // 1.600000 + 9.99981350059541806984952927450649440288543701171875e-01, // 1.700000 + 9.99988061331210875692931949743069708347320556640625e-01, // 1.800000 + 9.99992340071521379485375291551463305950164794921875e-01, // 1.900000 + 9.99995075192051996459952079021604731678962707519531e-01, // 2.000000 + 3.81824459744078470446737583188223652541637420654297e-01, // 0.000010 + 1.20743934975320044156177345939795486629009246826172e-01, // 0.000000 + 9.40661616417386547972512289561564102768898010253906e-01, // 0.123457 + 9.99999964784593187161476635083090513944625854492188e-01, // 3.141593 + 9.99999783864769620223000856640283018350601196289062e-01, // 2.718282 + 9.83788069985481405943517074774717912077903747558594e-01, // 0.314159 + 9.78987050620883314522302498517092317342758178710938e-01, // 0.271828 + }; + + public static final double[] SCIPY_GAMMA_PDF_01_4 = { // + 6.42906079231301164078615784092107787728309631347656e-01, // 0.100000 + 2.30941904807187847747940168119384907186031341552734e-01, // 0.200000 + 1.07473852696625640779082289100188063457608222961426e-01, // 0.300000 + 5.56083695858133400524714318180485861375927925109863e-02, // 0.400000 + 3.04932249412341663252767887115624034777283668518066e-02, // 0.500000 + 1.73469227014204988024381748346058884635567665100098e-02, // 0.600000 + 1.01216785315276381479465683810303744394332170486450e-02, // 0.700000 + 6.01647331475561961827702717187094094697386026382446e-03, // 0.800000 + 3.62732886466578426098994469839453813619911670684814e-03, // 0.900000 + 2.21150231109318090236270926141060044756159186363220e-03, // 1.000000 + 1.36055526775470518076327497425381807261146605014801e-03, // 1.100000 + 8.43312796876826494159307134879099976387806236743927e-04, // 1.200000 + 5.25999101373744552823108477923597092740237712860107e-04, // 1.300000 + 3.29838232387005033784815877240248482848983258008957e-04, // 1.400000 + 2.07786008544730114869572568103706089459592476487160e-04, // 1.500000 + 1.31423387335168583012201537485452718101441860198975e-04, // 1.600000 + 8.34178167284065351675878541293229773145867511630058e-05, // 1.700000 + 5.31128738330742547154618427018846205100999213755131e-05, // 1.800000 + 3.39116578546243936046616918034146692662034183740616e-05, // 1.900000 + 2.17061337053406243271114900439400230425235349684954e-05, // 2.000000 + 3.81810575458691437233937904238700866699218750000000e+03, // 0.000010 + 1.20743934931413173675537109375000000000000000000000e+08, // 0.000000 + 4.84212362445597399585039966041222214698791503906250e-01, // 0.123457 + 1.50288248944523529597303750440784320119291805895045e-07, // 3.141593 + 9.30810123491585766181913991212937276031880173832178e-07, // 2.718282 + 9.74271275055620944449685794097604230046272277832031e-02, // 0.314159 + 1.31457803694503660985049009468639269471168518066406e-01, // 0.271828 + }; + + public static final double[] GNUR_GAMMA_CDF_01_4 = { // + 9.27573833553704485410662528011016547679901123046875e-01, // 0.100000 + 9.66394653856728513829921212163753807544708251953125e-01, // 0.200000 + 9.82338840734720664116252919484395533800125122070312e-01, // 0.300000 + 9.90161634314574046200618795410264283418655395507812e-01, // 0.400000 + 9.94326176020188468029914474755059927701950073242188e-01, // 0.500000 + 9.96650923670122979025620679749408736824989318847656e-01, // 0.600000 + 9.97989690314743826959897887718398123979568481445312e-01, // 0.700000 + 9.98777830395332189006296630395809188485145568847656e-01, // 0.800000 + 9.99249499182313538625521687208674848079681396484375e-01, // 0.900000 + 9.99535388712767813856885368295479565858840942382812e-01, // 1.000000 + 9.99710441300151630983350514725316315889358520507812e-01, // 1.100000 + 9.99818519608100975126774301315890625119209289550781e-01, // 1.200000 + 9.99885709402319799998792859696550294756889343261719e-01, // 1.300000 + 9.99927723862083506212172778759850189089775085449219e-01, // 1.400000 + 9.99954127338806841862606233917176723480224609375000e-01, // 1.500000 + 9.99970792125264407168572233786107972264289855957031e-01, // 1.600000 + 9.99981350059541806984952927450649440288543701171875e-01, // 1.700000 + 9.99988061331210875692931949743069708347320556640625e-01, // 1.800000 + 9.99992340071521379485375291551463305950164794921875e-01, // 1.900000 + 9.99995075192051996459952079021604731678962707519531e-01, // 2.000000 + 3.81824459744078470446737583188223652541637420654297e-01, // 0.000010 + 1.20743934975320044156177345939795486629009246826172e-01, // 0.000000 + 9.40661616417386770017117214592872187495231628417969e-01, // 0.123457 + 9.99999964784593187161476635083090513944625854492188e-01, // 3.141593 + 9.99999783864769620223000856640283018350601196289062e-01, // 2.718282 + 9.83788069985483404344961400056490674614906311035156e-01, // 0.314159 + 9.78987050620895637997875837754691019654273986816406e-01, // 0.271828 + }; + + public static final double[] GNUR_GAMMA_PDF_01_4 = { // + 6.42906079231301386123220709123415872454643249511719e-01, // 0.100000 + 2.30941904807187903259091399377211928367614746093750e-01, // 0.200000 + 1.07473852696625682412445712543558329343795776367188e-01, // 0.300000 + 5.56083695858133747469409513541904743760824203491211e-02, // 0.400000 + 3.04932249412341767336176445724049699492752552032471e-02, // 0.500000 + 1.73469227014205092107790306954484549351036548614502e-02, // 0.600000 + 1.01216785315276381479465683810303744394332170486450e-02, // 0.700000 + 6.01647331475562135300050314867803535889834165573120e-03, // 0.800000 + 3.62732886466578339362820670999099093023687601089478e-03, // 0.900000 + 2.21150231109318176972444724981414765352383255958557e-03, // 1.000000 + 1.36055526775470539760370947135470487410202622413635e-03, // 1.100000 + 8.43312796876827144680610626181760380859486758708954e-04, // 1.200000 + 5.25999101373743902301804986620936688268557190895081e-04, // 1.300000 + 3.29838232387004816944381380139361681358423084020615e-04, // 1.400000 + 2.07786008544730223289789816654149490204872563481331e-04, // 1.500000 + 1.31423387335168528802092913210231017728801816701889e-04, // 1.600000 + 8.34178167284063996423162934412687263829866424202919e-05, // 1.700000 + 5.31128738330743021493068889427036083361599594354630e-05, // 1.800000 + 3.39116578546244545910338941130390821854234673082829e-05, // 1.900000 + 2.17061337053406344915068570955440918623935431241989e-05, // 2.000000 + 3.81810575458691755557083524763584136962890625000000e+03, // 0.000010 + 1.20743934931413218379020690917968750000000000000000e+08, // 0.000000 + 4.84212362445597510607342428556876257061958312988281e-01, // 0.123457 + 1.50288248944390466015246020196893805831450663390569e-07, // 3.141593 + 9.30810123487737271986064077622380352750042220577598e-07, // 2.718282 + 9.74271275055482721683119962108321487903594970703125e-02, // 0.314159 + 1.31457803694411845540912509022746235132217407226562e-01, // 0.271828 + }; + + public static final double[] SCIPY_GAMMA_CDF_01_1 = { // + 8.27551759585850255085404114652192220091819763183594e-01, // 0.100000 + 8.79419626790056918608229352685157209634780883789062e-01, // 0.200000 + 9.08357989730034232067623634065967053174972534179688e-01, // 0.300000 + 9.27573833553704485410662528011016547679901123046875e-01, // 0.400000 + 9.41402445890133443562319826014572754502296447753906e-01, // 0.500000 + 9.51832144669906954348448380187619477510452270507812e-01, // 0.600000 + 9.59944796430632019124118414765689522027969360351562e-01, // 0.700000 + 9.66394653856728291785316287132445722818374633789062e-01, // 0.800000 + 9.71606904600970766594514316238928586244583129882812e-01, // 0.900000 + 9.75872656273672145488262685830704867839813232421875e-01, // 1.000000 + 9.79399221790659857411753819178557023406028747558594e-01, // 1.100000 + 9.82338840734720442071647994453087449073791503906250e-01, // 1.200000 + 9.84806083916309082937345920072402805089950561523438e-01, // 1.300000 + 9.86888912463439682554167120542842894792556762695312e-01, // 1.400000 + 9.88655983362194623609298105293419212102890014648438e-01, // 1.500000 + 9.90161634314574046200618795410264283418655395507812e-01, // 1.600000 + 9.91449379513895889814989459409844130277633666992188e-01, // 1.700000 + 9.92554419222070327677442946878727525472640991210938e-01, // 1.800000 + 9.93505478144843912069461566716199740767478942871094e-01, // 1.900000 + 9.94326176020188468029914474755059927701950073242188e-01, // 2.000000 + 3.32398405040503219787240141158690676093101501464844e-01, // 0.000010 + 1.05113700610222168818630450459750136360526084899902e-01, // 0.000000 + 8.43459089726093846905996542773209512233734130859375e-01, // 0.123457 + 9.98686603432971331528733571758493781089782714843750e-01, // 3.141593 + 9.97771302397556292262947863491717725992202758789062e-01, // 2.718282 + 9.11526840920496961295782512024743482470512390136719e-01, // 0.314159 + 9.01485773262812539741162254358641803264617919921875e-01, // 0.271828 + }; + + public static final double[] SCIPY_GAMMA_PDF_01_1 = { // + 7.55492013825307062724334628001088276505470275878906e-01, // 0.100000 + 3.66330799305670196108764002929092384874820709228516e-01, // 0.200000 + 2.30123967106513782621135533190681599080562591552734e-01, // 0.300000 + 1.60726519807825291019653946023026946932077407836914e-01, // 0.400000 + 1.18970443671299608290325977577595040202140808105469e-01, // 0.500000 + 9.13579848700025454188988760506617836654186248779297e-02, // 0.600000 + 7.19556586194085856522661970302578993141651153564453e-02, // 0.700000 + 5.77354762017969619369850420298462267965078353881836e-02, // 0.800000 + 4.69868178615557574584471467460389249026775360107422e-02, // 0.900000 + 3.86691694403023741855029982161795487627387046813965e-02, // 1.000000 + 3.21130813374652795966213147949019912630319595336914e-02, // 1.100000 + 2.68684631741564101947705722750470158644020557403564e-02, // 1.200000 + 2.26218168783205615668396859518907149322330951690674e-02, // 1.300000 + 1.91483703787232971416099047701209201477468013763428e-02, // 1.400000 + 1.62830392611040195016691711771272821351885795593262e-02, // 1.500000 + 1.39020923964533350131178579545121465343981981277466e-02, // 1.600000 + 1.19111769796363797108140758496119815390557050704956e-02, // 1.700000 + 1.02372663794025164224477109087274584453552961349487e-02, // 1.800000 + 8.82310747761274466460790222299692686647176742553711e-03, // 1.900000 + 7.62330623530854158131919717789060086943209171295166e-03, // 2.000000 + 3.32395383249902215538895688951015472412109375000000e+03, // 0.000010 + 1.05113700600666418671607971191406250000000000000000e+08, // 0.000000 + 6.10491420220252267725413730659056454896926879882812e-01, // 0.123457 + 1.62124289852739653615409931575186419649980962276459e-03, // 3.141593 + 2.82006743783194159524985167308841482736170291900635e-03, // 2.718282 + 2.17664099893520729400719915247464086860418319702148e-01, // 0.314159 + 2.58666872715845264085743338000611402094364166259766e-01, // 0.271828 + }; + + public static final double[] GNUR_GAMMA_CDF_01_1 = { // + 8.27551759585850477130009039683500304818153381347656e-01, // 0.100000 + 8.79419626790056807585926890169503167271614074707031e-01, // 0.200000 + 9.08357989730034232067623634065967053174972534179688e-01, // 0.300000 + 9.27573833553704485410662528011016547679901123046875e-01, // 0.400000 + 9.41402445890133554584622288530226796865463256835938e-01, // 0.500000 + 9.51832144669907287415355767734581604599952697753906e-01, // 0.600000 + 9.59944796430632130146420877281343564391136169433594e-01, // 0.700000 + 9.66394653856728513829921212163753807544708251953125e-01, // 0.800000 + 9.71606904600970988639119241270236670970916748046875e-01, // 0.900000 + 9.75872656273672256510565148346358910202980041503906e-01, // 1.000000 + 9.79399221790659524344846431631594896316528320312500e-01, // 1.100000 + 9.82338840734720664116252919484395533800125122070312e-01, // 1.200000 + 9.84806083916308971915043457556748762726783752441406e-01, // 1.300000 + 9.86888912463439793576469583058496937155723571777344e-01, // 1.400000 + 9.88655983362194734631600567809073254466056823730469e-01, // 1.500000 + 9.90161634314574046200618795410264283418655395507812e-01, // 1.600000 + 9.91449379513895889814989459409844130277633666992188e-01, // 1.700000 + 9.92554419222070327677442946878727525472640991210938e-01, // 1.800000 + 9.93505478144843912069461566716199740767478942871094e-01, // 1.900000 + 9.94326176020188468029914474755059927701950073242188e-01, // 2.000000 + 3.32398405040503330809542603674344718456268310546875e-01, // 0.000010 + 1.05113700610222196574206066088663646951317787170410e-01, // 0.000000 + 8.43459089726094068950601467804517596960067749023438e-01, // 0.123457 + 9.98686603432971664595640959305455908179283142089844e-01, // 3.141593 + 9.97771302397558956798206963867414742708206176757812e-01, // 2.718282 + 9.11526840920501624232485937682213261723518371582031e-01, // 0.314159 + 9.01485773262837186692308932833839207887649536132812e-01, // 0.271828 + }; + + public static final double[] GNUR_GAMMA_PDF_01_1 = { // + 7.55492013825307506813544478063704445958137512207031e-01, // 0.100000 + 3.66330799305670307131066465444746427237987518310547e-01, // 0.200000 + 2.30123967106513865887862380077422130852937698364258e-01, // 0.300000 + 1.60726519807825346530805177280853968113660812377930e-01, // 0.400000 + 1.18970443671299608290325977577595040202140808105469e-01, // 0.500000 + 9.13579848700025592966866838651185389608144760131836e-02, // 0.600000 + 7.19556586194086134078418126591714099049568176269531e-02, // 0.700000 + 5.77354762017969758147728498443029820919036865234375e-02, // 0.800000 + 4.69868178615557713362349545604956801980733871459961e-02, // 0.900000 + 3.86691694403023811243969021234079264104366302490234e-02, // 1.000000 + 3.21130813374652934744091226093587465584278106689453e-02, // 1.100000 + 2.68684631741564206031114281358895823359489440917969e-02, // 1.200000 + 2.26218168783205719751805418127332814037799835205078e-02, // 1.300000 + 1.91483703787233006110568567237351089715957641601562e-02, // 1.400000 + 1.62830392611040264405630750843556597828865051269531e-02, // 1.500000 + 1.39020923964533436867352378385476185940206050872803e-02, // 1.600000 + 1.19111769796363866497079797568403591867536306381226e-02, // 1.700000 + 1.02372663794025233613416148159558360930532217025757e-02, // 1.800000 + 8.82310747761274292988442624618983245454728603363037e-03, // 1.900000 + 7.62330623530854418340441114310124248731881380081177e-03, // 2.000000 + 3.32395383249902442912571132183074951171875000000000e+03, // 0.000010 + 1.05113700600666448473930358886718750000000000000000e+08, // 0.000000 + 6.10491420220252156703111268143402412533760070800781e-01, // 0.123457 + 1.62124289852696415632771209658358202432282269001007e-03, // 3.141593 + 2.82006743782835809022935258383313339436426758766174e-03, // 2.718282 + 2.17664099893503326654808915918692946434020996093750e-01, // 0.314159 + 2.58666872715738793697681785488384775817394256591797e-01, // 0.271828 + }; + + public static final double[] P_QUANT = { // + 0.0001, 0.001, 0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99, 0.999, 0.9999 // + }; + + public static final double[] SCIPY_GAMMA_QUANT_1_1 = { // + 1.00005000333358370239768275755665172255248762667179e-04, // 0.000100 + 1.00050033358353349620395444929954464896582067012787e-03, // 0.001000 + 1.00503358535014367114435174244135851040482521057129e-02, // 0.010000 + 1.05360515657826322999390811219200259074568748474121e-01, // 0.100000 + 2.87682072451780956878764072826015762984752655029297e-01, // 0.250000 + 6.93147180559945286226763982995180413126945495605469e-01, // 0.500000 + 1.38629436111989079449813289102166891098022460937500e+00, // 0.750000 + 2.30258509299404590109361379290930926799774169921875e+00, // 0.900000 + 4.60517018598809002583038818556815385818481445312500e+00, // 0.990000 + 6.90775527898213592692400197847746312618255615234375e+00, // 0.999000 + 9.21034037197629196214165858691558241844177246093750e+00, // 0.999900 + }; + + public static final double[] GNUR_GAMMA_QUANT_1_1 = { // + 1.00005000333358343134713963618054322068928740918636e-04, // 0.000100 + 1.00050033358353349620395444929954464896582067012787e-03, // 0.001000 + 1.00503358535014419156139453548348683398216962814331e-02, // 0.010000 + 1.05360515657826309121603003404743503779172897338867e-01, // 0.100000 + 2.87682072451780901367612841568188741803169250488281e-01, // 0.250000 + 6.93147180559945286226763982995180413126945495605469e-01, // 0.500000 + 1.38629436111989057245352796599036082625389099121094e+00, // 0.750000 + 2.30258509299404590109361379290930926799774169921875e+00, // 0.900000 + 4.60517018598809002583038818556815385818481445312500e+00, // 0.990000 + 6.90775527898213592692400197847746312618255615234375e+00, // 0.999000 + 9.21034037197629196214165858691558241844177246093750e+00, // 0.999900 + }; + + public static final double[] SCIPY_GAMMA_QUANT_2_1 = { // + 1.42092376217775075153415542672519222833216190338135e-02, // 0.000100 + 4.54020177694895513131889686064823763445019721984863e-02, // 0.001000 + 1.48554740253265954441630469773372169584035873413086e-01, // 0.010000 + 5.31811608389612056058126654534135013818740844726562e-01, // 0.100000 + 9.61278763114777112797071367822354659438133239746094e-01, // 0.250000 + 1.67834699001666076689787132636411115527153015136719e+00, // 0.500000 + 2.69263452888969556653364634257741272449493408203125e+00, // 0.750000 + 3.88972016986742952227018577104900032281875610351562e+00, // 0.900000 + 6.63835206799381172260154926334507763385772705078125e+00, // 0.990000 + 9.23341347645158450063718191813677549362182617187500e+00, // 0.999000 + 1.17563712224955398966130815097130835056304931640625e+01, // 0.999900 + }; + + public static final double[] GNUR_GAMMA_QUANT_2_1 = { // + 1.42092376217774988417241743832164502236992120742798e-02, // 0.000100 + 4.54020177694895721298706803281675092875957489013672e-02, // 0.001000 + 1.48554740253265982197206085402285680174827575683594e-01, // 0.010000 + 5.31811608389611945035824192018480971455574035644531e-01, // 0.100000 + 9.61278763114777001774768905306700617074966430664062e-01, // 0.250000 + 1.67834699001666054485326640133280307054519653320312e+00, // 0.500000 + 2.69263452888969601062285619264002889394760131835938e+00, // 0.750000 + 3.88972016986742952227018577104900032281875610351562e+00, // 0.900000 + 6.63835206799381083442312956321984529495239257812500e+00, // 0.990000 + 9.23341347645158450063718191813677549362182617187500e+00, // 0.999000 + 1.17563712224955381202562421094626188278198242187500e+01, // 0.999900 + }; + + public static final double[] SCIPY_GAMMA_QUANT_4_1 = { // + 2.31796895172818012698101597379718441516160964965820e-01, // 0.000100 + 4.28552413628422945723883685786859132349491119384766e-01, // 0.001000 + 8.23248686345385172202782086969818919897079467773438e-01, // 0.010000 + 1.74476956282491135752366062661167234182357788085938e+00, // 0.100000 + 2.53532021190009304589807470620144158601760864257812e+00, // 0.250000 + 3.67206074885089650550185069732833653688430786132812e+00, // 0.500000 + 5.10942748512337985289377684239298105239868164062500e+00, // 0.750000 + 6.68078306825586487605050933780148625373840332031250e+00, // 0.900000 + 1.00451175148316167451412184163928031921386718750000e+01, // 0.990000 + 1.30622407791880714711396649363450706005096435546875e+01, // 0.999000 + 1.59138140006312944763067207532003521919250488281250e+01, // 0.999900 + }; + + public static final double[] GNUR_GAMMA_QUANT_4_1 = { // + 2.31796895172817957186950366121891420334577560424805e-01, // 0.000100 + 4.28552413628423112257337379560340195894241333007812e-01, // 0.001000 + 8.23248686345385172202782086969818919897079467773438e-01, // 0.010000 + 1.74476956282491157956826555164298042654991149902344e+00, // 0.100000 + 2.53532021190009304589807470620144158601760864257812e+00, // 0.250000 + 3.67206074885089650550185069732833653688430786132812e+00, // 0.500000 + 5.10942748512337896471535714226774871349334716796875e+00, // 0.750000 + 6.68078306825586398787208963767625391483306884765625e+00, // 0.900000 + 1.00451175148316149687843790161423385143280029296875e+01, // 0.990000 + 1.30622407791880696947828255360946059226989746093750e+01, // 0.999000 + 1.59138140006312944763067207532003521919250488281250e+01, // 0.999900 + }; + + public static final double[] SCIPY_GAMMA_QUANT_4_10 = { // + 2.31796895172818040453677213008631952106952667236328e-02, // 0.000100 + 4.28552413628423001235034917044686153531074523925781e-02, // 0.001000 + 8.23248686345385283225084549485472962260246276855469e-02, // 0.010000 + 1.74476956282491157956826555164298042654991149902344e-01, // 0.100000 + 2.53532021190009337896498209374840371310710906982422e-01, // 0.250000 + 3.67206074885089661652415315984399057924747467041016e-01, // 0.500000 + 5.10942748512338007493838176742428913712501525878906e-01, // 0.750000 + 6.68078306825586554218432411289541050791740417480469e-01, // 0.900000 + 1.00451175148316163010520085663301870226860046386719e+00, // 0.990000 + 1.30622407791880723593180846364703029394149780273438e+00, // 0.999000 + 1.59138140006312944763067207532003521919250488281250e+00, // 0.999900 + }; + + public static final double[] GNUR_GAMMA_QUANT_4_10 = { // + 2.31796895172817971064738173936348175629973411560059e-02, // 0.000100 + 4.28552413628423140012912995189253706485033035278320e-02, // 0.001000 + 8.23248686345385283225084549485472962260246276855469e-02, // 0.010000 + 1.74476956282491157956826555164298042654991149902344e-01, // 0.100000 + 2.53532021190009337896498209374840371310710906982422e-01, // 0.250000 + 3.67206074885089661652415315984399057924747467041016e-01, // 0.500000 + 5.10942748512337896471535714226774871349334716796875e-01, // 0.750000 + 6.68078306825586443196129948773887008428573608398438e-01, // 0.900000 + 1.00451175148316163010520085663301870226860046386719e+00, // 0.990000 + 1.30622407791880701388720353861572220921516418457031e+00, // 0.999000 + 1.59138140006312944763067207532003521919250488281250e+00, // 0.999900 + }; + + public static final double[] SCIPY_GAMMA_QUANT_01_10 = { // + 0.00000000000000000000000000000000000000000000000000e+00, // 0.000100 + 0.00000000000000000000000000000000000000000000000000e+00, // 0.001000 + 0.00000000000000000000000000000000000000000000000000e+00, // 0.010000 + 6.07304836274677903746487345757596094421304488975011e-12, // 0.100000 + 5.79171329496954621446304020682127600139210699126124e-08, // 0.250000 + 5.93391104460224945785218331195665086852386593818665e-05, // 0.500000 + 3.53063580735582700775854192443148349411785602569580e-03, // 0.750000 + 2.66154553738837257315097417631477583199739456176758e-02, // 0.900000 + 1.58847781792950559776045338367111980915069580078125e-01, // 0.990000 + 3.36367701171875421728429955692263320088386535644531e-01, // 0.999000 + 5.31624347769055760615231065457919612526893615722656e-01, // 0.999900 + }; + + public static final double[] GNUR_GAMMA_QUANT_01_10 = { // + 6.07304836240800246013734290758296395601191608171445e-42, // 0.000100 + 6.07304836240794013868566283512457787341965380549669e-32, // 0.001000 + 6.07304836240792099284182672275736116917595330090291e-22, // 0.010000 + 6.07304836274317627815629288782017336338625579372774e-12, // 0.100000 + 5.79171329496960180100020377028102736005621409276500e-08, // 0.250000 + 5.93391104460226165512662377388153345236787572503090e-05, // 0.500000 + 3.53063580735583307929070784325631393585354089736938e-03, // 0.750000 + 2.66154553738837638954262132529038353823125362396240e-02, // 0.900000 + 1.58847781792950504264894107109284959733486175537109e-01, // 0.990000 + 3.36367701171875477239581186950090341269969940185547e-01, // 0.999000 + 5.31624347769055760615231065457919612526893615722656e-01, // 0.999900 + }; + + public static final double[] SCIPY_GAMMA_QUANT_01_20 = { // + 0.00000000000000000000000000000000000000000000000000e+00, // 0.000100 + 0.00000000000000000000000000000000000000000000000000e+00, // 0.001000 + 0.00000000000000000000000000000000000000000000000000e+00, // 0.010000 + 3.03652418137338951873243672878798047210652244487505e-12, // 0.100000 + 2.89585664748477310723152010341063800069605349563062e-08, // 0.250000 + 2.96695552230112472892609165597832543426193296909332e-05, // 0.500000 + 1.76531790367791350387927096221574174705892801284790e-03, // 0.750000 + 1.33077276869418628657548708815738791599869728088379e-02, // 0.900000 + 7.94238908964752798880226691835559904575347900390625e-02, // 0.990000 + 1.68183850585937710864214977846131660044193267822266e-01, // 0.999000 + 2.65812173884527880307615532728959806263446807861328e-01, // 0.999900 + }; + + public static final double[] GNUR_GAMMA_QUANT_01_20 = { // + 3.03652418120400123006867145379148197800595804085722e-42, // 0.000100 + 3.03652418120397006934283141756228893670982690274834e-32, // 0.001000 + 3.03652418120396049642091336137868058458797665045146e-22, // 0.010000 + 3.03652418137158813907814644391008668169312789686387e-12, // 0.100000 + 2.89585664748480090050010188514051368002810704638250e-08, // 0.250000 + 2.96695552230113082756331188694076672618393786251545e-05, // 0.500000 + 1.76531790367791653964535392162815696792677044868469e-03, // 0.750000 + 1.33077276869418819477131066264519176911562681198120e-02, // 0.900000 + 7.94238908964752521324470535546424798667430877685547e-02, // 0.990000 + 1.68183850585937738619790593475045170634984970092773e-01, // 0.999000 + 2.65812173884527880307615532728959806263446807861328e-01, // 0.999900 + }; + + public static final double[] SCIPY_GAMMA_QUANT_01_4 = { // + 0.00000000000000000000000000000000000000000000000000e+00, // 0.000100 + 0.00000000000000000000000000000000000000000000000000e+00, // 0.001000 + 0.00000000000000000000000000000000000000000000000000e+00, // 0.010000 + 1.51826209068669467858686166976238136189225613748022e-11, // 0.100000 + 1.44792832374238655361576005170531900034802674781531e-07, // 0.250000 + 1.48347776115056236446304582798916271713096648454666e-04, // 0.500000 + 8.82658951838956665203461682267516152933239936828613e-03, // 0.750000 + 6.65386384347093073898804505006410181522369384765625e-02, // 0.900000 + 3.97119454482376343928962114659952931106090545654297e-01, // 0.990000 + 8.40919252929688498809923657972831279039382934570312e-01, // 0.999000 + 1.32906086942263934602692643238697201013565063476562e+00, // 0.999900 + }; + + public static final double[] GNUR_GAMMA_QUANT_01_4 = { // + 1.51826209060200048758698283629955882669254719901167e-41, // 0.000100 + 1.51826209060198492519497318340481080243853975684947e-31, // 0.001000 + 1.51826209060198020119068264779783997354452683633590e-21, // 0.010000 + 1.51826209068579382720100313806021671836354869356001e-11, // 0.100000 + 1.44792832374240031790115293408582886058866279199719e-07, // 0.250000 + 1.48347776115056534601902016312635623762616887688637e-04, // 0.500000 + 8.82658951838958226454590061393901123665273189544678e-03, // 0.750000 + 6.65386384347094045343951052018383052200078964233398e-02, // 0.900000 + 3.97119454482376232906659652144298888742923736572266e-01, // 0.990000 + 8.40919252929688609832226120488485321402549743652344e-01, // 0.999000 + 1.32906086942263934602692643238697201013565063476562e+00, // 0.999900 + }; + + public static final double[] SCIPY_GAMMA_QUANT_01_1 = { // + 0.00000000000000000000000000000000000000000000000000e+00, // 0.000100 + 0.00000000000000000000000000000000000000000000000000e+00, // 0.001000 + 0.00000000000000000000000000000000000000000000000000e+00, // 0.010000 + 6.07304836274677871434744667904952544756902454992087e-11, // 0.100000 + 5.79171329496954621446304020682127600139210699126124e-07, // 0.250000 + 5.93391104460224945785218331195665086852386593818665e-04, // 0.500000 + 3.53063580735582666081384672907006461173295974731445e-02, // 0.750000 + 2.66154553738837229559521802002564072608947753906250e-01, // 0.900000 + 1.58847781792950537571584845863981172442436218261719e+00, // 0.990000 + 3.36367701171875399523969463189132511615753173828125e+00, // 0.999000 + 5.31624347769055738410770572954788804054260253906250e+00, // 0.999900 + }; + + public static final double[] GNUR_GAMMA_QUANT_01_1 = { // + 6.07304836240800195034793134519823530677018879604667e-41, // 0.000100 + 6.07304836240793970077989273361924320975415902739790e-31, // 0.001000 + 6.07304836240792080476273059119135989417810734534360e-21, // 0.010000 + 6.07304836274317530880401255224086687345419477424002e-11, // 0.100000 + 5.79171329496960127160461173634331544235465116798878e-07, // 0.250000 + 5.93391104460226138407608065250542495050467550754547e-04, // 0.500000 + 3.53063580735583290581836024557560449466109275817871e-02, // 0.750000 + 2.66154553738837618137580420807353220880031585693359e-01, // 0.900000 + 1.58847781792950493162663860857719555497169494628906e+00, // 0.990000 + 3.36367701171875443932890448195394128561019897460938e+00, // 0.999000 + 5.31624347769055738410770572954788804054260253906250e+00, // 0.999900 + }; + + @Test + public void testPDF() { + checkPDF(new GammaDistribution(1., 1.), P_CDFPDF, SCIPY_GAMMA_PDF_1_1, 1e-12); + checkPDF(new GammaDistribution(2., 1.), P_CDFPDF, SCIPY_GAMMA_PDF_2_1, 1e-12); + checkPDF(new GammaDistribution(4., 1.), P_CDFPDF, SCIPY_GAMMA_PDF_4_1, 1e-12); + checkPDF(new GammaDistribution(4., 10), P_CDFPDF, SCIPY_GAMMA_PDF_4_10, 1e-13); + checkPDF(new GammaDistribution(.1, 10), P_CDFPDF, SCIPY_GAMMA_PDF_01_10, 1e-11); + checkPDF(new GammaDistribution(.1, 20), P_CDFPDF, SCIPY_GAMMA_PDF_01_20, 1e-14); + checkPDF(new GammaDistribution(.1, 4.), P_CDFPDF, SCIPY_GAMMA_PDF_01_4, 1e-12); + checkPDF(new GammaDistribution(.1, 1.), P_CDFPDF, SCIPY_GAMMA_PDF_01_1, 1e-12); + checkPDF(new GammaDistribution(1., 1.), P_CDFPDF, GNUR_GAMMA_PDF_1_1, 1e-15); + checkPDF(new GammaDistribution(2., 1.), P_CDFPDF, GNUR_GAMMA_PDF_2_1, 1e-16); + checkPDF(new GammaDistribution(4., 1.), P_CDFPDF, GNUR_GAMMA_PDF_4_1, 1e-14); + checkPDF(new GammaDistribution(4., 10), P_CDFPDF, GNUR_GAMMA_PDF_4_10, 1e-14); + checkPDF(new GammaDistribution(.1, 10), P_CDFPDF, GNUR_GAMMA_PDF_01_10, 1e-15); + checkPDF(new GammaDistribution(.1, 20), P_CDFPDF, GNUR_GAMMA_PDF_01_20, 1e-14); + checkPDF(new GammaDistribution(.1, 4.), P_CDFPDF, GNUR_GAMMA_PDF_01_4, 1e-15); + checkPDF(new GammaDistribution(.1, 1.), P_CDFPDF, GNUR_GAMMA_PDF_01_1, 1e-15); + } + + @Test + public void testCDF() { + checkCDF(new GammaDistribution(1., 1.), P_CDFPDF, SCIPY_GAMMA_CDF_1_1, 1e-13); + checkCDF(new GammaDistribution(2., 1.), P_CDFPDF, SCIPY_GAMMA_CDF_2_1, 1e-12); + checkCDF(new GammaDistribution(4., 1.), P_CDFPDF, SCIPY_GAMMA_CDF_4_1, 1e-12); + checkCDF(new GammaDistribution(4., 10), P_CDFPDF, SCIPY_GAMMA_CDF_4_10, 1e-12); + checkCDF(new GammaDistribution(.1, 10), P_CDFPDF, SCIPY_GAMMA_CDF_01_10, 1e-14); + checkCDF(new GammaDistribution(.1, 20), P_CDFPDF, SCIPY_GAMMA_CDF_01_20, 1e-15); + checkCDF(new GammaDistribution(.1, 4.), P_CDFPDF, SCIPY_GAMMA_CDF_01_4, 1e-13); + checkCDF(new GammaDistribution(.1, 1.), P_CDFPDF, SCIPY_GAMMA_CDF_01_1, 1e-13); + checkCDF(new GammaDistribution(1., 1.), P_CDFPDF, GNUR_GAMMA_CDF_1_1, 1e-15); + checkCDF(new GammaDistribution(2., 1.), P_CDFPDF, GNUR_GAMMA_CDF_2_1, 1e-15); + checkCDF(new GammaDistribution(4., 1.), P_CDFPDF, GNUR_GAMMA_CDF_4_1, 1e-14); + checkCDF(new GammaDistribution(4., 10), P_CDFPDF, GNUR_GAMMA_CDF_4_10, 1e-15); + checkCDF(new GammaDistribution(.1, 10), P_CDFPDF, GNUR_GAMMA_CDF_01_10, 1e-15); + checkCDF(new GammaDistribution(.1, 20), P_CDFPDF, GNUR_GAMMA_CDF_01_20, 1e-15); + checkCDF(new GammaDistribution(.1, 4.), P_CDFPDF, GNUR_GAMMA_CDF_01_4, 1e-15); + checkCDF(new GammaDistribution(.1, 1.), P_CDFPDF, GNUR_GAMMA_CDF_01_1, 1e-15); + } + + @Test + public void testProbit() { + checkQuantile(new GammaDistribution(1., 1.), P_QUANT, SCIPY_GAMMA_QUANT_1_1, 1e-14); + checkQuantile(new GammaDistribution(2., 1.), P_QUANT, SCIPY_GAMMA_QUANT_2_1, 1e-13); + checkQuantile(new GammaDistribution(4., 1.), P_QUANT, SCIPY_GAMMA_QUANT_4_1, 1e-13); + checkQuantile(new GammaDistribution(4., 10), P_QUANT, SCIPY_GAMMA_QUANT_4_10, 1e-13); + checkQuantile(new GammaDistribution(.1, 10), P_QUANT, SCIPY_GAMMA_QUANT_01_10, 1e-13); + checkQuantile(new GammaDistribution(.1, 20), P_QUANT, SCIPY_GAMMA_QUANT_01_20, 1e-14); + checkQuantile(new GammaDistribution(.1, 4.), P_QUANT, SCIPY_GAMMA_QUANT_01_4, 1e-13); + checkQuantile(new GammaDistribution(.1, 1.), P_QUANT, SCIPY_GAMMA_QUANT_01_1, 1e-13); + checkQuantile(new GammaDistribution(1., 1.), P_QUANT, GNUR_GAMMA_QUANT_1_1, 1e-14); + checkQuantile(new GammaDistribution(2., 1.), P_QUANT, GNUR_GAMMA_QUANT_2_1, 1e-13); + checkQuantile(new GammaDistribution(4., 1.), P_QUANT, GNUR_GAMMA_QUANT_4_1, 1e-13); + checkQuantile(new GammaDistribution(4., 10), P_QUANT, GNUR_GAMMA_QUANT_4_10, 1e-13); + checkQuantile(new GammaDistribution(.1, 10), P_QUANT, GNUR_GAMMA_QUANT_01_10, 1e-13); + checkQuantile(new GammaDistribution(.1, 20), P_QUANT, GNUR_GAMMA_QUANT_01_20, 1e-14); + checkQuantile(new GammaDistribution(.1, 4.), P_QUANT, GNUR_GAMMA_QUANT_01_4, 1e-13); + checkQuantile(new GammaDistribution(.1, 1.), P_QUANT, GNUR_GAMMA_QUANT_01_1, 1e-13); + } + + @Test + public void testRandomAndEstimation() { + GammaDistribution g = new GammaDistribution(1.2345, 0.12345, new Random(0)); + double[] data = new double[10000]; + for(int i = 0; i < data.length; i++) { + data[i] = g.nextRandom(); + } + GammaDistribution g2 = GammaDistribution.estimate(data); + assertEquals("k does not match.", g.getK(), g2.getK(), 1E-2); + assertEquals("theta does not match.", g.getTheta(), g2.getTheta(), 1E-5); + } +}
\ No newline at end of file diff --git a/test/de/lmu/ifi/dbs/elki/math/statistics/distribution/TestNormalDistribution.java b/test/de/lmu/ifi/dbs/elki/math/statistics/distribution/TestNormalDistribution.java new file mode 100644 index 00000000..c4924073 --- /dev/null +++ b/test/de/lmu/ifi/dbs/elki/math/statistics/distribution/TestNormalDistribution.java @@ -0,0 +1,519 @@ +package de.lmu.ifi.dbs.elki.math.statistics.distribution; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2012 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +import org.junit.Test; + +import de.lmu.ifi.dbs.elki.JUnit4Test; + +/** + * Unit test for the Normal distribution in ELKI. + * + * The reference values were computed using GNU R and SciPy. + * + * @author Erich Schubert + */ +public class TestNormalDistribution extends AbstractDistributionTest implements JUnit4Test { + public static final double[] P_CDFPDF = { // + 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0, 1e-05, 1e-10, 0.1234567, 3.14159265359, 2.71828182846, 0.314159265359, 0.271828182846 // + }; + + public static final double[] SCIPY_NORM_CDF_0_1 = { // + 5.39827837277028987905680423864396288990974426269531e-01, // 0.100000 + 5.79259709439102987715841663884930312633514404296875e-01, // 0.200000 + 6.17911422188952563772090798011049628257751464843750e-01, // 0.300000 + 6.55421741610324182225610911700641736388206481933594e-01, // 0.400000 + 6.91462461274013118206482886307640001177787780761719e-01, // 0.500000 + 7.25746882249926450469956762390211224555969238281250e-01, // 0.600000 + 7.58036347776926966446353617357090115547180175781250e-01, // 0.700000 + 7.88144601416603363119861569430213421583175659179688e-01, // 0.800000 + 8.15939874653240471147341850155498832464218139648438e-01, // 0.900000 + 8.41344746068542925776512220181757584214210510253906e-01, // 1.000000 + 8.64333939053617328340806125197559595108032226562500e-01, // 1.100000 + 8.84930329778291779874166422814596444368362426757812e-01, // 1.200000 + 9.03199515414389697554042868432588875293731689453125e-01, // 1.300000 + 9.19243340766228933524928379483753815293312072753906e-01, // 1.400000 + 9.33192798731141914814202209527138620615005493164062e-01, // 1.500000 + 9.45200708300442005160846292710630223155021667480469e-01, // 1.600000 + 9.55434537241456993683641485404223203659057617187500e-01, // 1.700000 + 9.64069680887074231812050584267126396298408508300781e-01, // 1.800000 + 9.71283440183998147965382941038114950060844421386719e-01, // 1.900000 + 9.77249868051820791414741051994496956467628479003906e-01, // 2.000000 + 5.00003989422803996056643427436938509345054626464844e-01, // 0.000010 + 5.00000000039894199055368062545312568545341491699219e-01, // 0.000000 + 5.49127269841833243191331348498351871967315673828125e-01, // 0.123457 + 9.99159841831736672546071531542111188173294067382812e-01, // 3.141593 + 9.96718904163704344512098032282665371894836425781250e-01, // 2.718282 + 6.23299960463814084477007781970314681529998779296875e-01, // 0.314159 + 6.07122932952487870394975288945715874433517456054688e-01, // 0.271828 + }; + + public static final double[] SCIPY_NORM_PDF_0_1 = { // + 3.96952547477011807863789272232679650187492370605469e-01, // 0.100000 + 3.91042693975455879495939370826818048954010009765625e-01, // 0.200000 + 3.81387815460524137733955285511910915374755859375000e-01, // 0.300000 + 3.68270140303323334496354846123722381889820098876953e-01, // 0.400000 + 3.52065326764299524331391921805334277451038360595703e-01, // 0.500000 + 3.33224602891799670523198528826469555497169494628906e-01, // 0.600000 + 3.12253933366761271539502331506810151040554046630859e-01, // 0.700000 + 2.89691552761482729039244077284820377826690673828125e-01, // 0.800000 + 2.66085249898754816477719487011199817061424255371094e-01, // 0.900000 + 2.41970724519143365327522587904240936040878295898438e-01, // 1.000000 + 2.17852177032550553548162497463636100292205810546875e-01, // 1.100000 + 1.94186054983212980085127696838753763586282730102539e-01, // 1.200000 + 1.71368592047807355438493459587334655225276947021484e-01, // 1.300000 + 1.49727465635744877436863475850259419530630111694336e-01, // 1.400000 + 1.29517595665891743772490940500574652105569839477539e-01, // 1.500000 + 1.10920834679455543314574583746434655040502548217773e-01, // 1.600000 + 9.40490773768869470217168782255612313747406005859375e-02, // 1.700000 + 7.89501583008941493213583839860802982002496719360352e-02, // 1.800000 + 6.56158147746765951779934766818769276142120361328125e-02, // 1.900000 + 5.39909665131880628363703067407186608761548995971680e-02, // 2.000000 + 3.98942280381485603335534051439026370644569396972656e-01, // 0.000010 + 3.98942280401432702863218082711682654917240142822266e-01, // 0.000000 + 3.95913584859803491244889528388739563524723052978516e-01, // 0.123457 + 2.86914634635447975477551274536835990147665143013000e-03, // 3.141593 + 9.91737923440347626802271463475335622206330299377441e-03, // 2.718282 + 3.79733132751562452167348737930296920239925384521484e-01, // 0.314159 + 3.84472192557199943241386108638835139572620391845703e-01, // 0.271828 + }; + + public static final double[] GNUR_NORM_CDF_0_1 = { // + 5.39827837277028987905680423864396288990974426269531e-01, // 0.100000 + 5.79259709439102987715841663884930312633514404296875e-01, // 0.200000 + 6.17911422188952674794393260526703670620918273925781e-01, // 0.300000 + 6.55421741610324182225610911700641736388206481933594e-01, // 0.400000 + 6.91462461274013007184180423791985958814620971679688e-01, // 0.500000 + 7.25746882249926450469956762390211224555969238281250e-01, // 0.600000 + 7.58036347776926966446353617357090115547180175781250e-01, // 0.700000 + 7.88144601416603363119861569430213421583175659179688e-01, // 0.800000 + 8.15939874653240471147341850155498832464218139648438e-01, // 0.900000 + 8.41344746068542925776512220181757584214210510253906e-01, // 1.000000 + 8.64333939053617328340806125197559595108032226562500e-01, // 1.100000 + 8.84930329778291779874166422814596444368362426757812e-01, // 1.200000 + 9.03199515414389697554042868432588875293731689453125e-01, // 1.300000 + 9.19243340766228933524928379483753815293312072753906e-01, // 1.400000 + 9.33192798731141914814202209527138620615005493164062e-01, // 1.500000 + 9.45200708300442005160846292710630223155021667480469e-01, // 1.600000 + 9.55434537241456993683641485404223203659057617187500e-01, // 1.700000 + 9.64069680887074231812050584267126396298408508300781e-01, // 1.800000 + 9.71283440183998147965382941038114950060844421386719e-01, // 1.900000 + 9.77249868051820791414741051994496956467628479003906e-01, // 2.000000 + 5.00003989422803996056643427436938509345054626464844e-01, // 0.000010 + 5.00000000039894199055368062545312568545341491699219e-01, // 0.000000 + 5.49127269841833243191331348498351871967315673828125e-01, // 0.123457 + 9.99159841831737227657583844120381399989128112792969e-01, // 3.141593 + 9.96718904163713781407807346113258972764015197753906e-01, // 2.718282 + 6.23299960463821967060482620581751689314842224121094e-01, // 0.314159 + 6.07122932952524618777090381627203896641731262207031e-01, // 0.271828 + }; + + public static final double[] GNUR_NORM_PDF_0_1 = { // + 3.96952547477011807863789272232679650187492370605469e-01, // 0.100000 + 3.91042693975455879495939370826818048954010009765625e-01, // 0.200000 + 3.81387815460524137733955285511910915374755859375000e-01, // 0.300000 + 3.68270140303323334496354846123722381889820098876953e-01, // 0.400000 + 3.52065326764299524331391921805334277451038360595703e-01, // 0.500000 + 3.33224602891799670523198528826469555497169494628906e-01, // 0.600000 + 3.12253933366761271539502331506810151040554046630859e-01, // 0.700000 + 2.89691552761482729039244077284820377826690673828125e-01, // 0.800000 + 2.66085249898754816477719487011199817061424255371094e-01, // 0.900000 + 2.41970724519143365327522587904240936040878295898438e-01, // 1.000000 + 2.17852177032550525792586881834722589701414108276367e-01, // 1.100000 + 1.94186054983212952329552081209840252995491027832031e-01, // 1.200000 + 1.71368592047807355438493459587334655225276947021484e-01, // 1.300000 + 1.49727465635744877436863475850259419530630111694336e-01, // 1.400000 + 1.29517595665891743772490940500574652105569839477539e-01, // 1.500000 + 1.10920834679455543314574583746434655040502548217773e-01, // 1.600000 + 9.40490773768869470217168782255612313747406005859375e-02, // 1.700000 + 7.89501583008941493213583839860802982002496719360352e-02, // 1.800000 + 6.56158147746765951779934766818769276142120361328125e-02, // 1.900000 + 5.39909665131880628363703067407186608761548995971680e-02, // 2.000000 + 3.98942280381485603335534051439026370644569396972656e-01, // 0.000010 + 3.98942280401432702863218082711682654917240142822266e-01, // 0.000000 + 3.95913584859803435733738297130912542343139648437500e-01, // 0.123457 + 2.86914634635261405967709968933831987669691443443298e-03, // 3.141593 + 9.91737923437773644108617077108647208660840988159180e-03, // 2.718282 + 3.79733132751559954165543331328080967068672180175781e-01, // 0.314159 + 3.84472192557189951234164482229971326887607574462891e-01, // 0.271828 + }; + + public static final double[] SCIPY_NORM_CDF_1_3 = { // + 3.82088577811047380716757970731123350560665130615234e-01, // 0.100000 + 3.94862910464025107515340096142608672380447387695312e-01, // 0.200000 + 4.07751288296895331164648723643040284514427185058594e-01, // 0.300000 + 4.20740290560896956773007104857242666184902191162109e-01, // 0.400000 + 4.33816167389096341011622826044913381338119506835938e-01, // 0.500000 + 4.46964883376386012958647597770323045551776885986328e-01, // 0.600000 + 4.60172162722971012094319576135603711009025573730469e-01, // 0.700000 + 4.73423535699634911999567066231975331902503967285156e-01, // 0.800000 + 4.86704386182907899893734793295152485370635986328125e-01, // 0.900000 + 5.00000000000000000000000000000000000000000000000000e-01, // 1.000000 + 5.13295613817092100106265206704847514629364013671875e-01, // 1.100000 + 5.26576464300365088000432933768024668097496032714844e-01, // 1.200000 + 5.39827837277028987905680423864396288990974426269531e-01, // 1.300000 + 5.53035116623614042552503633487503975629806518554688e-01, // 1.400000 + 5.66183832610903658988377173955086618661880493164062e-01, // 1.500000 + 5.79259709439102987715841663884930312633514404296875e-01, // 1.600000 + 5.92248711703104668835351276356959715485572814941406e-01, // 1.700000 + 6.05137089535974892484659903857391327619552612304688e-01, // 1.800000 + 6.17911422188952563772090798011049628257751464843750e-01, // 1.900000 + 6.30558659818236333371999080554815009236335754394531e-01, // 2.000000 + 3.69442598126554777593355538556352257728576660156250e-01, // 0.000010 + 3.69441340194343048608516255626454949378967285156250e-01, // 0.000000 + 3.85074080732704326912596570764435455203056335449219e-01, // 0.123457 + 7.62344427321841155986703597591258585453033447265625e-01, // 3.141593 + 7.16596603378728214117643346980912610888481140136719e-01, // 2.718282 + 4.09584635073669933724715974676655605435371398925781e-01, // 0.314159 + 4.04109630623745164079707592463819310069084167480469e-01, // 0.271828 + }; + + public static final double[] SCIPY_NORM_PDF_1_3 = { // + 1.27129271820174721829843633713608141988515853881836e-01, // 0.100000 + 1.28335624865337977196233509857847820967435836791992e-01, // 0.200000 + 1.29409556907848971585295316799602005630731582641602e-01, // 0.300000 + 1.30347564658485293165313123608939349651336669921875e-01, // 0.400000 + 1.31146572033979974536421764241822529584169387817383e-01, // 0.500000 + 1.31803946961939227877635971708514261990785598754883e-01, // 0.600000 + 1.32317515825670611873121629287197720259428024291992e-01, // 0.700000 + 1.32685575437984065727547999813396017998456954956055e-01, // 0.800000 + 1.32906902451659009178897008496278431266546249389648e-01, // 0.900000 + 1.32980760133810910206264566113532055169343948364258e-01, // 1.000000 + 1.32906902451659009178897008496278431266546249389648e-01, // 1.100000 + 1.32685575437984065727547999813396017998456954956055e-01, // 1.200000 + 1.32317515825670611873121629287197720259428024291992e-01, // 1.300000 + 1.31803946961939227877635971708514261990785598754883e-01, // 1.400000 + 1.31146572033979974536421764241822529584169387817383e-01, // 1.500000 + 1.30347564658485293165313123608939349651336669921875e-01, // 1.600000 + 1.29409556907848971585295316799602005630731582641602e-01, // 1.700000 + 1.28335624865337977196233509857847820967435836791992e-01, // 1.800000 + 1.27129271820174721829843633713608141988515853881836e-01, // 1.900000 + 1.25794409230997744630897727802221197634935379028320e-01, // 2.000000 + 1.25794549001942340682447252220299560576677322387695e-01, // 0.000010 + 1.25794409232395459907749568628787528723478317260742e-01, // 0.000000 + 1.27423930053037842702323700905253645032644271850586e-01, // 0.123457 + 1.03069487463836984009191155564622022211551666259766e-01, // 3.141593 + 1.12863271483980939779989682847372023388743400573730e-01, // 2.718282 + 1.29550708126044883394811790822132024914026260375977e-01, // 0.314159 + 1.29120619428010979801157986912585329264402389526367e-01, // 0.271828 + }; + + public static final double[] GNUR_NORM_CDF_1_3 = { // + 3.82088577811047325205606739473296329379081726074219e-01, // 0.100000 + 3.94862910464025163026491327400435693562030792236328e-01, // 0.200000 + 4.07751288296895275653497492385213263332843780517578e-01, // 0.300000 + 4.20740290560897012284158336115069687366485595703125e-01, // 0.400000 + 4.33816167389096341011622826044913381338119506835938e-01, // 0.500000 + 4.46964883376386012958647597770323045551776885986328e-01, // 0.600000 + 4.60172162722971012094319576135603711009025573730469e-01, // 0.700000 + 4.73423535699634911999567066231975331902503967285156e-01, // 0.800000 + 4.86704386182907899893734793295152485370635986328125e-01, // 0.900000 + 5.00000000000000000000000000000000000000000000000000e-01, // 1.000000 + 5.13295613817092100106265206704847514629364013671875e-01, // 1.100000 + 5.26576464300365088000432933768024668097496032714844e-01, // 1.200000 + 5.39827837277028987905680423864396288990974426269531e-01, // 1.300000 + 5.53035116623614042552503633487503975629806518554688e-01, // 1.400000 + 5.66183832610903658988377173955086618661880493164062e-01, // 1.500000 + 5.79259709439102987715841663884930312633514404296875e-01, // 1.600000 + 5.92248711703104668835351276356959715485572814941406e-01, // 1.700000 + 6.05137089535974892484659903857391327619552612304688e-01, // 1.800000 + 6.17911422188952674794393260526703670620918273925781e-01, // 1.900000 + 6.30558659818236333371999080554815009236335754394531e-01, // 2.000000 + 3.69442598126554777593355538556352257728576660156250e-01, // 0.000010 + 3.69441340194343048608516255626454949378967285156250e-01, // 0.000000 + 3.85074080732704326912596570764435455203056335449219e-01, // 0.123457 + 7.62344427321862472268776400596834719181060791015625e-01, // 3.141593 + 7.16596603378836016773334449680987745523452758789062e-01, // 2.718282 + 4.09584635073672598259975075052352622151374816894531e-01, // 0.314159 + 4.04109630623757543066432162959245033562183380126953e-01, // 0.271828 + }; + + public static final double[] GNUR_NORM_PDF_1_3 = { // + 1.27129271820174721829843633713608141988515853881836e-01, // 0.100000 + 1.28335624865337977196233509857847820967435836791992e-01, // 0.200000 + 1.29409556907848943829719701170688495039939880371094e-01, // 0.300000 + 1.30347564658485293165313123608939349651336669921875e-01, // 0.400000 + 1.31146572033979974536421764241822529584169387817383e-01, // 0.500000 + 1.31803946961939227877635971708514261990785598754883e-01, // 0.600000 + 1.32317515825670611873121629287197720259428024291992e-01, // 0.700000 + 1.32685575437984065727547999813396017998456954956055e-01, // 0.800000 + 1.32906902451659009178897008496278431266546249389648e-01, // 0.900000 + 1.32980760133810910206264566113532055169343948364258e-01, // 1.000000 + 1.32906902451659009178897008496278431266546249389648e-01, // 1.100000 + 1.32685575437984065727547999813396017998456954956055e-01, // 1.200000 + 1.32317515825670611873121629287197720259428024291992e-01, // 1.300000 + 1.31803946961939227877635971708514261990785598754883e-01, // 1.400000 + 1.31146572033979974536421764241822529584169387817383e-01, // 1.500000 + 1.30347564658485293165313123608939349651336669921875e-01, // 1.600000 + 1.29409556907848943829719701170688495039939880371094e-01, // 1.700000 + 1.28335624865337977196233509857847820967435836791992e-01, // 1.800000 + 1.27129271820174721829843633713608141988515853881836e-01, // 1.900000 + 1.25794409230997716875322112173307687044143676757812e-01, // 2.000000 + 1.25794549001942340682447252220299560576677322387695e-01, // 0.000010 + 1.25794409232395459907749568628787528723478317260742e-01, // 0.000000 + 1.27423930053037842702323700905253645032644271850586e-01, // 0.123457 + 1.03069487463831918616641303287906339392066001892090e-01, // 3.141593 + 1.12863271483960359020670694008003920316696166992188e-01, // 2.718282 + 1.29550708126045049928265484595613088458776473999023e-01, // 0.314159 + 1.29120619428011951246304533924558199942111968994141e-01, // 0.271828 + }; + + public static final double[] SCIPY_NORM_CDF_01_01 = { // + 5.00000000000000000000000000000000000000000000000000e-01, // 0.100000 + 8.41344746068542925776512220181757584214210510253906e-01, // 0.200000 + 9.77249868051820791414741051994496956467628479003906e-01, // 0.300000 + 9.98650101968369896532351503992686048150062561035156e-01, // 0.400000 + 9.99968328758166880021462930017150938510894775390625e-01, // 0.500000 + 9.99999713348428076464813329948810860514640808105469e-01, // 0.600000 + 9.99999999013412299575520592043176293373107910156250e-01, // 0.700000 + 9.99999999998720134897212119540199637413024902343750e-01, // 0.800000 + 9.99999999999999333866185224906075745820999145507812e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.58679452213762561818555241188732907176017761230469e-01, // 0.000010 + 1.58655254173427739949886472459184005856513977050781e-01, // 0.000000 + 5.92727587424917268243973467178875580430030822753906e-01, // 0.123457 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 9.83886862290707098210873482457827776670455932617188e-01, // 0.314159 + 9.57127388891955366290176243637688457965850830078125e-01, // 0.271828 + }; + + public static final double[] SCIPY_NORM_PDF_01_01 = { // + 3.98942280401432691760987836460117250680923461914062e+00, // 0.100000 + 2.41970724519143365327522587904240936040878295898438e+00, // 0.200000 + 5.39909665131880767141581145551754161715507507324219e-01, // 0.300000 + 4.43184841193799972103306572535075247287750244140625e-02, // 0.400000 + 1.33830225764885362219058784205572010250762104988098e-03, // 0.500000 + 1.48671951473429788965346931561839483038056641817093e-05, // 0.600000 + 6.07588284982330709772167754920135784857393446145579e-08, // 0.700000 + 9.13472040836459525705208369548147081390387924670904e-11, // 0.800000 + 5.05227108353689257466156274028694701290357130241127e-14, // 0.900000 + 1.02797735716689157526658972936723696508848598985854e-17, // 1.000000 + 7.69459862670641946739490369438934977209475249356985e-22, // 1.100000 + 2.11881925350939881499454444092795876174188364829642e-26, // 1.200000 + 2.14638373566309106931513939051468388983176710410808e-31, // 1.300000 + 7.99882775700703797498824580100503753079025371215284e-37, // 1.400000 + 1.09660655938900269201836083695141959890318473157378e-42, // 1.500000 + 5.53070954984441600375694607190945207837912395256854e-49, // 1.600000 + 1.02616307279193257019879511994839957950622370045046e-55, // 1.700000 + 7.00418213431858258321746607183141912867315677743039e-63, // 1.800000 + 1.75874954259520391793557171197265931013763303279246e-70, // 1.900000 + 1.62463603677370066317505050197138645763358359341841e-78, // 2.000000 + 2.41994921591514611236561904661357402801513671875000e+00, // 0.000010 + 2.41970724761114075462842265551444143056869506835938e+00, // 0.000000 + 3.88116637682825116328899639484006911516189575195312e+00, // 0.123457 + 5.14979409860350480381150213015443214774639694748316e-201, // 3.141593 + 5.46759448135644106738407952883081851114066212526125e-149, // 2.718282 + 4.02700170728412354392844463291112333536148071289062e-01, // 0.314159 + 9.11558359421968611435715956758940592408180236816406e-01, // 0.271828 + }; + + public static final double[] GNUR_NORM_CDF_01_01 = { // + 5.00000000000000000000000000000000000000000000000000e-01, // 0.100000 + 8.41344746068542925776512220181757584214210510253906e-01, // 0.200000 + 9.77249868051820791414741051994496956467628479003906e-01, // 0.300000 + 9.98650101968369896532351503992686048150062561035156e-01, // 0.400000 + 9.99968328758166880021462930017150938510894775390625e-01, // 0.500000 + 9.99999713348428076464813329948810860514640808105469e-01, // 0.600000 + 9.99999999013412299575520592043176293373107910156250e-01, // 0.700000 + 9.99999999998720134897212119540199637413024902343750e-01, // 0.800000 + 9.99999999999999333866185224906075745820999145507812e-01, // 0.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.000000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.100000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.200000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.300000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.400000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.500000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.600000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.700000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.800000 + 1.00000000000000000000000000000000000000000000000000e+00, // 1.900000 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.000000 + 1.58679452213762561818555241188732907176017761230469e-01, // 0.000010 + 1.58655254173427739949886472459184005856513977050781e-01, // 0.000000 + 5.92727587424917268243973467178875580430030822753906e-01, // 0.123457 + 1.00000000000000000000000000000000000000000000000000e+00, // 3.141593 + 1.00000000000000000000000000000000000000000000000000e+00, // 2.718282 + 9.83886862290715424883558171131880953907966613769531e-01, // 0.314159 + 9.57127388892042407775306855910457670688629150390625e-01, // 0.271828 + }; + + public static final double[] GNUR_NORM_PDF_01_01 = { // + 3.98942280401432691760987836460117250680923461914062e+00, // 0.100000 + 2.41970724519143365327522587904240936040878295898438e+00, // 0.200000 + 5.39909665131880767141581145551754161715507507324219e-01, // 0.300000 + 4.43184841193799972103306572535075247287750244140625e-02, // 0.400000 + 1.33830225764885362219058784205572010250762104988098e-03, // 0.500000 + 1.48671951473429755084029041389825920305156614631414e-05, // 0.600000 + 6.07588284982330709772167754920135784857393446145579e-08, // 0.700000 + 9.13472040836459267211266946726998684075171652807512e-11, // 0.800000 + 5.05227108353689257466156274028694701290357130241127e-14, // 0.900000 + 1.02797735716689157526658972936723696508848598985854e-17, // 1.000000 + 7.69459862670641946739490369438934977209475249356985e-22, // 1.100000 + 2.11881925350939881499454444092795876174188364829642e-26, // 1.200000 + 2.14638373566309063140936928900934922616627232600929e-31, // 1.300000 + 7.99882775700703797498824580100503753079025371215284e-37, // 1.400000 + 1.09660655938900253270916972370619189601514495480260e-42, // 1.500000 + 5.53070954984441600375694607190945207837912395256854e-49, // 1.600000 + 1.02616307279193257019879511994839957950622370045046e-55, // 1.700000 + 7.00418213431858258321746607183141912867315677743039e-63, // 1.800000 + 1.75874954259520391793557171197265931013763303279246e-70, // 1.900000 + 1.62463603677370042347322114173083210725409360385417e-78, // 2.000000 + 2.41994921591514611236561904661357402801513671875000e+00, // 0.000010 + 2.41970724761114031053921280545182526111602783203125e+00, // 0.000000 + 3.88116637682825116328899639484006911516189575195312e+00, // 0.123457 + 5.14979409827915730787384521426559029430228517673526e-201, // 3.141593 + 5.46759447998955731091850416560450583665483609253997e-149, // 2.718282 + 4.02700170728233830530484738119412213563919067382812e-01, // 0.314159 + 9.11558359420472030798521245742449536919593811035156e-01, // 0.271828 + }; + + public static final double[] P_PROBIT = { // + 0.0001, 0.001, 0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99, 0.999, 0.9999 // + }; + + public static final double[] SCIPY_NORM_PROBIT_0_1 = { // + -3.71901648545568042081299608980771154165267944335938e+00, // 0.000100 + -3.09023230616781319213259848766028881072998046875000e+00, // 0.001000 + -2.32634787404084075745913651189766824245452880859375e+00, // 0.010000 + -1.28155156554460036844034220848698168992996215820312e+00, // 0.100000 + -6.74489750196081816469018122006673365831375122070312e-01, // 0.250000 + 0.00000000000000000000000000000000000000000000000000e+00, // 0.500000 + 6.74489750196081816469018122006673365831375122070312e-01, // 0.750000 + 1.28155156554460036844034220848698168992996215820312e+00, // 0.900000 + 2.32634787404084075745913651189766824245452880859375e+00, // 0.990000 + 3.09023230616781319213259848766028881072998046875000e+00, // 0.999000 + 3.71901648545570839843321664375253021717071533203125e+00, // 0.999900 + }; + + public static final double[] GNUR_NORM_PROBIT_0_1 = { // + -3.71901648545567997672378623974509537220001220703125e+00, // 0.000100 + -3.09023230616781319213259848766028881072998046875000e+00, // 0.001000 + -2.32634787404084075745913651189766824245452880859375e+00, // 0.010000 + -1.28155156554460036844034220848698168992996215820312e+00, // 0.100000 + -6.74489750196081705446715659491019323468208312988281e-01, // 0.250000 + 0.00000000000000000000000000000000000000000000000000e+00, // 0.500000 + 6.74489750196081705446715659491019323468208312988281e-01, // 0.750000 + 1.28155156554460059048494713351828977465629577636719e+00, // 0.900000 + 2.32634787404084075745913651189766824245452880859375e+00, // 0.990000 + 3.09023230616781274804338863759767264127731323242188e+00, // 0.999000 + 3.71901648545570839843321664375253021717071533203125e+00, // 0.999900 + }; + + public static final double[] SCIPY_NORM_PROBIT_1_3 = { // + -1.01570494563670408183497784193605184555053710937500e+01, // 0.000100 + -8.27069691850343957639779546298086643218994140625000e+00, // 0.001000 + -5.97904362212252227237740953569300472736358642578125e+00, // 0.010000 + -2.84465469663380110532102662546094506978988647460938e+00, // 0.100000 + -1.02346925058824567145165929105132818222045898437500e+00, // 0.250000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.500000 + 3.02346925058824567145165929105132818222045898437500e+00, // 0.750000 + 4.84465469663380154941023647552356123924255371093750e+00, // 0.900000 + 7.97904362212252227237740953569300472736358642578125e+00, // 0.990000 + 1.02706969185034395763977954629808664321899414062500e+01, // 0.999000 + 1.21570494563671260834780696313828229904174804687500e+01, // 0.999900 + }; + + public static final double[] GNUR_NORM_PROBIT_1_3 = { // + -1.01570494563670408183497784193605184555053710937500e+01, // 0.000100 + -8.27069691850343957639779546298086643218994140625000e+00, // 0.001000 + -5.97904362212252227237740953569300472736358642578125e+00, // 0.010000 + -2.84465469663380110532102662546094506978988647460938e+00, // 0.100000 + -1.02346925058824500531784451595740392804145812988281e+00, // 0.250000 + 1.00000000000000000000000000000000000000000000000000e+00, // 0.500000 + 3.02346925058824522736244944098871201276779174804688e+00, // 0.750000 + 4.84465469663380154941023647552356123924255371093750e+00, // 0.900000 + 7.97904362212252227237740953569300472736358642578125e+00, // 0.990000 + 1.02706969185034395763977954629808664321899414062500e+01, // 0.999000 + 1.21570494563671243071212302311323583126068115234375e+01, // 0.999900 + }; + + public static final double[] SCIPY_NORM_PROBIT_01_01 = { // + -2.71901648545568086490220593987032771110534667968750e-01, // 0.000100 + -2.09023230616781324764374971891811583191156387329102e-01, // 0.001000 + -1.32634787404084075745913651189766824245452880859375e-01, // 0.010000 + -2.81551565544600479462644671002635732293128967285156e-02, // 0.100000 + 3.25510249803918211286557493622240144759416580200195e-02, // 0.250000 + 1.00000000000000005551115123125782702118158340454102e-01, // 0.500000 + 1.67448975019608203851362304703798145055770874023438e-01, // 0.750000 + 2.28155156554460059048494713351828977465629577636719e-01, // 0.900000 + 3.32634787404084086848143897441332228481769561767578e-01, // 0.990000 + 4.09023230616781363622180833772290498018264770507812e-01, // 0.999000 + 4.71901648545570817638861171872122213244438171386719e-01, // 0.999900 + }; + + public static final double[] GNUR_NORM_PROBIT_01_01 = { // + -2.71901648545568030979069362729205749928951263427734e-01, // 0.000100 + -2.09023230616781324764374971891811583191156387329102e-01, // 0.001000 + -1.32634787404084075745913651189766824245452880859375e-01, // 0.010000 + -2.81551565544600410073705631930351955816149711608887e-02, // 0.100000 + 3.25510249803918350064435571766807697713375091552734e-02, // 0.250000 + 1.00000000000000005551115123125782702118158340454102e-01, // 0.500000 + 1.67448975019608176095786689074884634464979171752930e-01, // 0.750000 + 2.28155156554460059048494713351828977465629577636719e-01, // 0.900000 + 3.32634787404084086848143897441332228481769561767578e-01, // 0.990000 + 4.09023230616781308111029602514463476836681365966797e-01, // 0.999000 + 4.71901648545570873150012403129949234426021575927734e-01, // 0.999900 + }; + + @Test + public void testPDF() { + checkPDF(new NormalDistribution(0., 1.), P_CDFPDF, SCIPY_NORM_PDF_0_1, 1e-11); + checkPDF(new NormalDistribution(1., 3.), P_CDFPDF, SCIPY_NORM_PDF_1_3, 1e-12); + checkPDF(new NormalDistribution(.1, .1), P_CDFPDF, SCIPY_NORM_PDF_01_01, 1e-11); + checkPDF(new NormalDistribution(0., 1.), P_CDFPDF, GNUR_NORM_PDF_0_1, 1e-15); + checkPDF(new NormalDistribution(1., 3.), P_CDFPDF, GNUR_NORM_PDF_1_3, 1e-15); + checkPDF(new NormalDistribution(.1, .1), P_CDFPDF, GNUR_NORM_PDF_01_01, 1e-15); + } + + @Test + public void testCDF() { + checkCDF(new NormalDistribution(0., 1.), P_CDFPDF, SCIPY_NORM_CDF_0_1, 1e-13); + checkCDF(new NormalDistribution(1., 3.), P_CDFPDF, SCIPY_NORM_CDF_1_3, 1e-12); + checkCDF(new NormalDistribution(.1, .1), P_CDFPDF, SCIPY_NORM_CDF_01_01, 1e-13); + checkCDF(new NormalDistribution(0., 1.), P_CDFPDF, GNUR_NORM_CDF_0_1, 1e-15); + checkCDF(new NormalDistribution(1., 3.), P_CDFPDF, GNUR_NORM_CDF_1_3, 1e-15); + checkCDF(new NormalDistribution(.1, .1), P_CDFPDF, GNUR_NORM_CDF_01_01, 1e-15); + } + + @Test + public void testProbit() { + // TODO: improve our implementation, to get more significant digits! + checkQuantile(new NormalDistribution(0., 1.), P_PROBIT, SCIPY_NORM_PROBIT_0_1, 1e-8); + checkQuantile(new NormalDistribution(1., 3.), P_PROBIT, SCIPY_NORM_PROBIT_1_3, 1e-9); + checkQuantile(new NormalDistribution(.1, .1), P_PROBIT, SCIPY_NORM_PROBIT_01_01, 1e-9); + checkQuantile(new NormalDistribution(0., 1.), P_PROBIT, GNUR_NORM_PROBIT_0_1, 1e-8); + checkQuantile(new NormalDistribution(1., 3.), P_PROBIT, GNUR_NORM_PROBIT_1_3, 1e-9); + checkQuantile(new NormalDistribution(.1, .1), P_PROBIT, GNUR_NORM_PROBIT_01_01, 1e-9); + } +}
\ No newline at end of file diff --git a/test/de/lmu/ifi/dbs/elki/utilities/datastructures/heap/TestHeapPerformance.java b/test/de/lmu/ifi/dbs/elki/utilities/datastructures/heap/TestHeapPerformance.java index 2f990e30..a10d3706 100644 --- a/test/de/lmu/ifi/dbs/elki/utilities/datastructures/heap/TestHeapPerformance.java +++ b/test/de/lmu/ifi/dbs/elki/utilities/datastructures/heap/TestHeapPerformance.java @@ -35,16 +35,13 @@ import java.util.Random; import org.junit.Test; -import de.lmu.ifi.dbs.elki.JUnit4Test; -import de.lmu.ifi.dbs.elki.utilities.datastructures.heap.Heap; - /** * Unit test to ensure that our heap is not significantly worse than SUN javas * regular PriorityQueue. * * @author Erich Schubert */ -public class TestHeapPerformance implements JUnit4Test { +public class TestHeapPerformance { final private int queueSize = 100000; final private int iterations = 20; |