blob: 35d6b6db22c29776515c6cbfe2b97f65f5a4063d (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
|
package de.lmu.ifi.dbs.elki.datasource.filter;
/*
This file is part of ELKI:
Environment for Developing KDD-Applications Supported by Index-Structures
Copyright (C) 2011
Ludwig-Maximilians-Universität München
Lehr- und Forschungseinheit für Datenbanksysteme
ELKI Development Team
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import java.util.BitSet;
import java.util.HashMap;
import java.util.Map;
import de.lmu.ifi.dbs.elki.data.SparseFloatVector;
/**
* Perform full TF-IDF Normalization as commonly used in text mining.
*
* Each record is first normalized using "term frequencies" to sum up to 1. Then
* it is globally normalized using the Inverse Document Frequency, so rare terms
* are weighted stronger than common terms.
*
* Restore will only undo the IDF part of the normalization!
*
* @author Erich Schubert
*/
public class TFIDFNormalization extends InverseDocumentFrequencyNormalization {
/**
* Constructor.
*/
public TFIDFNormalization() {
super();
}
@Override
protected SparseFloatVector filterSingleObject(SparseFloatVector featureVector) {
BitSet b = featureVector.getNotNullMask();
double sum = 0.0;
for(int i = b.nextSetBit(0); i >= 0; i = b.nextSetBit(i + 1)) {
sum += featureVector.doubleValue(i);
}
if(sum <= 0) {
sum = 1.0;
}
Map<Integer, Float> vals = new HashMap<Integer, Float>();
for(int i = b.nextSetBit(0); i >= 0; i = b.nextSetBit(i + 1)) {
vals.put(i, (float) (featureVector.doubleValue(i) / sum * idf.get(i).doubleValue()));
}
return new SparseFloatVector(vals, featureVector.getDimensionality());
}
}
|
