diff options
Diffstat (limited to 'src/de/lmu/ifi/dbs/elki/persistent')
27 files changed, 704 insertions, 266 deletions
diff --git a/src/de/lmu/ifi/dbs/elki/persistent/AbstractExternalizablePage.java b/src/de/lmu/ifi/dbs/elki/persistent/AbstractExternalizablePage.java index 194b594f..7e060012 100644 --- a/src/de/lmu/ifi/dbs/elki/persistent/AbstractExternalizablePage.java +++ b/src/de/lmu/ifi/dbs/elki/persistent/AbstractExternalizablePage.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.persistent; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2012 + Copyright (C) 2013 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team diff --git a/src/de/lmu/ifi/dbs/elki/persistent/AbstractPageFile.java b/src/de/lmu/ifi/dbs/elki/persistent/AbstractPageFile.java index e05c20c3..d6e750b8 100644 --- a/src/de/lmu/ifi/dbs/elki/persistent/AbstractPageFile.java +++ b/src/de/lmu/ifi/dbs/elki/persistent/AbstractPageFile.java @@ -1,10 +1,13 @@ package de.lmu.ifi.dbs.elki.persistent; +import de.lmu.ifi.dbs.elki.logging.Logging; +import de.lmu.ifi.dbs.elki.logging.statistics.Counter; + /* This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2012 + Copyright (C) 2013 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -23,7 +26,6 @@ package de.lmu.ifi.dbs.elki.persistent; along with this program. If not, see <http://www.gnu.org/licenses/>. */ - /** * Abstract base class for the page file API for both caches and true page files * (in-memory and on-disk). @@ -36,23 +38,31 @@ public abstract class AbstractPageFile<P extends Page> implements PageFile<P> { /** * The read I/O-Access of this file. */ - protected long readAccess; + private Counter readAccess; /** * The write I/O-Access of this file. */ - protected long writeAccess; + private Counter writeAccess; /** * Constructor. */ public AbstractPageFile() { super(); - this.readAccess = 0; - this.writeAccess = 0; + Logging log = getLogger(); + this.readAccess = log.isStatistics() ? log.newCounter(this.getClass().getName() + ".reads") : null; + this.writeAccess = log.isStatistics() ? log.newCounter(this.getClass().getName() + ".writes") : null; } /** + * Get the class logger. + * + * @return Logger + */ + abstract protected Logging getLogger(); + + /** * Writes a page into this file. The method tests if the page has already an * id, otherwise a new id is assigned and returned. * @@ -78,20 +88,32 @@ public abstract class AbstractPageFile<P extends Page> implements PageFile<P> { public void close() { clear(); } - + @Override - public final long getReadOperations() { - return readAccess; + public void logStatistics() { + if (readAccess != null) { + getLogger().statistics(readAccess); + } + if (writeAccess != null) { + getLogger().statistics(writeAccess); + } } - - @Override - public final long getWriteOperations() { - return writeAccess; + + /** + * Count a page read access. + */ + protected void countRead() { + if (readAccess != null) { + readAccess.increment(); + } } - @Override - public final void resetPageAccess() { - this.readAccess = 0; - this.writeAccess = 0; + /** + * Count a page write access. + */ + protected void countWrite() { + if (writeAccess != null) { + writeAccess.increment(); + } } -}
\ No newline at end of file +} diff --git a/src/de/lmu/ifi/dbs/elki/persistent/AbstractPageFileFactory.java b/src/de/lmu/ifi/dbs/elki/persistent/AbstractPageFileFactory.java new file mode 100644 index 00000000..16094f90 --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/persistent/AbstractPageFileFactory.java @@ -0,0 +1,100 @@ +package de.lmu.ifi.dbs.elki.persistent; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2013 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.GreaterConstraint; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.IntParameter; + +/** + * Abstract page file factory. + * + * @author Erich Schubert + * + * @param <P> Page type + */ +public abstract class AbstractPageFileFactory<P extends Page> implements PageFileFactory<P> { + /** + * Holds the value of {@link Parameterizer#PAGE_SIZE_ID}. + */ + protected int pageSize; + + /** + * Constructor. + * + * @param pageSize Page size + */ + public AbstractPageFileFactory(int pageSize) { + super(); + this.pageSize = pageSize; + } + + @Override + public int getPageSize() { + return pageSize; + } + + /** + * Parameterization class. + * + * @apiviz.exclude + * + * @author Erich Schubert + * + * @param <P> Page type + */ + public static abstract class Parameterizer<P extends Page> extends AbstractParameterizer { + /** + * Parameter to specify the size of a page in bytes, must be an integer + * greater than 0. + * <p> + * Default value: {@code 4000} + * </p> + * <p> + * Key: {@code -pagefile.pagesize} + * </p> + */ + public static final OptionID PAGE_SIZE_ID = new OptionID("pagefile.pagesize", "The size of a page in bytes."); + + /** + * Page size + */ + protected int pageSize; + + @Override + protected void makeOptions(Parameterization config) { + super.makeOptions(config); + final IntParameter pageSizeP = new IntParameter(PAGE_SIZE_ID, 4000); + pageSizeP.addConstraint(new GreaterConstraint(0)); + if (config.grab(pageSizeP)) { + pageSize = pageSizeP.getValue(); + } + } + + @Override + abstract protected PageFileFactory<P> makeInstance(); + } +} diff --git a/src/de/lmu/ifi/dbs/elki/persistent/AbstractStoringPageFile.java b/src/de/lmu/ifi/dbs/elki/persistent/AbstractStoringPageFile.java index 7151e6dd..6ac6eda9 100644 --- a/src/de/lmu/ifi/dbs/elki/persistent/AbstractStoringPageFile.java +++ b/src/de/lmu/ifi/dbs/elki/persistent/AbstractStoringPageFile.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.persistent; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2012 + Copyright (C) 2013 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -25,15 +25,14 @@ package de.lmu.ifi.dbs.elki.persistent; import java.util.Stack; +import de.lmu.ifi.dbs.elki.logging.statistics.LongStatistic; + /** * Abstract class implementing general methods of a PageFile. A PageFile stores * objects that implement the <code>Page</code> interface. * * @author Elke Achtert * - * @apiviz.has Page - * @apiviz.has PageFileStatistics - * * @param <P> Page type */ public abstract class AbstractStoringPageFile<P extends Page> extends AbstractPageFile<P> { @@ -56,7 +55,7 @@ public abstract class AbstractStoringPageFile<P extends Page> extends AbstractPa * Creates a new PageFile. */ protected AbstractStoringPageFile(int pageSize) { - this.emptyPages = new Stack<Integer>(); + this.emptyPages = new Stack<>(); this.nextPageID = 0; this.pageSize = pageSize; } @@ -148,8 +147,10 @@ public abstract class AbstractStoringPageFile<P extends Page> extends AbstractPa } @Override - public PageFileStatistics getInnerStatistics() { - // Default: no nested page file. - return null; + public void logStatistics() { + super.logStatistics(); + if (getLogger().isStatistics()) { + getLogger().statistics(new LongStatistic(this.getClass().getName() + ".numpages", nextPageID - emptyPages.size())); + } } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/persistent/ByteArrayUtil.java b/src/de/lmu/ifi/dbs/elki/persistent/ByteArrayUtil.java index ba9d108e..ba25435f 100644 --- a/src/de/lmu/ifi/dbs/elki/persistent/ByteArrayUtil.java +++ b/src/de/lmu/ifi/dbs/elki/persistent/ByteArrayUtil.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.persistent; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2012 + Copyright (C) 2013 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team diff --git a/src/de/lmu/ifi/dbs/elki/persistent/ByteBufferInputStream.java b/src/de/lmu/ifi/dbs/elki/persistent/ByteBufferInputStream.java index 586d5535..dbc5ed2d 100644 --- a/src/de/lmu/ifi/dbs/elki/persistent/ByteBufferInputStream.java +++ b/src/de/lmu/ifi/dbs/elki/persistent/ByteBufferInputStream.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.persistent; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2012 + Copyright (C) 2013 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team diff --git a/src/de/lmu/ifi/dbs/elki/persistent/ByteBufferOutputStream.java b/src/de/lmu/ifi/dbs/elki/persistent/ByteBufferOutputStream.java index 2a2971d9..28137393 100644 --- a/src/de/lmu/ifi/dbs/elki/persistent/ByteBufferOutputStream.java +++ b/src/de/lmu/ifi/dbs/elki/persistent/ByteBufferOutputStream.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.persistent; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2012 + Copyright (C) 2013 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team diff --git a/src/de/lmu/ifi/dbs/elki/persistent/ByteBufferSerializer.java b/src/de/lmu/ifi/dbs/elki/persistent/ByteBufferSerializer.java index 93f99ba9..8605dd07 100644 --- a/src/de/lmu/ifi/dbs/elki/persistent/ByteBufferSerializer.java +++ b/src/de/lmu/ifi/dbs/elki/persistent/ByteBufferSerializer.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.persistent; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2012 + Copyright (C) 2013 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -31,7 +31,8 @@ import java.nio.ByteBuffer; * * @author Erich Schubert * - * @apiviz.uses ByteBuffer + * @apiviz.has ByteBuffer - - oneway «serializes to/from» + * @apiviz.excludeSubtypes * * @param <T> Object type processed */ diff --git a/src/de/lmu/ifi/dbs/elki/persistent/DefaultPageHeader.java b/src/de/lmu/ifi/dbs/elki/persistent/DefaultPageHeader.java index e1ca2a41..5ae3312a 100644 --- a/src/de/lmu/ifi/dbs/elki/persistent/DefaultPageHeader.java +++ b/src/de/lmu/ifi/dbs/elki/persistent/DefaultPageHeader.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.persistent; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2012 + Copyright (C) 2013 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team diff --git a/src/de/lmu/ifi/dbs/elki/persistent/ExternalizablePage.java b/src/de/lmu/ifi/dbs/elki/persistent/ExternalizablePage.java index 703187ac..0ec0f7fb 100644 --- a/src/de/lmu/ifi/dbs/elki/persistent/ExternalizablePage.java +++ b/src/de/lmu/ifi/dbs/elki/persistent/ExternalizablePage.java @@ -6,7 +6,7 @@ import java.io.Externalizable; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2012 + Copyright (C) 2013 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team diff --git a/src/de/lmu/ifi/dbs/elki/persistent/FixedSizeByteBufferSerializer.java b/src/de/lmu/ifi/dbs/elki/persistent/FixedSizeByteBufferSerializer.java index 56f1f9b3..a1d8600c 100644 --- a/src/de/lmu/ifi/dbs/elki/persistent/FixedSizeByteBufferSerializer.java +++ b/src/de/lmu/ifi/dbs/elki/persistent/FixedSizeByteBufferSerializer.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.persistent; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2012 + Copyright (C) 2013 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -28,6 +28,7 @@ package de.lmu.ifi.dbs.elki.persistent; * Serializers with a fixed length serialization. * * @author Erich Schubert + * @apiviz.excludeSubtypes * * @param <T> Type */ diff --git a/src/de/lmu/ifi/dbs/elki/persistent/LRUCache.java b/src/de/lmu/ifi/dbs/elki/persistent/LRUCache.java index 7e06e5ea..18726ec7 100644 --- a/src/de/lmu/ifi/dbs/elki/persistent/LRUCache.java +++ b/src/de/lmu/ifi/dbs/elki/persistent/LRUCache.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.persistent; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2012 + Copyright (C) 2013 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -40,25 +40,25 @@ import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; * * @author Elke Achtert * - * @apiviz.composedOf PageFile + * @apiviz.uses PageFile * * @param <P> Page type */ public class LRUCache<P extends Page> extends AbstractPageFile<P> { /** - * Our logger + * Our class logger. */ private static final Logging LOG = Logging.getLogger(LRUCache.class); /** * Cache size in bytes. */ - protected long cacheSizeBytes; + protected int cacheSizeBytes; /** * The maximum number of objects in this cache. */ - protected long cacheSize; + protected int cacheSize; /** * The map holding the objects of this cache. @@ -74,11 +74,11 @@ public class LRUCache<P extends Page> extends AbstractPageFile<P> { /** * Initializes this cache with the specified parameters. * - * @param cacheSizeBytes the maximum number of pages in this cache + * @param cacheSizeBytes the maximum number of bytes for this cache * @param file the underlying file of this cache, if a page is dropped it is * written to the file */ - public LRUCache(long cacheSizeBytes, PageFile<P> file) { + public LRUCache(int cacheSizeBytes, PageFile<P> file) { this.file = file; this.cacheSizeBytes = cacheSizeBytes; } @@ -93,7 +93,7 @@ public class LRUCache<P extends Page> extends AbstractPageFile<P> { */ @Override public synchronized P readPage(int pageID) { - readAccess++; + countRead(); P page = map.get(pageID); if(page != null) { if(LOG.isDebuggingFine()) { @@ -112,7 +112,7 @@ public class LRUCache<P extends Page> extends AbstractPageFile<P> { @Override public synchronized void writePage(int pageID, P page) { - writeAccess++; + countWrite(); page.setDirty(true); map.put(pageID, page); if(LOG.isDebuggingFine()) { @@ -122,7 +122,7 @@ public class LRUCache<P extends Page> extends AbstractPageFile<P> { @Override public void deletePage(int pageID) { - writeAccess++; + countWrite(); map.remove(pageID); file.deletePage(pageID); } @@ -241,7 +241,7 @@ public class LRUCache<P extends Page> extends AbstractPageFile<P> { return; } - List<Integer> keys = new ArrayList<Integer>(map.keySet()); + List<Integer> keys = new ArrayList<>(map.keySet()); Collections.reverse(keys); for(Integer id : keys) { @@ -251,7 +251,13 @@ public class LRUCache<P extends Page> extends AbstractPageFile<P> { } @Override - public PageFileStatistics getInnerStatistics() { - return file; + public void logStatistics() { + super.logStatistics(); + file.logStatistics(); + } + + @Override + protected Logging getLogger() { + return LOG; } } diff --git a/src/de/lmu/ifi/dbs/elki/persistent/LRUCachePageFileFactory.java b/src/de/lmu/ifi/dbs/elki/persistent/LRUCachePageFileFactory.java new file mode 100644 index 00000000..a87f8e1f --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/persistent/LRUCachePageFileFactory.java @@ -0,0 +1,135 @@ +package de.lmu.ifi.dbs.elki.persistent; + +import de.lmu.ifi.dbs.elki.utilities.optionhandling.AbstractParameterizer; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.constraints.GreaterEqualConstraint; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.IntParameter; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.ObjectParameter; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2013 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +/** + * Page file factory for memory page files. + * + * @author Erich Schubert + * + * @apiviz.has LRUCache + * @apiviz.composedOf PageFileFactory + * + * @param <P> Page type + */ +public class LRUCachePageFileFactory<P extends Page> implements PageFileFactory<P> { + /** + * Inner page file factory. + */ + private PageFileFactory<P> pageFileFactory; + + /** + * Cache size, in bytes. + */ + private int cacheSize; + + /** + * Constructor. + * + * @param pageFileFactory Inner page file + * @param cacheSize Size of cache, in bytes. + */ + public LRUCachePageFileFactory(PageFileFactory<P> pageFileFactory, int cacheSize) { + super(); + this.cacheSize = cacheSize; + this.pageFileFactory = pageFileFactory; + } + + @Override + public PageFile<P> newPageFile(Class<P> cls) { + PageFile<P> inner = pageFileFactory.newPageFile(cls); + return new LRUCache<>(cacheSize, inner); + } + + @Override + public int getPageSize() { + return pageFileFactory.getPageSize(); + } + + /** + * Parameterization class. + * + * @author Erich Schubert + * + * @apiviz.exclude + */ + public static class Parameterizer extends AbstractParameterizer { + /** + * Parameter to specify the size of the cache in bytes, must be an integer + * equal to or greater than 0. + * <p> + * Default value: {@link Integer#MAX_VALUE} + * </p> + * <p> + * Key: {@code -pagefile.cachesize} + * </p> + */ + public static final OptionID CACHE_SIZE_ID = new OptionID("pagefile.cachesize", "The size of the cache in bytes."); + + /** + * Parameter to specify the inner pagefile. + * <p> + * Key: {@code -pagefile.pagefile} + * </p> + */ + public static final OptionID PAGEFILE_ID = new OptionID("pagefile.pagefile", "The backing pagefile for the cache."); + + /** + * Inner page file factory. + */ + PageFileFactory<Page> pageFileFactory; + + /** + * Cache size, in bytes. + */ + protected int cacheSize; + + @Override + protected void makeOptions(Parameterization config) { + super.makeOptions(config); + ObjectParameter<PageFileFactory<Page>> pffP = new ObjectParameter<>(PAGEFILE_ID, PageFileFactory.class, PersistentPageFileFactory.class); + if (config.grab(pffP)) { + pageFileFactory = pffP.instantiateClass(config); + } + + IntParameter cacheSizeP = new IntParameter(CACHE_SIZE_ID); + cacheSizeP.addConstraint(new GreaterEqualConstraint(0)); + if (config.grab(cacheSizeP)) { + cacheSize = cacheSizeP.getValue(); + } + } + + @Override + protected LRUCachePageFileFactory<Page> makeInstance() { + return new LRUCachePageFileFactory<>(pageFileFactory, cacheSize); + } + } +} diff --git a/src/de/lmu/ifi/dbs/elki/persistent/MemoryPageFile.java b/src/de/lmu/ifi/dbs/elki/persistent/MemoryPageFile.java index 7aba9a21..46d10fb4 100644 --- a/src/de/lmu/ifi/dbs/elki/persistent/MemoryPageFile.java +++ b/src/de/lmu/ifi/dbs/elki/persistent/MemoryPageFile.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.persistent; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2012 + Copyright (C) 2013 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -25,6 +25,7 @@ package de.lmu.ifi.dbs.elki.persistent; import gnu.trove.map.TIntObjectMap; import gnu.trove.map.hash.TIntObjectHashMap; +import de.lmu.ifi.dbs.elki.logging.Logging; /** * A memory based implementation of a PageFile that simulates I/O-access.<br> @@ -36,6 +37,11 @@ import gnu.trove.map.hash.TIntObjectHashMap; */ public class MemoryPageFile<P extends Page> extends AbstractStoringPageFile<P> { /** + * Class logger. + */ + private static final Logging LOG = Logging.getLogger(MemoryPageFile.class); + + /** * Holds the pages. */ private final TIntObjectMap<P> file; @@ -48,18 +54,18 @@ public class MemoryPageFile<P extends Page> extends AbstractStoringPageFile<P> { */ public MemoryPageFile(int pageSize) { super(pageSize); - this.file = new TIntObjectHashMap<P>(); + this.file = new TIntObjectHashMap<>(); } @Override public synchronized P readPage(int pageID) { - readAccess++; + countRead(); return file.get(pageID); } - + @Override protected void writePage(int pageID, P page) { - writeAccess++; + countWrite(); file.put(pageID, page); page.setDirty(false); } @@ -71,7 +77,7 @@ public class MemoryPageFile<P extends Page> extends AbstractStoringPageFile<P> { super.deletePage(pageID); // delete from file - writeAccess++; + countWrite(); file.remove(pageID); } @@ -79,4 +85,9 @@ public class MemoryPageFile<P extends Page> extends AbstractStoringPageFile<P> { public void clear() { file.clear(); } -}
\ No newline at end of file + + @Override + protected Logging getLogger() { + return LOG; + } +} diff --git a/src/de/lmu/ifi/dbs/elki/persistent/PageFileStatistics.java b/src/de/lmu/ifi/dbs/elki/persistent/MemoryPageFileFactory.java index df3de612..812e65c9 100644 --- a/src/de/lmu/ifi/dbs/elki/persistent/PageFileStatistics.java +++ b/src/de/lmu/ifi/dbs/elki/persistent/MemoryPageFileFactory.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.persistent; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2012 + Copyright (C) 2013 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -23,39 +23,41 @@ package de.lmu.ifi.dbs.elki.persistent; along with this program. If not, see <http://www.gnu.org/licenses/>. */ - /** - * Statistics API for a Page File. - * - * See {@link PageFileUtil} for related utility functions for analysing this - * data! + * Page file factory for memory page files. * * @author Erich Schubert + * + * @apiviz.has MemoryPageFile + * + * @param <P> Page type */ -public interface PageFileStatistics { +public class MemoryPageFileFactory<P extends Page> extends AbstractPageFileFactory<P> { /** - * Returns the read I/O-Accesses of this file. + * Constructor. * - * @return Number of physical read I/O accesses + * @param pageSize Size of a page */ - public long getReadOperations(); - + public MemoryPageFileFactory(int pageSize) { + super(pageSize); + } + + @Override + public PageFile<P> newPageFile(Class<P> cls) { + return new MemoryPageFile<>(pageSize); + } + /** - * Returns the write I/O-Accesses of this file. + * Parameterization class. * - * @return Number of physical write I/O accesses - */ - public long getWriteOperations(); - - /** - * Resets the counters for page accesses of this file and flushes the cache. - */ - public void resetPageAccess(); - - /** - * Get statistics for the inner page file, if present. + * @author Erich Schubert * - * @return Inner page file statistics, or null. + * @apiviz.exclude */ - public PageFileStatistics getInnerStatistics(); + public static class Parameterizer extends AbstractPageFileFactory.Parameterizer<Page> { + @Override + protected MemoryPageFileFactory<Page> makeInstance() { + return new MemoryPageFileFactory<>(pageSize); + } + } } diff --git a/src/de/lmu/ifi/dbs/elki/persistent/OnDiskArray.java b/src/de/lmu/ifi/dbs/elki/persistent/OnDiskArray.java index a37dcbdb..3f0965e2 100644 --- a/src/de/lmu/ifi/dbs/elki/persistent/OnDiskArray.java +++ b/src/de/lmu/ifi/dbs/elki/persistent/OnDiskArray.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.persistent; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2012 + Copyright (C) 2013 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -26,7 +26,6 @@ package de.lmu.ifi.dbs.elki.persistent; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; -import java.io.Serializable; import java.nio.ByteBuffer; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel.MapMode; @@ -45,11 +44,9 @@ import de.lmu.ifi.dbs.elki.utilities.exceptions.ExceptionMessages; * * @apiviz.composedOf RandomAccessFile */ -// TODO: make serializable - by just keeping the required information, restoring -// the other. // TODO: ensure file doesn't become to big - check for overflows in recordsize * // numrecs + headersize -public class OnDiskArray implements Serializable { +public class OnDiskArray { /** * Serial version. * @@ -60,7 +57,7 @@ public class OnDiskArray implements Serializable { private static final long serialVersionUID = 7586497243452875056L; /** - * Magic number used to identify files + * Magic number used to identify files. */ protected int magic; @@ -96,10 +93,13 @@ public class OnDiskArray implements Serializable { private FileLock lock = null; /** - * Writable or read-only object + * Writable or read-only object. */ private boolean writable; + /** + * The memory mapped buffer. + */ private MappedByteBuffer map; /** @@ -108,7 +108,7 @@ public class OnDiskArray implements Serializable { private static final int INTERNAL_HEADER_SIZE = 4 * ByteArrayUtil.SIZE_INT; /** - * Position of file size (in records) + * Position of file size (in records). */ private static final int HEADER_POS_SIZE = 3 * ByteArrayUtil.SIZE_INT; @@ -130,7 +130,7 @@ public class OnDiskArray implements Serializable { this.writable = true; // do not allow overwriting, unless empty - if(filename.exists() && filename.length() > 0) { + if (filename.exists() && filename.length() > 0) { throw new IOException(ExceptionMessages.FILE_EXISTS); } @@ -150,14 +150,14 @@ public class OnDiskArray implements Serializable { // write number of records // verify position. - if(file.getFilePointer() != HEADER_POS_SIZE) { + if (file.getFilePointer() != HEADER_POS_SIZE) { // TODO: more appropriate exception class? throw new IOException("File position doesn't match when writing file size."); } file.writeInt(initialsize); // we should have written the complete internal header now. - if(file.getFilePointer() != INTERNAL_HEADER_SIZE) { + if (file.getFilePointer() != INTERNAL_HEADER_SIZE) { // TODO: more appropriate exception class? throw new IOException("File position doesn't match header size after writing header."); } @@ -191,7 +191,7 @@ public class OnDiskArray implements Serializable { String mode = writable ? "rw" : "r"; file = new RandomAccessFile(filename, mode); - if(writable) { + if (writable) { // acquire a file write lock lock = file.getChannel().lock(); } @@ -219,7 +219,7 @@ public class OnDiskArray implements Serializable { String mode = writable ? "rw" : "r"; file = new RandomAccessFile(filename, mode); - if(writable) { + if (writable) { // acquire a file write lock lock = file.getChannel().lock(); } @@ -254,39 +254,38 @@ public class OnDiskArray implements Serializable { private void validateHeader(boolean validateRecordSize) throws IOException { int readmagic = file.readInt(); // Validate magic number - if(readmagic != this.magic) { + if (readmagic != this.magic) { file.close(); throw new IOException("Magic in LinearDiskCache does not match: " + readmagic + " instead of " + this.magic); } // Validate header size - if(file.readInt() != this.headersize) { + if (file.readInt() != this.headersize) { file.close(); throw new IOException("Header size in LinearDiskCache does not match."); } - if(validateRecordSize) { + if (validateRecordSize) { // Validate record size - if(file.readInt() != this.recordsize) { + if (file.readInt() != this.recordsize) { file.close(); throw new IOException("Recordsize in LinearDiskCache does not match."); } - } - else { + } else { // or just read it from file this.recordsize = file.readInt(); } // read the number of records and validate with file size. - if(file.getFilePointer() != HEADER_POS_SIZE) { + if (file.getFilePointer() != HEADER_POS_SIZE) { throw new IOException("Incorrect file position when reading header."); } this.numrecs = file.readInt(); - if(numrecs < 0 || file.length() != indexToFileposition(numrecs)) { + if (numrecs < 0 || file.length() != indexToFileposition(numrecs)) { throw new IOException("File size and number of records do not agree."); } // yet another sanity check. We should have read all of our internal header // now. - if(file.getFilePointer() != INTERNAL_HEADER_SIZE) { + if (file.getFilePointer() != INTERNAL_HEADER_SIZE) { throw new IOException("Incorrect file position after reading header."); } } @@ -325,7 +324,7 @@ public class OnDiskArray implements Serializable { * @throws IOException on IO errors */ public synchronized void resizeFile(int newsize) throws IOException { - if(!writable) { + if (!writable) { throw new IOException("File is not writeable!"); } // update the number of records @@ -346,11 +345,11 @@ public class OnDiskArray implements Serializable { * @throws IOException on IO errors */ public synchronized ByteBuffer getRecordBuffer(int index) throws IOException { - if(index < 0 || index >= numrecs) { + if (index < 0 || index >= numrecs) { throw new IOException("Access beyond end of file."); } // Adjust buffer view - synchronized(map) { + synchronized (map) { map.limit(recordsize * (index + 1)); map.position(recordsize * index); return map.slice(); @@ -413,7 +412,7 @@ public class OnDiskArray implements Serializable { */ public synchronized void close() throws IOException { writable = false; - if(lock != null) { + if (lock != null) { lock.release(); lock = null; } @@ -436,8 +435,8 @@ public class OnDiskArray implements Serializable { * @throws IOException */ public void ensureSize(int size) throws IOException { - if(size > getNumRecords()) { + if (size > getNumRecords()) { resizeFile(size); } } -}
\ No newline at end of file +} diff --git a/src/de/lmu/ifi/dbs/elki/persistent/OnDiskArrayPageFile.java b/src/de/lmu/ifi/dbs/elki/persistent/OnDiskArrayPageFile.java index 1f8c2a1a..061d4ce9 100644 --- a/src/de/lmu/ifi/dbs/elki/persistent/OnDiskArrayPageFile.java +++ b/src/de/lmu/ifi/dbs/elki/persistent/OnDiskArrayPageFile.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.persistent; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2012 + Copyright (C) 2013 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -32,6 +32,7 @@ import java.io.ObjectOutputStream; import java.nio.ByteBuffer; import java.util.logging.Level; +import de.lmu.ifi.dbs.elki.logging.Logging; import de.lmu.ifi.dbs.elki.logging.LoggingUtil; /** @@ -48,6 +49,11 @@ import de.lmu.ifi.dbs.elki.logging.LoggingUtil; */ public class OnDiskArrayPageFile<P extends Page> extends AbstractStoringPageFile<P> { /** + * Class logger. + */ + private static final Logging LOG = Logging.getLogger(OnDiskArrayPageFile.class); + + /** * Indicates an empty page. */ private static final int EMPTY_PAGE = 0; @@ -102,10 +108,9 @@ public class OnDiskArrayPageFile<P extends Page> extends AbstractStoringPageFile @Override public P readPage(int pageID) { try { - readAccess++; + countRead(); return byteBufferToPage(this.file.getRecordBuffer(pageID)); - } - catch(IOException e) { + } catch (IOException e) { throw new RuntimeException("IOException occurred during reading of page " + pageID, e); } } @@ -123,11 +128,10 @@ public class OnDiskArrayPageFile<P extends Page> extends AbstractStoringPageFile super.deletePage(pageID); // delete from file - writeAccess++; + countWrite(); byte[] array = pageToByteArray(null); file.getRecordBuffer(pageID).put(array); - } - catch(IOException e) { + } catch (IOException e) { throw new RuntimeException(e); } } @@ -140,14 +144,13 @@ public class OnDiskArrayPageFile<P extends Page> extends AbstractStoringPageFile */ @Override public void writePage(int pageID, P page) { - if(page.isDirty()) { + if (page.isDirty()) { try { - writeAccess++; + countWrite(); byte[] array = pageToByteArray(page); file.getRecordBuffer(pageID).put(array); page.setDirty(false); - } - catch(IOException e) { + } catch (IOException e) { throw new RuntimeException(e); } } @@ -161,8 +164,7 @@ public class OnDiskArrayPageFile<P extends Page> extends AbstractStoringPageFile try { super.close(); file.close(); - } - catch(IOException e) { + } catch (IOException e) { throw new RuntimeException(e); } } @@ -174,8 +176,7 @@ public class OnDiskArrayPageFile<P extends Page> extends AbstractStoringPageFile public void clear() { try { file.resizeFile(0); - } - catch(IOException e) { + } catch (IOException e) { throw new RuntimeException(e); } } @@ -188,25 +189,20 @@ public class OnDiskArrayPageFile<P extends Page> extends AbstractStoringPageFile */ @SuppressWarnings("unchecked") private P byteBufferToPage(ByteBuffer buffer) { - try { - InputStream bais = new ByteBufferInputStream(buffer); - ObjectInputStream ois = new ObjectInputStream(bais); + try (InputStream bais = new ByteBufferInputStream(buffer); + ObjectInputStream ois = new ObjectInputStream(bais)) { int type = ois.readInt(); - if(type == EMPTY_PAGE) { + if (type == EMPTY_PAGE) { return null; - } - else if(type == FILLED_PAGE) { + } else if (type == FILLED_PAGE) { return (P) ois.readObject(); - } - else { + } else { throw new IllegalArgumentException("Unknown type: " + type); } - } - catch(IOException e) { + } catch (IOException e) { LoggingUtil.exception(e); return null; - } - catch(ClassNotFoundException e) { + } catch (ClassNotFoundException e) { LoggingUtil.exception(e); return null; } @@ -220,7 +216,7 @@ public class OnDiskArrayPageFile<P extends Page> extends AbstractStoringPageFile */ private byte[] pageToByteArray(P page) { try { - if(page == null) { + if (page == null) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); ObjectOutputStream oos = new ObjectOutputStream(baos); oos.writeInt(EMPTY_PAGE); @@ -230,8 +226,7 @@ public class OnDiskArrayPageFile<P extends Page> extends AbstractStoringPageFile byte[] result = new byte[pageSize]; System.arraycopy(array, 0, result, 0, array.length); return result; - } - else { + } else { ByteArrayOutputStream baos = new ByteArrayOutputStream(); ObjectOutputStream oos = new ObjectOutputStream(baos); oos.writeInt(FILLED_PAGE); @@ -239,10 +234,9 @@ public class OnDiskArrayPageFile<P extends Page> extends AbstractStoringPageFile oos.close(); baos.close(); byte[] array = baos.toByteArray(); - if(array.length > this.pageSize) { + if (array.length > this.pageSize) { throw new IllegalArgumentException("Size of page " + page + " is greater than specified" + " pagesize: " + array.length + " > " + pageSize); - } - else if(array.length == this.pageSize) { + } else if (array.length == this.pageSize) { return array; } @@ -252,8 +246,7 @@ public class OnDiskArrayPageFile<P extends Page> extends AbstractStoringPageFile return result; } } - } - catch(IOException e) { + } catch (IOException e) { throw new RuntimeException("IOException occurred! ", e); } } @@ -262,7 +255,7 @@ public class OnDiskArrayPageFile<P extends Page> extends AbstractStoringPageFile public boolean initialize(PageHeader header) { this.header = header; try { - if(existed) { + if (existed) { LoggingUtil.logExpensive(Level.INFO, "Create from existing file."); this.file = new OnDiskArray(filename, 0, header.size(), pageSize, true); @@ -275,17 +268,15 @@ public class OnDiskArrayPageFile<P extends Page> extends AbstractStoringPageFile } // reading empty nodes in Stack - for(int i = 0; i < file.getNumRecords(); i++) { + for (int i = 0; i < file.getNumRecords(); i++) { ByteBuffer buffer = file.getRecordBuffer(i); int type = buffer.getInt(); - if(type == EMPTY_PAGE) { + if (type == EMPTY_PAGE) { emptyPages.push(i); - } - else if(type == FILLED_PAGE) { + } else if (type == FILLED_PAGE) { nextPageID = i + 1; - } - else { + } else { throw new IllegalArgumentException("Unknown type: " + type); } i++; @@ -304,9 +295,13 @@ public class OnDiskArrayPageFile<P extends Page> extends AbstractStoringPageFile buffer.put(header.asByteArray()); return false; } - } - catch(IOException e) { + } catch (IOException e) { throw new RuntimeException("IOException occurred.", e); } } + + @Override + protected Logging getLogger() { + return LOG; + } } diff --git a/src/de/lmu/ifi/dbs/elki/persistent/OnDiskArrayPageFileFactory.java b/src/de/lmu/ifi/dbs/elki/persistent/OnDiskArrayPageFileFactory.java new file mode 100644 index 00000000..4ddfdf4d --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/persistent/OnDiskArrayPageFileFactory.java @@ -0,0 +1,101 @@ +package de.lmu.ifi.dbs.elki.persistent; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2013 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.FileParameter; + +/** + * Page file factory for disk-based page files. + * + * @author Erich Schubert + * + * @apiviz.has OnDiskArrayPageFile + * + * @param <P> Page type + */ +public class OnDiskArrayPageFileFactory<P extends ExternalizablePage> extends AbstractPageFileFactory<P> { + /** + * File name. + */ + private String fileName; + + /** + * Constructor. + * + * @param pageSize Page size + */ + public OnDiskArrayPageFileFactory(int pageSize, String fileName) { + super(pageSize); + this.fileName = fileName; + } + + @Override + public PageFile<P> newPageFile(Class<P> cls) { + if (fileName == null) { + throw new AbortException("Disk-backed page file may only be instantiated once!"); + } + OnDiskArrayPageFile<P> pfile = new OnDiskArrayPageFile<>(pageSize, fileName); + fileName = null; // To avoid double instantiation. + return pfile; + } + + /** + * Parameterization class. + * + * @apiviz.exclude + * + * @author Erich Schubert + */ + public static class Parameterizer extends AbstractPageFileFactory.Parameterizer<ExternalizablePage> { + /** + * File name. + */ + private String fileName; + + /** + * Optional parameter that specifies the name of the file storing the index. + * <p> + * Key: {@code -pagefile.file} + * </p> + */ + public static final OptionID FILE_ID = new OptionID("pagefile.file", "The name of the file storing the page file."); + + @Override + protected void makeOptions(Parameterization config) { + super.makeOptions(config); + FileParameter fileNameP = new FileParameter(FILE_ID, FileParameter.FileType.OUTPUT_FILE); + if (config.grab(fileNameP)) { + fileName = fileNameP.getValue().getPath(); + } + } + + @Override + protected OnDiskArrayPageFileFactory<ExternalizablePage> makeInstance() { + return new OnDiskArrayPageFileFactory<>(pageSize, fileName); + } + } +} diff --git a/src/de/lmu/ifi/dbs/elki/persistent/OnDiskUpperTriangleMatrix.java b/src/de/lmu/ifi/dbs/elki/persistent/OnDiskUpperTriangleMatrix.java index 044657c5..6a6cad01 100644 --- a/src/de/lmu/ifi/dbs/elki/persistent/OnDiskUpperTriangleMatrix.java +++ b/src/de/lmu/ifi/dbs/elki/persistent/OnDiskUpperTriangleMatrix.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.persistent; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2012 + Copyright (C) 2013 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team diff --git a/src/de/lmu/ifi/dbs/elki/persistent/Page.java b/src/de/lmu/ifi/dbs/elki/persistent/Page.java index 2c3fb252..9be6141c 100644 --- a/src/de/lmu/ifi/dbs/elki/persistent/Page.java +++ b/src/de/lmu/ifi/dbs/elki/persistent/Page.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.persistent; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2012 + Copyright (C) 2013 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -28,6 +28,8 @@ package de.lmu.ifi.dbs.elki.persistent; * persistently saved. * * @author Elke Achtert + * + * @apiviz.excludeSubtypes */ public interface Page { /** diff --git a/src/de/lmu/ifi/dbs/elki/persistent/PageFile.java b/src/de/lmu/ifi/dbs/elki/persistent/PageFile.java index f50e829e..e3978a51 100644 --- a/src/de/lmu/ifi/dbs/elki/persistent/PageFile.java +++ b/src/de/lmu/ifi/dbs/elki/persistent/PageFile.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.persistent; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2012 + Copyright (C) 2013 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -23,15 +23,16 @@ package de.lmu.ifi.dbs.elki.persistent; along with this program. If not, see <http://www.gnu.org/licenses/>. */ - /** * Page file interface. * * @author Erich Schubert * + * @apiviz.has Page + * * @param <P> Page file */ -public interface PageFile<P extends Page> extends PageFileStatistics { +public interface PageFile<P extends Page> { /** * Sets the id of the given page. * @@ -103,4 +104,9 @@ public interface PageFile<P extends Page> extends PageFileStatistics { * @return true when the file already existed. */ public boolean initialize(PageHeader header); -}
\ No newline at end of file + + /** + * Log some statistics to the appropriate logger. + */ + public void logStatistics(); +} diff --git a/src/de/lmu/ifi/dbs/elki/persistent/PageFileFactory.java b/src/de/lmu/ifi/dbs/elki/persistent/PageFileFactory.java new file mode 100644 index 00000000..3c7fa86d --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/persistent/PageFileFactory.java @@ -0,0 +1,48 @@ +package de.lmu.ifi.dbs.elki.persistent; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2013 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +/** + * Factory interface for generating page files. + * + * @author Erich Schubert + * + * @param <P> Page type + */ +public interface PageFileFactory<P extends Page> { + /** + * Make a new page file. + * + * @param cls Page class + * @return Page file + */ + PageFile<P> newPageFile(Class<P> cls); + + /** + * Query the page size. + * + * @return page size + */ + int getPageSize(); +} diff --git a/src/de/lmu/ifi/dbs/elki/persistent/PageFileUtil.java b/src/de/lmu/ifi/dbs/elki/persistent/PageFileUtil.java deleted file mode 100644 index 80a77fca..00000000 --- a/src/de/lmu/ifi/dbs/elki/persistent/PageFileUtil.java +++ /dev/null @@ -1,100 +0,0 @@ -package de.lmu.ifi.dbs.elki.persistent; - -/* - This file is part of ELKI: - Environment for Developing KDD-Applications Supported by Index-Structures - - Copyright (C) 2012 - Ludwig-Maximilians-Universität München - Lehr- und Forschungseinheit für Datenbanksysteme - ELKI Development Team - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -/** - * Page file statistic utility functions. - * - * @author Erich Schubert - * - * @apiviz.landmark - * - * @apiviz.uses PageFileStatistics oneway - - analyses - */ -public final class PageFileUtil { - /** - * Append the page file statistics to the output buffer. - * - * @param buffer Buffer to append to - */ - public static void appendPageFileStatistics(StringBuilder buffer, PageFileStatistics statistics) { - if(statistics != null) { - buffer.append("Page File Layer: ").append(statistics.getClass()).append('\n'); - buffer.append("Read Operations: ").append(statistics.getReadOperations()).append('\n'); - buffer.append("Write Operations: ").append(statistics.getWriteOperations()).append('\n'); - PageFileStatistics inner = statistics.getInnerStatistics(); - if(inner != null) { - appendPageFileStatistics(buffer, inner); - } - } - } - - /** - * Get the number of (logical) read operations (without caching). - * - * @param statistics Statistics. - * @return logical read operations. - */ - public static long getLogicalReadOperations(PageFileStatistics statistics) { - return statistics.getReadOperations(); - } - - /** - * Get the number of (logical) write operations (without caching). - * - * @param statistics Statistics. - * @return logical write operations. - */ - public static long getLogicalWriteOperations(PageFileStatistics statistics) { - return statistics.getWriteOperations(); - } - - /** - * Get the number of (physical) read operations (with caching). - * - * @param statistics Statistics. - * @return physical read operations. - */ - public static long getPhysicalReadOperations(PageFileStatistics statistics) { - PageFileStatistics inner = statistics.getInnerStatistics(); - if(inner != null) { - return getPhysicalReadOperations(inner); - } - return statistics.getReadOperations(); - } - - /** - * Get the number of (physical) write operations (with caching). - * - * @param statistics Statistics. - * @return physical write operations. - */ - public static long getPhysicalWriteOperations(PageFileStatistics statistics) { - PageFileStatistics inner = statistics.getInnerStatistics(); - if(inner != null) { - return getPhysicalWriteOperations(inner); - } - return statistics.getWriteOperations(); - } -}
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/persistent/PageHeader.java b/src/de/lmu/ifi/dbs/elki/persistent/PageHeader.java index 70a31190..9729b1e7 100644 --- a/src/de/lmu/ifi/dbs/elki/persistent/PageHeader.java +++ b/src/de/lmu/ifi/dbs/elki/persistent/PageHeader.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.persistent; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2012 + Copyright (C) 2013 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team diff --git a/src/de/lmu/ifi/dbs/elki/persistent/PersistentPageFile.java b/src/de/lmu/ifi/dbs/elki/persistent/PersistentPageFile.java index 750ed646..5e0282c5 100644 --- a/src/de/lmu/ifi/dbs/elki/persistent/PersistentPageFile.java +++ b/src/de/lmu/ifi/dbs/elki/persistent/PersistentPageFile.java @@ -4,7 +4,7 @@ package de.lmu.ifi.dbs.elki.persistent; This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures - Copyright (C) 2012 + Copyright (C) 2013 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team @@ -114,7 +114,7 @@ public class PersistentPageFile<P extends ExternalizablePage> extends AbstractSt @Override public P readPage(int pageID) { try { - readAccess++; + countRead(); long offset = ((long) (header.getReservedPages() + pageID)) * (long) pageSize; byte[] buffer = new byte[pageSize]; file.seek(offset); @@ -138,7 +138,7 @@ public class PersistentPageFile<P extends ExternalizablePage> extends AbstractSt super.deletePage(pageID); // delete from file - writeAccess++; + countWrite(); byte[] array = pageToByteArray(null); long offset = ((long) (header.getReservedPages() + pageID)) * (long) pageSize; file.seek(offset); @@ -158,7 +158,7 @@ public class PersistentPageFile<P extends ExternalizablePage> extends AbstractSt @Override public void writePage(int pageID, P page) { try { - writeAccess++; + countWrite(); byte[] array = pageToByteArray(page); long offset = ((long) (header.getReservedPages() + pageID)) * (long) pageSize; assert offset >= 0 : header.getReservedPages() + " " + pageID + " " + pageSize + " " + offset; @@ -308,16 +308,18 @@ public class PersistentPageFile<P extends ExternalizablePage> extends AbstractSt * Increases the {@link AbstractStoringPageFile#readAccess readAccess} counter by * one. */ + @Deprecated public void increaseReadAccess() { - readAccess++; + countRead(); } /** * Increases the {@link AbstractStoringPageFile#writeAccess writeAccess} counter by * one. */ + @Deprecated public void increaseWriteAccess() { - writeAccess++; + countWrite(); } /** @@ -398,4 +400,9 @@ public class PersistentPageFile<P extends ExternalizablePage> extends AbstractSt // Return "new file" status return existed; } + + @Override + protected Logging getLogger() { + return LOG; + } }
\ No newline at end of file diff --git a/src/de/lmu/ifi/dbs/elki/persistent/PersistentPageFileFactory.java b/src/de/lmu/ifi/dbs/elki/persistent/PersistentPageFileFactory.java new file mode 100644 index 00000000..5dfe9ecb --- /dev/null +++ b/src/de/lmu/ifi/dbs/elki/persistent/PersistentPageFileFactory.java @@ -0,0 +1,101 @@ +package de.lmu.ifi.dbs.elki.persistent; + +/* + This file is part of ELKI: + Environment for Developing KDD-Applications Supported by Index-Structures + + Copyright (C) 2013 + Ludwig-Maximilians-Universität München + Lehr- und Forschungseinheit für Datenbanksysteme + ELKI Development Team + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +import de.lmu.ifi.dbs.elki.utilities.exceptions.AbortException; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.OptionID; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameterization.Parameterization; +import de.lmu.ifi.dbs.elki.utilities.optionhandling.parameters.FileParameter; + +/** + * Page file factory for disk-based page files. + * + * @author Erich Schubert + * + * @apiviz.has PersistentPageFile + * + * @param <P> Page type + */ +public class PersistentPageFileFactory<P extends ExternalizablePage> extends AbstractPageFileFactory<P> { + /** + * File name. + */ + private String fileName; + + /** + * Constructor. + * + * @param pageSize Page size + */ + public PersistentPageFileFactory(int pageSize, String fileName) { + super(pageSize); + this.fileName = fileName; + } + + @Override + public PageFile<P> newPageFile(Class<P> cls) { + if (fileName == null) { + throw new AbortException("Disk-backed page file may only be instantiated once!"); + } + PersistentPageFile<P> pfile = new PersistentPageFile<>(pageSize, fileName, cls); + fileName = null; // To avoid double instantiation. + return pfile; + } + + /** + * Parameterization class. + * + * @apiviz.exclude + * + * @author Erich Schubert + */ + public static class Parameterizer extends AbstractPageFileFactory.Parameterizer<ExternalizablePage> { + /** + * File name. + */ + private String fileName; + + /** + * Optional parameter that specifies the name of the file storing the index. + * <p> + * Key: {@code -pagefile.file} + * </p> + */ + public static final OptionID FILE_ID = new OptionID("pagefile.file", "The name of the file storing the page file."); + + @Override + protected void makeOptions(Parameterization config) { + super.makeOptions(config); + FileParameter fileNameP = new FileParameter(FILE_ID, FileParameter.FileType.OUTPUT_FILE); + if (config.grab(fileNameP)) { + fileName = fileNameP.getValue().getPath(); + } + } + + @Override + protected PersistentPageFileFactory<ExternalizablePage> makeInstance() { + return new PersistentPageFileFactory<>(pageSize, fileName); + } + } +} diff --git a/src/de/lmu/ifi/dbs/elki/persistent/package-info.java b/src/de/lmu/ifi/dbs/elki/persistent/package-info.java index c2d8614c..45f575c1 100644 --- a/src/de/lmu/ifi/dbs/elki/persistent/package-info.java +++ b/src/de/lmu/ifi/dbs/elki/persistent/package-info.java @@ -5,7 +5,7 @@ This file is part of ELKI: Environment for Developing KDD-Applications Supported by Index-Structures -Copyright (C) 2012 +Copyright (C) 2013 Ludwig-Maximilians-Universität München Lehr- und Forschungseinheit für Datenbanksysteme ELKI Development Team |