From b2ec1a2d459cfef3ff13133c1f7f5972e3740258 Mon Sep 17 00:00:00 2001 From: Emmanuel Bourg Date: Fri, 17 Jul 2015 00:28:57 +0200 Subject: Imported Upstream version 6.1.4 --- lang/.gitignore | 3 + lang/pom.xml | 175 ++ lang/src/main/java/net/openhft/lang/Compare.java | 176 ++ lang/src/main/java/net/openhft/lang/Jvm.java | 90 + .../main/java/net/openhft/lang/LongHashable.java | 26 + lang/src/main/java/net/openhft/lang/Maths.java | 154 ++ .../net/openhft/lang/collection/HugeArray.java | 68 + .../openhft/lang/collection/HugeCollections.java | 48 + .../net/openhft/lang/collection/HugeQueue.java | 69 + .../lang/collection/impl/HugeArrayImpl.java | 111 + .../lang/collection/impl/HugeQueueImpl.java | 89 + .../java/net/openhft/lang/io/AbstractBytes.java | 2323 ++++++++++++++++++++ .../java/net/openhft/lang/io/ByteBufferBytes.java | 441 ++++ .../net/openhft/lang/io/ByteStringAppender.java | 80 + .../java/net/openhft/lang/io/ByteStringParser.java | 83 + lang/src/main/java/net/openhft/lang/io/Bytes.java | 26 + .../main/java/net/openhft/lang/io/BytesCommon.java | 104 + .../main/java/net/openhft/lang/io/DirectBytes.java | 43 + .../main/java/net/openhft/lang/io/DirectStore.java | 96 + .../src/main/java/net/openhft/lang/io/IOTools.java | 79 + .../main/java/net/openhft/lang/io/MappedFile.java | 146 ++ .../java/net/openhft/lang/io/MappedMemory.java | 83 + .../java/net/openhft/lang/io/MultiStoreBytes.java | 40 + .../java/net/openhft/lang/io/MutableDecimal.java | 205 ++ .../main/java/net/openhft/lang/io/NativeBytes.java | 416 ++++ .../java/net/openhft/lang/io/RandomDataInput.java | 868 ++++++++ .../java/net/openhft/lang/io/RandomDataOutput.java | 819 +++++++ .../java/net/openhft/lang/io/RandomDataUpdate.java | 141 ++ .../java/net/openhft/lang/io/StopCharTester.java | 35 + .../java/net/openhft/lang/io/StopCharTesters.java | 93 + .../lang/io/serialization/BytesMarshallable.java | 43 + .../lang/io/serialization/BytesMarshaller.java | 45 + .../io/serialization/BytesMarshallerFactory.java | 31 + .../io/serialization/CompactBytesMarshaller.java | 27 + .../openhft/lang/io/serialization/RawCopier.java | 104 + .../direct/DirectSerializationFilter.java | 29 + .../direct/DirectSerializationMetadata.java | 89 + .../io/serialization/direct/FieldMetadata.java | 22 + .../lang/io/serialization/direct/Introspect.java | 63 + .../io/serialization/direct/ObjectMarshaller.java | 50 + .../io/serialization/direct/ObjectMarshallers.java | 72 + .../impl/BytesMarshallableMarshaller.java | 52 + .../io/serialization/impl/ClassMarshaller.java | 105 + .../impl/CompactEnumBytesMarshaller.java | 38 + .../lang/io/serialization/impl/DateMarshaller.java | 95 + .../io/serialization/impl/EnumBytesMarshaller.java | 93 + .../impl/ExternalizableMarshaller.java | 58 + .../serialization/impl/GenericEnumMarshaller.java | 93 + .../lang/io/serialization/impl/NoMarshaller.java | 38 + .../io/serialization/impl/StringMarshaller.java | 60 + .../impl/VanillaBytesMarshallerFactory.java | 82 + .../main/java/net/openhft/lang/model/Byteable.java | 34 + .../java/net/openhft/lang/model/ClassModel.java | 29 + .../java/net/openhft/lang/model/CodeGenerator.java | 20 + .../main/java/net/openhft/lang/model/Copyable.java | 31 + .../net/openhft/lang/model/DataValueGenerator.java | 483 ++++ .../net/openhft/lang/model/DataValueMetaModel.java | 60 + .../net/openhft/lang/model/DataValueModel.java | 37 + .../net/openhft/lang/model/DataValueModelImpl.java | 461 ++++ .../net/openhft/lang/model/DataValueModels.java | 47 + .../java/net/openhft/lang/model/FieldModel.java | 62 + .../net/openhft/lang/model/HeapCodeGenerator.java | 161 ++ .../java/net/openhft/lang/model/MethodFilter.java | 25 + .../net/openhft/lang/model/MethodTemplate.java | 23 + .../java/net/openhft/lang/model/VanillaFilter.java | 175 ++ .../net/openhft/lang/model/constraints/Digits.java | 36 + .../openhft/lang/model/constraints/MaxSize.java | 34 + .../net/openhft/lang/model/constraints/Range.java | 36 + .../java/net/openhft/lang/pool/StringInterner.java | 78 + .../java/net/openhft/lang/testing/Differencer.java | 26 + .../net/openhft/lang/testing/RunningMinimum.java | 55 + .../openhft/lang/testing/VanillaDifferencer.java | 29 + .../openhft/lang/thread/NamedThreadFactory.java | 51 + .../java/net/openhft/lang/values/BooleanValue.java | 28 + .../java/net/openhft/lang/values/ByteValue.java | 30 + .../java/net/openhft/lang/values/CharValue.java | 28 + .../java/net/openhft/lang/values/DoubleValue.java | 32 + .../java/net/openhft/lang/values/FloatValue.java | 32 + .../java/net/openhft/lang/values/Int24Value.java | 33 + .../java/net/openhft/lang/values/Int48Value.java | 33 + .../java/net/openhft/lang/values/IntValue.java | 43 + .../java/net/openhft/lang/values/LongValue.java | 34 + .../java/net/openhft/lang/values/ShortValue.java | 30 + .../java/net/openhft/lang/values/StringValue.java | 30 + .../net/openhft/lang/values/UnsignedByteValue.java | 32 + .../net/openhft/lang/values/UnsignedIntValue.java | 33 + .../openhft/lang/values/UnsignedShortValue.java | 32 + lang/src/test/java/net/openhft/lang/JvmTest.java | 36 + lang/src/test/java/net/openhft/lang/MathsTest.java | 53 + .../net/openhft/lang/collection/HugeArrayTest.java | 83 + .../openhft/lang/collection/HugePricesMain.java | 90 + .../net/openhft/lang/collection/HugeQueueTest.java | 71 + .../net/openhft/lang/io/AllocationRatesTest.java | 81 + .../net/openhft/lang/io/ByteBufferBytesTest.java | 920 ++++++++ .../openhft/lang/io/DirectByteBufferBytesTest.java | 921 ++++++++ .../java/net/openhft/lang/io/DirectBytesTest.java | 300 +++ .../test/java/net/openhft/lang/io/IOToolsTest.java | 47 + .../openhft/lang/io/LockingViaFileLockMain.java | 94 + .../net/openhft/lang/io/LockingViaMMapMain.java | 97 + .../java/net/openhft/lang/io/MappedFileTest.java | 97 + .../net/openhft/lang/io/MutableDecimalTest.java | 56 + .../java/net/openhft/lang/io/NativeBytesTest.java | 926 ++++++++ .../lang/io/examples/ParserExampleMain.java | 61 + .../ByteMarshallableMarshallerTest.java | 79 + .../ExternalizableMarshallerTest.java | 81 + .../lang/io/serialization/RawCopierTest.java | 93 + .../serialization/VanillaBytesMarshallerTest.java | 67 + .../direct/DirectSerializationFilterTest.java | 94 + .../direct/DirectSerializationMetadataTest.java | 78 + .../io/serialization/direct/IntrospectTest.java | 38 + .../serialization/direct/ObjectMarshallerTest.java | 245 +++ .../lang/io/serialization/direct/TestClasses.java | 90 + .../openhft/lang/model/DataValueGeneratorTest.java | 174 ++ .../net/openhft/lang/model/DataValueModelTest.java | 59 + .../net/openhft/lang/model/JavaBeanInterface.java | 70 + .../net/openhft/lang/model/MinimalInterface.java | 58 + .../test/java/net/openhft/lang/model/NestedA.java | 39 + .../openhft/lang/model/NestedArrayInterface.java | 35 + .../test/java/net/openhft/lang/model/NestedB.java | 34 + .../net/openhft/lang/model/StringInterface.java | 34 + .../openhft/lang/testing/RunningMinimumTest.java | 50 + .../openhft/lang/values/CheckValuesBuildTest.java | 51 + .../test/java/net/openhft/lang/values/NestAll.java | 53 + 123 files changed, 16317 insertions(+) create mode 100644 lang/.gitignore create mode 100755 lang/pom.xml create mode 100644 lang/src/main/java/net/openhft/lang/Compare.java create mode 100755 lang/src/main/java/net/openhft/lang/Jvm.java create mode 100644 lang/src/main/java/net/openhft/lang/LongHashable.java create mode 100755 lang/src/main/java/net/openhft/lang/Maths.java create mode 100644 lang/src/main/java/net/openhft/lang/collection/HugeArray.java create mode 100644 lang/src/main/java/net/openhft/lang/collection/HugeCollections.java create mode 100644 lang/src/main/java/net/openhft/lang/collection/HugeQueue.java create mode 100644 lang/src/main/java/net/openhft/lang/collection/impl/HugeArrayImpl.java create mode 100644 lang/src/main/java/net/openhft/lang/collection/impl/HugeQueueImpl.java create mode 100755 lang/src/main/java/net/openhft/lang/io/AbstractBytes.java create mode 100755 lang/src/main/java/net/openhft/lang/io/ByteBufferBytes.java create mode 100755 lang/src/main/java/net/openhft/lang/io/ByteStringAppender.java create mode 100755 lang/src/main/java/net/openhft/lang/io/ByteStringParser.java create mode 100755 lang/src/main/java/net/openhft/lang/io/Bytes.java create mode 100755 lang/src/main/java/net/openhft/lang/io/BytesCommon.java create mode 100755 lang/src/main/java/net/openhft/lang/io/DirectBytes.java create mode 100755 lang/src/main/java/net/openhft/lang/io/DirectStore.java create mode 100755 lang/src/main/java/net/openhft/lang/io/IOTools.java create mode 100644 lang/src/main/java/net/openhft/lang/io/MappedFile.java create mode 100644 lang/src/main/java/net/openhft/lang/io/MappedMemory.java create mode 100644 lang/src/main/java/net/openhft/lang/io/MultiStoreBytes.java create mode 100755 lang/src/main/java/net/openhft/lang/io/MutableDecimal.java create mode 100755 lang/src/main/java/net/openhft/lang/io/NativeBytes.java create mode 100755 lang/src/main/java/net/openhft/lang/io/RandomDataInput.java create mode 100755 lang/src/main/java/net/openhft/lang/io/RandomDataOutput.java create mode 100644 lang/src/main/java/net/openhft/lang/io/RandomDataUpdate.java create mode 100755 lang/src/main/java/net/openhft/lang/io/StopCharTester.java create mode 100755 lang/src/main/java/net/openhft/lang/io/StopCharTesters.java create mode 100644 lang/src/main/java/net/openhft/lang/io/serialization/BytesMarshallable.java create mode 100644 lang/src/main/java/net/openhft/lang/io/serialization/BytesMarshaller.java create mode 100644 lang/src/main/java/net/openhft/lang/io/serialization/BytesMarshallerFactory.java create mode 100644 lang/src/main/java/net/openhft/lang/io/serialization/CompactBytesMarshaller.java create mode 100644 lang/src/main/java/net/openhft/lang/io/serialization/RawCopier.java create mode 100644 lang/src/main/java/net/openhft/lang/io/serialization/direct/DirectSerializationFilter.java create mode 100644 lang/src/main/java/net/openhft/lang/io/serialization/direct/DirectSerializationMetadata.java create mode 100644 lang/src/main/java/net/openhft/lang/io/serialization/direct/FieldMetadata.java create mode 100644 lang/src/main/java/net/openhft/lang/io/serialization/direct/Introspect.java create mode 100644 lang/src/main/java/net/openhft/lang/io/serialization/direct/ObjectMarshaller.java create mode 100644 lang/src/main/java/net/openhft/lang/io/serialization/direct/ObjectMarshallers.java create mode 100644 lang/src/main/java/net/openhft/lang/io/serialization/impl/BytesMarshallableMarshaller.java create mode 100644 lang/src/main/java/net/openhft/lang/io/serialization/impl/ClassMarshaller.java create mode 100644 lang/src/main/java/net/openhft/lang/io/serialization/impl/CompactEnumBytesMarshaller.java create mode 100644 lang/src/main/java/net/openhft/lang/io/serialization/impl/DateMarshaller.java create mode 100644 lang/src/main/java/net/openhft/lang/io/serialization/impl/EnumBytesMarshaller.java create mode 100644 lang/src/main/java/net/openhft/lang/io/serialization/impl/ExternalizableMarshaller.java create mode 100644 lang/src/main/java/net/openhft/lang/io/serialization/impl/GenericEnumMarshaller.java create mode 100644 lang/src/main/java/net/openhft/lang/io/serialization/impl/NoMarshaller.java create mode 100644 lang/src/main/java/net/openhft/lang/io/serialization/impl/StringMarshaller.java create mode 100644 lang/src/main/java/net/openhft/lang/io/serialization/impl/VanillaBytesMarshallerFactory.java create mode 100644 lang/src/main/java/net/openhft/lang/model/Byteable.java create mode 100644 lang/src/main/java/net/openhft/lang/model/ClassModel.java create mode 100644 lang/src/main/java/net/openhft/lang/model/CodeGenerator.java create mode 100644 lang/src/main/java/net/openhft/lang/model/Copyable.java create mode 100644 lang/src/main/java/net/openhft/lang/model/DataValueGenerator.java create mode 100644 lang/src/main/java/net/openhft/lang/model/DataValueMetaModel.java create mode 100644 lang/src/main/java/net/openhft/lang/model/DataValueModel.java create mode 100644 lang/src/main/java/net/openhft/lang/model/DataValueModelImpl.java create mode 100644 lang/src/main/java/net/openhft/lang/model/DataValueModels.java create mode 100644 lang/src/main/java/net/openhft/lang/model/FieldModel.java create mode 100644 lang/src/main/java/net/openhft/lang/model/HeapCodeGenerator.java create mode 100644 lang/src/main/java/net/openhft/lang/model/MethodFilter.java create mode 100644 lang/src/main/java/net/openhft/lang/model/MethodTemplate.java create mode 100644 lang/src/main/java/net/openhft/lang/model/VanillaFilter.java create mode 100644 lang/src/main/java/net/openhft/lang/model/constraints/Digits.java create mode 100644 lang/src/main/java/net/openhft/lang/model/constraints/MaxSize.java create mode 100644 lang/src/main/java/net/openhft/lang/model/constraints/Range.java create mode 100755 lang/src/main/java/net/openhft/lang/pool/StringInterner.java create mode 100644 lang/src/main/java/net/openhft/lang/testing/Differencer.java create mode 100644 lang/src/main/java/net/openhft/lang/testing/RunningMinimum.java create mode 100644 lang/src/main/java/net/openhft/lang/testing/VanillaDifferencer.java create mode 100644 lang/src/main/java/net/openhft/lang/thread/NamedThreadFactory.java create mode 100644 lang/src/main/java/net/openhft/lang/values/BooleanValue.java create mode 100644 lang/src/main/java/net/openhft/lang/values/ByteValue.java create mode 100644 lang/src/main/java/net/openhft/lang/values/CharValue.java create mode 100644 lang/src/main/java/net/openhft/lang/values/DoubleValue.java create mode 100644 lang/src/main/java/net/openhft/lang/values/FloatValue.java create mode 100644 lang/src/main/java/net/openhft/lang/values/Int24Value.java create mode 100644 lang/src/main/java/net/openhft/lang/values/Int48Value.java create mode 100644 lang/src/main/java/net/openhft/lang/values/IntValue.java create mode 100644 lang/src/main/java/net/openhft/lang/values/LongValue.java create mode 100644 lang/src/main/java/net/openhft/lang/values/ShortValue.java create mode 100644 lang/src/main/java/net/openhft/lang/values/StringValue.java create mode 100644 lang/src/main/java/net/openhft/lang/values/UnsignedByteValue.java create mode 100644 lang/src/main/java/net/openhft/lang/values/UnsignedIntValue.java create mode 100644 lang/src/main/java/net/openhft/lang/values/UnsignedShortValue.java create mode 100644 lang/src/test/java/net/openhft/lang/JvmTest.java create mode 100644 lang/src/test/java/net/openhft/lang/MathsTest.java create mode 100644 lang/src/test/java/net/openhft/lang/collection/HugeArrayTest.java create mode 100644 lang/src/test/java/net/openhft/lang/collection/HugePricesMain.java create mode 100644 lang/src/test/java/net/openhft/lang/collection/HugeQueueTest.java create mode 100644 lang/src/test/java/net/openhft/lang/io/AllocationRatesTest.java create mode 100644 lang/src/test/java/net/openhft/lang/io/ByteBufferBytesTest.java create mode 100644 lang/src/test/java/net/openhft/lang/io/DirectByteBufferBytesTest.java create mode 100755 lang/src/test/java/net/openhft/lang/io/DirectBytesTest.java create mode 100644 lang/src/test/java/net/openhft/lang/io/IOToolsTest.java create mode 100644 lang/src/test/java/net/openhft/lang/io/LockingViaFileLockMain.java create mode 100644 lang/src/test/java/net/openhft/lang/io/LockingViaMMapMain.java create mode 100644 lang/src/test/java/net/openhft/lang/io/MappedFileTest.java create mode 100644 lang/src/test/java/net/openhft/lang/io/MutableDecimalTest.java create mode 100644 lang/src/test/java/net/openhft/lang/io/NativeBytesTest.java create mode 100644 lang/src/test/java/net/openhft/lang/io/examples/ParserExampleMain.java create mode 100644 lang/src/test/java/net/openhft/lang/io/serialization/ByteMarshallableMarshallerTest.java create mode 100644 lang/src/test/java/net/openhft/lang/io/serialization/ExternalizableMarshallerTest.java create mode 100644 lang/src/test/java/net/openhft/lang/io/serialization/RawCopierTest.java create mode 100644 lang/src/test/java/net/openhft/lang/io/serialization/VanillaBytesMarshallerTest.java create mode 100644 lang/src/test/java/net/openhft/lang/io/serialization/direct/DirectSerializationFilterTest.java create mode 100644 lang/src/test/java/net/openhft/lang/io/serialization/direct/DirectSerializationMetadataTest.java create mode 100644 lang/src/test/java/net/openhft/lang/io/serialization/direct/IntrospectTest.java create mode 100644 lang/src/test/java/net/openhft/lang/io/serialization/direct/ObjectMarshallerTest.java create mode 100644 lang/src/test/java/net/openhft/lang/io/serialization/direct/TestClasses.java create mode 100644 lang/src/test/java/net/openhft/lang/model/DataValueGeneratorTest.java create mode 100644 lang/src/test/java/net/openhft/lang/model/DataValueModelTest.java create mode 100644 lang/src/test/java/net/openhft/lang/model/JavaBeanInterface.java create mode 100644 lang/src/test/java/net/openhft/lang/model/MinimalInterface.java create mode 100644 lang/src/test/java/net/openhft/lang/model/NestedA.java create mode 100644 lang/src/test/java/net/openhft/lang/model/NestedArrayInterface.java create mode 100644 lang/src/test/java/net/openhft/lang/model/NestedB.java create mode 100644 lang/src/test/java/net/openhft/lang/model/StringInterface.java create mode 100644 lang/src/test/java/net/openhft/lang/testing/RunningMinimumTest.java create mode 100644 lang/src/test/java/net/openhft/lang/values/CheckValuesBuildTest.java create mode 100644 lang/src/test/java/net/openhft/lang/values/NestAll.java (limited to 'lang') diff --git a/lang/.gitignore b/lang/.gitignore new file mode 100644 index 0000000..7c73dad --- /dev/null +++ b/lang/.gitignore @@ -0,0 +1,3 @@ +*.ipr +*.iws +target/ \ No newline at end of file diff --git a/lang/pom.xml b/lang/pom.xml new file mode 100755 index 0000000..209a61a --- /dev/null +++ b/lang/pom.xml @@ -0,0 +1,175 @@ + + + + + 4.0.0 + + net.openhft + lang + 6.1.4-SNAPSHOT + bundle + + OpenHFT/Java-Lang/lang + Java Lang library for High Frequency Trading (Java 6+) + + + UTF-8 + 3.2.2 + 3.3.0 + 1.6.0 + + + + + org.kohsuke.jetbrains + annotations + 9.0 + compile + + + + net.openhft + compiler + 2.1 + true + + + + org.easymock + easymock + 3.1 + test + + + junit + junit + 4.11 + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 1.6 + 1.6 + UTF-8 + + + + org.apache.maven.plugins + maven-source-plugin + 2.2 + + + attach-sources + verify + + jar + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.9 + + + attach-javadocs + verify + + jar + + + + + public + true + + + + + + org.apache.felix + maven-bundle-plugin + 2.3.7 + true + + + ${project.groupId}.${project.artifactId} + ${project.artifactId} + + + + + + + manifest + + + + + + + + http://www.openhft.net + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + Peter Lawrey + peter.lawrey@higherfrequencytrading.com + + + + + scm:git:https://github.com/OpenHFT/Java-Lang.git + + \ No newline at end of file diff --git a/lang/src/main/java/net/openhft/lang/Compare.java b/lang/src/main/java/net/openhft/lang/Compare.java new file mode 100644 index 0000000..8d61054 --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/Compare.java @@ -0,0 +1,176 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang; + +/** + * A generic Compare class for many types. It can be sub-classed so you can give types a different behaviour. + */ +public abstract class Compare { + + private static final long NULL_HASHCODE = Long.MIN_VALUE; + + public static boolean isEqual(boolean a, boolean b) { + return a == b; + } + + public static long calcLongHashCode(boolean a) { + return a ? 1 : 0; + } + + public static boolean isEqual(Boolean a, Boolean b) { + return a == null ? b == null : b != null && a.booleanValue() == b.booleanValue(); + } + + public static long calcLongHashCode(Boolean a) { + return a == null ? NULL_HASHCODE : calcLongHashCode(a.booleanValue()); + } + + public static boolean isEqual(byte a, byte b) { + return a == b; + } + + public static long calcLongHashCode(byte a) { + return a; + } + + public static boolean isEqual(Byte a, Byte b) { + return a == null ? b == null : b != null && a.byteValue() == b.byteValue(); + } + + public static long calcLongHashCode(Byte a) { + return a == null ? NULL_HASHCODE : calcLongHashCode(a.byteValue()); + } + + public static boolean isEqual(char a, char b) { + return a == b; + } + + public static long calcLongHashCode(char a) { + return a; + } + + public static boolean isEqual(Character a, Character b) { + return a == null ? b == null : b != null && a.charValue() == b.charValue(); + } + + public static long calcLongHashCode(Character a) { + return a == null ? NULL_HASHCODE : calcLongHashCode(a.charValue()); + } + + public static boolean isEqual(short a, short b) { + return a == b; + } + + public static long calcLongHashCode(short a) { + return a; + } + + public static boolean isEqual(Short a, Short b) { + return a == null ? b == null : b != null && a.shortValue() == b.shortValue(); + } + + public static long calcLongHashCode(Short a) { + return a == null ? NULL_HASHCODE : calcLongHashCode(a.shortValue()); + } + + public static boolean isEqual(int a, int b) { + return a == b; + } + + public static long calcLongHashCode(int a) { + return a; + } + + public static long calcLongHashCode(Integer a) { + return a == null ? NULL_HASHCODE : calcLongHashCode(a.intValue()); + } + + public static boolean isEqual(Integer a, Integer b) { + return a == null ? b == null : b != null && a.intValue() == b.intValue(); + } + + public static boolean isEqual(long a, long b) { + return a == b; + } + + public static long calcLongHashCode(long a) { + return a; + } + + public static boolean isEqual(Long a, Long b) { + return a == null ? b == null : b != null && a.longValue() == b.longValue(); + } + + public static long calcLongHashCode(Long a) { + return a == null ? NULL_HASHCODE : calcLongHashCode(a.longValue()); + } + + public static boolean isEqual(float a, float b) { + return Float.floatToRawIntBits(a) == Float.floatToRawIntBits(b); + } + + public static long calcLongHashCode(float a) { + return Float.floatToRawIntBits(a); + } + + public static boolean isEqual(Float a, Float b) { + return a == null ? b == null : b != null && isEqual(a.floatValue(), b.floatValue()); + } + + public static long calcLongHashCode(Float a) { + return a == null ? NULL_HASHCODE : calcLongHashCode(a.floatValue()); + } + + public static boolean isEqual(double a, double b) { + return Double.doubleToRawLongBits(a) == Double.doubleToRawLongBits(b); + } + + public static long calcLongHashCode(double a) { + return Double.doubleToRawLongBits(a); + } + + public static boolean isEqual(Double a, Double b) { + return a == null ? b == null : b != null && isEqual(a.doubleValue(), b.doubleValue()); + } + + public static long calcLongHashCode(Double a) { + return a == null ? NULL_HASHCODE : calcLongHashCode(a.doubleValue()); + } + + public static boolean isEqual(T a, T b) { + return a == null ? b == null : b != null && a.equals(b); + } + + public static long calcLongHashCode(LongHashable t) { + return t.longHashCode(); + } + + public static long calcLongHashCode(Object t) { + return t == null ? NULL_HASHCODE : + t instanceof LongHashable ? calcLongHashCode((LongHashable) t) : + t instanceof CharSequence ? calcLongHashCode((CharSequence) t) : + t.hashCode(); + } + + public static long calcLongHashCode(CharSequence s) { + long hash = 0; + for (int i = 0, len = s.length(); i < len; i++) { + hash = 57 * hash + s.charAt(i); + } + return hash; + } +} diff --git a/lang/src/main/java/net/openhft/lang/Jvm.java b/lang/src/main/java/net/openhft/lang/Jvm.java new file mode 100755 index 0000000..99e07ab --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/Jvm.java @@ -0,0 +1,90 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang; + +import java.io.File; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.util.Random; +import java.util.logging.Logger; + +/** + * @author peter.lawrey + */ +public enum Jvm { + ; + public static final String TMP = System.getProperty("java.io.tmpdir"); + private static final boolean IS64BIT = is64Bit0(); + + public static boolean is64Bit() { + return IS64BIT; + } + + private static boolean is64Bit0() { + String systemProp; + systemProp = System.getProperty("com.ibm.vm.bitmode"); + if (systemProp != null) { + return "64".equals(systemProp); + } + systemProp = System.getProperty("sun.arch.data.model"); + if (systemProp != null) { + return "64".equals(systemProp); + } + systemProp = System.getProperty("java.vm.version"); + return systemProp != null && systemProp.contains("_64"); + } + + private static final int PROCESS_ID = getProcessId0(); + + public static int getProcessId() { + return PROCESS_ID; + } + + private static int getProcessId0() { + String pid = null; + final File self = new File("/proc/self"); + try { + if (self.exists()) + pid = self.getCanonicalFile().getName(); + } catch (IOException ignored) { + // ignored + } + if (pid == null) + pid = ManagementFactory.getRuntimeMXBean().getName().split("@")[0]; + if (pid == null) { + int rpid = new Random().nextInt(1 << 16); + Logger.getLogger(Jvm.class.getName()).warning("Unable to determine PID, picked a random number=" + rpid); + return rpid; + } else { + return Integer.parseInt(pid); + } + } + + /** + * This may or may not be the OS thread id, but should be unique across processes + * + * @return a unique tid of up to 48 bits. + */ + public static long getUniqueTid() { + return getUniqueTid(Thread.currentThread()); + } + + public static long getUniqueTid(Thread thread) { + return (long) getProcessId() << 32 | (thread.getId() & 0xFFFFFFFFL); + } + +} diff --git a/lang/src/main/java/net/openhft/lang/LongHashable.java b/lang/src/main/java/net/openhft/lang/LongHashable.java new file mode 100644 index 0000000..508ac7e --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/LongHashable.java @@ -0,0 +1,26 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang; + +/** + * User: peter.lawrey + * Date: 08/10/13 + * Time: 12:44 + */ +public interface LongHashable { + long longHashCode(); +} diff --git a/lang/src/main/java/net/openhft/lang/Maths.java b/lang/src/main/java/net/openhft/lang/Maths.java new file mode 100755 index 0000000..f293677 --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/Maths.java @@ -0,0 +1,154 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang; + +/** + * @author peter.lawrey + */ +public class Maths { + /** + * Numbers larger than this are whole numbers due to representation error. + */ + public static final double WHOLE_NUMBER = 1L << 53; + private static final long[] TENS = new long[19]; + + static { + TENS[0] = 1; + for (int i = 1; i < TENS.length; i++) + TENS[i] = TENS[i - 1] * 10; + } + + /** + * Performs a round which is accurate to within 1 ulp. i.e. for values very close to 0.5 it might be rounded up or + * down. This is a pragmatic choice for performance reasons as it is assumed you are not working on the edge of the + * precision of double. + * + * @param d value to round + */ + public static double round2(double d) { + final double factor = 1e2; + return d > WHOLE_NUMBER || d < -WHOLE_NUMBER ? d : + (long) (d < 0 ? d * factor - 0.5 : d * factor + 0.5) / factor; + } + + /** + * Performs a round which is accurate to within 1 ulp. i.e. for values very close to 0.5 it might be rounded up or + * down. This is a pragmatic choice for performance reasons as it is assumed you are not working on the edge of the + * precision of double. + * + * @param d value to round + */ + public static double round4(double d) { + final double factor = 1e4; + return d > Long.MAX_VALUE / factor || d < -Long.MAX_VALUE / factor ? d : + (long) (d < 0 ? d * factor - 0.5 : d * factor + 0.5) / factor; + } + + /** + * Performs a round which is accurate to within 1 ulp. i.e. for values very close to 0.5 it might be rounded up or + * down. This is a pragmatic choice for performance reasons as it is assumed you are not working on the edge of the + * precision of double. + * + * @param d value to round + */ + public static double round6(double d) { + final double factor = 1e6; + return d > Long.MAX_VALUE / factor || d < -Long.MAX_VALUE / factor ? d : + (long) (d < 0 ? d * factor - 0.5 : d * factor + 0.5) / factor; + } + + /** + * Performs a round which is accurate to within 1 ulp. i.e. for values very close to 0.5 it might be rounded up or + * down. This is a pragmatic choice for performance reasons as it is assumed you are not working on the edge of the + * precision of double. + * + * @param d value to round + */ + public static double round8(double d) { + final double factor = 1e8; + return d > Long.MAX_VALUE / factor || d < -Long.MAX_VALUE / factor ? d : + (long) (d < 0 ? d * factor - 0.5 : d * factor + 0.5) / factor; + } + + public static long power10(int n) { + if (n < 0 || n >= TENS.length) return -1; + return TENS[n]; + } + + public static int nextPower2(int n, int min) { + if (n < min) return min; + if ((n & (n - 1)) == 0) return n; + int i = min; + while (i < n) { + i *= 2; + if (i <= 0) return 1 << 30; + } + return i; + } + + public static long nextPower2(long n, long min) { + if (n < min) return min; + if ((n & (n - 1)) == 0) return n; + long i = min; + while (i < n) { + i *= 2; + if (i <= 0) return 1L << 62; + } + return i; + } + + + public static int hash(int n) { + n ^= (n >> 21) ^ (n >> 11); + n ^= (n >> 7) ^ (n >> 4); + return n; + } + + public static int hash(long n) { + n ^= (n >> 43) ^ (n >> 21); + n ^= (n >> 15) ^ (n >> 7); + return (int) n; + } + + public static long hash(CharSequence cs) { + long hash = 0; + for (int i = 0; i < cs.length(); i++) + hash = hash * 131 + cs.charAt(i); + return hash; + } + + + /** + * Compares two {@code long} values numerically. The value returned is identical to what would be returned by: + * 
+     *    Long.valueOf(x).compareTo(Long.valueOf(y))
+     *
+ * + * @param x the first {@code long} to compare + * @param y the second {@code long} to compare + * @return the value {@code 0} if {@code x == y}; a value less than {@code 0} if {@code x < y}; and a value greater + * than {@code 0} if {@code x > y} + */ + public static int compare(long x, long y) { + return (x < y) ? -1 : ((x == y) ? 0 : 1); + } + + public static int intLog2(long num) { + long l = Double.doubleToRawLongBits(num); + return (int) ((l >> 52) - 1023); + } +} diff --git a/lang/src/main/java/net/openhft/lang/collection/HugeArray.java b/lang/src/main/java/net/openhft/lang/collection/HugeArray.java new file mode 100644 index 0000000..cfe1ffb --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/collection/HugeArray.java @@ -0,0 +1,68 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.collection; + +/** + * User: peter.lawrey + * Date: 08/10/13 + * Time: 07:40 + */ +public interface HugeArray { + /** + * @return the capacity of the Array. + */ + long length(); + + /** + * Get a recycled object which is a reference to this element. + * + * @param index to look up. + * @return object reference to the element. + */ + T get(long index); + + /** + * Get a copy of the object in the array + * + * @param index of element to copy + * @param element Copyable element to copy to. + */ + void get(long index, T element); + + /** + * Copy the contents of an index to another object + * + * @param index to copy + * @param to object + */ + void copyTo(long index, T to); + + /** + * Set the data in the array to a copy of this element + * + * @param index to copy to + * @param t to copy + */ + void set(long index, T t); + + /** + * recycle the reference, + * + * @param t reference to recycle. + */ + void recycle(T t); +} diff --git a/lang/src/main/java/net/openhft/lang/collection/HugeCollections.java b/lang/src/main/java/net/openhft/lang/collection/HugeCollections.java new file mode 100644 index 0000000..4dbaad2 --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/collection/HugeCollections.java @@ -0,0 +1,48 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.collection; + +import net.openhft.lang.collection.impl.HugeArrayImpl; +import net.openhft.lang.collection.impl.HugeQueueImpl; +import net.openhft.lang.model.DataValueGenerator; + +/** + * User: peter.lawrey + * Date: 08/10/13 + * Time: 08:09 + */ +public enum HugeCollections { + ; + + private static final DataValueGenerator VALUE_GENERATOR = new DataValueGenerator(); + + static { + VALUE_GENERATOR.setDumpCode(true); + } + + public static DataValueGenerator getDataValueGenerator() { + return VALUE_GENERATOR; + } + + public static HugeArray newArray(Class tClass, long length) { + return new HugeArrayImpl(VALUE_GENERATOR, tClass, length); + } + + public static HugeQueue newQueue(Class tClass, long length) { + return new HugeQueueImpl(new HugeArrayImpl(VALUE_GENERATOR, tClass, length + 1), length + 1); + } +} diff --git a/lang/src/main/java/net/openhft/lang/collection/HugeQueue.java b/lang/src/main/java/net/openhft/lang/collection/HugeQueue.java new file mode 100644 index 0000000..c164fc4 --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/collection/HugeQueue.java @@ -0,0 +1,69 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.collection; + +/** + * User: peter.lawrey + * Date: 08/10/13 + * Time: 13:26 + */ +public interface HugeQueue { + /** + * @return is full + */ + boolean isFull(); + + /** + * @return is empty + */ + boolean isEmpty(); + + /** + * Add to the end of a queue or return false if full. + * + * @return an element to populate or null if full + */ + T offer(); + + /** + * Add to the end of a queue or return false if full. + * + * @param element to add + * @return true if added or false if full. + */ + boolean offer(T element); + + /** + * @return the element or null is non is available + */ + T take(); + + /** + * Copy data to an element + * + * @param element to copy to + * @return true if one was available or false if not. + */ + boolean takeCopy(T element); + + /** + * Recycle an element reference. + * + * @param element reference to recycle. + */ + void recycle(T element); +} diff --git a/lang/src/main/java/net/openhft/lang/collection/impl/HugeArrayImpl.java b/lang/src/main/java/net/openhft/lang/collection/impl/HugeArrayImpl.java new file mode 100644 index 0000000..97cf6dc --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/collection/impl/HugeArrayImpl.java @@ -0,0 +1,111 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.collection.impl; + +import net.openhft.lang.collection.HugeArray; +import net.openhft.lang.io.DirectBytes; +import net.openhft.lang.io.DirectStore; +import net.openhft.lang.model.Byteable; +import net.openhft.lang.model.Copyable; +import net.openhft.lang.model.DataValueGenerator; + +import java.util.ArrayList; +import java.util.List; + +/** + * User: peter.lawrey Date: 08/10/13 Time: 08:11 + */ +public class HugeArrayImpl implements HugeArray { + private static final int MAX_SIZE = 10; + + private final DataValueGenerator valueGenerator; + private final Class tClass; + private final long length; + private final int size; + private final DirectStore store; + private final List freeList = new ArrayList(MAX_SIZE); + + public HugeArrayImpl(DataValueGenerator valueGenerator, Class tClass, long length) { + this.valueGenerator = valueGenerator; + this.tClass = tClass; + this.length = length; + + T ref = valueGenerator.nativeInstance(tClass); + size = ((Byteable) ref).maxSize(); + store = new DirectStore(null, length * size); + ((Byteable) ref).bytes(store.createSlice()); + recycle(ref); + } + + private T createRef() { + T ref = valueGenerator.nativeInstance(tClass); + ((Byteable) ref).bytes(store.createSlice()); + return ref; + } + + @Override + public long length() { + return length; + } + + @Override + public T get(long index) { + T t = acquire(); + DirectBytes bytes = (DirectBytes) ((Byteable) t).bytes(); + bytes.positionAndSize(index * size, size); + return t; + } + + @Override + public void get(long index, T element) { + T t = acquire(); + DirectBytes bytes = (DirectBytes) ((Byteable) t).bytes(); + bytes.positionAndSize(index * size, size); + ((Copyable) element).copyFrom(t); + recycle(t); + } + + private T acquire() { + int size = freeList.size(); + if (size > 0) + return freeList.remove(size - 1); + return createRef(); + } + + @Override + public void copyTo(long index, T to) { + T from = get(index); + ((Copyable) to).copyFrom(from); + recycle(from); + } + + @Override + public void set(long index, T from) { + T to = get(index); + ((Copyable) to).copyFrom(from); + recycle(to); + } + + @Override + public void recycle(T t) { + if (freeList.size() < MAX_SIZE) { + assert ((DirectBytes) ((Byteable) t).bytes()).store() == store; + assert !freeList.contains(t) : "recycling object already recycled"; + freeList.add(t); + } + } +} diff --git a/lang/src/main/java/net/openhft/lang/collection/impl/HugeQueueImpl.java b/lang/src/main/java/net/openhft/lang/collection/impl/HugeQueueImpl.java new file mode 100644 index 0000000..620efc9 --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/collection/impl/HugeQueueImpl.java @@ -0,0 +1,89 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.collection.impl; + +import net.openhft.lang.collection.HugeArray; +import net.openhft.lang.collection.HugeQueue; + +/** + * User: peter.lawrey + * Date: 08/10/13 + * Time: 13:31 + */ +public class HugeQueueImpl implements HugeQueue { + private final HugeArray array; + private final long size; + private long start, end; + + public HugeQueueImpl(HugeArray tHugeArray, long size) { + array = tHugeArray; + this.size = size; + } + + public boolean isFull() { + return next(end) == start; + } + + private long next(long end) { + return (end + 1) % size; + } + + public boolean isEmpty() { + return end == start; + } + + @Override + public boolean offer(T element) { + if (isFull()) + return false; + array.set(end % size, element); + end = next(end); + return true; + } + + @Override + public T take() { + if (isEmpty()) + return null; + long pos = start % size; + start = next(start); + return array.get(pos); + } + + @Override + public boolean takeCopy(T element) { + if (isEmpty()) + return false; + array.get(start % size, element); + start = next(start); + return true; + } + + @Override + public T offer() { + if (isFull()) + return null; + T t = array.get(end % size); + end = next(end); + return t; + } + + @Override + public void recycle(T element) { + array.recycle(element); + } +} diff --git a/lang/src/main/java/net/openhft/lang/io/AbstractBytes.java b/lang/src/main/java/net/openhft/lang/io/AbstractBytes.java new file mode 100755 index 0000000..e167835 --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/io/AbstractBytes.java @@ -0,0 +1,2323 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.io; + +import net.openhft.lang.Jvm; +import net.openhft.lang.Maths; +import net.openhft.lang.io.serialization.BytesMarshallable; +import net.openhft.lang.io.serialization.BytesMarshaller; +import net.openhft.lang.io.serialization.BytesMarshallerFactory; +import net.openhft.lang.io.serialization.CompactBytesMarshaller; +import net.openhft.lang.io.serialization.impl.NoMarshaller; +import net.openhft.lang.io.serialization.impl.VanillaBytesMarshallerFactory; +import net.openhft.lang.pool.StringInterner; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.*; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.Charset; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * @author peter.lawrey + */ +@SuppressWarnings("MagicNumber") +public abstract class AbstractBytes implements Bytes { + public static final long BUSY_LOCK_LIMIT = 10L * 1000 * 1000 * 1000; + public static final int INT_LOCK_MASK = 0xFFFFFF; + public static final int UNSIGNED_BYTE_MASK = 0xFF; + public static final int UNSIGNED_SHORT_MASK = 0xFFFF; + public static final long UNSIGNED_INT_MASK = 0xFFFFFFFFL; + // extra 1 for decimal place. + static final int MAX_NUMBER_LENGTH = 1 + (int) Math.ceil(Math.log10(Long.MAX_VALUE)); + static final byte[] RADIX_PARSE = new byte[256]; + + static { + Arrays.fill(RADIX_PARSE, (byte) -1); + for (int i = 0; i < 10; i++) + RADIX_PARSE['0' + i] = (byte) i; + for (int i = 0; i < 26; i++) + RADIX_PARSE['A' + i] = RADIX_PARSE['a' + i] = (byte) (i + 10); + } + + private static final Logger LOGGER = Logger.getLogger(AbstractBytes.class.getName()); + private static final Charset ISO_8859_1 = Charset.forName("ISO-8859-1"); + private static final byte[] MIN_VALUE_TEXT = ("" + Long.MIN_VALUE).getBytes(); + private static final byte[] Infinity = "Infinity".getBytes(); + private static final byte[] NaN = "NaN".getBytes(); + private static final long MAX_VALUE_DIVIDE_5 = Long.MAX_VALUE / 5; + private static final byte BYTE_MIN_VALUE = Byte.MIN_VALUE; + private static final byte BYTE_EXTENDED = Byte.MIN_VALUE + 1; + private static final byte BYTE_MAX_VALUE = Byte.MIN_VALUE + 2; + private static final short UBYTE_EXTENDED = 0xff; + private static final short SHORT_MIN_VALUE = Short.MIN_VALUE; + private static final short SHORT_EXTENDED = Short.MIN_VALUE + 1; + private static final short SHORT_MAX_VALUE = Short.MIN_VALUE + 2; + private static final int USHORT_EXTENDED = UNSIGNED_SHORT_MASK; + // RandomDataInput + private static final int INT_MIN_VALUE = Integer.MIN_VALUE; + private static final int INT_EXTENDED = Integer.MIN_VALUE + 1; + private static final int INT_MAX_VALUE = Integer.MIN_VALUE + 2; + private static final long MAX_VALUE_DIVIDE_10 = Long.MAX_VALUE / 10; + private static final byte NULL = 'N'; + private static final byte ENUMED = 'E'; + private static final byte SERIALIZED = 'S'; + private static final byte[] RADIX = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ".getBytes(); + static boolean ID_LIMIT_WARNED = false; + private final byte[] numberBuffer = new byte[MAX_NUMBER_LENGTH]; + protected boolean finished; + protected BytesMarshallerFactory bytesMarshallerFactory; + private StringInterner stringInterner = null; + private BytesInputStream inputStream = null; + private BytesOutputStream outputStream = null; + private StringBuilder utfReader = null; + private SimpleDateFormat dateFormat = null; + private long lastDay = Long.MIN_VALUE; + @Nullable + private byte[] lastDateStr = null; + + protected AbstractBytes() { + this(new VanillaBytesMarshallerFactory()); + } + + protected AbstractBytes(BytesMarshallerFactory bytesMarshallerFactory) { + this.finished = false; + this.bytesMarshallerFactory = bytesMarshallerFactory; + } + + static boolean equalsCaseIgnore(StringBuilder sb, String s) { + if (sb.length() != s.length()) + return false; + for (int i = 0; i < s.length(); i++) + if (Character.toLowerCase(sb.charAt(i)) != s.charAt(i)) + return false; + return true; + } + + private static double asDouble(long value, int exp, boolean negative, int decimalPlaces) { + if (decimalPlaces > 0 && value < Long.MAX_VALUE / 2) { + if (value < Long.MAX_VALUE / (1L << 32)) { + exp -= 32; + value <<= 32; + } + if (value < Long.MAX_VALUE / (1L << 16)) { + exp -= 16; + value <<= 16; + } + if (value < Long.MAX_VALUE / (1L << 8)) { + exp -= 8; + value <<= 8; + } + if (value < Long.MAX_VALUE / (1L << 4)) { + exp -= 4; + value <<= 4; + } + if (value < Long.MAX_VALUE / (1L << 2)) { + exp -= 2; + value <<= 2; + } + if (value < Long.MAX_VALUE / (1L << 1)) { + exp -= 1; + value <<= 1; + } + } + for (; decimalPlaces > 0; decimalPlaces--) { + exp--; + long mod = value % 5; + value /= 5; + int modDiv = 1; + if (value < Long.MAX_VALUE / (1L << 4)) { + exp -= 4; + value <<= 4; + modDiv <<= 4; + } + if (value < Long.MAX_VALUE / (1L << 2)) { + exp -= 2; + value <<= 2; + modDiv <<= 2; + } + if (value < Long.MAX_VALUE / (1L << 1)) { + exp -= 1; + value <<= 1; + modDiv <<= 1; + } + if (decimalPlaces > 1) + value += modDiv * mod / 5; + else + value += (modDiv * mod + 4) / 5; + } + final double d = Math.scalb((double) value, exp); + return negative ? -d : d; + } + + private static void warnIdLimit(long id) { + LOGGER.log(Level.WARNING, "High thread id may result in collisions id: " + id); + + ID_LIMIT_WARNED = true; + } + + protected StringInterner stringInterner() { + if (stringInterner == null) + stringInterner = new StringInterner(8 * 1024); + return stringInterner; + } + + @Override + public Boolean parseBoolean(@NotNull StopCharTester tester) { + StringBuilder sb = acquireUtfReader(); + parseUTF(sb, tester); + if (sb.length() == 0) + return null; + switch (sb.charAt(0)) { + case 't': + case 'T': + return sb.length() == 1 || equalsCaseIgnore(sb, "true") ? true : null; + case 'y': + case 'Y': + return sb.length() == 1 || equalsCaseIgnore(sb, "yes") ? true : null; + case '0': + return sb.length() == 1 ? false : null; + case '1': + return sb.length() == 1 ? true : null; + case 'f': + case 'F': + return sb.length() == 1 || equalsCaseIgnore(sb, "false") ? false : null; + case 'n': + case 'N': + return sb.length() == 1 || equalsCaseIgnore(sb, "no") ? false : null; + } + return null; + } + + @Override + public void readFully(@NotNull byte[] bytes) { + readFully(bytes, 0, bytes.length); + } + + @Override + public int skipBytes(int n) { + long position = position(); + int n2 = (int) Math.min(n, capacity() - position); + position(position + n2); + return n2; + } + + @Override + public boolean readBoolean() { + return readByte() != 0; + } + + @Override + public boolean readBoolean(long offset) { + return readByte(offset) != 0; + } + + @Override + public int readUnsignedByte() { + return readByte() & UNSIGNED_BYTE_MASK; + } + + @Override + public int readUnsignedByte(long offset) { + return readByte(offset) & UNSIGNED_BYTE_MASK; + } + + @Override + public int readUnsignedShort() { + return readShort() & UNSIGNED_SHORT_MASK; + } + + @Override + public int readUnsignedShort(long offset) { + return readShort(offset) & UNSIGNED_SHORT_MASK; + } + + @NotNull + @Override + public String readLine() { + StringBuilder input = acquireUtfReader(); + EOL: + while (position() < capacity()) { + int c = readUnsignedByte(); + switch (c) { + case '\n': + break EOL; + case '\r': + long cur = position(); + if (cur < capacity() && readByte(cur) == '\n') + position(cur + 1); + break EOL; + default: + input.append((char) c); + break; + } + } + return stringInterner().intern(input); + } + + @Nullable + @Override + public String readUTFΔ() { + if (readUTFΔ(acquireUtfReader())) + return utfReader.length() == 0 ? "" : stringInterner().intern(utfReader); + return null; + } + + @Nullable + @Override + public String readUTFΔ(long offset) throws IllegalStateException { + long position = position(); + try { + position(offset); + return readUTFΔ(); + } finally { + position(position); + } + } + + // RandomDataOutput + + @NotNull + private StringBuilder acquireUtfReader() { + if (utfReader == null) + utfReader = new StringBuilder(); + else + utfReader.setLength(0); + return utfReader; + } + + @Override + public boolean readUTFΔ(@NotNull StringBuilder stringBuilder) { + try { + stringBuilder.setLength(0); + return appendUTF0(stringBuilder); + } catch (IOException unexpected) { + throw new IllegalStateException(unexpected); + } + } + + @SuppressWarnings("MagicNumber") + private boolean appendUTF0(@NotNull Appendable appendable) throws IOException { + long len = readStopBit(); + if (len < -1 || len > Integer.MAX_VALUE) + throw new StreamCorruptedException("UTF length invalid " + len); + if (len == -1) + return false; + int utflen = (int) len; + readUTF0(appendable, utflen); + return true; + } + + private void readUTF0(@NotNull Appendable appendable, int utflen) throws IOException { + int count = 0; + while (count < utflen) { + int c = readByte(); + if (c < 0) { + position(position() - 1); + break; + } + count++; + appendable.append((char) c); + } + + while (count < utflen) { + int c = readUnsignedByte(); + switch (c >> 4) { + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + /* 0xxxxxxx */ + count++; + appendable.append((char) c); + break; + case 12: + case 13: { + /* 110x xxxx 10xx xxxx */ + count += 2; + if (count > utflen) + throw new UTFDataFormatException( + "malformed input: partial character at end"); + int char2 = readUnsignedByte(); + if ((char2 & 0xC0) != 0x80) + throw new UTFDataFormatException( + "malformed input around byte " + count); + int c2 = (char) (((c & 0x1F) << 6) | + (char2 & 0x3F)); + appendable.append((char) c2); + break; + } + case 14: { + /* 1110 xxxx 10xx xxxx 10xx xxxx */ + count += 3; + if (count > utflen) + throw new UTFDataFormatException( + "malformed input: partial character at end"); + int char2 = readUnsignedByte(); + int char3 = readUnsignedByte(); + + if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80)) + throw new UTFDataFormatException( + "malformed input around byte " + (count - 1)); + int c3 = (char) (((c & 0x0F) << 12) | + ((char2 & 0x3F) << 6) | + (char3 & 0x3F)); + appendable.append((char) c3); + break; + } + default: + /* 10xx xxxx, 1111 xxxx */ + throw new UTFDataFormatException( + "malformed input around byte " + count); + } + } + } + + @NotNull + @Override + public String parseUTF(@NotNull StopCharTester tester) { + parseUTF(acquireUtfReader(), tester); + return stringInterner().intern(utfReader); + } + + @Override + public void parseUTF(@NotNull StringBuilder builder, @NotNull StopCharTester tester) { + builder.setLength(0); + try { + readUTF0(builder, tester); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + private void readUTF0(@NotNull Appendable appendable, @NotNull StopCharTester tester) throws IOException { + while (remaining() > 0) { + int c = readByte(); + if (c < 0) { + position(position() - 1); + break; + } + if (tester.isStopChar(c)) + return; + appendable.append((char) c); + } + + while (remaining() > 0) { + int c = readUnsignedByte(); + switch (c >> 4) { + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + /* 0xxxxxxx */ + if (tester.isStopChar(c)) + return; + appendable.append((char) c); + break; + case 12: + case 13: { + /* 110x xxxx 10xx xxxx */ + int char2 = readUnsignedByte(); + if ((char2 & 0xC0) != 0x80) + throw new UTFDataFormatException( + "malformed input around byte"); + int c2 = (char) (((c & 0x1F) << 6) | + (char2 & 0x3F)); + if (tester.isStopChar(c2)) + return; + appendable.append((char) c2); + break; + } + case 14: { + /* 1110 xxxx 10xx xxxx 10xx xxxx */ + + int char2 = readUnsignedByte(); + int char3 = readUnsignedByte(); + + if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80)) + throw new UTFDataFormatException( + "malformed input around byte "); + int c3 = (char) (((c & 0x0F) << 12) | + ((char2 & 0x3F) << 6) | + (char3 & 0x3F)); + if (tester.isStopChar(c3)) + return; + appendable.append((char) c3); + break; + } + default: + /* 10xx xxxx, 1111 xxxx */ + throw new UTFDataFormatException( + "malformed input around byte "); + } + } + } + + @Override + public boolean stepBackAndSkipTo(@NotNull StopCharTester tester) { + if (position() > 0) + position(position() - 1); + return skipTo(tester); + } + + @Override + public boolean skipTo(@NotNull StopCharTester tester) { + while (remaining() > 0) { + int ch = readByte(); + if (tester.isStopChar(ch)) + return true; + } + return false; + } + + @NotNull + @Override + public String readUTF() { + try { + int len = readUnsignedShort(); + readUTF0(acquireUtfReader(), len); + return utfReader.length() == 0 ? "" : stringInterner().intern(utfReader); + } catch (IOException unexpected) { + throw new AssertionError(unexpected); + } + } + + @Override + public short readCompactShort() { + byte b = readByte(); + switch (b) { + case BYTE_MIN_VALUE: + return Short.MIN_VALUE; + case BYTE_MAX_VALUE: + return Short.MAX_VALUE; + case BYTE_EXTENDED: + return readShort(); + default: + return b; + } + } + + @Override + public int readCompactUnsignedShort() { + int b = readUnsignedByte(); + if (b == UBYTE_EXTENDED) + return readUnsignedShort(); + return b; + } + + @Override + public int readInt24() { + int b = readUnsignedByte(); + int s = readUnsignedShort(); + if (byteOrder() == ByteOrder.BIG_ENDIAN) + return ((b << 24) + (s << 8)) >> 8; + // extra shifting to get sign extension. + return ((b << 8) + (s << 16)) >> 8; + } + + @Override + public int readInt24(long offset) { + int b = readUnsignedByte(offset); + int s = readUnsignedShort(offset + 1); + if (byteOrder() == ByteOrder.BIG_ENDIAN) + return ((b << 24) + (s << 8)) >> 8; + // extra shifting to get sign extension. + return ((b << 8) + (s << 16)) >> 8; + } + + @Override + public long readUnsignedInt() { + return readInt() & UNSIGNED_INT_MASK; + } + + @Override + public long readUnsignedInt(long offset) { + return readInt(offset) & UNSIGNED_INT_MASK; + } + + @Override + public int readCompactInt() { + short b = readShort(); + switch (b) { + case SHORT_MIN_VALUE: + return Integer.MIN_VALUE; + case SHORT_MAX_VALUE: + return Integer.MAX_VALUE; + case SHORT_EXTENDED: + return readInt(); + default: + return b; + } + } + + @Override + public long readCompactUnsignedInt() { + int b = readUnsignedShort(); + if (b == USHORT_EXTENDED) + return readUnsignedInt(); + return b; + } + + @Override + public long readInt48() { + long s = readUnsignedShort(); + long l = readUnsignedInt(); + if (byteOrder() == ByteOrder.BIG_ENDIAN) + return ((s << 48) + (l << 16)) >> 16; + // extra shifting to get sign extension. + return ((s << 16) + (l << 32)) >> 16; + } + + @Override + public long readInt48(long offset) { + long s = readUnsignedShort(offset); + long l = readUnsignedInt(offset + 2); + if (byteOrder() == ByteOrder.BIG_ENDIAN) + return ((s << 48) + (l << 16)) >> 16; + // extra shifting to get sign extension. + return ((s << 16) + (l << 32)) >> 16; + } + + @Override + public long readCompactLong() { + int b = readInt(); + switch (b) { + case INT_MIN_VALUE: + return Long.MIN_VALUE; + case INT_MAX_VALUE: + return Long.MAX_VALUE; + case INT_EXTENDED: + return readLong(); + default: + return b; + } + } + + @Override + public long readStopBit() { + long l = 0, b; + int count = 0; + while ((b = readByte()) < 0) { + l |= (b & 0x7FL) << count; + count += 7; + } + if (b == 0 && count > 0) + return ~l; + return l | (b << count); + } + + @Override + public double readCompactDouble() { + float f = readFloat(); + if (Float.isNaN(f)) + return readDouble(); + return f; + } + + @Override + public void read(@NotNull ByteBuffer bb) { + int len = (int) Math.min(bb.remaining(), remaining()); + if (bb.order() == byteOrder()) { + while (len >= 8) { + bb.putLong(readLong()); + len -= 8; + } + } + while (len > 0) { + bb.put(readByte()); + len--; + } + } + + // // RandomOutputStream + @Override + public void write(@NotNull byte[] bytes) { + int length = bytes.length; + checkWrite(length); + write(bytes, 0, length); + } + + private void checkWrite(int length) { + if (length > remaining()) + throw new IllegalStateException("Cannot write " + length + " only " + remaining() + " remaining"); + } + + @Override + public void writeBoolean(boolean v) { + write(v ? -1 : 0); + } + + @Override + public void writeBoolean(long offset, boolean v) { + writeByte(offset, v ? -1 : 0); + } + + @Override + public void writeBytes(@NotNull String s) { + int len = s.length(); + for (int i = 0; i < len; i++) + write(s.charAt(i)); + } + + @Override + public void writeChars(@NotNull String s) { + int len = s.length(); + for (int i = 0; i < len; i++) + writeChar(s.charAt(i)); + } + + @Override + public void writeUTF(@NotNull String str) { + long strlen = str.length(); + long utflen = findUTFLength(str, strlen); + if (utflen > 65535) + throw new IllegalStateException("String too long " + utflen + " when encoded, max: 65535"); + writeUnsignedShort((int) utflen); + writeUTF0(str, strlen); + } + + @Override + public void writeUTFΔ(@Nullable CharSequence str) { + if (str == null) { + writeStopBit(-1); + return; + } + long strlen = str.length(); + long utflen = findUTFLength(str, strlen); + writeStopBit(utflen); + writeUTF0(str, strlen); + } + + @Override + public void writeUTFΔ(long offset, int maxSize, @Nullable CharSequence s) throws IllegalStateException { + assert maxSize > 1; + if (s == null) { + writeStopBit(-1); + return; + } + long strlen = s.length(); + long utflen = findUTFLength(s, strlen); + long totalSize = IOTools.stopBitLength(utflen) + utflen; + if (totalSize > maxSize) + throw new IllegalStateException("Attempted to write " + totalSize + " byte String, when only " + maxSize + " allowed"); + long position = position(); + try { + position(offset); + writeStopBit(utflen); + writeUTF0(s, strlen); + } finally { + position(position); + } + } + + @NotNull + public ByteStringAppender append(@NotNull CharSequence str) { + if (str == null) + return this; + long strlen = str.length(); + writeUTF0(str, strlen); + return this; + } + + private long findUTFLength(@NotNull CharSequence str, long strlen) { + long utflen = 0, c;/* use charAt instead of copying String to char array */ + for (int i = 0; i < strlen; i++) { + c = str.charAt(i); + if ((c >= 0x0000) && (c <= 0x007F)) { + utflen++; + } else if (c > 0x07FF) { + utflen += 3; + } else { + utflen += 2; + } + } + + if (utflen > remaining()) + throw new IllegalArgumentException( + "encoded string too long: " + utflen + " bytes, remaining=" + remaining()); + return utflen; + } + + private void writeUTF0(@NotNull CharSequence str, long strlen) { + int c; + int i; + for (i = 0; i < strlen; i++) { + c = str.charAt(i); + if (!((c >= 0x0000) && (c <= 0x007F))) + break; + write(c); + } + + for (; i < strlen; i++) { + c = str.charAt(i); + if ((c >= 0x0000) && (c <= 0x007F)) { + write(c); + + } else if (c > 0x07FF) { + write((byte) (0xE0 | ((c >> 12) & 0x0F))); + write((byte) (0x80 | ((c >> 6) & 0x3F))); + write((byte) (0x80 | (c & 0x3F))); + } else { + write((byte) (0xC0 | ((c >> 6) & 0x1F))); + write((byte) (0x80 | c & 0x3F)); + } + } + } + + @Override + public void writeByte(int v) { + write(v); + } + + @Override + public void writeUnsignedByte(int v) { + writeByte(v); + } + + @Override + public void writeUnsignedByte(long offset, int v) { + writeByte(offset, v); + } + + @Override + public void write(long offset, @NotNull byte[] bytes) { + checkWrite(bytes.length); + for (int i = 0; i < bytes.length; i++) + writeByte(offset + i, bytes[i]); + } + + @Override + public void write(byte[] bytes, int off, int len) { + checkWrite(bytes.length); + + for (int i = 0; i < len; i++) + write(bytes[off + i]); + } + + @Override + public void writeUnsignedShort(int v) { + writeShort(v); + } + + @Override + public void writeUnsignedShort(long offset, int v) { + writeShort(offset, v); + } + + @Override + public void writeCompactShort(int v) { + if (v > BYTE_MAX_VALUE && v <= Byte.MAX_VALUE) + writeByte(v); + else + switch (v) { + case Short.MIN_VALUE: + writeByte(BYTE_MIN_VALUE); + break; + case Short.MAX_VALUE: + writeByte(BYTE_MAX_VALUE); + break; + default: + writeByte(BYTE_EXTENDED); + writeShort(v); + break; + } + } + + @Override + public void writeCompactUnsignedShort(int v) { + if (v >= 0 && v < USHORT_EXTENDED) { + writeByte(v); + } else { + writeUnsignedShort(USHORT_EXTENDED); + writeUnsignedShort(v); + } + } + + @Override + public void writeInt24(int v) { + if (byteOrder() == ByteOrder.BIG_ENDIAN) { + writeUnsignedByte(v >>> 16); + writeUnsignedShort(v); + } else { + writeUnsignedByte(v); + writeUnsignedShort(v >>> 8); + } + } + + @Override + public void writeInt24(long offset, int v) { + if (byteOrder() == ByteOrder.BIG_ENDIAN) { + writeUnsignedByte(offset, v >>> 16); + writeUnsignedShort(offset + 1, v); + } else { + writeUnsignedByte(offset, v); + writeUnsignedShort(offset + 1, v >>> 8); + } + } + + @Override + public void writeUnsignedInt(long v) { + writeInt((int) v); + } + + @Override + public void writeUnsignedInt(long offset, long v) { + writeInt(offset, (int) v); + } + + @Override + public void writeCompactInt(int v) { + if (v > SHORT_MAX_VALUE && v <= Short.MAX_VALUE) + writeShort(v); + else + switch (v) { + case Integer.MIN_VALUE: + writeShort(SHORT_MIN_VALUE); + break; + case Integer.MAX_VALUE: + writeShort(SHORT_MAX_VALUE); + break; + default: + writeShort(SHORT_EXTENDED); + writeInt(v); + break; + } + } + + @Override + public void writeCompactUnsignedInt(long v) { + if (v >= 0 && v < USHORT_EXTENDED) { + writeShort((int) v); + } else { + writeShort(USHORT_EXTENDED); + writeUnsignedInt(v); + } + } + + @Override + public void writeInt48(long v) { + if (byteOrder() == ByteOrder.BIG_ENDIAN) { + writeUnsignedShort((int) (v >>> 32)); + writeUnsignedInt(v); + } else { + writeUnsignedShort((int) v); + writeUnsignedInt(v >>> 16); + } + } + + @Override + public void writeInt48(long offset, long v) { + if (byteOrder() == ByteOrder.BIG_ENDIAN) { + writeUnsignedShort(offset, (int) (v >>> 32)); + writeUnsignedInt(offset + 2, v); + } else { + writeUnsignedShort(offset, (int) v); + writeUnsignedInt(offset + 2, v >>> 16); + } + } + + @Override + public void writeCompactLong(long v) { + if (v > INT_MAX_VALUE && v <= Integer.MAX_VALUE) { + writeInt((int) v); + + } else if (v == Long.MIN_VALUE) { + writeInt(INT_MIN_VALUE); + + } else if (v == Long.MAX_VALUE) { + writeInt(INT_MAX_VALUE); + + } else { + writeInt(INT_EXTENDED); + writeLong(v); + + } + } + + @Override + public void writeStopBit(long n) { + boolean neg = false; + if (n < 0) { + neg = true; + n = ~n; + } + while (true) { + long n2 = n >>> 7; + if (n2 != 0) { + writeByte((byte) (0x80 | (n & 0x7F))); + n = n2; + } else { + if (neg) { + writeByte((byte) (0x80 | (n & 0x7F))); + writeByte(0); + } else { + writeByte((byte) (n & 0x7F)); + } + break; + } + } + } + + @Override + public void writeCompactDouble(double v) { + float f = (float) v; + if (f == v) { + writeFloat(f); + } else { + writeFloat(Float.NaN); + writeDouble(v); + } + } + + @Override + public void write(@NotNull ByteBuffer bb) { + if (bb.order() == byteOrder()) + while (bb.remaining() >= 8) + writeLong(bb.getLong()); + while (bb.remaining() >= 1) + writeByte(bb.get()); + } + + @Deprecated + @Override + public void writeStartToPosition(@NotNull Bytes bb) { + final long position = bb.position(); + long offset = 0; + if (position > remaining()) + throw new IndexOutOfBoundsException("Trying to write " + position + " when only " + remaining() + " left"); + // TODO optimise this to use Unsafe copy memory + while (position - offset >= 8) { + writeLong(bb.readLong(offset)); + offset += 8; + } + while (position - offset >= 1) + writeByte(bb.readByte(offset++)); + } + + // // ByteStringAppender + @NotNull + @Override + public ByteStringAppender append(@NotNull CharSequence s, int start, int end) { + for (int i = start, len = Math.min(end, s.length()); i < len; i++) + writeByte(s.charAt(i)); + return this; + } + + @NotNull + @Override + public ByteStringAppender append(@Nullable Enum value) { + return value == null ? this : append(value.toString()); + } + + @NotNull + @Override + public ByteStringAppender append(boolean b) { + append(b ? "true" : "false"); + return this; + } + + @NotNull + @Override + public ByteStringAppender append(char c) { + writeByte(c); + return this; + } + + @NotNull + @Override + public ByteStringAppender append(int num) { + return append((long) num); + } + + @NotNull + @Override + public ByteStringAppender append(long num) { + if (num < 0) { + if (num == Long.MIN_VALUE) { + write(MIN_VALUE_TEXT); + return this; + } + writeByte('-'); + num = -num; + } + if (num == 0) { + writeByte('0'); + + } else { + appendLong0(num); + } + return this; + } + + @NotNull + @Override + public ByteStringAppender append(long num, int base) { + if (base < 2 || base > Character.MAX_RADIX) + throw new IllegalArgumentException("Invalid base: " + base); + if (num < 0) { + if (num == Long.MIN_VALUE) { + writeBytes(BigInteger.valueOf(num).toString(base)); + return this; + } + writeByte('-'); + num = -num; + } + if (num == 0) { + writeByte('0'); + + } else { + while (num > 0) { + writeByte(RADIX[((int) (num % base))]); + num /= base; + } + } + return this; + } + + @NotNull + @Override + public ByteStringAppender appendDateMillis(long timeInMS) { + if (dateFormat == null) { + dateFormat = new SimpleDateFormat("yyyy/MM/dd"); + dateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); + } + long date = timeInMS / 86400000; + if (lastDay != date) { + lastDateStr = dateFormat.format(new Date(timeInMS)).getBytes(ISO_8859_1); + lastDay = date; + } else { + assert lastDateStr != null; + } + write(lastDateStr); + return this; + } + + @NotNull + @Override + public ByteStringAppender appendDateTimeMillis(long timeInMS) { + appendDateMillis(timeInMS); + writeByte('T'); + appendTimeMillis(timeInMS % 86400000L); + return this; + } + + @NotNull + @Override + public ByteStringAppender appendTimeMillis(long timeInMS) { + int hours = (int) (timeInMS / (60 * 60 * 1000)); + if (hours > 99) { + appendLong0(hours); // can have over 24 hours. + } else { + writeByte((char) (hours / 10 + '0')); + writeByte((char) (hours % 10 + '0')); + } + writeByte(':'); + int minutes = (int) ((timeInMS / (60 * 1000)) % 60); + writeByte((char) (minutes / 10 + '0')); + writeByte((char) (minutes % 10 + '0')); + writeByte(':'); + int seconds = (int) ((timeInMS / 1000) % 60); + writeByte((char) (seconds / 10 + '0')); + writeByte((char) (seconds % 10 + '0')); + writeByte('.'); + int millis = (int) (timeInMS % 1000); + writeByte((char) (millis / 100 + '0')); + writeByte((char) (millis / 10 % 10 + '0')); + writeByte((char) (millis % 10 + '0')); + return this; + } + + @NotNull + @Override + public ByteStringAppender append(double d) { + long val = Double.doubleToRawLongBits(d); + int sign = (int) (val >>> 63); + int exp = (int) ((val >>> 52) & 2047); + long mantissa = val & ((1L << 52) - 1); + if (sign != 0) { + writeByte('-'); + } + if (exp == 0 && mantissa == 0) { + writeByte('0'); + return this; + } else if (exp == 2047) { + if (mantissa == 0) { + write(Infinity); + } else { + write(NaN); + } + return this; + } else if (exp > 0) { + mantissa += 1L << 52; + } + final int shift = (1023 + 52) - exp; + if (shift > 0) { + // integer and faction + if (shift < 53) { + long intValue = mantissa >> shift; + appendLong0(intValue); + mantissa -= intValue << shift; + if (mantissa > 0) { + writeByte('.'); + mantissa <<= 1; + mantissa++; + int precision = shift + 1; + long error = 1; + + long value = intValue; + int decimalPlaces = 0; + while (mantissa > error) { + // times 5*2 = 10 + mantissa *= 5; + error *= 5; + precision--; + long num = (mantissa >> precision); + value = value * 10 + num; + writeByte((char) ('0' + num)); + mantissa -= num << precision; + + final double parsedValue = asDouble(value, 0, sign != 0, ++decimalPlaces); + if (parsedValue == d) + break; + } + } + return this; + + } else { + // faction. + writeByte('0'); + writeByte('.'); + mantissa <<= 6; + mantissa += (1 << 5); + int precision = shift + 6; + + long error = (1 << 5); + + long value = 0; + int decimalPlaces = 0; + while (mantissa > error) { + while (mantissa > MAX_VALUE_DIVIDE_5) { + mantissa >>>= 1; + error = (error + 1) >>> 1; + precision--; + } + // times 5*2 = 10 + mantissa *= 5; + error *= 5; + precision--; + if (precision >= 64) { + decimalPlaces++; + writeByte('0'); + continue; + } + long num = (mantissa >>> precision); + value = value * 10 + num; + final char c = (char) ('0' + num); + assert !(c < '0' || c > '9'); + writeByte(c); + mantissa -= num << precision; + final double parsedValue = asDouble(value, 0, sign != 0, ++decimalPlaces); + if (parsedValue == d) + break; + } + return this; + } + } + // large number + mantissa <<= 10; + int precision = -10 - shift; + int digits = 0; + while ((precision > 53 || mantissa > Long.MAX_VALUE >> precision) && precision > 0) { + digits++; + precision--; + long mod = mantissa % 5; + mantissa /= 5; + int modDiv = 1; + while (mantissa < MAX_VALUE_DIVIDE_5 && precision > 1) { + precision -= 1; + mantissa <<= 1; + modDiv <<= 1; + } + mantissa += modDiv * mod / 5; + } + long val2 = precision > 0 ? mantissa << precision : mantissa >>> -precision; + + appendLong0(val2); + for (int i = 0; i < digits; i++) + writeByte('0'); + + return this; + } + + @Override + public double parseDouble() { + long value = 0; + int exp = 0; + boolean negative = false; + int decimalPlaces = Integer.MIN_VALUE; + while (true) { + byte ch = readByte(); + if (ch >= '0' && ch <= '9') { + while (value >= MAX_VALUE_DIVIDE_10) { + value >>>= 1; + exp++; + } + value = value * 10 + (ch - '0'); + decimalPlaces++; + } else if (ch == '-') { + negative = true; + } else if (ch == '.') { + decimalPlaces = 0; + } else { + break; + } + } + + return asDouble(value, exp, negative, decimalPlaces); + } + + @NotNull + @Override + public ByteStringAppender append(@NotNull Iterable list, @NotNull CharSequence separator) { + if (list instanceof RandomAccess && list instanceof List) { + return append((List) list, separator); + } + int i = 0; + for (E e : list) { + if (i++ > 0) + append(separator); + if (e != null) + append(e.toString()); + } + return this; + } + + @NotNull + public ByteStringAppender append(@NotNull List list, @NotNull CharSequence separator) { + for (int i = 0; i < list.size(); i++) { + if (i > 0) + append(separator); + E e = list.get(i); + if (e != null) + append(e.toString()); + } + return this; + } + + @NotNull + @Override + public MutableDecimal parseDecimal(@NotNull MutableDecimal decimal) { + long num = 0, scale = Long.MIN_VALUE; + boolean negative = false; + while (true) { + byte b = readByte(); + // if (b >= '0' && b <= '9') + if ((b - ('0' + Integer.MIN_VALUE)) <= 9 + Integer.MIN_VALUE) { + num = num * 10 + b - '0'; + scale++; + } else if (b == '.') { + scale = 0; + } else if (b == '-') { + negative = true; + } else { + break; + } + } + if (negative) + num = -num; + decimal.set(num, scale > 0 ? (int) scale : 0); + return decimal; + } + + @Override + public long parseLong() { + long num = 0; + boolean negative = false; + while (true) { + byte b = readByte(); + // if (b >= '0' && b <= '9') + if ((b - ('0' + Integer.MIN_VALUE)) <= 9 + Integer.MIN_VALUE) + num = num * 10 + b - '0'; + else if (b == '-') + negative = true; + else + break; + } + return negative ? -num : num; + } + + @Override + public long parseLong(int base) { + if (base < 2 || base > Character.MAX_RADIX) + throw new IllegalArgumentException("Invalid base:" + base); + long num = 0; + boolean negative = false; + while (true) { + byte b = readByte(); + byte rp = RADIX_PARSE[b]; + if (rp >= 0 && rp < base) { + num = num * base + rp; + } else if (b == '-') + negative = true; + else + break; + } + return negative ? -num : num; + } + + private void appendLong0(long num) { + // Extract digits into the end of the numberBuffer + int endIndex = appendLong1(num); + + // Bulk copy the digits into the front of the buffer + write(numberBuffer, endIndex, MAX_NUMBER_LENGTH - endIndex); + } + + private int appendLong1(long num) { + numberBuffer[19] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0) + return 19; + numberBuffer[18] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0) + return 18; + numberBuffer[17] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0) + return 17; + numberBuffer[16] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0) + return 16; + numberBuffer[15] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0) + return 15; + numberBuffer[14] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0) + return 14; + numberBuffer[13] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0) + return 13; + numberBuffer[12] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0) + return 12; + numberBuffer[11] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0) + return 11; + numberBuffer[10] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0) + return 10; + numberBuffer[9] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0) + return 9; + numberBuffer[8] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0) + return 8; + numberBuffer[7] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0) + return 7; + numberBuffer[6] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0) + return 6; + numberBuffer[5] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0) + return 5; + numberBuffer[4] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0) + return 4; + numberBuffer[3] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0) + return 3; + numberBuffer[2] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0) + return 2; + numberBuffer[1] = (byte) (num % 10L + '0'); + num /= 10; + return 1; + } + + @NotNull + @Override + public ByteStringAppender append(double d, int precision) { + if (precision < 0) + precision = 0; + long power10 = Maths.power10(precision); + if (power10 < 0) + power10 = 100000000000000000L; + if (d < 0) { + d = -d; + writeByte('-'); + } + double d2 = d * power10; + if (d2 > Long.MAX_VALUE || d2 < Long.MIN_VALUE + 1) + return append(d); + long val = (long) (d2 + 0.5); + while (precision > 1 && val % 10 == 0) { + val /= 10; + precision--; + } + if (precision > 0 && val % 10 == 0) { + val = (val + 5) / 10; + precision--; + } + if (precision > 0) + appendDouble0(val, precision); + else + appendLong0(val); + return this; + } + + private void appendDouble0(long num, int precision) { + // Extract digits into the end of the numberBuffer + // Once desired precision is reached, write the '.' + int endIndex = appendDouble1(num, precision); + + // Bulk copy the digits into the front of the buffer + // TODO: Can this be avoided with use of correctly offset bulk appends on Excerpt? + // Uses (numberBufferIdx - 1) because index was advanced one too many times + + write(numberBuffer, endIndex, MAX_NUMBER_LENGTH - endIndex); + } + + private int appendDouble1(long num, int precision) { + int endIndex = MAX_NUMBER_LENGTH; + int maxEnd = MAX_NUMBER_LENGTH - precision - 2; + numberBuffer[--endIndex] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0 && endIndex <= maxEnd) + return endIndex; + if (precision == 1) + numberBuffer[--endIndex] = (byte) '.'; + numberBuffer[--endIndex] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0 && endIndex <= maxEnd) + return endIndex; + if (precision == 2) + numberBuffer[--endIndex] = (byte) '.'; + numberBuffer[--endIndex] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0 && endIndex <= maxEnd) + return endIndex; + if (precision == 3) + numberBuffer[--endIndex] = (byte) '.'; + numberBuffer[--endIndex] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0 && endIndex <= maxEnd) + return endIndex; + if (precision == 4) + numberBuffer[--endIndex] = (byte) '.'; + numberBuffer[--endIndex] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0 && endIndex <= maxEnd) + return endIndex; + if (precision == 5) + numberBuffer[--endIndex] = (byte) '.'; + numberBuffer[--endIndex] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0 && endIndex <= maxEnd) + return endIndex; + if (precision == 6) + numberBuffer[--endIndex] = (byte) '.'; + numberBuffer[--endIndex] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0 && endIndex <= maxEnd) + return endIndex; + if (precision == 7) + numberBuffer[--endIndex] = (byte) '.'; + numberBuffer[--endIndex] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0 && endIndex <= maxEnd) + return endIndex; + if (precision == 8) + numberBuffer[--endIndex] = (byte) '.'; + numberBuffer[--endIndex] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0 && endIndex <= maxEnd) + return endIndex; + if (precision == 9) + numberBuffer[--endIndex] = (byte) '.'; + numberBuffer[--endIndex] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0 && endIndex <= maxEnd) + return endIndex; + if (precision == 10) + numberBuffer[--endIndex] = (byte) '.'; + numberBuffer[--endIndex] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0 && endIndex <= maxEnd) + return endIndex; + if (precision == 11) + numberBuffer[--endIndex] = (byte) '.'; + numberBuffer[--endIndex] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0 && endIndex <= maxEnd) + return endIndex; + if (precision == 12) + numberBuffer[--endIndex] = (byte) '.'; + numberBuffer[--endIndex] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0 && endIndex <= maxEnd) + return endIndex; + if (precision == 13) + numberBuffer[--endIndex] = (byte) '.'; + numberBuffer[--endIndex] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0 && endIndex <= maxEnd) + return endIndex; + if (precision == 14) + numberBuffer[--endIndex] = (byte) '.'; + numberBuffer[--endIndex] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0 && endIndex <= maxEnd) + return endIndex; + if (precision == 15) + numberBuffer[--endIndex] = (byte) '.'; + numberBuffer[--endIndex] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0 && endIndex <= maxEnd) + return endIndex; + if (precision == 16) + numberBuffer[--endIndex] = (byte) '.'; + numberBuffer[--endIndex] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0 && endIndex <= maxEnd) + return endIndex; + if (precision == 17) + numberBuffer[--endIndex] = (byte) '.'; + numberBuffer[--endIndex] = (byte) (num % 10L + '0'); + num /= 10; + if (num <= 0 && endIndex <= maxEnd) + return endIndex; + if (precision == 18) + numberBuffer[--endIndex] = (byte) '.'; + numberBuffer[--endIndex] = (byte) (num % 10L + '0'); + return endIndex; + } + + @NotNull + @Override + public ByteStringAppender append(@NotNull MutableDecimal md) { + StringBuilder sb = acquireUtfReader(); + md.toString(sb); + append(sb); + return this; + } + + @NotNull + @Override + public InputStream inputStream() { + if (inputStream == null) + inputStream = new BytesInputStream(); + return inputStream; + } + + @NotNull + @Override + public OutputStream outputStream() { + if (outputStream == null) + outputStream = new BytesOutputStream(); + return outputStream; + } + + @NotNull + @Override + public BytesMarshallerFactory bytesMarshallerFactory() { + return bytesMarshallerFactory == null ? bytesMarshallerFactory = new VanillaBytesMarshallerFactory() : bytesMarshallerFactory; + } + + @SuppressWarnings("unchecked") + @Override + public void writeEnum(@Nullable E e) { + Class aClass; + if (e == null || e instanceof CharSequence) + aClass = String.class; + else + aClass = (Class) e.getClass(); + BytesMarshaller em = bytesMarshallerFactory().acquireMarshaller(aClass, true); + em.write(this, e); + } + + @SuppressWarnings("unchecked") + @Override + public E readEnum(@NotNull Class eClass) { + BytesMarshaller em = bytesMarshallerFactory().acquireMarshaller(eClass, true); + return em.read(this); + } + + @SuppressWarnings("unchecked") + @Override + public > E parseEnum(@NotNull Class eClass, @NotNull StopCharTester tester) { + String text = parseUTF(tester); + if (text.isEmpty()) + return null; + return Enum.valueOf(eClass, text); + } + + @Override + public void writeList(@NotNull Collection list) { + writeStopBit(list.size()); + for (E e : list) + writeEnum(e); + } + + @Override + public void writeMap(@NotNull Map map) { + writeStopBit(map.size()); + for (Map.Entry entry : map.entrySet()) { + writeEnum(entry.getKey()); + writeEnum(entry.getValue()); + } + } + + @Override + public void readList(@NotNull Collection list, @NotNull Class eClass) { + int len = (int) readStopBit(); + list.clear(); + for (int i = 0; i < len; i++) { + @SuppressWarnings("unchecked") + E e = (E) readEnum(eClass); + list.add(e); + } + } + + @Override + @NotNull + public Map readMap(@NotNull Map map, @NotNull Class kClass, @NotNull Class vClass) { + int len = (int) readStopBit(); + map.clear(); + for (int i = 0; i < len; i++) + map.put(readEnum(kClass), readEnum(vClass)); + return map; + } + + @Override + public int available() { + return (int) Math.min(Integer.MAX_VALUE, remaining()); + } + + @Override + public int read() { + return remaining() > 0 ? readByte() : -1; + } + + @Override + public int read(@NotNull byte[] bytes) { + return read(bytes, 0, bytes.length); + } + + @Override + public abstract int read(@NotNull byte[] bytes, int off, int len); + + @Override + public long skip(long n) { + if (n < 0) + throw new IllegalArgumentException("Skip bytes out of range, was " + n); + if (n > remaining()) + n = remaining(); + skipBytes((int) n); + return n; + } + + @Override + public void close() { + if (!isFinished()) + finish(); + } + + @Override + public void finish() throws IndexOutOfBoundsException { + if (remaining() < 0) + throwOverflow(); + finished = true; + } + + private void throwOverflow() throws IndexOutOfBoundsException { + throw new IndexOutOfBoundsException("Buffer overflow, capacity: " + capacity() + " position: " + position()); + } + + @Override + public boolean isFinished() { + return finished; + } + + @Override + public void reset() { + finished = false; + position(0L); + } + + @Override + public void flush() { + checkEndOfBuffer(); + } + + @Nullable + @Override + public Object readObject() { + byte type = readByte(); + switch (type) { + case NULL: + return null; + case ENUMED: { + Class clazz = readEnum(Class.class); + return readEnum(clazz); + } + case SERIALIZED: { + try { + return new ObjectInputStream(this.inputStream()).readObject(); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + default: + BytesMarshaller m = bytesMarshallerFactory().getMarshaller(type); + if (m == null) + throw new IllegalStateException("Unknown type " + (char) type); + return m.read(this); + } + } + + @Nullable + @Override + public T readObject(Class tClass) throws IllegalStateException { + Object o = readObject(); + if (o == null || tClass.isInstance(o)) + return (T) o; + throw new ClassCastException("Cannot convert " + o.getClass().getName() + " to " + tClass.getName() + " was " + o); + } + + @SuppressWarnings("unchecked") + @Override + public void writeObject(@Nullable Object obj) { + if (obj == null) { + writeByte(NULL); + return; + } + + Class clazz = obj.getClass(); + final BytesMarshallerFactory bytesMarshallerFactory = bytesMarshallerFactory(); + BytesMarshaller em = bytesMarshallerFactory.acquireMarshaller(clazz, false); + if (em == NoMarshaller.INSTANCE && autoGenerateMarshaller(obj)) + em = bytesMarshallerFactory.acquireMarshaller(clazz, true); + + if (em != NoMarshaller.INSTANCE) { + if (em instanceof CompactBytesMarshaller) { + writeByte(((CompactBytesMarshaller) em).code()); + em.write(this, obj); + return; + } + writeByte(ENUMED); + writeEnum(clazz); + em.write(this, obj); + return; + } + writeByte(SERIALIZED); + // TODO this is the lame implementation, but it works. + try { + ObjectOutputStream oos = new ObjectOutputStream(this.outputStream()); + oos.writeObject(obj); + } catch (IOException e) { + throw new IllegalStateException(e); + } + checkEndOfBuffer(); + } + + protected boolean autoGenerateMarshaller(Object obj) { + return (obj instanceof Comparable && obj.getClass().getPackage().getName().startsWith("java")) + || obj instanceof Externalizable + || obj instanceof BytesMarshallable; + } + + @Override + public boolean tryLockInt(long offset) { + long id = shortThreadId(); + return tryLockNanos4a(offset, (int) id); + } + + @Override + public boolean tryLockNanosInt(long offset, long nanos) { + long id = shortThreadId(); + int limit = nanos <= 10000 ? (int) nanos / 10 : 1000; + for (int i = 0; i < limit; i++) + if (tryLockNanos4a(offset, (int) id)) + return true; + if (nanos <= 10000) + return false; + long end = System.nanoTime() + nanos - 10000; + do { + if (tryLockNanos4a(offset, (int) id)) + return true; + } while (end > System.nanoTime() && !currentThread().isInterrupted()); + return false; + } + + private boolean tryLockNanos4a(long offset, int id) { + int lowId = shortThreadId(); + int firstValue = ((1 << 24) | lowId); + if (compareAndSwapInt(offset, 0, firstValue)) + return true; + long currentValue = readUnsignedInt(offset); + if ((currentValue & INT_LOCK_MASK) == lowId) { + if (currentValue >= (255L << 24)) + throw new IllegalStateException("Reentred 255 times without an unlock"); + currentValue += 1 << 24; + writeOrderedInt(offset, (int) currentValue); + } + return false; + } + + @Override + public void busyLockInt(long offset) throws InterruptedException, IllegalStateException { + boolean success = tryLockNanosInt(offset, BUSY_LOCK_LIMIT); + if (!success) + if (currentThread().isInterrupted()) + throw new InterruptedException(); + else + throw new IllegalStateException("Failed to acquire lock after " + BUSY_LOCK_LIMIT / 1e9 + " seconds."); + } + + @Override + public void unlockInt(long offset) throws IllegalMonitorStateException { + int lowId = shortThreadId(); + int firstValue = ((1 << 24) | lowId); + if (compareAndSwapInt(offset, firstValue, 0)) + return; + // try to chek the lowId and the count. + unlockFailedInt(offset, lowId); + } + + private Thread currentThread; + private int shortThreadId = Integer.MIN_VALUE; + + public int shortThreadId() { + return shortThreadId > 0 ? shortThreadId : shortThreadId0(); + } + + protected int shortThreadId0() { + final int tid = (int) getId() & INT_LOCK_MASK; + if (!ID_LIMIT_WARNED && tid > 1 << 24) { + warnIdLimit(tid); + } + return tid; + } + + public void setCurrentThread() { + currentThread = Thread.currentThread(); + shortThreadId = shortThreadId0(); + } + + public Thread currentThread() { + return currentThread == null ? Thread.currentThread() : currentThread; + } + + @Override + public boolean tryLockLong(long offset) { + long id = uniqueTid(); + return tryLockNanos8a(offset, id); + } + + public long uniqueTid() { + return Jvm.getUniqueTid(currentThread()); + } + + @Override + public boolean tryLockNanosLong(long offset, long nanos) { + long id = uniqueTid(); + int limit = nanos <= 10000 ? (int) nanos / 10 : 1000; + for (int i = 0; i < limit; i++) + if (tryLockNanos8a(offset, id)) + return true; + if (nanos <= 10000) + return false; + long end = System.nanoTime() + nanos - 10000; + do { + if (tryLockNanos8a(offset, id)) + return true; + } while (end > System.nanoTime() && !currentThread().isInterrupted()); + return false; + } + + private boolean tryLockNanos8a(long offset, long id) { + long firstValue = (1L << 48) | id; + if (compareAndSwapLong(offset, 0, firstValue)) + return true; + long currentValue = readLong(offset); + if ((currentValue & (1L << 48) - 1) == id) { + if (currentValue >>> 48 == 65535) + throw new IllegalStateException("Reentred 65535 times without an unlock"); + currentValue += 1L << 48; + writeOrderedLong(offset, currentValue); + return true; + } + return false; + } + + @Override + public void busyLockLong(long offset) throws InterruptedException, IllegalStateException { + boolean success = tryLockNanosLong(offset, BUSY_LOCK_LIMIT); + if (!success) + if (currentThread().isInterrupted()) + throw new InterruptedException(); + else + throw new IllegalStateException("Failed to acquire lock after " + BUSY_LOCK_LIMIT / 1e9 + " seconds."); + } + + @Override + public void unlockLong(long offset) throws IllegalMonitorStateException { + long id = Jvm.getUniqueTid(); + long firstValue = (1L << 48) | id; + if (compareAndSwapLong(offset, firstValue, 0)) + return; + // try to check the lowId and the count. + unlockFailedLong(offset, id); + } + + protected long getId() { + return currentThread().getId(); + } + + private void unlockFailedInt(long offset, int lowId) throws IllegalMonitorStateException { + long currentValue = readUnsignedInt(offset); + long holderId = currentValue & INT_LOCK_MASK; + if (holderId == lowId) { + currentValue -= 1 << 24; + writeOrderedInt(offset, (int) currentValue); + } else if (currentValue == 0) { + throw new IllegalMonitorStateException("No thread holds this lock"); + } else { + throw new IllegalMonitorStateException("Thread " + holderId + " holds this lock, " + (currentValue >>> 24) + " times"); + } + } + + private void unlockFailedLong(long offset, long id) throws IllegalMonitorStateException { + long currentValue = readLong(offset); + long holderId = currentValue & (-1L >>> 16); + if (holderId == id) { + currentValue -= 1L << 48; + writeOrderedLong(offset, currentValue); + } else if (currentValue == 0) { + throw new IllegalMonitorStateException("No thread holds this lock"); + } else { + throw new IllegalMonitorStateException("Process " + ((currentValue >>> 32) & 0xFFFF) + + " thread " + (holderId & (-1L >>> 32)) + + " holds this lock, " + (currentValue >>> 48) + + " times, unlock from " + Jvm.getProcessId() + + " thread " + currentThread().getId()); + } + } + + @Override + public int getAndAdd(long offset, int delta) { + for (; ; ) { + int current = readVolatileInt(offset); + int next = current + delta; + if (compareAndSwapInt(offset, current, next)) + return current; + } + } + + @Override + public int addAndGetInt(long offset, int delta) { + for (; ; ) { + int current = readVolatileInt(offset); + int next = current + delta; + if (compareAndSwapInt(offset, current, next)) + return next; + } + } + + @Override + public byte addByte(long offset, byte b) { + byte b2 = readByte(offset); + b2 += b; + writeByte(offset, b2); + return b2; + } + + @Override + public int addUnsignedByte(long offset, int i) { + int b2 = readUnsignedByte(offset); + b2 += i; + writeUnsignedByte(offset, b2); + return b2 & 0xFF; + } + + @Override + public short addShort(long offset, short s) { + short s2 = readShort(offset); + s2 += s; + writeByte(offset, s2); + return s2; + } + + @Override + public int addUnsignedShort(long offset, int i) { + int b2 = readUnsignedShort(offset); + b2 += i; + writeUnsignedShort(offset, b2); + return b2 & 0xFFFF; + } + + @Override + public int addInt(long offset, int i) { + int b2 = readInt(offset); + b2 += i; + writeInt(offset, b2); + return b2; + } + + @Override + public long addUnsignedInt(long offset, long i) { + long b2 = readUnsignedInt(offset); + b2 += i; + writeUnsignedInt(offset, b2); + return b2 & 0xFFFFFFFFL; + } + + @Override + public long addLong(long offset, long i) { + long b2 = readLong(offset); + b2 += i; + writeLong(offset, b2); + return b2; + } + + @Override + public float addFloat(long offset, float f) { + float b2 = readFloat(offset); + b2 += f; + writeFloat(offset, b2); + return b2; + } + + @Override + public double addDouble(long offset, double d) { + double b2 = readDouble(offset); + b2 += d; + writeDouble(offset, b2); + return b2; + } + + @Override + public int addAtomicInt(long offset, int i) { + return addAndGetInt(offset, i); + } + + @Override + public long addAtomicLong(long offset, long delta) { + for (; ; ) { + long current = readVolatileLong(offset); + long next = current + delta; + if (compareAndSwapLong(offset, current, next)) + return next; + } + } + + @Override + public float addAtomicFloat(long offset, float delta) { + for (; ; ) { + int current = readVolatileInt(offset); + int next = Float.floatToRawIntBits(Float.intBitsToFloat(current) + delta); + if (compareAndSwapInt(offset, current, next)) + return next; + } + } + + @Override + public double addAtomicDouble(long offset, double delta) { + for (; ; ) { + long current = readVolatileLong(offset); + long next = Double.doubleToRawLongBits(Double.longBitsToDouble(current) + delta); + if (compareAndSwapLong(offset, current, next)) + return next; + } + } + + @Override + public float readVolatileFloat(long offset) { + return Float.intBitsToFloat(readVolatileInt(offset)); + } + + @Override + public double readVolatileDouble(long offset) { + return Double.longBitsToDouble(readVolatileLong(offset)); + } + + @Override + public void writeOrderedFloat(long offset, float v) { + writeOrderedInt(offset, Float.floatToRawIntBits(v)); + } + + @Override + public void writeOrderedDouble(long offset, double v) { + writeOrderedLong(offset, Double.doubleToRawLongBits(v)); + } + + @Override + public int length() { + return (int) Math.min(Integer.MAX_VALUE, remaining()); + } + + @Override + public char charAt(int index) { + return (char) readUnsignedByte(index); + } + + @Override + public CharSequence subSequence(int start, int end) { + StringBuilder sb = new StringBuilder(end - start + 1); + for (int i = start; i < end; i++) + sb.append(charAt(i)); + return sb; + } + + @Override + public void readMarshallable(@NotNull Bytes in) throws IllegalStateException { + long len = Math.min(remaining(), in.remaining()); + long inPosition = in.position(); + write(in, inPosition, len); + in.position(inPosition + len); + } + + @Override + public void writeMarshallable(@NotNull Bytes out) { + out.write(this, position(), remaining()); + + } + + @Override + public void write(BytesCommon bytes, long position, long length) { + if (length > bytes.remaining()) + throw new IllegalArgumentException("Attempt to write " + length + " bytes with " + remaining() + " remaining"); + RandomDataInput rdi = (RandomDataInput) bytes; + if (bytes.byteOrder() == byteOrder()) { + while (length >= 8) { + writeLong(rdi.readLong(position)); + position += 8; + length -= 8; + } + } + while (length >= 1) { + writeByte(rdi.readByte(position)); + position++; + length--; + } + } + + protected class BytesInputStream extends InputStream { + private long mark = 0; + + @Override + public int available() throws IOException { + long remaining = remaining(); + return (int) Math.min(Integer.MAX_VALUE, remaining); + } + + @Override + public void close() throws IOException { + finish(); + } + + @SuppressWarnings("NonSynchronizedMethodOverridesSynchronizedMethod") + @Override + public void mark(int readlimit) { + mark = position(); + } + + @Override + public boolean markSupported() { + return true; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + return AbstractBytes.this.read(b, off, len); + } + + @SuppressWarnings("NonSynchronizedMethodOverridesSynchronizedMethod") + @Override + public void reset() throws IOException { + position(mark); + } + + @Override + public long skip(long n) throws IOException { + if (n > Integer.MAX_VALUE) + throw new IOException("Skip too large"); + return skipBytes((int) n); + } + + @Override + public int read() throws IOException { + if (remaining() > 0) + return readUnsignedByte(); + return -1; + } + } + + protected class BytesOutputStream extends OutputStream { + @Override + public void close() throws IOException { + finish(); + } + + @Override + public void write(@NotNull byte[] b) throws IOException { + AbstractBytes.this.write(b); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + AbstractBytes.this.write(b, off, len); + } + + @Override + public void write(int b) throws IOException { + checkWrite(1); + writeUnsignedByte(b); + } + } +} diff --git a/lang/src/main/java/net/openhft/lang/io/ByteBufferBytes.java b/lang/src/main/java/net/openhft/lang/io/ByteBufferBytes.java new file mode 100755 index 0000000..5b85e87 --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/io/ByteBufferBytes.java @@ -0,0 +1,441 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.io; + +import org.jetbrains.annotations.NotNull; +import sun.nio.ch.DirectBuffer; + +import java.io.EOFException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * @author peter.lawrey + */ +public class ByteBufferBytes extends AbstractBytes { + protected final ByteBuffer buffer; + protected int start, position, limit; + protected AtomicBoolean barrier; + + public ByteBufferBytes(ByteBuffer buffer) { + this(buffer, 0, buffer.capacity()); + } + + public ByteBufferBytes(ByteBuffer buffer, int start, int limit) { + this.buffer = buffer; + this.start = position = start; + this.limit = limit; + } + + public ByteBuffer buffer() { + return buffer; + } + + protected void readBarrier() { + if (barrier == null) barrier = new AtomicBoolean(); + barrier.get(); + } + + protected void writeBarrier() { + if (barrier == null) barrier = new AtomicBoolean(); + barrier.lazySet(false); + } + + @Override + public int read(@NotNull byte[] bytes, int off, int len) { + if (len < 0 || off < 0 || off + len > bytes.length) + throw new IllegalArgumentException(); + long left = remaining(); + if (left <= 0) return -1; + int len2 = (int) Math.min(left, len); + for (int i = 0; i < len2; i++) + bytes[off + i] = readByte(); + return len2; + } + + @Override + public byte readByte() { + if (position < limit) + return buffer.get(position++); + throw new IndexOutOfBoundsException(); + } + + @Override + public byte readByte(long offset) { + int pos = (int) (start + offset); + if (pos < limit) + return buffer.get(pos); + throw new IndexOutOfBoundsException(); + } + + @Override + public void readFully(@NotNull byte[] b, int off, int len) { + if (len < 0 || off < 0 || off + len > b.length) + throw new IllegalArgumentException(); + long left = remaining(); + if (left < len) + throw new IllegalStateException(new EOFException()); + for (int i = 0; i < len; i++) + b[off + i] = readByte(); + } + + @Override + public short readShort() { + if (position + 2 <= limit) { + short s = buffer.getShort(position); + position += 2; + return s; + } + throw new IndexOutOfBoundsException(); + } + + @Override + public short readShort(long offset) { + int pos = (int) (start + offset); + if (pos + 2 <= limit) + return buffer.getShort(pos); + throw new IndexOutOfBoundsException(); + } + + @Override + public char readChar() { + if (position + 2 <= limit) { + char ch = buffer.getChar(position); + position += 2; + return ch; + } + throw new IndexOutOfBoundsException(); + } + + @Override + public char readChar(long offset) { + int pos = (int) (start + offset); + if (pos + 2 <= limit) + return buffer.getChar(pos); + throw new IndexOutOfBoundsException(); + } + + @Override + public int readInt() { + if (position + 4 <= limit) { + int i = buffer.getInt(position); + position += 4; + return i; + } + throw new IndexOutOfBoundsException(); + } + + @Override + public int readInt(long offset) { + int pos = (int) (start + offset); + if (pos + 4 <= limit) + return buffer.getInt(pos); + throw new IndexOutOfBoundsException(); + } + + @Override + public int readVolatileInt() { + readBarrier(); + return readInt(); + } + + @Override + public int readVolatileInt(long offset) { + readBarrier(); + return readInt(offset); + } + + @Override + public long readLong() { + if (position + 8 <= limit) { + long l = buffer.getLong(position); + position += 8; + return l; + } + throw new IndexOutOfBoundsException(); + } + + @Override + public long readLong(long offset) { + int pos = (int) (start + offset); + if (pos + 8 <= limit) + return buffer.getLong(pos); + throw new IndexOutOfBoundsException(); + } + + @Override + public long readVolatileLong() { + readBarrier(); + return readLong(); + } + + @Override + public long readVolatileLong(long offset) { + readBarrier(); + return readLong(offset); + } + + @Override + public float readFloat() { + if (position + 4 <= limit) { + float f = buffer.getFloat(position); + position += 4; + return f; + } + throw new IndexOutOfBoundsException(); + } + + @Override + public float readFloat(long offset) { + int pos = (int) (start + offset); + if (pos + 4 <= limit) + return buffer.getFloat(pos); + throw new IndexOutOfBoundsException(); + } + + @Override + public double readDouble() { + if (position + 8 <= limit) { + double d = buffer.getDouble(position); + position += 8; + return d; + } + throw new IndexOutOfBoundsException(); + } + + @Override + public double readDouble(long offset) { + int pos = (int) (start + offset); + if (pos + 8 <= limit) + return buffer.getDouble(pos); + throw new IndexOutOfBoundsException(); + } + + @Override + public void write(int b) { + if (position < limit) + buffer.put(position++, (byte) b); + else + throw new IndexOutOfBoundsException(); + } + + @Override + public void writeByte(long offset, int b) { + int pos = (int) (start + offset); + if (pos < limit) + buffer.put(pos, (byte) b); + else + throw new IndexOutOfBoundsException(); + } + + @Override + public void writeShort(int v) { + if (position + 2 <= limit) { + buffer.putShort(position, (short) v); + position += 2; + } else { + throw new IndexOutOfBoundsException(); + } + } + + @Override + public void writeShort(long offset, int v) { + int pos = (int) (start + offset); + if (pos + 2 <= limit) + buffer.putShort(pos, (short) v); + else + throw new IndexOutOfBoundsException(); + } + + @Override + public void writeChar(int v) { + if (position + 2 <= limit) { + buffer.putChar(position, (char) v); + position += 2; + } else { + throw new IndexOutOfBoundsException(); + } + } + + @Override + public void writeChar(long offset, int v) { + int pos = (int) (start + offset); + if (pos + 2 <= limit) + buffer.putChar(pos, (char) v); + else + throw new IndexOutOfBoundsException(); + } + + @Override + public void writeInt(int v) { + if (position + 4 <= limit) { + buffer.putInt(position, v); + position += 4; + } else { + throw new IndexOutOfBoundsException(); + } + } + + @Override + public void writeInt(long offset, int v) { + int pos = (int) (start + offset); + if (pos + 4 <= limit) + buffer.putInt(pos, v); + else + throw new IndexOutOfBoundsException(); + } + + @Override + public void writeOrderedInt(int v) { + writeInt(v); + writeBarrier(); + } + + @Override + public void writeOrderedInt(long offset, int v) { + writeInt(offset, v); + writeBarrier(); + } + + @Override + public boolean compareAndSwapInt(long offset, int expected, int x) { + if (buffer instanceof DirectBuffer) + return NativeBytes.UNSAFE.compareAndSwapInt(null, ((DirectBuffer) buffer).address() + offset, expected, x); + return NativeBytes.UNSAFE.compareAndSwapInt(buffer.array(), NativeBytes.BYTES_OFFSET + offset, expected, x); + } + + @Override + public void writeLong(long v) { + if (position + 8 <= limit) { + buffer.putLong(position, v); + position += 8; + } else { + throw new IndexOutOfBoundsException(); + } + } + + @Override + public void writeLong(long offset, long v) { + int pos = (int) (start + offset); + if (pos + 8 <= limit) + buffer.putLong(pos, v); + else + throw new IndexOutOfBoundsException(); + } + + @Override + public void writeOrderedLong(long v) { + writeLong(v); + writeBarrier(); + } + + @Override + public void writeOrderedLong(long offset, long v) { + writeLong((int) offset, v); + writeBarrier(); + } + + @Override + public boolean compareAndSwapLong(long offset, long expected, long x) { + if (buffer instanceof DirectBuffer) + return NativeBytes.UNSAFE.compareAndSwapLong(null, ((DirectBuffer) buffer).address() + offset, expected, x); + return NativeBytes.UNSAFE.compareAndSwapLong(buffer.array(), NativeBytes.BYTES_OFFSET + offset, expected, x); + } + + @Override + public void writeFloat(float v) { + if (position + 4 <= limit) { + buffer.putFloat(position, v); + position += 4; + } else { + throw new IndexOutOfBoundsException(); + } + } + + @Override + public void writeFloat(long offset, float v) { + int pos = (int) (start + offset); + if (pos + 4 <= limit) + buffer.putFloat(pos, v); + else + throw new IndexOutOfBoundsException(); + } + + @Override + public void writeDouble(double v) { + if (position + 8 <= limit) { + buffer.putDouble(position, v); + position += 8; + } else { + throw new IndexOutOfBoundsException(); + } + } + + @Override + public void writeDouble(long offset, double v) { + int pos = (int) (start + offset); + if (pos + 8 <= limit) + buffer.putDouble(pos, v); + else + throw new IndexOutOfBoundsException(); + } + + @Override + public void readObject(Object object, int start, int end) { + throw new UnsupportedOperationException(); + } + + @Override + public void writeObject(Object object, int start, int end) { + throw new UnsupportedOperationException(); + } + + @Override + public long position() { + return position - start; + } + + @Override + public void position(long position) { + if (start + position > Integer.MAX_VALUE) + throw new IndexOutOfBoundsException("Position to large"); + this.position = (int) (start + position); + } + + @Override + public long capacity() { + return limit - start; + } + + @Override + public long remaining() { + return limit - position; + } + + @NotNull + @Override + public ByteOrder byteOrder() { + return buffer.order(); + } + + @Override + public void checkEndOfBuffer() throws IndexOutOfBoundsException { + if (position < start || position > limit) + throw new IndexOutOfBoundsException(); + } +} diff --git a/lang/src/main/java/net/openhft/lang/io/ByteStringAppender.java b/lang/src/main/java/net/openhft/lang/io/ByteStringAppender.java new file mode 100755 index 0000000..fd36102 --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/io/ByteStringAppender.java @@ -0,0 +1,80 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.io; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * @author peter.lawrey + */ +public interface ByteStringAppender extends Appendable, BytesCommon { + @NotNull + ByteStringAppender append(@NotNull CharSequence s); + + @NotNull + ByteStringAppender append(@NotNull CharSequence s, int start, int end); + + /** + * Writes "true" or "false". + * + * @param b to write. + * @return this. + */ + @NotNull + ByteStringAppender append(boolean b); + + @NotNull + ByteStringAppender append(char c); + + @NotNull + ByteStringAppender append(@Nullable Enum value); + + @NotNull + ByteStringAppender append(int i); + + @NotNull + ByteStringAppender append(long l); + + @NotNull + ByteStringAppender append(long l, int base); + + @NotNull + ByteStringAppender appendTimeMillis(long timeInMS); + + @NotNull + ByteStringAppender appendDateMillis(long timeInMS); + + @NotNull + ByteStringAppender appendDateTimeMillis(long timeInMS); + +// ByteStringAppender append(float f); + +// ByteStringAppender append(float f, int precision); + + @NotNull + ByteStringAppender append(double d); + + @NotNull + ByteStringAppender append(double d, int precision); + + @NotNull + ByteStringAppender append(@NotNull MutableDecimal md); + + @NotNull + ByteStringAppender append(@NotNull Iterable list, @NotNull CharSequence separator); +} diff --git a/lang/src/main/java/net/openhft/lang/io/ByteStringParser.java b/lang/src/main/java/net/openhft/lang/io/ByteStringParser.java new file mode 100755 index 0000000..10132cc --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/io/ByteStringParser.java @@ -0,0 +1,83 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.io; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * @author peter.lawrey + */ +public interface ByteStringParser extends BytesCommon { + /** + * Return true or false, or null if it could not be detected as true or false. Case is not important + *

+ * false: f, false, n, no, 0 + *

+ * true: t, true, y, yes, 1 + * + * @param tester to detect the end of the text. + * @return true, false, or null if neither. + */ + Boolean parseBoolean(@NotNull StopCharTester tester); + + /** + * Populate a StringBuilder with the UTF encoded text until the end. + * + * @param builder to clear and append to. + * @param tester to detect when to stop. + */ + void parseUTF(@NotNull StringBuilder builder, @NotNull StopCharTester tester); + + @NotNull + String parseUTF(@NotNull StopCharTester tester); + + @Nullable + > E parseEnum(@NotNull Class eClass, @NotNull StopCharTester tester); + + @NotNull + MutableDecimal parseDecimal(@NotNull MutableDecimal decimal); + + /** + * @return the next long, stopping at the first invalid character + */ + long parseLong(); + + /** + * @param base to use. + * @return the next long, stopping at the first invalid character + */ + long parseLong(int base); + + double parseDouble(); + + /** + * Make sure we just read a stop character + * + * @param tester to stop at + * @return true if we stopped at a stop character, false if we ran out of data. + */ + boolean stepBackAndSkipTo(@NotNull StopCharTester tester); + + /** + * Wind from this positionAddr to the end of the field + * + * @param tester to stop at + * @return true if we stopped at a stop character, false if we ran out of data. + */ + boolean skipTo(@NotNull StopCharTester tester); +} diff --git a/lang/src/main/java/net/openhft/lang/io/Bytes.java b/lang/src/main/java/net/openhft/lang/io/Bytes.java new file mode 100755 index 0000000..c520613 --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/io/Bytes.java @@ -0,0 +1,26 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.io; + +import net.openhft.lang.io.serialization.BytesMarshallable; + +/** + * @author peter.lawrey + */ +public interface Bytes extends RandomDataInput, RandomDataOutput, RandomDataUpdate, + ByteStringAppender, ByteStringParser, CharSequence, BytesMarshallable { +} diff --git a/lang/src/main/java/net/openhft/lang/io/BytesCommon.java b/lang/src/main/java/net/openhft/lang/io/BytesCommon.java new file mode 100755 index 0000000..0549f61 --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/io/BytesCommon.java @@ -0,0 +1,104 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.io; + +import net.openhft.lang.io.serialization.BytesMarshallerFactory; +import org.jetbrains.annotations.NotNull; + +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteOrder; + +/** + * @author peter.lawrey + */ +public interface BytesCommon { + /** + * @return the offset read/written so far + */ + long position(); + + /** + * @param position to skip to + */ + void position(long position); + + /** + * @return space available + */ + long capacity(); + + /** + * @return space remaining in bytes + */ + long remaining(); + + /** + * Mark the end of the message if writing and check we are at the end of the message if reading. + * + * @throws IndexOutOfBoundsException if too much data was written. + */ + void finish() throws IndexOutOfBoundsException; + + /** + * @return has finish been called. + */ + boolean isFinished(); + + /** + * Start again, unfinished, position() == 0 + */ + void reset(); + + /** + * @return Byte order for reading binary + */ + @NotNull + ByteOrder byteOrder(); + + /** + * @return these Bytes as an InputStream + */ + @NotNull + InputStream inputStream(); + + /** + * @return these Bytes as an OutputStream + */ + @NotNull + OutputStream outputStream(); + + /** + * @return the factory for marshallers. + */ + @NotNull + BytesMarshallerFactory bytesMarshallerFactory(); + + /** + * @throws IndexOutOfBoundsException if the bounds of the Bytes has been exceeded. + */ + void checkEndOfBuffer() throws IndexOutOfBoundsException; + + /** + * Copy from one Bytes to another, moves the position by length + * + * @param bytes to copy + * @param position to copy from + * @param length to copy + */ + void write(BytesCommon bytes, long position, long length); +} diff --git a/lang/src/main/java/net/openhft/lang/io/DirectBytes.java b/lang/src/main/java/net/openhft/lang/io/DirectBytes.java new file mode 100755 index 0000000..7e783e4 --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/io/DirectBytes.java @@ -0,0 +1,43 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.io; + +import org.jetbrains.annotations.NotNull; + +/** + * @author peter.lawrey + */ +public class DirectBytes extends NativeBytes { + @NotNull + private final DirectStore store; + + DirectBytes(@NotNull DirectStore store) { + super(store.bytesMarshallerFactory, store.address, store.address, store.address + store.size); + this.store = store; + } + + public void positionAndSize(long offset, long size) { + if (offset < 0 || size < 0 || offset + size > store.size) + throw new IllegalArgumentException(); + startAddr = positionAddr = store.address + offset; + limitAddr = startAddr + size; + } + + public DirectStore store() { + return store; + } +} diff --git a/lang/src/main/java/net/openhft/lang/io/DirectStore.java b/lang/src/main/java/net/openhft/lang/io/DirectStore.java new file mode 100755 index 0000000..7099165 --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/io/DirectStore.java @@ -0,0 +1,96 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.io; + +import net.openhft.lang.io.serialization.BytesMarshallerFactory; +import net.openhft.lang.io.serialization.impl.VanillaBytesMarshallerFactory; +import org.jetbrains.annotations.NotNull; +import sun.misc.Cleaner; + +/** + * @author peter.lawrey + */ +public class DirectStore { + protected BytesMarshallerFactory bytesMarshallerFactory; + private final Cleaner cleaner; + protected long address; + protected long size; + + public DirectStore(long size) { + this(new VanillaBytesMarshallerFactory(), size); + } + + public DirectStore(BytesMarshallerFactory bytesMarshallerFactory, long size) { + this(bytesMarshallerFactory, size, true); + } + + public DirectStore(BytesMarshallerFactory bytesMarshallerFactory, long size, boolean zeroOut) { + this.bytesMarshallerFactory = bytesMarshallerFactory; + address = NativeBytes.UNSAFE.allocateMemory(size); + +// System.out.println("old value " + Integer.toHexString(NativeBytes.UNSAFE.getInt(null, address))); + if (zeroOut) { + NativeBytes.UNSAFE.setMemory(address, size, (byte) 0); + NativeBytes.UNSAFE.putLongVolatile(null, address, 0L); + } + + this.size = size; + cleaner = Cleaner.create(this, new Runnable() { + @Override + public void run() { + if (address != 0) + NativeBytes.UNSAFE.freeMemory(address); + address = DirectStore.this.size = 0; + } + }); + } + + @NotNull + public static DirectStore allocate(long size) { + return new DirectStore(null, size); + } + + @NotNull + public static DirectStore allocateLazy(long size) { + return new DirectStore(null, size, false); + } + +/* public void resize(long newSize) { + if (newSize == size) + return; + address = NativeBytes.UNSAFE.reallocateMemory(address, newSize); + size = newSize; + }*/ + + @NotNull + public DirectBytes createSlice() { + return new DirectBytes(this); + } + + public void free() { + cleaner.clean(); + } + + public long size() { + return size; + } + + public BytesMarshallerFactory bytesMarshallerFactory() { + return bytesMarshallerFactory; + } + +} diff --git a/lang/src/main/java/net/openhft/lang/io/IOTools.java b/lang/src/main/java/net/openhft/lang/io/IOTools.java new file mode 100755 index 0000000..854e8fe --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/io/IOTools.java @@ -0,0 +1,79 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.io; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.logging.Logger; + +/** + * @author peter.lawrey + */ +public enum IOTools { + ; + public static final Charset ISO_8859_1 = Charset.forName("ISO-8859-1"); + public static final Charset UTF_8 = Charset.forName("UTF-8"); + + public static void close(@Nullable Closeable closeable) { + if (closeable == null) return; + try { + closeable.close(); + } catch (IOException ignored) { + } + } + + public static void close(@NotNull Iterable closeables) { + for (Closeable closeable : closeables) { + close(closeable); + } + } + + public static int stopBitLength(long l) { + if (l < 0) return stopBitLength(~l) + 1; + int count = 1; + while (l >= 128) { + l >>>= 7; + count++; + } + return count; + } + + public static void deleteDir(String dirPath) { + deleteDir(new File(dirPath)); + } + + public static void deleteDir(File dir) { + // delete one level. + if (dir.isDirectory()) { + File[] files = dir.listFiles(); + if (files != null) + for (File file : files) + if (file.isDirectory()) { + deleteDir(file); + } else if (!file.delete()) { + Logger.getLogger(IOTools.class.getName()).info("... unable to delete " + file); + } + + } + dir.delete(); + } +} diff --git a/lang/src/main/java/net/openhft/lang/io/MappedFile.java b/lang/src/main/java/net/openhft/lang/io/MappedFile.java new file mode 100644 index 0000000..f4907b9 --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/io/MappedFile.java @@ -0,0 +1,146 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.io; + +import org.jetbrains.annotations.NotNull; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteOrder; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Logger; + +/** + * This manages the full life cycle of a file and its mappings. + */ +public class MappedFile { + private final FileChannel fileChannel; + private final String basePath; + private final long blockSize; + private final long overlapSize; + private final List maps = new ArrayList(); + // short list of the last two mappings. + private volatile MappedMemory map0, map1; + + public MappedFile(String basePath, long blockSize) throws FileNotFoundException { + this(basePath, blockSize, 0L); + } + + public MappedFile(String basePath, long blockSize, long overlapSize) throws FileNotFoundException { + this.basePath = basePath; + this.blockSize = blockSize; + this.overlapSize = overlapSize; + fileChannel = new RandomAccessFile(basePath, "rw").getChannel(); + } + + public static MappedByteBuffer getMap(@NotNull FileChannel fileChannel, long start, int size) throws IOException { + for (int i = 1; ; i++) { + try { +// long startTime = System.nanoTime(); + @SuppressWarnings("UnnecessaryLocalVariable") + MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size); + map.order(ByteOrder.nativeOrder()); +// long time = System.nanoTime() - startTime; +// System.out.printf("Took %,d us to map %,d MB%n", time / 1000, size / 1024 / 1024); +// System.out.println("Map size: "+size); + return map; + } catch (IOException e) { + if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) { + throw e; + } + if (i < 10) + //noinspection CallToThreadYield + Thread.yield(); + else + try { + //noinspection BusyWait + Thread.sleep(1); + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + throw e; + } + } + } + } + + public MappedMemory acquire(long index) throws IOException { + return acquire(index, false); + } + + public MappedMemory acquire(long index, boolean prefetch) throws IOException { + MappedMemory map0 = this.map0, map1 = this.map1; + if (map0 != null && map0.index() == index) { + map0.reserve(); + return map0; + } + if (map1 != null && map1.index() == index) { + map1.reserve(); + return map1; + } + return acquire0(index, prefetch); + } + + private synchronized MappedMemory acquire0(long index, boolean prefetch) throws IOException { + if (map1 != null) + map1.release(); + map1 = map0; + map0 = new MappedMemory(fileChannel.map(FileChannel.MapMode.READ_WRITE, index * blockSize, blockSize + overlapSize), index); + map0.reserve(); + maps.add(map0); + // clean up duds. + for (int i = maps.size() - 1; i >= 0; i--) { + if (maps.get(i).refCount() <= 0) + maps.remove(i); + } + return map0; + } + + public synchronized void close() throws IOException { + if (map1 != null) { + map1.release(); + map1 = null; + } + if (map0 != null) { + map0.release(); + map0 = null; + } + // clean up errant maps. + int count = 0; + for (int i = maps.size() - 1; i >= 0; i--) { + if (maps.get(i).refCount() <= 0) { + maps.get(i).close(); + count++; + } + } + if (count > 1) + Logger.getLogger(MappedFile.class.getName()).info(basePath + " memory mappings left unreleased, num= " + count); + maps.clear(); + fileChannel.close(); + } + + public long size() { + try { + return fileChannel.size(); + } catch (IOException e) { + return 0; + } + } +} diff --git a/lang/src/main/java/net/openhft/lang/io/MappedMemory.java b/lang/src/main/java/net/openhft/lang/io/MappedMemory.java new file mode 100644 index 0000000..df97ef5 --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/io/MappedMemory.java @@ -0,0 +1,83 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.io; + +import sun.misc.Cleaner; +import sun.nio.ch.DirectBuffer; + +import java.nio.MappedByteBuffer; +import java.util.concurrent.atomic.AtomicInteger; + +public class MappedMemory { + private final MappedByteBuffer buffer; + private final long index; + private final AtomicInteger refCount = new AtomicInteger(1); + private volatile boolean unmapped = false; + + public MappedMemory(MappedByteBuffer buffer, long index) { + this.buffer = buffer; + this.index = index; + } + + private static void unmap(MappedByteBuffer bb) { + Cleaner cl = ((DirectBuffer) bb).cleaner(); + if (cl != null) + cl.clean(); + } + + public long index() { + return index; + } + + public void reserve() { + if (unmapped) throw new IllegalStateException(); + refCount.incrementAndGet(); + } + + public void release() { + if (unmapped) throw new IllegalStateException(); + if (refCount.decrementAndGet() > 0) return; + close(); + } + + public MappedByteBuffer buffer() { + return buffer; + } + + public long address() { + return ((DirectBuffer) buffer).address(); + } + + public int refCount() { + return refCount.get(); + } + + public static void release(MappedMemory mapmem) { + if (mapmem != null) + mapmem.release(); + } + + public void force() { + if (!unmapped) + buffer.force(); + } + + public void close() { + unmap(buffer); + unmapped = true; + } +} diff --git a/lang/src/main/java/net/openhft/lang/io/MultiStoreBytes.java b/lang/src/main/java/net/openhft/lang/io/MultiStoreBytes.java new file mode 100644 index 0000000..c2b6679 --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/io/MultiStoreBytes.java @@ -0,0 +1,40 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.io; + +public class MultiStoreBytes extends NativeBytes { + private DirectStore store; + + public MultiStoreBytes() { + super(NO_PAGE, NO_PAGE, NO_PAGE); + this.store = store; + } + + public void storePositionAndSize(DirectStore store, long offset, long size) { + if (offset < 0 || size < 0 || offset + size > store.size) + throw new IllegalArgumentException(); + this.store = store; + this.bytesMarshallerFactory = store.bytesMarshallerFactory; + startAddr = positionAddr = store.address + offset; + limitAddr = startAddr + size; + } + + public DirectStore store() { + return store; + } +} + diff --git a/lang/src/main/java/net/openhft/lang/io/MutableDecimal.java b/lang/src/main/java/net/openhft/lang/io/MutableDecimal.java new file mode 100755 index 0000000..773ce07 --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/io/MutableDecimal.java @@ -0,0 +1,205 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.io; + +import org.jetbrains.annotations.NotNull; + +import java.math.BigDecimal; +import java.math.RoundingMode; + +/** + * @author peter.lawrey + */ +@SuppressWarnings({"CompareToUsesNonFinalVariable", "NonFinalFieldReferenceInEquals", "NonFinalFieldReferencedInHashCode"}) +public class MutableDecimal extends Number implements Comparable { + static final double[] TENS = new double[16]; + + static { + TENS[0] = 1; + for (int i = 1; i < TENS.length; i++) + TENS[i] = 10 * TENS[i - 1]; + } + + private long value; + private int scale; + + public MutableDecimal() { + this(0, Integer.MIN_VALUE); + } + + public MutableDecimal(long value, int scale) { + this.value = value; + this.scale = scale; + } + + public MutableDecimal(long value) { + this(value, 0); + } + + public MutableDecimal(double d, int precision) { + set(d, precision); + } + + public void set(double d, int precision) { + double d2 = precision > 0 ? d * tens(precision) : d / tens(-precision); + scale = precision; + if (Math.abs(d2) < 1e16) { + value = Math.round(d2); + } else { + BigDecimal bd = BigDecimal.valueOf(d).setScale(precision, RoundingMode.HALF_UP); + value = bd.unscaledValue().longValue(); + } + } + + public void set(long value, int scale) { + this.value = value; + this.scale = scale; + } + + public long value() { + return value; + } + + public int scale() { + return scale; + } + + public void clear() { + scale = Integer.MIN_VALUE; + } + + public boolean isSet() { + return scale > Integer.MIN_VALUE; + } + + @Override + public int hashCode() { + int hash = (int) value; + hash ^= value >>> 32; + hash ^= scale * 37; + return hash; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof MutableDecimal)) + return false; + MutableDecimal md = (MutableDecimal) obj; + return value == md.value && scale == md.scale; + } + + @NotNull + @Override + public String toString() { + StringBuilder sb = new StringBuilder(20); + toString(sb); + return sb.toString(); + } + + public void toString(@NotNull StringBuilder sb) { + if (scale == Integer.MIN_VALUE) { + sb.append("not set"); + return; + } + if (value == 0 && scale <= 0) { + sb.append('0'); + return; + } + + boolean neg = false; + long v = value; + int s = scale; + if (v < 0) { + v = -value; + neg = true; + } + for (int s2 = scale; s2 < 0; s2++) + sb.append('0'); + while (v != 0 || s >= 0) { + int digit = (int) (v % 10); + // MIN_VALUE + if (digit < 0) { + digit = 8; + v = (v >>> 1) / 5; + } else { + v /= 10; + } + sb.append((char) ('0' + digit)); + if (--s == 0) + sb.append('.'); + } + if (neg) + sb.append('-'); + sb.reverse(); + } + + @NotNull + @Override + public MutableDecimal clone() throws CloneNotSupportedException { + return (MutableDecimal) super.clone(); + } + + @Override + public int intValue() { + return (int) longValue(); + } + + @Override + public long longValue() { + if (scale == 0) + return value; + if (scale > 0 && scale < TENS.length) + return value / (long) TENS[scale]; + if (scale < 0 && -scale < TENS.length) + return value * (long) TENS[-scale]; + return (long) doubleValue(); + } + + @Override + public float floatValue() { + return (float) doubleValue(); + } + + public double doubleValue() { + if (scale == 0) + return value; + return scale <= 0 ? value * tens(-scale) : value / tens(scale); + } + + private static double tens(int scale) { + return scale < TENS.length ? TENS[scale] : Math.pow(10, scale); + } + + @Override + public int compareTo(@NotNull MutableDecimal o) { + long value = this.value, ovalue = o.value; + if (scale == o.scale) + return longCompareTo(value, ovalue); + if (value == 0 && ovalue == 0) + return 0; + double d = doubleValue(), od = o.doubleValue(); + double err = (Math.abs(d) + Math.abs(od)) / 1e15; + if (d + err < od) return -1; + if (d > od + err) return +1; + // fallback. + return BigDecimal.valueOf(value, scale).compareTo(BigDecimal.valueOf(ovalue, o.scale())); + } + + private static int longCompareTo(long value, long ovalue) { + return value < ovalue ? -1 : value > ovalue ? +1 : 0; + } +} diff --git a/lang/src/main/java/net/openhft/lang/io/NativeBytes.java b/lang/src/main/java/net/openhft/lang/io/NativeBytes.java new file mode 100755 index 0000000..2a974d7 --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/io/NativeBytes.java @@ -0,0 +1,416 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.io; + +import net.openhft.lang.io.serialization.BytesMarshallerFactory; +import org.jetbrains.annotations.NotNull; +import sun.misc.Unsafe; + +import java.io.EOFException; +import java.lang.reflect.Field; +import java.nio.ByteOrder; + +/** + * @author peter.lawrey + */ +public class NativeBytes extends AbstractBytes { + /** + * *** Access the Unsafe class ***** + */ + @NotNull + @SuppressWarnings("ALL") + public static final Unsafe UNSAFE; + static final int BYTES_OFFSET; + + static { + try { + @SuppressWarnings("ALL") + Field theUnsafe = Unsafe.class.getDeclaredField("theUnsafe"); + theUnsafe.setAccessible(true); + UNSAFE = (Unsafe) theUnsafe.get(null); + BYTES_OFFSET = UNSAFE.arrayBaseOffset(byte[].class); + + } catch (Exception e) { + throw new AssertionError(e); + } + } + + protected static final long NO_PAGE = UNSAFE.allocateMemory(UNSAFE.pageSize()); + + protected long startAddr; + protected long positionAddr; + protected long limitAddr; + + public NativeBytes(long startAddr, long positionAddr, long limitAddr) { + this.startAddr = startAddr; + this.positionAddr = positionAddr; + this.limitAddr = limitAddr; + } + + public NativeBytes(BytesMarshallerFactory bytesMarshallerFactory, long startAddr, long positionAddr, long limitAddr) { + super(bytesMarshallerFactory); + this.startAddr = startAddr; + this.positionAddr = positionAddr; + this.limitAddr = limitAddr; + } + + public NativeBytes(NativeBytes bytes) { + super(bytes.bytesMarshallerFactory()); + this.startAddr = bytes.startAddr; + this.positionAddr = bytes.positionAddr; + this.limitAddr = bytes.limitAddr; + } + + public static long longHash(byte[] bytes, int off, int len) { + long hash = 0; + int pos = 0; + for (; pos < len - 7; pos += 8) + hash = hash * 10191 + UNSAFE.getLong(bytes, (long) BYTES_OFFSET + off + pos); + for (; pos < len; pos++) + hash = hash * 57 + bytes[off + pos]; + return hash; + } + + @Override + public int read(@NotNull byte[] bytes, int off, int len) { + if (len < 0 || off < 0 || off + len > bytes.length) + throw new IllegalArgumentException(); + long left = remaining(); + if (left <= 0) return -1; + int len2 = (int) Math.min(len, left); + UNSAFE.copyMemory(null, positionAddr, bytes, BYTES_OFFSET + off, len2); + positionAddr += len2; + return len2; + } + + @Override + public byte readByte() { + return UNSAFE.getByte(positionAddr++); + } + + @Override + public byte readByte(long offset) { + return UNSAFE.getByte(startAddr + offset); + } + + @Override + public void readFully(@NotNull byte[] b, int off, int len) { + if (len < 0 || off < 0 || off + len > b.length) + throw new IllegalArgumentException(); + long left = remaining(); + if (left < len) + throw new IllegalStateException(new EOFException()); + UNSAFE.copyMemory(null, positionAddr, b, BYTES_OFFSET + off, len); + positionAddr += len; + } + + @Override + public short readShort() { + short s = UNSAFE.getShort(positionAddr); + positionAddr += 2; + return s; + } + + @Override + public short readShort(long offset) { + return UNSAFE.getShort(startAddr + offset); + } + + @Override + public char readChar() { + char ch = UNSAFE.getChar(positionAddr); + positionAddr += 2; + return ch; + } + + @Override + public char readChar(long offset) { + return UNSAFE.getChar(startAddr + offset); + } + + @Override + public int readInt() { + int i = UNSAFE.getInt(positionAddr); + positionAddr += 4; + return i; + } + + @Override + public int readInt(long offset) { + return UNSAFE.getInt(startAddr + offset); + } + + @Override + public int readVolatileInt() { + int i = UNSAFE.getIntVolatile(null, positionAddr); + positionAddr += 4; + return i; + } + + @Override + public int readVolatileInt(long offset) { + return UNSAFE.getIntVolatile(null, startAddr + offset); + } + + @Override + public long readLong() { + long l = UNSAFE.getLong(positionAddr); + positionAddr += 8; + return l; + } + + @Override + public long readLong(long offset) { + return UNSAFE.getLong(startAddr + offset); + } + + @Override + public long readVolatileLong() { + long l = UNSAFE.getLongVolatile(null, positionAddr); + positionAddr += 8; + return l; + } + + @Override + public long readVolatileLong(long offset) { + return UNSAFE.getLongVolatile(null, startAddr + offset); + } + + @Override + public float readFloat() { + float f = UNSAFE.getFloat(positionAddr); + positionAddr += 4; + return f; + } + + @Override + public float readFloat(long offset) { + return UNSAFE.getFloat(startAddr + offset); + } + + @Override + public double readDouble() { + double d = UNSAFE.getDouble(positionAddr); + positionAddr += 8; + return d; + } + + @Override + public double readDouble(long offset) { + return UNSAFE.getDouble(startAddr + offset); + } + + @Override + public void write(int b) { + UNSAFE.putByte(positionAddr++, (byte) b); + } + + @Override + public void writeByte(long offset, int b) { + UNSAFE.putByte(startAddr + offset, (byte) b); + } + + @Override + public void write(long offset, @NotNull byte[] bytes) { + if (offset < 0 || offset + bytes.length > capacity()) + throw new IllegalArgumentException(); + UNSAFE.copyMemory(bytes, BYTES_OFFSET, null, startAddr + offset, bytes.length); + positionAddr += bytes.length; + } + + @Override + public void write(byte[] bytes, int off, int len) { + UNSAFE.copyMemory(bytes, BYTES_OFFSET + off, null, positionAddr, len); + positionAddr += len; + } + + @Override + public void writeShort(int v) { + UNSAFE.putShort(positionAddr, (short) v); + positionAddr += 2; + } + + @Override + public void writeShort(long offset, int v) { + UNSAFE.putShort(startAddr + offset, (short) v); + } + + @Override + public void writeChar(int v) { + UNSAFE.putChar(positionAddr, (char) v); + positionAddr += 2; + } + + @Override + public void writeChar(long offset, int v) { + UNSAFE.putChar(startAddr + offset, (char) v); + } + + @Override + public void writeInt(int v) { + UNSAFE.putInt(null, positionAddr, v); + positionAddr += 4; + } + + @Override + public void writeInt(long offset, int v) { + UNSAFE.putInt(startAddr + offset, v); + } + + @Override + public void writeOrderedInt(int v) { + UNSAFE.putOrderedInt(null, positionAddr, v); + positionAddr += 4; + } + + @Override + public void writeOrderedInt(long offset, int v) { + UNSAFE.putOrderedInt(null, startAddr + offset, v); + } + + @Override + public boolean compareAndSwapInt(long offset, int expected, int x) { + return UNSAFE.compareAndSwapInt(null, startAddr + offset, expected, x); + } + + @Override + public void writeLong(long v) { + UNSAFE.putLong(positionAddr, v); + positionAddr += 8; + } + + @Override + public void writeLong(long offset, long v) { + UNSAFE.putLong(startAddr + offset, v); + } + + @Override + public void writeOrderedLong(long v) { + UNSAFE.putOrderedLong(null, positionAddr, v); + positionAddr += 8; + } + + @Override + public void writeOrderedLong(long offset, long v) { + UNSAFE.putOrderedLong(null, startAddr + offset, v); + } + + @Override + public boolean compareAndSwapLong(long offset, long expected, long x) { + return UNSAFE.compareAndSwapLong(null, startAddr + offset, expected, x); + } + + @Override + public void writeFloat(float v) { + UNSAFE.putFloat(positionAddr, v); + positionAddr += 4; + } + + @Override + public void writeFloat(long offset, float v) { + UNSAFE.putFloat(startAddr + offset, v); + } + + @Override + public void writeDouble(double v) { + UNSAFE.putDouble(positionAddr, v); + positionAddr += 8; + } + + @Override + public void writeDouble(long offset, double v) { + UNSAFE.putDouble(startAddr + offset, v); + } + + @Override + public void readObject(Object object, int start, int end) { + int len = end - start; + for (; len >= 8; len -= 8) { + UNSAFE.putLong(object, (long) start, UNSAFE.getLong(positionAddr)); + positionAddr += 8; + start += 8; + } + for (; len > 0; len--) { + UNSAFE.putByte(object, (long) start, UNSAFE.getByte(positionAddr)); + positionAddr++; + start++; + } + } + + @Override + public void writeObject(Object object, int start, int end) { + int len = end - start; + for (; len >= 8; len -= 8) { + UNSAFE.putLong(positionAddr, UNSAFE.getLong(object, (long) start)); + positionAddr += 8; + start += 8; + } + for (; len > 0; len--) { + UNSAFE.putByte(positionAddr, UNSAFE.getByte(object, (long) start)); + positionAddr++; + start++; + } + } + + @Override + public long position() { + return (positionAddr - startAddr); + } + + @Override + public void position(long position) { + this.positionAddr = startAddr + position; + } + + @Override + public long capacity() { + return (limitAddr - startAddr); + } + + @Override + public long remaining() { + return (limitAddr - positionAddr); + } + + @NotNull + @Override + public ByteOrder byteOrder() { + return ByteOrder.nativeOrder(); + } + + @Override + public void checkEndOfBuffer() throws IndexOutOfBoundsException { + if (position() > capacity()) + throw new IndexOutOfBoundsException("position is beyond the end of the buffer " + position() + " > " + capacity()); + } + + public long startAddr() { + return startAddr; + } + + public long positionAddr() { + return positionAddr; + } + + public void positionAddr(long positionAddr) { + this.positionAddr = positionAddr; + } + + public long limitAddr() { + return limitAddr; + } +} diff --git a/lang/src/main/java/net/openhft/lang/io/RandomDataInput.java b/lang/src/main/java/net/openhft/lang/io/RandomDataInput.java new file mode 100755 index 0000000..3581b0f --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/io/RandomDataInput.java @@ -0,0 +1,868 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.io; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.ObjectInput; +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.Map; +import java.util.RandomAccess; + +/** + * @author peter.lawrey + */ +public interface RandomDataInput extends ObjectInput, RandomAccess, BytesCommon { + /** + * Reads some bytes from an input stream and stores them into the buffer array b. The number of bytes + * read is equal to the length of b. + *

+ * This method blocks until one of the following conditions occurs:

  • b.length bytes of input + * data are available, in which case a normal return is made. + *

    + *

  • End of file is detected, in which case an EOFException is thrown. + *

    + *

  • An I/O error occurs, in which case an IOException other than EOFException is + * thrown.
+ *

+ * If b is null, a NullPointerException is thrown. If b.length + * is zero, then no bytes are read. Otherwise, the first byte read is stored into element b[0], the + * next one into b[1], and so on. If an exception is thrown from this method, then it may be that some + * but not all bytes of b have been updated with data from the input stream. + * + * @param bytes the buffer into which the data is read. + */ + @Override + void readFully(@NotNull byte[] bytes); + + /** + * Reads len bytes from an input stream. + *

+ * This method blocks until one of the following conditions occurs:

  • len bytes of input data + * are available, in which case a normal return is made. + *

    + *

  • End of file is detected, in which case an EOFException is thrown. + *

    + *

  • An I/O error occurs, in which case an IOException other than EOFException is + * thrown.
+ *

+ * If b is null, a NullPointerException is thrown. If off is + * negative, or len is negative, or off+len is greater than the length of the array + * b, then an IndexOutOfBoundsException is thrown. If len is zero, then no + * bytes are read. Otherwise, the first byte read is stored into element b[off], the next one into + * b[off+1], and so on. The number of bytes read is, at most, equal to len. + * + * @param bytes the buffer into which the data is read. + * @param off an int specifying the offset into the data. + * @param len an int specifying the number of bytes to read. + */ + @Override + void readFully(@NotNull byte[] bytes, int off, int len); + + /** + * Makes an attempt to skip over n bytes of data from the input stream, discarding the skipped bytes. + * However, it may skip over some smaller number of bytes, possibly zero. This may result from any of a number of + * conditions; reaching end of file before n bytes have been skipped is only one possibility. This + * method never throws an EOFException. The actual number of bytes skipped is returned. + * + * @param n the number of bytes to be skipped. + * @return the number of bytes actually skipped. + */ + @Override + int skipBytes(int n); + + /** + * Reads one input byte and returns true if that byte is nonzero, false if that byte is + * zero. This method is suitable for reading the byte written by the writeBoolean method of interface + * DataOutput. + * + * @return the boolean value read. + */ + @Override + boolean readBoolean(); + + /** + * Reads one input byte and returns true if that byte is nonzero, false if that byte is + * zero. This method is suitable for reading the byte written by the writeBoolean method of interface + * RandomDataOutput. + * + * @param offset to read byte translated into a boolean + * @return the boolean value read. + */ + boolean readBoolean(long offset); + + /** + * Reads and returns one input byte. The byte is treated as a signed value in the range -128 through + * 127, inclusive. This method is suitable for reading the byte written by the writeByte + * method of interface DataOutput. + * + * @return the 8-bit value read. + */ + @Override + byte readByte(); + + /** + * Reads and returns one input byte. The byte is treated as a signed value in the range -128 through + * 127, inclusive. This method is suitable for reading the byte written by the writeByte + * method of interface RandomDataOutput. + * + * @param offset of byte to read. + * @return the 8-bit value read. + */ + byte readByte(long offset); + + /** + * Reads one input byte, zero-extends it to type int, and returns the result, which is therefore in the + * range 0 through 255. This method is suitable for reading the byte written by the + * writeByte method of interface DataOutput if the argument to writeByte was + * intended to be a value in the range 0 through 255. + * + * @return the unsigned 8-bit value read. + */ + @Override + int readUnsignedByte(); + + /** + * Reads one input byte, zero-extends it to type int, and returns the result, which is therefore in the + * range 0 through 255. This method is suitable for reading the byte written by the + * writeByte method of interface RandomDataOutput if the argument to + * writeByte was intended to be a value in the range 0 through 255. + * + * @param offset of byte to read + * @return the unsigned 8-bit value read. + */ + int readUnsignedByte(long offset); + + /** + * Reads two input bytes and returns a short value. Let a be the first byte read and + * b be the second byte on big endian machines, and the opposite on little endian machines. The value + * returned is: + * 

(short)((a << 8) | (b & 0xff))
+     *
+ * This method is suitable for reading the bytes written by the writeShort method of interface + * DataOutput. + * + * @return the 16-bit value read. + */ + @Override + short readShort(); + + /** + * Reads two input bytes and returns a short value. Let a be the first byte read and + * b be the second byte on big endian machines, and the opposite on little endian machines. The value + * returned is: + * 


+     *     (short)((a << 8) | (b & 0xff))
+     *
+ * This method is suitable for reading the bytes written by the writeShort method of interface + * RandomDataOutput. + * + * @param offset of short to read. + * @return the 16-bit value read. + */ + short readShort(long offset); + + /** + * Reads two input bytes and returns an int value in the range 0 through + * 65535. Let a be the first byte read and b be the second byte on big endian + * machines, and the opposite on little endian machines. The value returned is: + * 


+     *     (((a & 0xff) << 8) | (b & 0xff))
+     *
+ * This method is suitable for reading the bytes written by the writeUnsignedShort method of interface + * DataOutput if the argument to writeUnsignedShort was intended to be a value in the + * range 0 through 65535. + * + * @return the unsigned 16-bit value read. + */ + @Override + int readUnsignedShort(); + + /** + * Reads two input bytes and returns an int value in the range 0 through + * 65535. Let a be the first byte read and b be the second byte on big endian + * machines, and the opposite on little endian machines. The value returned is: + * 


+     *     (((a & 0xff) << 8) | (b & 0xff))
+     *
+ * This method is suitable for reading the bytes written by the writeShort method of interface + * RandomDataOutput if the argument to writeUnsignedShort was intended to be a value in + * the range 0 through 65535. + * + * @param offset of short to read. + * @return the unsigned 16-bit value read. + */ + int readUnsignedShort(long offset); + + /** + * Reads one or three input bytes and returns a short value. Let a be the first byte read. + * This mapped as follows; Byte.MIN_VALUE => Short.MIN_VALUE, Byte.MAX_VALUE => Short.MAX_VALUE, Byte.MIN_VALUE+2 to + * Byte.MAX_VALUE-1 => same as short value, Byte.MIN_VALUE+1 => readShort(). + *

+ * This method is suitable for reading the bytes written by the writeCompactShort method of interface + * RandomDataOutput. + * + * @return the 16-bit value read. + */ + short readCompactShort(); + + /** + * Reads one or three input bytes and returns a short value. Let a be the first byte read. + * This mapped as follows; -1 => readUnsignedShort(), default => (a & 0xFF) + *

+ * This method is suitable for reading the bytes written by the writeCompactUnsignedShort method of + * interface RandomDataOutput. + * + * @return the unsigned 16-bit value read. + */ + int readCompactUnsignedShort(); + + /** + * Reads two input bytes and returns a char value. Let a be the first byte read and + * b be the second byte on big endian machines, and the opposite on little endian machines. The value + * returned is: + * 

(char)((a << 8) | (b & 0xff))
+     *
+ * This method is suitable for reading bytes written by the writeChar method of interface + * DataOutput. + * + * @return the char value read. + */ + @Override + char readChar(); + + /** + * Reads two input bytes and returns a char value. Let a be the first byte read and + * b be the second byte on big endian machines, and the opposite on little endian machines. The value + * returned is: + * 

(char)((a << 8) | (b & 0xff))
+     *
+ * This method is suitable for reading bytes written by the writeChar method of interface + * RandomDataOutput. + * + * @param offset of the char to read. + * @return the char value read. + */ + char readChar(long offset); + + /** + * Reads three input bytes and returns a 24-bit int value. Let a-c be the first through + * third bytes read on big endian machines, and the opposite on little endian machines. The value returned is: + * 

+     * 
+     * ((((a & 0xff) << 24) | ((b & 0xff) << 16) | ((c & 0xff) << 8))) >> 8
+     *
+ * This method is suitable for reading bytes written by the writeInt24 method of interface + * RandomDataOutput. + * + * @return the int value read. + */ + int readInt24(); + + /** + * Reads three input bytes and returns a 24-bit int value. Let a-c be the first through + * third bytes read on big endian machines, and the opposite on little endian machines. The value returned is: + * 

+     * 
+     * ((((a & 0xff) << 24) | ((b & 0xff) << 16) | ((c & 0xff) << 8))) >> 8
+     *
+ * This method is suitable for reading bytes written by the writeInt24 method of interface + * RandomDataOutput. + * + * @param offset to read from + * @return the int value read. + */ + int readInt24(long offset); + + /** + * Reads four input bytes and returns an int value. Let a-d be the first through fourth + * bytes read on big endian machines, and the opposite on little endian machines. The value returned is: + * 

+     * 
+     * (((a & 0xff) << 24) | ((b & 0xff) << 16) | ((c & 0xff) << 8) | (d & 0xff))
+     *
+ * This method is suitable for reading bytes written by the writeInt method of interface + * DataOutput. + * + * @return the int value read. + */ + @Override + int readInt(); + + /** + * Reads four input bytes and returns an int value. Let a-d be the first through fourth + * bytes read on big endian machines, and the opposite on little endian machines. The value returned is: + * 

+     * 
+     * (((a & 0xff) << 24) | ((b & 0xff) << 16) | ((c & 0xff) << 8) | (d & 0xff))
+     *
+ * This method is suitable for reading bytes written by the writeInt method of interface + * RandomDataOutput. + * + * @param offset to read from + * @return the int value read. + */ + int readInt(long offset); + + /** + * This is the same as readInt() except a read barrier is performed first.

Reads four input bytes and returns + * an int value. Let a-d be the first through fourth bytes read on big endian machines, + * and the opposite on little endian machines. The value returned is: + * 

+     * 
+     * (((a & 0xff) << 24) | ((b & 0xff) << 16) | ((c & 0xff) << 8) | (d & 0xff))
+     *
+ * This method is suitable for reading bytes written by the writeOrderedInt or + * writeVolatileInt method of interface RandomDataOutput. + * + * @return the int value read. + */ + int readVolatileInt(); + + /** + * This is the same as readInt() except a read barrier is performed first.

Reads four input bytes and returns + * an int value. Let a-d be the first through fourth bytes read on big endian machines, + * and the opposite on little endian machines. The value returned is: + * 

+     * 
+     * (((a & 0xff) << 24) | ((b & 0xff) << 16) | ((c & 0xff) << 8) | (d & 0xff))
+     *
+ * This method is suitable for reading bytes written by the writeOrderedInt or + * writeVolatileInt method of interface RandomDataOutput. + * + * @param offset to read from + * @return the int value read. + */ + int readVolatileInt(long offset); + + /** + * Reads four input bytes and returns an int value. Let a-d be the first through fourth + * bytes read on big endian machines, and the opposite on little endian machines. The value returned is: + * 

+     * 
+     * ((((long) a & 0xff) << 24) | ((b & 0xff) << 16) | ((c & 0xff) << 8) | (d &
+     * 0xff))
+     *
+ * This method is suitable for reading bytes written by the writeUnsignedInt method of interface + * RandomDataOutput. + * + * @return the unsigned int value read. + */ + long readUnsignedInt(); + + /** + * Reads four input bytes and returns an int value. Let a-d be the first through fourth + * bytes read on big endian machines, and the opposite on little endian machines. The value returned is: + * 

+     * 
+     * ((((long) a & 0xff) << 24) | ((b & 0xff) << 16) | ((c & 0xff) << 8) | (d &
+     * 0xff))
+     *
+ * This method is suitable for reading bytes written by the writeUnsignedInt method of interface + * RandomDataOutput. + * + * @param offset to read from + * @return the unsigned int value read. + */ + long readUnsignedInt(long offset); + + /** + * Reads two or six input bytes and returns an int value. Let a be the first short read + * with readShort(). This mapped as follows; Short.MIN_VALUE => Integer.MIN_VALUE, Short.MAX_VALUE => + * Integer.MAX_VALUE, Short.MIN_VALUE+2 to Short.MAX_VALUE-1 => same as short value, Short.MIN_VALUE+1 => + * readInt(). + *

+ * This method is suitable for reading the bytes written by the writeCompactInt method of interface + * RandomDataOutput. + * + * @return the 32-bit value read. + */ + int readCompactInt(); + + /** + * Reads two or six input bytes and returns an int value. Let a be the first short read + * with readShort(). This mapped as follows; -1 => readUnsignedInt(), default => (a & 0xFFFF) + *

+ * This method is suitable for reading the bytes written by the writeCompactUnsignedInt method of + * interface RandomDataOutput. + * + * @return the unsigned 32-bit value read. + */ + long readCompactUnsignedInt(); + + /** + * Reads eight input bytes and returns a long value. Let a-h be the first through eighth + * bytes read on big endian machines, and the opposite on little endian machines. The value returned is: + * 

 
+     * (((long)(a & 0xff) << 56) |
+     *  ((long)(b & 0xff) << 48) |
+     *  ((long)(c & 0xff) << 40) |
+     *  ((long)(d & 0xff) << 32) |
+     *  ((long)(e & 0xff) << 24) |
+     *  ((long)(f & 0xff) << 16) |
+     *  ((long)(g & 0xff) <<  8) |
+     *  ((long)(h & 0xff)))
+     *
+ *

+ * This method is suitable for reading bytes written by the writeLong method of interface + * DataOutput. + * + * @return the long value read. + */ + @Override + long readLong(); + + /** + * Reads eight input bytes and returns a long value. Let a-h be the first through eighth + * bytes read on big endian machines, and the opposite on little endian machines. The value returned is: + * 

 
+     * (((long)(a & 0xff) << 56) |
+     *  ((long)(b & 0xff) << 48) |
+     *  ((long)(c & 0xff) << 40) |
+     *  ((long)(d & 0xff) << 32) |
+     *  ((long)(e & 0xff) << 24) |
+     *  ((long)(f & 0xff) << 16) |
+     *  ((long)(g & 0xff) <<  8) |
+     *  ((long)(h & 0xff)))
+     *
+ *

+ * This method is suitable for reading bytes written by the writeLong method of interface + * RandomDataOutput. + * + * @param offset of the long to read + * @return the long value read. + */ + long readLong(long offset); + + /** + * This is the same readLong() except a dread barrier is performed first + *

+ * Reads eight input bytes and returns a long value. Let a-h be the first through eighth + * bytes read on big endian machines, and the opposite on little endian machines. The value returned is: + * 

 
+     * (((long)(a & 0xff) << 56) |
+     *  ((long)(b & 0xff) << 48) |
+     *  ((long)(c & 0xff) << 40) |
+     *  ((long)(d & 0xff) << 32) |
+     *  ((long)(e & 0xff) << 24) |
+     *  ((long)(f & 0xff) << 16) |
+     *  ((long)(g & 0xff) <<  8) |
+     *  ((long)(h & 0xff)))
+     *
+ *

+ * This method is suitable for reading bytes written by the writeOrderedLong or + * writeVolatileLong method of interface RandomDataOutput. + * + * @return the long value read. + */ + long readVolatileLong(); + + /** + * This is the same readLong() except a dread barrier is performed first + *

+ * Reads eight input bytes and returns a long value. Let a-h be the first through eighth + * bytes read on big endian machines, and the opposite on little endian machines. The value returned is: + * 

 
+     * (((long)(a & 0xff) << 56) |
+     *  ((long)(b & 0xff) << 48) |
+     *  ((long)(c & 0xff) << 40) |
+     *  ((long)(d & 0xff) << 32) |
+     *  ((long)(e & 0xff) << 24) |
+     *  ((long)(f & 0xff) << 16) |
+     *  ((long)(g & 0xff) <<  8) |
+     *  ((long)(h & 0xff)))
+     *
+ *

+ * This method is suitable for reading bytes written by the writeOrderedLong or + * writeVolatileLong method of interface RandomDataOutput. + * + * @param offset of the long to read + * @return the long value read. + */ + long readVolatileLong(long offset); + + /** + * Reads six input bytes and returns a long value. Let a-f be the first through sixth + * bytes read on big endian machines, and the opposite on little endian machines. The value returned is: + * 

 
+     * (((long)(a & 0xff) << 56) |
+     *  ((long)(b & 0xff) << 48) |
+     *  ((long)(c & 0xff) << 40) |
+     *  ((long)(d & 0xff) << 32) |
+     *  ((long)(e & 0xff) << 24) |
+     *  ((long)(f & 0xff) << 16)) >> 16
+     *
+ *

+ * This method is suitable for reading bytes written by the writeInt48 method of interface + * RandomDataOutput. + * + * @return the long value read. + */ + long readInt48(); + + /** + * Reads six input bytes and returns a long value. Let a-f be the first through sixth + * bytes read on big endian machines, and the opposite on little endian machines. The value returned is: + * 

 
+     * (((long)(a & 0xff) << 56) |
+     *  ((long)(b & 0xff) << 48) |
+     *  ((long)(c & 0xff) << 40) |
+     *  ((long)(d & 0xff) << 32) |
+     *  ((long)(e & 0xff) << 24) |
+     *  ((long)(f & 0xff) << 16)) >> 16
+     *
+ *

+ * This method is suitable for reading bytes written by the writeInt48 method of interface + * RandomDataOutput. + * + * @param offset of the long to read + * @return the long value read. + */ + long readInt48(long offset); + + /** + * Reads four or twelve input bytes and returns a long value. Let a be the first int read + * with readInt(). This mapped as follows; Integer.MIN_VALUE => Long.MIN_VALUE, Integer.MAX_VALUE => Long.MAX_VALUE, + * Integer.MIN_VALUE+2 to Integer.MAX_VALUE-1 => same as short value, Integer.MIN_VALUE+1 => readLong(). + *

+ * This method is suitable for reading the bytes written by the writeCompactLong method of interface + * RandomDataOutput. + * + * @return the 64-bit value read. + */ + long readCompactLong(); + + /** + * Reads between one and ten bytes with are stop encoded with support for negative numbers + * 


+     * long l = 0, b;
+     * int count = 0;
+     * while ((b = readByte()) < 0) {
+     *     l |= (b & 0x7FL) << count;
+     *     count += 7;
+     * }
+     * if (b == 0 && count > 0)
+     *     return ~l;
+     * return l | (b << count);
+     *
+ * + * @return a stop bit encoded number as a long. + */ + long readStopBit(); + + /** + * Reads four input bytes and returns a float value. It does this by first constructing an + * int value in exactly the manner of the readInt method, then converting this + * int value to a float in exactly the manner of the method + * Float.intBitsToFloat. This method is suitable for reading bytes written by the + * writeFloat method of interface DataOutput. + * + * @return the float value read. + */ + @Override + float readFloat(); + + /** + * Reads four input bytes and returns a float value. It does this by first constructing an + * int value in exactly the manner of the readInt method, then converting this + * int value to a float in exactly the manner of the method + * Float.intBitsToFloat. This method is suitable for reading bytes written by the + * writeFloat method of interface DataOutput. + * + * @param offset to read from + * @return the float value read. + */ + float readFloat(long offset); + + /** + * This is the same as readFloat() except a read barrier is performed first.

Reads four input bytes and returns + * a float value. + *

+ * This method is suitable for reading bytes written by the writeOrderedFloat method of interface RandomDataOutput. + * + * @param offset to read from + * @return the int value read. + */ + float readVolatileFloat(long offset); + + /** + * Reads eight input bytes and returns a double value. It does this by first constructing a + * long value in exactly the manner of the readLong method, then converting this + * long value to a double in exactly the manner of the method + * Double.longBitsToDouble. This method is suitable for reading bytes written by the + * writeDouble method of interface DataOutput. + * + * @return the double value read. + */ + @Override + double readDouble(); + + /** + * Reads eight input bytes and returns a double value. It does this by first constructing a + * long value in exactly the manner of the readLong method, then converting this + * long value to a double in exactly the manner of the method + * Double.longBitsToDouble. This method is suitable for reading bytes written by the + * writeDouble method of interface DataOutput. + * + * @param offset to read from + * @return the double value read. + */ + double readDouble(long offset); + + /** + * Reads the first four bytes as readFloat(). If this is Float.NaN, the next eight bytes are read as readDouble() + * + * @return the double value read. + */ + double readCompactDouble(); + + /** + * This is the same as readDouble() except a read barrier is performed first.

Reads four input bytes and returns + * a float value. + *

+ * This method is suitable for reading bytes written by the writeOrderedFloat method of interface RandomDataOutput. + * + * @param offset to read from + * @return the int value read. + */ + double readVolatileDouble(long offset); + + /** + * Reads the next line of text from the input stream. It reads successive bytes, converting each byte separately + * into a character, until it encounters a line terminator or end of file; the characters read are then returned as + * a String. Note that because this method processes bytes, it does not support input of the full + * Unicode character set. + *

+ * If end of file is encountered before even one byte can be read, then null is returned. Otherwise, + * each byte that is read is converted to type char by zero-extension. If the character + * '\n' is encountered, it is discarded and reading ceases. If the character '\r' is + * encountered, it is discarded and, if the following byte converts to the character '\n', then + * that is discarded also; reading then ceases. If end of file is encountered before either of the characters + * '\n' and '\r' is encountered, reading ceases. Once reading has ceased, a + * String is returned that contains all the characters read and not discarded, taken in order. Note + * that every character in this string will have a value less than \u0100, that is, + * (char)256. + * + * @return the next line of text from the input stream, or null if the end of file is encountered + * before a byte can be read. + */ + @Override + @Nullable + String readLine(); + + /** + * Reads in a string that has been encoded using a modified UTF-8 format. The general + * contract of readUTF is that it reads a representation of a Unicode character string encoded in + * modified UTF-8 format; this string of characters is then returned as a String. + *

+ * First, two bytes are read and used to construct an unsigned 16-bit integer in exactly the manner of the + * readUnsignedShort method . This integer value is called the UTF length and specifies the + * number of additional bytes to be read. These bytes are then converted to characters by considering them in + * groups. The length of each group is computed from the value of the first byte of the group. The byte following a + * group, if any, is the first byte of the next group. + *

+ * If the first byte of a group matches the bit pattern 0xxxxxxx (where x means "may be + * 0 or 1"), then the group consists of just that byte. The byte is zero-extended to form + * a character. + *

+ * If the first byte of a group matches the bit pattern 110xxxxx, then the group consists of that byte + * a and a second byte b. If there is no byte b (because byte a + * was the last of the bytes to be read), or if byte b does not match the bit pattern + * 10xxxxxx, then a UTFDataFormatException is thrown. Otherwise, the group is converted to + * the character:

+ * 

(char)(((a& 0x1F) << 6) | (b & 0x3F))
+     *
+ * If the first byte of a group matches the bit pattern 1110xxxx, then the group consists of that byte + * a and two more bytes b and c. If there is no byte c (because + * byte a was one of the last two of the bytes to be read), or either byte b or byte + * c does not match the bit pattern 10xxxxxx, then a UTFDataFormatException + * is thrown. Otherwise, the group is converted to the character:

+ * 


+     * (char)(((a & 0x0F) << 12) | ((b & 0x3F) << 6) | (c & 0x3F))
+     *
+ * If the first byte of a group matches the pattern 1111xxxx or the pattern 10xxxxxx, then + * a UTFDataFormatException is thrown. + *

+ * If end of file is encountered at any time during this entire process, then an EOFException is + * thrown. + *

+ * After every group has been converted to a character by this process, the characters are gathered, in the same + * order in which their corresponding groups were read from the input stream, to form a String, which + * is returned. + *

+ * The writeUTF method of interface DataOutput may be used to write data that is suitable + * for reading by this method. + * + * @return a Unicode string. + * @throws IllegalStateException if the bytes do not represent a valid modified UTF-8 encoding of a string. + */ + @Override + @NotNull + String readUTF(); + + /** + * The same as readUTF() except the length is stop bit encoded. This saves one byte for strings shorter than 128 + * chars. null values are also supported + * + * @return a Unicode string or null if writeUTFΔ(null) was called + */ + @Nullable + String readUTFΔ(); + + /** + * The same as readUTFΔ() except an offset and maximum length is given. + * + * @param offset to read from + * @return a Unicode string or null if writeUTFΔ(null) was called + * @throws IllegalStateException if the length to be read is out of range. + */ + @Nullable + String readUTFΔ(long offset) throws IllegalStateException; + + /** + * The same as readUTFΔ() except the chars are copied to a truncated StringBuilder. + * + * @return true if there was a String, or false if it was null + */ + boolean readUTFΔ(@NotNull StringBuilder stringBuilder); + + /** + * Copy bytes into a ByteBuffer to the minimum of the length remaining() in the ByteBuffer or the + * Excerpt. + * + * @param bb to copy into + */ + void read(@NotNull ByteBuffer bb); + + /** + * Read a String with readUTFΔ which is converted to an enumerable type. i.e. where there is a one to + * one mapping between an object and it's toString(). + *

+ * This is suitable to read an object written using writeEnum() in the RandomDataOutput + * interface + * + * @param eClass to decode the String as + * @return the decoded value. null with be return if null was written. + */ + @Nullable + E readEnum(@NotNull Class eClass); + + /** + * Read a stop bit encoded length and populates this Collection after clear()ing it. + *

+ * This is suitable to reading a list written using writeList() in the RandomDataOutput + * interface + * + * @param list to populate + */ + void readList(@NotNull Collection list, @NotNull Class eClass); + + /** + * Read a stop bit encoded length and populates this Map after clear()ing it. + *

+ * This is suitable to reading a list written using writeMap() in the RandomDataOutput + * interface + * + * @param map to populate + * @return the map + */ + Map readMap(@NotNull Map map, @NotNull Class kClass, @NotNull Class vClass); + + // ObjectInput + + /** + * Read and return an object. The class that implements this interface defines where the object is "read" from. + * + * @return the object read from the stream + * @throws IllegalStateException the class of a serialized object cannot be found or any of the usual Input/Output + * related exceptions occur. + */ + @Nullable + @Override + Object readObject() throws IllegalStateException; + + /** + * Read and return an object. The class that implements this interface defines where the object is "read" from. + * + * @return the object read from the stream + * @throws IllegalStateException the class of a serialized object cannot be found or any of the usual Input/Output + * related exceptions occur. + * @throws ClassCastException The class cannot be cast or converted to the type given. + */ + @Nullable + T readObject(Class tClass) throws IllegalStateException; + + /** + * Reads a byte of data. This method will block if no input is available. + * + * @return the byte read, or -1 if the end of the stream is reached. + */ + @Override + int read(); + + /** + * Reads into an array of bytes. This method will block until some input is available. + * + * @param bytes the buffer into which the data is read + * @return the actual number of bytes read, -1 is returned when the end of the stream is reached. + */ + @Override + int read(@NotNull byte[] bytes); + + /** + * Reads into an array of bytes. This method will block until some input is available. + * + * @param bytes the buffer into which the data is read + * @param off the start offset of the data + * @param len the maximum number of bytes read + * @return the actual number of bytes read, -1 is returned when the end of the stream is reached. + */ + @Override + int read(@NotNull byte[] bytes, int off, int len); + + /** + * Read the object from start to end bytes + * + * @param object to read into + * @param start byte inclusive + * @param end byte exclusive + */ + void readObject(Object object, int start, int end); + + /** + * Skips n bytes of input. + * + * @param n the number of bytes to be skipped + * @return the actual number of bytes skipped. + */ + @Override + long skip(long n); + + /** + * @return remaining() or Integer.MAX_VALUE if larger. + */ + @Override + int available(); + + /** + * Finishes the excerpt entry if not finished() + */ + @Override + void close(); +} diff --git a/lang/src/main/java/net/openhft/lang/io/RandomDataOutput.java b/lang/src/main/java/net/openhft/lang/io/RandomDataOutput.java new file mode 100755 index 0000000..92a1a4c --- /dev/null +++ b/lang/src/main/java/net/openhft/lang/io/RandomDataOutput.java @@ -0,0 +1,819 @@ +/* + * Copyright 2013 Peter Lawrey + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.openhft.lang.io; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.ObjectOutput; +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.Map; +import java.util.RandomAccess; + +/** + * @author peter.lawrey + */ +public interface RandomDataOutput extends ObjectOutput, RandomAccess, BytesCommon { + /** + * Copy from one Bytes to another. Copied from the start to the current position. + * + * @param bytes to copy from + * @deprecated Use write(BytesCommon bytes, long position, long length) instead. + */ + @Deprecated + void writeStartToPosition(Bytes bytes); + + /** + * Writes to the output stream the eight low-order bits of the argument b. The 24 high-order bits of + * b are ignored. + * + * @param b the byte to be written. + */ + @Override + void write(int b); + + /** + * Writes to the output stream the eight low- order bits of the argument v. The 24 high-order bits of + * v are ignored. (This means that writeByte does exactly the same thing as + * write for an integer argument.) The byte written by this method may be read by the + * readByte method of interface DataInput, which will then return a byte + * equal to (byte)v. + * + * @param v the byte value to be written. + */ + @Override + void writeByte(int v); + + /** + * Writes to the output stream the eight low- order bits of the argument v. The 24 high-order bits of + * v are ignored. (This means that writeByte does exactly the same thing as + * write for an integer argument.) The byte written by this method may be read by the + * readUnsignedByte method of interface DataInput, which will then return a + * byte equal to (byte)v. + * + * @param v the byte value to be written. + */ + void writeUnsignedByte(int v); + + /** + * Writes to the output stream the eight low-order bits of the argument b. The 24 high-order bits of + * b are ignored. + * + * @param offset to write byte + * @param b the byte to be written. + */ + void writeByte(long offset, int b); + + /** + * Writes to the output stream the eight low- order bits of the argument v. The 24 high-order bits of + * v are ignored. (This means that writeByte does exactly the same thing as + * write for an integer argument.) The byte written by this method may be read by the + * readUnsignedByte method of interface DataInput, which will then return a + * byte equal to v & 0xFF. + * + * @param offset to write byte + * @param v the unsigned byte value to be written. + */ + void writeUnsignedByte(long offset, int v); + + /** + * Writes to the output stream all the bytes in array bytes. If bytes is + * null, a NullPointerException is thrown. If bytes.length is zero, then no + * bytes are written. Otherwise, the byte bytes[0] is written first, then bytes[1], and so + * on; the last byte written is bytes[bytes.length-1]. + * + * @param bytes the data. + */ + @Override + void write(byte[] bytes); + + /** + * Writes to the output stream all the bytes in array bytes. If bytes is + * null, a NullPointerException is thrown. If bytes.length is zero, then no + * bytes are written. Otherwise, the byte bytes[0] is written first, then bytes[1], and so + * on; the last byte written is bytes[bytes.length-1]. + * + * @param offset to be written + * @param bytes the data. + */ + void write(long offset, byte[] bytes); + + /** + * Writes len bytes from array bytes, in order, to the output stream. If + * bytes is null, a NullPointerException is thrown. If off is + * negative, or len is negative, or off+len is greater than the length of the array + * bytes, then an IndexOutOfBoundsException is thrown. If len is zero, then + * no bytes are written. Otherwise, the byte bytes[off] is written first, then + * bytes[off+1], and so on; the last byte written is bytes[off+len-1]. + * + * @param bytes the data. + * @param off the start offset in the data. + * @param len the number of bytes to write. + */ + @Override + void write(byte[] bytes, int off, int len); + + /** + * Writes a boolean value to this output stream. If the argument v is true, + * the value (byte)1 is written; if v is false, the value + * (byte)0 is written. The byte written by this method may be read by the readBoolean + * method of interface DataInput, which will then return a boolean equal to + * v. + * + * @param v the boolean to be written. + */ + @Override + void writeBoolean(boolean v); + + /** + * Writes a boolean value to this output stream. If the argument v is true, + * the value (byte)1 is written; if v is false, the value + * (byte)0 is written. The byte written by this method may be read by the readBoolean + * method of interface DataInput, which will then return a boolean equal to + * v. + * + * @param offset to write boolean + * @param v the boolean to be written. + */ + + void writeBoolean(long offset, boolean v); + + /** + * Writes two bytes to the output stream to represent the value of the argument. The byte values to be written, in + * the order shown for big endian machines and the opposite for little endian, are:

+ * 


+     * (byte)(0xff & (v >> 8))
+     * (byte)(0xff & v)
+     *

+ * The bytes written by this method may be read by the readShort method of interface + * DataInput , which will then return a short equal to (short)v. + * + * @param v the short value to be written. + */ + @Override + void writeShort(int v); + + /** + * Writes two bytes to the output stream to represent the value of the argument. The byte values to be written, in + * the order shown for big endian machines and the opposite for little endian, are:

+ * 


+     * (byte)(0xff & (v >> 8))
+     * (byte)(0xff & v)
+     *

+ * The bytes written by this method may be read by the readShort method of interface + * DataInput , which will then return a short equal to (short)v. + * + * @param offset to be written to + * @param v the short value to be written. + */ + void writeShort(long offset, int v); + + /** + * Writes two bytes to the output stream to represent the value of the argument. The byte values to be written, in + * the order shown for big endian machines and the opposite for little endian, are:

+ * 


+     * (byte)(0xff & (v >> 8))
+     * (byte)(0xff & v)
+     *

+ * The bytes written by this method may be read by the readUnsignedShort method of interface + * RandomDataInput , which will then return a short equal to (short)v. + * + * @param v the unsigned short value to be written. + */ + void writeUnsignedShort(int v); + + /** + * Writes two bytes to the output stream to represent the value of the argument. The byte values to be written, in + * the order shown for big endian machines and the opposite for little endian, are:

+ * 


+     * (byte)(0xff & (v >> 8))
+     * (byte)(0xff & v)
+     *

+ * The bytes written by this method may be read by the readShort method of interface + * RandomDataInput , which will then return a short equal to (short)v. + * + * @param offset to be written to + * @param v the unsigned short value to be written. + */ + void writeUnsignedShort(long offset, int v); + + /** + * Writes one or three bytes as follows; Short.MIN_VALUE => Byte.MIN_VALUE, Short.MAX_VALUE => Byte.MAX_VALUE, + * Short.MIN_VALUE+2 to Short.MAX_VALUE-1 => writeByte(x), default => writeByte(Byte.MIN_VALUE+1; + * writeShort(x) + *

+ * The bytes written by this method may be read by the readCompactShort method of interface + * RandomDataInput , which will then return a short equal to (short)v. + * + * @param v the short value to be written. + */ + void writeCompactShort(int v); + + /** + * Writes one or three bytes as follows; 0 to 254 => writeByte(x); otherwise writeByte(255); writeByteShort(x); + *

+ * The bytes written by this method may be read by the readCompactUnsignedShort method of interface + * RandomDataInput , which will then return a short equal to v & 0xFFFF. + * + * @param v the unsigned short value to be written. + */ + void writeCompactUnsignedShort(int v); + + /** + * Writes a char value, which is comprised of two bytes, to the output stream. The byte values to be + * written, in the order shown for big endian machines and the opposite for little endian, are: + * 


+     * (byte)(0xff & (v >> 8))
+     * (byte)(0xff & v)
+     *

+ * The bytes written by this method may be read by the readChar method of interface + * DataInput , which will then return a char equal to (char)v. + * + * @param v the char value to be written. + */ + @Override + void writeChar(int v); + + /** + * Writes a char value, which is comprised of two bytes, to the output stream. The byte values to be + * written, in the order shown for big endian machines and the opposite for little endian, are: + * 


+     * (byte)(0xff & (v >> 8))
+     * (byte)(0xff & v)
+     *

+ * The bytes written by this method may be read by the readChar method of interface + * DataInput , which will then return a char equal to (char)v. + * + * @param offset to be written to + * @param v the char value to be written. + */ + void writeChar(long offset, int v); + + /** + * Writes an int value, which is comprised of three bytes, to the output stream. The byte values to be + * written, in the order shown for big endian machines and the opposite for little endian, are: + * 


+     * (byte)(0xff & (v >> 16))
+     * (byte)(0xff & (v >>    8))
+     * (byte)(0xff & v)
+     *

+ * The bytes written by this method may be read by the readInt24 method of interface + * RandomDataInput , which will then return an int equal to v. + * + * @param v the int value to be written. + */ + void writeInt24(int v); + + /** + * Writes an int value, which is comprised of three bytes, to the output stream. The byte values to be + * written, in the order shown for big endian machines and the opposite for little endian, are: + * 


+     * (byte)(0xff & (v >> 16))
+     * (byte)(0xff & (v >>    8))
+     * (byte)(0xff & v)
+     *

+ * The bytes written by this method may be read by the readInt24 method of interface + * RandomDataInput , which will then return an int equal to v. + * + * @param offset to be written to + * @param v the int value to be written. + */ + void writeInt24(long offset, int v); + + /** + * Writes an int value, which is comprised of four bytes, to the output stream. The byte values to be + * written, in the order shown for big endian machines and the opposite for little endian, are: + * 


+     * (byte)(0xff & (v >> 24))
+     * (byte)(0xff & (v >> 16))
+     * (byte)(0xff & (v >>    8))
+     * (byte)(0xff & v)
+     *

+ * The bytes written by this method may be read by the readInt method of interface + * DataInput , which will then return an int equal to v. + * + * @param v the int value to be written. + */ + @Override + void writeInt(int v); + + /** + * Writes an int value, which is comprised of four bytes, to the output stream. The byte values to be + * written, in the order shown for big endian machines and the opposite for little endian, are: + * 


+     * (byte)(0xff & (v >> 24))
+     * (byte)(0xff & (v >> 16))
+     * (byte)(0xff & (v >>    8))
+     * (byte)(0xff & v)
+     *

+ * The bytes written by this method may be read by the readInt method of interface + * DataInput , which will then return an int equal to v. + * + * @param offset to be written to + * @param v the int value to be written. + */ + void writeInt(long offset, int v); + + /** + * Writes an int value, which is comprised of four bytes, to the output stream. The byte values to be + * written, in the order shown for big endian machines and the opposite for little endian, are: + * 


+     * (byte)(0xff & (v >> 24))
+     * (byte)(0xff & (v >> 16))
+     * (byte)(0xff & (v >>    8))
+     * (byte)(0xff & v)
+     *

+ * The bytes written by this method may be read by the readUnsignedInt method of interface + * RandomDataInput , which will then return an long equal to v & + * 0xFFFFFFFF. + * + * @param v the int value to be written. + */ + void writeUnsignedInt(long v); + + /** + * Writes an int value, which is comprised of four bytes, to the output stream. The byte values to be + * written, in the order shown for big endian machines and the opposite for little endian, are: + * 


+     * (byte)(0xff & (v >> 24))
+     * (byte)(0xff & (v >> 16))
+     * (byte)(0xff & (v >>    8))
+     * (byte)(0xff & v)
+     *

+ * The bytes written by this method may be read by the readUnsignedInt method of interface + * RandomDataInput , which will then return an long equal to v & + * 0xFFFFFFFF. + * + * @param offset to be written to + * @param v the int value to be written. + */ + void writeUnsignedInt(long offset, long v); + + /** + * Writes two or six bytes as follows; Integer.MIN_VALUE => Short.MIN_VALUE, Integer.MAX_VALUE => + * Short.MAX_VALUE, Short.MIN_VALUE+2 to Short.MAX_VALUE-1 => writeShort(x), default => + * writeShort(Short.MIN_VALUE+1; writeInt(x) + *

+ * The bytes written by this method may be read by the readCompactInt method of interface + * RandomDataInput , which will then return a int equal to v. + * + * @param v the int value to be written. + */ + void writeCompactInt(int v); + + /** + * Writes two or six bytes as follows; 0 to (1 << 16) - 2 => writeInt(x), otherwise writeShort(-1); + * writeInt(x) + *

+ * The bytes written by this method may be read by the readCompactUnsignedInt method of interface + * RandomDataInput , which will then return a int equal to v & + * 0xFFFFFFFF. + * + * @param v the short value to be written. + */ + void writeCompactUnsignedInt(long v); + + /** + * Same as writeInt but include an ordered write barrier. This means all writes will be visible on a read barrier + * if this write is visible. This might not be visible to be same thread for some clock cycles so an immediate read + * could see an old value + *

+ * This is much faster than a volatile write which stalls the pipeline. The data is visible to other threads at the + * same time. + * + * @param v value to write + */ + void writeOrderedInt(int v); + + /** + * Same as writeInt but include an ordered write barrier. This means all writes will be visible on a read barrier + * if this write is visible. This might not be visible to be same thread for some clock cycles so an immediate read + * could see an old value + *

+ * This is much faster than writeVolatileInt as the volatile write stalls the pipeline. The data is + * visible to other threads at the same time. + * + * @param offset to write to + * @param v value to write + */ + void writeOrderedInt(long offset, int v); + + /** + * Perform a compare and set operation. The value is set to x provided the expected value + * is set already. This operation is atomic. + * + * @param offset to write to. + * @param expected to expect + * @param x to set if expected was found + * @return true if set, false if the value was not expected + */ + boolean compareAndSwapInt(long offset, int expected, int x); + + /** + * Atomically adds the given value to the current value. + * + * @param offset of the int value to use. + * @param delta the value to add + * @return the previous value + */ + int getAndAdd(long offset, int delta); + + /** + * Atomically adds the given value to the current value. + * + * @param offset of the int value to use. + * @param delta the value to add + * @return the updated value + */ + int addAndGetInt(long offset, int delta); + + /** + * Writes a long value, which is comprised of eight bytes, to the output stream. The byte values to be + * written, in the order shown for big endian machines and the opposite for little endian, are: + * 


+     * (byte)(0xff & (v >> 40))
+     * (byte)(0xff & (v >> 32))
+     * (byte)(0xff & (v >> 24))
+     * (byte)(0xff & (v >> 16))
+     * (byte)(0xff & (v >>  8))
+     * (byte)(0xff & v)
+     *

+ * The bytes written by this method may be read by the readInt48 method of interface + * RandomDataInput , which will then return a long equal to v & ((1L << 48) + * - 1). + * + * @param v the long value to be written. + */ + void writeInt48(long v); + + /** + * Writes a long value, which is comprised of eight bytes, to the output stream. The byte values to be + * written, in the order shown for big endian machines and the opposite for little endian, are: + * 


+     * (byte)(0xff & (v >> 40))
+     * (byte)(0xff & (v >> 32))
+     * (byte)(0xff & (v >> 24))
+     * (byte)(0xff & (v >> 16))
+     * (byte)(0xff & (v >>  8))
+     * (byte)(0xff & v)
+     *

+ * The bytes written by this method may be read by the readInt48 method of interface + * RandomDataInput , which will then return a long equal to v & ((1L << 48) + * - 1). + * + * @param offset to be written to + * @param v the long value to be written. + */ + void writeInt48(long offset, long v); + + /** + * Writes a long value, which is comprised of eight bytes, to the output stream. The byte values to be + * written, in the order shown for big endian machines and the opposite for little endian, are: + * 


+     * (byte)(0xff & (v >> 56))
+     * (byte)(0xff & (v >> 48))
+     * (byte)(0xff & (v >> 40))
+     * (byte)(0xff & (v >> 32))
+     * (byte)(0xff & (v >> 24))
+     * (byte)(0xff & (v >> 16))
+     * (byte)(0xff & (v >>  8))
+     * (byte)(0xff & v)
+     *

+ * The bytes written by this method may be read by the readLong method of interface + * DataInput , which will then return a long equal to v. + * + * @param v the long value to be written. + */ + @Override + void writeLong(long v); + + /** + * Writes a long value, which is comprised of eight bytes, to the output stream. The byte values to be + * written, in the order shown for big endian machines and the opposite for little endian, are: + * 


+     * (byte)(0xff & (v >> 56))
+     * (byte)(0xff & (v >> 48))
+     * (byte)(0xff & (v >> 40))
+     * (byte)(0xff & (v >> 32))
+     * (byte)(0xff & (v >> 24))
+     * (byte)(0xff & (v >> 16))
+     * (byte)(0xff & (v >>  8))
+     * (byte)(0xff & v)
+     *

+ * The bytes written by this method may be read by the readLong method of interface + * DataInput , which will then return a long equal to v. + * + * @param offset to be written to + * @param v the long value to be written. + */ + void writeLong(long offset, long v); + + /** + * Writes four or twelve bytes as follows Long.MIN_VALUE => Integer.MIN_VALUE, Long.MAX_VALUE => + * Integer.MAX_VALUE, Integer.MIN_VALUE+2 to Integer.MAX_VALUE-1 => writeInt(x), default => + * writeInt(Integer.MIN_VALUE+1; writeLong(x) + *

+ * The bytes written by this method may be read by the readCompactLong method of interface + * RandomDataInput , which will then return a long equal to v. + * + * @param v the long value to be written. + */ + void writeCompactLong(long v); + + /** + * Same as writeLong but include an ordered write barrier. This means all writes will be visible on a read barrier + * if this write is visible. This might not be visible to be same thread for some clock cycles so an immediate read + * could see an old value + *

+ * This is much faster than a volatile write which stalls the pipeline. The data is visible to other threads at the + * same time. + * + * @param v value to write + */ + void writeOrderedLong(long v); + + /** + * Same as writeLong but include an ordered write barrier. This means all writes will be visible on a read barrier + * if this write is visible. This might not be visible to be same thread for some clock cycles so an immediate read + * could see an old value + *

+ * This is much faster than a volatile write which stalls the pipeline. The data is visible to other threads at the + * same time. + * + * @param offset to be written to + * @param v value to write + */ + void writeOrderedLong(long offset, long v); + + /** + * Perform a compare and set operation. The value is set to x provided the expected value + * is set already. This operation is atomic. + * + * @param offset to write to. + * @param expected to expect + * @param x to set if expected was found + * @return true if set, false if the value was not expected + */ + boolean compareAndSwapLong(long offset, long expected, long x); + + /** + * Stop bit encoding numbers. This will write the same number of bytes whether you used a byte, short or int. + */ + void writeStopBit(long n); + + /** + * Writes a float value, which is comprised of four bytes, to the output stream. It does this as if it + * first converts this float value to an int in exactly the manner of the + * Float.floatToIntBits method and then writes the int value in exactly the manner of the + * writeInt method. The bytes written by this method may be read by the readFloat method + * of interface DataInput, which will then return a float equal to v. + * + * @param v the float value to be written. + */ + @Override + void writeFloat(float v); + + /** + * Writes a float value, which is comprised of four bytes, to the output stream. It does this as if it + * first converts this float value to an int in exactly the manner of the + * Float.floatToIntBits method and then writes the int value in exactly the manner of the + * writeInt method. The bytes written by this method may be read by the readFloat method + * of interface DataInput, which will then return a float equal to v. + * + * @param offset to write to + * @param v the float value to be written. + */ + void writeFloat(long offset, float v); + + /** + * Same as writeFloat but include an ordered write barrier. This means all writes will be visible on a read barrier + * if this write is visible. This might not be visible to be same thread for some clock cycles so an immediate read + * could see an old value + *

+ * This is much faster than a volatile write which stalls the pipeline. The data is visible to other threads at the + * same time. + * + * @param v value to write + */ + void writeOrderedFloat(long offset, float v); + + /** + * Writes a double value, which is comprised of eight bytes, to the output stream. It does this as if + * it first converts this double value to a long in exactly the manner of the + * Double.doubleToLongBits method and then writes the long value in exactly the manner of + * the writeLong method. The bytes written by this method may be read by the readDouble + * method of interface DataInput, which will then return a double equal to + * v. + * + * @param v the double value to be written. + */ + @Override + void writeDouble(double v); + + /** + * Writes a double value, which is comprised of eight bytes, to the output stream. It does this as if + * it first converts this double value to a long in exactly the manner of the + * Double.doubleToLongBits method and then writes the long value in exactly the manner of + * the writeLong method. The bytes written by this method may be read by the readDouble + * method of interface DataInput, which will then return a double equal to + * v. + * + * @param offset to write to + * @param v the double value to be written. + */ + void writeDouble(long offset, double v); + + /** + * Writes four or twelve bytes as follow; + * 


+     * if ((float) d == d) {
+     *     writeFloat((float) d);
+     * } else {
+     *     writeFloat(Float.NaN);
+     *     writeDouble(d);
+     * }
+     * 

+     * The bytes written by this method may be read by the readCompactDouble method of interface
+     * RandomDataInput , which will then return a double equal to v.
+     *
+     * @param v the double value to be written.
+     */
+    void writeCompactDouble(double v);
+
+    /**
+     * Same as writeDouble but include an ordered write barrier.  This means all writes will be visible on a read
+     * barrier if this write is visible. This might not be visible to be same thread for some clock cycles so an
+     * immediate read could see an old value
+     * 

+     * This is much faster than a volatile write which stalls the pipeline.  The data is visible to other threads at the
+     * same time.
+     *
+     * @param v value to write
+     */
+    void writeOrderedDouble(long offset, double v);
+
+    /**
+     * Writes a string to the output stream. For every character in the string s,  taken in order, one byte
+     * is written to the output stream.  If s is null, a NullPointerException is
+     * thrown.
  If s.length is zero, then no bytes are written. Otherwise, the character
+     * s[0] is written first, then s[1], and so on; the last character written is
+     * s[s.length-1]. For each character, one byte is written, the low-order byte, in exactly the manner of
+     * the writeByte method . The high-order eight bits of each character in the string are ignored.
+     *
+     * @param s the string of bytes to be written. Cannot be null.
+     */
+    @Override
+    void writeBytes(@NotNull String s);
+
+    /**
+     * Writes every character in the string s, to the output stream, in order, two bytes per character. If
+     * s is null, a NullPointerException is thrown.  If s.length is
+     * zero, then no characters are written. Otherwise, the character s[0] is written first, then
+     * s[1], and so on; the last character written is s[s.length-1]. For each character, two
+     * bytes are actually written, high-order byte first, in exactly the manner of the writeChar method.
+     *
+     * @param s the string value to be written. Cannot be null.
+     */
+    @Override
+    void writeChars(@NotNull String s);
+
+    /**
+     * Writes two bytes of length information to the output stream, followed by the modified UTF-8 representation of  every character in the string
+     * s. If s is null, a NullPointerException is thrown. Each
+     * character in the string s is converted to a group of one, two, or three bytes, depending on the
+     * value of the character.
 If a character c is in the range \u0001 through
+     * \u007f, it is represented by one byte:

+     * 
(byte)c 
  

+     * If a character c is \u0000 or is in the range \u0080 through
+     * \u07ff, then it is represented by two bytes, to be written
+     * in the order shown:
 

+     * (byte)(0xc0 | (0x1f & (c >> 6)))
+     * (byte)(0x80 | (0x3f & c))
+     *  
  
 If a character
+     * c is in the range \u0800 through uffff, then it is represented by
+     * three bytes, to be written
+     * in the order shown:
 

+     * (byte)(0xe0 | (0x0f & (c >> 12)))
+     * (byte)(0x80 | (0x3f & (c >>  6)))
+     * (byte)(0x80 | (0x3f & c))
+     *  
  
 First,
+     * the total number of bytes needed to represent all the characters of s is calculated. If this number
+     * is larger than 65535, then a UTFDataFormatException is thrown. Otherwise, this length
+     * is written to the output stream in exactly the manner of the writeShort method; after this, the
+     * one-, two-, or three-byte representation of each character in the string s is written.
  The bytes
+     * written by this method may be read by the readUTF method of interface DataInput , which
+     * will then return a String equal to s.
+     *
+     * @param s the string value to be written. Cannot be null
+     */
+    @Override
+    void writeUTF(@NotNull String s);
+
+    /**
+     * Write the same encoding as writeUTF with the following changes.  1) The length is stop bit encoded
+     * i.e. one byte longer for short strings, but is not limited in length. 2) The string can be null.
+     *
+     * @param s the string value to be written. Can be null.
+     */
+    void writeUTFΔ(@Nullable CharSequence s);
+
+
+    /**
+     * Write the same encoding as writeUTF with the following changes.  1) The length is stop bit encoded
+     * i.e. one byte longer for short strings, but is not limited in length. 2) The string can be null.
+     *
+     * @param offset  to write to
+     * @param maxSize maximum number of bytes to use
+     * @param s       the string value to be written. Can be null.
+     * @throws IllegalStateException if the size is too large.
+     */
+    void writeUTFΔ(long offset, int maxSize, @Nullable CharSequence s) throws IllegalStateException;
+
+    /**
+     * Copies the contents of a ByteBuffer from the potision ot the limit.
+     *
+     * @param bb to copy.
+     */
+    void write(@NotNull ByteBuffer bb);
+
+    /**
+     * Write the object in a form which can be uniquely recreated by readEnum.  This type of "enumerable objects" has
+     * the following constraints; 1) each object must have a one to one mapping with a toString() representation, 2) be
+     * immutable, 3) ideally appears more than once, 4) Must have a constructor which takes a single String or a
+     * valueOf(String) method.
+     *
+     * @param e   to enumerate
+     * @param  element class
+     */
+     void writeEnum(@Nullable E e);
+
+    /**
+     * Write an ordered collection of "enumerable objects" (See writeEnum).  This writes the stop bit encoded length,
+     * followed by multiple calls to writeEnum  All the elements must be of the same type.
+     * 

+     * This can be read by the readList method of RandomInputStream and the reader must know
+     * the type of each element.  You can send the class first by using writeEnum of the element class
+     *
+     * @param list to be written
+     */
+     void writeList(@NotNull Collection list);
+
+    /**
+     * Write the keys and values of a Map of "enumerable objects" (See writeEnum). This writes the stop bit encoded
+     * length, followed by multiple calls to writeEnum for each key and value.  All the keys must be of the
+     * same type. All values must be of the same type.
+     *
+     * @param map to write out
+     */
+
+     void writeMap(@NotNull Map map);
+
+    // ObjectOutput
+
+    /**
+     * Write an object as either an "enumerable object" or a Serializable/Externalizable object using Java
+     * Serialization. Java Serialization is much slower but sometimes more convenient than using
+     * BytesMarshallable.
+     *
+     * @param object to write
+     */
+    @Override
+    void writeObject(@Nullable Object object);
+
+    /**
+     * Copy data from an Object from bytes start to end.
+     *
+     * @param object to copy from
+     * @param start  first byte inclusive
+     * @param end    last byte exclusive.
+     */
+    void writeObject(Object object, int start, int end);
+
+    /**
+     * Check the end of the stream has not overflowed.  Otherwise this doesn't do anything.
+     */
+    @Override
+    void flush();
+
+    /**
+     * The same as calling finish();
+     */
+    @Override
+    void close();
+}
diff --git a/lang/src/main/java/net/openhft/lang/io/RandomDataUpdate.java b/lang/src/main/java/net/openhft/lang/io/RandomDataUpdate.java
new file mode 100644
index 0000000..c019276
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/RandomDataUpdate.java
@@ -0,0 +1,141 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io;
+
+/**
+ * This class supports updates which are a companion to RandomDataInput and RandomDataOutput
+ */
+public interface RandomDataUpdate {
+
+    byte addByte(long offset, byte b);
+
+    int addUnsignedByte(long offset, int i);
+
+    short addShort(long offset, short s);
+
+    int addUnsignedShort(long offset, int i);
+
+    int addInt(long offset, int i);
+
+    long addUnsignedInt(long offset, long i);
+
+    long addLong(long offset, long i);
+
+    float addFloat(long offset, float f);
+
+    double addDouble(long offset, double d);
+
+    int addAtomicInt(long offset, int i);
+
+    long addAtomicLong(long offset, long l);
+
+    float addAtomicFloat(long offset, float f);
+
+    double addAtomicDouble(long offset, double d);
+
+    /**
+     * Lock which uses 4 bytes.  It store the lower 24 bits of the Thread Id and the re-entrant count as 8 bit.  This
+     * means if you create more than 16 million threads you can get a collision, and if you try to re-enter 255 times
+     * you will get an ISE
+     *
+     * @param offset of the start of the 4-byte lock
+     * @return did it lock or not.
+     */
+    boolean tryLockInt(long offset);
+
+    /**
+     * Lock which uses 4 bytes.  It store the lower 24 bits of the Thread Id and the re-entrant count as 8 bit.  This
+     * means if you create more than 16 million threads you can get a collision, and if you try to re-enter 255 times
+     * you will get an ISE
+     *
+     * @param offset of the start of the 4-byte lock
+     * @param nanos  to try to lock for
+     * @return did it lock or not.
+     */
+    boolean tryLockNanosInt(long offset, long nanos);
+
+    /**
+     * Lock which uses 4 bytes.  It store the lower 24 bits of the Thread Id and the re-entrant count as 8 bit.  This
+     * means if you create more than 16 million threads you can get a collision, and if you try to re-enter 255 times
+     * you will get an ISE
+     *
+     * @param offset of the start of the 4-byte lock
+     * @throws InterruptedException  if interrupted
+     * @throws IllegalStateException if the thread tries to lock it 255 nested time (without an unlock)
+     */
+    void busyLockInt(long offset) throws InterruptedException, IllegalStateException;
+
+    /**
+     * Lock which uses 4 bytes.  Unlock this It store the lower 24 bits of the Thread Id and the re-entrant count as 8
+     * bit.  This means if you create more than 16 million threads you can get a collision, and if you try to re-enter
+     * 255 times you will get an ISE
+     *
+     * @param offset of the start of the 4-byte lock
+     * @throws IllegalMonitorStateException if this thread doesn't hold the lock
+     */
+    void unlockInt(long offset) throws IllegalMonitorStateException;
+
+    /**
+     * Lock across processes
+     * 

+     * Lock which uses 8 bytes.  It store the lower 32 bits of the Thread Id, 16 bits are the process id and the re-entrant count as 16 bit.  This
+     * means if you create more than 16 million threads you can get a collision, and if you try to re-enter 65535 times
+     * you will get an ISE
+     *
+     * @param offset of the start of the 8-byte lock
+     * @return did it lock or not.
+     */
+    boolean tryLockLong(long offset);
+
+    /**
+     * Lock across processes
+     * 

+     * Lock which uses 8 bytes.  It store the lower 32 bits of the Thread Id, 16 bits are the process id and the re-entrant count as 16 bit.  This
+     * means if you create more than 16 million threads you can get a collision, and if you try to re-enter 65535 times
+     * you will get an ISE
+     *
+     * @param offset of the start of the 8-byte lock
+     * @param nanos  to try to lock for
+     * @return did it lock or not.
+     */
+    boolean tryLockNanosLong(long offset, long nanos);
+
+    /**
+     * Lock across processes
+     * 

+     * Lock which uses 8 bytes.  It store the lower 32 bits of the Thread Id, 16 bits are the process id and the re-entrant count as 16 bit.  This
+     * means if you create more than 16 million threads you can get a collision, and if you try to re-enter 65535 times
+     * you will get an ISE
+     *
+     * @param offset of the start of the 8-byte lock
+     * @throws InterruptedException  if interrupted
+     * @throws IllegalStateException if the thread tries to lock it 65535 nested time (without an unlock)
+     */
+    void busyLockLong(long offset) throws InterruptedException, IllegalStateException;
+
+    /**
+     * Lock across processes
+     * 

+     * Lock which uses 8 bytes.  It store the lower 32 bits of the Thread Id, 16 bits are the process id and the re-entrant count as 16 bit.  This
+     * means if you create more than 16 million threads you can get a collision, and if you try to re-enter 65535 times
+     * you will get an ISE
+     *
+     * @param offset of the start of the 8-byte lock
+     * @throws IllegalMonitorStateException if this thread doesn't hold the lock
+     */
+    void unlockLong(long offset) throws IllegalMonitorStateException;
+}
diff --git a/lang/src/main/java/net/openhft/lang/io/StopCharTester.java b/lang/src/main/java/net/openhft/lang/io/StopCharTester.java
new file mode 100755
index 0000000..8f965e3
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/StopCharTester.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io;
+
+/**
+ * @author peter.lawrey
+ */
+public interface StopCharTester {
+    /**
+     * Detect which byte stops the string to be parsed
+     * 

+     * This should be changed to support char instead.
+     * 

+     * Note: for safety reasons, you should stop on a 0 byte or throw an IllegalStateException.
+     *
+     * @param ch to test, 0 should return true or throw an exception.
+     * @return if this byte is a stop character.
+     * @throws IllegalStateException if an invalid character like 0 was detected.
+     */
+    boolean isStopChar(int ch) throws IllegalStateException;
+}
diff --git a/lang/src/main/java/net/openhft/lang/io/StopCharTesters.java b/lang/src/main/java/net/openhft/lang/io/StopCharTesters.java
new file mode 100755
index 0000000..f711706
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/StopCharTesters.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io;
+
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * @author peter.lawrey
+ */
+public enum StopCharTesters implements StopCharTester {
+    COMMA_STOP {
+        @Override
+        public boolean isStopChar(int ch) {
+            return ch < ' ' || ch == ',';
+        }
+    }, CONTROL_STOP {
+        @Override
+        public boolean isStopChar(int ch) {
+            return ch < ' ';
+        }
+    },
+    SPACE_STOP {
+        @Override
+        public boolean isStopChar(int ch) {
+            return Character.isWhitespace(ch) || ch == 0;
+        }
+    },
+    XML_TEXT {
+        @Override
+        public boolean isStopChar(int ch) {
+            return ch == '"' || ch == '<' || ch == '>' || ch == 0;
+        }
+    },
+    FIX_TEXT {
+        @Override
+        public boolean isStopChar(int ch) {
+            return ch <= 1;
+        }
+    };
+
+    @NotNull
+    public static StopCharTester forChars(@NotNull CharSequence sequence) {
+        if (sequence.length() == 1)
+            return forChar(sequence.charAt(0));
+        return new CSCSTester(sequence);
+    }
+
+    @NotNull
+    public static StopCharTester forChar(char ch) {
+        return new CharCSTester(ch);
+    }
+
+    static class CSCSTester implements StopCharTester {
+        @NotNull
+        private final String seperators;
+
+        public CSCSTester(@NotNull CharSequence cs) {
+            seperators = cs.toString();
+        }
+
+        @Override
+        public boolean isStopChar(int ch) {
+            return seperators.indexOf(ch) >= 0;
+        }
+    }
+
+    static class CharCSTester implements StopCharTester {
+        private final char ch;
+
+        public CharCSTester(char ch) {
+            this.ch = ch;
+        }
+
+        @Override
+        public boolean isStopChar(int ch) {
+            return this.ch == ch;
+        }
+    }
+}
diff --git a/lang/src/main/java/net/openhft/lang/io/serialization/BytesMarshallable.java b/lang/src/main/java/net/openhft/lang/io/serialization/BytesMarshallable.java
new file mode 100644
index 0000000..a92b316
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/serialization/BytesMarshallable.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io.serialization;
+
+import net.openhft.lang.io.Bytes;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Integrated marshaller for objects.
+ *
+ * @author peter.lawrey
+ * @see BytesMarshaller
+ */
+public interface BytesMarshallable {
+    /**
+     * read an object from bytes
+     *
+     * @param in to read from
+     * @throws IllegalStateException if the object could not be read.
+     */
+    void readMarshallable(@NotNull Bytes in) throws IllegalStateException;
+
+    /**
+     * write an object to bytes
+     *
+     * @param out to write to
+     */
+    void writeMarshallable(@NotNull Bytes out);
+}
diff --git a/lang/src/main/java/net/openhft/lang/io/serialization/BytesMarshaller.java b/lang/src/main/java/net/openhft/lang/io/serialization/BytesMarshaller.java
new file mode 100644
index 0000000..d9a4fa2
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/serialization/BytesMarshaller.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io.serialization;
+
+import net.openhft.lang.io.Bytes;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * External marshaller for classes.
+ *
+ * @author peter.lawrey
+ * @see BytesMarshallable
+ */
+public interface BytesMarshaller {
+    /**
+     * write the object out as bytes.
+     *
+     * @param bytes to write to
+     * @param e     to write
+     */
+    void write(Bytes bytes, E e);
+
+    /**
+     * Read bytes and obtain an object
+     *
+     * @param bytes to read
+     * @return the object
+     */
+    @Nullable
+    E read(Bytes bytes);
+}
diff --git a/lang/src/main/java/net/openhft/lang/io/serialization/BytesMarshallerFactory.java b/lang/src/main/java/net/openhft/lang/io/serialization/BytesMarshallerFactory.java
new file mode 100644
index 0000000..87218b7
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/serialization/BytesMarshallerFactory.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io.serialization;
+
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * @author peter.lawrey
+ */
+public interface BytesMarshallerFactory {
+    @NotNull
+     BytesMarshaller acquireMarshaller(@NotNull Class eClass, boolean create);
+
+     BytesMarshaller getMarshaller(byte code);
+
+     void addMarshaller(Class eClass, BytesMarshaller marshaller);
+}
diff --git a/lang/src/main/java/net/openhft/lang/io/serialization/CompactBytesMarshaller.java b/lang/src/main/java/net/openhft/lang/io/serialization/CompactBytesMarshaller.java
new file mode 100644
index 0000000..6ee1f87
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/serialization/CompactBytesMarshaller.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io.serialization;
+
+/**
+ * A BytesMarshaller with a byte code for the class.
+ */
+public interface CompactBytesMarshaller extends BytesMarshaller {
+    /**
+     * the code for this marshaller
+     */
+    byte code();
+}
diff --git a/lang/src/main/java/net/openhft/lang/io/serialization/RawCopier.java b/lang/src/main/java/net/openhft/lang/io/serialization/RawCopier.java
new file mode 100644
index 0000000..eaf5760
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/serialization/RawCopier.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io.serialization;
+
+import net.openhft.lang.io.Bytes;
+
+import java.lang.reflect.Array;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+import static net.openhft.lang.io.NativeBytes.UNSAFE;
+
+/**
+ * User: peter.lawrey Date: 22/09/13 Time: 16:51
+ */
+public class RawCopier {
+    final int start, end;
+    private final Class tClass;
+
+    public RawCopier(Class tClass) {
+        this.tClass = tClass;
+        List fields = new ArrayList();
+        addAllFields(fields, tClass);
+        Collections.sort(fields, new Comparator() {
+            @Override
+            public int compare(Field o1, Field o2) {
+                long off1 = UNSAFE.objectFieldOffset(o1);
+                long off2 = UNSAFE.objectFieldOffset(o2);
+                return Double.compare(off1, off2);
+            }
+        });
+        start = (int) UNSAFE.objectFieldOffset(fields.get(0));
+        Field lastField = null;
+        for (Field field : fields) {
+            if (Modifier.isTransient(field.getModifiers()) || !field.getType().isPrimitive())
+                break;
+            lastField = field;
+        }
+        end = (int) UNSAFE.objectFieldOffset(lastField) + sizeOf(lastField.getType());
+
+        assert end > start : "end <= start, start: " + start + ", end: " + end;
+    }
+
+    public static  RawCopier copies(Class tClass) {
+        return new RawCopier(tClass);
+    }
+
+    private static int sizeOf(Class type) {
+        return UNSAFE.arrayIndexScale(Array.newInstance(type, 0).getClass());
+    }
+
+    public int start() {
+        return start;
+    }
+
+    public int end() {
+        return end;
+    }
+
+    public void toBytes(Object obj, Bytes bytes) {
+        bytes.writeObject(obj, start, end);
+    }
+
+    public void fromBytes(Bytes bytes, Object obj) {
+        bytes.readObject(obj, start, end);
+    }
+
+    public void copy(T from, T to) {
+        long i;
+        for (i = start; i < end - 7; i += 8) {
+            UNSAFE.putLong(to, i, UNSAFE.getLong(from, i));
+        }
+        for (; i < end; i++) {
+            UNSAFE.putByte(to, i, UNSAFE.getByte(from, i));
+        }
+    }
+
+    private void addAllFields(List fields, Class tClass) {
+        if (tClass != null && tClass != Object.class)
+            addAllFields(fields, tClass.getSuperclass());
+        for (Field field : tClass.getDeclaredFields()) {
+            if (!Modifier.isStatic(field.getModifiers()))
+                fields.add(field);
+        }
+    }
+}
diff --git a/lang/src/main/java/net/openhft/lang/io/serialization/direct/DirectSerializationFilter.java b/lang/src/main/java/net/openhft/lang/io/serialization/direct/DirectSerializationFilter.java
new file mode 100644
index 0000000..c6b5b2e
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/serialization/direct/DirectSerializationFilter.java
@@ -0,0 +1,29 @@
+package net.openhft.lang.io.serialization.direct;
+
+import java.lang.reflect.Field;
+import java.util.*;
+
+import static net.openhft.lang.io.serialization.direct.FieldMetadata.*;
+
+public final class DirectSerializationFilter {
+    public static List stopAtFirstIneligibleField(List fields) {
+        ArrayList eligibleFields = new ArrayList();
+        for (Field f : fields) {
+            if (checkEligible(f)) {
+                eligibleFields.add(f);
+            } else {
+                break;
+            }
+        }
+
+        return eligibleFields.isEmpty() ?
+                Collections.emptyList() :
+                eligibleFields;
+    }
+
+    private static boolean checkEligible(Field f) {
+        return isPrimitive(f) &&
+                !isStatic(f) &&
+                !isTransient(f);
+    }
+}
\ No newline at end of file
diff --git a/lang/src/main/java/net/openhft/lang/io/serialization/direct/DirectSerializationMetadata.java b/lang/src/main/java/net/openhft/lang/io/serialization/direct/DirectSerializationMetadata.java
new file mode 100644
index 0000000..2fcd50c
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/serialization/direct/DirectSerializationMetadata.java
@@ -0,0 +1,89 @@
+package net.openhft.lang.io.serialization.direct;
+
+import net.openhft.lang.Jvm;
+
+import java.lang.reflect.Field;
+import java.util.List;
+
+import static net.openhft.lang.io.NativeBytes.UNSAFE;
+
+public class DirectSerializationMetadata {
+    private static final int OBJECT_ALIGNMENT = 8;
+    private static final int OBJECT_ALIGNMENT_MASK = OBJECT_ALIGNMENT - 1;
+
+    static final long NATIVE_WORD_SIZE = Jvm.is64Bit() ? 8 : 4;
+    static final long OOP_SIZE = UNSAFE.arrayIndexScale(Object[].class);
+    static final long OBJECT_HEADER_SIZE = NATIVE_WORD_SIZE + OOP_SIZE; // Object header has a native sized mark word + variable sized oop to klass meta object
+
+    public static final SerializationMetadata EmptyObjectMetadata = new SerializationMetadata(0, 0);
+
+    public static final class SerializationMetadata {
+        final long start;
+        final long length;
+
+        SerializationMetadata(long start, long length) {
+            this.start = start;
+            this.length = length;
+        }
+
+        @Override
+        public String toString() {
+            return String.format("SerializationMetadata: Start %s Length %s", start, length);
+        }
+    }
+
+    public static SerializationMetadata extractMetadata(List fields) {
+        if (fields.isEmpty()) return EmptyObjectMetadata;
+
+        Offsets offsets = minMaxOffsets(fields);
+
+        long totalSize = OBJECT_HEADER_SIZE + offsets.max - offsets.min + 1;
+        return new SerializationMetadata(offsets.min, padToObjectAlignment(totalSize) - OBJECT_HEADER_SIZE);
+    }
+
+    public static SerializationMetadata extractMetadataForPartialCopy(List fields) {
+        if (fields.isEmpty()) return EmptyObjectMetadata;
+
+        Offsets offsets = minMaxOffsets(fields);
+
+        Field lastField = fields.get(fields.size() - 1);
+
+        return new SerializationMetadata(offsets.min, offsets.max + sizeOf(lastField) - OBJECT_HEADER_SIZE);
+    }
+
+    private static Offsets minMaxOffsets(List fields) {
+        long minOffset = UNSAFE.objectFieldOffset(fields.get(0));
+        long maxOffset = UNSAFE.objectFieldOffset(fields.get(fields.size() - 1));
+
+        return new Offsets(minOffset, maxOffset);
+    }
+
+    static long padToObjectAlignment(long length) {
+        if ((length & OBJECT_ALIGNMENT_MASK) != 0) {
+            long padding = OBJECT_ALIGNMENT - (length & OBJECT_ALIGNMENT_MASK);
+            length += padding;
+        }
+
+        return length;
+    }
+
+    private static long sizeOf(Field field) {
+        if (boolean.class.equals(field.getType())) return 1;
+        else if (byte.class.equals(field.getType())) return 1;
+        else if (short.class.equals(field.getType())) return 2;
+        else if (char.class.equals(field.getType())) return 2;
+        else if (int.class.equals(field.getType())) return 4;
+        else if (float.class.equals(field.getType())) return 4;
+        else return 8;
+    }
+
+    private static final class Offsets {
+        public final long min;
+        public final long max;
+
+        private Offsets(long min, long max) {
+            this.min = min;
+            this.max = max;
+        }
+    }
+}
\ No newline at end of file
diff --git a/lang/src/main/java/net/openhft/lang/io/serialization/direct/FieldMetadata.java b/lang/src/main/java/net/openhft/lang/io/serialization/direct/FieldMetadata.java
new file mode 100644
index 0000000..d186c77
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/serialization/direct/FieldMetadata.java
@@ -0,0 +1,22 @@
+package net.openhft.lang.io.serialization.direct;
+
+import java.lang.reflect.*;
+
+public final class FieldMetadata {
+    public static boolean isPrimitive(Field f) {
+        return f.getType().isPrimitive();
+    }
+
+    public static boolean isPrimitiveArray(Field f) {
+        Class clazz = f.getType();
+        return clazz.isArray() && clazz.getComponentType().isPrimitive();
+    }
+
+    public static boolean isStatic(Field f) {
+        return Modifier.isStatic(f.getModifiers());
+    }
+
+    public static boolean isTransient(Field f) {
+        return Modifier.isTransient(f.getModifiers());
+    }
+}
diff --git a/lang/src/main/java/net/openhft/lang/io/serialization/direct/Introspect.java b/lang/src/main/java/net/openhft/lang/io/serialization/direct/Introspect.java
new file mode 100644
index 0000000..28ff782
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/serialization/direct/Introspect.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io.serialization.direct;
+
+import net.openhft.lang.Maths;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+import static net.openhft.lang.io.NativeBytes.UNSAFE;
+import static net.openhft.lang.io.serialization.direct.FieldMetadata.isStatic;
+
+public class Introspect {
+    public static List fields(Class clazz) {
+        ArrayList fields = new ArrayList();
+
+        addToFields(clazz, fields);
+        Collections.sort(fields, FieldOffsetComparator.Instance);
+
+        return fields;
+    }
+
+    private static List addToFields(Class clazz, ArrayList accumulator) {
+        Collections.addAll(accumulator, clazz.getDeclaredFields());
+        Class maybeSuper = clazz.getSuperclass();
+
+        return maybeSuper != null ?
+                addToFields(maybeSuper, accumulator) :
+                accumulator;
+    }
+
+    private static final class FieldOffsetComparator implements Comparator {
+        public static final FieldOffsetComparator Instance = new FieldOffsetComparator();
+
+        @Override
+        public int compare(Field first, Field second) {
+            return Maths.compare(offset(first), offset(second));
+        }
+
+        private static long offset(Field field) {
+            return isStatic(field) ?
+                    UNSAFE.staticFieldOffset(field) :
+                    UNSAFE.objectFieldOffset(field);
+        }
+    }
+}
\ No newline at end of file
diff --git a/lang/src/main/java/net/openhft/lang/io/serialization/direct/ObjectMarshaller.java b/lang/src/main/java/net/openhft/lang/io/serialization/direct/ObjectMarshaller.java
new file mode 100644
index 0000000..24f8a48
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/serialization/direct/ObjectMarshaller.java
@@ -0,0 +1,50 @@
+package net.openhft.lang.io.serialization.direct;
+
+import net.openhft.lang.io.Bytes;
+
+import static net.openhft.lang.io.NativeBytes.UNSAFE;
+import static net.openhft.lang.io.serialization.direct.DirectSerializationMetadata.SerializationMetadata;
+
+public final class ObjectMarshaller {
+    private final SerializationMetadata metadata;
+
+    public ObjectMarshaller(SerializationMetadata metadata) {
+        this.metadata = metadata;
+    }
+
+    public void write(Bytes bytes, T tObject) {
+        long i = metadata.start;
+        long end = metadata.start + metadata.length;
+
+        while (i < end - 7) {
+            bytes.writeLong(UNSAFE.getLong(tObject, i));
+            i += 8;
+        }
+
+        while (i < end) {
+            bytes.writeByte(UNSAFE.getByte(tObject, i));
+            ++i;
+        }
+    }
+
+    public T read(Bytes bytes, T tObject) {
+        long i = metadata.start;
+        long end = metadata.start + metadata.length;
+
+        while (i < end - 7) {
+            UNSAFE.putLong(tObject, i, bytes.readLong());
+            i += 8;
+        }
+
+        while (i < end) {
+            UNSAFE.putByte(tObject, i, bytes.readByte());
+            ++i;
+        }
+
+        return tObject;
+    }
+
+    public long length() {
+        return metadata.length;
+    }
+}
\ No newline at end of file
diff --git a/lang/src/main/java/net/openhft/lang/io/serialization/direct/ObjectMarshallers.java b/lang/src/main/java/net/openhft/lang/io/serialization/direct/ObjectMarshallers.java
new file mode 100644
index 0000000..bb9e35d
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/serialization/direct/ObjectMarshallers.java
@@ -0,0 +1,72 @@
+package net.openhft.lang.io.serialization.direct;
+
+import net.openhft.lang.io.NativeBytes;
+
+import java.lang.reflect.Field;
+import java.util.*;
+import java.util.logging.Logger;
+
+import static java.util.logging.Level.WARNING;
+import static net.openhft.lang.io.serialization.direct.DirectSerializationMetadata.SerializationMetadata;
+
+public final class ObjectMarshallers {
+    private static final Logger Log = Logger.getLogger(ObjectMarshallers.class.getName());
+
+    private static final Map metadata = new HashMap();
+
+    @SuppressWarnings("unchecked")
+    public static  ObjectMarshaller forClass(Class clazz) {
+        ObjectMarshaller om = metadata.get(clazz);
+        if (om == null) {
+            List fields = Introspect.fields(clazz);
+            List eligibleFields = DirectSerializationFilter.stopAtFirstIneligibleField(fields);
+
+            SerializationMetadata serializationMetadata;
+
+            if (hasIneligibleFields(fields, eligibleFields)) {
+                WarnAboutIneligibleFields.apply(clazz, fields, eligibleFields);
+                serializationMetadata = DirectSerializationMetadata.extractMetadataForPartialCopy(eligibleFields);
+            } else {
+                serializationMetadata = DirectSerializationMetadata.extractMetadata(eligibleFields);
+            }
+
+            om = new ObjectMarshaller(serializationMetadata);
+            Log.log(WARNING, String.format("Class %s has metadata %s", clazz.getName(), serializationMetadata));
+            metadata.put(clazz, om);
+        }
+
+        return (ObjectMarshaller) om;
+    }
+
+    private static boolean hasIneligibleFields(List allFields, List eligibleFields) {
+        return allFields.size() != eligibleFields.size();
+    }
+
+    private static class WarnAboutIneligibleFields {
+        static void apply(Class clazz, List allFields, List eligibleFields) {
+            List ineligibleFields = allFields.subList(eligibleFields.size(), allFields.size());
+            Log.log(WARNING, String.format(
+                    "The following fields in Class %s will not be copied by ObjectMarshaller:\n%s",
+                    clazz.getName(),
+                    commaSeparate(ineligibleFields)
+            ));
+        }
+
+        private static String commaSeparate(Collection fields) {
+            StringBuilder sb = new StringBuilder();
+            boolean first = true;
+            for (Field field : fields) {
+                if (first) {
+                    sb.append("\t");
+                    sb.append(field.getName());
+                    first = false;
+                } else {
+                    sb.append("\n\t");
+                    sb.append(field.getName());
+                }
+            }
+
+            return sb.toString();
+        }
+    }
+}
diff --git a/lang/src/main/java/net/openhft/lang/io/serialization/impl/BytesMarshallableMarshaller.java b/lang/src/main/java/net/openhft/lang/io/serialization/impl/BytesMarshallableMarshaller.java
new file mode 100644
index 0000000..ebcb95f
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/serialization/impl/BytesMarshallableMarshaller.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io.serialization.impl;
+
+import net.openhft.lang.io.Bytes;
+import net.openhft.lang.io.serialization.BytesMarshallable;
+import net.openhft.lang.io.serialization.BytesMarshaller;
+import net.openhft.lang.io.NativeBytes;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * @author peter.lawrey
+ */
+public class BytesMarshallableMarshaller implements BytesMarshaller {
+    @NotNull
+    private final Class classMarshaled;
+
+    public BytesMarshallableMarshaller(@NotNull Class classMarshaled) {
+        this.classMarshaled = classMarshaled;
+    }
+
+    @Override
+    public void write(@NotNull Bytes bytes, @NotNull E e) {
+        e.writeMarshallable(bytes);
+    }
+
+    @Override
+    public E read(@NotNull Bytes bytes) {
+        E e;
+        try {
+            e = (E) NativeBytes.UNSAFE.allocateInstance(classMarshaled);
+        } catch (Exception e2) {
+            throw new IllegalStateException(e2);
+        }
+        e.readMarshallable(bytes);
+        return e;
+    }
+}
diff --git a/lang/src/main/java/net/openhft/lang/io/serialization/impl/ClassMarshaller.java b/lang/src/main/java/net/openhft/lang/io/serialization/impl/ClassMarshaller.java
new file mode 100644
index 0000000..db36c37
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/serialization/impl/ClassMarshaller.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io.serialization.impl;
+
+import net.openhft.lang.Compare;
+import net.openhft.lang.io.Bytes;
+import net.openhft.lang.io.serialization.CompactBytesMarshaller;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.lang.ref.WeakReference;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * @author peter.lawrey
+ */
+public class ClassMarshaller implements CompactBytesMarshaller {
+    private static final int CACHE_SIZE = 1019;
+    private static final Map SC_SHORT_NAME = new LinkedHashMap();
+    private static final Map CS_SHORT_NAME = new LinkedHashMap();
+
+    static {
+        Class[] classes = {Boolean.class, Byte.class, Character.class, Short.class, Integer.class, Long.class, Float.class, Double.class,
+                String.class, Class.class, BigInteger.class, BigDecimal.class, Date.class};
+        for (Class clazz : classes) {
+            String simpleName = clazz.getSimpleName();
+            SC_SHORT_NAME.put(simpleName, clazz);
+            CS_SHORT_NAME.put(clazz, simpleName);
+        }
+    }
+
+    private final ClassLoader classLoader;
+    private final StringBuilder className = new StringBuilder(40);
+    @Nullable
+    @SuppressWarnings("unchecked")
+    private WeakReference[] classWeakReference = null;
+
+    public ClassMarshaller(ClassLoader classLoader) {
+        this.classLoader = classLoader;
+    }
+
+    @Override
+    public void write(@NotNull Bytes bytes, @NotNull Class aClass) {
+        String s = CS_SHORT_NAME.get(aClass);
+        if (s == null)
+            s = aClass.getName();
+        bytes.writeUTFΔ(s);
+    }
+
+    @Nullable
+    @Override
+    public Class read(@NotNull Bytes bytes) {
+        className.setLength(0);
+        bytes.readUTFΔ(className);
+        return load(className);
+    }
+
+    @Nullable
+    private Class load(@NotNull CharSequence name) {
+        int hash = (int) (Compare.calcLongHashCode(name) & 0x7ffffff) % CACHE_SIZE;
+        if (classWeakReference == null)
+            //noinspection unchecked
+            classWeakReference = new WeakReference[CACHE_SIZE];
+        WeakReference ref = classWeakReference[hash];
+        if (ref != null) {
+            Class clazz = ref.get();
+            if (clazz != null && clazz.getName().equals(name))
+                return clazz;
+        }
+        try {
+
+            Class clazz = SC_SHORT_NAME.get(name.toString());
+            if (clazz != null)
+                return clazz;
+            clazz = classLoader.loadClass(name.toString());
+            classWeakReference[hash] = new WeakReference(clazz);
+            return clazz;
+        } catch (ClassNotFoundException e) {
+            throw new IllegalArgumentException(e);
+        }
+    }
+
+    @Override
+    public byte code() {
+        return 'C' & 31; // control C
+    }
+}
diff --git a/lang/src/main/java/net/openhft/lang/io/serialization/impl/CompactEnumBytesMarshaller.java b/lang/src/main/java/net/openhft/lang/io/serialization/impl/CompactEnumBytesMarshaller.java
new file mode 100644
index 0000000..08f66e4
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/serialization/impl/CompactEnumBytesMarshaller.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io.serialization.impl;
+
+import net.openhft.lang.io.serialization.CompactBytesMarshaller;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Created with IntelliJ IDEA. User: peter Date: 09/12/13 Time: 17:05 To change this template use File | Settings | File
+ * Templates.
+ */
+public class CompactEnumBytesMarshaller extends GenericEnumMarshaller implements CompactBytesMarshaller {
+    private final byte code;
+
+    public CompactEnumBytesMarshaller(@NotNull Class classMarshaled, int capacity, byte code) {
+        super(classMarshaled, capacity);
+        this.code = code;
+    }
+
+    @Override
+    public byte code() {
+        return code;
+    }
+}
diff --git a/lang/src/main/java/net/openhft/lang/io/serialization/impl/DateMarshaller.java b/lang/src/main/java/net/openhft/lang/io/serialization/impl/DateMarshaller.java
new file mode 100644
index 0000000..03f6bb1
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/serialization/impl/DateMarshaller.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io.serialization.impl;
+
+import net.openhft.lang.io.Bytes;
+import net.openhft.lang.io.serialization.CompactBytesMarshaller;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.Date;
+
+/**
+ * @author peter.lawrey
+ */
+public class DateMarshaller implements CompactBytesMarshaller {
+    final int size1;
+    private final StringBuilder sb = new StringBuilder();
+    @Nullable
+    private Date[] interner = null;
+
+    public DateMarshaller(int size) {
+        int size2 = 128;
+        while (size2 < size && size2 < (1 << 20)) size2 <<= 1;
+        this.size1 = size2 - 1;
+    }
+
+    private static long parseLong(@NotNull CharSequence sb) {
+        long num = 0;
+        boolean negative = false;
+        for (int i = 0; i < sb.length(); i++) {
+            char b = sb.charAt(i);
+//            if (b >= '0' && b <= '9')
+            if ((b - ('0' + Integer.MIN_VALUE)) <= 9 + Integer.MIN_VALUE)
+                num = num * 10 + b - '0';
+            else if (b == '-')
+                negative = true;
+            else
+                break;
+        }
+        return negative ? -num : num;
+    }
+
+    @Override
+    public void write(@NotNull Bytes bytes, @NotNull Date date) {
+        long pos = bytes.position();
+        bytes.writeUnsignedByte(0);
+        bytes.append(date.getTime());
+        bytes.writeUnsignedByte(pos, (int) (bytes.position() - 1 - pos));
+    }
+
+    @Nullable
+    @Override
+    public Date read(@NotNull Bytes bytes) {
+        bytes.readUTFΔ(sb);
+        long time = parseLong(sb);
+        return lookupDate(time);
+    }
+
+    @Nullable
+    private Date lookupDate(long time) {
+        int idx = hashFor(time);
+        if (interner == null)
+            interner = new Date[size1 + 1];
+        Date date = interner[idx];
+        if (date != null && date.getTime() == time)
+            return date;
+        return interner[idx] = new Date(time);
+    }
+
+    private int hashFor(long time) {
+        long h = time;
+        h ^= (h >>> 41) ^ (h >>> 20);
+        h ^= (h >>> 14) ^ (h >>> 7);
+        return (int) (h & size1);
+    }
+
+    @Override
+    public byte code() {
+        return 'T' & 31; // Control T.
+    }
+}
diff --git a/lang/src/main/java/net/openhft/lang/io/serialization/impl/EnumBytesMarshaller.java b/lang/src/main/java/net/openhft/lang/io/serialization/impl/EnumBytesMarshaller.java
new file mode 100644
index 0000000..11f060b
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/serialization/impl/EnumBytesMarshaller.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io.serialization.impl;
+
+import net.openhft.lang.io.Bytes;
+import net.openhft.lang.io.serialization.BytesMarshaller;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.BitSet;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * @author peter.lawrey
+ */
+public class EnumBytesMarshaller> implements BytesMarshaller {
+    @NotNull
+    private final Class classMarshaled;
+    @SuppressWarnings("unchecked")
+    private final E[] interner = (E[]) new Enum[1024];
+    private final BitSet internerDup = new BitSet(1024);
+    private final Map map = new LinkedHashMap();
+    private final E defaultValue;
+    private final int mask;
+    private final StringBuilder reader = new StringBuilder();
+
+    public EnumBytesMarshaller(@NotNull Class classMarshaled, E defaultValue) {
+        this.classMarshaled = classMarshaled;
+        this.defaultValue = defaultValue;
+
+        mask = interner.length - 1;
+        for (E e : classMarshaled.getEnumConstants()) {
+            map.put(e.name(), e);
+            int idx = hashFor(e.name());
+            if (!internerDup.get(idx)) {
+                if (interner[idx] != null) {
+                    //noinspection UnqualifiedFieldAccess,AssignmentToNull
+                    interner[idx] = null;
+                    internerDup.set(idx);
+                } else {
+                    interner[idx] = e;
+                }
+            }
+        }
+    }
+
+    @Override
+    public void write(@NotNull Bytes bytes, @Nullable E e) {
+        bytes.writeUTFΔ(e == null ? "" : e.name());
+    }
+
+    private int hashFor(@NotNull CharSequence cs) {
+        int h = 0;
+
+        for (int i = 0, length = cs.length(); i < length; i++)
+            h = 57 * h + cs.charAt(i);
+
+        h ^= (h >>> 20) ^ (h >>> 12);
+        h ^= (h >>> 7) ^ (h >>> 4);
+        return h & mask;
+    }
+
+    @Override
+    public E read(@NotNull Bytes bytes) {
+        bytes.readUTFΔ(reader);
+        return builderToEnum();
+    }
+
+    private E builderToEnum() {
+        int num = hashFor(reader);
+        int idx = num & mask;
+        E e = interner[idx];
+        if (e != null) return e;
+        if (!internerDup.get(idx)) return defaultValue;
+        e = map.get(reader.toString());
+        return e == null ? defaultValue : e;
+    }
+}
diff --git a/lang/src/main/java/net/openhft/lang/io/serialization/impl/ExternalizableMarshaller.java b/lang/src/main/java/net/openhft/lang/io/serialization/impl/ExternalizableMarshaller.java
new file mode 100644
index 0000000..05f5390
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/serialization/impl/ExternalizableMarshaller.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io.serialization.impl;
+
+import net.openhft.lang.io.Bytes;
+import net.openhft.lang.io.serialization.BytesMarshaller;
+import net.openhft.lang.io.NativeBytes;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.Externalizable;
+import java.io.IOException;
+
+/**
+ * @author peter.lawrey
+ */
+public class ExternalizableMarshaller implements BytesMarshaller {
+    @NotNull
+    private final Class classMarshaled;
+
+    public ExternalizableMarshaller(@NotNull Class classMarshaled) {
+        this.classMarshaled = classMarshaled;
+    }
+
+    @Override
+    public void write(Bytes bytes, @NotNull E e) {
+        try {
+            e.writeExternal(bytes);
+        } catch (IOException e2) {
+            throw new IllegalStateException(e2);
+        }
+    }
+
+    @Override
+    public E read(Bytes bytes) {
+        E e;
+        try {
+            e = (E) NativeBytes.UNSAFE.allocateInstance(classMarshaled);
+            e.readExternal(bytes);
+        } catch (Exception e2) {
+            throw new IllegalStateException(e2);
+        }
+        return e;
+    }
+}
diff --git a/lang/src/main/java/net/openhft/lang/io/serialization/impl/GenericEnumMarshaller.java b/lang/src/main/java/net/openhft/lang/io/serialization/impl/GenericEnumMarshaller.java
new file mode 100644
index 0000000..8a1b52d
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/serialization/impl/GenericEnumMarshaller.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io.serialization.impl;
+
+import net.openhft.lang.io.Bytes;
+import net.openhft.lang.io.serialization.BytesMarshaller;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * @author peter.lawrey
+ */
+public class GenericEnumMarshaller implements BytesMarshaller {
+    @NotNull
+    private final Class classMarshaled;
+    @Nullable
+    private final Constructor constructor;
+    @Nullable
+    private final Method valueOf;
+    @NotNull
+    private final Map map;
+
+    public GenericEnumMarshaller(@NotNull Class classMarshaled, final int capacity) {
+        this.classMarshaled = classMarshaled;
+        Constructor constructor = null;
+        Method valueOf = null;
+        try {
+            valueOf = classMarshaled.getMethod("valueOf", String.class);
+        } catch (NoSuchMethodException e) {
+            try {
+                constructor = classMarshaled.getConstructor(String.class);
+            } catch (NoSuchMethodException e1) {
+                throw new IllegalArgumentException(classMarshaled + " doesn't have a valueOf(String) or a Constructor(String)");
+            }
+        }
+        this.constructor = constructor;
+        this.valueOf = valueOf;
+        map = new LinkedHashMap(128, 0.7f, true) {
+            @Override
+            protected boolean removeEldestEntry(Map.Entry eldest) {
+                return size() > capacity;
+            }
+        };
+    }
+
+    @Override
+    public void write(@NotNull Bytes bytes, @Nullable E e) {
+        bytes.writeUTFΔ(e == null ? null : e.toString());
+    }
+
+    @Nullable
+    @Override
+    public E read(@NotNull Bytes bytes) {
+        String s = bytes.readUTFΔ();
+        return s == null ? null : valueOf(s);
+    }
+
+    private E valueOf(String s) {
+        E e = map.get(s);
+        if (e == null)
+            try {
+                if (constructor != null) {
+                    map.put(s, e = constructor.newInstance(s));
+                } else {
+                    @SuppressWarnings("unchecked")
+                    E invoke = (E) valueOf.invoke(null, s);
+                    map.put(s, e = invoke);
+                }
+            } catch (Exception t) {
+                throw new AssertionError(t.getCause());
+            }
+        return e;
+    }
+}
diff --git a/lang/src/main/java/net/openhft/lang/io/serialization/impl/NoMarshaller.java b/lang/src/main/java/net/openhft/lang/io/serialization/impl/NoMarshaller.java
new file mode 100644
index 0000000..1369c4a
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/serialization/impl/NoMarshaller.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io.serialization.impl;
+
+import net.openhft.lang.io.Bytes;
+import net.openhft.lang.io.serialization.BytesMarshaller;
+
+/**
+ * Created with IntelliJ IDEA. User: peter.lawrey Date: 19/09/13 Time: 18:26 To change this template use File | Settings | File
+ * Templates.
+ */
+public enum NoMarshaller implements BytesMarshaller {
+    INSTANCE;
+
+    @Override
+    public void write(Bytes bytes, Void aVoid) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public Void read(Bytes bytes) {
+        throw new UnsupportedOperationException();
+    }
+}
diff --git a/lang/src/main/java/net/openhft/lang/io/serialization/impl/StringMarshaller.java b/lang/src/main/java/net/openhft/lang/io/serialization/impl/StringMarshaller.java
new file mode 100644
index 0000000..2fb2ee5
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/serialization/impl/StringMarshaller.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io.serialization.impl;
+
+import net.openhft.lang.io.Bytes;
+import net.openhft.lang.io.serialization.CompactBytesMarshaller;
+import net.openhft.lang.pool.StringInterner;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * @author peter.lawrey
+ */
+public class StringMarshaller implements CompactBytesMarshaller {
+    private final int size;
+    private final StringBuilder reader = new StringBuilder();
+    private StringInterner interner;
+
+    public StringMarshaller(int size) {
+        this.size = size;
+    }
+
+    @Override
+    public void write(@NotNull Bytes bytes, String s) {
+        bytes.writeUTFΔ(s);
+    }
+
+    @Nullable
+    @Override
+    public String read(@NotNull Bytes bytes) {
+        if (bytes.readUTFΔ(reader))
+            return builderToString();
+        return null;
+    }
+
+
+    private String builderToString() {
+        if (interner == null)
+            interner = new StringInterner(size);
+        return interner.intern(reader);
+    }
+
+    public byte code() {
+        return 'S' & 31;
+    }
+}
diff --git a/lang/src/main/java/net/openhft/lang/io/serialization/impl/VanillaBytesMarshallerFactory.java b/lang/src/main/java/net/openhft/lang/io/serialization/impl/VanillaBytesMarshallerFactory.java
new file mode 100644
index 0000000..6fc66ee
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/io/serialization/impl/VanillaBytesMarshallerFactory.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io.serialization.impl;
+
+import net.openhft.lang.io.serialization.BytesMarshallable;
+import net.openhft.lang.io.serialization.BytesMarshaller;
+import net.openhft.lang.io.serialization.BytesMarshallerFactory;
+import net.openhft.lang.io.serialization.CompactBytesMarshaller;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.Externalizable;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * @author peter.lawrey
+ */
+public class VanillaBytesMarshallerFactory implements BytesMarshallerFactory {
+
+    private final Map marshallerMap = new LinkedHashMap();
+    private final BytesMarshaller[] compactMarshallerMap = new BytesMarshaller[256];
+
+    //    private final Map marshallerTextMap = new LinkedHashMap();
+    {
+        BytesMarshaller stringMarshaller = new StringMarshaller(16 * 1024);
+        addMarshaller(String.class, stringMarshaller);
+        addMarshaller(CharSequence.class, stringMarshaller);
+        addMarshaller(Class.class, new ClassMarshaller(Thread.currentThread().getContextClassLoader()));
+        addMarshaller(Date.class, new DateMarshaller(10191));
+        addMarshaller(Integer.class, new CompactEnumBytesMarshaller(Integer.class, 10191, (byte) ('I' & 31)));
+        addMarshaller(Long.class, new CompactEnumBytesMarshaller(Long.class, 10191, (byte) ('L' & 31)));
+        addMarshaller(Double.class, new CompactEnumBytesMarshaller(Double.class, 10191, (byte) ('D' & 31)));
+    }
+
+    @NotNull
+    @SuppressWarnings("unchecked")
+    @Override
+    public  BytesMarshaller acquireMarshaller(@NotNull Class eClass, boolean create) {
+        BytesMarshaller em = marshallerMap.get(eClass);
+        if (em == null)
+            if (eClass.isEnum())
+                marshallerMap.put(eClass, em = new EnumBytesMarshaller(eClass, null));
+            else if (BytesMarshallable.class.isAssignableFrom(eClass))
+                marshallerMap.put(eClass, em = new BytesMarshallableMarshaller((Class) eClass));
+            else if (Externalizable.class.isAssignableFrom(eClass))
+                marshallerMap.put(eClass, em = new ExternalizableMarshaller((Class) eClass));
+            else {
+                try {
+                    marshallerMap.put(eClass, em = new GenericEnumMarshaller(eClass, 1000));
+                } catch (Exception e) {
+                    marshallerMap.put(eClass, em = NoMarshaller.INSTANCE);
+                }
+            }
+        return em;
+    }
+
+    @Override
+    public  BytesMarshaller getMarshaller(byte code) {
+        return compactMarshallerMap[code & 0xFF];
+    }
+
+    public  void addMarshaller(Class eClass, BytesMarshaller marshaller) {
+        marshallerMap.put(eClass, marshaller);
+        if (marshaller instanceof CompactBytesMarshaller)
+            compactMarshallerMap[((CompactBytesMarshaller) marshaller).code()] = marshaller;
+    }
+}
diff --git a/lang/src/main/java/net/openhft/lang/model/Byteable.java b/lang/src/main/java/net/openhft/lang/model/Byteable.java
new file mode 100644
index 0000000..a6c1a54
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/model/Byteable.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model;
+
+import net.openhft.lang.io.Bytes;
+
+/**
+ * User: peter.lawrey
+ * Date: 07/10/13
+ * Time: 21:38
+ */
+public interface Byteable {
+    void bytes(Bytes bytes);
+
+    void bytes(Bytes bytes, long offset);
+
+    Bytes bytes();
+
+    int maxSize();
+}
\ No newline at end of file
diff --git a/lang/src/main/java/net/openhft/lang/model/ClassModel.java b/lang/src/main/java/net/openhft/lang/model/ClassModel.java
new file mode 100644
index 0000000..40bc614
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/model/ClassModel.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model;
+
+import java.lang.reflect.Method;
+import java.util.Map;
+import java.util.Set;
+
+public interface ClassModel {
+    void setMethod(String name, MethodFilter filter, Method method);
+
+    Set getNames();
+
+    Map filterMapFor(String name);
+}
diff --git a/lang/src/main/java/net/openhft/lang/model/CodeGenerator.java b/lang/src/main/java/net/openhft/lang/model/CodeGenerator.java
new file mode 100644
index 0000000..d1df93c
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/model/CodeGenerator.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model;
+
+public interface CodeGenerator {
+}
diff --git a/lang/src/main/java/net/openhft/lang/model/Copyable.java b/lang/src/main/java/net/openhft/lang/model/Copyable.java
new file mode 100644
index 0000000..79baf90
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/model/Copyable.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model;
+
+/**
+ * User: peter.lawrey
+ * Date: 08/10/13
+ * Time: 07:45
+ */
+public interface Copyable {
+    /**
+     * Copy from this type.
+     *
+     * @param from to copy from
+     */
+    void copyFrom(T from);
+}
diff --git a/lang/src/main/java/net/openhft/lang/model/DataValueGenerator.java b/lang/src/main/java/net/openhft/lang/model/DataValueGenerator.java
new file mode 100644
index 0000000..97ff8fe
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/model/DataValueGenerator.java
@@ -0,0 +1,483 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model;
+
+import net.openhft.compiler.CachedCompiler;
+import net.openhft.lang.Compare;
+import net.openhft.lang.Maths;
+import net.openhft.lang.io.Bytes;
+import net.openhft.lang.io.serialization.BytesMarshallable;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.lang.reflect.Method;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.logging.Logger;
+
+/**
+ * User: peter.lawrey
+ * Date: 06/10/13
+ * Time: 19:17
+ */
+public class DataValueGenerator {
+    public static final Comparator COMPARATOR = new Comparator() {
+        @Override
+        public int compare(Class o1, Class o2) {
+            return o1.getName().compareTo(o2.getName());
+        }
+    };
+    public static final Comparator> COMPARE_BY_HEAP_SIZE = new Comparator>() {
+        @Override
+        public int compare(Map.Entry o1, Map.Entry o2) {
+            // descending
+            int cmp = -Maths.compare(o1.getValue().heapSize(), o2.getValue().heapSize());
+            return cmp == 0 ? o1.getKey().compareTo(o2.getKey()) : cmp;
+        }
+    };
+    private static final Logger LOGGER = Logger.getLogger(DataValueGenerator.class.getName());
+    final CachedCompiler cc = new CachedCompiler(null, null);
+    private final Map heapClassMap = new ConcurrentHashMap();
+    private final Map nativeClassMap = new ConcurrentHashMap();
+    private boolean dumpCode = false;
+
+    private static String bytesType(Class type) {
+        if (type.isPrimitive())
+            return Character.toUpperCase(type.getName().charAt(0)) + type.getName().substring(1);
+        if (CharSequence.class.isAssignableFrom(type))
+            return "UTFΔ";
+        return "Object";
+    }
+
+    public  T heapInstance(Class tClass) {
+        try {
+            return (T) acquireHeapClass(tClass).newInstance();
+        } catch (Exception e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    public  Class acquireHeapClass(Class tClass) throws ClassNotFoundException {
+        Class heapClass = heapClassMap.get(tClass);
+        if (heapClass != null)
+            return heapClass;
+        String actual = new DataValueGenerator().generateHeapObject(tClass);
+        if (dumpCode)
+            LOGGER.info(actual);
+        heapClass = cc.loadFromJava(tClass.getClassLoader(), tClass.getName() + "£heap", actual);
+        heapClassMap.put(tClass, heapClass);
+        return heapClass;
+    }
+
+    public String generateHeapObject(Class tClass) {
+        return generateHeapObject(DataValueModels.acquireModel(tClass));
+    }
+
+    public String generateHeapObject(DataValueModel dvmodel) {
+        SortedSet imported = new TreeSet(COMPARATOR);
+        imported.add(BytesMarshallable.class);
+        imported.add(Bytes.class);
+        imported.add(IOException.class);
+        imported.add(Copyable.class);
+        imported.add(dvmodel.type());
+
+        StringBuilder fieldDeclarations = new StringBuilder();
+        StringBuilder getterSetters = new StringBuilder();
+        StringBuilder writeMarshal = new StringBuilder();
+        StringBuilder readMarshal = new StringBuilder();
+        StringBuilder copy = new StringBuilder();
+        Map fieldMap = dvmodel.fieldMap();
+        Map.Entry[] entries = fieldMap.entrySet().toArray(new Map.Entry[fieldMap.size()]);
+        Arrays.sort(entries, COMPARE_BY_HEAP_SIZE);
+        for (Map.Entry entry : entries) {
+            String name = entry.getKey();
+            FieldModel model = entry.getValue();
+            Class type = model.type();
+            if (!type.isPrimitive() && !type.getPackage().getName().equals("java.lang"))
+                imported.add(type);
+            fieldDeclarations.append("    private ").append(type.getName()).append(" _").append(name).append(";\n");
+
+            final Method setter = model.setter();
+            final Method getter = model.getter();
+            if (setter == null) {
+                copy.append("        ((Copyable) ").append(getter.getName()).append("()).copyFrom(from.").append(getter.getName()).append("());\n");
+            } else {
+                copy.append("        ").append(setter.getName()).append("(from.").append(getter.getName()).append("());\n");
+                Class setterType = setter.getParameterTypes()[0];
+                getterSetters.append("    public void ").append(setter.getName()).append('(').append(setterType.getName()).append(" _) {\n");
+                if (type == String.class && setterType != String.class)
+                    getterSetters.append("        _").append(name).append(" = _.toString();\n");
+                else
+                    getterSetters.append("        _").append(name).append(" = _;\n");
+                getterSetters.append("    }\n\n");
+            }
+            getterSetters.append("    public ").append(type.getName()).append(' ').append(getter.getName()).append("() {\n");
+            getterSetters.append("        return _").append(name).append(";\n");
+            getterSetters.append("    }\n\n");
+            Method adder = model.adder();
+            if (adder != null) {
+                getterSetters.append("    public ").append(type.getName()).append(' ').append(adder.getName())
+                        .append("(").append(adder.getParameterTypes()[0].getName()).append(" _) {\n")
+                        .append("        return _").append(name).append(" += _;\n")
+                        .append("    }");
+            }
+            Method atomicAdder = model.atomicAdder();
+            if (atomicAdder != null) {
+                getterSetters.append("    public synchronized ").append(type.getName()).append(' ').append(atomicAdder.getName())
+                        .append("(").append(adder.getParameterTypes()[0].getName()).append(" _) {\n")
+                        .append("        return _").append(name).append(" += _;\n")
+                        .append("    }");
+            }
+            Method cas = model.cas();
+            if (cas != null) {
+                getterSetters.append("    public synchronized boolean ").append(cas.getName()).append("(")
+                        .append(type.getName()).append(" _1, ")
+                        .append(type.getName()).append(" _2) {\n")
+                        .append("        if (_").append(name).append(" == _1) {\n")
+                        .append("            _").append(name).append(" = _2;\n")
+                        .append("            return true;\n")
+                        .append("        }\n")
+                        .append("        return false;\n")
+                        .append("    }\n");
+            }
+            Method tryLockNanos = model.tryLockNanos();
+            if (tryLockNanos != null) {
+                getterSetters.append("    public boolean ").append(tryLockNanos.getName()).append("(long nanos) {\n")
+                        .append("        throw new UnsupportedOperationException();\n")
+                        .append("    }");
+            }
+            Method tryLock = model.tryLock();
+            if (tryLock != null) {
+                getterSetters.append("    public boolean ").append(tryLock.getName()).append("() {\n")
+                        .append("        throw new UnsupportedOperationException();\n")
+                        .append("    }");
+            }
+            Method unlock = model.unlock();
+            if (unlock != null) {
+                getterSetters.append("    public void ").append(unlock.getName()).append("() {\n")
+                        .append("        throw new UnsupportedOperationException();\n")
+                        .append("    }");
+            }
+            Method busyLock = model.busyLock();
+            if (busyLock != null) {
+                getterSetters.append("    public void ").append(busyLock.getName()).append("() {\n")
+                        .append("        throw new UnsupportedOperationException();\n")
+                        .append("    }");
+            }
+            writeMarshal.append("         out.write").append(bytesType(type)).append("(_").append(name).append(");\n");
+            readMarshal.append("         _").append(name).append(" = in.read").append(bytesType(type)).append("(");
+            if ("Object".equals(bytesType(type)))
+                readMarshal.append(type.getName()).append(".class");
+            readMarshal.append(");\n");
+        }
+        StringBuilder sb = new StringBuilder();
+        sb.append("package ").append(dvmodel.type().getPackage().getName()).append(";\n\n");
+        sb.append("import static ").append(Compare.class.getName()).append(".*;\n");
+        for (Class aClass : imported) {
+            sb.append("import ").append(aClass.getName()).append(";\n");
+        }
+        sb.append("\npublic class ").append(dvmodel.type().getSimpleName())
+                .append("£heap implements ").append(dvmodel.type().getSimpleName())
+                .append(", BytesMarshallable, Copyable<").append(dvmodel.type().getName()).append(">  {\n");
+        sb.append(fieldDeclarations).append('\n');
+        sb.append(getterSetters);
+        sb.append("        @SuppressWarnings(\"unchecked\")\n" +
+                "        public void copyFrom(").append(dvmodel.type().getName()).append(" from) {\n");
+        sb.append(copy);
+        sb.append("    }\n\n");
+        sb.append("    public void writeMarshallable(Bytes out) {\n");
+        sb.append(writeMarshal);
+        sb.append("    }\n");
+        sb.append("    public void readMarshallable(Bytes in) {\n");
+        sb.append(readMarshal);
+        sb.append("    }\n");
+        if (Byteable.class.isAssignableFrom(dvmodel.type())) {
+            sb.append("    public void bytes(Bytes bytes) {\n");
+            sb.append("       throw new UnsupportedOperationException();\n");
+            sb.append("    }\n");
+            sb.append("    public void bytes(Bytes bytes, long l) {\n");
+            sb.append("       throw new UnsupportedOperationException();\n");
+            sb.append("    }\n");
+            sb.append("    public Bytes bytes() {\n");
+            sb.append("       return null;\n");
+            sb.append("    }\n");
+            sb.append("    public int maxSize() {\n");
+            sb.append("       throw new UnsupportedOperationException();\n");
+            sb.append("    }\n");
+        }
+        generateObjectMethods(sb, dvmodel, entries);
+        sb.append("}\n");
+//        System.out.println(sb);
+        return sb.toString();
+    }
+
+    private static void generateObjectMethods(StringBuilder sb, DataValueModel dvmodel, Map.Entry[] entries) {
+        int count = 0;
+        StringBuilder hashCode = new StringBuilder();
+        StringBuilder equals = new StringBuilder();
+        StringBuilder toString = new StringBuilder();
+        for (Map.Entry entry : entries) {
+            String name = entry.getKey();
+            FieldModel model = entry.getValue();
+            Class type = model.type();
+            if (count > 0)
+                hashCode.append(") * 10191 +\n            ");
+            String getterName = model.getter().getName();
+            hashCode.append("calcLongHashCode(").append(getterName).append("())");
+            equals.append("        if(!isEqual(").append(getterName).append("(), that.").append(getterName).append("())) return false;\n");
+            toString.append("            \", ").append(name).append("= \" + ").append(getterName).append("() +\n");
+            count++;
+        }
+        sb.append("    public int hashCode() {\n" +
+                "        long lhc = longHashCode();\n" +
+                "        return (int) ((lhc >>> 32) ^ lhc);\n" +
+                "    }\n" +
+                "\n" +
+                "    public long longHashCode() {\n" +
+                "        return ");
+        for (int i = 1; i < count; i++)
+            sb.append('(');
+        sb.append(hashCode);
+        String simpleName = dvmodel.type().getSimpleName();
+        sb.append(";\n")
+                .append("    }\n")
+                .append("\n")
+                .append("    public boolean equals(Object o) {\n")
+                .append("        if (this == o) return true;\n")
+                .append("        if (!(o instanceof ").append(simpleName).append(")) return false;\n")
+                .append("        ").append(simpleName).append(" that = (").append(simpleName).append(") o;\n")
+                .append("\n")
+                .append(equals)
+                .append("        return true;\n" +
+                        "    }\n" +
+                        "\n" +
+                        "    public String toString() {\n" +
+                        "        return \"").append(simpleName).append(" {\" +\n")
+                .append(toString.substring(0, toString.length() - 3)).append(";\n")
+                .append("    }");
+
+
+    }
+
+    public  T nativeInstance(Class tClass) {
+        try {
+            return (T) acquireNativeClass(tClass).newInstance();
+        } catch (Exception e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    public  Class acquireNativeClass(Class tClass) throws ClassNotFoundException {
+        Class nativeClass = nativeClassMap.get(tClass);
+        if (nativeClass != null)
+            return nativeClass;
+        DataValueModel dvmodel = DataValueModels.acquireModel(tClass);
+        for (Class clazz : dvmodel.nestedModels()) {
+            Class clazz2 = acquireNativeClass(clazz);
+        }
+        String actual = new DataValueGenerator().generateNativeObject(dvmodel);
+        if (dumpCode)
+            LOGGER.info(actual);
+        nativeClass = cc.loadFromJava(tClass.getClassLoader(), tClass.getName() + "£native", actual);
+        nativeClassMap.put(tClass, nativeClass);
+        return nativeClass;
+    }
+
+    public String generateNativeObject(Class tClass) {
+        return generateNativeObject(DataValueModels.acquireModel(tClass));
+    }
+
+    public String generateNativeObject(DataValueModel dvmodel) {
+        SortedSet imported = new TreeSet(COMPARATOR);
+        imported.add(BytesMarshallable.class);
+        imported.add(ObjectOutput.class);
+        imported.add(ObjectInput.class);
+        imported.add(IOException.class);
+        imported.add(Copyable.class);
+        imported.add(Byteable.class);
+        imported.add(Bytes.class);
+
+        StringBuilder staticFieldDeclarations = new StringBuilder();
+        StringBuilder fieldDeclarations = new StringBuilder();
+        StringBuilder getterSetters = new StringBuilder();
+        StringBuilder writeMarshal = new StringBuilder();
+        StringBuilder readMarshal = new StringBuilder();
+        StringBuilder copy = new StringBuilder();
+        StringBuilder nestedBytes = new StringBuilder();
+        Map fieldMap = dvmodel.fieldMap();
+        Map.Entry[] entries = fieldMap.entrySet().toArray(new Map.Entry[fieldMap.size()]);
+        Arrays.sort(entries, COMPARE_BY_HEAP_SIZE);
+        int offset = 0;
+        for (Map.Entry entry : entries) {
+            String name = entry.getKey();
+            FieldModel model = entry.getValue();
+            Class type = model.type();
+            if (!type.isPrimitive() && !type.getPackage().getName().equals("java.lang"))
+                imported.add(type);
+            String NAME = "_offset + " + name.toUpperCase();
+            final Method setter = model.setter();
+            final Method getter = model.getter();
+            if (dvmodel.isScalar(type)) {
+                staticFieldDeclarations.append("    private static final int ").append(name.toUpperCase()).append(" = ").append(offset).append(";\n");
+                copy.append("        ").append(setter.getName()).append("(from.").append(getter.getName()).append("());\n");
+                Class setterType = setter.getParameterTypes()[0];
+                getterSetters.append("    public void ").append(setter.getName()).append('(').append(setterType.getName()).append(" _) {\n");
+                getterSetters.append("        _bytes.write").append(bytesType(type)).append("(").append(NAME).append(", ");
+                if (CharSequence.class.isAssignableFrom(type))
+                    getterSetters.append(model.size().value()).append(", ");
+                getterSetters.append("_);\n");
+                getterSetters.append("    }\n\n");
+                getterSetters.append("    public ").append(type.getName()).append(' ').append(getter.getName()).append("() {\n");
+                getterSetters.append("        return _bytes.read").append(bytesType(type)).append("(").append(NAME).append(");\n");
+                getterSetters.append("    }\n\n");
+                Method adder = model.adder();
+                if (adder != null) {
+                    getterSetters.append("    public ").append(type.getName()).append(' ').append(adder.getName())
+                            .append("(").append(adder.getParameterTypes()[0].getName()).append(" _) {\n")
+                            .append("        return _bytes.add").append(bytesType(type)).append("(").append(NAME).append(", _);\n")
+                            .append("    }");
+                }
+                Method atomicAdder = model.atomicAdder();
+                if (atomicAdder != null) {
+                    getterSetters.append("    public ").append(type.getName()).append(' ').append(atomicAdder.getName())
+                            .append("(").append(adder.getParameterTypes()[0].getName()).append(" _) {\n")
+                            .append("        return _bytes.addAtomic").append(bytesType(type)).append("(").append(NAME).append(", _);\n")
+                            .append("    }");
+                }
+                Method cas = model.cas();
+                if (cas != null) {
+                    getterSetters.append("    public boolean ").append(cas.getName()).append("(")
+                            .append(type.getName()).append(" _1, ")
+                            .append(type.getName()).append(" _2) {\n")
+                            .append("        return _bytes.compareAndSwap").append(bytesType(type)).append('(').append(NAME).append(", _1, _2);\n")
+                            .append("    }");
+                }
+                Method tryLockNanos = model.tryLockNanos();
+                if (tryLockNanos != null) {
+                    getterSetters.append("    public boolean ").append(tryLockNanos.getName()).append("(long nanos) {\n")
+                            .append("        return _bytes.tryLockNanos").append(bytesType(type)).append('(').append(NAME).append(", nanos);\n")
+                            .append("    }");
+                }
+                Method tryLock = model.tryLock();
+                if (tryLock != null) {
+                    getterSetters.append("    public boolean ").append(tryLock.getName()).append("() {\n")
+                            .append("        return _bytes.tryLock").append(bytesType(type)).append('(').append(NAME).append(");\n")
+                            .append("    }");
+                }
+                Method unlock = model.unlock();
+                if (unlock != null) {
+                    getterSetters.append("    public void ").append(unlock.getName()).append("() {\n")
+                            .append("         _bytes.unlock").append(bytesType(type)).append('(').append(NAME).append(");\n")
+                            .append("    }");
+                }
+                Method busyLock = model.busyLock();
+                if (busyLock != null) {
+                    getterSetters.append("    public void ").append(busyLock.getName()).append("() throws InterruptedException {\n")
+                            .append("         _bytes.busyLock").append(bytesType(type)).append('(').append(NAME).append(");\n")
+                            .append("    }");
+                }
+                writeMarshal.append("         out.write").append(bytesType(type)).append("(")
+                        .append(getter.getName()).append("());\n");
+                readMarshal.append("         ").append(setter.getName()).append("(in.read").append(bytesType(type)).append("());\n");
+                offset += (model.nativeSize() + 7) >> 3;
+            } else {
+                staticFieldDeclarations.append("    private static final int ").append(name.toUpperCase()).append(" = ").append(offset).append(";\n");
+                fieldDeclarations.append("    private final ").append(type.getName()).append("£native _").append(name).append(" = new ").append(type.getName()).append("£native();\n");
+                if (setter == null) {
+                    copy.append("        _").append(name).append(".copyFrom(from.").append(getter.getName()).append("());\n");
+                } else {
+                    copy.append("        ").append(setter.getName()).append("(from.").append(getter.getName()).append("());\n");
+                    Class setterType = setter.getParameterTypes()[0];
+                    getterSetters.append("    public void ").append(setter.getName()).append('(').append(setterType.getName()).append(" _) {\n");
+                    if (type == String.class && setterType != String.class)
+                        getterSetters.append("        _").append(name).append(" = _.toString();\n");
+                    else
+                        getterSetters.append("        _").append(name).append(".copyFrom(_);\n");
+                    getterSetters.append("    }\n\n");
+                }
+
+                getterSetters.append("    public ").append(type.getName()).append(' ').append(getter.getName()).append("() {\n");
+                getterSetters.append("        return _").append(name).append(";\n");
+                getterSetters.append("    }\n\n");
+
+                writeMarshal.append("         _").append(name).append(".writeMarshallable(out);\n");
+                readMarshal.append("         _").append(name).append(".readMarshallable(in);\n");
+
+                nestedBytes.append("        ((Byteable) _").append(name).append(").bytes(bytes, ").append(NAME).append(");\n");
+                DataValueModel dvmodel2 = dvmodel.nestedModel(type);
+                Map fieldMap2 = dvmodel2.fieldMap();
+                Map.Entry[] entries2 = fieldMap2.entrySet().toArray(new Map.Entry[fieldMap2.size()]);
+                Arrays.sort(entries2, COMPARE_BY_HEAP_SIZE);
+                for (Map.Entry entry2 : entries2) {
+                    FieldModel model2 = entry2.getValue();
+                    offset += (model2.nativeSize() + 7) >> 3;
+                }
+            }
+        }
+        fieldDeclarations.append("\n")
+                .append("    private Bytes _bytes;\n")
+                .append("    private long _offset;\n");
+        StringBuilder sb = new StringBuilder();
+        sb.append("package ").append(dvmodel.type().getPackage().getName()).append(";\n\n");
+        sb.append("import static ").append(Compare.class.getName()).append(".*;\n");
+        for (Class aClass : imported) {
+            sb.append("import ").append(aClass.getName()).append(";\n");
+        }
+        sb.append("\npublic class ").append(dvmodel.type().getSimpleName())
+                .append("£native implements ").append(dvmodel.type().getSimpleName())
+                .append(", BytesMarshallable, Byteable, Copyable<").append(dvmodel.type().getName()).append("> {\n");
+        sb.append(staticFieldDeclarations).append('\n');
+        sb.append(fieldDeclarations).append('\n');
+        sb.append(getterSetters);
+        sb.append("    public void copyFrom(").append(dvmodel.type().getName()).append(" from) {\n");
+        sb.append(copy);
+        sb.append("    }\n\n");
+        sb.append("    public void writeMarshallable(Bytes out) {\n");
+        sb.append(writeMarshal);
+        sb.append("    }\n");
+        sb.append("    public void readMarshallable(Bytes in) {\n");
+        sb.append(readMarshal);
+        sb.append("    }\n");
+        sb.append("    public void bytes(Bytes bytes) {\n");
+        sb.append("       bytes(bytes, 0L);\n");
+        sb.append("    }\n");
+        sb.append("    public void bytes(Bytes bytes, long offset) {\n");
+        sb.append("       this._bytes = bytes;\n");
+        sb.append("       this._offset = offset;\n");
+        sb.append(nestedBytes);
+        sb.append("    }\n");
+        sb.append("    public Bytes bytes() {\n");
+        sb.append("       return _bytes;\n");
+        sb.append("    }\n");
+        sb.append("    public int maxSize() {\n");
+        sb.append("       return ").append(offset).append(";\n");
+        sb.append("    }\n");
+        generateObjectMethods(sb, dvmodel, entries);
+        sb.append("}\n");
+//        System.out.println(sb);
+        return sb.toString();
+    }
+
+    public boolean isDumpCode() {
+        return dumpCode;
+    }
+
+    public void setDumpCode(boolean dumpCode) {
+        this.dumpCode = dumpCode;
+    }
+}
diff --git a/lang/src/main/java/net/openhft/lang/model/DataValueMetaModel.java b/lang/src/main/java/net/openhft/lang/model/DataValueMetaModel.java
new file mode 100644
index 0000000..75263f5
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/model/DataValueMetaModel.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model;
+
+import net.openhft.lang.io.serialization.BytesMarshallable;
+
+import java.io.Externalizable;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public class DataValueMetaModel {
+    private final Set ignoredClasses = new HashSet();
+    private final List filters = new ArrayList();
+
+    public DataValueMetaModel() {
+        addIgnoredClass(Object.class);
+        addIgnoredClass(Externalizable.class);
+        addIgnoredClass(Copyable.class);
+        addIgnoredClass(Byteable.class);
+        addIgnoredClass(BytesMarshallable.class);
+
+        for (VanillaFilter vanillaFilter : VanillaFilter.values()) {
+            addMethodFilter(vanillaFilter);
+        }
+    }
+
+    public void addIgnoredClass(Class aClass) {
+        ignoredClasses.add(aClass);
+    }
+
+    public void addMethodFilter(MethodFilter filter) {
+        int pos = insertionPoint(filter);
+        filters.add(pos, filter);
+    }
+
+    private int insertionPoint(MethodFilter filter) {
+        for (int i = 0; i < filters.size(); i++)
+            if (filters.get(i).matches() < filter.matches())
+                return i;
+        return filters.size();
+    }
+
+
+}
diff --git a/lang/src/main/java/net/openhft/lang/model/DataValueModel.java b/lang/src/main/java/net/openhft/lang/model/DataValueModel.java
new file mode 100644
index 0000000..23e1567
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/model/DataValueModel.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model;
+
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * User: peter.lawrey
+ * Date: 06/10/13
+ * Time: 17:06
+ */
+public interface DataValueModel {
+    Map fieldMap();
+
+    boolean isScalar(Class nClass);
+
+    Set nestedModels();
+
+     DataValueModel nestedModel(Class nClass);
+
+    Class type();
+}
\ No newline at end of file
diff --git a/lang/src/main/java/net/openhft/lang/model/DataValueModelImpl.java b/lang/src/main/java/net/openhft/lang/model/DataValueModelImpl.java
new file mode 100644
index 0000000..bc5b03b
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/model/DataValueModelImpl.java
@@ -0,0 +1,461 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model;
+
+import net.openhft.lang.io.serialization.BytesMarshallable;
+import net.openhft.lang.model.constraints.Digits;
+import net.openhft.lang.model.constraints.MaxSize;
+import net.openhft.lang.model.constraints.Range;
+
+import java.io.Externalizable;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+/**
+ * User: peter.lawrey
+ * Date: 06/10/13
+ * Time: 17:23
+ */
+public class DataValueModelImpl implements DataValueModel {
+    static final Map HEAP_SIZE_MAP = new HashMap();
+
+    static {
+        HEAP_SIZE_MAP.put(boolean.class, 1);
+        HEAP_SIZE_MAP.put(byte.class, 8);
+        HEAP_SIZE_MAP.put(char.class, 16);
+        HEAP_SIZE_MAP.put(short.class, 16);
+        HEAP_SIZE_MAP.put(int.class, 32);
+        HEAP_SIZE_MAP.put(float.class, 32);
+        HEAP_SIZE_MAP.put(long.class, 64);
+        HEAP_SIZE_MAP.put(double.class, 64);
+    }
+
+    private final Map fieldModelMap = new TreeMap();
+    private final Class type;
+    private final Map nestedMap = new HashMap();
+
+    public DataValueModelImpl(Class type) {
+        this.type = type;
+        if (!type.isInterface())
+            throw new IllegalArgumentException("type must be an interface, was " + type);
+
+        Method[] methods = type.getMethods();
+        for (Method method : methods) {
+            Class declaringClass = method.getDeclaringClass();
+            if (declaringClass == Object.class
+                    || declaringClass == Externalizable.class
+                    || declaringClass == BytesMarshallable.class
+                    || declaringClass == Copyable.class
+                    || declaringClass == Byteable.class)
+                continue;
+            String name = method.getName();
+            Class[] parameterTypes = method.getParameterTypes();
+            final Class returnType = method.getReturnType();
+            switch (parameterTypes.length) {
+                case 0: {
+                    String name5 = getUnlock(name);
+                    if (name5 != null && returnType == void.class) {
+                        FieldModelImpl fm = acquireField(name5);
+                        fm.unlock(method);
+                        break;
+                    }
+                    String name4 = getBusyLock(name);
+                    if (name4 != null && returnType == void.class) {
+                        FieldModelImpl fm = acquireField(name4);
+                        fm.busyLock(method);
+                        break;
+                    }
+                    String name3 = getTryLock(name);
+                    if (name3 != null && returnType == boolean.class) {
+                        FieldModelImpl fm = acquireField(name3);
+                        fm.tryLock(method);
+                        break;
+                    }
+                    if (returnType == void.class)
+                        throw new IllegalArgumentException("void () not supported " + method);
+                    String name2 = getGetter(name, returnType);
+                    FieldModelImpl fm = acquireField(name2);
+                    fm.getter(method);
+                    break;
+                }
+                case 1: {
+                    String name5 = getTryLockNanos(name);
+                    if (name5 != null && returnType == boolean.class) {
+                        FieldModelImpl fm = acquireField(name5);
+                        fm.tryLockNanos(method);
+                        break;
+                    }
+
+                    String name4 = getAtomicAdder(name);
+                    if (name4 != null) {
+                        FieldModelImpl fm = acquireField(name4);
+                        fm.atomicAdder(method);
+                        break;
+                    }
+
+                    String name3 = getAdder(name);
+                    if (name3 != null) {
+                        FieldModelImpl fm = acquireField(name3);
+                        fm.adder(method);
+                        break;
+                    }
+
+                    String name6 = getGetter(name, returnType);
+                    if (name6 != null && parameterTypes[0] == int.class && returnType != void.class) {
+                        FieldModelImpl fm = acquireField(name6);
+                        fm.indexedGetter(method);
+                        break;
+                    }
+
+                    if (returnType != void.class)
+                        throw new IllegalArgumentException("setter must be void " + method);
+                    String name2 = getSetter(name);
+                    FieldModelImpl fm = acquireField(name2);
+                    fm.setter(method);
+                    break;
+                }
+                case 2: {
+                    String name2 = getCAS(name);
+                    if (name2 != null && returnType == boolean.class) {
+                        FieldModelImpl fm = acquireField(name2);
+                        fm.cas(method);
+                        break;
+                    }
+                    String name3 = getSetter(name);
+                    if (name3 != null && parameterTypes[0] == int.class && returnType == void.class) {
+                        FieldModelImpl fm = acquireField(name3);
+                        fm.indexedSetter(method);
+                        break;
+                    }
+                }
+                default: {
+                    throw new IllegalArgumentException("method not supported " + method);
+                }
+            }
+        }
+        for (Map.Entry entry : fieldModelMap.entrySet()) {
+            FieldModelImpl model = entry.getValue();
+            if (model.getter() == null || (model.setter() == null && model.getter().getReturnType().isPrimitive()))
+                if (model.indexedGetter() == null || (model.indexedSetter() == null && model.indexedGetter().getReturnType().isPrimitive()))
+                    throw new IllegalArgumentException("Field " + entry.getKey() + " must have a getter and setter.");
+            Class ftype = model.type();
+            if (!isScalar(ftype) && !nestedMap.containsKey(ftype))
+                nestedMap.put(ftype, new DataValueModelImpl(ftype));
+        }
+    }
+
+    private static String getCAS(String name) {
+        final int len = 14;
+        if (name.length() > len && name.startsWith("compareAndSwap") && Character.isUpperCase(name.charAt(len)))
+            return Character.toLowerCase(name.charAt(len)) + name.substring(len + 1);
+        return null;
+    }
+
+    private static String getAtomicAdder(String name) {
+        final int len = 9;
+        if (name.length() > len && name.startsWith("addAtomic") && Character.isUpperCase(name.charAt(len)))
+            return Character.toLowerCase(name.charAt(len)) + name.substring(len + 1);
+        return null;
+    }
+
+    private static String getAdder(String name) {
+        final int len = 3;
+        if (name.length() > len && name.startsWith("add") && Character.isUpperCase(name.charAt(len)))
+            return Character.toLowerCase(name.charAt(len)) + name.substring(len + 1);
+        return null;
+    }
+
+    private static String getSetter(String name) {
+        final int len = 3;
+        if (name.length() > len && name.startsWith("set") && Character.isUpperCase(name.charAt(len)))
+            return Character.toLowerCase(name.charAt(len)) + name.substring(len + 1);
+        return name;
+    }
+
+    private static String getGetter(String name, Class returnType) {
+        if (name.length() > 3 && name.startsWith("get") && Character.isUpperCase(name.charAt(3)))
+            return Character.toLowerCase(name.charAt(3)) + name.substring(4);
+        if ((returnType == boolean.class || returnType == Boolean.class)
+                && name.length() > 2 && name.startsWith("is") && Character.isUpperCase(name.charAt(2)))
+            return Character.toLowerCase(name.charAt(2)) + name.substring(3);
+        return name;
+    }
+
+    private String getBusyLock(String name) {
+        final int len = 8;
+        if (name.length() > len && name.startsWith("busyLock") && Character.isUpperCase(name.charAt(len)))
+            return Character.toLowerCase(name.charAt(len)) + name.substring(len + 1);
+        return null;
+    }
+
+    private String getUnlock(String name) {
+        final int len = 6;
+        if (name.length() > len && name.startsWith("unlock") && Character.isUpperCase(name.charAt(len)))
+            return Character.toLowerCase(name.charAt(len)) + name.substring(len + 1);
+        return null;
+    }
+
+    private String getTryLockNanos(String name) {
+        final int len = 12;
+        if (name.length() > len && name.startsWith("tryLockNanos") && Character.isUpperCase(name.charAt(len)))
+            return Character.toLowerCase(name.charAt(len)) + name.substring(len + 1);
+        return null;
+    }
+
+    private String getTryLock(String name) {
+        final int len = 7;
+        if (name.length() > len && name.startsWith("tryLock") && Character.isUpperCase(name.charAt(len)))
+            return Character.toLowerCase(name.charAt(len)) + name.substring(len + 1);
+        return null;
+    }
+
+    private FieldModelImpl acquireField(String name) {
+        FieldModelImpl fieldModelImpl = fieldModelMap.get(name);
+        if (fieldModelImpl == null)
+            fieldModelMap.put(name, fieldModelImpl = new FieldModelImpl(name));
+
+        return fieldModelImpl;
+    }
+
+    @Override
+    public Map fieldMap() {
+        return fieldModelMap;
+    }
+
+    public boolean isScalar(Class type) {
+        return type.isPrimitive() || CharSequence.class.isAssignableFrom(type);
+    }
+
+    @Override
+    public Set nestedModels() {
+        return nestedMap.keySet();
+    }
+
+    @Override
+    public  DataValueModel nestedModel(Class nClass) {
+        @SuppressWarnings("unchecked")
+        DataValueModel model = (DataValueModel) (nClass == type ? this : nestedMap.get(nClass));
+        return model;
+    }
+
+    @Override
+    public Class type() {
+        return type;
+    }
+
+    static class FieldModelImpl implements FieldModel {
+
+        private final String name;
+        private Method getter, setter;
+        private Digits digits;
+        private Range range;
+        private MaxSize maxSize;
+        private MaxSize indexSize;
+        private Method adder;
+        private Method atomicAdder;
+        private Method cas;
+        private Method tryLockNanos;
+        private Method tryLock;
+        private Method busyLock;
+        private Method unlock;
+        private Method indexedGetter;
+        private Method indexedSetter;
+
+        public FieldModelImpl(String name) {
+            this.name = name;
+        }
+
+        public String name() {
+            return name;
+        }
+
+        public void getter(Method getter) {
+            this.getter = getter;
+        }
+
+        public Method getter() {
+            return getter;
+        }
+
+        public void setter(Method setter) {
+            this.setter = setter;
+            for (Annotation a : setter.getParameterAnnotations()[0]) {
+                if (a instanceof Digits)
+                    digits = (Digits) a;
+                if (a instanceof Range)
+                    range = (Range) a;
+                if (a instanceof MaxSize)
+                    maxSize = (MaxSize) a;
+            }
+        }
+
+        public Method setter() {
+            return setter;
+        }
+
+        @Override
+        public Class type() {
+            return (Class) (getter != null ? getter.getReturnType() :
+                    indexedGetter != null ? indexedGetter.getReturnType() : null);
+        }
+
+        public void adder(Method method) {
+            adder = method;
+        }
+
+        public Method adder() {
+            return adder;
+        }
+
+        @Override
+        public int heapSize() {
+            Integer size = HEAP_SIZE_MAP.get(type());
+            if (size == null) return -1;
+            return size;
+        }
+
+        // maxSize in bits.
+        @Override
+        public int nativeSize() {
+            Integer size = HEAP_SIZE_MAP.get(type());
+            if (size != null)
+                return size;
+            MaxSize maxSize2 = size();
+            if (maxSize2 == null)
+                throw new AssertionError(type() + " without a @MaxSize not supported for native types");
+            return maxSize2.value() << 3;
+        }
+
+        @Override
+        public Digits digits() {
+            return digits;
+        }
+
+        @Override
+        public Range range() {
+            return range;
+        }
+
+        @Override
+        public MaxSize size() {
+            return maxSize;
+        }
+
+        @Override
+        public String toString() {
+            return "FieldModel{" +
+                    "name='" + name + '\'' +
+                    ", getter=" + (indexedGetter != null ? indexedGetter : getter) +
+                    ", setter=" + (indexedSetter != null ? indexedSetter : setter) +
+                    (digits == null ? "" : ", digits= " + digits) +
+                    (range == null ? "" : ", range= " + range) +
+                    (maxSize == null ? "" : ", size= " + maxSize) +
+                    ((indexedGetter == null && indexedSetter == null) ? "" : ", indexSize= " + indexSize) +
+                    '}';
+        }
+
+        public void atomicAdder(Method method) {
+            atomicAdder = method;
+        }
+
+        public Method atomicAdder() {
+            return atomicAdder;
+        }
+
+        public void cas(Method method) {
+            cas = method;
+        }
+
+        public Method cas() {
+            return cas;
+        }
+
+        public void tryLockNanos(Method method) {
+            tryLockNanos = method;
+        }
+
+        public Method tryLockNanos() {
+            return tryLockNanos;
+        }
+
+        public void tryLock(Method tryLock) {
+            this.tryLock = tryLock;
+        }
+
+        public Method tryLock() {
+            return tryLock;
+        }
+
+        public void busyLock(Method busyLock) {
+            this.busyLock = busyLock;
+        }
+
+        public Method busyLock() {
+            return busyLock;
+        }
+
+        public void unlock(Method unlock) {
+            this.unlock = unlock;
+        }
+
+        public Method unlock() {
+            return unlock;
+        }
+
+        public void indexSize(MaxSize indexSize) {
+            if (indexSize != null)
+                this.indexSize = indexSize;
+        }
+
+        public MaxSize indexSize() {
+            return indexSize;
+        }
+
+        public void indexedGetter(Method indexedGetter) {
+            this.indexedGetter = indexedGetter;
+            indexAnnotations(indexedGetter);
+        }
+
+        public Method indexedGetter() {
+            return indexedGetter;
+        }
+
+        public void indexedSetter(Method indexedSetter) {
+            this.indexedSetter = indexedSetter;
+            indexAnnotations(indexedSetter);
+        }
+
+        public void indexAnnotations(Method method) {
+            for (Annotation a : method.getParameterAnnotations()[0]) {
+//                if (a instanceof Digits)
+//                    digits = (Digits) a;
+//                if (a instanceof Range)
+//                    range = (Range) a;
+                if (a instanceof MaxSize)
+                    indexSize = (MaxSize) a;
+            }
+        }
+
+        public Method indexedSetter() {
+            return indexedSetter;
+        }
+    }
+}
diff --git a/lang/src/main/java/net/openhft/lang/model/DataValueModels.java b/lang/src/main/java/net/openhft/lang/model/DataValueModels.java
new file mode 100644
index 0000000..bda9620
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/model/DataValueModels.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model;
+
+import java.util.Map;
+import java.util.WeakHashMap;
+
+/**
+ * User: peter.lawrey
+ * Date: 06/10/13
+ * Time: 17:14
+ */
+public enum DataValueModels {
+    ;
+    private static final Map MODEL_MAP = new WeakHashMap();
+
+    public static synchronized  DataValueModel getModel(Class tClass) {
+        return MODEL_MAP.get(tClass);
+    }
+
+    public static synchronized  void putModel(Class tClass, DataValueModel model) {
+        MODEL_MAP.put(tClass, model);
+    }
+
+    public static  DataValueModel acquireModel(Class tClass) {
+        DataValueModel model = getModel(tClass);
+        if (model == null) {
+            model = new DataValueModelImpl(tClass);
+            putModel(tClass, model);
+        }
+        return model;
+    }
+}
diff --git a/lang/src/main/java/net/openhft/lang/model/FieldModel.java b/lang/src/main/java/net/openhft/lang/model/FieldModel.java
new file mode 100644
index 0000000..33869d0
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/model/FieldModel.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model;
+
+import net.openhft.lang.model.constraints.Digits;
+import net.openhft.lang.model.constraints.MaxSize;
+import net.openhft.lang.model.constraints.Range;
+
+import java.lang.reflect.Method;
+
+/**
+ * User: peter.lawrey
+ * Date: 06/10/13
+ * Time: 18:22
+ */
+public interface FieldModel {
+    String name();
+
+    Method getter();
+
+    Method setter();
+
+    Method adder();
+
+    Method atomicAdder();
+
+    Method cas();
+
+    Method tryLockNanos();
+
+    Method tryLock();
+
+    Method busyLock();
+
+    Method unlock();
+
+    Class type();
+
+    int heapSize();
+
+    int nativeSize();
+
+    Digits digits();
+
+    Range range();
+
+    MaxSize size();
+}
diff --git a/lang/src/main/java/net/openhft/lang/model/HeapCodeGenerator.java b/lang/src/main/java/net/openhft/lang/model/HeapCodeGenerator.java
new file mode 100644
index 0000000..78886a4
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/model/HeapCodeGenerator.java
@@ -0,0 +1,161 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model;
+
+public enum HeapCodeGenerator implements CodeGenerator {
+  /*  GET_VALUE {
+        public void addCode(CodeModel codeModel, Method method) {
+               StringBuilder sb = codeModel.acquireMethod()
+        }
+    },
+    SET_VALUE {
+        @Override
+        public int matches() {
+            return 3;
+        }
+
+        @Override
+        public String nameFor(Method method, Class[] parameterTypes) {
+            if (parameterTypes.length != 1 || method.getReturnType() == void.class)
+                return null;
+            final String name = method.getName();
+            final int pos = matches();
+            if (name.startsWith("set") && Character.isUpperCase(name.charAt(pos)))
+                return Character.toLowerCase(name.charAt(pos)) + name.substring(pos + 1);
+            return null;
+        }
+    },
+    ADD_VALUE {
+        @Override
+        public int matches() {
+            return 3;
+        }
+
+        @Override
+        public String nameFor(Method method, Class[] parameterTypes) {
+            if (parameterTypes.length != 1)
+                return null;
+            final String name = method.getName();
+            final int pos = matches();
+            if (name.startsWith("add") && Character.isUpperCase(name.charAt(pos)))
+                return Character.toLowerCase(name.charAt(pos)) + name.substring(pos + 1);
+            return null;
+        }
+    },
+    ADD_ATOMIC_VALUE {
+        @Override
+        public int matches() {
+            return 8;
+        }
+
+        @Override
+        public String nameFor(Method method, Class[] parameterTypes) {
+            if (parameterTypes.length != 1)
+                return null;
+            final String name = method.getName();
+            final int pos = matches();
+            if (name.startsWith("addAtomic") && Character.isUpperCase(name.charAt(pos)))
+                return Character.toLowerCase(name.charAt(pos)) + name.substring(pos + 1);
+            return null;
+        }
+    },
+    COMPARE_AND_SWAP_VALUE {
+        @Override
+        public int matches() {
+            return 15;
+        }
+
+        @Override
+        public String nameFor(Method method, Class[] parameterTypes) {
+            if (parameterTypes.length != 1)
+                return null;
+            final String name = method.getName();
+            final int pos = matches();
+            if (name.startsWith("compareAndSwap") && Character.isUpperCase(name.charAt(pos)))
+                return Character.toLowerCase(name.charAt(pos)) + name.substring(pos + 1);
+            return null;
+        }
+    },
+    TRY_LOCK_VALUE {
+        @Override
+        public int matches() {
+            return 7;
+        }
+
+        @Override
+        public String nameFor(Method method, Class[] parameterTypes) {
+            if (parameterTypes.length != 1)
+                return null;
+            final String name = method.getName();
+            final int pos = matches();
+            if (name.startsWith("tryLock") && Character.isUpperCase(name.charAt(pos)))
+                return Character.toLowerCase(name.charAt(pos)) + name.substring(pos + 1);
+            return null;
+        }
+    },
+    TRY_LOCK_NANOS_VALUE {
+        @Override
+        public int matches() {
+            return 12;
+        }
+
+        @Override
+        public String nameFor(Method method, Class[] parameterTypes) {
+            if (parameterTypes.length != 1)
+                return null;
+            final String name = method.getName();
+            final int pos = matches();
+            if (name.startsWith("tryLockNanos") && Character.isUpperCase(name.charAt(pos)))
+                return Character.toLowerCase(name.charAt(pos)) + name.substring(pos + 1);
+            return null;
+        }
+    },
+    BUSY_LOCK_VALUE {
+        @Override
+        public int matches() {
+            return 8;
+        }
+
+        @Override
+        public String nameFor(Method method, Class[] parameterTypes) {
+            if (parameterTypes.length != 1)
+                return null;
+            final String name = method.getName();
+            final int pos = matches();
+            if (name.startsWith("busyLock") && Character.isUpperCase(name.charAt(pos)))
+                return Character.toLowerCase(name.charAt(pos)) + name.substring(pos + 1);
+            return null;
+        }
+    },
+    UNLOCK_VALUE {
+        @Override
+        public int matches() {
+            return 6;
+        }
+
+        @Override
+        public String nameFor(Method method, Class[] parameterTypes) {
+            if (parameterTypes.length != 1)
+                return null;
+            final String name = method.getName();
+            final int pos = matches();
+            if (name.startsWith("unlock") && Character.isUpperCase(name.charAt(pos)))
+                return Character.toLowerCase(name.charAt(pos)) + name.substring(pos + 1);
+            return null;
+        }
+    },*/
+}
diff --git a/lang/src/main/java/net/openhft/lang/model/MethodFilter.java b/lang/src/main/java/net/openhft/lang/model/MethodFilter.java
new file mode 100644
index 0000000..32a3908
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/model/MethodFilter.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model;
+
+import java.lang.reflect.Method;
+
+public interface MethodFilter {
+    int matches();
+
+    String nameFor(Method method, Class[] parameterTypes);
+}
diff --git a/lang/src/main/java/net/openhft/lang/model/MethodTemplate.java b/lang/src/main/java/net/openhft/lang/model/MethodTemplate.java
new file mode 100644
index 0000000..1a38eee
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/model/MethodTemplate.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model;
+
+import java.lang.reflect.Method;
+
+public interface MethodTemplate {
+    void generate(Method method, ClassModel classModel);
+}
diff --git a/lang/src/main/java/net/openhft/lang/model/VanillaFilter.java b/lang/src/main/java/net/openhft/lang/model/VanillaFilter.java
new file mode 100644
index 0000000..3ec00bc
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/model/VanillaFilter.java
@@ -0,0 +1,175 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model;
+
+import java.lang.reflect.Method;
+
+public enum VanillaFilter implements MethodFilter {
+    GET_VALUE {
+        @Override
+        public int matches() {
+            return 3;
+        }
+
+        @Override
+        public String nameFor(Method method, Class[] parameterTypes) {
+            if (parameterTypes.length != 0 || method.getReturnType() != void.class)
+                return null;
+            final String name = method.getName();
+            final int pos = matches();
+            if (name.startsWith("get") && Character.isUpperCase(name.charAt(pos)))
+                return Character.toLowerCase(name.charAt(pos)) + name.substring(pos + 1);
+            return null;
+        }
+    },
+    SET_VALUE {
+        @Override
+        public int matches() {
+            return 3;
+        }
+
+        @Override
+        public String nameFor(Method method, Class[] parameterTypes) {
+            if (parameterTypes.length != 1 || method.getReturnType() == void.class)
+                return null;
+            final String name = method.getName();
+            final int pos = matches();
+            if (name.startsWith("set") && Character.isUpperCase(name.charAt(pos)))
+                return Character.toLowerCase(name.charAt(pos)) + name.substring(pos + 1);
+            return null;
+        }
+    },
+    ADD_VALUE {
+        @Override
+        public int matches() {
+            return 3;
+        }
+
+        @Override
+        public String nameFor(Method method, Class[] parameterTypes) {
+            if (parameterTypes.length != 1)
+                return null;
+            final String name = method.getName();
+            final int pos = matches();
+            if (name.startsWith("add") && Character.isUpperCase(name.charAt(pos)))
+                return Character.toLowerCase(name.charAt(pos)) + name.substring(pos + 1);
+            return null;
+        }
+    },
+    ADD_ATOMIC_VALUE {
+        @Override
+        public int matches() {
+            return 8;
+        }
+
+        @Override
+        public String nameFor(Method method, Class[] parameterTypes) {
+            if (parameterTypes.length != 1)
+                return null;
+            final String name = method.getName();
+            final int pos = matches();
+            if (name.startsWith("addAtomic") && Character.isUpperCase(name.charAt(pos)))
+                return Character.toLowerCase(name.charAt(pos)) + name.substring(pos + 1);
+            return null;
+        }
+    },
+    COMPARE_AND_SWAP_VALUE {
+        @Override
+        public int matches() {
+            return 15;
+        }
+
+        @Override
+        public String nameFor(Method method, Class[] parameterTypes) {
+            if (parameterTypes.length != 1)
+                return null;
+            final String name = method.getName();
+            final int pos = matches();
+            if (name.startsWith("compareAndSwap") && Character.isUpperCase(name.charAt(pos)))
+                return Character.toLowerCase(name.charAt(pos)) + name.substring(pos + 1);
+            return null;
+        }
+    },
+    TRY_LOCK_VALUE {
+        @Override
+        public int matches() {
+            return 7;
+        }
+
+        @Override
+        public String nameFor(Method method, Class[] parameterTypes) {
+            if (parameterTypes.length != 1)
+                return null;
+            final String name = method.getName();
+            final int pos = matches();
+            if (name.startsWith("tryLock") && Character.isUpperCase(name.charAt(pos)))
+                return Character.toLowerCase(name.charAt(pos)) + name.substring(pos + 1);
+            return null;
+        }
+    },
+    TRY_LOCK_NANOS_VALUE {
+        @Override
+        public int matches() {
+            return 12;
+        }
+
+        @Override
+        public String nameFor(Method method, Class[] parameterTypes) {
+            if (parameterTypes.length != 1)
+                return null;
+            final String name = method.getName();
+            final int pos = matches();
+            if (name.startsWith("tryLockNanos") && Character.isUpperCase(name.charAt(pos)))
+                return Character.toLowerCase(name.charAt(pos)) + name.substring(pos + 1);
+            return null;
+        }
+    },
+    BUSY_LOCK_VALUE {
+        @Override
+        public int matches() {
+            return 8;
+        }
+
+        @Override
+        public String nameFor(Method method, Class[] parameterTypes) {
+            if (parameterTypes.length != 1)
+                return null;
+            final String name = method.getName();
+            final int pos = matches();
+            if (name.startsWith("busyLock") && Character.isUpperCase(name.charAt(pos)))
+                return Character.toLowerCase(name.charAt(pos)) + name.substring(pos + 1);
+            return null;
+        }
+    },
+    UNLOCK_VALUE {
+        @Override
+        public int matches() {
+            return 6;
+        }
+
+        @Override
+        public String nameFor(Method method, Class[] parameterTypes) {
+            if (parameterTypes.length != 1)
+                return null;
+            final String name = method.getName();
+            final int pos = matches();
+            if (name.startsWith("unlock") && Character.isUpperCase(name.charAt(pos)))
+                return Character.toLowerCase(name.charAt(pos)) + name.substring(pos + 1);
+            return null;
+        }
+    },
+}
diff --git a/lang/src/main/java/net/openhft/lang/model/constraints/Digits.java b/lang/src/main/java/net/openhft/lang/model/constraints/Digits.java
new file mode 100644
index 0000000..4a1b1ee
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/model/constraints/Digits.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model.constraints;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+/**
+ * The maximum size in encoded bytes for a variable length data type.
+ */
+@Target(PARAMETER)
+@Retention(RUNTIME)
+@Documented
+public @interface Digits {
+    int integer() default 12;
+
+    int decimal() default 6;
+}
diff --git a/lang/src/main/java/net/openhft/lang/model/constraints/MaxSize.java b/lang/src/main/java/net/openhft/lang/model/constraints/MaxSize.java
new file mode 100644
index 0000000..4ca6c4a
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/model/constraints/MaxSize.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model.constraints;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+/**
+ * The maximum size in encoded bytes for a variable length data type.  The units is either elements, or bytes.
+ */
+@Target(PARAMETER)
+@Retention(RUNTIME)
+@Documented
+public @interface MaxSize {
+    int value() default 64;
+}
diff --git a/lang/src/main/java/net/openhft/lang/model/constraints/Range.java b/lang/src/main/java/net/openhft/lang/model/constraints/Range.java
new file mode 100644
index 0000000..2e99420
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/model/constraints/Range.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model.constraints;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+/**
+ * The lowest to highest values allowed (inclusive)
+ */
+@Target(PARAMETER)
+@Retention(RUNTIME)
+@Documented
+public @interface Range {
+    long min() default Long.MIN_VALUE;
+
+    long max() default Long.MAX_VALUE;
+}
diff --git a/lang/src/main/java/net/openhft/lang/pool/StringInterner.java b/lang/src/main/java/net/openhft/lang/pool/StringInterner.java
new file mode 100755
index 0000000..3796bc0
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/pool/StringInterner.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.pool;
+
+import net.openhft.lang.Maths;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * @author peter.lawrey
+ */
+public class StringInterner {
+    @NotNull
+    private final String[] interner;
+    private final int mask;
+
+    public StringInterner(int capacity) {
+        int n = Maths.nextPower2(capacity, 128);
+        interner = new String[n];
+        mask = n - 1;
+    }
+
+    private static boolean isEqual(@Nullable CharSequence s, @NotNull CharSequence cs) {
+        if (s == null) return false;
+        if (s.length() != cs.length()) return false;
+        for (int i = 0; i < cs.length(); i++)
+            if (s.charAt(i) != cs.charAt(i))
+                return false;
+        return true;
+    }
+
+/*    private static boolean isEqual(@Nullable CharSequence s, @NotNull byte[] bytes, int off, int len) {
+        if (s == null) return false;
+        if (s.length() != len) return false;
+        for (int i = 0; i < len; i++)
+            if (s.charAt(i) != (bytes[off + i] & 0xFF))
+                return false;
+        return true;
+    }
+
+    @NotNull
+    public String intern(@NotNull byte[] bytes, int off, int len) {
+        long hash = NativeBytes.longHash(bytes, off, len);
+        int h = Maths.hash(hash) & mask;
+        String s = interner[h];
+        if (isEqual(s, bytes, off, len))
+            return s;
+        String s2 = new String(bytes, off, len, IOTools.ISO_8859_1);
+        return interner[h] = s2;
+    }*/
+
+    @NotNull
+    public String intern(@NotNull CharSequence cs) {
+        long hash = 0;
+        for (int i = 0; i < cs.length(); i++)
+            hash = 57 * hash + cs.charAt(i);
+        int h = Maths.hash(hash) & mask;
+        String s = interner[h];
+        if (isEqual(s, cs))
+            return s;
+        String s2 = cs.toString();
+        return interner[h] = s2;
+    }
+}
diff --git a/lang/src/main/java/net/openhft/lang/testing/Differencer.java b/lang/src/main/java/net/openhft/lang/testing/Differencer.java
new file mode 100644
index 0000000..29d44f1
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/testing/Differencer.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.testing;
+
+/**
+ * User: peter.lawrey
+ * Date: 05/08/13
+ * Time: 19:06
+ */
+public interface Differencer {
+    long sample(long startTime, long endTime);
+}
diff --git a/lang/src/main/java/net/openhft/lang/testing/RunningMinimum.java b/lang/src/main/java/net/openhft/lang/testing/RunningMinimum.java
new file mode 100644
index 0000000..983f114
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/testing/RunningMinimum.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.testing;
+
+/**
+ * User: peter.lawrey
+ * Date: 05/08/13
+ * Time: 19:06
+ */
+public class RunningMinimum implements Differencer {
+    private final long actualMinimum;
+    private final int drift;
+    private long lastStartTime = Long.MIN_VALUE;
+    private long minimum = Long.MAX_VALUE;
+
+    public RunningMinimum(long actualMinimum) {
+        this(actualMinimum, 100 * 1000);
+    }
+
+    public RunningMinimum(long actualMinimum, int drift) {
+        this.actualMinimum = actualMinimum;
+        this.drift = drift;
+    }
+
+    @Override
+    public long sample(long startTime, long endTime) {
+        if (lastStartTime + drift <= startTime) {
+            if (lastStartTime != Long.MIN_VALUE)
+                minimum += (startTime - lastStartTime) / drift;
+            lastStartTime = startTime;
+        }
+        long delta = endTime - startTime;
+        if (minimum > delta)
+            minimum = delta;
+        return delta - minimum + actualMinimum;
+    }
+
+    public long minimum() {
+        return minimum;
+    }
+}
diff --git a/lang/src/main/java/net/openhft/lang/testing/VanillaDifferencer.java b/lang/src/main/java/net/openhft/lang/testing/VanillaDifferencer.java
new file mode 100644
index 0000000..3853861
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/testing/VanillaDifferencer.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.testing;
+
+/**
+ * User: peter.lawrey
+ * Date: 05/08/13
+ * Time: 19:07
+ */
+public class VanillaDifferencer implements Differencer {
+    @Override
+    public long sample(long startTime, long endTime) {
+        return endTime - startTime;
+    }
+}
diff --git a/lang/src/main/java/net/openhft/lang/thread/NamedThreadFactory.java b/lang/src/main/java/net/openhft/lang/thread/NamedThreadFactory.java
new file mode 100644
index 0000000..fc9895e
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/thread/NamedThreadFactory.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.thread;
+
+import org.jetbrains.annotations.NotNull;
+
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * User: peter.lawrey Date: 18/08/13 Time: 11:37
+ */
+public class NamedThreadFactory implements ThreadFactory {
+    private final AtomicInteger id = new AtomicInteger();
+    private final String name;
+    private final Boolean daemon;
+
+    public NamedThreadFactory(@NotNull String name) {
+        this(name, null);
+    }
+
+    public NamedThreadFactory(@NotNull String name, Boolean daemon) {
+        this.name = name;
+        this.daemon = daemon;
+    }
+
+    @NotNull
+    @Override
+    public Thread newThread(@NotNull Runnable r) {
+        int id = this.id.getAndIncrement();
+        String nameN = id == 0 ? name : (name + '-' + id);
+        Thread t = new Thread(r, nameN);
+        if (daemon != null)
+            t.setDaemon(daemon);
+        return t;
+    }
+}
diff --git a/lang/src/main/java/net/openhft/lang/values/BooleanValue.java b/lang/src/main/java/net/openhft/lang/values/BooleanValue.java
new file mode 100644
index 0000000..5af319c
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/values/BooleanValue.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.values;
+
+/**
+ * User: peter.lawrey
+ * Date: 10/10/13
+ * Time: 07:19
+ */
+public interface BooleanValue {
+    boolean getValue();
+
+    void setValue(boolean value);
+}
diff --git a/lang/src/main/java/net/openhft/lang/values/ByteValue.java b/lang/src/main/java/net/openhft/lang/values/ByteValue.java
new file mode 100644
index 0000000..b0d1f62
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/values/ByteValue.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.values;
+
+/**
+ * User: peter.lawrey
+ * Date: 10/10/13
+ * Time: 07:19
+ */
+public interface ByteValue {
+    byte getValue();
+
+    void setValue(byte value);
+
+    byte addValue(byte b);
+}
diff --git a/lang/src/main/java/net/openhft/lang/values/CharValue.java b/lang/src/main/java/net/openhft/lang/values/CharValue.java
new file mode 100644
index 0000000..d798208
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/values/CharValue.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.values;
+
+/**
+ * User: peter.lawrey
+ * Date: 10/10/13
+ * Time: 07:17
+ */
+public interface CharValue {
+    char getValue();
+
+    void setValue(char value);
+}
diff --git a/lang/src/main/java/net/openhft/lang/values/DoubleValue.java b/lang/src/main/java/net/openhft/lang/values/DoubleValue.java
new file mode 100644
index 0000000..8954f73
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/values/DoubleValue.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.values;
+
+/**
+ * User: peter.lawrey
+ * Date: 10/10/13
+ * Time: 07:12
+ */
+public interface DoubleValue {
+    double getValue();
+
+    void setValue(double value);
+
+    double addValue(double delta);
+
+    double addAtomicValue(double delta);
+}
diff --git a/lang/src/main/java/net/openhft/lang/values/FloatValue.java b/lang/src/main/java/net/openhft/lang/values/FloatValue.java
new file mode 100644
index 0000000..b56152b
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/values/FloatValue.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.values;
+
+/**
+ * User: peter.lawrey
+ * Date: 10/10/13
+ * Time: 07:15
+ */
+public interface FloatValue {
+    float getValue();
+
+    void setValue(float value);
+
+    float addValue(float delta);
+
+    float addAtomicValue(float delta);
+}
diff --git a/lang/src/main/java/net/openhft/lang/values/Int24Value.java b/lang/src/main/java/net/openhft/lang/values/Int24Value.java
new file mode 100644
index 0000000..538ea42
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/values/Int24Value.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.values;
+
+import net.openhft.lang.model.constraints.Range;
+
+/**
+ * 24-bit signed value
+ * User: peter.lawrey
+ * Date: 10/10/13
+ * Time: 07:15
+ */
+public interface Int24Value {
+    int getValue();
+
+    void setValue(@Range(min = -1 << 23, max = (1 << 23) - 1) int value);
+
+    int addValue(int delta);
+}
diff --git a/lang/src/main/java/net/openhft/lang/values/Int48Value.java b/lang/src/main/java/net/openhft/lang/values/Int48Value.java
new file mode 100644
index 0000000..bb7482b
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/values/Int48Value.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.values;
+
+import net.openhft.lang.model.constraints.Range;
+
+/**
+ * 48-bit signed value
+ * User: peter.lawrey
+ * Date: 10/10/13
+ * Time: 07:15
+ */
+public interface Int48Value {
+    long getValue();
+
+    void setValue(@Range(min = -1L << 47, max = (1L << 47) - 1) long value);
+
+    long addValue(long delta);
+}
diff --git a/lang/src/main/java/net/openhft/lang/values/IntValue.java b/lang/src/main/java/net/openhft/lang/values/IntValue.java
new file mode 100644
index 0000000..892d640
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/values/IntValue.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.values;
+
+/**
+ * User: peter.lawrey
+ * Date: 10/10/13
+ * Time: 07:15
+ */
+public interface IntValue {
+    int getValue();
+
+    void setValue(int value);
+
+    int addValue(int delta);
+
+    int addAtomicValue(int delta);
+
+    boolean compareAndSwapValue(int expected, int value);
+
+    boolean tryLockValue();
+
+    boolean tryLockNanosValue(long nanos);
+
+    void busyLockValue() throws InterruptedException, IllegalStateException;
+
+    void unlockValue() throws IllegalMonitorStateException;
+
+}
diff --git a/lang/src/main/java/net/openhft/lang/values/LongValue.java b/lang/src/main/java/net/openhft/lang/values/LongValue.java
new file mode 100644
index 0000000..b59b707
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/values/LongValue.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.values;
+
+/**
+ * User: peter.lawrey
+ * Date: 10/10/13
+ * Time: 07:11
+ */
+public interface LongValue {
+    long getValue();
+
+    void setValue(long value);
+
+    long addValue(long delta);
+
+    long addAtomicValue(long delta);
+
+    boolean compareAndSwapValue(long expected, long value);
+}
diff --git a/lang/src/main/java/net/openhft/lang/values/ShortValue.java b/lang/src/main/java/net/openhft/lang/values/ShortValue.java
new file mode 100644
index 0000000..4dd5547
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/values/ShortValue.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.values;
+
+/**
+ * User: peter.lawrey
+ * Date: 10/10/13
+ * Time: 07:16
+ */
+public interface ShortValue {
+    short getValue();
+
+    void setValue(short value);
+
+    short addValue(short delta);
+}
diff --git a/lang/src/main/java/net/openhft/lang/values/StringValue.java b/lang/src/main/java/net/openhft/lang/values/StringValue.java
new file mode 100644
index 0000000..2a0c587
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/values/StringValue.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.values;
+
+import net.openhft.lang.model.constraints.MaxSize;
+
+/**
+ * User: peter.lawrey
+ * Date: 10/10/13
+ * Time: 07:13
+ */
+public interface StringValue {
+    String getValue();
+
+    void setValue(@MaxSize CharSequence value);
+}
diff --git a/lang/src/main/java/net/openhft/lang/values/UnsignedByteValue.java b/lang/src/main/java/net/openhft/lang/values/UnsignedByteValue.java
new file mode 100644
index 0000000..7cc5007
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/values/UnsignedByteValue.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.values;
+
+import net.openhft.lang.model.constraints.Range;
+
+/**
+ * User: peter.lawrey
+ * Date: 10/10/13
+ * Time: 07:19
+ */
+public interface UnsignedByteValue {
+    int getValue();
+
+    void setValue(@Range(min = 0, max = 255) int value);
+
+    int addValue(int delta);
+}
diff --git a/lang/src/main/java/net/openhft/lang/values/UnsignedIntValue.java b/lang/src/main/java/net/openhft/lang/values/UnsignedIntValue.java
new file mode 100644
index 0000000..8eae1a3
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/values/UnsignedIntValue.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.values;
+
+import net.openhft.lang.model.constraints.Range;
+
+/**
+ * User: peter.lawrey
+ * Date: 10/10/13
+ * Time: 07:19
+ */
+public interface UnsignedIntValue {
+    long getValue();
+
+    void setValue(@Range(min = 0, max = (1L << 32) - 1) long value);
+
+    long addValue(long delta);
+
+}
diff --git a/lang/src/main/java/net/openhft/lang/values/UnsignedShortValue.java b/lang/src/main/java/net/openhft/lang/values/UnsignedShortValue.java
new file mode 100644
index 0000000..b9d06d6
--- /dev/null
+++ b/lang/src/main/java/net/openhft/lang/values/UnsignedShortValue.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.values;
+
+import net.openhft.lang.model.constraints.Range;
+
+/**
+ * User: peter.lawrey
+ * Date: 10/10/13
+ * Time: 07:19
+ */
+public interface UnsignedShortValue {
+    int getValue();
+
+    void setValue(@Range(min = 0, max = 65535) int value);
+
+    int addValue(int delta);
+}
diff --git a/lang/src/test/java/net/openhft/lang/JvmTest.java b/lang/src/test/java/net/openhft/lang/JvmTest.java
new file mode 100644
index 0000000..af3eae8
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/JvmTest.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang;
+
+import org.junit.Test;
+
+/**
+ * User: peter.lawrey
+ * Date: 20/09/13
+ * Time: 10:04
+ */
+public class JvmTest {
+    @Test
+    public void testIs64Bit() {
+        System.out.println("is64 = " + Jvm.is64Bit());
+    }
+
+    @Test
+    public void testGetProcessId() {
+        System.out.println("pid = " + Jvm.getProcessId());
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/MathsTest.java b/lang/src/test/java/net/openhft/lang/MathsTest.java
new file mode 100644
index 0000000..ab96efb
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/MathsTest.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang;
+
+import org.junit.Test;
+
+import java.math.BigDecimal;
+import java.util.Random;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * User: peter.lawrey
+ * Date: 20/09/13
+ * Time: 10:31
+ */
+public class MathsTest {
+    @Test
+    public void testIntLog2() {
+        for (int i = 0; i < 63; i++) {
+            long l = 1L << i;
+            assertEquals(i, Maths.intLog2(l));
+        }
+    }
+
+    @Test
+    public void testRounding() {
+        Random rand = new Random(1);
+        for (int i = 0; i < 1000; i++) {
+            double d = Math.pow(1e18, rand.nextDouble()) / 1e6;
+            BigDecimal bd = new BigDecimal(d);
+            assertEquals(bd.setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue(), Maths.round2(d), 5e-2);
+            assertEquals(bd.setScale(4, BigDecimal.ROUND_HALF_UP).doubleValue(), Maths.round4(d), 5e-4);
+            assertEquals(bd.setScale(6, BigDecimal.ROUND_HALF_UP).doubleValue(), Maths.round6(d), 5e-6);
+            if (d < 1e8)
+                assertEquals(bd.setScale(8, BigDecimal.ROUND_HALF_UP).doubleValue(), Maths.round8(d), 5e-8);
+        }
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/collection/HugeArrayTest.java b/lang/src/test/java/net/openhft/lang/collection/HugeArrayTest.java
new file mode 100644
index 0000000..0b10e01
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/collection/HugeArrayTest.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.collection;
+
+import net.openhft.lang.model.JavaBeanInterface;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * User: peter.lawrey
+ * Date: 08/10/13
+ * Time: 08:30
+ */
+public class HugeArrayTest {
+    static void assertEquals2(long a, long b) {
+        if (a != b)
+            org.junit.Assert.assertEquals(a, b);
+    }
+
+    @Test
+    public void testHugeArray() throws InterruptedException {
+        // runs with a maximum heap size of 32 MB.
+        int length = 10 * 1000 * 1000;
+        HugeArray array =
+                HugeCollections.newArray(JavaBeanInterface.class, length);
+        long start = System.nanoTime();
+        for (int i = 0; i < array.length(); i++) {
+            JavaBeanInterface jbi = array.get(i);
+//            jbi.busyLockRecord();
+//            try {
+            jbi.setByte((byte) i);
+            jbi.setChar((char) i);
+            jbi.setShort((short) i);
+            jbi.setInt(i);
+            jbi.setFloat(i);
+            jbi.setLong(i); // System.nanoTime());
+            jbi.setDouble(i);
+            jbi.setFlag((i & 3) == 0);
+            jbi.setString("hello");
+//            } finally {
+//                jbi.unlockRecord();
+//            }
+            array.recycle(jbi);
+        }
+        for (int i = 0; i < array.length(); i++) {
+            JavaBeanInterface jbi = array.get(i);
+//            jbi.busyLockRecord();
+//            try {
+            assertEquals2((byte) i, jbi.getByte());
+            assertEquals2((char) i, jbi.getChar());
+            assertEquals2((short) i, jbi.getShort());
+            assertEquals2(i, jbi.getInt());
+            assertEquals(i, jbi.getFloat(), 0);
+//            long time = System.nanoTime() - jbi.getLong();
+            assertEquals2(i, jbi.getLong());
+            assertEquals(i, jbi.getDouble(), 0.0);
+            assertEquals((i & 3) == 0, jbi.getFlag());
+            assertEquals("hello", jbi.getString());
+//            } finally {
+//                jbi.unlockRecord();
+//            }
+            array.recycle(jbi);
+        }
+        long time = System.nanoTime() - start;
+        double avg = time / 2.0 / length;
+        System.out.printf("Average time to access a JavaBeanInterface was %.1f ns%n", avg);
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/collection/HugePricesMain.java b/lang/src/test/java/net/openhft/lang/collection/HugePricesMain.java
new file mode 100644
index 0000000..2613063
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/collection/HugePricesMain.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.collection;
+
+import net.openhft.lang.model.constraints.MaxSize;
+import net.openhft.lang.model.constraints.Range;
+
+import static org.junit.Assert.assertEquals;
+
+interface Price {
+    String getInstrument();
+
+    void setInstrument(@MaxSize CharSequence instrument);
+
+    TimedQuote getAsk();
+
+    void setAsk(TimedQuote ask);
+
+    TimedQuote getBid();
+
+    void setBid(TimedQuote bid);
+}
+
+interface QuoteHeader {
+    int getMajor();
+
+    // @Start
+    void setMajor(@Range(min = 0, max = (1 << 24) - 1) int major);
+
+    void setType(char ch);
+
+    char getType();
+}
+
+interface Quote extends QuoteHeader {
+    double getPrice();
+
+    void setPrice(double price);
+
+    int getAmount();
+
+    void setAmount(int amount);
+}
+
+interface TimedQuote extends Quote {
+    long getTimestamp();
+
+    void setTimestamp(long price);
+}
+
+public class HugePricesMain {
+    public static void main(String[] args) {
+        int length = 1000;
+        final HugeArray prices =
+                HugeCollections.newArray(Price.class, length);
+        for (int i = 0; i < length; i++) {
+            final Price price = prices.get(i);
+            price.setInstrument("ID" + i);
+            price.getAsk().setPrice(100.1);
+            price.getAsk().setAmount(1000);
+            price.getBid().setPrice(99.8);
+            price.getBid().setAmount(2000);
+            prices.recycle(price);
+        }
+        for (int i = 0; i < length; i++) {
+            final Price price = prices.get(i);
+            assertEquals("ID" + i, price.getInstrument());
+            assertEquals(100.1, price.getAsk().getPrice(), 0.0);
+            assertEquals(1000, price.getAsk().getAmount());
+            assertEquals(99.8, price.getBid().getPrice(), 0.0);
+            assertEquals(2000, price.getBid().getAmount());
+            prices.recycle(price);
+        }
+    }
+}
+
diff --git a/lang/src/test/java/net/openhft/lang/collection/HugeQueueTest.java b/lang/src/test/java/net/openhft/lang/collection/HugeQueueTest.java
new file mode 100644
index 0000000..5c5d01c
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/collection/HugeQueueTest.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.collection;
+
+import net.openhft.lang.model.JavaBeanInterface;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * User: peter.lawrey
+ * Date: 08/10/13
+ * Time: 13:47
+ */
+public class HugeQueueTest {
+    @Test
+    public void testQueue() {
+        // runs with a maximum heap size of 32 MB.
+        int tests = 5 * 1000 * 1000;
+        int length = 1000;
+        HugeQueue queue = HugeCollections.newQueue(JavaBeanInterface.class, length);
+        long start = System.nanoTime();
+        for (int j = 0; j < tests; j += length) {
+            for (int i = 0; i < length; i++) {
+                JavaBeanInterface jbi = queue.offer();
+                jbi.setByte((byte) i);
+                jbi.setChar((char) i);
+                jbi.setShort((short) i);
+                jbi.setInt(i);
+                jbi.setFloat(i);
+                jbi.setLong(i);
+                jbi.setDouble(i);
+                jbi.setFlag((i & 3) == 0);
+                queue.recycle(jbi);
+            }
+            assertTrue(queue.isFull());
+            for (int i = 0; i < length; i++) {
+                JavaBeanInterface jbi = queue.take();
+                assertEquals((byte) i, jbi.getByte());
+                assertEquals((char) i, jbi.getChar());
+                assertEquals((short) i, jbi.getShort());
+                assertEquals(i, jbi.getInt());
+                assertEquals(i, jbi.getFloat(), 0);
+                assertEquals(i, jbi.getLong());
+                assertEquals(i, jbi.getDouble(), 0.0);
+                assertEquals((i & 3) == 0, jbi.getFlag());
+                queue.recycle(jbi);
+            }
+            assertTrue(queue.isEmpty());
+        }
+        long time = System.nanoTime() - start;
+        double avg = time / 2.0 / tests;
+        System.out.printf("Average time to access a JavaBeanInterface was %.1f ns%n", avg);
+
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/io/AllocationRatesTest.java b/lang/src/test/java/net/openhft/lang/io/AllocationRatesTest.java
new file mode 100644
index 0000000..da8696a
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/io/AllocationRatesTest.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io;
+
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+
+/**
+ * User: peter Date: 24/12/13 Time: 19:43
+ */
+/*
+buffers 128 KB took an average of 18,441 ns for heap ByteBuffer, 33,683 ns for direct ByteBuffer and 1,761 for DirectStore
+buffers 128 KB took an average of 13,062 ns for heap ByteBuffer, 17,855 ns for direct ByteBuffer and 903 for DirectStore
+buffers 128 KB took an average of 12,809 ns for heap ByteBuffer, 21,602 ns for direct ByteBuffer and 922 for DirectStore
+buffers 128 KB took an average of 10,768 ns for heap ByteBuffer, 21,444 ns for direct ByteBuffer and 894 for DirectStore
+buffers 128 KB took an average of 8,739 ns for heap ByteBuffer, 22,684 ns for direct ByteBuffer and 890 for DirectStore
+ */
+public class AllocationRatesTest {
+    static final int BUFFER_SIZE = 128 * 1024;
+    static final int ALLOCATIONS = 25000;
+    public static final int BATCH = 10;
+
+    @Test
+    public void compareAllocationRates() {
+        for (int i = 0; i < 5; i++) {
+            long timeHBB = timeHeapByteBufferAllocations();
+            long timeDBB = timeDirectByteBufferAllocations();
+            long timeDS = timeDirectStoreAllocations();
+            System.out.printf("buffers %d KB took an average of %,d ns for heap ByteBuffer, %,d ns for direct ByteBuffer and %,d for DirectStore%n",
+                    BUFFER_SIZE / 1024, timeHBB / ALLOCATIONS, timeDBB / ALLOCATIONS, timeDS / ALLOCATIONS
+            );
+        }
+    }
+
+    private long timeHeapByteBufferAllocations() {
+        long start = System.nanoTime();
+        for (int i = 0; i < ALLOCATIONS; i += BATCH) {
+            ByteBuffer[] bb = new ByteBuffer[BATCH];
+            for (int j = 0; j < BATCH; j++)
+                bb[j] = ByteBuffer.allocate(BUFFER_SIZE);
+        }
+        return System.nanoTime() - start;
+    }
+
+    private long timeDirectByteBufferAllocations() {
+        long start = System.nanoTime();
+        for (int i = 0; i < ALLOCATIONS; i += BATCH) {
+            ByteBuffer[] bb = new ByteBuffer[BATCH];
+            for (int j = 0; j < BATCH; j++)
+                bb[j] = ByteBuffer.allocateDirect(BUFFER_SIZE);
+        }
+        return System.nanoTime() - start;
+    }
+
+    private long timeDirectStoreAllocations() {
+        long start = System.nanoTime();
+        for (int i = 0; i < ALLOCATIONS; i += BATCH) {
+            DirectStore[] ds = new DirectStore[BATCH];
+            for (int j = 0; j < BATCH; j++)
+                ds[j] = DirectStore.allocateLazy(BUFFER_SIZE);
+            for (int j = 0; j < BATCH; j++)
+                ds[j].free();
+        }
+        return System.nanoTime() - start;
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/io/ByteBufferBytesTest.java b/lang/src/test/java/net/openhft/lang/io/ByteBufferBytesTest.java
new file mode 100644
index 0000000..9c2ec03
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/io/ByteBufferBytesTest.java
@@ -0,0 +1,920 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io;
+
+import net.openhft.lang.Maths;
+import net.openhft.lang.thread.NamedThreadFactory;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.*;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+import static net.openhft.lang.io.StopCharTesters.CONTROL_STOP;
+import static net.openhft.lang.io.StopCharTesters.SPACE_STOP;
+import static org.junit.Assert.*;
+
+/**
+ * Created with IntelliJ IDEA. User: peter.lawrey Date: 17/09/13 Time: 16:09 To change this template use File | Settings | File
+ * Templates.
+ */
+public class ByteBufferBytesTest {
+    public static final int SIZE = 128;
+    private ByteBufferBytes bytes;
+    private ByteBuffer byteBuffer;
+
+    @Before
+    public void beforeTest() {
+        byteBuffer = ByteBuffer.allocate(SIZE).order(ByteOrder.nativeOrder());
+        bytes = new ByteBufferBytes(byteBuffer);
+    }
+
+    @Test
+    public void testLongHash() throws Exception {
+        byte[] bytes = {1, 2, 3, 4, 5, 6, 7, 8};
+        long h = NativeBytes.longHash(bytes, 0, bytes.length);
+        assertFalse(h == 0);
+        byte[] bytes2 = {1, 2, 3, 4, 5, 6, 7, 8, 9};
+        long h2 = NativeBytes.longHash(bytes2, 0, bytes2.length);
+        assertFalse(h2 == 0);
+        assertFalse(h2 == h);
+    }
+
+    @Test
+    public void testRead() throws Exception {
+        for (int i = 0; i < bytes.capacity(); i++)
+            bytes.writeByte(i, i);
+        bytes.position(0);
+        for (int i = 0; i < bytes.capacity(); i++)
+            assertEquals((byte) i, bytes.read());
+        for (int i = (int) (bytes.capacity() - 1); i >= 0; i--) {
+            assertEquals((byte) i, bytes.readByte(i));
+        }
+
+    }
+
+    @Test
+    public void testReadFully() throws Exception {
+        for (int i = 0; i < bytes.capacity(); i++)
+            bytes.write(i);
+        bytes.position(0);
+        byte[] bytes = new byte[(int) this.bytes.capacity()];
+        this.bytes.readFully(bytes);
+        for (int i = 0; i < this.bytes.capacity(); i++)
+            assertEquals((byte) i, bytes[i]);
+    }
+
+    @Test
+    public void testCompareAndSetLong() throws Exception {
+        assertTrue(bytes.compareAndSwapLong(0, 0, 1));
+        assertFalse(bytes.compareAndSwapLong(0, 0, 1));
+        assertTrue(bytes.compareAndSwapLong(8, 0, 1));
+        assertTrue(bytes.compareAndSwapLong(0, 1, 2));
+
+    }
+
+    @Test
+    public void testPosition() throws Exception {
+        for (int i = 0; i < bytes.capacity(); i++)
+            bytes.write(i);
+        for (int i = (int) (bytes.capacity() - 1); i >= 0; i--) {
+            bytes.position(i);
+            assertEquals((byte) i, bytes.read());
+        }
+    }
+
+    @Test
+    public void testCapacity() throws Exception {
+        assertEquals(SIZE, bytes.capacity());
+        assertEquals(10, new NativeBytes(0, 0, 10).capacity());
+    }
+
+    @Test
+    public void testRemaining() throws Exception {
+        assertEquals(SIZE, bytes.remaining());
+        bytes.position(10);
+        assertEquals(SIZE - 10, bytes.remaining());
+    }
+
+    @Test
+    public void testByteOrder() throws Exception {
+        assertEquals(ByteOrder.nativeOrder(), bytes.byteOrder());
+    }
+
+    @Test
+    public void testCheckEndOfBuffer() throws Exception {
+        bytes.checkEndOfBuffer();
+
+        bytes.position(SIZE + 2);
+        try {
+            bytes.checkEndOfBuffer();
+            fail();
+        } catch (IndexOutOfBoundsException expected) {
+        }
+    }
+
+    @Test
+    public void testAppendDouble() {
+        testAppendDouble0(-6.895305375646115E24);
+        Random random = new Random(1);
+        for (int i = 0; i < 100000; i++) {
+            double d = Math.pow(1e32, random.nextDouble()) / 1e6;
+            if (i % 3 == 0) d = -d;
+            testAppendDouble0(d);
+        }
+    }
+
+    private void testAppendDouble0(double d) {
+        bytes.position(0);
+        bytes.append(d).append(' ');
+        bytes.position(0);
+        double d2 = bytes.parseDouble();
+        assertEquals(d, d2, 0);
+    }
+
+    @Test
+    public void testAppendDouble2() {
+//        testAppendDouble0(-0.93879148954440506, 14);
+//        testAppendDouble0(-0.214980202661, 12);
+//        testAppendDouble0(-0.937082148896, 12);
+        testAppendDouble0(0.17805, 5);
+        Random random = new Random(1);
+
+        for (int j = 0; j < 20000; j++) {
+            double d = random.nextDouble();
+            if (j % 3 == 0) d = -d;
+//            if (j % 5 == 0) d *= 1e6;
+            for (int i = 0; i < 4; i++) {
+                testAppendDouble0(d, i);
+            }
+        }
+    }
+
+    private void testAppendDouble0(double d, int precision) {
+        bytes.position(0);
+        bytes.append(d, precision).append(' ');
+        bytes.position(0);
+        String text = bytes.parseUTF(SPACE_STOP);
+        bytes.position(0);
+        assertEquals(0, bytes.position());
+        double d2 = bytes.parseDouble();
+        double d3 = (double) Math.round(d * Maths.power10(precision)) / Maths.power10(precision);
+//        if (precision >= 14)
+//            assertEquals("'" + text + "' p: " + precision + " v: " + new BigDecimal(d), d3, d2, 5e-29 * Maths.power10(precision));
+//        else
+        assertEquals("'" + text + "' p: " + precision, d3, d2, 0);
+    }
+
+    @Test
+    public void testWriteReadBytes() {
+        byte[] bytes = "Hello World!".getBytes();
+        this.bytes.write(bytes);
+        byte[] bytes2 = new byte[bytes.length];
+        this.bytes.position(0);
+        this.bytes.read(bytes2);
+        assertTrue(Arrays.equals(bytes, bytes2));
+
+        this.bytes.write(22, bytes);
+        byte[] bytes3 = new byte[bytes.length];
+        this.bytes.skipBytes((int) (22 - this.bytes.position()));
+        assertEquals(bytes3.length, this.bytes.read(bytes3));
+        assertTrue(Arrays.equals(bytes, bytes3));
+        this.bytes.position(this.bytes.capacity());
+        assertEquals(-1, this.bytes.read(bytes3));
+    }
+
+    @Test
+    public void testWriteReadUTFΔ() {
+        String[] words = "Hello,World!,Bye£€!".split(",");
+        for (String word : words) {
+            bytes.writeUTFΔ(word);
+        }
+        bytes.writeUTFΔ(null);
+        bytes.position(0);
+        for (String word : words) {
+            assertEquals(word, bytes.readUTFΔ());
+        }
+        assertEquals(null, bytes.readUTFΔ());
+        assertEquals("", bytes.readUTFΔ());
+        assertEquals(26, bytes.position()); // check the size
+
+        bytes.position(0);
+        StringBuilder sb = new StringBuilder();
+        for (String word : words) {
+            assertTrue(bytes.readUTFΔ(sb));
+            assertEquals(word, sb.toString());
+        }
+        assertFalse(bytes.readUTFΔ(sb));
+        assertTrue(bytes.readUTFΔ(sb));
+        assertEquals("", sb.toString());
+    }
+
+    @Test
+    public void testWriteReadUTF() {
+        String[] words = "Hello,World!,Bye£€!".split(",");
+        for (String word : words) {
+            bytes.writeUTF(word);
+        }
+        bytes.writeUTF("");
+        assertEquals(28, bytes.position()); // check the size, more bytes for less strings than writeUTFΔ
+        bytes.position(0);
+        for (String word : words) {
+            assertEquals(word, bytes.readUTF());
+        }
+        assertEquals("", bytes.readUTF());
+    }
+
+    @Test
+    public void testAppendParseUTF() {
+        String[] words = "Hello,World!,Bye£€!".split(",");
+        for (String word : words) {
+            bytes.append(word).append('\t');
+        }
+        bytes.append('\t');
+        bytes.position(0);
+        for (String word : words) {
+            assertEquals(word, bytes.parseUTF(CONTROL_STOP));
+        }
+        assertEquals("", bytes.parseUTF(CONTROL_STOP));
+
+        bytes.position(0);
+        StringBuilder sb = new StringBuilder();
+        for (String word : words) {
+            bytes.parseUTF(sb, CONTROL_STOP);
+            assertEquals(word, sb.toString());
+        }
+        bytes.parseUTF(sb, CONTROL_STOP);
+        assertEquals("", sb.toString());
+
+        bytes.position(0);
+        bytes.skipTo(CONTROL_STOP);
+        assertEquals(6, bytes.position());
+        bytes.skipTo(CONTROL_STOP);
+        assertEquals(13, bytes.position());
+        bytes.skipTo(CONTROL_STOP);
+        assertEquals(17, bytes.position());
+        bytes.skipTo(CONTROL_STOP);
+        assertEquals(18, bytes.position());
+
+        bytes.position(0);
+        bytes.stepBackAndSkipTo(CONTROL_STOP);
+        assertEquals(6, bytes.position());
+        bytes.stepBackAndSkipTo(CONTROL_STOP);
+        assertEquals(6, bytes.position());
+        bytes.position(10);
+        bytes.stepBackAndSkipTo(CONTROL_STOP);
+        assertEquals(13, bytes.position());
+
+    }
+
+    @Test
+    public void testWriteReadLines() {
+        byte[] bytes = "Hello\nWorld!\r\nBye".getBytes();
+        this.bytes.write(bytes);
+        this.bytes.position(0);
+        assertEquals("Hello", this.bytes.readLine());
+        assertEquals("World!", this.bytes.readLine());
+        assertTrue(this.bytes.readLine().startsWith("Bye"));
+    }
+
+    @Test
+    public void testWriteReadByteBuffer() {
+        byte[] bytes = "Hello\nWorld!\r\nBye".getBytes();
+        this.bytes.write(ByteBuffer.wrap(bytes));
+        this.bytes.position(0);
+        byte[] bytes2 = new byte[bytes.length + 1];
+        ByteBuffer bb2 = ByteBuffer.wrap(bytes2);
+        this.bytes.read(bb2);
+
+        assertEquals(bytes2.length, bb2.position());
+        assertTrue(Arrays.equals(bytes, Arrays.copyOf(bytes2, bytes.length)));
+    }
+
+    @Test
+    public void testReadWriteBoolean() {
+        for (int i = 0; i < 32; i++)
+            bytes.writeBoolean(i, (i & 3) == 0);
+        bytes.position(32);
+        for (int i = 32; i < 64; i++)
+            bytes.writeBoolean((i & 5) == 0);
+        bytes.position(0);
+        for (int i = 0; i < 32; i++)
+            assertEquals((i & 3) == 0, bytes.readBoolean());
+        for (int i = 32; i < 64; i++)
+            assertEquals((i & 5) == 0, bytes.readBoolean(i));
+    }
+
+    @Test
+    public void testReadWriteShort() {
+        for (int i = 0; i < 32; i += 2)
+            bytes.writeShort(i, i);
+        bytes.position(32);
+        for (int i = 32; i < 64; i += 2)
+            bytes.writeShort(i);
+        bytes.position(0);
+        for (int i = 0; i < 32; i += 2)
+            assertEquals(i, bytes.readShort());
+        for (int i = 32; i < 64; i += 2)
+            assertEquals(i, bytes.readShort(i));
+    }
+
+    @Test
+    public void testReadWriteCompactShort() {
+        int[] ints = {Short.MIN_VALUE, Short.MAX_VALUE, -125, 0, 127, -10000, 10000};
+        for (int i : ints) {
+            bytes.writeCompactShort(i);
+//            System.out.println(i + " " + bytes.position());
+        }
+        assertEquals(5 + 2 * 3, bytes.position());
+
+        bytes.position(0);
+        for (int i : ints)
+            assertEquals(i, bytes.readCompactShort());
+    }
+
+    @Test
+    public void testReadWriteCompactInt() {
+        int[] ints = {-10000000, Integer.MIN_VALUE, Integer.MAX_VALUE, Short.MIN_VALUE + 3, 0, Short.MAX_VALUE, 10000000};
+        for (int i : ints) {
+            bytes.writeCompactInt(i);
+//            System.out.println(i + " " + bytes.position());
+        }
+        assertEquals(5 * 2 + 2 * 6, bytes.position());
+
+        bytes.position(0);
+        for (int i : ints)
+            assertEquals(i, bytes.readCompactInt());
+    }
+
+    @Test
+    public void testReadWriteCompactLong() {
+        long[] ints = {Long.MAX_VALUE, -100000000000L, Long.MIN_VALUE, Integer.MIN_VALUE + 3, 0, Integer.MAX_VALUE, 100000000000L};
+        for (long i : ints) {
+            bytes.writeCompactLong(i);
+//            System.out.println(i + " " + bytes.position());
+        }
+        assertEquals(5 * 4 + 2 * 12, bytes.position());
+
+        bytes.position(0);
+        for (long i : ints)
+            assertEquals(i, bytes.readCompactLong());
+    }
+
+    @Test
+    public void testReadWriteCompactDouble() {
+        double[] doubles = {1, 1000, 1000000, -100000000, 0.1f, 0.1, 0.5, 0.51};
+        for (double i : doubles) {
+            bytes.writeCompactDouble(i);
+//            System.out.println(i + " " + bytes.position());
+        }
+        assertEquals(6 * 4 + 2 * 12, bytes.position());
+
+        bytes.position(0);
+        for (double i : doubles)
+            assertEquals(i, bytes.readCompactDouble(), 0.0);
+    }
+
+    @Test
+    public void testReadWriteStop() {
+        long[] longs = {Long.MIN_VALUE, Long.MAX_VALUE, Integer.MIN_VALUE, Integer.MAX_VALUE};
+        for (long i : longs) {
+            bytes.writeStopBit(i);
+//            System.out.println(i + " " + bytes.position());
+        }
+        assertEquals(9 + 10, +5 + 6, bytes.position());
+
+        bytes.position(0);
+        for (long i : longs)
+            assertEquals(i, bytes.readStopBit());
+    }
+
+    @Test
+    public void testReadWriteChar() {
+        for (int i = 0; i < 32; i += 2)
+            bytes.writeChar(i, i);
+        bytes.position(32);
+        for (int i = 32; i < 64; i += 2)
+            bytes.writeChar(i);
+        bytes.position(0);
+        for (int i = 0; i < 32; i += 2)
+            assertEquals(i, bytes.readChar());
+        for (int i = 32; i < 64; i += 2)
+            assertEquals(i, bytes.readChar(i));
+    }
+
+    @Test
+    public void testReadWriteUnsignedShort() {
+        for (int i = 0; i < 32; i += 2)
+            bytes.writeUnsignedShort(i, ~i);
+        bytes.position(32);
+        for (int i = 32; i < 64; i += 2)
+            bytes.writeUnsignedShort(~i);
+        bytes.position(0);
+        for (int i = 0; i < 32; i += 2)
+            assertEquals(~i & 0xFFFF, bytes.readUnsignedShort());
+        for (int i = 32; i < 64; i += 2)
+            assertEquals(~i & 0xFFFF, bytes.readUnsignedShort(i));
+    }
+
+    @Test
+    public void testReadWriteInt() {
+        for (int i = 0; i < 32; i += 4)
+            bytes.writeInt(i, i);
+        bytes.position(32);
+        for (int i = 32; i < 64; i += 4)
+            bytes.writeInt(i);
+        bytes.position(0);
+        for (int i = 0; i < 32; i += 4)
+            assertEquals(i, bytes.readInt());
+        for (int i = 32; i < 64; i += 4)
+            assertEquals(i, bytes.readInt(i));
+    }
+
+    @Test
+    public void testReadWriteThreadeSafeInt() {
+        for (int i = 0; i < 32; i += 4)
+            bytes.writeOrderedInt(i, i);
+        bytes.position(32);
+        for (int i = 32; i < 64; i += 4)
+            bytes.writeOrderedInt(i);
+        bytes.position(0);
+        for (int i = 0; i < 32; i += 4)
+            assertEquals(i, bytes.readVolatileInt());
+        for (int i = 32; i < 64; i += 4)
+            assertEquals(i, bytes.readVolatileInt(i));
+    }
+
+    @Test
+    public void testReadWriteFloat() {
+        for (int i = 0; i < 32; i += 4)
+            bytes.writeFloat(i, i);
+        bytes.position(32);
+        for (int i = 32; i < 64; i += 4)
+            bytes.writeFloat(i);
+        bytes.position(0);
+        for (int i = 0; i < 32; i += 4)
+            assertEquals(i, bytes.readFloat(), 0);
+        for (int i = 32; i < 64; i += 4)
+            assertEquals(i, bytes.readFloat(i), 0);
+    }
+
+    @Test
+    public void testReadWriteUnsignedInt() {
+        for (int i = 0; i < 32; i += 4)
+            bytes.writeUnsignedInt(i, ~i);
+        bytes.position(32);
+        for (int i = 32; i < 64; i += 4)
+            bytes.writeUnsignedInt(~i);
+        bytes.position(0);
+        for (int i = 0; i < 32; i += 4)
+            assertEquals(~i & 0xFFFFFFFFL, bytes.readUnsignedInt());
+        for (int i = 32; i < 64; i += 4)
+            assertEquals(~i & 0xFFFFFFFFL, bytes.readUnsignedInt(i));
+    }
+
+    @Test
+    public void testReadWriteInt24() {
+        for (int i = 0; i < 30; i += 3)
+            bytes.writeInt24(i, ~i & 0x7FFFFF);
+        bytes.position(30);
+        for (int i = 30; i < 63; i += 3)
+            bytes.writeInt24(~i & 0x7FFFFF);
+        assertEquals(63, bytes.position());
+        bytes.position(0);
+        for (int i = 0; i < 30; i += 3)
+            assertEquals("i: " + i, ~i & 0x7FFFFFL, bytes.readInt24());
+        for (int i = 30; i < 63; i += 3)
+            assertEquals("i: " + i, ~i & 0x7FFFFFL, bytes.readInt24(i));
+        // now negative
+        bytes.position(0);
+        for (int i = 0; i < 30; i += 3)
+            bytes.writeInt24(i, ~i);
+        bytes.position(30);
+        for (int i = 30; i < 63; i += 3)
+            bytes.writeInt24(~i);
+        assertEquals(63, bytes.position());
+        bytes.position(0);
+        for (int i = 0; i < 30; i += 3)
+            assertEquals("i: " + i, ~i << 8 >> 8, bytes.readInt24());
+        for (int i = 30; i < 63; i += 3)
+            assertEquals("i: " + i, ~i << 8 >> 8, bytes.readInt24(i));
+    }
+
+    @Test
+    public void testReadWriteInt48() {
+        for (long i = 0; i < 30; i += 6)
+            bytes.writeInt48(i, ~i & 0x7FFFFFFFFFFFL);
+        bytes.position(30);
+        for (long i = 30; i < 60; i += 6)
+            bytes.writeInt48(~i & 0x7FFFFFFFFFFFL);
+        assertEquals(60, bytes.position());
+        bytes.position(0);
+        for (long i = 0; i < 30; i += 6)
+            assertEquals("i: " + i, ~i & 0x7FFFFFFFFFFFL, bytes.readInt48());
+        for (long i = 30; i < 60; i += 6)
+            assertEquals("i: " + i, ~i & 0x7FFFFFFFFFFFL, bytes.readInt48(i));
+        // now negative
+        bytes.position(0);
+        for (long i = 0; i < 30; i += 6)
+            bytes.writeInt48(i, ~i);
+        bytes.position(30);
+        for (long i = 30; i < 60; i += 6)
+            bytes.writeInt48(~i);
+        assertEquals(60, bytes.position());
+        bytes.position(0);
+        for (long i = 0; i < 30; i += 6)
+            assertEquals("i: " + i, ~i << 16 >> 16, bytes.readInt48());
+        for (long i = 30; i < 60; i += 6)
+            assertEquals("i: " + i, ~i << 16 >> 16, bytes.readInt48(i));
+    }
+
+    @Test
+    public void testDateTimes() {
+        long now = System.currentTimeMillis();
+        bytes.appendDateTimeMillis(now);
+        bytes.append(' ');
+        bytes.appendDateMillis(now);
+        bytes.append('T');
+        bytes.appendTimeMillis(now % 86400000L);
+        assertEquals(23 * 2 + 1, bytes.position());
+        bytes.position(0);
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd'T'HH:mm:ss.SSS");
+        sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
+        String asStr = sdf.format(new Date(now));
+        assertEquals(asStr, bytes.parseUTF(SPACE_STOP));
+        assertEquals(asStr, bytes.parseUTF(SPACE_STOP));
+    }
+
+    @Test
+    public void testReadWriteCompactUnsignedShort() {
+        for (int i = 0; i < 64; i += 4)
+            bytes.writeCompactUnsignedShort(i);
+        assertEquals(64 / 4, bytes.position());
+        bytes.position(0);
+        for (int i = 0; i < 64; i += 4)
+            assertEquals(i, bytes.readByte(i / 4));
+        for (int i = 0; i < 64; i += 4)
+            assertEquals(i, bytes.readCompactUnsignedShort());
+    }
+
+    @Test
+    public void testReadWriteCompactUnsignedInt() {
+        for (int i = 0; i < 64; i += 4)
+            bytes.writeCompactUnsignedInt(i);
+        assertEquals(64 / 2, bytes.position());
+        bytes.position(0);
+        for (int i = 0; i < 64; i += 4)
+            assertEquals(i, bytes.readShort(i / 2));
+        for (int i = 0; i < 64; i += 4)
+            assertEquals(i, bytes.readCompactUnsignedInt());
+    }
+
+    @Test
+    public void testReadWriteLong() {
+        for (long i = 0; i < 32; i += 8)
+            bytes.writeLong(i, i);
+        bytes.position(32);
+        for (long i = 32; i < 64; i += 8)
+            bytes.writeLong(i);
+        bytes.position(0);
+        for (long i = 0; i < 32; i += 8)
+            assertEquals(i, bytes.readLong());
+        for (long i = 32; i < 64; i += 8)
+            assertEquals(i, bytes.readLong(i));
+    }
+
+    @Test
+    public void testReadWriteThreadSafeLong() {
+        for (long i = 0; i < 32; i += 8)
+            bytes.writeOrderedLong(i, i);
+        bytes.position(32);
+        for (long i = 32; i < 64; i += 8)
+            bytes.writeOrderedLong(i);
+        bytes.position(0);
+        for (long i = 0; i < 32; i += 8)
+            assertEquals(i, bytes.readVolatileLong());
+        for (long i = 32; i < 64; i += 8)
+            assertEquals(i, bytes.readVolatileLong(i));
+    }
+
+    @Test
+    public void testReadWriteDouble() {
+        for (long i = 0; i < 32; i += 8)
+            bytes.writeDouble(i, i);
+        bytes.position(32);
+        for (long i = 32; i < 64; i += 8)
+            bytes.writeDouble(i);
+        bytes.position(0);
+        for (long i = 0; i < 32; i += 8)
+            assertEquals(i, bytes.readDouble(), 0);
+        for (long i = 32; i < 64; i += 8)
+            assertEquals(i, bytes.readDouble(i), 0);
+    }
+
+    @Test
+    public void testAppendSubstring() {
+        bytes.append("Hello World", 2, 7).append("\n");
+        bytes.position(0);
+        assertEquals("Hello World".substring(2, 7), bytes.parseUTF(CONTROL_STOP));
+    }
+
+    @Test
+    public void testWriteReadEnum() {
+        bytes.append(BuySell.Buy).append("\t").append(BuySell.Sell);
+        bytes.position(0);
+        assertEquals(BuySell.Buy, bytes.parseEnum(BuySell.class, CONTROL_STOP));
+        assertEquals(BuySell.Sell, bytes.parseEnum(BuySell.class, CONTROL_STOP));
+        assertEquals(null, bytes.parseEnum(BuySell.class, CONTROL_STOP));
+    }
+
+    @Test
+    public void testAppendParse() {
+        bytes.append(false).append(' ');
+        bytes.append(true).append(' ');
+        bytes.append("what?").append(' ');
+        bytes.append("word£€").append(' ');
+        bytes.append(BuySell.Buy).append(' ');
+        bytes.append(1234).append(' ');
+        bytes.append(123456L).append(' ');
+        bytes.append(1.2345).append(' ');
+        bytes.append(1.5555, 3).append(' ');
+        bytes.position(0);
+        assertEquals(false, bytes.parseBoolean(SPACE_STOP));
+        assertEquals(true, bytes.parseBoolean(SPACE_STOP));
+        assertEquals(null, bytes.parseBoolean(SPACE_STOP));
+        assertEquals("word£€", bytes.parseUTF(SPACE_STOP));
+        assertEquals(BuySell.Buy, bytes.parseEnum(BuySell.class, SPACE_STOP));
+        assertEquals(1234, bytes.parseLong());
+        assertEquals(123456L, bytes.parseLong());
+        assertEquals(1.2345, bytes.parseDouble(), 0);
+        assertEquals(1.556, bytes.parseDouble(), 0);
+
+    }
+
+    @Test
+    public void testWriteByteChar() throws UnsupportedEncodingException {
+        bytes.writeBytes("Hello \u00ff\u01fe\u02fc\n");
+        bytes.writeChars("Hello \u00ff\u01fe\u02fc\n");
+        byte[] bytes = new byte[(int) this.bytes.position()];
+        this.bytes.position(0);
+        this.bytes.readFully(bytes);
+        assertEquals("Hello \u00ff\u00fe\u00fc\n" +
+                "H\u0000e\u0000l\u0000l\u0000o\u0000 \u0000ÿ\u0000þ\u0001ü\u0002\n" +
+                "\u0000", new String(bytes, "ISO-8859-1"));
+    }
+
+    @Test
+    public void testWriteBytes() {
+        bytes.write("Hello World\n".getBytes(), 0, 10);
+        bytes.write("good bye\n".getBytes(), 4, 4);
+        bytes.write(4, "0 w".getBytes());
+        bytes.position(0);
+        assertEquals("Hell0 worl bye", bytes.parseUTF(CONTROL_STOP));
+    }
+
+    @Test
+    public void testAppendIterable() {
+        bytes.append(Arrays.asList(1, 2, 3, 4, 5), ";").append(' ');
+        bytes.append(new TreeSet(Arrays.asList(21, 2, 13, 4, 5)), ";");
+        bytes.position(0);
+        assertEquals("1;2;3;4;5 2;4;5;13;21", bytes.parseUTF(CONTROL_STOP));
+    }
+
+    @Test
+    public void readWriteMutableDecimal() {
+        Random rand = new Random(2);
+        MutableDecimal md = new MutableDecimal();
+        MutableDecimal md2 = new MutableDecimal();
+//        md.set(1260042744, 0);
+
+        for (int i = 0; i < 20000; i++) {
+            int n = rand.nextInt();
+            for (int j = 0; j < 6; j++) {
+                testDecimal0(md, md2, n, j);
+            }
+        }
+    }
+
+    private void testDecimal0(MutableDecimal md, MutableDecimal md2, int n, int j) {
+        md.set(n, j);
+        bytes.position(0);
+        bytes.append(md).append('\n');
+        bytes.position(0);
+        bytes.parseDecimal(md2);
+        bytes.position(0);
+        String text = bytes.parseUTF(CONTROL_STOP);
+        if (!md.equals(md2))
+            assertEquals("n: " + n + ", s: " + j + " t: " + text, md, md2);
+    }
+
+    @Test
+    public void testStream() throws IOException {
+        bytes = new ByteBufferBytes(ByteBuffer.allocate(1000));
+        GZIPOutputStream out = new GZIPOutputStream(bytes.outputStream());
+        out.write("Hello world\n".getBytes());
+        out.close();
+        bytes.position(0);
+        GZIPInputStream in = new GZIPInputStream(bytes.inputStream());
+        byte[] bytes = new byte[12];
+        for (int i = 0; i < 12; i++)
+            bytes[i] = (byte) in.read();
+        assertEquals(-1, in.read());
+        assertEquals("Hello world\n", new String(bytes));
+        in.close();
+    }
+
+    @Test
+    public void testStream2() throws IOException {
+        OutputStream out = bytes.outputStream();
+        out.write(11);
+        out.write(22);
+        out.write(33);
+        out.write(44);
+        out.write(55);
+
+        bytes.position(0);
+        InputStream in = bytes.inputStream();
+        assertTrue(in.markSupported());
+        assertEquals(11, in.read());
+        in.mark(1);
+        assertEquals(1, bytes.position());
+        assertEquals(22, in.read());
+        assertEquals(2, bytes.position());
+
+        assertEquals(33, in.read());
+        in.reset();
+
+        assertEquals(1, bytes.position());
+        assertEquals(22, in.read());
+
+        assertEquals(2, in.skip(2));
+        assertEquals(4, bytes.position());
+        assertEquals(SIZE - 4, bytes.available());
+        assertEquals(55, in.read());
+        in.close();
+    }
+
+    @Test
+    public void testWriteObject() {
+        for (Object o : new Object[]{10, 9.9, "string", new Date(), BigDecimal.valueOf(1.1)}) {
+            bytes.position(0);
+            bytes.writeObject(o);
+//            System.out.println(o +" size: "+bytes.position());
+            assertTrue(bytes.position() < 21);
+            bytes.position(0);
+            Object o2 = bytes.readObject();
+            bytes.position(0);
+            Object o3 = bytes.readObject(o.getClass());
+            assertEquals(o, o2);
+            assertEquals(o, o3);
+        }
+    }
+
+    @Test
+    public void testWriteSerializable() {
+        int capacity = 16 * 1024;
+        byteBuffer = ByteBuffer.allocateDirect(capacity);
+        bytes = new ByteBufferBytes(byteBuffer);
+        Calendar cal = Calendar.getInstance();
+        bytes.writeObject(cal);
+        Dummy d = new Dummy();
+        bytes.writeObject(d);
+        bytes.position(0);
+        Calendar cal2 = bytes.readObject(Calendar.class);
+        Dummy d2 = bytes.readObject(Dummy.class);
+        assertEquals(cal, cal2);
+        assertEquals(d, d2);
+    }
+
+    @Test
+    public void testAddAndGet() {
+        for (int i = 0; i < 10; i++)
+            bytes.addAndGetInt(0L, 10);
+        assertEquals(100, bytes.readInt(0L));
+        assertEquals(0, bytes.readInt(4L));
+
+        for (int i = 0; i < 11; i++)
+            bytes.getAndAdd(4L, 11);
+        assertEquals(100, bytes.readInt(0L));
+        assertEquals(11 * 11, bytes.readInt(4L));
+    }
+
+    enum BuySell {
+        Buy, Sell
+    }
+
+    static class Dummy implements Serializable {
+        @Override
+        public boolean equals(Object obj) {
+            return obj instanceof Dummy;
+        }
+    }
+
+    @Test
+    public void testErrors() {
+        int capacity = 1024;
+        byteBuffer = ByteBuffer.allocate(capacity);
+        // it is actually much bigger than it believes
+        bytes = new ByteBufferBytes(byteBuffer, 0, 16);
+        bytes.writeLong(8);
+        assertFalse(bytes.isFinished());
+        bytes.finish();
+        assertTrue(bytes.isFinished());
+        bytes.flush();
+        bytes.writeLong(16);
+        bytes.finish();
+        bytes.flush();
+        try {
+            bytes.writeLong(24);
+            fail();
+        } catch (IndexOutOfBoundsException expected) {
+        }
+        bytes.finish();
+        bytes.flush();
+
+        bytes.reset();
+        assertEquals(0, bytes.position());
+        assertEquals(8, bytes.skip(8));
+        assertEquals(8, bytes.position());
+        bytes.writeLong(22);
+        bytes.close();
+    }
+
+    @Test
+    public void testWriteList() {
+        List ints = Arrays.asList(1, 2, 3, 4);
+        bytes.writeList(ints);
+        bytes.reset();
+        List ints2 = new ArrayList();
+        bytes.readList(ints2, Integer.class);
+        assertEquals(ints, ints2);
+
+        bytes.reset();
+        List words = Arrays.asList("Hello word byte for now".split(" "));
+        bytes.writeList(words);
+        bytes.reset();
+        List words2 = new ArrayList();
+        bytes.readList(words2, String.class);
+    }
+
+    @Test
+    public void testWriteMap() {
+        Map map = new LinkedHashMap() {
+            {
+                put("one", 1);
+                put("two", 2);
+                put("three", 3);
+                put("four", 4);
+            }
+        };
+
+        bytes.writeMap(map);
+        bytes.finish();
+
+        bytes.reset();
+        Map map2 = new LinkedHashMap();
+        bytes.readMap(map2, String.class, Integer.class);
+        assertEquals(map, map2);
+    }
+
+    @Test
+    public void unloadFailed() throws InterruptedException {
+        bytes.busyLockInt(0);
+        ExecutorService es = Executors.newSingleThreadExecutor(new NamedThreadFactory("unloadFailed"));
+        Future future = es.submit(new Callable() {
+            @Override
+            public Void call() throws Exception {
+                bytes.unlockInt(0);
+                return null;
+            }
+        });
+        es.shutdown();
+        try {
+            future.get();
+            fail();
+        } catch (ExecutionException e) {
+            assertEquals(IllegalMonitorStateException.class, e.getCause().getClass());
+        }
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/io/DirectByteBufferBytesTest.java b/lang/src/test/java/net/openhft/lang/io/DirectByteBufferBytesTest.java
new file mode 100644
index 0000000..bba0933
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/io/DirectByteBufferBytesTest.java
@@ -0,0 +1,921 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io;
+
+import net.openhft.lang.Maths;
+import net.openhft.lang.thread.NamedThreadFactory;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.*;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+import static net.openhft.lang.io.StopCharTesters.CONTROL_STOP;
+import static net.openhft.lang.io.StopCharTesters.SPACE_STOP;
+import static org.junit.Assert.*;
+
+/**
+ * Created with IntelliJ IDEA. User: peter.lawrey Date: 17/09/13 Time: 16:09 To change this template use File | Settings | File
+ * Templates.
+ */
+public class DirectByteBufferBytesTest {
+    public static final int SIZE = 128;
+    private ByteBufferBytes bytes;
+    private ByteBuffer byteBuffer;
+
+    @Before
+    public void beforeTest() {
+        byteBuffer = ByteBuffer.allocateDirect(SIZE).order(ByteOrder.nativeOrder());
+        bytes = new ByteBufferBytes(byteBuffer);
+    }
+
+    @Test
+    public void testLongHash() throws Exception {
+        byte[] bytes = {1, 2, 3, 4, 5, 6, 7, 8};
+        long h = NativeBytes.longHash(bytes, 0, bytes.length);
+        assertFalse(h == 0);
+        byte[] bytes2 = {1, 2, 3, 4, 5, 6, 7, 8, 9};
+        long h2 = NativeBytes.longHash(bytes2, 0, bytes2.length);
+        assertFalse(h2 == 0);
+        assertFalse(h2 == h);
+    }
+
+    @Test
+    public void testRead() throws Exception {
+        for (int i = 0; i < bytes.capacity(); i++)
+            bytes.writeByte(i, i);
+        bytes.position(0);
+        for (int i = 0; i < bytes.capacity(); i++)
+            assertEquals((byte) i, bytes.read());
+        for (int i = (int) (bytes.capacity() - 1); i >= 0; i--) {
+            assertEquals((byte) i, bytes.readByte(i));
+        }
+
+    }
+
+    @Test
+    public void testReadFully() throws Exception {
+        for (int i = 0; i < bytes.capacity(); i++)
+            bytes.write(i);
+        bytes.position(0);
+        byte[] bytes = new byte[(int) this.bytes.capacity()];
+        this.bytes.readFully(bytes);
+        for (int i = 0; i < this.bytes.capacity(); i++)
+            assertEquals((byte) i, bytes[i]);
+    }
+
+    @Test
+    public void testCompareAndSetLong() throws Exception {
+        assertTrue(bytes.compareAndSwapLong(0, 0, 1));
+        assertFalse(bytes.compareAndSwapLong(0, 0, 1));
+        assertTrue(bytes.compareAndSwapLong(8, 0, 1));
+        assertTrue(bytes.compareAndSwapLong(0, 1, 2));
+
+    }
+
+    @Test
+    public void testPosition() throws Exception {
+        for (int i = 0; i < bytes.capacity(); i++)
+            bytes.write(i);
+        for (int i = (int) (bytes.capacity() - 1); i >= 0; i--) {
+            bytes.position(i);
+            assertEquals((byte) i, bytes.read());
+        }
+    }
+
+    @Test
+    public void testCapacity() throws Exception {
+        assertEquals(SIZE, bytes.capacity());
+        assertEquals(10, new NativeBytes(0, 0, 10).capacity());
+    }
+
+    @Test
+    public void testRemaining() throws Exception {
+        assertEquals(SIZE, bytes.remaining());
+        bytes.position(10);
+        assertEquals(SIZE - 10, bytes.remaining());
+    }
+
+    @Test
+    public void testByteOrder() throws Exception {
+        assertEquals(ByteOrder.nativeOrder(), bytes.byteOrder());
+    }
+
+    @Test
+    public void testCheckEndOfBuffer() throws Exception {
+        bytes.checkEndOfBuffer();
+
+        bytes.position(SIZE + 2);
+        try {
+            bytes.checkEndOfBuffer();
+            fail();
+        } catch (IndexOutOfBoundsException expected) {
+        }
+    }
+
+    @Test
+    public void testAppendDouble() {
+        testAppendDouble0(-6.895305375646115E24);
+        Random random = new Random(1);
+        for (int i = 0; i < 100000; i++) {
+            double d = Math.pow(1e32, random.nextDouble()) / 1e6;
+            if (i % 3 == 0) d = -d;
+            testAppendDouble0(d);
+        }
+    }
+
+    private void testAppendDouble0(double d) {
+        bytes.position(0);
+        bytes.append(d).append(' ');
+        bytes.position(0);
+        double d2 = bytes.parseDouble();
+        assertEquals(d, d2, 0);
+    }
+
+    @Test
+    public void testAppendDouble2() {
+//        testAppendDouble0(-0.93879148954440506, 14);
+//        testAppendDouble0(-0.214980202661, 12);
+//        testAppendDouble0(-0.937082148896, 12);
+        testAppendDouble0(0.17805, 5);
+        Random random = new Random(1);
+
+        for (int j = 0; j < 20000; j++) {
+            double d = random.nextDouble();
+            if (j % 3 == 0) d = -d;
+//            if (j % 5 == 0) d *= 1e6;
+            for (int i = 0; i < 4; i++) {
+                testAppendDouble0(d, i);
+            }
+        }
+    }
+
+    private void testAppendDouble0(double d, int precision) {
+        bytes.position(0);
+        bytes.append(d, precision).append(' ');
+        bytes.position(0);
+        String text = bytes.parseUTF(SPACE_STOP);
+        bytes.position(0);
+        assertEquals(0, bytes.position());
+        double d2 = bytes.parseDouble();
+        double d3 = (double) Math.round(d * Maths.power10(precision)) / Maths.power10(precision);
+//        if (precision >= 14)
+//            assertEquals("'" + text + "' p: " + precision + " v: " + new BigDecimal(d), d3, d2, 5e-29 * Maths.power10(precision));
+//        else
+        assertEquals("'" + text + "' p: " + precision, d3, d2, 0);
+    }
+
+    @Test
+    public void testWriteReadBytes() {
+        byte[] bytes = "Hello World!".getBytes();
+        this.bytes.write(bytes);
+        byte[] bytes2 = new byte[bytes.length];
+        this.bytes.position(0);
+        this.bytes.read(bytes2);
+        assertTrue(Arrays.equals(bytes, bytes2));
+
+        this.bytes.write(22, bytes);
+        byte[] bytes3 = new byte[bytes.length];
+        this.bytes.skipBytes((int) (22 - this.bytes.position()));
+        assertEquals(bytes3.length, this.bytes.read(bytes3));
+        assertTrue(Arrays.equals(bytes, bytes3));
+        this.bytes.position(this.bytes.capacity());
+        assertEquals(-1, this.bytes.read(bytes3));
+    }
+
+    @Test
+    public void testWriteReadUTFΔ() {
+        String[] words = "Hello,World!,Bye£€!".split(",");
+        for (String word : words) {
+            bytes.writeUTFΔ(word);
+        }
+        bytes.writeUTFΔ(null);
+        bytes.position(0);
+        for (String word : words) {
+            assertEquals(word, bytes.readUTFΔ());
+        }
+        assertEquals(null, bytes.readUTFΔ());
+        assertEquals("", bytes.readUTFΔ());
+        assertEquals(26, bytes.position()); // check the size
+
+        bytes.position(0);
+        StringBuilder sb = new StringBuilder();
+        for (String word : words) {
+            assertTrue(bytes.readUTFΔ(sb));
+            assertEquals(word, sb.toString());
+        }
+        assertFalse(bytes.readUTFΔ(sb));
+        assertTrue(bytes.readUTFΔ(sb));
+        assertEquals("", sb.toString());
+    }
+
+    @Test
+    public void testWriteReadUTF() {
+        String[] words = "Hello,World!,Bye£€!".split(",");
+        for (String word : words) {
+            bytes.writeUTF(word);
+        }
+        bytes.writeUTF("");
+        assertEquals(28, bytes.position()); // check the size, more bytes for less strings than writeUTFΔ
+        bytes.position(0);
+        for (String word : words) {
+            assertEquals(word, bytes.readUTF());
+        }
+        assertEquals("", bytes.readUTF());
+    }
+
+    @Test
+    public void testAppendParseUTF() {
+        String[] words = "Hello,World!,Bye£€!".split(",");
+        for (String word : words) {
+            bytes.append(word).append('\t');
+        }
+        bytes.append('\t');
+        bytes.position(0);
+        for (String word : words) {
+            assertEquals(word, bytes.parseUTF(CONTROL_STOP));
+        }
+        assertEquals("", bytes.parseUTF(CONTROL_STOP));
+
+        bytes.position(0);
+        StringBuilder sb = new StringBuilder();
+        for (String word : words) {
+            bytes.parseUTF(sb, CONTROL_STOP);
+            assertEquals(word, sb.toString());
+        }
+        bytes.parseUTF(sb, CONTROL_STOP);
+        assertEquals("", sb.toString());
+
+        bytes.position(0);
+        bytes.skipTo(CONTROL_STOP);
+        assertEquals(6, bytes.position());
+        bytes.skipTo(CONTROL_STOP);
+        assertEquals(13, bytes.position());
+        bytes.skipTo(CONTROL_STOP);
+        assertEquals(17, bytes.position());
+        bytes.skipTo(CONTROL_STOP);
+        assertEquals(18, bytes.position());
+
+        bytes.position(0);
+        bytes.stepBackAndSkipTo(CONTROL_STOP);
+        assertEquals(6, bytes.position());
+        bytes.stepBackAndSkipTo(CONTROL_STOP);
+        assertEquals(6, bytes.position());
+        bytes.position(10);
+        bytes.stepBackAndSkipTo(CONTROL_STOP);
+        assertEquals(13, bytes.position());
+
+    }
+
+    @Test
+    public void testWriteReadLines() {
+        byte[] bytes = "Hello\nWorld!\r\nBye".getBytes();
+        this.bytes.write(bytes);
+        this.bytes.position(0);
+        assertEquals("Hello", this.bytes.readLine());
+        assertEquals("World!", this.bytes.readLine());
+        assertTrue(this.bytes.readLine().startsWith("Bye"));
+    }
+
+    @Test
+    public void testWriteReadByteBuffer() {
+        byte[] bytes = "Hello\nWorld!\r\nBye".getBytes();
+        this.bytes.write(ByteBuffer.wrap(bytes));
+        this.bytes.position(0);
+        byte[] bytes2 = new byte[bytes.length + 1];
+        ByteBuffer bb2 = ByteBuffer.wrap(bytes2);
+        this.bytes.read(bb2);
+
+        assertEquals(bytes2.length, bb2.position());
+        assertTrue(Arrays.equals(bytes, Arrays.copyOf(bytes2, bytes.length)));
+    }
+
+    @Test
+    public void testReadWriteBoolean() {
+        for (int i = 0; i < 32; i++)
+            bytes.writeBoolean(i, (i & 3) == 0);
+        bytes.position(32);
+        for (int i = 32; i < 64; i++)
+            bytes.writeBoolean((i & 5) == 0);
+        bytes.position(0);
+        for (int i = 0; i < 32; i++)
+            assertEquals((i & 3) == 0, bytes.readBoolean());
+        for (int i = 32; i < 64; i++)
+            assertEquals((i & 5) == 0, bytes.readBoolean(i));
+    }
+
+    @Test
+    public void testReadWriteShort() {
+        for (int i = 0; i < 32; i += 2)
+            bytes.writeShort(i, i);
+        bytes.position(32);
+        for (int i = 32; i < 64; i += 2)
+            bytes.writeShort(i);
+        bytes.position(0);
+        for (int i = 0; i < 32; i += 2)
+            assertEquals(i, bytes.readShort());
+        for (int i = 32; i < 64; i += 2)
+            assertEquals(i, bytes.readShort(i));
+    }
+
+    @Test
+    public void testReadWriteCompactShort() {
+        int[] ints = {Short.MIN_VALUE, Short.MAX_VALUE, -125, 0, 127, -10000, 10000};
+        for (int i : ints) {
+            bytes.writeCompactShort(i);
+//            System.out.println(i + " " + bytes.position());
+        }
+        assertEquals(5 + 2 * 3, bytes.position());
+
+        bytes.position(0);
+        for (int i : ints)
+            assertEquals(i, bytes.readCompactShort());
+    }
+
+    @Test
+    public void testReadWriteCompactInt() {
+        int[] ints = {-10000000, Integer.MIN_VALUE, Integer.MAX_VALUE, Short.MIN_VALUE + 3, 0, Short.MAX_VALUE, 10000000};
+        for (int i : ints) {
+            bytes.writeCompactInt(i);
+//            System.out.println(i + " " + bytes.position());
+        }
+        assertEquals(5 * 2 + 2 * 6, bytes.position());
+
+        bytes.position(0);
+        for (int i : ints)
+            assertEquals(i, bytes.readCompactInt());
+    }
+
+    @Test
+    public void testReadWriteCompactLong() {
+        long[] ints = {Long.MAX_VALUE, -100000000000L, Long.MIN_VALUE, Integer.MIN_VALUE + 3, 0, Integer.MAX_VALUE, 100000000000L};
+        for (long i : ints) {
+            bytes.writeCompactLong(i);
+//            System.out.println(i + " " + bytes.position());
+        }
+        assertEquals(5 * 4 + 2 * 12, bytes.position());
+
+        bytes.position(0);
+        for (long i : ints)
+            assertEquals(i, bytes.readCompactLong());
+    }
+
+    @Test
+    public void testReadWriteCompactDouble() {
+        double[] doubles = {1, 1000, 1000000, -100000000, 0.1f, 0.1, 0.5, 0.51};
+        for (double i : doubles) {
+            bytes.writeCompactDouble(i);
+//            System.out.println(i + " " + bytes.position());
+        }
+        assertEquals(6 * 4 + 2 * 12, bytes.position());
+
+        bytes.position(0);
+        for (double i : doubles)
+            assertEquals(i, bytes.readCompactDouble(), 0.0);
+    }
+
+    @Test
+    public void testReadWriteStop() {
+        long[] longs = {Long.MIN_VALUE, Long.MAX_VALUE, Integer.MIN_VALUE, Integer.MAX_VALUE};
+        for (long i : longs) {
+            bytes.writeStopBit(i);
+//            System.out.println(i + " " + bytes.position());
+        }
+        assertEquals(9 + 10, +5 + 6, bytes.position());
+
+        bytes.position(0);
+        for (long i : longs)
+            assertEquals(i, bytes.readStopBit());
+    }
+
+    @Test
+    public void testReadWriteChar() {
+        for (int i = 0; i < 32; i += 2)
+            bytes.writeChar(i, i);
+        bytes.position(32);
+        for (int i = 32; i < 64; i += 2)
+            bytes.writeChar(i);
+        bytes.position(0);
+        for (int i = 0; i < 32; i += 2)
+            assertEquals(i, bytes.readChar());
+        for (int i = 32; i < 64; i += 2)
+            assertEquals(i, bytes.readChar(i));
+    }
+
+    @Test
+    public void testReadWriteUnsignedShort() {
+        for (int i = 0; i < 32; i += 2)
+            bytes.writeUnsignedShort(i, ~i);
+        bytes.position(32);
+        for (int i = 32; i < 64; i += 2)
+            bytes.writeUnsignedShort(~i);
+        bytes.position(0);
+        for (int i = 0; i < 32; i += 2)
+            assertEquals(~i & 0xFFFF, bytes.readUnsignedShort());
+        for (int i = 32; i < 64; i += 2)
+            assertEquals(~i & 0xFFFF, bytes.readUnsignedShort(i));
+    }
+
+    @Test
+    public void testReadWriteInt() {
+        for (int i = 0; i < 32; i += 4)
+            bytes.writeInt(i, i);
+        bytes.position(32);
+        for (int i = 32; i < 64; i += 4)
+            bytes.writeInt(i);
+        bytes.position(0);
+        for (int i = 0; i < 32; i += 4)
+            assertEquals(i, bytes.readInt());
+        for (int i = 32; i < 64; i += 4)
+            assertEquals(i, bytes.readInt(i));
+    }
+
+    @Test
+    public void testReadWriteThreadeSafeInt() {
+        for (int i = 0; i < 32; i += 4)
+            bytes.writeOrderedInt(i, i);
+        bytes.position(32);
+        for (int i = 32; i < 64; i += 4)
+            bytes.writeOrderedInt(i);
+        bytes.position(0);
+        for (int i = 0; i < 32; i += 4)
+            assertEquals(i, bytes.readVolatileInt());
+        for (int i = 32; i < 64; i += 4)
+            assertEquals(i, bytes.readVolatileInt(i));
+    }
+
+    @Test
+    public void testReadWriteFloat() {
+        for (int i = 0; i < 32; i += 4)
+            bytes.writeFloat(i, i);
+        bytes.position(32);
+        for (int i = 32; i < 64; i += 4)
+            bytes.writeFloat(i);
+        bytes.position(0);
+        for (int i = 0; i < 32; i += 4)
+            assertEquals(i, bytes.readFloat(), 0);
+        for (int i = 32; i < 64; i += 4)
+            assertEquals(i, bytes.readFloat(i), 0);
+    }
+
+    @Test
+    public void testReadWriteUnsignedInt() {
+        for (int i = 0; i < 32; i += 4)
+            bytes.writeUnsignedInt(i, ~i);
+        bytes.position(32);
+        for (int i = 32; i < 64; i += 4)
+            bytes.writeUnsignedInt(~i);
+        bytes.position(0);
+        for (int i = 0; i < 32; i += 4)
+            assertEquals(~i & 0xFFFFFFFFL, bytes.readUnsignedInt());
+        for (int i = 32; i < 64; i += 4)
+            assertEquals(~i & 0xFFFFFFFFL, bytes.readUnsignedInt(i));
+    }
+
+    @Test
+    public void testReadWriteInt24() {
+        for (int i = 0; i < 30; i += 3)
+            bytes.writeInt24(i, ~i & 0x7FFFFF);
+        bytes.position(30);
+        for (int i = 30; i < 63; i += 3)
+            bytes.writeInt24(~i & 0x7FFFFF);
+        assertEquals(63, bytes.position());
+        bytes.position(0);
+        for (int i = 0; i < 30; i += 3)
+            assertEquals("i: " + i, ~i & 0x7FFFFFL, bytes.readInt24());
+        for (int i = 30; i < 63; i += 3)
+            assertEquals("i: " + i, ~i & 0x7FFFFFL, bytes.readInt24(i));
+        // now negative
+        bytes.position(0);
+        for (int i = 0; i < 30; i += 3)
+            bytes.writeInt24(i, ~i);
+        bytes.position(30);
+        for (int i = 30; i < 63; i += 3)
+            bytes.writeInt24(~i);
+        assertEquals(63, bytes.position());
+        bytes.position(0);
+        for (int i = 0; i < 30; i += 3)
+            assertEquals("i: " + i, ~i << 8 >> 8, bytes.readInt24());
+        for (int i = 30; i < 63; i += 3)
+            assertEquals("i: " + i, ~i << 8 >> 8, bytes.readInt24(i));
+    }
+
+    @Test
+    public void testReadWriteInt48() {
+        for (long i = 0; i < 30; i += 6)
+            bytes.writeInt48(i, ~i & 0x7FFFFFFFFFFFL);
+        bytes.position(30);
+        for (long i = 30; i < 60; i += 6)
+            bytes.writeInt48(~i & 0x7FFFFFFFFFFFL);
+        assertEquals(60, bytes.position());
+        bytes.position(0);
+        for (long i = 0; i < 30; i += 6)
+            assertEquals("i: " + i, ~i & 0x7FFFFFFFFFFFL, bytes.readInt48());
+        for (long i = 30; i < 60; i += 6)
+            assertEquals("i: " + i, ~i & 0x7FFFFFFFFFFFL, bytes.readInt48(i));
+        // now negative
+        bytes.position(0);
+        for (long i = 0; i < 30; i += 6)
+            bytes.writeInt48(i, ~i);
+        bytes.position(30);
+        for (long i = 30; i < 60; i += 6)
+            bytes.writeInt48(~i);
+        assertEquals(60, bytes.position());
+        bytes.position(0);
+        for (long i = 0; i < 30; i += 6)
+            assertEquals("i: " + i, ~i << 16 >> 16, bytes.readInt48());
+        for (long i = 30; i < 60; i += 6)
+            assertEquals("i: " + i, ~i << 16 >> 16, bytes.readInt48(i));
+    }
+
+    @Test
+    public void testDateTimes() {
+        long now = System.currentTimeMillis();
+        bytes.appendDateTimeMillis(now);
+        bytes.append(' ');
+        bytes.appendDateMillis(now);
+        bytes.append('T');
+        bytes.appendTimeMillis(now % 86400000L);
+        assertEquals(23 * 2 + 1, bytes.position());
+        bytes.position(0);
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd'T'HH:mm:ss.SSS");
+        sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
+        String asStr = sdf.format(new Date(now));
+        assertEquals(asStr, bytes.parseUTF(SPACE_STOP));
+        assertEquals(asStr, bytes.parseUTF(SPACE_STOP));
+    }
+
+    @Test
+    public void testReadWriteCompactUnsignedShort() {
+        for (int i = 0; i < 64; i += 4)
+            bytes.writeCompactUnsignedShort(i);
+        assertEquals(64 / 4, bytes.position());
+        bytes.position(0);
+        for (int i = 0; i < 64; i += 4)
+            assertEquals(i, bytes.readByte(i / 4));
+        for (int i = 0; i < 64; i += 4)
+            assertEquals(i, bytes.readCompactUnsignedShort());
+    }
+
+    @Test
+    public void testReadWriteCompactUnsignedInt() {
+        for (int i = 0; i < 64; i += 4)
+            bytes.writeCompactUnsignedInt(i);
+        assertEquals(64 / 2, bytes.position());
+        bytes.position(0);
+        for (int i = 0; i < 64; i += 4)
+            assertEquals(i, bytes.readShort(i / 2));
+        for (int i = 0; i < 64; i += 4)
+            assertEquals(i, bytes.readCompactUnsignedInt());
+    }
+
+    @Test
+    public void testReadWriteLong() {
+        for (long i = 0; i < 32; i += 8)
+            bytes.writeLong(i, i);
+        bytes.position(32);
+        for (long i = 32; i < 64; i += 8)
+            bytes.writeLong(i);
+        bytes.position(0);
+        for (long i = 0; i < 32; i += 8)
+            assertEquals(i, bytes.readLong());
+        for (long i = 32; i < 64; i += 8)
+            assertEquals(i, bytes.readLong(i));
+    }
+
+    @Test
+    public void testReadWriteThreadSafeLong() {
+        for (long i = 0; i < 32; i += 8)
+            bytes.writeOrderedLong(i, i);
+        bytes.position(32);
+        for (long i = 32; i < 64; i += 8)
+            bytes.writeOrderedLong(i);
+        bytes.position(0);
+        for (long i = 0; i < 32; i += 8)
+            assertEquals(i, bytes.readVolatileLong());
+        for (long i = 32; i < 64; i += 8)
+            assertEquals(i, bytes.readVolatileLong(i));
+    }
+
+    @Test
+    public void testReadWriteDouble() {
+        for (long i = 0; i < 32; i += 8)
+            bytes.writeDouble(i, i);
+        bytes.position(32);
+        for (long i = 32; i < 64; i += 8)
+            bytes.writeDouble(i);
+        bytes.position(0);
+        for (long i = 0; i < 32; i += 8)
+            assertEquals(i, bytes.readDouble(), 0);
+        for (long i = 32; i < 64; i += 8)
+            assertEquals(i, bytes.readDouble(i), 0);
+    }
+
+    @Test
+    public void testAppendSubstring() {
+        bytes.append("Hello World", 2, 7).append("\n");
+        bytes.position(0);
+        assertEquals("Hello World".substring(2, 7), bytes.parseUTF(CONTROL_STOP));
+    }
+
+    @Test
+    public void testWriteReadEnum() {
+        bytes.append(BuySell.Buy).append("\t").append(BuySell.Sell);
+        bytes.position(0);
+        assertEquals(BuySell.Buy, bytes.parseEnum(BuySell.class, CONTROL_STOP));
+        assertEquals(BuySell.Sell, bytes.parseEnum(BuySell.class, CONTROL_STOP));
+        assertEquals(null, bytes.parseEnum(BuySell.class, CONTROL_STOP));
+    }
+
+    @Test
+    public void testAppendParse() {
+        bytes.append(false).append(' ');
+        bytes.append(true).append(' ');
+        bytes.append("what?").append(' ');
+        bytes.append("word£€").append(' ');
+        bytes.append(BuySell.Buy).append(' ');
+        bytes.append(1234).append(' ');
+        bytes.append(123456L).append(' ');
+        bytes.append(1.2345).append(' ');
+        bytes.append(1.5555, 3).append(' ');
+        bytes.position(0);
+        assertEquals(false, bytes.parseBoolean(SPACE_STOP));
+        assertEquals(true, bytes.parseBoolean(SPACE_STOP));
+        assertEquals(null, bytes.parseBoolean(SPACE_STOP));
+        assertEquals("word£€", bytes.parseUTF(SPACE_STOP));
+        assertEquals(BuySell.Buy, bytes.parseEnum(BuySell.class, SPACE_STOP));
+        assertEquals(1234, bytes.parseLong());
+        assertEquals(123456L, bytes.parseLong());
+        assertEquals(1.2345, bytes.parseDouble(), 0);
+        assertEquals(1.556, bytes.parseDouble(), 0);
+
+    }
+
+    @Test
+    public void testWriteByteChar() throws UnsupportedEncodingException {
+        bytes.writeBytes("Hello \u00ff\u01fe\u02fc\n");
+        bytes.writeChars("Hello \u00ff\u01fe\u02fc\n");
+        byte[] bytes = new byte[(int) this.bytes.position()];
+        this.bytes.position(0);
+        this.bytes.readFully(bytes);
+        assertEquals("Hello \u00ff\u00fe\u00fc\n" +
+                "H\u0000e\u0000l\u0000l\u0000o\u0000 \u0000ÿ\u0000þ\u0001ü\u0002\n" +
+                "\u0000", new String(bytes, "ISO-8859-1"));
+    }
+
+    @Test
+    public void testWriteBytes() {
+        bytes.write("Hello World\n".getBytes(), 0, 10);
+        bytes.write("good bye\n".getBytes(), 4, 4);
+        bytes.write(4, "0 w".getBytes());
+        bytes.position(0);
+        assertEquals("Hell0 worl bye", bytes.parseUTF(CONTROL_STOP));
+    }
+
+    @Test
+    public void testAppendIterable() {
+        bytes.append(Arrays.asList(1, 2, 3, 4, 5), ";").append(' ');
+        bytes.append(new TreeSet(Arrays.asList(21, 2, 13, 4, 5)), ";");
+        bytes.position(0);
+        assertEquals("1;2;3;4;5 2;4;5;13;21", bytes.parseUTF(CONTROL_STOP));
+    }
+
+    @Test
+    public void readWriteMutableDecimal() {
+        Random rand = new Random(2);
+        MutableDecimal md = new MutableDecimal();
+        MutableDecimal md2 = new MutableDecimal();
+//        md.set(1260042744, 0);
+
+        for (int i = 0; i < 20000; i++) {
+            int n = rand.nextInt();
+            for (int j = 0; j < 6; j++) {
+                testDecimal0(md, md2, n, j);
+            }
+        }
+    }
+
+    private void testDecimal0(MutableDecimal md, MutableDecimal md2, int n, int j) {
+        md.set(n, j);
+        bytes.position(0);
+        bytes.append(md).append('\n');
+        bytes.position(0);
+        bytes.parseDecimal(md2);
+        bytes.position(0);
+        String text = bytes.parseUTF(CONTROL_STOP);
+        if (!md.equals(md2))
+            assertEquals("n: " + n + ", s: " + j + " t: " + text, md, md2);
+    }
+
+    @Test
+    public void testStream() throws IOException {
+        bytes = new ByteBufferBytes(ByteBuffer.allocateDirect(1000));
+        GZIPOutputStream out = new GZIPOutputStream(bytes.outputStream());
+        out.write("Hello world\n".getBytes());
+        out.close();
+        bytes.position(0);
+        GZIPInputStream in = new GZIPInputStream(bytes.inputStream());
+        byte[] bytes = new byte[12];
+        for (int i = 0; i < 12; i++)
+            bytes[i] = (byte) in.read();
+        assertEquals(-1, in.read());
+        assertEquals("Hello world\n", new String(bytes));
+        in.close();
+    }
+
+    @Test
+    public void testStream2() throws IOException {
+        OutputStream out = bytes.outputStream();
+        out.write(11);
+        out.write(22);
+        out.write(33);
+        out.write(44);
+        out.write(55);
+
+        bytes.position(0);
+        InputStream in = bytes.inputStream();
+        assertTrue(in.markSupported());
+        assertEquals(11, in.read());
+        in.mark(1);
+        assertEquals(1, bytes.position());
+        assertEquals(22, in.read());
+        assertEquals(2, bytes.position());
+
+        assertEquals(33, in.read());
+        in.reset();
+
+        assertEquals(1, bytes.position());
+        assertEquals(22, in.read());
+
+        assertEquals(2, in.skip(2));
+        assertEquals(4, bytes.position());
+        assertEquals(SIZE - 4, bytes.available());
+        assertEquals(55, in.read());
+        in.close();
+    }
+
+    @Test
+    public void testWriteObject() {
+        for (Object o : new Object[]{10, 9.9, "string", new Date(), BigDecimal.valueOf(1.1)}) {
+            bytes.position(0);
+            bytes.writeObject(o);
+//            System.out.println(o +" size: "+bytes.position());
+            assertTrue(bytes.position() < 21);
+            bytes.position(0);
+            Object o2 = bytes.readObject();
+            bytes.position(0);
+            Object o3 = bytes.readObject(o.getClass());
+            assertEquals(o, o2);
+            assertEquals(o, o3);
+        }
+    }
+
+    @Test
+    public void testWriteSerializable() {
+        int capacity = 16 * 1024;
+        byteBuffer = ByteBuffer.allocateDirect(capacity);
+        bytes = new ByteBufferBytes(byteBuffer);
+        Calendar cal = Calendar.getInstance();
+        bytes.writeObject(cal);
+        Dummy d = new Dummy();
+        bytes.writeObject(d);
+        bytes.position(0);
+        Calendar cal2 = bytes.readObject(Calendar.class);
+        Dummy d2 = bytes.readObject(Dummy.class);
+        assertEquals(cal, cal2);
+        assertEquals(d, d2);
+    }
+
+    @Test
+    public void testAddAndGet() {
+        for (int i = 0; i < 10; i++)
+            bytes.addAndGetInt(0L, 10);
+        assertEquals(100, bytes.readInt(0L));
+        assertEquals(0, bytes.readInt(4L));
+
+        for (int i = 0; i < 11; i++)
+            bytes.getAndAdd(4L, 11);
+        assertEquals(100, bytes.readInt(0L));
+        assertEquals(11 * 11, bytes.readInt(4L));
+    }
+
+    enum BuySell {
+        Buy, Sell
+    }
+
+    static class Dummy implements Serializable {
+        @Override
+        public boolean equals(Object obj) {
+            return obj instanceof Dummy;
+        }
+    }
+
+    @Test
+    public void testErrors() {
+        int capacity = 1024;
+        byteBuffer = ByteBuffer.allocateDirect(capacity);
+        // it is actually much bigger than it believes
+        bytes = new ByteBufferBytes(byteBuffer, 0, 16);
+        bytes.writeLong(8);
+        assertFalse(bytes.isFinished());
+        bytes.finish();
+        assertTrue(bytes.isFinished());
+        bytes.flush();
+        bytes.writeLong(16);
+        bytes.finish();
+        bytes.flush();
+
+        try {
+            bytes.writeLong(24);
+            fail();
+        } catch (IndexOutOfBoundsException expected) {
+        }
+        bytes.finish();
+        bytes.flush();
+
+        bytes.reset();
+        assertEquals(0, bytes.position());
+        assertEquals(8, bytes.skip(8));
+        assertEquals(8, bytes.position());
+        bytes.writeLong(22);
+        bytes.close();
+    }
+
+    @Test
+    public void testWriteList() {
+        List ints = Arrays.asList(1, 2, 3, 4);
+        bytes.writeList(ints);
+        bytes.reset();
+        List ints2 = new ArrayList();
+        bytes.readList(ints2, Integer.class);
+        assertEquals(ints, ints2);
+
+        bytes.reset();
+        List words = Arrays.asList("Hello word byte for now".split(" "));
+        bytes.writeList(words);
+        bytes.reset();
+        List words2 = new ArrayList();
+        bytes.readList(words2, String.class);
+    }
+
+    @Test
+    public void testWriteMap() {
+        Map map = new LinkedHashMap() {
+            {
+                put("one", 1);
+                put("two", 2);
+                put("three", 3);
+                put("four", 4);
+            }
+        };
+
+        bytes.writeMap(map);
+        bytes.finish();
+
+        bytes.reset();
+        Map map2 = new LinkedHashMap();
+        bytes.readMap(map2, String.class, Integer.class);
+        assertEquals(map, map2);
+    }
+
+    @Test
+    public void unloadFailed() throws InterruptedException {
+        bytes.busyLockInt(0);
+        ExecutorService es = Executors.newSingleThreadExecutor(new NamedThreadFactory("unloadFailed"));
+        Future future = es.submit(new Callable() {
+            @Override
+            public Void call() throws Exception {
+                bytes.unlockInt(0);
+                return null;
+            }
+        });
+        es.shutdown();
+        try {
+            future.get();
+            fail();
+        } catch (ExecutionException e) {
+            assertEquals(IllegalMonitorStateException.class, e.getCause().getClass());
+        }
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/io/DirectBytesTest.java b/lang/src/test/java/net/openhft/lang/io/DirectBytesTest.java
new file mode 100755
index 0000000..090b0b5
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/io/DirectBytesTest.java
@@ -0,0 +1,300 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io;
+
+import org.jetbrains.annotations.NotNull;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author peter.lawrey
+ */
+public class DirectBytesTest {
+    @Test
+    public void testSimpleLock() {
+        DirectBytes bytes = new DirectStore(64).createSlice();
+        bytes.writeLong(0, -1L);
+        assertFalse(bytes.tryLockLong(0));
+        bytes.writeLong(0, 0L);
+        assertTrue(bytes.tryLockLong(0));
+        long val1 = bytes.readLong(0);
+        assertTrue(bytes.tryLockLong(0));
+        bytes.unlockLong(0);
+        assertEquals(val1, bytes.readLong(0));
+        bytes.unlockLong(0);
+        assertEquals(0L, bytes.readLong(0));
+        try {
+            bytes.unlockLong(0);
+            fail();
+        } catch (IllegalMonitorStateException e) {
+            // expected.
+        }
+    }
+
+    private static void manyToggles(@NotNull DirectStore store1, int lockCount, int from, int to) {
+        long id = Thread.currentThread().getId();
+        assertEquals(0, id >>> 24);
+        System.out.println("Thread " + id);
+
+        DirectBytes slice1 = store1.createSlice();
+
+        int records = 64;
+        for (int i = 0; i < lockCount; i += records) {
+            for (long j = 0; j < records * 64; j += 64) {
+                slice1.positionAndSize(j, 64);
+                boolean condition = false;
+                for (int k = 0; k < 20; k++) {
+                    condition = slice1.tryLockInt(0L) || slice1.tryLockNanosInt(0L, 5 * 1000 * 1000);
+                    if (condition)
+                        break;
+                    if (k > 0)
+                        System.out.println("k: " + (5 + k * 5));
+                }
+                if (!condition)
+                    assertTrue("i= " + i, condition);
+                int toggle1 = slice1.readInt(4);
+                if (toggle1 == from) {
+                    slice1.writeInt(4L, to);
+                } else {
+                    // noinspection AssignmentToForLoopParameter,AssignmentToForLoopParameter
+                    i--;
+                }
+                slice1.unlockInt(0L);
+                System.currentTimeMillis(); // small delay
+            }
+        }
+    }
+
+    private static void manyLongToggles(@NotNull DirectStore store1, int lockCount, int from, int to) {
+        long id = Thread.currentThread().getId();
+        assertEquals(0, id >>> 32);
+        System.out.println("Thread " + id);
+
+        DirectBytes slice1 = store1.createSlice();
+
+        int records = 64;
+        for (int i = 0; i < lockCount; i += records) {
+            for (long j = 0; j < records * 64; j += 64) {
+                slice1.positionAndSize(j, 64);
+                boolean condition = false;
+                for (int k = 0; k < 20; k++) {
+                    condition = slice1.tryLockLong(0L) || slice1.tryLockNanosLong(0L, 5 * 1000 * 1000);
+                    if (condition)
+                        break;
+                    if (k > 0)
+                        System.out.println("k: " + (5 + k * 5));
+                }
+                if (!condition)
+                    assertTrue("i= " + i, condition);
+                long toggle1 = slice1.readLong(8);
+                if (toggle1 == from) {
+                    slice1.writeLong(8L, to);
+                } else {
+                    // noinspection AssignmentToForLoopParameter,AssignmentToForLoopParameter
+                    i--;
+                }
+                slice1.unlockLong(0L);
+                System.currentTimeMillis(); // small delay
+            }
+        }
+    }
+
+    @Test
+    public void testAllocate() throws Exception {
+        long size = 1L << 24; // 31; don't overload cloud-bees
+        DirectStore store = DirectStore.allocate(size);
+        assertEquals(size, store.size());
+        DirectBytes slice = store.createSlice();
+        slice.positionAndSize(0, size);
+        slice.writeLong(0, size);
+        slice.writeLong(size - 8, size);
+        store.free();
+    }
+
+    @Test
+    public void testLocking() {
+        if (Runtime.getRuntime().availableProcessors() < 2) {
+            System.err.println("Test requires 2 CPUs, skipping");
+            return;
+        }
+        long start = System.nanoTime();
+        // a page
+        final DirectStore store1 = DirectStore.allocate(1 << 12);
+        final int lockCount = 10 * 1000 * 1000;
+        new Thread(new Runnable() {
+            @Override
+            public void run() {
+                manyToggles(store1, lockCount, 1, 0);
+            }
+        }).start();
+
+        manyToggles(store1, lockCount, 0, 1);
+
+        store1.free();
+        long time = System.nanoTime() - start;
+        System.out.printf("Contended lock rate was %,d per second%n", (int) (lockCount * 2 * 1e9 / time));
+    }
+
+    @Test
+    public void testLockingLong() {
+        if (Runtime.getRuntime().availableProcessors() < 2) {
+            System.err.println("Test requires 2 CPUs, skipping");
+            return;
+        }
+        long start = System.nanoTime();
+        // a page
+        final DirectStore store1 = DirectStore.allocate(1 << 12);
+        final int lockCount = 10 * 1000 * 1000;
+        new Thread(new Runnable() {
+            @Override
+            public void run() {
+                manyLongToggles(store1, lockCount, 1, 0);
+            }
+        }).start();
+
+        manyLongToggles(store1, lockCount, 0, 1);
+
+        store1.free();
+        long time = System.nanoTime() - start;
+        System.out.printf("Contended lock rate was %,d per second%n", (int) (lockCount * 2 * 1e9 / time));
+    }
+
+    @Test
+    public void testLocking2() throws Exception {
+        if (Runtime.getRuntime().availableProcessors() < 2) {
+            System.err.println("Test requires 2 CPUs, skipping");
+            return;
+        }
+        // a page
+        final DirectStore store1 = DirectStore.allocate(1 << 12);
+        final DirectStore store2 = DirectStore.allocate(1 << 12);
+        final int lockCount = 1000000;
+
+        Thread t = new Thread(new Runnable() {
+            @Override
+            public void run() {
+                long id = Thread.currentThread().getId();
+                System.out.println("Thread " + id);
+                assertEquals(0, id >>> 24);
+                int expected = (1 << 24) | (int) id;
+                try {
+                    DirectBytes slice1 = store1.createSlice();
+                    DirectBytes slice2 = store2.createSlice();
+
+                    for (int i = 0; i < lockCount; i += 2) {
+                        slice1.busyLockInt(0);
+                        slice2.busyLockInt(0);
+                        int lockValue1 = slice1.readInt(0);
+                        if (lockValue1 != expected)
+                            assertEquals(expected, lockValue1);
+                        int lockValue2 = slice2.readInt(0);
+                        if (lockValue2 != expected)
+                            assertEquals(expected, lockValue2);
+                        int toggle1 = slice1.readInt(4);
+                        if (toggle1 == 1) {
+                            slice1.writeInt(4, 0);
+                            // if (i % 10000== 0)
+                            // System.out.println("t: " + i);
+                        } else {
+                            // noinspection AssignmentToForLoopParameter
+                            i--;
+                        }
+                        int toggle2 = slice2.readInt(4);
+                        if (toggle2 == 1) {
+                            slice2.writeInt(4, 0);
+                            // if (i % 10000== 0)
+                            // System.out.println("t: " + i);
+                        } else {
+                            // noinspection AssignmentToForLoopParameter
+                            i--;
+                        }
+                        int lockValue1A = slice1.readInt(0);
+                        int lockValue2A = slice1.readInt(0);
+                        try {
+                            slice2.unlockInt(0);
+                            slice1.unlockInt(0);
+                        } catch (IllegalStateException e) {
+                            int lockValue1B = slice1.readInt(0);
+                            int lockValue2B = slice2.readInt(0);
+                            System.err.println("i= " + i +
+                                    " lock: " + Integer.toHexString(lockValue1A) + " / " + Integer.toHexString(lockValue2A) +
+                                    " lock: " + Integer.toHexString(lockValue1B) + " / " + Integer.toHexString(lockValue2B));
+                            throw e;
+                        }
+                    }
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+            }
+        });
+        t.start();
+
+        long id = Thread.currentThread().getId();
+        assertEquals(0, id >>> 24);
+        int expected = (1 << 24) | (int) id;
+        System.out.println("Thread " + id);
+
+        DirectBytes slice1 = store1.createSlice();
+        DirectBytes slice2 = store2.createSlice();
+        for (int i = 0; i < lockCount; i += 2) {
+            slice1.busyLockInt(0);
+            slice2.busyLockInt(0);
+            int lockValue1 = slice1.readInt(0);
+            if (lockValue1 != expected)
+                assertEquals(expected, lockValue1);
+            int lockValue2 = slice2.readInt(0);
+            if (lockValue2 != expected)
+                assertEquals(expected, lockValue2);
+            int toggle1 = slice1.readInt(4);
+            if (toggle1 == 0) {
+                slice1.writeInt(4, 1);
+                // if (i % 10000== 0)
+                // System.out.println("t: " + i);
+            } else {
+                // noinspection AssignmentToForLoopParameter
+                i--;
+            }
+            int toggle2 = slice2.readInt(4);
+            if (toggle2 == 0) {
+                slice2.writeInt(4, 1);
+                // if (i % 10000== 0)
+                // System.out.println("t: " + i);
+            } else {
+                // noinspection AssignmentToForLoopParameter
+                i--;
+            }
+            int lockValue1A = slice1.readInt(0);
+            int lockValue2A = slice1.readInt(0);
+            try {
+                slice2.unlockInt(0);
+                slice1.unlockInt(0);
+            } catch (IllegalStateException e) {
+                int lockValue1B = slice1.readInt(0);
+                int lockValue2B = slice2.readInt(0);
+                System.err.println("i= " + i +
+                        " lock: " + Integer.toHexString(lockValue1A) + " / " + Integer.toHexString(lockValue2A) +
+                        " lock: " + Integer.toHexString(lockValue1B) + " / " + Integer.toHexString(lockValue2B));
+                throw e;
+            }
+        }
+
+        store1.free();
+        store2.free();
+    }
+
+}
diff --git a/lang/src/test/java/net/openhft/lang/io/IOToolsTest.java b/lang/src/test/java/net/openhft/lang/io/IOToolsTest.java
new file mode 100644
index 0000000..086749d
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/io/IOToolsTest.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io;
+
+import org.junit.Test;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Arrays;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * User: peter.lawrey
+ * Date: 20/09/13
+ * Time: 09:54
+ */
+public class IOToolsTest {
+    @Test
+    public void testClose() {
+        final int[] count = {0};
+        Closeable c = new Closeable() {
+            @Override
+            public void close() throws IOException {
+                count[0]++;
+            }
+        };
+        IOTools.close(c);
+        assertEquals(1, count[0]);
+        IOTools.close(Arrays.asList(c, c, c));
+        assertEquals(4, count[0]);
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/io/LockingViaFileLockMain.java b/lang/src/test/java/net/openhft/lang/io/LockingViaFileLockMain.java
new file mode 100644
index 0000000..46abc86
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/io/LockingViaFileLockMain.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteOrder;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+
+/**
+ * User: peter
+ * Date: 22/12/13
+ * Time: 11:05
+ *
+ * Toggled 10,000,128 times with an average delay of 2,402 ns
+ */
+public class LockingViaFileLockMain {
+    static int RECORDS = Integer.getInteger("records", 128);
+    static int RECORD_SIZE = Integer.getInteger("record_size", 64); // cache line size
+    static int WARMUP = Integer.getInteger("warmup", RECORDS * 100);
+    static int RUNS = Integer.getInteger("runs", 5 * 1000 * 1000);
+
+    // offsets
+//    static int LOCK = 0;
+    static int FLAG = 8;
+
+    public static void main(String... args) throws IOException, InterruptedException {
+        boolean toggleTo = Boolean.parseBoolean(args[0]);
+        File tmpFile = new File(System.getProperty("java.io.tmpdir"), "lock-test.dat");
+        FileChannel fc = new RandomAccessFile(tmpFile, "rw").getChannel();
+        MappedByteBuffer mbb = fc.map(FileChannel.MapMode.READ_WRITE, 0, RECORDS * RECORD_SIZE);
+        ByteBufferBytes bytes = new ByteBufferBytes(mbb.order(ByteOrder.nativeOrder()));
+
+        long start = 0;
+        for (int i = -WARMUP / RECORDS; i < (RUNS + RECORDS - 1) / RECORDS; i++) {
+            if (i == 0) {
+                start = System.nanoTime();
+                System.out.println("Started");
+            }
+
+            for (int j = 0; j < RECORDS; j++) {
+                int recordOffset = j * RECORD_SIZE;
+                for (int t = 9999; t >= 0; t--) {
+                    if (t == 0)
+                        if (i >= 0) {
+                            throw new AssertionError("Didn't toggle in time !??");
+                        } else {
+                            System.out.println("waiting");
+                            t = 99999;
+                            Thread.sleep(200);
+                        }
+                    final FileLock lock = fc.lock();
+                    try {
+                        bytes.readBarrier();
+                        boolean flag = bytes.readBoolean(recordOffset + FLAG);
+                        if (flag == toggleTo) {
+                            if (t % 100 == 0)
+                                System.out.println("j: " + j + " is " + flag);
+                            continue;
+                        }
+                        bytes.writeBoolean(recordOffset + FLAG, toggleTo);
+                        bytes.writeBarrier();
+                        break;
+                    } finally {
+                        lock.release();
+                    }
+                }
+            }
+        }
+        long time = System.nanoTime() - start;
+        final int toggles = (RUNS + RECORDS - 1) / RECORDS * RECORDS * 2; // one for each of two processes.
+        System.out.printf("Toggled %,d times with an average delay of %,d ns%n",
+                toggles, time / toggles);
+        fc.close();
+        tmpFile.deleteOnExit();
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/io/LockingViaMMapMain.java b/lang/src/test/java/net/openhft/lang/io/LockingViaMMapMain.java
new file mode 100644
index 0000000..a1cddd1
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/io/LockingViaMMapMain.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteOrder;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+
+/**
+ * User: peter
+ * Date: 22/12/13
+ * Time: 11:05
+ *
+ * Toggled 10,000,128 times with an average delay of 28 ns
+ */
+public class LockingViaMMapMain {
+    static int RECORDS = Integer.getInteger("records", 128);
+    static int RECORD_SIZE = Integer.getInteger("record_size", 64); // cache line size
+    static int WARMUP = Integer.getInteger("warmup", RECORDS * 100);
+    static int RUNS = Integer.getInteger("runs", 5 * 1000 * 1000);
+
+    // offsets
+    static int LOCK = 0;
+    static int FLAG = 8;
+    static int LENGTH = 16;
+
+    public static void main(String... args) throws IOException, InterruptedException {
+        boolean toggleTo = Boolean.parseBoolean(args[0]);
+        File tmpFile = new File(System.getProperty("java.io.tmpdir"), "lock-test.dat");
+        FileChannel fc = new RandomAccessFile(tmpFile, "rw").getChannel();
+        MappedByteBuffer mbb = fc.map(FileChannel.MapMode.READ_WRITE, 0, RECORDS * RECORD_SIZE);
+        ByteBufferBytes bytes = new ByteBufferBytes(mbb.order(ByteOrder.nativeOrder()));
+        bytes.setCurrentThread();
+
+        long start = 0;
+        for (int i = -WARMUP / RECORDS; i < (RUNS + RECORDS - 1) / RECORDS; i++) {
+            if (i == 0) {
+                start = System.nanoTime();
+                System.out.println("Started");
+            }
+
+            for (int j = 0; j < RECORDS; j++) {
+                int recordOffset = j * RECORD_SIZE;
+                for (int t = 1; t < 10000; t++) {
+                    if (t == 0)
+                        if (i >= 0) {
+                            throw new AssertionError("Didn't toggle in time !??");
+                        } else {
+                            Thread.sleep(200);
+                        }
+                    bytes.busyLockLong(recordOffset + LOCK);
+                    try {
+                        boolean flag = bytes.readBoolean(recordOffset + FLAG);
+                        if (flag != toggleTo) {
+                            bytes.writeBoolean(recordOffset + FLAG, toggleTo);
+                            break;
+                        }
+                    } finally {
+                        bytes.unlockLong(recordOffset + LOCK);
+                    }
+                    if (t % 100 == 0)
+                        System.out.println("waiting for " + j
+                                + " pid " + (int) bytes.readLong(recordOffset + LOCK)
+                                + " is " + bytes.readBoolean(recordOffset + FLAG));
+                    if (t > 100)
+                        if (t > 200)
+                            Thread.sleep(1);
+                        else
+                            Thread.yield();
+                }
+            }
+        }
+        long time = System.nanoTime() - start;
+        final int toggles = (RUNS + RECORDS - 1) / RECORDS * RECORDS * 2; // one for each of two processes.
+        System.out.printf("Toggled %,d times with an average delay of %,d ns%n",
+                toggles, time / toggles);
+        fc.close();
+        tmpFile.deleteOnExit();
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/io/MappedFileTest.java b/lang/src/test/java/net/openhft/lang/io/MappedFileTest.java
new file mode 100644
index 0000000..6456888
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/io/MappedFileTest.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io;
+
+import org.junit.Test;
+
+import java.io.*;
+import java.nio.MappedByteBuffer;
+
+import static org.junit.Assert.assertTrue;
+
+public class MappedFileTest {
+    public static void printMappings() throws IOException {
+        BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream("/proc/self/maps")));
+        try {
+            for (String line; (line = br.readLine()) != null; ) {
+                System.out.println(line);
+            }
+        } catch (IOException ioe) {
+            br.close();
+            throw ioe;
+        }
+
+    }
+
+    public static void delete(File file) throws IOException {
+        if (file.delete() || !file.exists()) return;
+        // get an error message as to why.
+        ProcessBuilder pb = new ProcessBuilder("/bin/rm", file.getAbsolutePath());
+        pb.redirectErrorStream(true);
+        Process p = pb.start();
+        StringWriter sw = new StringWriter();
+        char[] chars = new char[512];
+        Reader r = new InputStreamReader(p.getInputStream());
+        for (int len; (len = r.read(chars)) > 0; )
+            sw.write(chars, 0, len);
+        String msg = sw.toString().trim();
+        if (msg.length() > 0)
+            throw new IOException(msg);
+    }
+
+    @Test
+    public void testUnmap() throws IOException, InterruptedException {
+
+        String TMP = System.getProperty("java.io.tmpdir");
+        String basePath = TMP + "/testUnmap";
+        File file = new File(basePath);
+        File dir = file.getParentFile();
+        long free0 = dir.getFreeSpace();
+
+        MappedFile mfile = new MappedFile(basePath, 1024 * 1024);
+        MappedMemory map0 = mfile.acquire(0);
+        fill(map0.buffer());
+        MappedMemory map1 = mfile.acquire(1);
+        fill(map1.buffer().force());
+        long free1 = dir.getFreeSpace();
+
+        map1.release();
+        map0.release();
+        mfile.close();
+
+//        printMappings();
+        long free2 = dir.getFreeSpace();
+        delete(file);
+        long free3 = 0;
+        for (int i = 0; i < 100; i++) {
+            free3 = dir.getFreeSpace();
+            System.out.println("Freed " + free0 + " ~ " + free1 + " ~ " + free2 + " ~ " + free3 + ", delete = " + file.delete());
+            if (free3 > free1)
+                break;
+            Thread.sleep(500);
+        }
+        assertTrue("free3-free1: " + (free3 - free1), free3 > free1);
+
+
+    }
+
+    private void fill(MappedByteBuffer buffer) {
+        buffer.position(0);
+        while (buffer.remaining() >= 8)
+            buffer.putLong(0x123456789ABCDEFL);
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/io/MutableDecimalTest.java b/lang/src/test/java/net/openhft/lang/io/MutableDecimalTest.java
new file mode 100644
index 0000000..ba6796e
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/io/MutableDecimalTest.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * User: peter.lawrey
+ * Date: 20/09/13
+ * Time: 10:40
+ */
+public class MutableDecimalTest {
+    @Test
+    public void testConstructor() {
+        MutableDecimal md = new MutableDecimal(Long.MAX_VALUE);
+        assertEquals(Long.MAX_VALUE, md.longValue());
+        assertEquals("" + Long.MAX_VALUE, md.toString());
+
+        MutableDecimal md2 = new MutableDecimal(Long.MAX_VALUE, 10);
+        assertEquals(Long.MAX_VALUE / 1e10, md2.doubleValue(), 0);
+        assertEquals("922337203.6854775807", md2.toString());
+
+        MutableDecimal md3 = new MutableDecimal(Math.PI, 6);
+        assertEquals(3.141593, md3.doubleValue(), 0);
+        assertEquals(3.141593f, md3.floatValue(), 0);
+        assertEquals(3, md3.intValue());
+        assertEquals("3.141593", md3.toString());
+
+        assertEquals(1, md.compareTo(md2));
+        assertEquals(-1, md2.compareTo(md));
+        assertEquals(1, md2.compareTo(md3));
+        assertEquals(-1, md3.compareTo(md2));
+
+        assertEquals(3141593, md3.value());
+        assertEquals(6, md3.scale());
+        assertTrue(md3.isSet());
+        md3.clear();
+        assertFalse(md3.isSet());
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/io/NativeBytesTest.java b/lang/src/test/java/net/openhft/lang/io/NativeBytesTest.java
new file mode 100644
index 0000000..2b50825
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/io/NativeBytesTest.java
@@ -0,0 +1,926 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io;
+
+import net.openhft.lang.Maths;
+import net.openhft.lang.thread.NamedThreadFactory;
+import org.junit.Before;
+import org.junit.Test;
+import sun.nio.ch.DirectBuffer;
+
+import java.io.*;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+import static net.openhft.lang.io.StopCharTesters.CONTROL_STOP;
+import static net.openhft.lang.io.StopCharTesters.SPACE_STOP;
+import static org.junit.Assert.*;
+
+/**
+ * Created with IntelliJ IDEA. User: peter.lawrey Date: 17/09/13 Time: 16:09 To change this template use File | Settings | File
+ * Templates.
+ */
+public class NativeBytesTest {
+    public static final int SIZE = 128;
+    private NativeBytes bytes;
+    private ByteBuffer byteBuffer;
+
+    @Before
+    public void beforeTest() {
+        byteBuffer = ByteBuffer.allocateDirect(SIZE);
+        long addr = ((DirectBuffer) byteBuffer).address();
+        bytes = new NativeBytes(addr, addr, addr + SIZE);
+    }
+
+    @Test
+    public void testLongHash() throws Exception {
+        byte[] bytes = {1, 2, 3, 4, 5, 6, 7, 8};
+        long h = NativeBytes.longHash(bytes, 0, bytes.length);
+        assertFalse(h == 0);
+        byte[] bytes2 = {1, 2, 3, 4, 5, 6, 7, 8, 9};
+        long h2 = NativeBytes.longHash(bytes2, 0, bytes2.length);
+        assertFalse(h2 == 0);
+        assertFalse(h2 == h);
+    }
+
+    @Test
+    public void testRead() throws Exception {
+        for (int i = 0; i < bytes.capacity(); i++)
+            bytes.writeByte(i, i);
+        bytes.position(0);
+        for (int i = 0; i < bytes.capacity(); i++)
+            assertEquals((byte) i, bytes.read());
+        for (int i = (int) (bytes.capacity() - 1); i >= 0; i--) {
+            assertEquals((byte) i, bytes.readByte(i));
+        }
+
+    }
+
+    @Test
+    public void testReadFully() throws Exception {
+        for (int i = 0; i < bytes.capacity(); i++)
+            bytes.write(i);
+        bytes.position(0);
+        byte[] bytes = new byte[(int) this.bytes.capacity()];
+        this.bytes.readFully(bytes);
+        for (int i = 0; i < this.bytes.capacity(); i++)
+            assertEquals((byte) i, bytes[i]);
+    }
+
+    @Test
+    public void testCompareAndSetLong() throws Exception {
+        assertTrue(bytes.compareAndSwapLong(0, 0, 1));
+        assertFalse(bytes.compareAndSwapLong(0, 0, 1));
+        assertTrue(bytes.compareAndSwapLong(8, 0, 1));
+        assertTrue(bytes.compareAndSwapLong(0, 1, 2));
+
+    }
+
+    @Test
+    public void testPosition() throws Exception {
+        for (int i = 0; i < bytes.capacity(); i++)
+            bytes.write(i);
+        for (int i = (int) (bytes.capacity() - 1); i >= 0; i--) {
+            bytes.position(i);
+            assertEquals((byte) i, bytes.read());
+        }
+    }
+
+    @Test
+    public void testCapacity() throws Exception {
+        assertEquals(SIZE, bytes.capacity());
+        assertEquals(10, new NativeBytes(0, 0, 10).capacity());
+    }
+
+    @Test
+    public void testRemaining() throws Exception {
+        assertEquals(SIZE, bytes.remaining());
+        bytes.position(10);
+        assertEquals(SIZE - 10, bytes.remaining());
+    }
+
+    @Test
+    public void testByteOrder() throws Exception {
+        assertEquals(ByteOrder.nativeOrder(), bytes.byteOrder());
+    }
+
+    @Test
+    public void testCheckEndOfBuffer() throws Exception {
+        bytes.checkEndOfBuffer();
+
+        bytes.position(SIZE + 2);
+        try {
+            bytes.checkEndOfBuffer();
+            fail();
+        } catch (IndexOutOfBoundsException expected) {
+        }
+    }
+
+    @Test
+    public void testAppendDouble() {
+        testAppendDouble0(-6.895305375646115E24);
+        Random random = new Random(1);
+        for (int i = 0; i < 100000; i++) {
+            double d = Math.pow(1e32, random.nextDouble()) / 1e6;
+            if (i % 3 == 0) d = -d;
+            testAppendDouble0(d);
+        }
+    }
+
+    private void testAppendDouble0(double d) {
+        bytes.position(0);
+        bytes.append(d).append(' ');
+        bytes.position(0);
+        double d2 = bytes.parseDouble();
+        assertEquals(d, d2, 0);
+    }
+
+    @Test
+    public void testAppendDouble2() {
+//        testAppendDouble0(-0.93879148954440506, 14);
+//        testAppendDouble0(-0.214980202661, 12);
+//        testAppendDouble0(-0.937082148896, 12);
+        testAppendDouble0(0.17805, 5);
+        Random random = new Random(1);
+
+        for (int j = 0; j < 20000; j++) {
+            double d = random.nextDouble();
+            if (j % 3 == 0) d = -d;
+//            if (j % 5 == 0) d *= 1e6;
+            for (int i = 0; i < 4; i++) {
+                testAppendDouble0(d, i);
+            }
+        }
+    }
+
+    private void testAppendDouble0(double d, int precision) {
+        bytes.position(0);
+        bytes.append(d, precision).append(' ');
+        bytes.position(0);
+        String text = bytes.parseUTF(SPACE_STOP);
+        bytes.position(0);
+        assertEquals(0, bytes.position());
+        double d2 = bytes.parseDouble();
+        double d3 = (double) Math.round(d * Maths.power10(precision)) / Maths.power10(precision);
+//        if (precision >= 14)
+//            assertEquals("'" + text + "' p: " + precision + " v: " + new BigDecimal(d), d3, d2, 5e-29 * Maths.power10(precision));
+//        else
+        assertEquals("'" + text + "' p: " + precision, d3, d2, 0);
+    }
+
+    @Test
+    public void testWriteReadBytes() {
+        byte[] bytes = "Hello World!".getBytes();
+        this.bytes.write(bytes);
+        byte[] bytes2 = new byte[bytes.length];
+        this.bytes.position(0);
+        this.bytes.read(bytes2);
+        assertTrue(Arrays.equals(bytes, bytes2));
+
+        this.bytes.write(22, bytes);
+        byte[] bytes3 = new byte[bytes.length];
+        this.bytes.skipBytes((int) (22 - this.bytes.position()));
+        assertEquals(bytes3.length, this.bytes.read(bytes3));
+        assertTrue(Arrays.equals(bytes, bytes3));
+        this.bytes.position(this.bytes.capacity());
+        assertEquals(-1, this.bytes.read(bytes3));
+    }
+
+    @Test
+    public void testWriteReadUTFΔ() {
+        String[] words = "Hello,World!,Bye£€!".split(",");
+        for (String word : words) {
+            bytes.writeUTFΔ(word);
+        }
+        bytes.writeUTFΔ(null);
+        bytes.position(0);
+        for (String word : words) {
+            assertEquals(word, bytes.readUTFΔ());
+        }
+        assertEquals(null, bytes.readUTFΔ());
+        assertEquals("", bytes.readUTFΔ());
+        assertEquals(26, bytes.position()); // check the size
+
+        bytes.position(0);
+        StringBuilder sb = new StringBuilder();
+        for (String word : words) {
+            assertTrue(bytes.readUTFΔ(sb));
+            assertEquals(word, sb.toString());
+        }
+        assertFalse(bytes.readUTFΔ(sb));
+        assertTrue(bytes.readUTFΔ(sb));
+        assertEquals("", sb.toString());
+    }
+
+    @Test
+    public void testWriteReadUTF() {
+        String[] words = "Hello,World!,Bye£€!".split(",");
+        for (String word : words) {
+            bytes.writeUTF(word);
+        }
+        bytes.writeUTF("");
+        assertEquals(28, bytes.position()); // check the size, more bytes for less strings than writeUTFΔ
+        bytes.position(0);
+        for (String word : words) {
+            assertEquals(word, bytes.readUTF());
+        }
+        assertEquals("", bytes.readUTF());
+    }
+
+    @Test
+    public void testAppendParseUTF() {
+        String[] words = "Hello,World!,Bye£€!".split(",");
+        for (String word : words) {
+            bytes.append(word).append('\t');
+        }
+        bytes.append('\t');
+        bytes.position(0);
+        for (String word : words) {
+            assertEquals(word, bytes.parseUTF(CONTROL_STOP));
+        }
+        assertEquals("", bytes.parseUTF(CONTROL_STOP));
+
+        bytes.position(0);
+        StringBuilder sb = new StringBuilder();
+        for (String word : words) {
+            bytes.parseUTF(sb, CONTROL_STOP);
+            assertEquals(word, sb.toString());
+        }
+        bytes.parseUTF(sb, CONTROL_STOP);
+        assertEquals("", sb.toString());
+
+        bytes.position(0);
+        bytes.skipTo(CONTROL_STOP);
+        assertEquals(6, bytes.position());
+        bytes.skipTo(CONTROL_STOP);
+        assertEquals(13, bytes.position());
+        bytes.skipTo(CONTROL_STOP);
+        assertEquals(17, bytes.position());
+        bytes.skipTo(CONTROL_STOP);
+        assertEquals(18, bytes.position());
+
+        bytes.position(0);
+        bytes.stepBackAndSkipTo(CONTROL_STOP);
+        assertEquals(6, bytes.position());
+        bytes.stepBackAndSkipTo(CONTROL_STOP);
+        assertEquals(6, bytes.position());
+        bytes.position(10);
+        bytes.stepBackAndSkipTo(CONTROL_STOP);
+        assertEquals(13, bytes.position());
+
+    }
+
+    @Test
+    public void testWriteReadLines() {
+        byte[] bytes = "Hello\nWorld!\r\nBye".getBytes();
+        this.bytes.write(bytes);
+        this.bytes.position(0);
+        assertEquals("Hello", this.bytes.readLine());
+        assertEquals("World!", this.bytes.readLine());
+        assertTrue(this.bytes.readLine().startsWith("Bye"));
+    }
+
+    @Test
+    public void testWriteReadByteBuffer() {
+        byte[] bytes = "Hello\nWorld!\r\nBye".getBytes();
+        this.bytes.write(ByteBuffer.wrap(bytes));
+        this.bytes.position(0);
+        byte[] bytes2 = new byte[bytes.length + 1];
+        ByteBuffer bb2 = ByteBuffer.wrap(bytes2);
+        this.bytes.read(bb2);
+
+        assertEquals(bytes2.length, bb2.position());
+        assertTrue(Arrays.equals(bytes, Arrays.copyOf(bytes2, bytes.length)));
+    }
+
+    @Test
+    public void testReadWriteBoolean() {
+        for (int i = 0; i < 32; i++)
+            bytes.writeBoolean(i, (i & 3) == 0);
+        bytes.position(32);
+        for (int i = 32; i < 64; i++)
+            bytes.writeBoolean((i & 5) == 0);
+        bytes.position(0);
+        for (int i = 0; i < 32; i++)
+            assertEquals((i & 3) == 0, bytes.readBoolean());
+        for (int i = 32; i < 64; i++)
+            assertEquals((i & 5) == 0, bytes.readBoolean(i));
+    }
+
+    @Test
+    public void testReadWriteShort() {
+        for (int i = 0; i < 32; i += 2)
+            bytes.writeShort(i, i);
+        bytes.position(32);
+        for (int i = 32; i < 64; i += 2)
+            bytes.writeShort(i);
+        bytes.position(0);
+        for (int i = 0; i < 32; i += 2)
+            assertEquals(i, bytes.readShort());
+        for (int i = 32; i < 64; i += 2)
+            assertEquals(i, bytes.readShort(i));
+    }
+
+    @Test
+    public void testReadWriteCompactShort() {
+        int[] ints = {Short.MIN_VALUE, Short.MAX_VALUE, -125, 0, 127, -10000, 10000};
+        for (int i : ints) {
+            bytes.writeCompactShort(i);
+//            System.out.println(i + " " + bytes.position());
+        }
+        assertEquals(5 + 2 * 3, bytes.position());
+
+        bytes.position(0);
+        for (int i : ints)
+            assertEquals(i, bytes.readCompactShort());
+    }
+
+    @Test
+    public void testReadWriteCompactInt() {
+        int[] ints = {-10000000, Integer.MIN_VALUE, Integer.MAX_VALUE, Short.MIN_VALUE + 3, 0, Short.MAX_VALUE, 10000000};
+        for (int i : ints) {
+            bytes.writeCompactInt(i);
+//            System.out.println(i + " " + bytes.position());
+        }
+        assertEquals(5 * 2 + 2 * 6, bytes.position());
+
+        bytes.position(0);
+        for (int i : ints)
+            assertEquals(i, bytes.readCompactInt());
+    }
+
+    @Test
+    public void testReadWriteCompactLong() {
+        long[] ints = {Long.MAX_VALUE, -100000000000L, Long.MIN_VALUE, Integer.MIN_VALUE + 3, 0, Integer.MAX_VALUE, 100000000000L};
+        for (long i : ints) {
+            bytes.writeCompactLong(i);
+//            System.out.println(i + " " + bytes.position());
+        }
+        assertEquals(5 * 4 + 2 * 12, bytes.position());
+
+        bytes.position(0);
+        for (long i : ints)
+            assertEquals(i, bytes.readCompactLong());
+    }
+
+    @Test
+    public void testReadWriteCompactDouble() {
+        double[] doubles = {1, 1000, 1000000, -100000000, 0.1f, 0.1, 0.5, 0.51};
+        for (double i : doubles) {
+            bytes.writeCompactDouble(i);
+//            System.out.println(i + " " + bytes.position());
+        }
+        assertEquals(6 * 4 + 2 * 12, bytes.position());
+
+        bytes.position(0);
+        for (double i : doubles)
+            assertEquals(i, bytes.readCompactDouble(), 0.0);
+    }
+
+    @Test
+    public void testReadWriteStop() {
+        long[] longs = {Long.MIN_VALUE, Long.MAX_VALUE, Integer.MIN_VALUE, Integer.MAX_VALUE};
+        for (long i : longs) {
+            bytes.writeStopBit(i);
+//            System.out.println(i + " " + bytes.position());
+        }
+        assertEquals(9 + 10, +5 + 6, bytes.position());
+
+        bytes.position(0);
+        for (long i : longs)
+            assertEquals(i, bytes.readStopBit());
+    }
+
+    @Test
+    public void testReadWriteChar() {
+        for (int i = 0; i < 32; i += 2)
+            bytes.writeChar(i, i);
+        bytes.position(32);
+        for (int i = 32; i < 64; i += 2)
+            bytes.writeChar(i);
+        bytes.position(0);
+        for (int i = 0; i < 32; i += 2)
+            assertEquals(i, bytes.readChar());
+        for (int i = 32; i < 64; i += 2)
+            assertEquals(i, bytes.readChar(i));
+    }
+
+    @Test
+    public void testReadWriteUnsignedShort() {
+        for (int i = 0; i < 32; i += 2)
+            bytes.writeUnsignedShort(i, ~i);
+        bytes.position(32);
+        for (int i = 32; i < 64; i += 2)
+            bytes.writeUnsignedShort(~i);
+        bytes.position(0);
+        for (int i = 0; i < 32; i += 2)
+            assertEquals(~i & 0xFFFF, bytes.readUnsignedShort());
+        for (int i = 32; i < 64; i += 2)
+            assertEquals(~i & 0xFFFF, bytes.readUnsignedShort(i));
+    }
+
+    @Test
+    public void testReadWriteInt() {
+        for (int i = 0; i < 32; i += 4)
+            bytes.writeInt(i, i);
+        bytes.position(32);
+        for (int i = 32; i < 64; i += 4)
+            bytes.writeInt(i);
+        bytes.position(0);
+        for (int i = 0; i < 32; i += 4)
+            assertEquals(i, bytes.readInt());
+        for (int i = 32; i < 64; i += 4)
+            assertEquals(i, bytes.readInt(i));
+    }
+
+    @Test
+    public void testReadWriteThreadeSafeInt() {
+        for (int i = 0; i < 32; i += 4)
+            bytes.writeOrderedInt(i, i);
+        bytes.position(32);
+        for (int i = 32; i < 64; i += 4)
+            bytes.writeOrderedInt(i);
+        bytes.position(0);
+        for (int i = 0; i < 32; i += 4)
+            assertEquals(i, bytes.readVolatileInt());
+        for (int i = 32; i < 64; i += 4)
+            assertEquals(i, bytes.readVolatileInt(i));
+    }
+
+    @Test
+    public void testReadWriteFloat() {
+        for (int i = 0; i < 32; i += 4)
+            bytes.writeFloat(i, i);
+        bytes.position(32);
+        for (int i = 32; i < 64; i += 4)
+            bytes.writeFloat(i);
+        bytes.position(0);
+        for (int i = 0; i < 32; i += 4)
+            assertEquals(i, bytes.readFloat(), 0);
+        for (int i = 32; i < 64; i += 4)
+            assertEquals(i, bytes.readFloat(i), 0);
+    }
+
+    @Test
+    public void testReadWriteUnsignedInt() {
+        for (int i = 0; i < 32; i += 4)
+            bytes.writeUnsignedInt(i, ~i);
+        bytes.position(32);
+        for (int i = 32; i < 64; i += 4)
+            bytes.writeUnsignedInt(~i);
+        bytes.position(0);
+        for (int i = 0; i < 32; i += 4)
+            assertEquals(~i & 0xFFFFFFFFL, bytes.readUnsignedInt());
+        for (int i = 32; i < 64; i += 4)
+            assertEquals(~i & 0xFFFFFFFFL, bytes.readUnsignedInt(i));
+    }
+
+    @Test
+    public void testReadWriteInt24() {
+        for (int i = 0; i < 30; i += 3)
+            bytes.writeInt24(i, ~i & 0x7FFFFF);
+        bytes.position(30);
+        for (int i = 30; i < 63; i += 3)
+            bytes.writeInt24(~i & 0x7FFFFF);
+        assertEquals(63, bytes.position());
+        bytes.position(0);
+        for (int i = 0; i < 30; i += 3)
+            assertEquals("i: " + i, ~i & 0x7FFFFFL, bytes.readInt24());
+        for (int i = 30; i < 63; i += 3)
+            assertEquals("i: " + i, ~i & 0x7FFFFFL, bytes.readInt24(i));
+        // now negative
+        bytes.position(0);
+        for (int i = 0; i < 30; i += 3)
+            bytes.writeInt24(i, ~i);
+        bytes.position(30);
+        for (int i = 30; i < 63; i += 3)
+            bytes.writeInt24(~i);
+        assertEquals(63, bytes.position());
+        bytes.position(0);
+        for (int i = 0; i < 30; i += 3)
+            assertEquals("i: " + i, ~i << 8 >> 8, bytes.readInt24());
+        for (int i = 30; i < 63; i += 3)
+            assertEquals("i: " + i, ~i << 8 >> 8, bytes.readInt24(i));
+    }
+
+    @Test
+    public void testReadWriteInt48() {
+        for (long i = 0; i < 30; i += 6)
+            bytes.writeInt48(i, ~i & 0x7FFFFFFFFFFFL);
+        bytes.position(30);
+        for (long i = 30; i < 60; i += 6)
+            bytes.writeInt48(~i & 0x7FFFFFFFFFFFL);
+        assertEquals(60, bytes.position());
+        bytes.position(0);
+        for (long i = 0; i < 30; i += 6)
+            assertEquals("i: " + i, ~i & 0x7FFFFFFFFFFFL, bytes.readInt48());
+        for (long i = 30; i < 60; i += 6)
+            assertEquals("i: " + i, ~i & 0x7FFFFFFFFFFFL, bytes.readInt48(i));
+        // now negative
+        bytes.position(0);
+        for (long i = 0; i < 30; i += 6)
+            bytes.writeInt48(i, ~i);
+        bytes.position(30);
+        for (long i = 30; i < 60; i += 6)
+            bytes.writeInt48(~i);
+        assertEquals(60, bytes.position());
+        bytes.position(0);
+        for (long i = 0; i < 30; i += 6)
+            assertEquals("i: " + i, ~i << 16 >> 16, bytes.readInt48());
+        for (long i = 30; i < 60; i += 6)
+            assertEquals("i: " + i, ~i << 16 >> 16, bytes.readInt48(i));
+    }
+
+    @Test
+    public void testDateTimes() {
+        long now = System.currentTimeMillis();
+        bytes.appendDateTimeMillis(now);
+        bytes.append(' ');
+        bytes.appendDateMillis(now);
+        bytes.append('T');
+        bytes.appendTimeMillis(now % 86400000L);
+        assertEquals(23 * 2 + 1, bytes.position());
+        bytes.position(0);
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd'T'HH:mm:ss.SSS");
+        sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
+        String asStr = sdf.format(new Date(now));
+        assertEquals(asStr, bytes.parseUTF(SPACE_STOP));
+        assertEquals(asStr, bytes.parseUTF(SPACE_STOP));
+    }
+
+    @Test
+    public void testReadWriteCompactUnsignedShort() {
+        for (int i = 0; i < 64; i += 4)
+            bytes.writeCompactUnsignedShort(i);
+        assertEquals(64 / 4, bytes.position());
+        bytes.position(0);
+        for (int i = 0; i < 64; i += 4)
+            assertEquals(i, bytes.readByte(i / 4));
+        for (int i = 0; i < 64; i += 4)
+            assertEquals(i, bytes.readCompactUnsignedShort());
+    }
+
+    @Test
+    public void testReadWriteCompactUnsignedInt() {
+        for (int i = 0; i < 64; i += 4)
+            bytes.writeCompactUnsignedInt(i);
+        assertEquals(64 / 2, bytes.position());
+        bytes.position(0);
+        for (int i = 0; i < 64; i += 4)
+            assertEquals(i, bytes.readShort(i / 2));
+        for (int i = 0; i < 64; i += 4)
+            assertEquals(i, bytes.readCompactUnsignedInt());
+    }
+
+    @Test
+    public void testReadWriteLong() {
+        for (long i = 0; i < 32; i += 8)
+            bytes.writeLong(i, i);
+        bytes.position(32);
+        for (long i = 32; i < 64; i += 8)
+            bytes.writeLong(i);
+        bytes.position(0);
+        for (long i = 0; i < 32; i += 8)
+            assertEquals(i, bytes.readLong());
+        for (long i = 32; i < 64; i += 8)
+            assertEquals(i, bytes.readLong(i));
+    }
+
+    @Test
+    public void testReadWriteThreadSafeLong() {
+        for (long i = 0; i < 32; i += 8)
+            bytes.writeOrderedLong(i, i);
+        bytes.position(32);
+        for (long i = 32; i < 64; i += 8)
+            bytes.writeOrderedLong(i);
+        bytes.position(0);
+        for (long i = 0; i < 32; i += 8)
+            assertEquals(i, bytes.readVolatileLong());
+        for (long i = 32; i < 64; i += 8)
+            assertEquals(i, bytes.readVolatileLong(i));
+    }
+
+    @Test
+    public void testReadWriteDouble() {
+        for (long i = 0; i < 32; i += 8)
+            bytes.writeDouble(i, i);
+        bytes.position(32);
+        for (long i = 32; i < 64; i += 8)
+            bytes.writeDouble(i);
+        bytes.position(0);
+        for (long i = 0; i < 32; i += 8)
+            assertEquals(i, bytes.readDouble(), 0);
+        for (long i = 32; i < 64; i += 8)
+            assertEquals(i, bytes.readDouble(i), 0);
+    }
+
+    @Test
+    public void testAppendSubstring() {
+        bytes.append("Hello World", 2, 7).append("\n");
+        bytes.position(0);
+        assertEquals("Hello World".substring(2, 7), bytes.parseUTF(CONTROL_STOP));
+    }
+
+    @Test
+    public void testWriteReadEnum() {
+        bytes.append(BuySell.Buy).append("\t").append(BuySell.Sell);
+        bytes.position(0);
+        assertEquals(BuySell.Buy, bytes.parseEnum(BuySell.class, CONTROL_STOP));
+        assertEquals(BuySell.Sell, bytes.parseEnum(BuySell.class, CONTROL_STOP));
+        assertEquals(null, bytes.parseEnum(BuySell.class, CONTROL_STOP));
+    }
+
+    @Test
+    public void testAppendParse() {
+        bytes.append(false).append(' ');
+        bytes.append(true).append(' ');
+        bytes.append("what?").append(' ');
+        bytes.append("word£€").append(' ');
+        bytes.append(BuySell.Buy).append(' ');
+        bytes.append(1234).append(' ');
+        bytes.append(123456L).append(' ');
+        bytes.append(1.2345).append(' ');
+        bytes.append(1.5555, 3).append(' ');
+        bytes.position(0);
+        assertEquals(false, bytes.parseBoolean(SPACE_STOP));
+        assertEquals(true, bytes.parseBoolean(SPACE_STOP));
+        assertEquals(null, bytes.parseBoolean(SPACE_STOP));
+        assertEquals("word£€", bytes.parseUTF(SPACE_STOP));
+        assertEquals(BuySell.Buy, bytes.parseEnum(BuySell.class, SPACE_STOP));
+        assertEquals(1234, bytes.parseLong());
+        assertEquals(123456L, bytes.parseLong());
+        assertEquals(1.2345, bytes.parseDouble(), 0);
+        assertEquals(1.556, bytes.parseDouble(), 0);
+
+    }
+
+    @Test
+    public void testWriteByteChar() throws UnsupportedEncodingException {
+        bytes.writeBytes("Hello \u00ff\u01fe\u02fc\n");
+        bytes.writeChars("Hello \u00ff\u01fe\u02fc\n");
+        byte[] bytes = new byte[(int) this.bytes.position()];
+        this.bytes.position(0);
+        this.bytes.readFully(bytes);
+        assertEquals("Hello \u00ff\u00fe\u00fc\n" +
+                "H\u0000e\u0000l\u0000l\u0000o\u0000 \u0000ÿ\u0000þ\u0001ü\u0002\n" +
+                "\u0000", new String(bytes, "ISO-8859-1"));
+    }
+
+    @Test
+    public void testWriteBytes() {
+        bytes.write("Hello World\n".getBytes(), 0, 10);
+        bytes.write("good bye\n".getBytes(), 4, 4);
+        bytes.write(4, "0 w".getBytes());
+        bytes.position(0);
+        assertEquals("Hell0 worl bye", bytes.parseUTF(CONTROL_STOP));
+    }
+
+    @Test
+    public void testAppendIterable() {
+        bytes.append(Arrays.asList(1, 2, 3, 4, 5), ";").append(' ');
+        bytes.append(new TreeSet(Arrays.asList(21, 2, 13, 4, 5)), ";");
+        bytes.position(0);
+        assertEquals("1;2;3;4;5 2;4;5;13;21", bytes.parseUTF(CONTROL_STOP));
+    }
+
+    @Test
+    public void readWriteMutableDecimal() {
+        Random rand = new Random(2);
+        MutableDecimal md = new MutableDecimal();
+        MutableDecimal md2 = new MutableDecimal();
+//        md.set(1260042744, 0);
+
+        for (int i = 0; i < 20000; i++) {
+            int n = rand.nextInt();
+            for (int j = 0; j < 6; j++) {
+                testDecimal0(md, md2, n, j);
+            }
+        }
+    }
+
+    private void testDecimal0(MutableDecimal md, MutableDecimal md2, int n, int j) {
+        md.set(n, j);
+        bytes.position(0);
+        bytes.append(md).append('\n');
+        bytes.position(0);
+        bytes.parseDecimal(md2);
+        bytes.position(0);
+        String text = bytes.parseUTF(CONTROL_STOP);
+        if (!md.equals(md2))
+            assertEquals("n: " + n + ", s: " + j + " t: " + text, md, md2);
+    }
+
+    @Test
+    public void testStream() throws IOException {
+        GZIPOutputStream out = new GZIPOutputStream(bytes.outputStream());
+        out.write("Hello world\n".getBytes());
+        out.close();
+        bytes.position(0);
+        GZIPInputStream in = new GZIPInputStream(bytes.inputStream());
+        byte[] bytes = new byte[12];
+        for (int i = 0; i < 12; i++)
+            bytes[i] = (byte) in.read();
+        assertEquals(-1, in.read());
+        assertEquals("Hello world\n", new String(bytes));
+        in.close();
+    }
+
+    @Test
+    public void testStream2() throws IOException {
+        OutputStream out = bytes.outputStream();
+        out.write(11);
+        out.write(22);
+        out.write(33);
+        out.write(44);
+        out.write(55);
+
+        bytes.position(0);
+        InputStream in = bytes.inputStream();
+        assertTrue(in.markSupported());
+        assertEquals(11, in.read());
+        in.mark(1);
+        assertEquals(1, bytes.position());
+        assertEquals(22, in.read());
+        assertEquals(2, bytes.position());
+
+        assertEquals(33, in.read());
+        in.reset();
+
+        assertEquals(1, bytes.position());
+        assertEquals(22, in.read());
+
+        assertEquals(2, in.skip(2));
+        assertEquals(4, bytes.position());
+        assertEquals(SIZE - 4, bytes.available());
+        assertEquals(55, in.read());
+        in.close();
+    }
+
+    @Test
+    public void testWriteObject() {
+        for (Object o : new Object[]{10, 9.9, "string", new Date(), BigDecimal.valueOf(1.1)}) {
+            bytes.position(0);
+            bytes.writeObject(o);
+//            System.out.println(o +" size: "+bytes.position());
+            assertTrue(bytes.position() < 21);
+            bytes.position(0);
+            Object o2 = bytes.readObject();
+            bytes.position(0);
+            Object o3 = bytes.readObject(o.getClass());
+            assertEquals(o, o2);
+            assertEquals(o, o3);
+        }
+    }
+
+    @Test
+    public void testWriteSerializable() {
+        int capacity = 16 * 1024;
+        byteBuffer = ByteBuffer.allocateDirect(capacity);
+        long addr = ((DirectBuffer) byteBuffer).address();
+        bytes = new NativeBytes(addr, addr, addr + capacity);
+        Calendar cal = Calendar.getInstance();
+        bytes.writeObject(cal);
+        Dummy d = new Dummy();
+        bytes.writeObject(d);
+        bytes.position(0);
+        Calendar cal2 = bytes.readObject(Calendar.class);
+        Dummy d2 = bytes.readObject(Dummy.class);
+        assertEquals(cal, cal2);
+        assertEquals(d, d2);
+    }
+
+    @Test
+    public void testAddAndGet() {
+        for (int i = 0; i < 10; i++)
+            bytes.addAndGetInt(0L, 10);
+        assertEquals(100, bytes.readInt(0L));
+        assertEquals(0, bytes.readInt(4L));
+
+        for (int i = 0; i < 11; i++)
+            bytes.getAndAdd(4L, 11);
+        assertEquals(100, bytes.readInt(0L));
+        assertEquals(11 * 11, bytes.readInt(4L));
+    }
+
+    enum BuySell {
+        Buy, Sell
+    }
+
+    static class Dummy implements Serializable {
+        @Override
+        public boolean equals(Object obj) {
+            return obj instanceof Dummy;
+        }
+    }
+
+    @Test
+    public void testErrors() {
+        int capacity = 1024;
+        byteBuffer = ByteBuffer.allocateDirect(capacity);
+        long addr = ((DirectBuffer) byteBuffer).address();
+        // it is actually much bigger than it believes
+        bytes = new NativeBytes(addr, addr, addr + 16);
+        bytes.writeLong(8);
+        assertFalse(bytes.isFinished());
+        bytes.finish();
+        assertTrue(bytes.isFinished());
+        bytes.flush();
+        bytes.writeLong(16);
+        bytes.finish();
+        bytes.flush();
+        bytes.writeLong(24);
+        try {
+            bytes.finish();
+            fail();
+        } catch (IndexOutOfBoundsException expected) {
+        }
+        try {
+            bytes.flush();
+            fail();
+        } catch (IndexOutOfBoundsException expected) {
+        }
+        bytes.reset();
+        assertEquals(0, bytes.position());
+        assertEquals(8, bytes.skip(8));
+        assertEquals(8, bytes.position());
+        bytes.writeLong(22);
+        bytes.close();
+    }
+
+    @Test
+    public void testWriteList() {
+        List ints = Arrays.asList(1, 2, 3, 4);
+        bytes.writeList(ints);
+        bytes.reset();
+        List ints2 = new ArrayList();
+        bytes.readList(ints2, Integer.class);
+        assertEquals(ints, ints2);
+
+        bytes.reset();
+        List words = Arrays.asList("Hello word byte for now".split(" "));
+        bytes.writeList(words);
+        bytes.reset();
+        List words2 = new ArrayList();
+        bytes.readList(words2, String.class);
+    }
+
+    @Test
+    public void testWriteMap() {
+        Map map = new LinkedHashMap() {
+            {
+                put("one", 1);
+                put("two", 2);
+                put("three", 3);
+                put("four", 4);
+            }
+        };
+
+        bytes.writeMap(map);
+        bytes.finish();
+
+        bytes.reset();
+        Map map2 = new LinkedHashMap();
+        bytes.readMap(map2, String.class, Integer.class);
+        assertEquals(map, map2);
+    }
+
+    @Test
+    public void unloadFailed() throws InterruptedException {
+        bytes.busyLockInt(0);
+        ExecutorService es = Executors.newSingleThreadExecutor(new NamedThreadFactory("unloadFailed"));
+        Future future = es.submit(new Callable() {
+            @Override
+            public Void call() throws Exception {
+                bytes.unlockInt(0);
+                return null;
+            }
+        });
+        es.shutdown();
+        try {
+            future.get();
+            fail();
+        } catch (ExecutionException e) {
+            assertEquals(IllegalMonitorStateException.class, e.getCause().getClass());
+        }
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/io/examples/ParserExampleMain.java b/lang/src/test/java/net/openhft/lang/io/examples/ParserExampleMain.java
new file mode 100644
index 0000000..c9ec1b5
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/io/examples/ParserExampleMain.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io.examples;
+
+import net.openhft.lang.io.ByteBufferBytes;
+import net.openhft.lang.io.StopCharTesters;
+
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Run with -verbosegc -Xmx32m
+ * 

+ * Average time was 282 nano-seconds
+ */
+public class ParserExampleMain {
+    public static void main(String... ignored) {
+        ByteBuffer wrap = ByteBuffer.allocate(1024);
+        ByteBufferBytes bufferBytes = new ByteBufferBytes(wrap);
+        byte[] bytes = "BAC,12.32,12.54,12.56,232443".getBytes();
+
+        int runs = 10000000;
+        long start = System.nanoTime();
+        for (int i = 0; i < runs; i++) {
+            bufferBytes.reset();
+            // read the next message.
+            bufferBytes.write(bytes);
+            bufferBytes.position(0);
+            // decode message
+            String word = bufferBytes.parseUTF(StopCharTesters.COMMA_STOP);
+            double low = bufferBytes.parseDouble();
+            double curr = bufferBytes.parseDouble();
+            double high = bufferBytes.parseDouble();
+            long sequence = bufferBytes.parseLong();
+            if (i == 0) {
+                assertEquals("BAC", word);
+                assertEquals(12.32, low, 0.0);
+                assertEquals(12.54, curr, 0.0);
+                assertEquals(12.56, high, 0.0);
+                assertEquals(232443, sequence);
+            }
+        }
+        long time = System.nanoTime() - start;
+        System.out.println("Average time was " + time / runs + " nano-seconds");
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/io/serialization/ByteMarshallableMarshallerTest.java b/lang/src/test/java/net/openhft/lang/io/serialization/ByteMarshallableMarshallerTest.java
new file mode 100644
index 0000000..95df903
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/io/serialization/ByteMarshallableMarshallerTest.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io.serialization;
+
+import net.openhft.lang.io.Bytes;
+import net.openhft.lang.io.NativeBytes;
+import org.jetbrains.annotations.NotNull;
+import org.junit.Test;
+import sun.nio.ch.DirectBuffer;
+
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * User: peter.lawrey
+ * Date: 20/09/13
+ * Time: 09:28
+ */
+public class ByteMarshallableMarshallerTest {
+    @Test
+    public void testMarshallable() {
+        int capacity = 2 * 1024;
+        ByteBuffer byteBuffer = ByteBuffer.allocateDirect(capacity);
+        long addr = ((DirectBuffer) byteBuffer).address();
+        NativeBytes nativeBytes = new NativeBytes(addr, addr, addr + capacity);
+
+        BytesMarshallable bm = new MockBytesMarshallable(12345678);
+        nativeBytes.writeObject(bm);
+        nativeBytes.finish();
+        nativeBytes.reset();
+        BytesMarshallable bm2 = nativeBytes.readObject(MockBytesMarshallable.class);
+        assertEquals(bm, bm2);
+    }
+
+    static class MockBytesMarshallable implements BytesMarshallable {
+        long number;
+
+        MockBytesMarshallable(long number) {
+            this.number = number;
+        }
+
+        @Override
+        public void readMarshallable(@NotNull Bytes in) throws IllegalStateException {
+            number = in.readLong();
+        }
+
+        @Override
+        public void writeMarshallable(@NotNull Bytes out) {
+            out.writeLong(number);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+
+            MockBytesMarshallable that = (MockBytesMarshallable) o;
+
+            if (number != that.number) return false;
+
+            return true;
+        }
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/io/serialization/ExternalizableMarshallerTest.java b/lang/src/test/java/net/openhft/lang/io/serialization/ExternalizableMarshallerTest.java
new file mode 100644
index 0000000..70bad3e
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/io/serialization/ExternalizableMarshallerTest.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io.serialization;
+
+import net.openhft.lang.io.NativeBytes;
+import org.junit.Test;
+import sun.nio.ch.DirectBuffer;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * User: peter.lawrey
+ * Date: 20/09/13
+ * Time: 09:28
+ */
+public class ExternalizableMarshallerTest {
+    @Test
+    public void testMarshallable() {
+        int capacity = 2 * 1024;
+        ByteBuffer byteBuffer = ByteBuffer.allocateDirect(capacity);
+        long addr = ((DirectBuffer) byteBuffer).address();
+        NativeBytes nativeBytes = new NativeBytes(addr, addr, addr + capacity);
+
+        Externalizable bm = new MockExternalizable(12345678);
+        nativeBytes.writeObject(bm);
+        nativeBytes.finish();
+        nativeBytes.reset();
+        Externalizable bm2 = nativeBytes.readObject(MockExternalizable.class);
+        assertEquals(bm, bm2);
+    }
+
+    static class MockExternalizable implements Externalizable {
+        long number;
+
+        MockExternalizable(long number) {
+            this.number = number;
+        }
+
+        @Override
+        public void writeExternal(ObjectOutput out) throws IOException {
+            out.writeLong(number);
+        }
+
+        @Override
+        public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+            number = in.readLong();
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+
+            MockExternalizable that = (MockExternalizable) o;
+
+            if (number != that.number) return false;
+
+            return true;
+        }
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/io/serialization/RawCopierTest.java b/lang/src/test/java/net/openhft/lang/io/serialization/RawCopierTest.java
new file mode 100644
index 0000000..1b89806
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/io/serialization/RawCopierTest.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io.serialization;
+
+import net.openhft.lang.io.DirectBytes;
+import net.openhft.lang.io.DirectStore;
+import net.openhft.lang.io.NativeBytes;
+import org.junit.Test;
+
+import java.awt.geom.Point2D;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * User: peter.lawrey Date: 22/09/13 Time: 17:06
+ */
+public class RawCopierTest {
+    static void printInts(Object o, int len) {
+        for (long i = 0; i < len; i += 4) {
+            System.out.print(NativeBytes.UNSAFE.getInt(o, i) + " ");
+        }
+        System.out.println();
+    }
+
+    @Test
+    public void testStartEnd() {
+        RawCopier aRawCopier = RawCopier.copies(A.class);
+        if (aRawCopier.start != 8)
+            assertEquals(12, aRawCopier.start);
+        assertEquals(aRawCopier.start + 3 * 4, aRawCopier.end);
+
+        RawCopier bRawCopier = RawCopier.copies(B.class);
+        if (aRawCopier.start != 8)
+            assertEquals(16, bRawCopier.start);
+        assertEquals(bRawCopier.start + 4 * 8, bRawCopier.end);
+    }
+
+    @Test
+    public void testReadWrite() {
+        DirectStore ds = new DirectStore(null, 1024);
+        DirectBytes db = ds.createSlice();
+        RawCopier aRawCopier = RawCopier.copies(A.class);
+        A a = new A();
+        a.i = 111;
+        a.j = -222;
+        a.k = 333;
+        a.s = "Hello";
+        aRawCopier.toBytes(a, db);
+        assertEquals(12, db.position());
+
+        assertEquals(111, db.readInt(0));
+        assertEquals(-222, db.readInt(4));
+        assertEquals(333, db.readInt(8));
+
+        A a2 = new A();
+        a2.i = 1;
+        a2.j = 2;
+        a2.k = 3;
+//        printInts(a2, 28);
+        db.position(0);
+        aRawCopier.fromBytes(db, a2);
+//        printInts(a2, 28);
+        assertEquals(111, a2.i);
+        assertEquals(-222, a2.j);
+        assertEquals(333, a2.k);
+        assertEquals(null, a2.s);
+    }
+
+    static class A {
+        int i, j, k;
+        String s;
+        transient Map map;
+    }
+
+    static class B extends Point2D.Double {
+        double z, w;
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/io/serialization/VanillaBytesMarshallerTest.java b/lang/src/test/java/net/openhft/lang/io/serialization/VanillaBytesMarshallerTest.java
new file mode 100644
index 0000000..b258e32
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/io/serialization/VanillaBytesMarshallerTest.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.io.serialization;
+
+import net.openhft.lang.io.DirectBytes;
+import net.openhft.lang.io.DirectStore;
+import net.openhft.lang.io.NativeBytes;
+import org.junit.Test;
+import sun.nio.ch.DirectBuffer;
+
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * User: peter.lawrey Date: 20/09/13 Time: 09:28
+ */
+public class VanillaBytesMarshallerTest {
+    @Test
+    public void testObjects() {
+        DirectBytes bytes = new DirectStore(1024).createSlice();
+        Object[] objects = {1, 1L, 1.0, "Hello"};
+        for (Object o : objects) {
+            long pos = bytes.position();
+            bytes.writeObject(o);
+            System.out.printf("%s used %,d bytes%n", o.getClass(), bytes.position() - pos);
+        }
+        bytes.reset();
+        for (Object o : objects) {
+            Object o2 = bytes.readObject();
+            assertEquals(o, o2);
+        }
+    }
+
+    @Test
+    public void testMarshallable() {
+        int capacity = 2 * 1024;
+        ByteBuffer byteBuffer = ByteBuffer.allocateDirect(capacity);
+        long addr = ((DirectBuffer) byteBuffer).address();
+        NativeBytes nativeBytes = new NativeBytes(addr, addr, addr + capacity);
+
+        nativeBytes.writeObject(BuySell.BUY);
+        nativeBytes.writeObject(BuySell.SELL);
+        nativeBytes.finish();
+        nativeBytes.reset();
+        assertEquals(BuySell.BUY, nativeBytes.readObject());
+        assertEquals(BuySell.SELL, nativeBytes.readObject());
+    }
+
+    enum BuySell {
+        BUY, SELL
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/io/serialization/direct/DirectSerializationFilterTest.java b/lang/src/test/java/net/openhft/lang/io/serialization/direct/DirectSerializationFilterTest.java
new file mode 100644
index 0000000..35b5c6e
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/io/serialization/direct/DirectSerializationFilterTest.java
@@ -0,0 +1,94 @@
+package net.openhft.lang.io.serialization.direct;
+
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.util.*;
+
+import static net.openhft.lang.io.serialization.direct.DirectSerializationFilter.stopAtFirstIneligibleField;
+import static org.junit.Assert.*;
+
+public class DirectSerializationFilterTest {
+
+    @Test
+    public void instancePrimitivesAreEligible() {
+        List field = fieldsNamed("intField");
+        assertEquals(field, stopAtFirstIneligibleField(field));
+    }
+
+    @Test
+    public void instancePrimitiveArraysAreNotEligible() {
+        List field = fieldsNamed("doubleArray");
+        assertTrue(stopAtFirstIneligibleField(field).isEmpty());
+    }
+
+    @Test
+    public void instanceReferencesAreNotEligible() {
+        List field = fieldsNamed("stringList");
+        assertTrue(stopAtFirstIneligibleField(field).isEmpty());
+    }
+
+    @Test
+    public void instanceReferenceArraysAreNotEligible() {
+        List field = fieldsNamed("objectArray");
+        assertTrue(stopAtFirstIneligibleField(field).isEmpty());
+    }
+
+    @Test
+    public void staticPrimitivesAreNotEligible() {
+        List field = fieldsNamed("staticIntField");
+        assertTrue(stopAtFirstIneligibleField(field).isEmpty());
+    }
+
+    @Test
+    public void staticPrimitiveArraysAreNotEligible() {
+        List field = fieldsNamed("staticDoubleArray");
+        assertTrue(stopAtFirstIneligibleField(field).isEmpty());
+    }
+
+    @Test
+    public void staticReferencesAreNotEligible() {
+        List field = fieldsNamed("staticStringList");
+        assertTrue(stopAtFirstIneligibleField(field).isEmpty());
+    }
+
+    @Test
+    public void staticReferenceArraysAreNotEligible() {
+        List field = fieldsNamed("staticObjectArray");
+        assertTrue(stopAtFirstIneligibleField(field).isEmpty());
+    }
+
+    @Test
+    public void transientPrimitivesAreNotEligible() {
+        List field = fieldsNamed("transientShort");
+        assertTrue(stopAtFirstIneligibleField(field).isEmpty());
+    }
+
+    @Test
+    public void transientReferencsAreNotEligible() {
+        List field = fieldsNamed("transientObject");
+        assertTrue(stopAtFirstIneligibleField(field).isEmpty());
+    }
+
+    @Test
+    public void includesAllEligibleFields() {
+        List fields = fieldsNamed("intField", "shortField", "longField", "byteField", "stringList");
+        List expectedfields = fields.subList(0, 4);
+
+        assertEquals(expectedfields, stopAtFirstIneligibleField(fields));
+    }
+
+    private static List fieldsNamed(String... names) {
+        ArrayList fields = new ArrayList();
+
+        for (String name : names) {
+            try {
+                fields.add(TestClasses.MixedFields.class.getDeclaredField(name));
+            } catch (NoSuchFieldException e) {
+                throw new RuntimeException("Exception retrieving " + name, e);
+            }
+        }
+
+        return fields;
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/io/serialization/direct/DirectSerializationMetadataTest.java b/lang/src/test/java/net/openhft/lang/io/serialization/direct/DirectSerializationMetadataTest.java
new file mode 100644
index 0000000..1c02082
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/io/serialization/direct/DirectSerializationMetadataTest.java
@@ -0,0 +1,78 @@
+package net.openhft.lang.io.serialization.direct;
+
+import org.junit.Test;
+
+import static net.openhft.lang.io.AbstractBytes.UNSIGNED_INT_MASK;
+import static net.openhft.lang.io.NativeBytes.UNSAFE;
+import static net.openhft.lang.io.serialization.direct.DirectSerializationMetadata.*;
+import static net.openhft.lang.io.serialization.direct.TestClasses.*;
+import static org.junit.Assert.assertEquals;
+
+public class DirectSerializationMetadataTest {
+
+    @Test
+    public void primitives1Metadata() {
+        unsafeClassHeaderSize64BitCompressedOops(new Primitives1());
+
+        SerializationMetadata serializationMetadata = extractMetadata(Introspect.fields(Primitives1.class));
+        assertEquals(OBJECT_HEADER_SIZE, serializationMetadata.start);
+        assertEquals(4, serializationMetadata.length);
+    }
+
+    @Test
+    public void primitives2Metadata() {
+        unsafeClassHeaderSize64BitCompressedOops(new Primitives2());
+
+        SerializationMetadata serializationMetadata = extractMetadata(Introspect.fields(Primitives2.class));
+        assertEquals(OBJECT_HEADER_SIZE, serializationMetadata.start);
+        assertEquals(12, serializationMetadata.length);
+    }
+
+    @Test
+    public void primitives3Metadata() {
+        unsafeClassHeaderSize64BitCompressedOops(new Primitives3());
+
+        SerializationMetadata serializationMetadata = extractMetadata(Introspect.fields(Primitives3.class));
+        assertEquals(OBJECT_HEADER_SIZE, serializationMetadata.start);
+        assertEquals(12, serializationMetadata.length);
+    }
+
+    @Test
+    public void primitives4Metadata() {
+        unsafeClassHeaderSize64BitCompressedOops(new Primitives4());
+
+        SerializationMetadata serializationMetadata = extractMetadata(Introspect.fields(Primitives4.class));
+        assertEquals(OBJECT_HEADER_SIZE, serializationMetadata.start);
+        assertEquals(4, serializationMetadata.length);
+    }
+
+    @Test
+    public void primitives5Metadata() {
+        unsafeClassHeaderSize64BitCompressedOops(new Primitives5());
+
+        SerializationMetadata serializationMetadata = extractMetadata(Introspect.fields(Primitives5.class));
+        assertEquals(OBJECT_HEADER_SIZE, serializationMetadata.start);
+        assertEquals(12, serializationMetadata.length);
+    }
+
+    @Test
+    public void primitives6Metadata() {
+        unsafeClassHeaderSize64BitCompressedOops(new Primitives6());
+
+        SerializationMetadata serializationMetadata = extractMetadata(Introspect.fields(Primitives6.class));
+        assertEquals(OBJECT_HEADER_SIZE, serializationMetadata.start);
+        assertEquals(28, serializationMetadata.length);
+    }
+
+    private void unsafeClassHeaderSize64BitCompressedOops(Object instance) {
+        if (System.getProperty("useUnsafeClassHeaders") != null) {
+            long narrowKlassPointer = UNSAFE.getInt(instance, NATIVE_WORD_SIZE) & UNSIGNED_INT_MASK;
+            long wideKlassPointer = narrowKlassPointer << 3;
+            long sizeField = wideKlassPointer + 3 * UNSAFE.addressSize();
+
+            System.out.println(String.format("Size of %s is %s",
+                    instance.getClass().getName(),
+                    UNSAFE.getAddress(sizeField) & UNSIGNED_INT_MASK));
+        }
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/io/serialization/direct/IntrospectTest.java b/lang/src/test/java/net/openhft/lang/io/serialization/direct/IntrospectTest.java
new file mode 100644
index 0000000..1c5879b
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/io/serialization/direct/IntrospectTest.java
@@ -0,0 +1,38 @@
+package net.openhft.lang.io.serialization.direct;
+
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.util.*;
+
+import static net.openhft.lang.io.serialization.direct.TestClasses.*;
+import static org.junit.Assert.assertEquals;
+
+public class IntrospectTest {
+
+    @Test
+    public void returnsFieldsRegardlessOfVisibility() {
+        List fields = asFieldList(InstanceOnlyNoStaticFields.class);
+
+        assertEquals(3, fields.size());
+    }
+
+    @Test
+    public void returnsStaticAsWellAsInstanceFields() {
+        List fields = asFieldList(HasInstanceAndStaticFields.class);
+
+        assertEquals(6, fields.size());
+    }
+
+    @Test
+    public void walksClassHierarchyForAllFields() {
+        List fields = asFieldList(LevelTwoDerivedClass.class);
+
+        assertEquals(6, fields.size());
+    }
+
+    private static List asFieldList(Class clazz) {
+        return new ArrayList(Introspect.fields(clazz));
+    }
+}
+
diff --git a/lang/src/test/java/net/openhft/lang/io/serialization/direct/ObjectMarshallerTest.java b/lang/src/test/java/net/openhft/lang/io/serialization/direct/ObjectMarshallerTest.java
new file mode 100644
index 0000000..7226a17
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/io/serialization/direct/ObjectMarshallerTest.java
@@ -0,0 +1,245 @@
+package net.openhft.lang.io.serialization.direct;
+
+import net.openhft.lang.io.*;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.util.Collections;
+
+import static net.openhft.lang.io.serialization.direct.TestClasses.*;
+import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+
+public class ObjectMarshallerTest {
+
+    @Test
+    public void marshalAndUnmarshalPrimitives1() {
+        Primitives1 p = new Primitives1();
+        p.a = 55;
+
+        Bytes b = createByteStore();
+        ObjectMarshaller marshaller = ObjectMarshallers.forClass(Primitives1.class);
+        marshaller.write(b, p);
+
+        p.a = 0;
+        b.reset();
+        marshaller.read(b, p);
+
+        assertEquals(55, p.a);
+
+        b.reset();
+
+        Primitives1 newP = new Primitives1();
+        marshaller.read(b, newP);
+
+        assertEquals(55, newP.a);
+    }
+
+    @Test
+    public void marshalAndUnmarshalPrimitives2() {
+        Primitives2 p = new Primitives2();
+        p.a = 55;
+        p.b = -10;
+
+        Bytes b = createByteStore();
+        ObjectMarshaller marshaller = ObjectMarshallers.forClass(Primitives2.class);
+        marshaller.write(b, p);
+
+        p.a = 0;
+        p.b = 0;
+        b.reset();
+        marshaller.read(b, p);
+
+        assertEquals(55, p.a);
+        assertEquals(-10, p.b);
+
+        b.reset();
+
+        Primitives2 newP = new Primitives2();
+        marshaller.read(b, newP);
+
+        assertEquals(55, newP.a);
+        assertEquals(-10, newP.b);
+    }
+
+    @Test
+    public void marshalAndUnmarshalPrimitives3() {
+        Primitives3 p = new Primitives3();
+        p.a = 55;
+        p.b = -10;
+        p.c = 92;
+
+        Bytes b = createByteStore();
+        ObjectMarshaller marshaller = ObjectMarshallers.forClass(Primitives3.class);
+        marshaller.write(b, p);
+
+        p.a = 0;
+        p.b = 0;
+        p.c = 0;
+        b.reset();
+        marshaller.read(b, p);
+
+        assertEquals(55, p.a);
+        assertEquals(-10, p.b);
+        assertEquals(92, p.c);
+
+        b.reset();
+
+        Primitives3 newP = new Primitives3();
+        marshaller.read(b, newP);
+
+        assertEquals(55, newP.a);
+        assertEquals(-10, newP.b);
+        assertEquals(92, newP.c);
+    }
+
+    @Test
+    public void marshalAndUnmarshalPrimitives4() {
+        Primitives4 p = new Primitives4();
+        p.a = true;
+
+        Bytes b = createByteStore();
+        ObjectMarshaller marshaller = ObjectMarshallers.forClass(Primitives4.class);
+        marshaller.write(b, p);
+
+        p.a = false;
+        b.reset();
+        marshaller.read(b, p);
+
+        assertTrue(p.a);
+
+        b.reset();
+
+        Primitives4 newP = new Primitives4();
+        marshaller.read(b, newP);
+
+        assertTrue(newP.a);
+    }
+
+    @Test
+    public void marshalAndUnmarshalPrimitives5() {
+        Primitives5 p = new Primitives5();
+        p.a = true;
+        p.b = Long.MIN_VALUE;
+
+        Bytes b = createByteStore();
+        ObjectMarshaller marshaller = ObjectMarshallers.forClass(Primitives5.class);
+        marshaller.write(b, p);
+
+        p.a = false;
+        p.b = 0;
+        b.reset();
+        marshaller.read(b, p);
+
+        assertTrue(p.a);
+        assertEquals(Long.MIN_VALUE, p.b);
+
+        b.reset();
+
+        Primitives5 newP = new Primitives5();
+        marshaller.read(b, newP);
+
+        assertTrue(newP.a);
+        assertEquals(Long.MIN_VALUE, newP.b);
+    }
+
+    @Test
+    public void marshalAndUnmarshalPrimitives6() {
+        Primitives6 p = new Primitives6();
+        p.a = true;
+        p.b = Integer.MAX_VALUE;
+        p.c = Short.MAX_VALUE;
+        p.d = Long.MAX_VALUE;
+        p.e = Double.MAX_VALUE;
+
+        Bytes b = createByteStore();
+        ObjectMarshaller marshaller = ObjectMarshallers.forClass(Primitives6.class);
+        marshaller.write(b, p);
+
+        p.a = false;
+        p.b = 0;
+        p.c = 0;
+        p.d = 0;
+        p.e = 0;
+        b.reset();
+        marshaller.read(b, p);
+
+        assertTrue(p.a);
+        assertEquals(Integer.MAX_VALUE, p.b);
+        assertEquals(Short.MAX_VALUE, p.c);
+        assertEquals(Long.MAX_VALUE, p.d);
+        assertEquals(Double.MAX_VALUE, p.e, 0);
+
+        b.reset();
+
+        Primitives6 newP = new Primitives6();
+        marshaller.read(b, newP);
+
+        assertTrue(newP.a);
+        assertEquals(Integer.MAX_VALUE, p.b);
+        assertEquals(Short.MAX_VALUE, p.c);
+        assertEquals(Long.MAX_VALUE, p.d);
+        assertEquals(Double.MAX_VALUE, p.e, 0);
+    }
+
+    @Test
+    public void marshalAndUnmarshalMixedClass() {
+        MixedFields m = new MixedFields();
+        m.intField = Integer.MIN_VALUE + 1;
+        m.byteField = Byte.MAX_VALUE - 1;
+        m.shortField = Short.MIN_VALUE + 1;
+        m.longField = Long.MAX_VALUE - 1;
+
+        m.doubleArray = new double[]{-50.0, 50.0};
+        m.stringList = Collections.singletonList("Foo");
+        m.objectArray = new Object[]{new Primitives1()};
+        m.transientShort = 12;
+        m.transientObject = new Primitives2();
+
+        Bytes b = createByteStore();
+        ObjectMarshaller marshaller = ObjectMarshallers.forClass(MixedFields.class);
+        marshaller.write(b, m);
+
+        m.intField = 0;
+        m.byteField = 0;
+        m.shortField = 0;
+        m.longField = 0;
+        m.doubleArray = null;
+        m.stringList = null;
+        m.objectArray = null;
+        m.transientShort = 0;
+        m.transientObject = null;
+
+        b.reset();
+        marshaller.read(b, m);
+
+        assertEquals(Integer.MIN_VALUE + 1, m.intField);
+        assertEquals(0, m.byteField); // because of jvm field rearrangement the two shorts are packed into 4 bytes and this eligible field is skipped
+        assertEquals(Short.MIN_VALUE + 1, m.shortField);
+        assertEquals(Long.MAX_VALUE - 1, m.longField);
+        assertNull(m.doubleArray);
+        assertNull(m.stringList);
+        assertNull(m.objectArray);
+        assertNull(m.transientObject);
+        assertEquals(0, m.transientShort);
+
+        b.reset();
+
+        MixedFields newM = new MixedFields();
+        marshaller.read(b, newM);
+
+        assertEquals(Integer.MIN_VALUE + 1, newM.intField);
+        assertEquals(0, newM.byteField);
+        assertEquals(Short.MIN_VALUE + 1, newM.shortField);
+        assertEquals(Long.MAX_VALUE - 1, newM.longField);
+        assertNull(newM.doubleArray);
+        assertNull(newM.stringList);
+        assertNull(newM.objectArray);
+        assertNull(newM.transientObject);
+        assertEquals(0, newM.transientShort);
+    }
+
+    private ByteBufferBytes createByteStore() {
+        return new ByteBufferBytes(ByteBuffer.allocate(64));
+    }
+}
\ No newline at end of file
diff --git a/lang/src/test/java/net/openhft/lang/io/serialization/direct/TestClasses.java b/lang/src/test/java/net/openhft/lang/io/serialization/direct/TestClasses.java
new file mode 100644
index 0000000..7b173dd
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/io/serialization/direct/TestClasses.java
@@ -0,0 +1,90 @@
+package net.openhft.lang.io.serialization.direct;
+
+import java.util.*;
+
+class TestClasses {
+
+    public static class InstanceOnlyNoStaticFields {
+        public int intField;
+        protected String stringField;
+        private Object objectField;
+    }
+
+    public static class HasInstanceAndStaticFields {
+        public static int staticIntField;
+        protected static String staticStringField;
+        private static Object staticObjectField;
+
+        public int intField;
+        protected String stringField;
+        private Object objectField;
+    }
+
+    public static class BaseClass {
+        private static double staticBaseDoubleField;
+
+        protected List baseListField;
+    }
+
+    public static class LevelOneDerivedClass extends BaseClass {
+        protected static Object staticLevelOneObjectField;
+
+        public long levelOneLongField;
+    }
+
+    public static class LevelTwoDerivedClass extends LevelOneDerivedClass {
+        public static String staticLevelTwoStringField;
+
+        final int levelTwoDerivedIntField = 0;
+    }
+
+    public static class MixedFields {
+        public static int staticIntField ;
+        public static double[] staticDoubleArray;
+
+        public static List staticStringList = Collections.singletonList("S1");
+        public static Object[] staticObjectArray = {new Object()};
+
+        public int intField;
+        public byte byteField;
+        public short shortField;
+        public long longField;
+        public double[] doubleArray;
+
+        public List stringList;
+        public Object[] objectArray;
+
+        transient short transientShort;
+        public transient Object transientObject;
+    }
+
+    public static class Primitives1 {
+        public int a;
+    }
+
+    public static class Primitives2 {
+        public int a, b;
+    }
+
+    public static class Primitives3 {
+        public int a, b;
+        public short c;
+    }
+
+    public static class Primitives4 {
+        public boolean a;
+    }
+
+    public static class Primitives5 {
+        public boolean a;
+        public long b;
+    }
+
+    public static class Primitives6 {
+        public boolean a;
+        public int b;
+        public short c;
+        public long d;
+        public double e;
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/model/DataValueGeneratorTest.java b/lang/src/test/java/net/openhft/lang/model/DataValueGeneratorTest.java
new file mode 100644
index 0000000..1310fde
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/model/DataValueGeneratorTest.java
@@ -0,0 +1,174 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model;
+
+import net.openhft.compiler.CachedCompiler;
+import net.openhft.lang.io.ByteBufferBytes;
+import net.openhft.lang.io.Bytes;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * User: peter.lawrey Date: 06/10/13 Time: 20:13
+ */
+public class DataValueGeneratorTest {
+    @Test
+    public void testGenerateJavaCode() throws Exception {
+        DataValueGenerator dvg = new DataValueGenerator();
+//        dvg.setDumpCode(true);
+        JavaBeanInterface jbi = dvg.heapInstance(JavaBeanInterface.class);
+        jbi.setByte((byte) 1);
+        jbi.setChar('2');
+        jbi.setShort((short) 3);
+        jbi.setInt(4);
+        jbi.setFloat(5);
+        jbi.setLong(6);
+        jbi.setDouble(7);
+        jbi.setFlag(true);
+        assertEquals(1, jbi.getByte());
+        assertEquals('2', jbi.getChar());
+        assertEquals(3, jbi.getShort());
+        assertEquals(4, jbi.getInt());
+        assertEquals(5.0, jbi.getFloat(), 0);
+        assertEquals(6, jbi.getLong());
+        assertEquals(7.0, jbi.getDouble(), 0.0);
+        assertTrue(jbi.getFlag());
+    }
+
+    @Test
+    public void testGenerateJavaCode2() throws Exception {
+        DataValueGenerator dvg = new DataValueGenerator();
+        MinimalInterface mi = dvg.heapInstance(MinimalInterface.class);
+
+        mi.byte$((byte) 1);
+        mi.char$('2');
+        mi.short$((short) 3);
+        mi.int$(4);
+        mi.float$(5);
+        mi.long$(6);
+        mi.double$(7);
+        mi.flag(true);
+
+        assertEquals(1, mi.byte$());
+        assertEquals('2', mi.char$());
+        assertEquals(3, mi.short$());
+        assertEquals(4, mi.int$());
+        assertEquals(5.0, mi.float$(), 0);
+        assertEquals(6, mi.long$());
+        assertEquals(7.0, mi.double$(), 0.0);
+        assertTrue(mi.flag());
+
+        ByteBufferBytes bbb = new ByteBufferBytes(ByteBuffer.allocate(64));
+        mi.writeMarshallable(bbb);
+        System.out.println("size: " + bbb.position());
+
+        MinimalInterface mi2 = dvg.heapInstance(MinimalInterface.class);
+        bbb.position(0);
+        mi2.readMarshallable(bbb);
+
+
+        assertEquals(1, mi2.byte$());
+        assertEquals('2', mi2.char$());
+        assertEquals(3, mi2.short$());
+        assertEquals(4, mi2.int$());
+        assertEquals(5.0, mi2.float$(), 0);
+        assertEquals(6, mi2.long$());
+        assertEquals(7.0, mi2.double$(), 0.0);
+        assertTrue(mi2.flag());
+    }
+
+    @Test
+    public void testGenerateNative() throws Exception {
+        String actual = new DataValueGenerator().generateNativeObject(JavaBeanInterface.class);
+//        System.out.println(actual);
+        CachedCompiler cc = new CachedCompiler(null, null);
+        Class aClass = cc.loadFromJava(JavaBeanInterface.class.getName() + "£native", actual);
+        JavaBeanInterface jbi = (JavaBeanInterface) aClass.asSubclass(JavaBeanInterface.class).newInstance();
+        Bytes bytes = new ByteBufferBytes(ByteBuffer.allocate(64));
+        ((Byteable) jbi).bytes(bytes);
+        jbi.setByte((byte) 1);
+        jbi.setChar('2');
+        jbi.setShort((short) 3);
+        jbi.setInt(4);
+        jbi.setFloat(5);
+        jbi.setLong(6);
+        jbi.setDouble(7);
+        jbi.setFlag(true);
+        assertEquals("", jbi.getString());
+        jbi.setString("G'day");
+        assertEquals(1, jbi.getByte());
+        assertEquals('2', jbi.getChar());
+        assertEquals(3, jbi.getShort());
+        assertEquals(4, jbi.getInt());
+        assertEquals(5.0, jbi.getFloat(), 0);
+        assertEquals(6, jbi.getLong());
+        assertEquals(7.0, jbi.getDouble(), 0.0);
+        assertTrue(jbi.getFlag());
+        assertEquals("G'day", jbi.getString());
+        assertEquals(42, ((Byteable) jbi).maxSize());
+    }
+
+
+    @Test
+    public void testStringFields() {
+        DataValueGenerator dvg = new DataValueGenerator();
+        StringInterface si = dvg.heapInstance(StringInterface.class);
+        si.setString("Hello world");
+        assertEquals("Hello world", si.getString());
+
+        StringInterface si2 = dvg.nativeInstance(StringInterface.class);
+        Bytes bytes = new ByteBufferBytes(ByteBuffer.allocate(192));
+        ((Byteable) si2).bytes(bytes);
+        si2.setString("Hello world £€");
+        si2.setText("Hello world £€");
+        assertEquals("Hello world £€", si2.getString());
+        assertEquals("Hello world £€", si2.getText());
+    }
+
+    @Test
+    public void testNested() {
+        DataValueGenerator dvg = new DataValueGenerator();
+//        dvg.setDumpCode(true);
+        NestedB nestedB1 = dvg.heapInstance(NestedB.class);
+        nestedB1.ask(100);
+        nestedB1.bid(100);
+        NestedB nestedB2 = dvg.heapInstance(NestedB.class);
+        nestedB2.ask(91);
+        nestedB2.bid(92);
+
+//        dvg.setDumpCode(true);
+        NestedA nestedA = dvg.nativeInstance(NestedA.class);
+        Bytes bytes = new ByteBufferBytes(ByteBuffer.allocate(192));
+        ((Byteable) nestedA).bytes(bytes);
+        nestedA.key("key");
+        nestedA.one(nestedB1);
+        nestedA.two(nestedB2);
+        assertEquals("key", nestedA.key());
+        assertEquals(nestedB1.ask(), nestedA.one().ask(), 0.0);
+        assertEquals(nestedB1.bid(), nestedA.one().bid(), 0.0);
+        assertEquals(nestedB2.ask(), nestedA.two().ask(), 0.0);
+        assertEquals(nestedB2.bid(), nestedA.two().bid(), 0.0);
+        assertEquals(nestedB1, nestedA.one());
+        assertEquals(nestedB2, nestedA.two());
+        assertEquals(nestedB1.hashCode(), nestedA.one().hashCode());
+        assertEquals(nestedB2.hashCode(), nestedA.two().hashCode());
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/model/DataValueModelTest.java b/lang/src/test/java/net/openhft/lang/model/DataValueModelTest.java
new file mode 100644
index 0000000..0f7a8f0
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/model/DataValueModelTest.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * User: peter.lawrey Date: 06/10/13 Time: 18:12
+ */
+public class DataValueModelTest {
+    @Test
+    public void testAcquire() {
+        DataValueModel midvm = DataValueModels.acquireModel(MinimalInterface.class);
+        assertEquals("{byte$=FieldModel{name='byte$', getter=public abstract byte net.openhft.lang.model.MinimalInterface.byte$(), setter=public abstract void net.openhft.lang.model.MinimalInterface.byte$(byte)}\n" +
+                " char$=FieldModel{name='char$', getter=public abstract char net.openhft.lang.model.MinimalInterface.char$(), setter=public abstract void net.openhft.lang.model.MinimalInterface.char$(char)}\n" +
+                " double$=FieldModel{name='double$', getter=public abstract double net.openhft.lang.model.MinimalInterface.double$(), setter=public abstract void net.openhft.lang.model.MinimalInterface.double$(double)}\n" +
+                " flag=FieldModel{name='flag', getter=public abstract boolean net.openhft.lang.model.MinimalInterface.flag(), setter=public abstract void net.openhft.lang.model.MinimalInterface.flag(boolean)}\n" +
+                " float$=FieldModel{name='float$', getter=public abstract float net.openhft.lang.model.MinimalInterface.float$(), setter=public abstract void net.openhft.lang.model.MinimalInterface.float$(float)}\n" +
+                " int$=FieldModel{name='int$', getter=public abstract int net.openhft.lang.model.MinimalInterface.int$(), setter=public abstract void net.openhft.lang.model.MinimalInterface.int$(int)}\n" +
+                " long$=FieldModel{name='long$', getter=public abstract long net.openhft.lang.model.MinimalInterface.long$(), setter=public abstract void net.openhft.lang.model.MinimalInterface.long$(long)}\n" +
+                " short$=FieldModel{name='short$', getter=public abstract short net.openhft.lang.model.MinimalInterface.short$(), setter=public abstract void net.openhft.lang.model.MinimalInterface.short$(short)}}"
+                , midvm.fieldMap().toString().replaceAll("},", "}\n"));
+        DataValueModel jbdvm = DataValueModels.acquireModel(JavaBeanInterface.class);
+        assertEquals("{byte=FieldModel{name='byte', getter=public abstract byte net.openhft.lang.model.JavaBeanInterface.getByte(), setter=public abstract void net.openhft.lang.model.JavaBeanInterface.setByte(byte)}\n" +
+                " char=FieldModel{name='char', getter=public abstract char net.openhft.lang.model.JavaBeanInterface.getChar(), setter=public abstract void net.openhft.lang.model.JavaBeanInterface.setChar(char)}\n" +
+                " double=FieldModel{name='double', getter=public abstract double net.openhft.lang.model.JavaBeanInterface.getDouble(), setter=public abstract void net.openhft.lang.model.JavaBeanInterface.setDouble(double)}\n" +
+                " flag=FieldModel{name='flag', getter=public abstract boolean net.openhft.lang.model.JavaBeanInterface.getFlag(), setter=public abstract void net.openhft.lang.model.JavaBeanInterface.setFlag(boolean)}\n" +
+                " float=FieldModel{name='float', getter=public abstract float net.openhft.lang.model.JavaBeanInterface.getFloat(), setter=public abstract void net.openhft.lang.model.JavaBeanInterface.setFloat(float)}\n" +
+                " int=FieldModel{name='int', getter=public abstract int net.openhft.lang.model.JavaBeanInterface.getInt(), setter=public abstract void net.openhft.lang.model.JavaBeanInterface.setInt(int)}\n" +
+                " long=FieldModel{name='long', getter=public abstract long net.openhft.lang.model.JavaBeanInterface.getLong(), setter=public abstract void net.openhft.lang.model.JavaBeanInterface.setLong(long)}\n" +
+                " record=FieldModel{name='record', getter=public abstract int net.openhft.lang.model.JavaBeanInterface.getRecord(), setter=public abstract void net.openhft.lang.model.JavaBeanInterface.setRecord(int)}\n" +
+                " short=FieldModel{name='short', getter=public abstract short net.openhft.lang.model.JavaBeanInterface.getShort(), setter=public abstract void net.openhft.lang.model.JavaBeanInterface.setShort(short)}\n" +
+                " string=FieldModel{name='string', getter=public abstract java.lang.String net.openhft.lang.model.JavaBeanInterface.getString(), setter=public abstract void net.openhft.lang.model.JavaBeanInterface.setString(java.lang.String), size= @net.openhft.lang.model.constraints.MaxSize(value=8)}}"
+                , jbdvm.fieldMap().toString().replaceAll("},", "}\n"));
+
+        DataValueModel nais = DataValueModels.acquireModel(NestedArrayInterface.class);
+        assertEquals("{int=FieldModel{name='int', getter=public abstract int net.openhft.lang.model.NestedArrayInterface.getInt(int), setter=public abstract void net.openhft.lang.model.NestedArrayInterface.setInt(int,int), indexSize= @net.openhft.lang.model.constraints.MaxSize(value=16)}\n" +
+                " jBI=FieldModel{name='jBI', getter=public abstract net.openhft.lang.model.JavaBeanInterface net.openhft.lang.model.NestedArrayInterface.getJBI(int), setter=public abstract void net.openhft.lang.model.NestedArrayInterface.setJBI(int,net.openhft.lang.model.JavaBeanInterface), indexSize= @net.openhft.lang.model.constraints.MaxSize(value=32)}\n" +
+                " text=FieldModel{name='text', getter=public abstract java.lang.String net.openhft.lang.model.NestedArrayInterface.getText(), setter=public abstract void net.openhft.lang.model.NestedArrayInterface.setText(java.lang.String)}}"
+                , nais.fieldMap().toString().replaceAll("},", "}\n"));
+
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/model/JavaBeanInterface.java b/lang/src/test/java/net/openhft/lang/model/JavaBeanInterface.java
new file mode 100644
index 0000000..0cf0675
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/model/JavaBeanInterface.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model;
+
+import net.openhft.lang.model.constraints.MaxSize;
+
+/**
+ * User: peter.lawrey
+ * Date: 06/10/13
+ * Time: 16:59
+ */
+public interface JavaBeanInterface {
+    int getRecord();
+
+    void setRecord(int record);
+
+    void busyLockRecord() throws InterruptedException;
+
+    void unlockRecord();
+
+    void setFlag(boolean flag);
+
+    boolean getFlag();
+
+    void setByte(byte b);
+
+    byte getByte();
+
+    void setShort(short s);
+
+    short getShort();
+
+    void setChar(char ch);
+
+    char getChar();
+
+    void setInt(int i);
+
+    int getInt();
+
+    void setFloat(float f);
+
+    float getFloat();
+
+    void setLong(long l);
+
+    long getLong();
+
+    void setDouble(double d);
+
+    double getDouble();
+
+    void setString(@MaxSize(8) String s);
+
+    String getString();
+}
diff --git a/lang/src/test/java/net/openhft/lang/model/MinimalInterface.java b/lang/src/test/java/net/openhft/lang/model/MinimalInterface.java
new file mode 100644
index 0000000..2379b2e
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/model/MinimalInterface.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model;
+
+import net.openhft.lang.io.serialization.BytesMarshallable;
+
+/**
+ * User: peter.lawrey
+ * Date: 06/10/13
+ * Time: 16:59
+ */
+public interface MinimalInterface extends BytesMarshallable, Copyable, Byteable {
+    void flag(boolean flag);
+
+    boolean flag();
+
+    void byte$(byte b);
+
+    byte byte$();
+
+    void short$(short s);
+
+    short short$();
+
+    void char$(char ch);
+
+    char char$();
+
+    void int$(int i);
+
+    int int$();
+
+    void float$(float f);
+
+    float float$();
+
+    void long$(long l);
+
+    long long$();
+
+    void double$(double d);
+
+    double double$();
+}
diff --git a/lang/src/test/java/net/openhft/lang/model/NestedA.java b/lang/src/test/java/net/openhft/lang/model/NestedA.java
new file mode 100644
index 0000000..70d5f8a
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/model/NestedA.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model;
+
+import net.openhft.lang.model.constraints.MaxSize;
+
+/**
+ * User: peter.lawrey
+ * Date: 08/10/13
+ * Time: 10:11
+ */
+public interface NestedA {
+    void key(@MaxSize String key);
+
+    String key();
+
+    void one(NestedB one);
+
+    NestedB one();
+
+    void two(NestedB one);
+
+    NestedB two();
+
+}
diff --git a/lang/src/test/java/net/openhft/lang/model/NestedArrayInterface.java b/lang/src/test/java/net/openhft/lang/model/NestedArrayInterface.java
new file mode 100644
index 0000000..22caef0
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/model/NestedArrayInterface.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model;
+
+import net.openhft.lang.model.constraints.MaxSize;
+
+/**
+ */
+public interface NestedArrayInterface {
+    String getText();
+
+    void setText(String text);
+
+    int getInt(int index);
+
+    void setInt(@MaxSize(16) int index, int value);
+
+    JavaBeanInterface getJBI(@MaxSize(32) int index);
+
+    void setJBI(int index, JavaBeanInterface jbi);
+}
diff --git a/lang/src/test/java/net/openhft/lang/model/NestedB.java b/lang/src/test/java/net/openhft/lang/model/NestedB.java
new file mode 100644
index 0000000..5b4807b
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/model/NestedB.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model;
+
+import net.openhft.lang.model.constraints.Digits;
+
+/**
+ * User: peter.lawrey
+ * Date: 08/10/13
+ * Time: 10:24
+ */
+public interface NestedB {
+    void bid(@Digits double bid);
+
+    double bid();
+
+    void ask(@Digits double ask);
+
+    double ask();
+}
diff --git a/lang/src/test/java/net/openhft/lang/model/StringInterface.java b/lang/src/test/java/net/openhft/lang/model/StringInterface.java
new file mode 100644
index 0000000..14d5d19
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/model/StringInterface.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.model;
+
+import net.openhft.lang.model.constraints.MaxSize;
+
+/**
+ * User: peter.lawrey
+ * Date: 08/10/13
+ * Time: 09:09
+ */
+public interface StringInterface {
+    void setString(@MaxSize(64) String s);
+
+    String getString();
+
+    void setText(@MaxSize String s);
+
+    String getText();
+}
diff --git a/lang/src/test/java/net/openhft/lang/testing/RunningMinimumTest.java b/lang/src/test/java/net/openhft/lang/testing/RunningMinimumTest.java
new file mode 100644
index 0000000..a6c0ddb
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/testing/RunningMinimumTest.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.testing;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * User: peter.lawrey
+ * Date: 05/08/13
+ * Time: 19:14
+ */
+public class RunningMinimumTest {
+    @Test
+    public void testSample() throws Exception {
+        for (int k = 0; k < 1000; k++) {
+            for (long delta : new long[]{0, Integer.MIN_VALUE, Integer.MAX_VALUE}) {
+                RunningMinimum rm = new RunningMinimum(50 * 1000);
+                int j;
+                for (j = 0; j < 50 * 1000000; j += 1000000) {
+                    long startTime = System.nanoTime() + j;
+                    long endTime = System.nanoTime() + j + delta + (long) (Math.pow(10 * 1000, Math.random()) * 1000);
+                    rm.sample(startTime, endTime);
+                }
+                assertEquals("delta=" + delta, delta, rm.minimum(), 40 * 1000);
+            }
+        }
+    }
+
+    @Test
+    public void testVanillaDiff() {
+        VanillaDifferencer vd = new VanillaDifferencer();
+        assertEquals(100, vd.sample(123400, 123500));
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/values/CheckValuesBuildTest.java b/lang/src/test/java/net/openhft/lang/values/CheckValuesBuildTest.java
new file mode 100644
index 0000000..f10ae6e
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/values/CheckValuesBuildTest.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.values;
+
+import net.openhft.lang.model.DataValueGenerator;
+import org.junit.Test;
+
+/**
+ * User: peter.lawrey
+ * Date: 10/10/13
+ * Time: 11:38
+ */
+public class CheckValuesBuildTest {
+    @Test
+    public void testValuesCompile() throws ClassNotFoundException {
+        DataValueGenerator dvg = new DataValueGenerator();
+        for (Class clazz : new Class[]{
+                IntValue.class,
+                ByteValue.class,
+                CharValue.class,
+                ShortValue.class,
+                Int24Value.class,
+                FloatValue.class,
+                DoubleValue.class,
+                LongValue.class,
+                StringValue.class,
+                BooleanValue.class,
+                UnsignedByteValue.class,
+                UnsignedIntValue.class,
+                UnsignedShortValue.class,
+                NestAll.class
+        }) {
+            System.out.println(dvg.acquireHeapClass(clazz).getName() + " "
+                    + dvg.acquireNativeClass(clazz).getName());
+        }
+    }
+}
diff --git a/lang/src/test/java/net/openhft/lang/values/NestAll.java b/lang/src/test/java/net/openhft/lang/values/NestAll.java
new file mode 100644
index 0000000..cb629b6
--- /dev/null
+++ b/lang/src/test/java/net/openhft/lang/values/NestAll.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2013 Peter Lawrey
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package net.openhft.lang.values;
+
+/**
+ * User: peter.lawrey
+ * Date: 11/10/13
+ * Time: 08:59
+ */
+public interface NestAll {
+    BooleanValue getBoolean();
+
+    ByteValue getByte();
+
+    CharValue getChar();
+
+    DoubleValue getDouble();
+
+    FloatValue getFloat();
+
+    Int24Value getInt24();
+
+    Int48Value getInt48();
+
+    IntValue getInt();
+
+    LongValue getLong();
+
+    ShortValue getShort();
+
+    StringValue getString();
+
+    UnsignedByteValue getUB();
+
+    UnsignedIntValue getUI();
+
+    UnsignedShortValue getUS();
+
+}
-- 
cgit v1.2.3