From 681f31895f47227e1ed676ebd1d29ede01493084 Mon Sep 17 00:00:00 2001 From: varunnvs92 Date: Fri, 1 Jun 2018 14:41:39 -0700 Subject: [PATCH 001/490] Add Automatic-Module-Name in manifest so Java9 modular applications can depend on this library (#142) --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index b962bd298b..d13b85b9cb 100644 --- a/pom.xml +++ b/pom.xml @@ -120,6 +120,7 @@ ${build.time} ${project.version} + software.amazon.ion From 3748ed1558616b4e1258949567b709a68ac73c5b Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Wed, 20 Jun 2018 18:39:43 -0700 Subject: [PATCH 002/490] Decouples IonReader from IonSystem and adds IonReaderBuilder. (#143) --- .../ion/impl/IonReaderBinarySystemX.java | 9 +- .../amazon/ion/impl/IonReaderBinaryUserX.java | 32 +- .../amazon/ion/impl/IonReaderTextSystemX.java | 14 +- .../amazon/ion/impl/IonReaderTextUserX.java | 56 +-- .../amazon/ion/impl/IonReaderTreeSystem.java | 26 +- .../amazon/ion/impl/IonReaderTreeUserX.java | 38 +- .../amazon/ion/impl/IonWriterSystem.java | 3 +- .../amazon/ion/impl/IonWriterSystemTree.java | 17 +- .../amazon/ion/impl/IonWriterUser.java | 7 +- .../amazon/ion/impl/LocalSymbolTable.java | 335 +++--------------- .../ion/impl/LocalSymbolTableAsStruct.java | 265 ++++++++++++++ .../ion/impl/PrivateIonReaderFactory.java | 152 ++++---- .../ion/impl/PrivateIonTextWriterBuilder.java | 2 +- .../impl/PrivateLocalSymbolTableFactory.java | 80 +++++ .../amazon/ion/impl/PrivateUtils.java | 83 ++--- .../amazon/ion/impl/PrivateValueFactory.java | 32 ++ .../amazon/ion/impl/lite/IonDatagramLite.java | 5 +- .../amazon/ion/impl/lite/IonLoaderLite.java | 12 +- .../amazon/ion/impl/lite/IonSystemLite.java | 54 +-- .../impl/lite/PrivateLiteDomTrampoline.java | 7 +- .../ion/impl/lite/ValueFactoryLite.java | 18 +- .../amazon/ion/system/IonReaderBuilder.java | 307 ++++++++++++++++ .../amazon/ion/system/IonSystemBuilder.java | 3 +- test/AllTests.java | 2 + test/software/amazon/ion/DatagramTest.java | 3 +- test/software/amazon/ion/LoaderTest.java | 1 + .../amazon/ion/SharedSymtabMaker.java | 2 +- .../amazon/ion/SystemProcessingTestCase.java | 3 +- .../amazon/ion/impl/BinaryWriterTest.java | 1 - .../BinaryWriterWithLocalSymtabsTest.java | 7 +- .../amazon/ion/impl/IonWriterTestCase.java | 3 +- .../LocalSymbolTableImportAdapterTest.java | 14 +- .../amazon/ion/impl/LocalSymbolTableTest.java | 7 +- .../OptimizedBinaryWriterSymbolTableTest.java | 3 +- .../impl/OptimizedBinaryWriterTestCase.java | 2 +- .../ion/impl/OutputStreamWriterTestCase.java | 1 - .../ion/impl/SharedSymbolTableTest.java | 3 +- .../amazon/ion/impl/SymbolTableTest.java | 17 +- .../amazon/ion/{ => impl}/Symtabs.java | 42 +-- .../amazon/ion/impl/TextWriterTest.java | 1 - .../amazon/ion/impl/ValueWriterTest.java | 1 - .../amazon/ion/impl/lite/IonContextTest.java | 4 +- .../amazon/ion/streaming/ReaderTest.java | 16 +- .../system/IonBinaryWriterBuilderTest.java | 15 +- .../ion/system/IonReaderBuilderTest.java | 113 ++++++ .../ion/system/IonTextWriterBuilderTest.java | 2 +- 46 files changed, 1172 insertions(+), 648 deletions(-) create mode 100644 src/software/amazon/ion/impl/LocalSymbolTableAsStruct.java create mode 100644 src/software/amazon/ion/impl/PrivateLocalSymbolTableFactory.java create mode 100644 src/software/amazon/ion/impl/PrivateValueFactory.java create mode 100644 src/software/amazon/ion/system/IonReaderBuilder.java rename test/software/amazon/ion/{ => impl}/Symtabs.java (90%) create mode 100644 test/software/amazon/ion/system/IonReaderBuilderTest.java diff --git a/src/software/amazon/ion/impl/IonReaderBinarySystemX.java b/src/software/amazon/ion/impl/IonReaderBinarySystemX.java index 544d4fcd81..1bf2a98e19 100644 --- a/src/software/amazon/ion/impl/IonReaderBinarySystemX.java +++ b/src/software/amazon/ion/impl/IonReaderBinarySystemX.java @@ -24,7 +24,6 @@ import java.util.Iterator; import software.amazon.ion.Decimal; import software.amazon.ion.IntegerSize; -import software.amazon.ion.IonSystem; import software.amazon.ion.IonType; import software.amazon.ion.NullValueException; import software.amazon.ion.SymbolTable; @@ -38,16 +37,14 @@ class IonReaderBinarySystemX extends IonReaderBinaryRawX implements PrivateReaderWriter { - IonSystem _system; SymbolTable _symbols; - // ValueVariant _v; actually owned by the raw reader so it can be cleared at appropriate times - IonReaderBinarySystemX(IonSystem system, UnifiedInputStreamX in) + IonReaderBinarySystemX(UnifiedInputStreamX in) { super(); init_raw(in); - _system = system; - _symbols = system.getSystemSymbolTable(); + // TODO check IVM to determine version: amznlabs/ion-java#19, amznlabs/ion-java#24 + _symbols = SharedSymbolTable.getSystemSymbolTable(1); } diff --git a/src/software/amazon/ion/impl/IonReaderBinaryUserX.java b/src/software/amazon/ion/impl/IonReaderBinaryUserX.java index 28727f3b0e..aa07fab0d3 100644 --- a/src/software/amazon/ion/impl/IonReaderBinaryUserX.java +++ b/src/software/amazon/ion/impl/IonReaderBinaryUserX.java @@ -16,11 +16,9 @@ import static software.amazon.ion.SystemSymbols.ION_1_0_SID; import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; -import static software.amazon.ion.impl.PrivateUtils.newLocalSymtab; import java.io.IOException; import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonSystem; import software.amazon.ion.IonType; import software.amazon.ion.OffsetSpan; import software.amazon.ion.SeekableReader; @@ -41,6 +39,7 @@ final class IonReaderBinaryUserX * {@link OffsetSpan}s. */ private final int _physical_start_offset; + private final PrivateLocalSymbolTableFactory _lstFactory; IonCatalog _catalog; @@ -63,29 +62,22 @@ public long getFinishOffset() } } - public IonReaderBinaryUserX(IonSystem system, - IonCatalog catalog, + public IonReaderBinaryUserX(IonCatalog catalog, + PrivateLocalSymbolTableFactory lstFactory, UnifiedInputStreamX userBytes, int physicalStartOffset) { - super(system, userBytes); + super(userBytes); _physical_start_offset = physicalStartOffset; init_user(catalog); - } - - public IonReaderBinaryUserX(IonSystem system, - IonCatalog catalog, - UnifiedInputStreamX userBytes) - { - super(system, userBytes); - _physical_start_offset = 0; - init_user(catalog); + _lstFactory = lstFactory; } //FIXME: PERF_TEST was :private final void init_user(IonCatalog catalog) { - _symbols = _system.getSystemSymbolTable(); + // TODO check IVM to determine version: amzn/ion-java#19, amzn/ion-java#24 + _symbols = SharedSymbolTable.getSystemSymbolTable(1); _catalog = catalog; } @@ -214,7 +206,7 @@ private final void has_next_helper_user() throws IOException load_cached_value(AS_TYPE.int_value); int sid = _v.getInt(); if (sid == ION_1_0_SID) { - _symbols = _system.getSystemSymbolTable(); + _symbols = SharedSymbolTable.getSystemSymbolTable(1); push_symbol_table(_symbols); _has_next_needed = true; } @@ -224,13 +216,7 @@ else if (_value_tid == PrivateIonConstants.tidStruct) { int count = load_annotations(); for(int ii=0; ii _iter; protected IonValue _parent; protected PrivateIonValue _next; @@ -69,16 +68,17 @@ public IonReaderTreeSystem(IonValue value) { if (value == null) { // do nothing + _system_symtab = null; _symbolTableAccessor = null; } else { - _system = value.getSystem(); + _system_symtab = value.getSystem().getSystemSymbolTable(); re_init(value, /* hoisted */ false); _symbolTableAccessor = new SymbolTableProvider() { public SymbolTable getSymbolTable() { - return null == _symbols ? _system.getSystemSymbolTable() : _symbols; + return IonReaderTreeSystem.this.getSymbolTable(); } }; } @@ -97,7 +97,6 @@ public T asFacet(Class facetType) void re_init(IonValue value, boolean hoisted) { - _symbols = null; _curr = null; _eof = false; _top = 0; @@ -121,12 +120,6 @@ public void close() _eof = true; } - protected void set_symbol_table(SymbolTable symtab) - { - _symbols = symtab; - return; - } - private void push() { int oldlen = _stack.length; if (_top + 1 >= oldlen) { // we're going to do a "+2" on top so we need extra space @@ -208,16 +201,7 @@ public final int getDepth() { public SymbolTable getSymbolTable() { - SymbolTable symboltable = null; - - if (_curr != null) { - symboltable = _curr.getSymbolTable(); - } - else if (_parent != null) { - symboltable = _parent.getSymbolTable(); - } - - return symboltable; + return _system_symtab; } public IonType getType() diff --git a/src/software/amazon/ion/impl/IonReaderTreeUserX.java b/src/software/amazon/ion/impl/IonReaderTreeUserX.java index 19b642d492..2ffef5b3a3 100644 --- a/src/software/amazon/ion/impl/IonReaderTreeUserX.java +++ b/src/software/amazon/ion/impl/IonReaderTreeUserX.java @@ -17,7 +17,6 @@ import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; import static software.amazon.ion.SystemSymbols.ION_1_0_SID; import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; -import static software.amazon.ion.impl.PrivateUtils.newLocalSymtab; import software.amazon.ion.IonCatalog; import software.amazon.ion.IonDatagram; @@ -35,17 +34,34 @@ final class IonReaderTreeUserX extends IonReaderTreeSystem implements PrivateReaderWriter { + + private final PrivateLocalSymbolTableFactory _lstFactory; + IonCatalog _catalog; + private SymbolTable _symbols; - public IonReaderTreeUserX(IonValue value, IonCatalog catalog) + public IonReaderTreeUserX(IonValue value, IonCatalog catalog, PrivateLocalSymbolTableFactory lstFactory) { - super(value); + super(value); // calls re_init _catalog = catalog; + _lstFactory = lstFactory; } + @Override + void re_init(IonValue value, boolean hoisted) + { + super.re_init(value, hoisted); + _symbols = _system_symtab; + } //======================================================================== + @Override + public SymbolTable getSymbolTable() + { + return _symbols; + } + @Override public IonType next() { @@ -85,14 +101,14 @@ private boolean next_helper_user() if (sid == UNKNOWN_SYMBOL_ID) { String name = sym.stringValue(); if (name != null) { - sid = _system.getSystemSymbolTable().findSymbol(name); + sid = _system_symtab.findSymbol(name); } } if (sid == ION_1_0_SID && _next.getTypeAnnotationSymbols().length == 0) { // $ion_1_0 is read as an IVM only if it is not annotated - SymbolTable symbols = _system.getSystemSymbolTable(); - set_symbol_table(symbols); + SymbolTable symbols = _system_symtab; + _symbols = symbols; push_symbol_table(symbols); _next = null; continue; @@ -103,13 +119,9 @@ else if (IonType.STRUCT.equals(next_type) ) { assert(_next instanceof IonStruct); // read a local symbol table - IonReader reader = new IonReaderTreeUserX(_next, _catalog); - SymbolTable symtab = - newLocalSymtab(_system, - _system.getSystemSymbolTable(), - _system.getCatalog(), - reader, false); - set_symbol_table(symtab); + IonReader reader = new IonReaderTreeUserX(_next, _catalog, _lstFactory); + SymbolTable symtab = _lstFactory.newLocalSymtab(_catalog, reader, false); + _symbols = symtab; push_symbol_table(symtab); _next = null; continue; diff --git a/src/software/amazon/ion/impl/IonWriterSystem.java b/src/software/amazon/ion/impl/IonWriterSystem.java index 8a3f823e34..7550732888 100644 --- a/src/software/amazon/ion/impl/IonWriterSystem.java +++ b/src/software/amazon/ion/impl/IonWriterSystem.java @@ -15,7 +15,6 @@ package software.amazon.ion.impl; import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; -import static software.amazon.ion.impl.PrivateUtils.newLocalSymtab; import static software.amazon.ion.impl.PrivateUtils.newSymbolToken; import static software.amazon.ion.impl.PrivateUtils.newSymbolTokens; @@ -226,7 +225,7 @@ SymbolTable inject_local_symbol_table() throws IOException assert _symbol_table.isSystemTable(); // no catalog since it doesn't matter as this is a // pure local table, with no imports - return newLocalSymtab(null /*system*/, _symbol_table); + return LocalSymbolTable.DEFAULT_LST_FACTORY.newLocalSymtab(_symbol_table); } @Override diff --git a/src/software/amazon/ion/impl/IonWriterSystemTree.java b/src/software/amazon/ion/impl/IonWriterSystemTree.java index a4c9214c85..4223053445 100644 --- a/src/software/amazon/ion/impl/IonWriterSystemTree.java +++ b/src/software/amazon/ion/impl/IonWriterSystemTree.java @@ -14,7 +14,6 @@ package software.amazon.ion.impl; -import static software.amazon.ion.impl.PrivateUtils.newLocalSymtab; import static software.amazon.ion.impl.PrivateUtils.valueIsLocalSymbolTable; import java.io.IOException; @@ -51,12 +50,17 @@ final class IonWriterSystemTree extends IonWriterSystem { + + /** Factory for constructing local symbol tables. Not null. */ + private final LocalSymbolTableAsStruct.Factory _lst_factory; + + private final ValueFactory _factory; /** Used to construct new local symtabs. May be null */ private final IonCatalog _catalog; - private final int _initialDepth; + private final int _initialDepth; private boolean _in_struct; @@ -82,9 +86,8 @@ protected IonWriterSystemTree(SymbolTable defaultSystemSymbolTable, super(defaultSystemSymbolTable, initialIvmHandling, IvmMinimizing.ADJACENT); - if (rootContainer == null) throw new NullPointerException(); - _factory = rootContainer.getSystem(); + _lst_factory = (LocalSymbolTableAsStruct.Factory)((PrivateValueFactory)_factory).getLstFactory(); _catalog = catalog; _current_parent = rootContainer; _in_struct = (_current_parent instanceof IonStruct); @@ -223,9 +226,7 @@ public void stepOut() throws IOException && valueIsLocalSymbolTable(prior)) { // We just finish writing a symbol table! - SymbolTable symbol_table = - newLocalSymtab(_default_system_symbol_table, - _catalog, (IonStruct) prior); + SymbolTable symbol_table = _lst_factory.newLocalSymtab(_catalog, (IonStruct) prior); setSymbolTable(symbol_table); } } @@ -255,7 +256,7 @@ void writeLocalSymtab(SymbolTable symtab) @Override final SymbolTable inject_local_symbol_table() throws IOException { - return newLocalSymtab(_factory, getSymbolTable()); + return _lst_factory.newLocalSymtab(getSymbolTable()); } //======================================================================== diff --git a/src/software/amazon/ion/impl/IonWriterUser.java b/src/software/amazon/ion/impl/IonWriterUser.java index 9d05c2e76e..adf089de9a 100644 --- a/src/software/amazon/ion/impl/IonWriterUser.java +++ b/src/software/amazon/ion/impl/IonWriterUser.java @@ -256,10 +256,9 @@ private void close_local_symbol_table_copy() throws IOException // convert the struct we just wrote with the TreeWriter to a // local symbol table - SymbolTable symtab = - PrivateUtils.newLocalSymtab(activeSystemSymbolTable(), - _catalog, - _symbol_table_value); + LocalSymbolTableAsStruct.Factory lstFactory = + (LocalSymbolTableAsStruct.Factory)((PrivateValueFactory)_symtab_value_factory).getLstFactory(); + SymbolTable symtab = lstFactory.newLocalSymtab(_catalog, _symbol_table_value); _symbol_table_value = null; _current_writer = _system_writer; diff --git a/src/software/amazon/ion/impl/LocalSymbolTable.java b/src/software/amazon/ion/impl/LocalSymbolTable.java index 62311fcbf3..0367184683 100644 --- a/src/software/amazon/ion/impl/LocalSymbolTable.java +++ b/src/software/amazon/ion/impl/LocalSymbolTable.java @@ -18,13 +18,9 @@ import static software.amazon.ion.SystemSymbols.IMPORTS_SID; import static software.amazon.ion.SystemSymbols.ION; import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; -import static software.amazon.ion.SystemSymbols.MAX_ID; import static software.amazon.ion.SystemSymbols.MAX_ID_SID; -import static software.amazon.ion.SystemSymbols.NAME; import static software.amazon.ion.SystemSymbols.NAME_SID; -import static software.amazon.ion.SystemSymbols.SYMBOLS; import static software.amazon.ion.SystemSymbols.SYMBOLS_SID; -import static software.amazon.ion.SystemSymbols.VERSION; import static software.amazon.ion.SystemSymbols.VERSION_SID; import static software.amazon.ion.impl.PrivateUtils.copyOf; import static software.amazon.ion.impl.PrivateUtils.getSidForSymbolTableField; @@ -39,16 +35,12 @@ import java.util.NoSuchElementException; import software.amazon.ion.IonCatalog; import software.amazon.ion.IonException; -import software.amazon.ion.IonList; import software.amazon.ion.IonReader; -import software.amazon.ion.IonStruct; import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; import software.amazon.ion.IonWriter; import software.amazon.ion.ReadOnlyValueException; import software.amazon.ion.SymbolTable; import software.amazon.ion.SymbolToken; -import software.amazon.ion.ValueFactory; import software.amazon.ion.util.IonTextUtils; /** @@ -56,9 +48,44 @@ *

* Instances of this class are safe for use by multiple threads. */ -final class LocalSymbolTable +class LocalSymbolTable implements SymbolTable { + + static class Factory implements PrivateLocalSymbolTableFactory + { + + private Factory(){} // Should be accessed through the singleton + + @Override + public SymbolTable newLocalSymtab(IonCatalog catalog, + IonReader reader, + boolean alreadyInStruct) + { + List symbolsList = new ArrayList(); + LocalSymbolTableImports imports = readLocalSymbolTable(reader, + catalog, + alreadyInStruct, + symbolsList); + return new LocalSymbolTable(imports, symbolsList); + } + + @Override + public SymbolTable newLocalSymtab(SymbolTable defaultSystemSymtab, + SymbolTable... imports) + { + LocalSymbolTableImports unifiedSymtabImports = + new LocalSymbolTableImports(defaultSystemSymtab, imports); + + return new LocalSymbolTable(unifiedSymtabImports, + null /* local symbols */); + } + + } + + static final Factory DEFAULT_LST_FACTORY = new Factory(); + + /** * The initial length of {@link #mySymbolNames}. */ @@ -72,13 +99,6 @@ final class LocalSymbolTable */ private final LocalSymbolTableImports myImportsList; - /** - * The factory used to build the {@link #myImage} of a local symtab. - * It's used by the datagram level to maintain the tree representation. - * It cannot be changed since local symtabs can't be moved between trees. - */ - private final ValueFactory myImageFactory; - /** * Map of symbol names to symbol ids of local symbols that are not in * imports. @@ -90,30 +110,24 @@ final class LocalSymbolTable */ private boolean isReadOnly; - /** - * Memoized result of {@link #getIonRepresentation(ValueFactory)}; - * Once this is created, we maintain it as symbols are added. - */ - private IonStruct myImage; - /** * The local symbol names declared in this symtab; never null. * The sid of the first element is {@link #myFirstLocalSid}. * Only the first {@link #mySymbolsCount} elements are valid. */ - private String[] mySymbolNames; + String[] mySymbolNames; /** * This is the number of symbols defined in this symbol table * locally, that is not imported from some other table. */ - private int mySymbolsCount; + int mySymbolsCount; /** * The sid of the first local symbol, which is stored at * {@link #mySymbolNames}[0]. */ - private final int myFirstLocalSid; + final int myFirstLocalSid; //========================================================================== // Private constructor(s) and static factory methods @@ -134,13 +148,10 @@ private void buildSymbolsMap() /** - * @param imageFactory never null * @param imports never null * @param symbolsList may be null or empty */ - private LocalSymbolTable(ValueFactory imageFactory, - LocalSymbolTableImports imports, - List symbolsList) + protected LocalSymbolTable(LocalSymbolTableImports imports, List symbolsList) { if (symbolsList == null || symbolsList.isEmpty()) { @@ -153,7 +164,6 @@ private LocalSymbolTable(ValueFactory imageFactory, mySymbolNames = symbolsList.toArray(new String[mySymbolsCount]); } - myImageFactory = imageFactory; myImportsList = imports; myFirstLocalSid = myImportsList.getMaxId() + 1; @@ -166,12 +176,10 @@ private LocalSymbolTable(ValueFactory imageFactory, * Copy-constructor, performs defensive copying of member fields where * necessary. The returned instance is mutable. */ - private LocalSymbolTable(LocalSymbolTable other, int maxId) + protected LocalSymbolTable(LocalSymbolTable other, int maxId) { isReadOnly = false; myFirstLocalSid = other.myFirstLocalSid; - myImage = null; - myImageFactory = other.myImageFactory; myImportsList = other.myImportsList; mySymbolsCount = maxId - myImportsList.getMaxId(); @@ -190,96 +198,10 @@ private LocalSymbolTable(LocalSymbolTable other, int maxId) } } - /** - * Constructs a new local symtab with given imports and local symbols. - * - * @param imageFactory - * the factory to use when building a DOM image, may be null - * @param defaultSystemSymtab - * the default system symtab, which will be used if the first - * import in {@code imports} isn't a system symtab, never null - * @param localSymbols - * the list of local symbols; may be null or empty to indicate - * no local symbols - * @param imports - * the set of shared symbol tables to import; the first (and only - * the first) may be a system table, in which case the - * {@code defaultSystemSymtab} is ignored - * - * @throws IllegalArgumentException - * if any import is a local table, or if any but the first is a - * system table - * @throws NullPointerException - * if any import is null - */ - static LocalSymbolTable - makeNewLocalSymbolTable(ValueFactory imageFactory, - SymbolTable defaultSystemSymtab, - List localSymbols, - SymbolTable... imports) - { - LocalSymbolTableImports unifiedSymtabImports = - new LocalSymbolTableImports(defaultSystemSymtab, imports); - - return new LocalSymbolTable(imageFactory, - unifiedSymtabImports, - localSymbols); - } - - /** - * Constructs a new local symbol table represented by the passed in - * {@link IonStruct}. - * - * @param systemSymbolTable - * never null - * @param catalog - * may be null - * @param ionRep - * the struct represented the local symtab - */ - static LocalSymbolTable - makeNewLocalSymbolTable(SymbolTable systemSymbolTable, - IonCatalog catalog, - IonStruct ionRep) - { - ValueFactory imageFactory = ionRep.getSystem(); - IonReader reader = new IonReaderTreeSystem(ionRep); - LocalSymbolTable table = - makeNewLocalSymbolTable(imageFactory, systemSymbolTable, - catalog, reader, false); - - table.myImage = ionRep; - - return table; - } - - /** - * Constructs a new local symbol table represented by the current value of - * the passed in {@link IonReader}. - *

- * NOTE: It is assumed that the passed in reader is positioned - * properly on/before a value that represents a local symtab semantically. - * That is, no exception-checks are made on the {@link IonType} - * and annotation, callers are responsible for checking this! - * - * @param imageFactory - * @param systemSymbolTable - * @param catalog - * the catalog containing shared symtabs referenced by import - * declarations within the local symtab - * @param reader - * the reader positioned on the local symbol table represented as - * a struct - * @param isOnStruct - * denotes whether the reader is already positioned on the struct; - * false if it is positioned before the struct - */ - static LocalSymbolTable - makeNewLocalSymbolTable(ValueFactory imageFactory, - SymbolTable systemSymbolTable, - IonCatalog catalog, - IonReader reader, - boolean isOnStruct) + protected static LocalSymbolTableImports readLocalSymbolTable(IonReader reader, + IonCatalog catalog, + boolean isOnStruct, + List symbolsListOut) { if (! isOnStruct) { @@ -295,9 +217,8 @@ private LocalSymbolTable(LocalSymbolTable other, int maxId) reader.stepIn(); - List symbolsList = new ArrayList(); List importsList = new ArrayList(); - importsList.add(systemSymbolTable); + importsList.add(reader.getSymbolTable().getSystemSymbolTable()); IonType fieldType; boolean foundImportList = false; @@ -346,7 +267,7 @@ private LocalSymbolTable(LocalSymbolTable other, int maxId) text = null; } - symbolsList.add(text); + symbolsListOut.add(text); } reader.stepOut(); } @@ -382,11 +303,7 @@ else if (fieldType == IonType.SYMBOL) reader.stepOut(); - LocalSymbolTableImports imports = - new LocalSymbolTableImports(importsList); - - // We have all necessary data, pass it over to the private constructor - return new LocalSymbolTable(imageFactory, imports, symbolsList); + return new LocalSymbolTableImports(importsList); } synchronized LocalSymbolTable makeCopy() @@ -603,10 +520,8 @@ private static final void validateSymbol(String name) /** * NOT SYNCHRONIZED! Call within constructor or from synch'd method. - * - * @param symbolName must be nonempty. */ - private int putSymbol(String symbolName) + int putSymbol(String symbolName) { if (isReadOnly) { @@ -625,19 +540,17 @@ private int putSymbol(String symbolName) mySymbolNames = temp; } - int sid = mySymbolsCount + myFirstLocalSid; - assert sid == getMaxId() + 1; - - putToMapIfNotThere(mySymbolsMap, symbolName, sid); + int sid = -1; + if (symbolName != null) + { + sid = mySymbolsCount + myFirstLocalSid; + assert sid == getMaxId() + 1; + putToMapIfNotThere(mySymbolsMap, symbolName, sid); + } mySymbolNames[mySymbolsCount] = symbolName; mySymbolsCount++; - if (myImage != null) - { - recordLocalSymbolInIonRep(myImage, symbolName, sid); - } - return sid; } @@ -690,132 +603,6 @@ public void writeTo(IonWriter writer) throws IOException writer.writeValues(reader); } - // - // TODO: there needs to be a better way to associate a System with - // the symbol table, which is required if someone is to be - // able to generate an instance. The other way to resolve - // this dependency would be for the IonSystem object to be - // able to take a SymbolTable and synthesize an Ion - // value from it, by using the public API's to see the useful - // contents. But what about open content? If the origin of - // the symbol table was an IonValue you could get the sys - // from it, and update it, thereby preserving any extra bits. - // If, OTOH, it was synthesized from scratch (a common case) - // then extra content doesn't matter. - // - - /** - * Only valid on local symtabs that already have an _image_factory set. - * - * @param imageFactory is used to check that the image uses the correct - * DOM implementation. - * It must be identical to the {@link #myImageFactory} and not null. - * @return Not null. - */ - IonStruct getIonRepresentation(ValueFactory imageFactory) - { - if (imageFactory == null) - { - throw new IonExceptionNoSystem("can't create representation without a system"); - } - - if (imageFactory != myImageFactory) - { - throw new IonException("wrong system"); - } - - IonStruct image; - synchronized (this) - { - image = myImage; - - if (image == null) - { - // Start a new image from scratch - myImage = image = makeIonRepresentation(myImageFactory); - } - } - - return image; - } - - /** - * NOT SYNCHRONIZED! Call only from a synch'd method. - * - * @return a new struct, not null. - */ - private IonStruct makeIonRepresentation(ValueFactory factory) - { - IonStruct ionRep = factory.newEmptyStruct(); - - ionRep.addTypeAnnotation(ION_SYMBOL_TABLE); - - SymbolTable[] importedTables = getImportedTablesNoCopy(); - - if (importedTables.length > 1) - { - IonList importsList = factory.newEmptyList(); - for (int i = 1; i < importedTables.length; i++) - { - SymbolTable importedTable = importedTables[i]; - IonStruct importStruct = factory.newEmptyStruct(); - - importStruct.add(NAME, - factory.newString(importedTable.getName())); - importStruct.add(VERSION, - factory.newInt(importedTable.getVersion())); - importStruct.add(MAX_ID, - factory.newInt(importedTable.getMaxId())); - - importsList.add(importStruct); - } - ionRep.add(IMPORTS, importsList); - } - - if (mySymbolsCount > 0) - { - int sid = myFirstLocalSid; - for (int offset = 0; offset < mySymbolsCount; offset++, sid++) - { - String symbolName = mySymbolNames[offset]; - recordLocalSymbolInIonRep(ionRep, symbolName, sid); - } - } - - return ionRep; - } - - /** - * NOT SYNCHRONIZED! Call within constructor or from synched method. - * @param symbolName can be null when there's a gap in the local symbols list. - */ - private void recordLocalSymbolInIonRep(IonStruct ionRep, - String symbolName, - int sid) - { - assert sid >= myFirstLocalSid; - - ValueFactory sys = ionRep.getSystem(); - - // TODO this is crazy inefficient and not as reliable as it looks - // since it doesn't handle the case where's theres more than one list - IonValue syms = ionRep.get(SYMBOLS); - while (syms != null && syms.getType() != IonType.LIST) - { - ionRep.remove(syms); - syms = ionRep.get(SYMBOLS); - } - if (syms == null) - { - syms = sys.newEmptyList(); - ionRep.put(SYMBOLS, syms); - } - - int this_offset = sid - myFirstLocalSid; - IonValue name = sys.newString(symbolName); - ((IonList)syms).add(this_offset, name); - } - /** * Collects the necessary imports from the reader and catalog, and load * them into the passed-in {@code importsList}. @@ -976,16 +763,6 @@ public String toString() return "(LocalSymbolTable max_id:" + getMaxId() + ')'; } - static class IonExceptionNoSystem extends IonException - { - private static final long serialVersionUID = 1L; - - IonExceptionNoSystem(String msg) - { - super(msg); - } - } - private static final class SymbolIterator implements Iterator { diff --git a/src/software/amazon/ion/impl/LocalSymbolTableAsStruct.java b/src/software/amazon/ion/impl/LocalSymbolTableAsStruct.java new file mode 100644 index 0000000000..7cc6f3021e --- /dev/null +++ b/src/software/amazon/ion/impl/LocalSymbolTableAsStruct.java @@ -0,0 +1,265 @@ +/* + * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at: + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific + * language governing permissions and limitations under the License. + */ + +package software.amazon.ion.impl; + +import static software.amazon.ion.SystemSymbols.IMPORTS; +import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; +import static software.amazon.ion.SystemSymbols.MAX_ID; +import static software.amazon.ion.SystemSymbols.NAME; +import static software.amazon.ion.SystemSymbols.SYMBOLS; +import static software.amazon.ion.SystemSymbols.VERSION; + +import java.util.ArrayList; +import java.util.List; +import software.amazon.ion.IonCatalog; +import software.amazon.ion.IonList; +import software.amazon.ion.IonReader; +import software.amazon.ion.IonStruct; +import software.amazon.ion.IonType; +import software.amazon.ion.IonValue; +import software.amazon.ion.SymbolTable; +import software.amazon.ion.ValueFactory; + +/** + * A LocalSymbolTable that memoizes its IonStruct representation. + * @deprecated Should not be used in new code without data demonstrating its + * benefits. Instead, use {@link LocalSymbolTable}. + */ +@Deprecated +class LocalSymbolTableAsStruct + extends LocalSymbolTable +{ + + static class Factory implements PrivateLocalSymbolTableFactory + { + + private final ValueFactory imageFactory; + + /** + * @param imageFactory + * the factory to use when building a DOM image, not null + */ + public Factory(ValueFactory imageFactory) + { + this.imageFactory = imageFactory; + } + + @Override + public SymbolTable newLocalSymtab(IonCatalog catalog, + IonReader reader, + boolean alreadyInStruct) + { + List symbolsList = new ArrayList(); + LocalSymbolTableImports imports = readLocalSymbolTable(reader, + catalog, + alreadyInStruct, + symbolsList); + return new LocalSymbolTableAsStruct(imageFactory, imports, symbolsList); + } + + @Override + public SymbolTable newLocalSymtab(SymbolTable defaultSystemSymtab, + SymbolTable... imports) + { + LocalSymbolTableImports unifiedSymtabImports = + new LocalSymbolTableImports(defaultSystemSymtab, imports); + + return new LocalSymbolTableAsStruct(imageFactory, + unifiedSymtabImports, + null /* local symbols */); + } + + /** + * Constructs a new local symbol table represented by the passed in + * {@link IonStruct}. + * + * @param catalog + * may be null + * @param ionRep + * the struct represented the local symtab + */ + // TODO this should die with the 'backed' DOM + public SymbolTable newLocalSymtab(IonCatalog catalog, + IonStruct ionRep) + { + assert imageFactory == ionRep.getSystem(); + IonReader reader = new IonReaderTreeSystem(ionRep); + + List symbolsList = new ArrayList(); + LocalSymbolTableImports imports = readLocalSymbolTable(reader, + catalog, + false, + symbolsList); + + LocalSymbolTableAsStruct table = new LocalSymbolTableAsStruct(imageFactory, + imports, + symbolsList); + table.myImage = ionRep; + + return table; + } + + } + + /** + * The factory used to build the {@link #myImage} of a local symtab. + * It's used by the datagram level to maintain the tree representation. + * It cannot be changed since local symtabs can't be moved between trees. + */ + private final ValueFactory myImageFactory; + + /** + * Memoized result of {@link #getIonRepresentation()}; + * Once this is created, we maintain it as symbols are added. + */ + private IonStruct myImage; + + /** + * @param imageFactory never null + * @param imports never null + * @param symbolsList may be null or empty + */ + private LocalSymbolTableAsStruct(ValueFactory imageFactory, + LocalSymbolTableImports imports, + List symbolsList) + { + super(imports, symbolsList); + myImageFactory = imageFactory; + } + + @Override + int putSymbol(String symbolName) + { + int sid = super.putSymbol(symbolName); + if (myImage != null) + { + recordLocalSymbolInIonRep(myImage, symbolName, sid); + } + return sid; + } + + // + // TODO: there needs to be a better way to associate a System with + // the symbol table, which is required if someone is to be + // able to generate an instance. The other way to resolve + // this dependency would be for the IonSystem object to be + // able to take a SymbolTable and synthesize an Ion + // value from it, by using the public API's to see the useful + // contents. But what about open content? If the origin of + // the symbol table was an IonValue you could get the sys + // from it, and update it, thereby preserving any extra bits. + // If, OTOH, it was synthesized from scratch (a common case) + // then extra content doesn't matter. + // + + /** + * Only valid on local symtabs that already have an _image_factory set. + * + * @return Not null. + */ + IonStruct getIonRepresentation() + { + synchronized (this) + { + IonStruct image = myImage; + + if (image == null) + { + // Start a new image from scratch + myImage = image = makeIonRepresentation(myImageFactory); + } + + return image; + } + } + + /** + * NOT SYNCHRONIZED! Call only from a synch'd method. + * + * @return a new struct, not null. + */ + private IonStruct makeIonRepresentation(ValueFactory factory) + { + IonStruct ionRep = factory.newEmptyStruct(); + + ionRep.addTypeAnnotation(ION_SYMBOL_TABLE); + + SymbolTable[] importedTables = getImportedTablesNoCopy(); + + if (importedTables.length > 1) + { + IonList importsList = factory.newEmptyList(); + for (int i = 1; i < importedTables.length; i++) + { + SymbolTable importedTable = importedTables[i]; + IonStruct importStruct = factory.newEmptyStruct(); + + importStruct.add(NAME, + factory.newString(importedTable.getName())); + importStruct.add(VERSION, + factory.newInt(importedTable.getVersion())); + importStruct.add(MAX_ID, + factory.newInt(importedTable.getMaxId())); + + importsList.add(importStruct); + } + ionRep.add(IMPORTS, importsList); + } + + if (mySymbolsCount > 0) + { + int sid = myFirstLocalSid; + for (int offset = 0; offset < mySymbolsCount; offset++, sid++) + { + String symbolName = mySymbolNames[offset]; + recordLocalSymbolInIonRep(ionRep, symbolName, sid); + } + } + + return ionRep; + } + + /** + * NOT SYNCHRONIZED! Call within constructor or from synched method. + * @param symbolName can be null when there's a gap in the local symbols list. + */ + private void recordLocalSymbolInIonRep(IonStruct ionRep, + String symbolName, + int sid) + { + assert sid >= myFirstLocalSid; + + ValueFactory sys = ionRep.getSystem(); + + // TODO this is crazy inefficient and not as reliable as it looks + // since it doesn't handle the case where's theres more than one list + IonValue syms = ionRep.get(SYMBOLS); + while (syms != null && syms.getType() != IonType.LIST) + { + ionRep.remove(syms); + syms = ionRep.get(SYMBOLS); + } + if (syms == null) + { + syms = sys.newEmptyList(); + ionRep.put(SYMBOLS, syms); + } + + int this_offset = sid - myFirstLocalSid; + IonValue name = sys.newString(symbolName); + ((IonList)syms).add(this_offset, name); + } + +} diff --git a/src/software/amazon/ion/impl/PrivateIonReaderFactory.java b/src/software/amazon/ion/impl/PrivateIonReaderFactory.java index 7f672a0d15..28f7cc4c31 100644 --- a/src/software/amazon/ion/impl/PrivateIonReaderFactory.java +++ b/src/software/amazon/ion/impl/PrivateIonReaderFactory.java @@ -36,21 +36,26 @@ @Deprecated public final class PrivateIonReaderFactory { - public static final IonReader makeReader(IonSystem system, - IonCatalog catalog, + public static final IonReader makeReader(IonCatalog catalog, byte[] bytes) { - return makeReader(system, catalog, bytes, 0, bytes.length); + return makeReader(catalog, bytes, 0, bytes.length); } - public static IonReader makeSystemReader(IonSystem system, byte[] bytes) + public static final IonReader makeReader(IonCatalog catalog, + byte[] bytes, + PrivateLocalSymbolTableFactory lstFactory) + { + return makeReader(catalog, bytes, 0, bytes.length, lstFactory); + } + + public static IonReader makeSystemReader(byte[] bytes) { - return makeSystemReader(system, bytes, 0, bytes.length); + return makeSystemReader(bytes, 0, bytes.length); } - public static final IonReader makeReader(IonSystem system, - IonCatalog catalog, + public static final IonReader makeReader(IonCatalog catalog, byte[] bytes, int offset, int length) @@ -58,7 +63,7 @@ public static final IonReader makeReader(IonSystem system, try { UnifiedInputStreamX uis = makeUnifiedStream(bytes, offset, length); - return makeReader(system, catalog, uis, offset); + return makeReader(catalog, uis, offset, LocalSymbolTable.DEFAULT_LST_FACTORY); } catch (IOException e) { @@ -66,15 +71,31 @@ public static final IonReader makeReader(IonSystem system, } } - public static IonReader makeSystemReader(IonSystem system, + public static final IonReader makeReader(IonCatalog catalog, byte[] bytes, + int offset, + int length, + PrivateLocalSymbolTableFactory lstFactory) + { + try + { + UnifiedInputStreamX uis = makeUnifiedStream(bytes, offset, length); + return makeReader(catalog, uis, offset, lstFactory); + } + catch (IOException e) + { + throw new IonException(e); + } + } + + public static IonReader makeSystemReader(byte[] bytes, int offset, int length) { try { UnifiedInputStreamX uis = makeUnifiedStream(bytes, offset, length); - return makeSystemReader(system, uis, offset); + return makeSystemReader(uis, offset); } catch (IOException e) { @@ -83,122 +104,129 @@ public static IonReader makeSystemReader(IonSystem system, } - public static final IonReader makeReader(IonSystem system, - IonCatalog catalog, - char[] chars) + public static final IonReader makeReader(IonCatalog catalog, + char[] chars) { - return makeReader(system, catalog, chars, 0, chars.length); + return makeReader(catalog, chars, 0, chars.length); } - public static final IonReader makeSystemReader(IonSystem system, - char[] chars) + public static final IonReader makeSystemReader(char[] chars) { UnifiedInputStreamX in = makeStream(chars); - return new IonReaderTextSystemX(system, in); + return new IonReaderTextSystemX(in); } - public static final IonReader makeReader(IonSystem system, - IonCatalog catalog, + public static final IonReader makeReader(IonCatalog catalog, char[] chars, int offset, int length) { UnifiedInputStreamX in = makeStream(chars, offset, length); - return new IonReaderTextUserX(system, catalog, in, offset); + return new IonReaderTextUserX(catalog, LocalSymbolTable.DEFAULT_LST_FACTORY, in, offset); } - public static final IonReader makeSystemReader(IonSystem system, - char[] chars, + public static final IonReader makeSystemReader(char[] chars, int offset, int length) { UnifiedInputStreamX in = makeStream(chars, offset, length); - return new IonReaderTextSystemX(system, in); + return new IonReaderTextSystemX(in); } - public static final IonReader makeReader(IonSystem system, - IonCatalog catalog, - CharSequence chars) + public static final IonReader makeReader(IonCatalog catalog, + CharSequence chars) + { + return makeReader(catalog, chars, LocalSymbolTable.DEFAULT_LST_FACTORY); + } + + public static final IonReader makeReader(IonCatalog catalog, + CharSequence chars, + PrivateLocalSymbolTableFactory lstFactory) { UnifiedInputStreamX in = makeStream(chars); - return new IonReaderTextUserX(system, catalog, in); + return new IonReaderTextUserX(catalog, lstFactory, in); } - public static final IonReader makeSystemReader(IonSystem system, - CharSequence chars) + public static final IonReader makeSystemReader(CharSequence chars) { UnifiedInputStreamX in = makeStream(chars); - return new IonReaderTextSystemX(system, in); + return new IonReaderTextSystemX(in); } - public static final IonReader makeReader(IonSystem system, - IonCatalog catalog, + public static final IonReader makeReader(IonCatalog catalog, CharSequence chars, int offset, int length) { UnifiedInputStreamX in = makeStream(chars, offset, length); - return new IonReaderTextUserX(system, catalog, in, offset); + return new IonReaderTextUserX(catalog, LocalSymbolTable.DEFAULT_LST_FACTORY, in, offset); } - public static final IonReader makeSystemReader(IonSystem system, - CharSequence chars, + public static final IonReader makeSystemReader(CharSequence chars, int offset, int length) { UnifiedInputStreamX in = makeStream(chars, offset, length); - return new IonReaderTextSystemX(system, in); + return new IonReaderTextSystemX(in); } - - public static final IonReader makeReader(IonSystem system, - IonCatalog catalog, + public static final IonReader makeReader(IonCatalog catalog, InputStream is) + { + return makeReader(catalog, is, LocalSymbolTable.DEFAULT_LST_FACTORY); + } + + public static final IonReader makeReader(IonCatalog catalog, + InputStream is, + PrivateLocalSymbolTableFactory lstFactory) { try { UnifiedInputStreamX uis = makeUnifiedStream(is); - return makeReader(system, catalog, uis, 0); + return makeReader(catalog, uis, 0, lstFactory); } catch (IOException e) { throw new IonException(e); } } - public static IonReader makeSystemReader(IonSystem system, - InputStream is) + public static IonReader makeSystemReader(InputStream is) { try { UnifiedInputStreamX uis = makeUnifiedStream(is); - return makeSystemReader(system, uis, 0); + return makeSystemReader(uis, 0); } catch (IOException e) { throw new IonException(e); } } - - public static final IonReader makeReader(IonSystem system, - IonCatalog catalog, + public static final IonReader makeReader(IonCatalog catalog, Reader chars) + { + return makeReader(catalog, chars, LocalSymbolTable.DEFAULT_LST_FACTORY); + } + + public static final IonReader makeReader(IonCatalog catalog, + Reader chars, + PrivateLocalSymbolTableFactory lstFactory) { try { UnifiedInputStreamX in = makeStream(chars); - return new IonReaderTextUserX(system, catalog, in); + return new IonReaderTextUserX(catalog, lstFactory, in); } catch (IOException e) { throw new IonException(e); } } - public static final IonReader makeSystemReader(IonSystem system, - Reader chars) + public static final IonReader makeSystemReader(Reader chars) { try { UnifiedInputStreamX in = makeStream(chars); - return new IonReaderTextSystemX(system, in); + return new IonReaderTextSystemX(in); } catch (IOException e) { throw new IonException(e); @@ -206,11 +234,10 @@ public static final IonReader makeSystemReader(IonSystem system, } - public static final IonReader makeReader(IonSystem system, - IonCatalog catalog, + public static final IonReader makeReader(IonCatalog catalog, IonValue value) { - return new IonReaderTreeUserX(value, catalog); + return new IonReaderTreeUserX(value, catalog, LocalSymbolTable.DEFAULT_LST_FACTORY); } public static final IonReader makeSystemReader(IonSystem system, @@ -227,35 +254,34 @@ public static final IonReader makeSystemReader(IonSystem system, - private static IonReader makeReader(IonSystem system, - IonCatalog catalog, + private static IonReader makeReader(IonCatalog catalog, UnifiedInputStreamX uis, - int offset) + int offset, + PrivateLocalSymbolTableFactory lstFactory) throws IOException { IonReader r; if (has_binary_cookie(uis)) { - r = new IonReaderBinaryUserX(system, catalog, uis, offset); + r = new IonReaderBinaryUserX(catalog, lstFactory, uis, offset); } else { - r = new IonReaderTextUserX(system, catalog, uis, offset); + r = new IonReaderTextUserX(catalog, lstFactory, uis, offset); } return r; } - private static IonReader makeSystemReader(IonSystem system, - UnifiedInputStreamX uis, + private static IonReader makeSystemReader(UnifiedInputStreamX uis, int offset) throws IOException { IonReader r; if (has_binary_cookie(uis)) { // TODO pass offset, or spans will be incorrect - r = new IonReaderBinarySystemX(system, uis); + r = new IonReaderBinarySystemX(uis); } else { // TODO pass offset, or spans will be incorrect - r = new IonReaderTextSystemX(system, uis); + r = new IonReaderTextSystemX(uis); } return r; } @@ -284,7 +310,6 @@ private static UnifiedInputStreamX makeUnifiedStream(byte[] bytes, return uis; } - private static UnifiedInputStreamX makeUnifiedStream(InputStream in) throws IOException { @@ -296,7 +321,6 @@ private static UnifiedInputStreamX makeUnifiedStream(InputStream in) return uis; } - private static final boolean has_binary_cookie(UnifiedInputStreamX uis) throws IOException { diff --git a/src/software/amazon/ion/impl/PrivateIonTextWriterBuilder.java b/src/software/amazon/ion/impl/PrivateIonTextWriterBuilder.java index a47693f3d9..fb3245e420 100644 --- a/src/software/amazon/ion/impl/PrivateIonTextWriterBuilder.java +++ b/src/software/amazon/ion/impl/PrivateIonTextWriterBuilder.java @@ -191,7 +191,7 @@ private IonWriter build(PrivateFastAppendable appender) appender); SymbolTable initialSymtab = - initialSymtab(system, defaultSystemSymtab, imports); + initialSymtab(((PrivateValueFactory)system).getLstFactory(), defaultSystemSymtab, imports); return new IonWriterUser(catalog, system, systemWriter, initialSymtab); } diff --git a/src/software/amazon/ion/impl/PrivateLocalSymbolTableFactory.java b/src/software/amazon/ion/impl/PrivateLocalSymbolTableFactory.java new file mode 100644 index 0000000000..2e1121f699 --- /dev/null +++ b/src/software/amazon/ion/impl/PrivateLocalSymbolTableFactory.java @@ -0,0 +1,80 @@ +/* + * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at: + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific + * language governing permissions and limitations under the License. + */ + +package software.amazon.ion.impl; + +import software.amazon.ion.IonCatalog; +import software.amazon.ion.IonReader; +import software.amazon.ion.IonType; +import software.amazon.ion.SymbolTable; + +/** + * NOT FOR APPLICATION USE + * + * Implementations of this interface may be provided to IonReaders in order + * to force them to construct LocalSymbolTables in a different way. + * + * In practice, this is used to construct a different LocalSymbolTable + * implementation for use with the DOM than is used purely by readers + * and writers. + * + * If {@link LocalSymbolTableAsStruct} is ever deleted, this can go away + * too. + */ +@SuppressWarnings("javadoc") +public interface PrivateLocalSymbolTableFactory +{ + /** + * Constructs a new local symbol table represented by the current value of + * the passed in {@link IonReader}. + *

+ * NOTE: It is assumed that the passed in reader is positioned + * properly on/before a value that represents a local symtab semantically. + * That is, no exception-checks are made on the {@link IonType} + * and annotation, callers are responsible for checking this! + * + * @param catalog + * the catalog containing shared symtabs referenced by import + * declarations within the local symtab + * @param reader + * the reader positioned on the local symbol table represented as + * a struct + * @param alreadyInStruct + * denotes whether the reader is already positioned on the struct; + * false if it is positioned before the struct + */ + public SymbolTable newLocalSymtab(IonCatalog catalog, + IonReader reader, + boolean alreadyInStruct); + + /** + * Constructs a new local symtab with given imports and local symbols. + * + * @param defaultSystemSymtab + * the default system symtab, which will be used if the first + * import in {@code imports} isn't a system symtab, never null + * @param imports + * the set of shared symbol tables to import; the first (and only + * the first) may be a system table, in which case the + * {@code defaultSystemSymtab} is ignored + * + * @throws IllegalArgumentException + * if any import is a local table, or if any but the first is a + * system table + * @throws NullPointerException + * if any import is null + */ + public SymbolTable newLocalSymtab(SymbolTable defaultSystemSymtab, + SymbolTable... imports); +} diff --git a/src/software/amazon/ion/impl/PrivateUtils.java b/src/software/amazon/ion/impl/PrivateUtils.java index 2ecabe4cef..ed8ce655a3 100644 --- a/src/software/amazon/ion/impl/PrivateUtils.java +++ b/src/software/amazon/ion/impl/PrivateUtils.java @@ -42,11 +42,9 @@ import java.util.Arrays; import java.util.Collection; import java.util.Iterator; -import java.util.List; import java.util.ListIterator; import java.util.NoSuchElementException; import java.util.TimeZone; -import software.amazon.ion.IonCatalog; import software.amazon.ion.IonException; import software.amazon.ion.IonReader; import software.amazon.ion.IonStruct; @@ -739,53 +737,6 @@ public static SymbolTable newSharedSymtab(String name, symbols); } - - public static SymbolTable newLocalSymtab(ValueFactory imageFactory, - SymbolTable systemSymtab, - List localSymbols, - SymbolTable... imports) - { - return LocalSymbolTable.makeNewLocalSymbolTable(imageFactory, - systemSymtab, - localSymbols, - imports); - } - - - public static SymbolTable newLocalSymtab(ValueFactory imageFactory, - SymbolTable systemSymtab, - SymbolTable... imports) - { - return LocalSymbolTable.makeNewLocalSymbolTable(imageFactory, - systemSymtab, - null /*localSymbols*/, - imports); - } - - - public static SymbolTable newLocalSymtab(SymbolTable systemSymbtab, - IonCatalog catalog, - IonStruct ionRep) - { - return LocalSymbolTable.makeNewLocalSymbolTable(systemSymbtab, - catalog, - ionRep); - } - - - public static SymbolTable newLocalSymtab(ValueFactory imageFactory, - SymbolTable systemSymbolTable, - IonCatalog catalog, - IonReader reader, - boolean alreadyInStruct) - { - return LocalSymbolTable.makeNewLocalSymbolTable(imageFactory, - systemSymbolTable, - catalog, - reader, - alreadyInStruct); - } - public static SymbolTable newSubstituteSymtab(SymbolTable original, int version, int maxId) @@ -845,14 +796,27 @@ public static SymbolTable copyLocalSymbolTable(SymbolTable symtab) return ((LocalSymbolTable) symtab).makeCopy(); } + /** + * Trampoline to {@link LocalSymbolTableAsStruct.Factory#Factory(ValueFactory)} + * @param imageFactory + * the ValueFactory from which to construct the IonStruct representation of the LST + * @return a new {@link LocalSymbolTableAsStruct.Factory} + * @deprecated due to DOM entanglement. Streaming applications should use + * {@value LocalSymbolTable#DEFAULT_LST_FACTORY}. + */ + @Deprecated + public static PrivateLocalSymbolTableFactory newLocalSymbolTableAsStructFactory(ValueFactory imageFactory) + { + return new LocalSymbolTableAsStruct.Factory(imageFactory); + } /** * Returns a minimal symtab, either system or local depending on the - * given values. If the imports are empty, the default system symtab is - * returned. + * given values, that supports representation as an IonStruct. If the + * imports are empty, the default system symtab is returned. * - * @param imageFactory - * the factory to use when building a DOM image, may be null + * @param lstFactory + * the factory to use to build the local symbol table, never null * @param defaultSystemSymtab * the default system symtab, which will be used if the first * import in {@code imports} isn't a system symtab, never null @@ -861,7 +825,7 @@ public static SymbolTable copyLocalSymbolTable(SymbolTable symtab) * The first (and only the first) may be a system table, in which case the * {@code defaultSystemSymtab} is ignored. */ - public static SymbolTable initialSymtab(ValueFactory imageFactory, + public static SymbolTable initialSymtab(PrivateLocalSymbolTableFactory lstFactory, SymbolTable defaultSystemSymtab, SymbolTable... imports) { @@ -875,20 +839,17 @@ public static SymbolTable initialSymtab(ValueFactory imageFactory, return imports[0]; } - return LocalSymbolTable.makeNewLocalSymbolTable(imageFactory, - defaultSystemSymtab, - null, /*localSymbols*/ - imports); + return lstFactory.newLocalSymtab(defaultSystemSymtab, imports); } /** * Trampoline to - * {@link LocalSymbolTable#getIonRepresentation(ValueFactory)}; + * {@link LocalSymbolTableAsStruct#getIonRepresentation()}; */ - public static IonStruct symtabTree(ValueFactory vf, SymbolTable symtab) + public static IonStruct symtabTree(SymbolTable symtab) { - return ((LocalSymbolTable)symtab).getIonRepresentation(vf); + return ((LocalSymbolTableAsStruct)symtab).getIonRepresentation(); } /** diff --git a/src/software/amazon/ion/impl/PrivateValueFactory.java b/src/software/amazon/ion/impl/PrivateValueFactory.java new file mode 100644 index 0000000000..e4e2517ee5 --- /dev/null +++ b/src/software/amazon/ion/impl/PrivateValueFactory.java @@ -0,0 +1,32 @@ +/* + * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at: + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific + * language governing permissions and limitations under the License. + */ + +package software.amazon.ion.impl; + +import software.amazon.ion.IonStruct; +import software.amazon.ion.ValueFactory; + +public interface PrivateValueFactory extends ValueFactory +{ + /** + * Gets the {@link LocalSymbolTableAsStruct.Factory} associated with this + * {@link ValueFactory}. This is used to construct local symbol tables + * backed by {@link IonStruct}s. Note that this should not be used in new + * code; use {@link LocalSymbolTable} instead. + * + * @return a LocalSymbolTableAsStruct.Factory; never null. + */ + @SuppressWarnings("javadoc") + public PrivateLocalSymbolTableFactory getLstFactory(); +} diff --git a/src/software/amazon/ion/impl/lite/IonDatagramLite.java b/src/software/amazon/ion/impl/lite/IonDatagramLite.java index ad8cd0c20f..d8158b97a9 100644 --- a/src/software/amazon/ion/impl/lite/IonDatagramLite.java +++ b/src/software/amazon/ion/impl/lite/IonDatagramLite.java @@ -1010,8 +1010,7 @@ void load_current_symbol_table(IonValueLite prev_user_value) rep = __iterator.get_datagram_ivm(); } else { - IonSystem sys = __iterator.get_datagram_system(); - rep = PrivateUtils.symtabTree(sys, new_symbol_table); + rep = PrivateUtils.symtabTree(new_symbol_table); } assert(rep != null && __iterator.get_datagram_system() == rep.getSystem()); @@ -1118,7 +1117,7 @@ private static int count_system_values(IonSystem sys, int count = 0; while (curr.isLocalTable()) { count++; - curr = PrivateUtils.symtabTree(sys, curr).getSymbolTable(); + curr = PrivateUtils.symtabTree(curr).getSymbolTable(); } // we should terminate when the symbol tables symbol table is the system symbol table assert(curr != null); diff --git a/src/software/amazon/ion/impl/lite/IonLoaderLite.java b/src/software/amazon/ion/impl/lite/IonLoaderLite.java index 5d771c3b91..1c50234e65 100644 --- a/src/software/amazon/ion/impl/lite/IonLoaderLite.java +++ b/src/software/amazon/ion/impl/lite/IonLoaderLite.java @@ -29,6 +29,7 @@ import software.amazon.ion.IonSystem; import software.amazon.ion.IonWriter; import software.amazon.ion.impl.PrivateIonWriterFactory; +import software.amazon.ion.impl.PrivateLocalSymbolTableFactory; final class IonLoaderLite implements IonLoader @@ -38,6 +39,8 @@ final class IonLoaderLite /** Not null. */ private final IonCatalog _catalog; + private final PrivateLocalSymbolTableFactory _lstFactory; + /** * @param system must not be null. * @param catalog must not be null. @@ -49,6 +52,7 @@ public IonLoaderLite(IonSystemLite system, IonCatalog catalog) _system = system; _catalog = catalog; + _lstFactory = system.getLstFactory(); } public IonSystem getSystem() @@ -93,7 +97,7 @@ public IonDatagram load(File ionFile) throws IonException, IOException public IonDatagram load(String ionText) throws IonException { try { - IonReader reader = makeReader(_system, _catalog, ionText); + IonReader reader = makeReader(_catalog, ionText, _lstFactory); IonDatagramLite datagram = load_helper(reader); return datagram; } @@ -105,7 +109,7 @@ public IonDatagram load(String ionText) throws IonException public IonDatagram load(Reader ionText) throws IonException, IOException { try { - IonReader reader = makeReader(_system, _catalog, ionText); + IonReader reader = makeReader(_catalog, ionText, _lstFactory); IonDatagramLite datagram = load_helper(reader); return datagram; } @@ -119,7 +123,7 @@ public IonDatagram load(Reader ionText) throws IonException, IOException public IonDatagram load(byte[] ionData) throws IonException { try { - IonReader reader = makeReader(_system, _catalog, ionData, 0, ionData.length); + IonReader reader = makeReader(_catalog, ionData, 0, ionData.length, _lstFactory); IonDatagramLite datagram = load_helper(reader); return datagram; } @@ -132,7 +136,7 @@ public IonDatagram load(InputStream ionData) throws IonException, IOException { try { - IonReader reader = makeReader(_system, _catalog, ionData); + IonReader reader = makeReader(_catalog, ionData, _lstFactory); IonDatagramLite datagram = load_helper(reader); return datagram; } diff --git a/src/software/amazon/ion/impl/lite/IonSystemLite.java b/src/software/amazon/ion/impl/lite/IonSystemLite.java index 33d813ff6a..d7e94b36eb 100644 --- a/src/software/amazon/ion/impl/lite/IonSystemLite.java +++ b/src/software/amazon/ion/impl/lite/IonSystemLite.java @@ -52,6 +52,7 @@ import software.amazon.ion.impl.PrivateIonSystem; import software.amazon.ion.impl.PrivateIonWriterFactory; import software.amazon.ion.impl.PrivateUtils; +import software.amazon.ion.system.IonReaderBuilder; import software.amazon.ion.system.IonTextWriterBuilder; final class IonSystemLite @@ -63,19 +64,22 @@ final class IonSystemLite /** Not null. */ private final IonCatalog _catalog; - private ValueFactoryLite _value_factory; private final IonLoader _loader; /** Immutable. */ private final IonTextWriterBuilder myTextWriterBuilder; /** Immutable. */ private final PrivateIonBinaryWriterBuilder myBinaryWriterBuilder; + /** Immutable. **/ + private final IonReaderBuilder myReaderBuilder; public IonSystemLite(IonTextWriterBuilder twb, - PrivateIonBinaryWriterBuilder bwb) + PrivateIonBinaryWriterBuilder bwb, + IonReaderBuilder rb) { IonCatalog catalog = twb.getCatalog(); assert catalog != null; assert catalog == bwb.getCatalog(); + assert catalog == rb.getCatalog(); _catalog = catalog; _loader = new IonLoaderLite(this, catalog); @@ -84,12 +88,12 @@ public IonSystemLite(IonTextWriterBuilder twb, myTextWriterBuilder = twb.immutable(); - // whacked but I'm not going to figure this out right now - _value_factory = this; - _value_factory.set_system(this); + set_system(this); - bwb.setSymtabValueFactory(_value_factory); + bwb.setSymtabValueFactory(this); myBinaryWriterBuilder = bwb.immutable(); + + myReaderBuilder = rb.immutable(); } //========================================================================== @@ -168,28 +172,28 @@ public SymbolTable getSystemSymbolTable(String ionVersionId) public Iterator iterate(Reader ionText) { - IonReader reader = makeReader(this, _catalog, ionText); + IonReader reader = makeReader(_catalog, ionText, _lstFactory); ReaderIterator iterator = new ReaderIterator(this, reader); return iterator; } public Iterator iterate(InputStream ionData) { - IonReader reader = newReader(ionData); + IonReader reader = makeReader(_catalog, ionData, _lstFactory); ReaderIterator iterator = new ReaderIterator(this, reader); return iterator; } public Iterator iterate(String ionText) { - IonReader reader = makeReader(this, _catalog, ionText); + IonReader reader = makeReader(_catalog, ionText, _lstFactory); ReaderIterator iterator = new ReaderIterator(this, reader); return iterator; } public Iterator iterate(byte[] ionData) { - IonReader reader = makeReader(this, _catalog, ionData); + IonReader reader = makeReader(_catalog, ionData, _lstFactory); ReaderIterator iterator = new ReaderIterator(this, reader); return iterator; } @@ -224,10 +228,7 @@ public IonWriter newTextWriter(OutputStream out, SymbolTable... imports) public SymbolTable newLocalSymbolTable(SymbolTable... imports) { - return PrivateUtils.newLocalSymtab(this, - getSystemSymbolTable(), - null /* localSymbols */, - imports); + return _lstFactory.newLocalSymtab(getSystemSymbolTable(), imports); } public SymbolTable newSharedSymbolTable(IonStruct ionRep) @@ -646,8 +647,7 @@ public IonDatagram newDatagram(SymbolTable... imports) public IonDatagram newDatagram(IonCatalog catalog, SymbolTable... imports) { SymbolTable defaultSystemSymtab = getSystemSymbolTable(); - SymbolTable symbols = - initialSymtab(this, defaultSystemSymtab, imports); + SymbolTable symbols = initialSymtab(_lstFactory, defaultSystemSymtab, imports); IonDatagramLite dg = newDatagram(catalog); dg.appendTrailingSymbolTable(symbols); return dg; @@ -655,55 +655,55 @@ public IonDatagram newDatagram(IonCatalog catalog, SymbolTable... imports) public IonReader newReader(byte[] ionData) { - return makeReader(this, _catalog, ionData); + return myReaderBuilder.build(ionData); } public IonReader newSystemReader(byte[] ionData) { - return makeSystemReader(this, ionData); + return makeSystemReader(ionData); } public IonReader newReader(byte[] ionData, int offset, int len) { - return makeReader(this, _catalog, ionData, offset, len); + return myReaderBuilder.build(ionData, offset, len); } public IonReader newSystemReader(byte[] ionData, int offset, int len) { - return makeSystemReader(this, ionData, offset, len); + return makeSystemReader(ionData, offset, len); } public IonReader newReader(String ionText) { - return makeReader(this, _catalog, ionText); + return myReaderBuilder.build(ionText); } public IonReader newSystemReader(String ionText) { - return makeSystemReader(this, ionText); + return makeSystemReader(ionText); } public IonReader newReader(InputStream ionData) { - return makeReader(this, _catalog, ionData); + return myReaderBuilder.build(ionData); } public IonReader newSystemReader(InputStream ionData) { - return makeSystemReader(this, ionData); + return makeSystemReader(ionData); } public IonReader newReader(Reader ionText) { - return makeReader(this, _catalog, ionText); + return myReaderBuilder.build(ionText); } public IonReader newReader(IonValue value) { - return makeReader(this, _catalog, value); + return myReaderBuilder.build(value); } //========================================================================== @@ -712,7 +712,7 @@ public IonReader newReader(IonValue value) public IonReader newSystemReader(Reader ionText) { - return makeSystemReader(this, ionText); + return makeSystemReader(ionText); } public IonReader newSystemReader(IonValue value) diff --git a/src/software/amazon/ion/impl/lite/PrivateLiteDomTrampoline.java b/src/software/amazon/ion/impl/lite/PrivateLiteDomTrampoline.java index dcc8bde54c..eeeddab03b 100644 --- a/src/software/amazon/ion/impl/lite/PrivateLiteDomTrampoline.java +++ b/src/software/amazon/ion/impl/lite/PrivateLiteDomTrampoline.java @@ -17,6 +17,7 @@ import software.amazon.ion.IonSystem; import software.amazon.ion.SymbolTable; import software.amazon.ion.impl.PrivateIonBinaryWriterBuilder; +import software.amazon.ion.system.IonReaderBuilder; import software.amazon.ion.system.IonTextWriterBuilder; /** @@ -27,10 +28,12 @@ @Deprecated public final class PrivateLiteDomTrampoline { + public static IonSystem newLiteSystem(IonTextWriterBuilder twb, - PrivateIonBinaryWriterBuilder bwb) + PrivateIonBinaryWriterBuilder bwb, + IonReaderBuilder rb) { - return new IonSystemLite(twb, bwb); + return new IonSystemLite(twb, bwb, rb); } public static boolean isLiteSystem(IonSystem system) diff --git a/src/software/amazon/ion/impl/lite/ValueFactoryLite.java b/src/software/amazon/ion/impl/lite/ValueFactoryLite.java index 83140bec13..a0609d0888 100644 --- a/src/software/amazon/ion/impl/lite/ValueFactoryLite.java +++ b/src/software/amazon/ion/impl/lite/ValueFactoryLite.java @@ -28,7 +28,9 @@ import software.amazon.ion.IonValue; import software.amazon.ion.SymbolToken; import software.amazon.ion.Timestamp; -import software.amazon.ion.ValueFactory; +import software.amazon.ion.impl.PrivateLocalSymbolTableFactory; +import software.amazon.ion.impl.PrivateUtils; +import software.amazon.ion.impl.PrivateValueFactory; /** * This class handles all of the IonValueLite @@ -36,10 +38,16 @@ * */ abstract class ValueFactoryLite - implements ValueFactory + implements PrivateValueFactory { + protected final PrivateLocalSymbolTableFactory _lstFactory; private ContainerlessContext _context; + ValueFactoryLite() + { + _lstFactory = PrivateUtils.newLocalSymbolTableAsStructFactory(this); + } + protected void set_system(IonSystemLite system) { _context = ContainerlessContext.wrap(system); } @@ -436,4 +444,10 @@ private ArrayList newInts(long[] elements) return e; } + + @Override + public PrivateLocalSymbolTableFactory getLstFactory() + { + return _lstFactory; + } } diff --git a/src/software/amazon/ion/system/IonReaderBuilder.java b/src/software/amazon/ion/system/IonReaderBuilder.java new file mode 100644 index 0000000000..7366ba52d9 --- /dev/null +++ b/src/software/amazon/ion/system/IonReaderBuilder.java @@ -0,0 +1,307 @@ +/* + * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at: + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific + * language governing permissions and limitations under the License. + */ + +package software.amazon.ion.system; + +import static software.amazon.ion.impl.PrivateIonReaderFactory.makeReader; + +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import software.amazon.ion.IonCatalog; +import software.amazon.ion.IonException; +import software.amazon.ion.IonReader; +import software.amazon.ion.IonStruct; +import software.amazon.ion.IonSystem; +import software.amazon.ion.IonValue; + +/** + * Build a new {@link IonReader} from the given {@link IonCatalog} and data + * source. A data source is required, while an IonCatalog is optional. If no + * IonCatalog is provided, an empty {@link SimpleCatalog} will be used. + *

+ * {@link IonReader}s parse incrementally, so syntax errors in the input data + * will not be detected as side effects of any of the {@code build} methods + * in this class. + */ +@SuppressWarnings("deprecation") +public class IonReaderBuilder +{ + + private IonCatalog catalog = null; + + private IonReaderBuilder() + { + } + + private IonReaderBuilder(IonReaderBuilder that) + { + this.catalog = that.catalog; + } + + /** + * The standard builder of {@link IonReader}s, with all configuration + * properties having their default values. + * + * @return a new, mutable builder instance. + */ + public static IonReaderBuilder standard() + { + return new Mutable(); + } + + /** + * Creates a mutable copy of this builder. + * + * @return a new builder with the same configuration as {@code this}. + */ + public IonReaderBuilder copy() + { + return new Mutable(this); + } + + /** + * Returns an immutable builder configured exactly like this one. + * + * @return this builder instance, if immutable; + * otherwise an immutable copy of this builder. + */ + public IonReaderBuilder immutable() + { + return this; + } + + /** + * Returns a mutable builder configured exactly like this one. + * + * @return this instance, if mutable; + * otherwise a mutable copy of this instance. + */ + public IonReaderBuilder mutable() + { + return copy(); + } + + /** NOT FOR APPLICATION USE! */ + protected void mutationCheck() + { + throw new UnsupportedOperationException("This builder is immutable"); + } + + /** + * Declares the catalog to use when building an {@link IonReader}, + * returning a new mutable builder the current one is immutable. + * + * @param catalog the catalog to use in built readers. + * If null, a new {@link SimpleCatalog} will be used. + * + * @return this builder instance, if mutable; + * otherwise a mutable copy of this builder. + * + * @see #setCatalog(IonCatalog) + * @see #withCatalog(IonCatalog) + */ + public IonReaderBuilder withCatalog(IonCatalog catalog) + { + IonReaderBuilder b = mutable(); + b.setCatalog(catalog); + return b; + } + + /** + * Sets the catalog to use when building an {@link IonReader}. + * + * @param catalog the catalog to use in built readers. + * If null, a new {@link SimpleCatalog} will be used. + * + * @see #getCatalog() + * @see #withCatalog(IonCatalog) + * + * @throws UnsupportedOperationException if this builder is immutable. + */ + public void setCatalog(IonCatalog catalog) + { + mutationCheck(); + this.catalog = catalog; + } + + /** + * Gets the catalog to use when building an {@link IonReader}, or null + * if none has been manually set. The catalog is needed to resolve shared + * symbol table imports. + * + * @see #setCatalog(IonCatalog) + * @see #withCatalog(IonCatalog) + */ + public IonCatalog getCatalog() + { + return catalog; + } + + private IonCatalog validateCatalog() + { + // matches behavior in IonSystemBuilder when no catalog provided + return catalog != null ? catalog : new SimpleCatalog(); + } + + /** + * Based on the builder's configuration properties, creates a new IonReader + * instance over the given block of Ion data, detecting whether it's text or + * binary data. + *

+ * This method will auto-detect and uncompress GZIPped Ion data. + * + * @param ionData the source of the Ion data, which may be either Ion binary + * data or UTF-8 Ion text. The reader retains a reference to the array, so + * its data must not be modified while the reader is active. Must not be + * null. + * + * @return a new {@link IonReader} instance; not {@code null}. + * + * @see IonSystem#newReader(byte[]) + */ + public IonReader build(byte[] ionData) + { + return makeReader(validateCatalog(), ionData); + } + + /** + * Based on the builder's configuration properties, creates a new IonReader + * instance over the given block of Ion data, detecting whether it's text or + * binary data. + *

+ * This method will auto-detect and uncompress GZIPped Ion data. + * + * @param ionData the source of the Ion data, which is used only within the + * range of bytes starting at {@code offset} for {@code len} bytes. + * The data in that range may be either Ion binary data or UTF-8 Ion text. + * The reader retains a reference to the array, so its data must not be + * modified while the reader is active. Must not be null. + * @param offset must be non-negative and less than {@code ionData.length}. + * @param length must be non-negative and {@code offset+length} must not + * exceed {@code ionData.length}. + * + * @see IonSystem#newReader(byte[], int, int) + */ + public IonReader build(byte[] ionData, int offset, int length) + { + return makeReader(validateCatalog(), ionData, offset, length); + } + + /** + * Based on the builder's configuration properties, creates a new IonReader + * instance over the given stream of Ion data, detecting whether it's text or + * binary data. + *

+ * This method will auto-detect and uncompress GZIPped Ion data. + *

+ * Because this library performs its own buffering, it's recommended that + * users avoid adding additional buffering to the given stream. + * + * @param ionData the source of the Ion data, which may be either Ion binary + * data or UTF-8 Ion text. Must not be null. + * + * @return a new reader instance. + * Callers must call {@link IonReader#close()} when finished with it. + * + * @throws IonException if the source throws {@link IOException}. + * + * @see IonSystem#newReader(InputStream) + */ + public IonReader build(InputStream ionData) + { + return makeReader(validateCatalog(), ionData); + } + + /** + * Based on the builder's configuration properties, creates a new + * {@link IonReader} instance over Ion text data. + *

+ * Applications should generally use {@link #build(InputStream)} + * whenever possible, since this library has much faster Unicode decoding + * than the Java IO framework. + *

+ * Because this library performs its own buffering, it's recommended that + * you avoid adding additional buffering to the given stream. + * + * @param ionText the source of the Ion text data. Must not be null. + * + * @throws IonException if the source throws {@link IOException}. + * + * @see IonSystem#newReader(Reader) + */ + public IonReader build(Reader ionText) + { + return makeReader(validateCatalog(), ionText); + } + + /** + * Based on the builder's configuration properties, creates a new + * {@link IonReader} instance over an {@link IonValue} data model. Typically + * this is used to iterate over a collection, such as an {@link IonStruct}. + * + * @param value must not be null. + * + * @see IonSystem#newReader(IonValue) + */ + public IonReader build(IonValue value) + { + return makeReader(validateCatalog(), value); + } + + /** + * Based on the builder's configuration properties, creates an new + * {@link IonReader} instance over Ion text data. + * + * @param ionText the source of the Ion text data. Must not be null. + * + * @see IonSystem#newReader(String) + */ + public IonReader build(String ionText) + { + return makeReader(validateCatalog(), ionText); + } + + private static class Mutable extends IonReaderBuilder + { + + private Mutable() + { + } + + private Mutable(IonReaderBuilder that) + { + super(that); + } + + @Override + public IonReaderBuilder immutable() + { + return new IonReaderBuilder(this); + } + + @Override + public IonReaderBuilder mutable() + { + return this; + } + + @Override + protected void mutationCheck() + { + } + + } + +} diff --git a/src/software/amazon/ion/system/IonSystemBuilder.java b/src/software/amazon/ion/system/IonSystemBuilder.java index 2e813dfc18..982f251d1f 100644 --- a/src/software/amazon/ion/system/IonSystemBuilder.java +++ b/src/software/amazon/ion/system/IonSystemBuilder.java @@ -286,7 +286,8 @@ public final IonSystem build() // This is what we need, more or less. // bwb = bwb.fillDefaults(); - return newLiteSystem(twb, bwb); + IonReaderBuilder rb = IonReaderBuilder.standard().withCatalog(catalog); + return newLiteSystem(twb, bwb, rb); } //========================================================================= diff --git a/test/AllTests.java b/test/AllTests.java index 392e8bcd5f..b5b44ef293 100644 --- a/test/AllTests.java +++ b/test/AllTests.java @@ -83,6 +83,7 @@ import software.amazon.ion.streaming.RoundTripStreamingTest; import software.amazon.ion.streaming.SpanTests; import software.amazon.ion.system.IonBinaryWriterBuilderTest; +import software.amazon.ion.system.IonReaderBuilderTest; import software.amazon.ion.system.IonSystemBuilderTest; import software.amazon.ion.system.IonTextWriterBuilderTest; import software.amazon.ion.system.SimpleCatalogTest; @@ -186,6 +187,7 @@ IonBinaryWriterBuilderTest.class, IonReaderToIonValueTest.class, BinaryReaderWrappedValueLengthTest.class, + IonReaderBuilderTest.class, // experimental binary writer tests PooledBlockAllocatorProviderTest.class, diff --git a/test/software/amazon/ion/DatagramTest.java b/test/software/amazon/ion/DatagramTest.java index ae6ec5442c..6491d6fd97 100644 --- a/test/software/amazon/ion/DatagramTest.java +++ b/test/software/amazon/ion/DatagramTest.java @@ -15,11 +15,11 @@ package software.amazon.ion; import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; -import static software.amazon.ion.Symtabs.FRED_MAX_IDS; import static software.amazon.ion.SystemSymbols.ION_1_0; import static software.amazon.ion.SystemSymbols.ION_1_0_SID; import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; import static software.amazon.ion.SystemSymbols.SYMBOLS; +import static software.amazon.ion.impl.Symtabs.FRED_MAX_IDS; import static software.amazon.ion.junit.IonAssert.assertIonEquals; import java.io.ByteArrayOutputStream; @@ -45,6 +45,7 @@ import software.amazon.ion.SymbolTable; import software.amazon.ion.impl.PrivateIonSystem; import software.amazon.ion.impl.PrivateIonValue; +import software.amazon.ion.impl.Symtabs; public class DatagramTest diff --git a/test/software/amazon/ion/LoaderTest.java b/test/software/amazon/ion/LoaderTest.java index f0f92f0014..18e9a76c89 100644 --- a/test/software/amazon/ion/LoaderTest.java +++ b/test/software/amazon/ion/LoaderTest.java @@ -42,6 +42,7 @@ import software.amazon.ion.IonSymbol; import software.amazon.ion.IonSystem; import software.amazon.ion.IonValue; +import software.amazon.ion.impl.Symtabs; import software.amazon.ion.system.SimpleCatalog; public class LoaderTest diff --git a/test/software/amazon/ion/SharedSymtabMaker.java b/test/software/amazon/ion/SharedSymtabMaker.java index 2101e2c27d..554854f6bb 100644 --- a/test/software/amazon/ion/SharedSymtabMaker.java +++ b/test/software/amazon/ion/SharedSymtabMaker.java @@ -15,7 +15,7 @@ package software.amazon.ion; import static junit.framework.Assert.assertSame; -import static software.amazon.ion.Symtabs.sharedSymtabStruct; +import static software.amazon.ion.impl.Symtabs.sharedSymtabStruct; import software.amazon.ion.IonReader; import software.amazon.ion.IonStruct; diff --git a/test/software/amazon/ion/SystemProcessingTestCase.java b/test/software/amazon/ion/SystemProcessingTestCase.java index 1fc491e57a..98fcc99f94 100644 --- a/test/software/amazon/ion/SystemProcessingTestCase.java +++ b/test/software/amazon/ion/SystemProcessingTestCase.java @@ -15,12 +15,12 @@ package software.amazon.ion; import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; -import static software.amazon.ion.Symtabs.LocalSymbolTablePrefix; import static software.amazon.ion.SystemSymbols.ION_1_0; import static software.amazon.ion.SystemSymbols.ION_1_0_SID; import static software.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE; import static software.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE_SID; import static software.amazon.ion.TestUtils.FERMATA; +import static software.amazon.ion.impl.Symtabs.LocalSymbolTablePrefix; import org.junit.Ignore; import org.junit.Test; @@ -29,6 +29,7 @@ import software.amazon.ion.SymbolTable; import software.amazon.ion.Timestamp; import software.amazon.ion.impl.SymbolTableTest; +import software.amazon.ion.impl.Symtabs; import software.amazon.ion.system.SimpleCatalog; diff --git a/test/software/amazon/ion/impl/BinaryWriterTest.java b/test/software/amazon/ion/impl/BinaryWriterTest.java index 10dd9cc7f9..b7f51f8c1b 100644 --- a/test/software/amazon/ion/impl/BinaryWriterTest.java +++ b/test/software/amazon/ion/impl/BinaryWriterTest.java @@ -26,7 +26,6 @@ import software.amazon.ion.IonWriter; import software.amazon.ion.SymbolTable; import software.amazon.ion.SymbolToken; -import software.amazon.ion.Symtabs; import software.amazon.ion.junit.IonAssert; public class BinaryWriterTest diff --git a/test/software/amazon/ion/impl/BinaryWriterWithLocalSymtabsTest.java b/test/software/amazon/ion/impl/BinaryWriterWithLocalSymtabsTest.java index 0188c11d71..6997522fcb 100644 --- a/test/software/amazon/ion/impl/BinaryWriterWithLocalSymtabsTest.java +++ b/test/software/amazon/ion/impl/BinaryWriterWithLocalSymtabsTest.java @@ -14,10 +14,10 @@ package software.amazon.ion.impl; -import static software.amazon.ion.Symtabs.FRED_MAX_IDS; -import static software.amazon.ion.Symtabs.LOCAL_SYMBOLS_ABC; -import static software.amazon.ion.Symtabs.makeLocalSymtab; import static software.amazon.ion.impl.PrivateUtils.EMPTY_STRING_ARRAY; +import static software.amazon.ion.impl.Symtabs.FRED_MAX_IDS; +import static software.amazon.ion.impl.Symtabs.LOCAL_SYMBOLS_ABC; +import static software.amazon.ion.impl.Symtabs.makeLocalSymtab; import java.io.ByteArrayOutputStream; import org.junit.After; @@ -27,7 +27,6 @@ import software.amazon.ion.IonValue; import software.amazon.ion.IonWriter; import software.amazon.ion.SymbolTable; -import software.amazon.ion.Symtabs; import software.amazon.ion.system.IonBinaryWriterBuilder; public class BinaryWriterWithLocalSymtabsTest diff --git a/test/software/amazon/ion/impl/IonWriterTestCase.java b/test/software/amazon/ion/impl/IonWriterTestCase.java index 0bf6626c0b..2c7e435489 100644 --- a/test/software/amazon/ion/impl/IonWriterTestCase.java +++ b/test/software/amazon/ion/impl/IonWriterTestCase.java @@ -14,13 +14,13 @@ package software.amazon.ion.impl; -import static software.amazon.ion.Symtabs.FRED_MAX_IDS; import static software.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE; import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; import static software.amazon.ion.SystemSymbols.NAME_SID; import static software.amazon.ion.TestUtils.FERMATA; import static software.amazon.ion.impl.PrivateIonWriterBase.ERROR_MISSING_FIELD_NAME; import static software.amazon.ion.impl.PrivateUtils.newSymbolToken; +import static software.amazon.ion.impl.Symtabs.FRED_MAX_IDS; import static software.amazon.ion.junit.IonAssert.assertIonEquals; import static software.amazon.ion.junit.IonAssert.expectNextField; @@ -54,7 +54,6 @@ import software.amazon.ion.IonWriter; import software.amazon.ion.SymbolTable; import software.amazon.ion.SymbolToken; -import software.amazon.ion.Symtabs; import software.amazon.ion.SystemSymbols; import software.amazon.ion.TestUtils; import software.amazon.ion.impl.PrivateIonWriter; diff --git a/test/software/amazon/ion/impl/LocalSymbolTableImportAdapterTest.java b/test/software/amazon/ion/impl/LocalSymbolTableImportAdapterTest.java index 25cd944d45..b2dd6fce4f 100644 --- a/test/software/amazon/ion/impl/LocalSymbolTableImportAdapterTest.java +++ b/test/software/amazon/ion/impl/LocalSymbolTableImportAdapterTest.java @@ -1,15 +1,15 @@ package software.amazon.ion.impl; +import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; + import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.util.Arrays; import org.junit.Assert; import org.junit.Test; import software.amazon.ion.IonDatagram; import software.amazon.ion.IonWriter; import software.amazon.ion.ReadOnlyValueException; import software.amazon.ion.SymbolTable; -import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; import software.amazon.ion.SymbolToken; public class LocalSymbolTableImportAdapterTest extends BaseSymbolTableWrapperTest @@ -251,12 +251,16 @@ private LocalSymbolTableBuilder(PrivateIonSystem system) public LocalSymbolTable build() { - return LocalSymbolTable.makeNewLocalSymbolTable( - system, + LocalSymbolTable lst = (LocalSymbolTable) LocalSymbolTable.DEFAULT_LST_FACTORY.newLocalSymtab( system.getSystemSymbolTable(), - Arrays.asList(symbols), importedTables ); + + for(String symbol : symbols) { + lst.intern(symbol); + } + + return lst; } public LocalSymbolTableBuilder setSymbols(final String... symbols) diff --git a/test/software/amazon/ion/impl/LocalSymbolTableTest.java b/test/software/amazon/ion/impl/LocalSymbolTableTest.java index 14e2fe49af..a680f9055c 100644 --- a/test/software/amazon/ion/impl/LocalSymbolTableTest.java +++ b/test/software/amazon/ion/impl/LocalSymbolTableTest.java @@ -14,11 +14,11 @@ package software.amazon.ion.impl; -import static software.amazon.ion.Symtabs.FRED_MAX_IDS; -import static software.amazon.ion.Symtabs.LOCAL_SYMBOLS_ABC; -import static software.amazon.ion.Symtabs.makeLocalSymtab; import static software.amazon.ion.impl.PrivateUtils.EMPTY_STRING_ARRAY; import static software.amazon.ion.impl.PrivateUtils.copyLocalSymbolTable; +import static software.amazon.ion.impl.Symtabs.FRED_MAX_IDS; +import static software.amazon.ion.impl.Symtabs.LOCAL_SYMBOLS_ABC; +import static software.amazon.ion.impl.Symtabs.makeLocalSymtab; import org.junit.Test; import software.amazon.ion.IonException; @@ -26,7 +26,6 @@ import software.amazon.ion.SubstituteSymbolTableException; import software.amazon.ion.SymbolTable; import software.amazon.ion.SymbolToken; -import software.amazon.ion.Symtabs; import software.amazon.ion.impl.LocalSymbolTable; import software.amazon.ion.impl.SubstituteSymbolTable; diff --git a/test/software/amazon/ion/impl/OptimizedBinaryWriterSymbolTableTest.java b/test/software/amazon/ion/impl/OptimizedBinaryWriterSymbolTableTest.java index 4f533910f4..cbd0532431 100644 --- a/test/software/amazon/ion/impl/OptimizedBinaryWriterSymbolTableTest.java +++ b/test/software/amazon/ion/impl/OptimizedBinaryWriterSymbolTableTest.java @@ -14,9 +14,9 @@ package software.amazon.ion.impl; -import static software.amazon.ion.Symtabs.printLocalSymtab; import static software.amazon.ion.impl.PrivateUtils.isNonSymbolScalar; import static software.amazon.ion.impl.PrivateUtils.symtabExtends; +import static software.amazon.ion.impl.Symtabs.printLocalSymtab; import static software.amazon.ion.junit.IonAssert.assertIonEquals; import static software.amazon.ion.junit.IonAssert.assertIonIteratorEquals; @@ -24,7 +24,6 @@ import software.amazon.ion.IonDatagram; import software.amazon.ion.IonType; import software.amazon.ion.SymbolTable; -import software.amazon.ion.Symtabs; import software.amazon.ion.impl.PrivateUtils; /** diff --git a/test/software/amazon/ion/impl/OptimizedBinaryWriterTestCase.java b/test/software/amazon/ion/impl/OptimizedBinaryWriterTestCase.java index 0c909cea5b..2941a201c2 100644 --- a/test/software/amazon/ion/impl/OptimizedBinaryWriterTestCase.java +++ b/test/software/amazon/ion/impl/OptimizedBinaryWriterTestCase.java @@ -15,8 +15,8 @@ package software.amazon.ion.impl; import static java.lang.reflect.Proxy.newProxyInstance; -import static software.amazon.ion.Symtabs.makeLocalSymtab; import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; +import static software.amazon.ion.impl.Symtabs.makeLocalSymtab; import java.io.ByteArrayOutputStream; import java.io.IOException; diff --git a/test/software/amazon/ion/impl/OutputStreamWriterTestCase.java b/test/software/amazon/ion/impl/OutputStreamWriterTestCase.java index f3d9451b8e..7749a64518 100644 --- a/test/software/amazon/ion/impl/OutputStreamWriterTestCase.java +++ b/test/software/amazon/ion/impl/OutputStreamWriterTestCase.java @@ -26,7 +26,6 @@ import software.amazon.ion.IonValue; import software.amazon.ion.IonWriter; import software.amazon.ion.SymbolTable; -import software.amazon.ion.Symtabs; import software.amazon.ion.SystemSymbols; import software.amazon.ion.Timestamp; diff --git a/test/software/amazon/ion/impl/SharedSymbolTableTest.java b/test/software/amazon/ion/impl/SharedSymbolTableTest.java index 25554ee059..07d9eb5fd0 100644 --- a/test/software/amazon/ion/impl/SharedSymbolTableTest.java +++ b/test/software/amazon/ion/impl/SharedSymbolTableTest.java @@ -14,10 +14,10 @@ package software.amazon.ion.impl; -import static software.amazon.ion.Symtabs.sharedSymtabStruct; import static software.amazon.ion.impl.PrivateUtils.EMPTY_STRING_ARRAY; import static software.amazon.ion.impl.PrivateUtils.stringIterator; import static software.amazon.ion.impl.SymbolTableTest.checkSharedTable; +import static software.amazon.ion.impl.Symtabs.sharedSymtabStruct; import java.io.IOException; import org.junit.Test; @@ -30,7 +30,6 @@ import software.amazon.ion.SharedSymtabMaker; import software.amazon.ion.SymbolTable; import software.amazon.ion.SymbolToken; -import software.amazon.ion.Symtabs; import software.amazon.ion.SystemSymbols; import software.amazon.ion.junit.Injected.Inject; diff --git a/test/software/amazon/ion/impl/SymbolTableTest.java b/test/software/amazon/ion/impl/SymbolTableTest.java index 66651f4612..32e08bdd8c 100644 --- a/test/software/amazon/ion/impl/SymbolTableTest.java +++ b/test/software/amazon/ion/impl/SymbolTableTest.java @@ -14,11 +14,7 @@ package software.amazon.ion.impl; -import org.junit.Assert; -import static org.junit.Assert.assertEquals; import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; -import software.amazon.ion.SymbolToken; -import static software.amazon.ion.Symtabs.printLocalSymtab; import static software.amazon.ion.SystemSymbols.ION; import static software.amazon.ion.SystemSymbols.ION_1_0; import static software.amazon.ion.SystemSymbols.ION_1_0_SID; @@ -30,6 +26,7 @@ import static software.amazon.ion.impl.PrivateUtils.EMPTY_STRING_ARRAY; import static software.amazon.ion.impl.PrivateUtils.stringIterator; import static software.amazon.ion.impl.PrivateUtils.symtabTree; +import static software.amazon.ion.impl.Symtabs.printLocalSymtab; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -56,7 +53,7 @@ import software.amazon.ion.IonWriter; import software.amazon.ion.ReadOnlyValueException; import software.amazon.ion.SymbolTable; -import software.amazon.ion.Symtabs; +import software.amazon.ion.SymbolToken; import software.amazon.ion.SystemSymbols; import software.amazon.ion.Timestamp; import software.amazon.ion.system.SimpleCatalog; @@ -939,11 +936,12 @@ public void testBasicLocalSymtabCreation() @Test public void testSymtabImageMaintenance() { - SymbolTable st = system().newLocalSymbolTable(); + IonSystem system = system(); + SymbolTable st = ((PrivateValueFactory)system).getLstFactory().newLocalSymtab(system.getSystemSymbolTable()); st.intern("foo"); - IonStruct image = symtabTree(system(), st); + IonStruct image = symtabTree(st); st.intern("bar"); - image = symtabTree(system(), st); + image = symtabTree(st); IonList symbols = (IonList) image.get(SYMBOLS); assertEquals("[\"foo\",\"bar\"]", symbols.toString()); } @@ -1214,9 +1212,6 @@ SymbolTable checkFirstImport(String name, int version, IonStruct stStruct = writeIonRep(st); checkFirstImport(name, version, expectedSymbols, stStruct); - stStruct = PrivateUtils.symtabTree(system(), st); - checkFirstImport(name, version, expectedSymbols, stStruct); - SymbolTable importedTable = st.getImportedTables()[0]; checkSharedTable(name, version, expectedSymbols, importedTable); diff --git a/test/software/amazon/ion/Symtabs.java b/test/software/amazon/ion/impl/Symtabs.java similarity index 90% rename from test/software/amazon/ion/Symtabs.java rename to test/software/amazon/ion/impl/Symtabs.java index fb265cdd18..c1c0ebf084 100644 --- a/test/software/amazon/ion/Symtabs.java +++ b/test/software/amazon/ion/impl/Symtabs.java @@ -12,14 +12,13 @@ * language governing permissions and limitations under the License. */ -package software.amazon.ion; +package software.amazon.ion.impl; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static software.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE; import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; -import static software.amazon.ion.impl.PrivateUtils.newLocalSymtab; import static software.amazon.ion.util.IonTextUtils.printString; import java.io.IOException; @@ -31,6 +30,7 @@ import software.amazon.ion.SystemSymbols; import software.amazon.ion.ValueFactory; import software.amazon.ion.impl.PrivateIonSystem; +import software.amazon.ion.impl.PrivateLocalSymbolTableFactory; import software.amazon.ion.system.IonSystemBuilder; import software.amazon.ion.system.SimpleCatalog; @@ -203,24 +203,6 @@ public static String printLocalSymtab(String... symbols) return s.toString(); } - - /** - * Creates a local symtab with local symbols but no imports. - * - * @param system - * @param localSymbols - * the array (var-args) of local symbols that the resulting - * local symtab to contain; may be {@code null} to indicate no - * local symbols; elements may be {@code null} to indicate a gap - */ - public static SymbolTable makeLocalSymtab(IonSystem system, - String... localSymbols) - { - return newLocalSymtab(system, system.getSystemSymbolTable(), - Arrays.asList(localSymbols)); - } - - /** * Creates a local symtab with local symbols and imports. */ @@ -232,9 +214,27 @@ public static SymbolTable makeLocalSymtab(IonSystem system, for (String localSymbol : localSymbols) { - localSymtab.intern(localSymbol); + if (localSymbol == null) + { + // This injects a gap. + ((LocalSymbolTable)localSymtab).putSymbol(null); + } + else + { + localSymtab.intern(localSymbol); + } } return localSymtab; } + + /** + * Trampoline to {@link LocalSymbolTable#DEFAULT_LST_FACTORY} + * @return the {@link LocalSymbolTable.Factory} singleton. + */ + public static PrivateLocalSymbolTableFactory localSymbolTableFactory() + { + return LocalSymbolTable.DEFAULT_LST_FACTORY; + } + } diff --git a/test/software/amazon/ion/impl/TextWriterTest.java b/test/software/amazon/ion/impl/TextWriterTest.java index fc58162c30..536e47e534 100644 --- a/test/software/amazon/ion/impl/TextWriterTest.java +++ b/test/software/amazon/ion/impl/TextWriterTest.java @@ -29,7 +29,6 @@ import software.amazon.ion.IonStruct; import software.amazon.ion.IonWriter; import software.amazon.ion.SymbolTable; -import software.amazon.ion.Symtabs; import software.amazon.ion.SystemSymbols; import software.amazon.ion.impl.PrivateUtils; import software.amazon.ion.system.IonTextWriterBuilder; diff --git a/test/software/amazon/ion/impl/ValueWriterTest.java b/test/software/amazon/ion/impl/ValueWriterTest.java index 05ac70e623..c8141fb9c1 100644 --- a/test/software/amazon/ion/impl/ValueWriterTest.java +++ b/test/software/amazon/ion/impl/ValueWriterTest.java @@ -23,7 +23,6 @@ import software.amazon.ion.IonSymbol; import software.amazon.ion.IonWriter; import software.amazon.ion.SymbolTable; -import software.amazon.ion.Symtabs; public class ValueWriterTest extends IonWriterTestCase diff --git a/test/software/amazon/ion/impl/lite/IonContextTest.java b/test/software/amazon/ion/impl/lite/IonContextTest.java index 63ef347d52..823d507706 100644 --- a/test/software/amazon/ion/impl/lite/IonContextTest.java +++ b/test/software/amazon/ion/impl/lite/IonContextTest.java @@ -15,7 +15,7 @@ package software.amazon.ion.impl.lite; import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; -import static software.amazon.ion.Symtabs.CATALOG; +import static software.amazon.ion.impl.Symtabs.CATALOG; import org.junit.Test; import software.amazon.ion.IonDatagram; @@ -27,7 +27,7 @@ import software.amazon.ion.IonValue; import software.amazon.ion.SymbolTable; import software.amazon.ion.SymbolToken; -import software.amazon.ion.Symtabs; +import software.amazon.ion.impl.Symtabs; import software.amazon.ion.impl.lite.ContainerlessContext; import software.amazon.ion.impl.lite.IonContainerLite; import software.amazon.ion.impl.lite.IonContext; diff --git a/test/software/amazon/ion/streaming/ReaderTest.java b/test/software/amazon/ion/streaming/ReaderTest.java index 2733a3d0d7..1a5eb9517b 100644 --- a/test/software/amazon/ion/streaming/ReaderTest.java +++ b/test/software/amazon/ion/streaming/ReaderTest.java @@ -14,7 +14,7 @@ package software.amazon.ion.streaming; -import static software.amazon.ion.Symtabs.printLocalSymtab; +import static software.amazon.ion.impl.Symtabs.printLocalSymtab; import static software.amazon.ion.junit.IonAssert.checkNullSymbol; import java.io.IOException; @@ -27,9 +27,10 @@ import software.amazon.ion.Decimal; import software.amazon.ion.IonType; import software.amazon.ion.ReaderMaker; +import software.amazon.ion.SymbolTable; import software.amazon.ion.SymbolToken; -import software.amazon.ion.junit.IonAssert; import software.amazon.ion.junit.Injected.Inject; +import software.amazon.ion.junit.IonAssert; public class ReaderTest extends ReaderTestCase @@ -367,4 +368,15 @@ public void testSkippingLobInStruct() testSkippingLob("{a:1, b:{ c:", " }}"); testSkippingLob("{a:1, b:{ c:", "}}"); } + + @Test + public void testGetSymbolTableBeforeFirstValue() + { + read("123"); + SymbolTable symtab = in.getSymbolTable(); + expectNoCurrentValue(); + assertNotNull(symtab); + assertTrue(symtab.isSystemTable()); + } + } diff --git a/test/software/amazon/ion/system/IonBinaryWriterBuilderTest.java b/test/software/amazon/ion/system/IonBinaryWriterBuilderTest.java index 248a23b91b..bee7917c68 100644 --- a/test/software/amazon/ion/system/IonBinaryWriterBuilderTest.java +++ b/test/software/amazon/ion/system/IonBinaryWriterBuilderTest.java @@ -21,25 +21,20 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static software.amazon.ion.TestUtils.symbolTableEquals; -import static software.amazon.ion.impl.PrivateUtils.newLocalSymtab; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.util.Collections; import org.junit.Assert; import org.junit.Test; import software.amazon.ion.IonCatalog; import software.amazon.ion.IonSystem; import software.amazon.ion.IonWriter; import software.amazon.ion.SymbolTable; -import software.amazon.ion.Symtabs; import software.amazon.ion.impl.PrivateIonBinaryWriterBuilder; import software.amazon.ion.impl.PrivateIonWriter; import software.amazon.ion.impl.PrivateUtils; -import software.amazon.ion.system.IonBinaryWriterBuilder; -import software.amazon.ion.system.IonSystemBuilder; -import software.amazon.ion.system.SimpleCatalog; +import software.amazon.ion.impl.Symtabs; public class IonBinaryWriterBuilderTest { @@ -240,11 +235,9 @@ public void testSymtabValueFactoryImmutability() public void testInitialSymtab() throws IOException { - IonSystem system = IonSystemBuilder.standard().build(); SymbolTable sst = PrivateUtils.systemSymtab(1); - SymbolTable lst0 = newLocalSymtab(system, sst, - Collections.emptyList()); + SymbolTable lst0 = Symtabs.localSymbolTableFactory().newLocalSymtab(sst); lst0.intern("hello"); PrivateIonBinaryWriterBuilder b = @@ -284,12 +277,10 @@ public void testInitialSymtab() @Test public void testImmutableInitialSymtab() { - IonSystem system = IonSystemBuilder.standard().build(); SymbolTable sst = PrivateUtils.systemSymtab(1); // Immutable local symtabs shouldn't get copied. - SymbolTable lst = newLocalSymtab(system, sst, - Collections.emptyList()); + SymbolTable lst = Symtabs.localSymbolTableFactory().newLocalSymtab(sst); lst.intern("hello"); lst.makeReadOnly(); diff --git a/test/software/amazon/ion/system/IonReaderBuilderTest.java b/test/software/amazon/ion/system/IonReaderBuilderTest.java new file mode 100644 index 0000000000..06b938396e --- /dev/null +++ b/test/software/amazon/ion/system/IonReaderBuilderTest.java @@ -0,0 +1,113 @@ +/* + * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at: + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific + * language governing permissions and limitations under the License. + */ + +package software.amazon.ion.system; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import software.amazon.ion.IonCatalog; +import software.amazon.ion.IonReader; +import software.amazon.ion.IonType; +import software.amazon.ion.IonWriter; +import software.amazon.ion.impl.PrivateIonBinaryWriterBuilder; + +/** + * Note: because the IonReaderBuilder is used by IonSystem.newReader(...), + * its build() methods are well-exercised elsewhere. See: ReaderMaker. + */ +public class IonReaderBuilderTest +{ + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test + public void testMutable() + { + IonCatalog catalog = new SimpleCatalog(); + IonReaderBuilder mutable = IonReaderBuilder.standard().withCatalog(catalog); + assertSame(catalog, mutable.getCatalog()); + IonReaderBuilder mutableSame = mutable.withCatalog(new SimpleCatalog()); + assertNotSame(catalog, mutable.getCatalog()); + assertSame(mutable, mutableSame); + } + + @Test + public void testImmutable() + { + IonCatalog catalog = new SimpleCatalog(); + IonReaderBuilder mutable = IonReaderBuilder.standard().withCatalog(catalog); + IonReaderBuilder immutable = mutable.immutable(); + mutable.withCatalog(new SimpleCatalog()); + assertNotSame(catalog, mutable.getCatalog()); + assertSame(catalog, immutable.getCatalog()); + } + + @Test + public void testMutatingImmutableFails() + { + IonReaderBuilder immutable = IonReaderBuilder.standard().immutable(); + thrown.expect(UnsupportedOperationException.class); + immutable.setCatalog(new SimpleCatalog()); + } + + @Test + public void testMutateCopiedImmutable() + { + IonCatalog catalog = new SimpleCatalog(); + IonReaderBuilder immutable = IonReaderBuilder.standard().withCatalog(catalog).immutable(); + IonReaderBuilder mutableCopy = immutable.copy(); + assertSame(immutable, immutable.immutable()); + assertNotSame(immutable, mutableCopy); + assertSame(catalog, mutableCopy.getCatalog()); + mutableCopy.withCatalog(new SimpleCatalog()); + assertNotSame(catalog, mutableCopy.getCatalog()); + } + + @Test + public void testMutateCopiedMutable() + { + IonCatalog catalog = new SimpleCatalog(); + IonReaderBuilder mutable = IonReaderBuilder.standard().withCatalog(catalog); + IonReaderBuilder mutableCopy = mutable.copy(); + assertNotSame(mutable, mutable.immutable()); + assertNotSame(mutable, mutableCopy); + assertSame(mutable, mutable.mutable()); + assertSame(catalog, mutableCopy.getCatalog()); + IonReaderBuilder mutableSame = mutableCopy.withCatalog(new SimpleCatalog()); + assertNotSame(catalog, mutableCopy.getCatalog()); + assertSame(mutableCopy, mutableSame); + } + + @Test + public void testSystemFreeRoundtrip() throws IOException + { + // No IonSystem in sight. + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IonWriter writer = PrivateIonBinaryWriterBuilder.standard().build(out); + writer.writeInt(42); + writer.finish(); + IonReader reader = IonReaderBuilder.standard().build(out.toByteArray()); + assertEquals(IonType.INT, reader.next()); + assertEquals(42, reader.intValue()); + } + +} diff --git a/test/software/amazon/ion/system/IonTextWriterBuilderTest.java b/test/software/amazon/ion/system/IonTextWriterBuilderTest.java index a84ef37595..0e10244fe1 100644 --- a/test/software/amazon/ion/system/IonTextWriterBuilderTest.java +++ b/test/software/amazon/ion/system/IonTextWriterBuilderTest.java @@ -36,8 +36,8 @@ import software.amazon.ion.IonCatalog; import software.amazon.ion.IonWriter; import software.amazon.ion.SymbolTable; -import software.amazon.ion.Symtabs; import software.amazon.ion.impl.PrivateIonWriter; +import software.amazon.ion.impl.Symtabs; import software.amazon.ion.system.IonTextWriterBuilder; import software.amazon.ion.system.SimpleCatalog; From 357388b31d3e4c75858eec5a4f214c5b4948b6cb Mon Sep 17 00:00:00 2001 From: barbosf Date: Fri, 22 Jun 2018 10:46:05 -0700 Subject: [PATCH 003/490] Release 1.2.0 --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a18834b8e1..04cb20b202 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ dependency into your project's `pom.xml`: software.amazon.ion ion-java - 1.1.1 + 1.2.0 ``` diff --git a/pom.xml b/pom.xml index d13b85b9cb..2298eba7c8 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 software.amazon.ion ion-java - 1.1.3-SNAPSHOT + 1.2.0 bundle ${project.groupId}:${project.artifactId} From 7992a822ba85fb538b3dd872f1a86b4d0e80906a Mon Sep 17 00:00:00 2001 From: barbosf Date: Fri, 22 Jun 2018 12:11:23 -0700 Subject: [PATCH 004/490] Bumps version to 1.2.1-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2298eba7c8..9c2203e472 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 software.amazon.ion ion-java - 1.2.0 + 1.2.1-SNAPSHOT bundle ${project.groupId}:${project.artifactId} From 28309af0ffe96a045a270f92fa722642cc382df0 Mon Sep 17 00:00:00 2001 From: Fernando Raganhan Barbosa Date: Tue, 7 Aug 2018 16:53:30 -0700 Subject: [PATCH 005/490] Correctly detects overflow for VarUInt and VarInt (#147) Correctly detects overflow for VarUInt and VarInt https://github.com/amzn/ion-java/issues/146 --- .../amazon/ion/impl/IonReaderBinaryRawX.java | 230 +++++++----------- test/software/amazon/ion/impl/VarIntTest.java | 96 ++++++++ 2 files changed, 188 insertions(+), 138 deletions(-) create mode 100644 test/software/amazon/ion/impl/VarIntTest.java diff --git a/src/software/amazon/ion/impl/IonReaderBinaryRawX.java b/src/software/amazon/ion/impl/IonReaderBinaryRawX.java index 196d416872..ae391f7f32 100644 --- a/src/software/amazon/ion/impl/IonReaderBinaryRawX.java +++ b/src/software/amazon/ion/impl/IonReaderBinaryRawX.java @@ -886,129 +886,98 @@ protected final BigInteger readBigInteger(int len, boolean is_negative) throws I } return value; } + protected final int readVarInt() throws IOException { - int retvalue = 0; - boolean is_negative = false; - int b; - // synthetic label "done" (yuck) -done: for (;;) { - // read the first byte - it has the sign bit - if ((b = read()) < 0) throwUnexpectedEOFException(); - if ((b & 0x40) != 0) { - is_negative = true; - } - retvalue = (b & 0x3F); - if ((b & 0x80) != 0) break done; - // for the second byte we shift our eariler bits just as much, - // but there are fewer of them there to shift - if ((b = read()) < 0) throwUnexpectedEOFException(); - retvalue = (retvalue << 7) | (b & 0x7F); - if ((b & 0x80) != 0) break done; - // for the rest, they're all the same - if ((b = read()) < 0) throwUnexpectedEOFException(); - retvalue = (retvalue << 7) | (b & 0x7F); - if ((b & 0x80) != 0) break done; - // for the rest, they're all the same - if ((b = read()) < 0) throwUnexpectedEOFException(); - retvalue = (retvalue << 7) | (b & 0x7F); - if ((b & 0x80) != 0) break done; - // for the rest, they're all the same - if ((b = read()) < 0) throwUnexpectedEOFException(); - retvalue = (retvalue << 7) | (b & 0x7F); - if ((b & 0x80) != 0) break done; - // if we get here we have more bits than we have room for :( - throwIntOverflowExeption(); - } - if (is_negative) { - retvalue = -retvalue; - } - return retvalue; - } - protected final long readVarLong() throws IOException - { - long retvalue = 0; - boolean is_negative = false; - int b; - // synthetic label "done" (yuck) -done: for (;;) { - // read the first byte - it has the sign bit - if ((b = read()) < 0) throwUnexpectedEOFException(); - if ((b & 0x40) != 0) { - is_negative = true; - } - retvalue = (b & 0x3F); - if ((b & 0x80) != 0) break done; - // for the second byte we shift our eariler bits just as much, - // but there are fewer of them there to shift - if ((b = read()) < 0) throwUnexpectedEOFException(); - retvalue = (retvalue << 7) | (b & 0x7F); - if ((b & 0x80) != 0) break done; - // for the rest, they're all the same - for (;;) { - if ((b = read()) < 0) throwUnexpectedEOFException(); - if ((retvalue & 0xFE00000000000000L) != 0) throwIntOverflowExeption(); - retvalue = (retvalue << 7) | (b & 0x7F); - if ((b & 0x80) != 0) break done; - } - } - if (is_negative) { - retvalue = -retvalue; - } - return retvalue; + return readVarInt(read()); } + /** * Reads an integer value, returning null to mean -0. * @throws IOException */ protected final Integer readVarInteger() throws IOException { - int retvalue = 0; - boolean is_negative = false; - int b; - // Synthetic label "done" (yuck) -done: for (;;) { - // read the first byte - it has the sign bit - if ((b = read()) < 0) throwUnexpectedEOFException(); + int firstByte = read(); + + // if byte represents -0 returns null + if (firstByte == 0xC0) { + return null; + } + + return readVarInt(firstByte); + } + + /** + * reads a varInt after the first byte was read. The first byte is used to specify the sign and -0 has different + * representation on the protected API that was called + * + * @param firstByte last varInt octet + */ + private int readVarInt(int firstByte) throws IOException { + // VarInt uses the high-order bit of the last octet as a marker; some (but not all) 5-byte VarInts can fit + // into a Java int. + // To validate overflows we accumulate the VarInt in a long and then check if it can be represented by an int + // + // see http://amzn.github.io/ion-docs/docs/binary.html#varuint-and-varint-fields + + long retValue = 0; + int b = firstByte; + boolean isNegative = false; + + for (;;) { + if (b < 0) throwUnexpectedEOFException(); + if ((b & 0x40) != 0) { - is_negative = true; + isNegative = true; } - retvalue = (b & 0x3F); - if ((b & 0x80) != 0) break done; - // for the second byte we shift our eariler bits just as much, - // but there are fewer of them there to shift + retValue = (b & 0x3F); + if ((b & 0x80) != 0) break; + if ((b = read()) < 0) throwUnexpectedEOFException(); - retvalue = (retvalue << 7) | (b & 0x7F); - if ((b & 0x80) != 0) break done; + retValue = (retValue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break; // for the rest, they're all the same + if ((b = read()) < 0) throwUnexpectedEOFException(); - retvalue = (retvalue << 7) | (b & 0x7F); - if ((b & 0x80) != 0) break done; + retValue = (retValue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break; // for the rest, they're all the same + if ((b = read()) < 0) throwUnexpectedEOFException(); - retvalue = (retvalue << 7) | (b & 0x7F); - if ((b & 0x80) != 0) break done; + retValue = (retValue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break; // for the rest, they're all the same + if ((b = read()) < 0) throwUnexpectedEOFException(); - retvalue = (retvalue << 7) | (b & 0x7F); - if ((b & 0x80) != 0) break done; - // if we get here we have more bits than we have room for :( - throwIntOverflowExeption(); - } - Integer retInteger = null; - if (is_negative) { - if (retvalue != 0) { - retInteger = new Integer(-retvalue); - } + retValue = (retValue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break; + + // Don't support anything above a 5-byte VarInt for now, see https://github.com/amzn/ion-java/issues/146 + throwVarIntOverflowException(); } - else { - retInteger = new Integer(retvalue); + + if (isNegative) { + retValue = -retValue; + } + + int retValueAsInt = (int) retValue; + if (retValue != ((long) retValueAsInt)) { + throwVarIntOverflowException(); } - return retInteger; + + return retValueAsInt; } + protected final int readVarUIntOrEOF() throws IOException { - int retvalue = 0; + // VarUInt uses the high-order bit of the last octet as a marker; some (but not all) 5-byte VarUInt can fit + // into a Java int. + // To validate overflows we accumulate the VarInt in a long and then check if it can be represented by an int + // + // see http://amzn.github.io/ion-docs/docs/binary.html#varuint-and-varint-fields + + long retvalue = 0; int b; for (;;) { // fake loop to create a "goto done" if ((b = read()) < 0) { @@ -1016,47 +985,43 @@ protected final int readVarUIntOrEOF() throws IOException } retvalue = (retvalue << 7) | (b & 0x7F); if ((b & 0x80) != 0) break; + if ((b = read()) < 0) throwUnexpectedEOFException(); retvalue = (retvalue << 7) | (b & 0x7F); if ((b & 0x80) != 0) break; + if ((b = read()) < 0) throwUnexpectedEOFException(); retvalue = (retvalue << 7) | (b & 0x7F); if ((b & 0x80) != 0) break; + if ((b = read()) < 0) throwUnexpectedEOFException(); retvalue = (retvalue << 7) | (b & 0x7F); if ((b & 0x80) != 0) break; + if ((b = read()) < 0) throwUnexpectedEOFException(); retvalue = (retvalue << 7) | (b & 0x7F); if ((b & 0x80) != 0) break; - // if we get here we have more bits than we have room for :( - throwIntOverflowExeption(); + + // Don't support anything above a 5-byte VarUInt for now, see https://github.com/amzn/ion-java/issues/146 + throwVarIntOverflowException(); } - return retvalue; + + int retValueAsInt = (int) retvalue; + if (retvalue != ((long) retValueAsInt)) { + throwVarIntOverflowException(); + } + + return retValueAsInt; } + protected final int readVarUInt() throws IOException { - int retvalue = 0; - int b; - for (;;) { // fake loop to create a "goto done" - if ((b = read()) < 0) throwUnexpectedEOFException(); - retvalue = (retvalue << 7) | (b & 0x7F); - if ((b & 0x80) != 0) break; - if ((b = read()) < 0) throwUnexpectedEOFException(); - retvalue = (retvalue << 7) | (b & 0x7F); - if ((b & 0x80) != 0) break; - if ((b = read()) < 0) throwUnexpectedEOFException(); - retvalue = (retvalue << 7) | (b & 0x7F); - if ((b & 0x80) != 0) break; - if ((b = read()) < 0) throwUnexpectedEOFException(); - retvalue = (retvalue << 7) | (b & 0x7F); - if ((b & 0x80) != 0) break; - if ((b = read()) < 0) throwUnexpectedEOFException(); - retvalue = (retvalue << 7) | (b & 0x7F); - if ((b & 0x80) != 0) break; - // if we get here we have more bits than we have room for :( - throwIntOverflowExeption(); + int varUInt = readVarUIntOrEOF(); + if (varUInt == UnifiedInputStreamX.EOF) { + throwUnexpectedEOFException(); } - return retvalue; + + return varUInt; } protected final double readFloat(int len) throws IOException { @@ -1076,18 +1041,7 @@ protected final double readFloat(int len) throws IOException ? (double) Float.intBitsToFloat((int) (dBits & 0xffffffffL)) : Double.longBitsToDouble(dBits); } - protected final long readVarULong() throws IOException - { - long retvalue = 0; - int b; - for (;;) { - if ((b = read()) < 0) throwUnexpectedEOFException(); - if ((retvalue & 0xFE00000000000000L) != 0) throwIntOverflowExeption(); - retvalue = (retvalue << 7) | (b & 0x7F); - if ((b & 0x80) != 0) break; - } - return retvalue; - } + protected final Decimal readDecimal(int len) throws IOException { MathContext mathContext = MathContext.UNLIMITED; @@ -1276,8 +1230,8 @@ private final void throwUTF8Exception() throws IOException private final void throwUnexpectedEOFException() throws IOException { throwErrorAt("unexpected EOF in value"); } - private final void throwIntOverflowExeption() throws IOException { - throwErrorAt("int in stream is too long for a Java int 32 use readLong()"); + private final void throwVarIntOverflowException() throws IOException { + throwErrorAt("int in stream is too long for a Java int 32"); } protected IonException newErrorAt(String msg) { diff --git a/test/software/amazon/ion/impl/VarIntTest.java b/test/software/amazon/ion/impl/VarIntTest.java new file mode 100644 index 0000000000..ae0e856cd1 --- /dev/null +++ b/test/software/amazon/ion/impl/VarIntTest.java @@ -0,0 +1,96 @@ +package software.amazon.ion.impl; + +import org.junit.Test; +import software.amazon.ion.IonException; +import software.amazon.ion.IonTestCase; +import software.amazon.ion.system.SimpleCatalog; + +import javax.xml.bind.DatatypeConverter; +import java.io.ByteArrayInputStream; + +public class VarIntTest extends IonTestCase { + + @Test + public void readMaxVarUInt() throws Exception { + assertEquals(Integer.MAX_VALUE, makeReader("077F7F7FFF").readVarUInt()); + } + + @Test(expected = IonException.class) + public void overflowVarUInt() throws Exception { + makeReader("0800000080").readVarUInt(); // Integer.MAX_VALUE + 1 + } + + @Test(expected = IonException.class) + public void readEOFVarUInt() throws Exception { + makeReader("").readVarUInt(); + } + + @Test + public void readMaxVarUIntOrEOF() throws Exception { + assertEquals(Integer.MAX_VALUE, makeReader("077F7F7FFF").readVarUIntOrEOF()); + } + + @Test(expected = IonException.class) + public void overflowVarUIntOrEOF() throws Exception { + makeReader("0800000080").readVarUIntOrEOF(); // Integer.MAX_VALUE + 1 + } + + @Test + public void readEOFVarUIntOrEOF() throws Exception { + assertEquals(UnifiedInputStreamX.EOF, makeReader("").readVarUIntOrEOF()); + } + + @Test + public void readMaxVarInt() throws Exception { + assertEquals(Integer.MAX_VALUE, makeReader("077F7F7FFF").readVarInt()); + } + + @Test + public void readMinVarInt() throws Exception { + assertEquals(Integer.MIN_VALUE, makeReader("4800000080").readVarInt()); + } + + @Test(expected = IonException.class) + public void readVarIntOverflow() throws Exception { + makeReader("0800000080").readVarInt(); // Integer.MAX_VALUE + 1 + } + + @Test(expected = IonException.class) + public void readVarIntUnderflow() throws Exception { + makeReader("4800000081").readVarInt(); // Integer.MIN_VALUE - 1 + } + + @Test + public void readMaxVarInteger() throws Exception { + assertEquals(Integer.MAX_VALUE, (int) makeReader("077F7F7FFF").readVarInteger()); + } + + @Test + public void readMinVarInteger() throws Exception { + assertEquals(Integer.MIN_VALUE, (int) makeReader("4800000080").readVarInteger()); + } + + @Test(expected = IonException.class) + public void readVarIntegerOverflow() throws Exception { + makeReader("0800000080").readVarInteger(); // Integer.MAX_VALUE + 1 + } + + @Test(expected = IonException.class) + public void readVarIntegerUnderflow() throws Exception { + makeReader("4800000081").readVarInt(); // Integer.MIN_VALUE - 1 + } + + @Test + public void readVarIntegerNegativeZero() throws Exception { + assertNull(makeReader("C0").readVarInteger()); + } + + private IonReaderBinaryUserX makeReader(String hex) throws Exception { + ByteArrayInputStream input = new ByteArrayInputStream(DatatypeConverter.parseHexBinary("E00100EA" + hex)); + + UnifiedInputStreamX uis = UnifiedInputStreamX.makeStream(input); + uis.skip(4); + + return new IonReaderBinaryUserX(new SimpleCatalog(), LocalSymbolTable.DEFAULT_LST_FACTORY, uis, 0); + } +} \ No newline at end of file From e8818dbfc0be2eb59e2d044bb9667a97a304fa97 Mon Sep 17 00:00:00 2001 From: Fernando Raganhan Barbosa Date: Fri, 10 Aug 2018 13:00:09 -0700 Subject: [PATCH 006/490] uses decimal.longValue when casting from decimal to long (#149) was using intValue previously resulting in overflow. Inludes UTs to verify the behavior --- .../ion/impl/PrivateScalarConversions.java | 2 +- .../impl/PrivateScalarConversionsTest.java | 33 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 test/software/amazon/ion/impl/PrivateScalarConversionsTest.java diff --git a/src/software/amazon/ion/impl/PrivateScalarConversions.java b/src/software/amazon/ion/impl/PrivateScalarConversions.java index d02d2f8e49..5c71c1586a 100644 --- a/src/software/amazon/ion/impl/PrivateScalarConversions.java +++ b/src/software/amazon/ion/impl/PrivateScalarConversions.java @@ -800,7 +800,7 @@ private final void fn_from_decimal_to_long() { ) { throw new CantConvertException("BigDecimal value is too large to fit in a long"); } - _long_value = _decimal_value.intValue(); + _long_value = _decimal_value.longValue(); add_value_type(AS_TYPE.long_value); } private final void fn_from_double_to_long() { diff --git a/test/software/amazon/ion/impl/PrivateScalarConversionsTest.java b/test/software/amazon/ion/impl/PrivateScalarConversionsTest.java new file mode 100644 index 0000000000..b19b301839 --- /dev/null +++ b/test/software/amazon/ion/impl/PrivateScalarConversionsTest.java @@ -0,0 +1,33 @@ +package software.amazon.ion.impl; + +import org.junit.Assert; +import org.junit.Test; +import software.amazon.ion.Decimal; + +import static org.junit.Assert.*; + +public class PrivateScalarConversionsTest { + + private long decimalToLong(final Decimal d) { + PrivateScalarConversions.ValueVariant v = new PrivateScalarConversions.ValueVariant(); + v.setValue(d); + + v.cast(PrivateScalarConversions.FNID_FROM_DECIMAL_TO_LONG); + return v.getLong(); + } + + @Test + public void decimalToLong() { + assertEquals(1, decimalToLong(Decimal.valueOf(1L))); + } + + @Test + public void decimalToMinLong() { + assertEquals(Long.MAX_VALUE, decimalToLong(Decimal.valueOf(Long.MAX_VALUE))); + } + + @Test + public void decimalToMaxLong() { + assertEquals(Long.MIN_VALUE, decimalToLong(Decimal.valueOf(Long.MIN_VALUE))); + } +} \ No newline at end of file From 9e4c03f066d165166a6c858c6d701196537b2c3c Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Fri, 24 Aug 2018 17:38:37 -0700 Subject: [PATCH 007/490] Requires the annotation to be a structs first annotation in order to carry LST semantics. --- ion-tests | 2 +- .../amazon/ion/impl/IonReaderBinaryUserX.java | 11 ++--- .../amazon/ion/impl/IonReaderTextUserX.java | 19 +++----- .../amazon/ion/impl/IonReaderTreeUserX.java | 2 +- .../amazon/ion/impl/IonWriterSystem.java | 46 +++---------------- .../amazon/ion/impl/IonWriterSystemTree.java | 2 +- .../amazon/ion/impl/IonWriterUser.java | 8 ++-- .../amazon/ion/impl/PrivateIonValue.java | 7 +++ .../amazon/ion/impl/PrivateIonWriterBase.java | 8 +++- .../amazon/ion/impl/PrivateUtils.java | 4 +- .../amazon/ion/impl/lite/IonValueLite.java | 8 ++-- 11 files changed, 43 insertions(+), 74 deletions(-) diff --git a/ion-tests b/ion-tests index f7a4999a41..20bc2a6d4f 160000 --- a/ion-tests +++ b/ion-tests @@ -1 +1 @@ -Subproject commit f7a4999a416b98a4ab454ba18843a177404b7a45 +Subproject commit 20bc2a6d4f22f757ed8abbeb0c961296d42a97b4 diff --git a/src/software/amazon/ion/impl/IonReaderBinaryUserX.java b/src/software/amazon/ion/impl/IonReaderBinaryUserX.java index aa07fab0d3..5a53c251fc 100644 --- a/src/software/amazon/ion/impl/IonReaderBinaryUserX.java +++ b/src/software/amazon/ion/impl/IonReaderBinaryUserX.java @@ -214,13 +214,10 @@ private final void has_next_helper_user() throws IOException } else if (_value_tid == PrivateIonConstants.tidStruct) { int count = load_annotations(); - for(int ii=0; ii 0 && _annotation_ids[0] == ION_SYMBOL_TABLE_SID) { + _symbols = _lstFactory.newLocalSymtab(_catalog, this, false); + push_symbol_table(_symbols); + _has_next_needed = true; } } else { diff --git a/src/software/amazon/ion/impl/IonReaderTextUserX.java b/src/software/amazon/ion/impl/IonReaderTextUserX.java index d98d504b27..5906f6b666 100644 --- a/src/software/amazon/ion/impl/IonReaderTextUserX.java +++ b/src/software/amazon/ion/impl/IonReaderTextUserX.java @@ -128,19 +128,12 @@ private final boolean has_next_user_value() if (_value_type != null && !isNullValue() && IonType.DATAGRAM.equals(getContainerType())) { switch (_value_type) { case STRUCT: - if (_annotation_count > 0) { - for (int ii=0; ii<_annotation_count; ii++) { - SymbolToken a = _annotations[ii]; - // TODO SID only? - if (ION_SYMBOL_TABLE.equals(a.getText())) { - _symbols = _lstFactory.newLocalSymtab(_catalog, - this, - true); - push_symbol_table(_symbols); - _has_next_called = false; - break; - } - } + if (_annotation_count > 0 && ION_SYMBOL_TABLE.equals(_annotations[0].getText())) { + _symbols = _lstFactory.newLocalSymtab(_catalog, + this, + true); + push_symbol_table(_symbols); + _has_next_called = false; } break; case SYMBOL: diff --git a/src/software/amazon/ion/impl/IonReaderTreeUserX.java b/src/software/amazon/ion/impl/IonReaderTreeUserX.java index 2ffef5b3a3..9b24511825 100644 --- a/src/software/amazon/ion/impl/IonReaderTreeUserX.java +++ b/src/software/amazon/ion/impl/IonReaderTreeUserX.java @@ -115,7 +115,7 @@ private boolean next_helper_user() } } else if (IonType.STRUCT.equals(next_type) - && _next.hasTypeAnnotation(ION_SYMBOL_TABLE) + && _next.findTypeAnnotation(ION_SYMBOL_TABLE) == 0 ) { assert(_next instanceof IonStruct); // read a local symbol table diff --git a/src/software/amazon/ion/impl/IonWriterSystem.java b/src/software/amazon/ion/impl/IonWriterSystem.java index 7550732888..e4f3b5534c 100644 --- a/src/software/amazon/ion/impl/IonWriterSystem.java +++ b/src/software/amazon/ion/impl/IonWriterSystem.java @@ -434,58 +434,26 @@ final void ensureAnnotationCapacity(int length) { } - final int[] internAnnotationsAndGetSids() throws IOException - { - int count = _annotation_count; - if (count == 0) return PrivateUtils.EMPTY_INT_ARRAY; - - int[] sids = new int[count]; - for (int i = 0; i < count; i++) - { - SymbolToken sym = _annotations[i]; - int sid = sym.getSid(); - if (sid == UNKNOWN_SYMBOL_ID) - { - String text = sym.getText(); - sid = add_symbol(text); - _annotations[i] = new SymbolTokenImpl(text, sid); - } - sids[i] = sid; - } - return sids; - } - - final boolean hasAnnotations() { return _annotation_count != 0; } - final int annotationCount() - { - return _annotation_count; - } - final void clearAnnotations() { _annotation_count = 0; } - @Override - final boolean has_annotation(String name, int id) - { - assert(this._symbol_table.findKnownSymbol(id).equals(name)); - if (_annotation_count < 1) { - return false; - } - - for (int ii=0; ii<_annotation_count; ii++) { - if (name.equals(_annotations[ii].getText())) { - return true; + final int findAnnotation(String name) { + if (_annotation_count > 0) { + for (int ii=0; ii<_annotation_count; ii++) { + if (name.equals(_annotations[ii].getText())) { + return ii; + } } } - return false; + return -1; } final SymbolToken[] getTypeAnnotationSymbols() diff --git a/src/software/amazon/ion/impl/IonWriterSystemTree.java b/src/software/amazon/ion/impl/IonWriterSystemTree.java index 4223053445..663008a768 100644 --- a/src/software/amazon/ion/impl/IonWriterSystemTree.java +++ b/src/software/amazon/ion/impl/IonWriterSystemTree.java @@ -219,7 +219,7 @@ public void stepIn(IonType containerType) throws IOException public void stepOut() throws IOException { - IonValue prior = _current_parent; + PrivateIonValue prior = (PrivateIonValue)_current_parent; popParent(); if (_current_parent instanceof IonDatagram diff --git a/src/software/amazon/ion/impl/IonWriterUser.java b/src/software/amazon/ion/impl/IonWriterUser.java index adf089de9a..b5b3fd2dca 100644 --- a/src/software/amazon/ion/impl/IonWriterUser.java +++ b/src/software/amazon/ion/impl/IonWriterUser.java @@ -148,11 +148,9 @@ public IonCatalog getCatalog() return _catalog; } - @Override - boolean has_annotation(String name, int id) - { - return _current_writer.has_annotation(name, id); + int findAnnotation(String name) { + return _current_writer.findAnnotation(name); } @Override @@ -374,7 +372,7 @@ public void stepIn(IonType containerType) throws IOException // see if it looks like we're starting a local symbol table if (containerType == IonType.STRUCT && _current_writer.getDepth() == 0 - && has_annotation(ION_SYMBOL_TABLE, ION_SYMBOL_TABLE_SID)) + && findAnnotation(ION_SYMBOL_TABLE) == 0) { open_local_symbol_table_copy(); } diff --git a/src/software/amazon/ion/impl/PrivateIonValue.java b/src/software/amazon/ion/impl/PrivateIonValue.java index d7a4c2533f..cce85f4188 100644 --- a/src/software/amazon/ion/impl/PrivateIonValue.java +++ b/src/software/amazon/ion/impl/PrivateIonValue.java @@ -59,6 +59,13 @@ public interface SymbolTableProvider { */ public SymbolToken[] getTypeAnnotationSymbols(SymbolTableProvider symbolTableProvider); + /** + * Returns the given annotation's index in the value's annotations list, or -1 if not present. + * @param annotation the annotation to find. + * @return the index or -1. + */ + int findTypeAnnotation(String annotation); + /** * Makes this symbol table current for this value. * This may directly apply to this IonValue if this diff --git a/src/software/amazon/ion/impl/PrivateIonWriterBase.java b/src/software/amazon/ion/impl/PrivateIonWriterBase.java index 16659f0030..0fda9b92b0 100644 --- a/src/software/amazon/ion/impl/PrivateIonWriterBase.java +++ b/src/software/amazon/ion/impl/PrivateIonWriterBase.java @@ -124,8 +124,12 @@ public abstract void setSymbolTable(SymbolTable symbols) //======================================================================== // Annotations - - abstract boolean has_annotation(String name, int id); + /** + * Returns the given annotation's index in the value's annotations list, or -1 if not present. + * @param name the annotation to find. + * @return the index or -1. + */ + abstract int findAnnotation(String name); /** diff --git a/src/software/amazon/ion/impl/PrivateUtils.java b/src/software/amazon/ion/impl/PrivateUtils.java index ed8ce655a3..f047a87a52 100644 --- a/src/software/amazon/ion/impl/PrivateUtils.java +++ b/src/software/amazon/ion/impl/PrivateUtils.java @@ -644,10 +644,10 @@ public static Iterator iterate(ValueFactory valueFactory, * * @return boolean true if v can be a local symbol table otherwise false */ - public static boolean valueIsLocalSymbolTable(IonValue v) + public static boolean valueIsLocalSymbolTable(PrivateIonValue v) { return (v instanceof IonStruct - && v.hasTypeAnnotation(ION_SYMBOL_TABLE)); + && v.findTypeAnnotation(ION_SYMBOL_TABLE) == 0); } diff --git a/src/software/amazon/ion/impl/lite/IonValueLite.java b/src/software/amazon/ion/impl/lite/IonValueLite.java index e974eabe6f..cc2e80de65 100644 --- a/src/software/amazon/ion/impl/lite/IonValueLite.java +++ b/src/software/amazon/ion/impl/lite/IonValueLite.java @@ -661,14 +661,16 @@ public void setTypeAnnotations(String... annotations) public final boolean hasTypeAnnotation(String annotation) { if (annotation != null && annotation.length() > 0) { - int pos = find_type_annotation(annotation); + int pos = findTypeAnnotation(annotation); if (pos >= 0) { return true; } } return false; } - private final int find_type_annotation(String annotation) + + @Override + public final int findTypeAnnotation(String annotation) { assert(annotation != null && annotation.length() > 0); @@ -802,7 +804,7 @@ public void removeTypeAnnotation(String annotation) checkForLock(); if (annotation != null && annotation.length() > 0) { - int pos = find_type_annotation(annotation); + int pos = findTypeAnnotation(annotation); if (pos < 0) { return; } From d9388a70be2a29609640b8ad16087651a7654ff2 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Fri, 24 Aug 2018 16:21:06 -0700 Subject: [PATCH 008/490] Disallows writing and reading symbol IDs out of range of the local symbol table. --- ion-tests | 2 +- .../ion/impl/IonReaderBinarySystemX.java | 4 +- .../amazon/ion/impl/IonReaderBinaryUserX.java | 33 +++++++ .../amazon/ion/impl/IonReaderTextSystemX.java | 2 +- .../amazon/ion/impl/IonReaderTextUserX.java | 31 +++++++ .../amazon/ion/impl/IonWriterSystem.java | 4 + .../amazon/ion/impl/PrivateIonSystem.java | 6 +- .../amazon/ion/impl/PrivateIonWriterBase.java | 13 ++- .../ion/impl/bin/IonManagedBinaryWriter.java | 7 ++ test/software/amazon/ion/CloneTest.java | 20 +---- .../amazon/ion/HashCodeCorrectnessTest.java | 49 +++++----- test/software/amazon/ion/IonValueTest.java | 5 +- test/software/amazon/ion/StructTest.java | 89 +++++++------------ test/software/amazon/ion/SymbolTest.java | 2 +- .../amazon/ion/SystemProcessingTestCase.java | 15 ++-- test/software/amazon/ion/TestUtils.java | 2 +- .../amazon/ion/TrBwBrProcessingTest.java | 12 +++ .../amazon/ion/impl/IonWriterTestCase.java | 37 +++++--- .../ion/streaming/SeekableReaderTest.java | 2 +- 19 files changed, 207 insertions(+), 128 deletions(-) diff --git a/ion-tests b/ion-tests index 20bc2a6d4f..9bbb4ea124 160000 --- a/ion-tests +++ b/ion-tests @@ -1 +1 @@ -Subproject commit 20bc2a6d4f22f757ed8abbeb0c961296d42a97b4 +Subproject commit 9bbb4ea1244c707d77d0d0ff0d35a90d44b4d885 diff --git a/src/software/amazon/ion/impl/IonReaderBinarySystemX.java b/src/software/amazon/ion/impl/IonReaderBinarySystemX.java index 1bf2a98e19..27b9900d48 100644 --- a/src/software/amazon/ion/impl/IonReaderBinarySystemX.java +++ b/src/software/amazon/ion/impl/IonReaderBinarySystemX.java @@ -403,7 +403,7 @@ public final String stringValue() return _v.getString(); } - public final SymbolToken symbolValue() + public SymbolToken symbolValue() { if (_value_type != SYMBOL) throw new IllegalStateException(); if (_value_is_null) return null; @@ -439,7 +439,7 @@ public final String getFieldName() return name; } - public final SymbolToken getFieldNameSymbol() + public SymbolToken getFieldNameSymbol() { if (_value_field_id == SymbolTable.UNKNOWN_SYMBOL_ID) return null; int sid = _value_field_id; diff --git a/src/software/amazon/ion/impl/IonReaderBinaryUserX.java b/src/software/amazon/ion/impl/IonReaderBinaryUserX.java index 5a53c251fc..16c04d6192 100644 --- a/src/software/amazon/ion/impl/IonReaderBinaryUserX.java +++ b/src/software/amazon/ion/impl/IonReaderBinaryUserX.java @@ -25,6 +25,8 @@ import software.amazon.ion.Span; import software.amazon.ion.SpanProvider; import software.amazon.ion.SymbolTable; +import software.amazon.ion.SymbolToken; +import software.amazon.ion.UnknownSymbolException; import software.amazon.ion.impl.PrivateScalarConversions.AS_TYPE; import software.amazon.ion.impl.UnifiedInputStreamX.FromByteArray; import software.amazon.ion.impl.UnifiedSavePointManagerX.SavePoint; @@ -226,6 +228,37 @@ else if (_value_tid == PrivateIonConstants.tidStruct) { } } + private void validateSymbolToken(SymbolToken symbol) { + if (symbol != null) { + if (symbol.getText() == null && symbol.getSid() > getSymbolTable().getMaxId()) { + throw new UnknownSymbolException(symbol.getSid()); + } + } + } + + @Override + public SymbolToken[] getTypeAnnotationSymbols() { + SymbolToken[] annotations = super.getTypeAnnotationSymbols(); + for (SymbolToken annotation : annotations) { + validateSymbolToken(annotation); + } + return annotations; + } + + @Override + public final SymbolToken getFieldNameSymbol() { + SymbolToken fieldName = super.getFieldNameSymbol(); + validateSymbolToken(fieldName); + return fieldName; + } + + @Override + public final SymbolToken symbolValue() { + SymbolToken symbol = super.symbolValue(); + validateSymbolToken(symbol); + return symbol; + } + // // This code handles the skipped symbol table // support - it is cloned in IonReaderTextUserX, diff --git a/src/software/amazon/ion/impl/IonReaderTextSystemX.java b/src/software/amazon/ion/impl/IonReaderTextSystemX.java index 537a26dd4b..459a067f10 100644 --- a/src/software/amazon/ion/impl/IonReaderTextSystemX.java +++ b/src/software/amazon/ion/impl/IonReaderTextSystemX.java @@ -662,7 +662,7 @@ public final String getFieldName() } @Override - public final SymbolToken getFieldNameSymbol() + public SymbolToken getFieldNameSymbol() { SymbolToken sym = super.getFieldNameSymbol(); if (sym != null) diff --git a/src/software/amazon/ion/impl/IonReaderTextUserX.java b/src/software/amazon/ion/impl/IonReaderTextUserX.java index 5906f6b666..64fa372ac8 100644 --- a/src/software/amazon/ion/impl/IonReaderTextUserX.java +++ b/src/software/amazon/ion/impl/IonReaderTextUserX.java @@ -27,6 +27,7 @@ import software.amazon.ion.SymbolTable; import software.amazon.ion.SymbolToken; import software.amazon.ion.TextSpan; +import software.amazon.ion.UnknownSymbolException; import software.amazon.ion.UnsupportedIonVersionException; /** @@ -177,6 +178,36 @@ private final void symbol_table_reset() return; } + private void validateSymbolToken(SymbolToken symbol) { + if (symbol != null) { + if (symbol.getText() == null && symbol.getSid() > getSymbolTable().getMaxId()) { + throw new UnknownSymbolException(symbol.getSid()); + } + } + } + + @Override + public SymbolToken[] getTypeAnnotationSymbols() { + SymbolToken[] annotations = super.getTypeAnnotationSymbols(); + for (SymbolToken annotation : annotations) { + validateSymbolToken(annotation); + } + return annotations; + } + + @Override + public final SymbolToken getFieldNameSymbol() { + SymbolToken fieldName = super.getFieldNameSymbol(); + validateSymbolToken(fieldName); + return fieldName; + } + + @Override + public final SymbolToken symbolValue() { + SymbolToken symbol = super.symbolValue(); + validateSymbolToken(symbol); + return symbol; + } @Override public SymbolTable getSymbolTable() diff --git a/src/software/amazon/ion/impl/IonWriterSystem.java b/src/software/amazon/ion/impl/IonWriterSystem.java index e4f3b5534c..6237a7c58a 100644 --- a/src/software/amazon/ion/impl/IonWriterSystem.java +++ b/src/software/amazon/ion/impl/IonWriterSystem.java @@ -391,6 +391,7 @@ public final void setFieldNameSymbol(SymbolToken name) if (sid < 0) { throw new IllegalArgumentException(); } + validateSymbolId(sid); _field_name_type = IonType.INT; _field_name_sid = sid; @@ -483,6 +484,9 @@ public final void setTypeAnnotationSymbols(SymbolToken... annotations) for (int i = 0; i < count; i++) { SymbolToken sym = annotations[i]; + if (sym.getText() == null) { + validateSymbolId(sym.getSid()); + } sym = PrivateUtils.localize(symtab, sym); _annotations[i] = sym; } diff --git a/src/software/amazon/ion/impl/PrivateIonSystem.java b/src/software/amazon/ion/impl/PrivateIonSystem.java index a55dbc0b1b..640d0ae6f8 100644 --- a/src/software/amazon/ion/impl/PrivateIonSystem.java +++ b/src/software/amazon/ion/impl/PrivateIonSystem.java @@ -36,19 +36,19 @@ public interface PrivateIonSystem public SymbolTable newSharedSymbolTable(IonStruct ionRep); /** - * TODO Must correct amzn/ion-java#63 before exposing this or using from public API. + * TODO Must correct amzn/ion-java#14 before exposing this or using from public API. */ public Iterator systemIterate(String ionText); /** - * TODO Must correct amzn/ion-java#63 before exposing this or using from public API. + * TODO Must correct amzn/ion-java#14 before exposing this or using from public API. */ public Iterator systemIterate(Reader ionText); public Iterator systemIterate(byte[] ionData); /** - * TODO Must correct amzn/ion-java#63 before exposing this or using from public API. + * TODO Must correct amzn/ion-java#14 before exposing this or using from public API. */ public Iterator systemIterate(InputStream ionData); diff --git a/src/software/amazon/ion/impl/PrivateIonWriterBase.java b/src/software/amazon/ion/impl/PrivateIonWriterBase.java index 0fda9b92b0..a5cdc96716 100644 --- a/src/software/amazon/ion/impl/PrivateIonWriterBase.java +++ b/src/software/amazon/ion/impl/PrivateIonWriterBase.java @@ -165,9 +165,8 @@ public abstract void setSymbolTable(SymbolTable symbols) abstract int[] getTypeAnnotationIds(); /** - * Write symbolId out as an IonSymbol value. The value does not - * have to be valid in the symbol table, unless the output is - * text, in which case it does. + * Write symbolId out as an IonSymbol value. The value must + * be valid in the symbol table. * * @param symbolId symbol table id to write */ @@ -217,6 +216,13 @@ public void writeNull() throws IOException writeNull(IonType.NULL); } + final void validateSymbolId(int sid) { + if (sid > getSymbolTable().getMaxId()) { + // There is no slot for this symbol ID in the symbol table, + // so an error would be raised on read. Fail early on write. + throw new UnknownSymbolException(sid); + } + } public final void writeSymbolToken(SymbolToken tok) throws IOException @@ -234,6 +240,7 @@ public final void writeSymbolToken(SymbolToken tok) else { int sid = tok.getSid(); + validateSymbolId(sid); writeSymbol(sid); } } diff --git a/src/software/amazon/ion/impl/bin/IonManagedBinaryWriter.java b/src/software/amazon/ion/impl/bin/IonManagedBinaryWriter.java index ec0300a9e8..61d77afbc7 100644 --- a/src/software/amazon/ion/impl/bin/IonManagedBinaryWriter.java +++ b/src/software/amazon/ion/impl/bin/IonManagedBinaryWriter.java @@ -48,6 +48,7 @@ import software.amazon.ion.SymbolTable; import software.amazon.ion.SymbolToken; import software.amazon.ion.Timestamp; +import software.amazon.ion.UnknownSymbolException; import software.amazon.ion.impl.PrivateUtils; import software.amazon.ion.impl.bin.IonRawBinaryWriter.StreamCloseMode; import software.amazon.ion.impl.bin.IonRawBinaryWriter.StreamFlushMode; @@ -843,6 +844,12 @@ private SymbolToken intern(final SymbolToken token) // string content always makes us intern return intern(text); } + final int sid = token.getSid(); + if (sid > getSymbolTable().getMaxId()) { + // There is no slot for this symbol ID in the symbol table, + // so an error would be raised on read. Fail early on write. + throw new UnknownSymbolException(sid); + } // no text, we just return what we got return token; } diff --git a/test/software/amazon/ion/CloneTest.java b/test/software/amazon/ion/CloneTest.java index 682b223c9c..0e94379515 100644 --- a/test/software/amazon/ion/CloneTest.java +++ b/test/software/amazon/ion/CloneTest.java @@ -74,8 +74,6 @@ public void testDifferentValueFactoryCloneWithUnknownSymbolText() assertEquals(99, copy.symbolValue().getSid()); } - static final boolean DEFECTIVE_CLONE_OF_UNKNOWN_ANNOTATION_TEXT = true; - @Test public void testIonValueCloneWithUnknownAnnotationText() { @@ -83,15 +81,8 @@ public void testIonValueCloneWithUnknownAnnotationText() IonInt original = system().newInt(5); original.setTypeAnnotationSymbols(tok); - if (! DEFECTIVE_CLONE_OF_UNKNOWN_ANNOTATION_TEXT) { - thrown.expect(UnknownSymbolException.class); - thrown.expectMessage("$99"); - } - IonInt actual = original.clone(); - if (DEFECTIVE_CLONE_OF_UNKNOWN_ANNOTATION_TEXT) { - assertEquals(original, actual); - } + assertEquals(original, actual); } @Test @@ -101,15 +92,8 @@ public void testValueFactoryCloneWithUnknownAnnotationText() IonInt original = system().newInt(5); original.setTypeAnnotationSymbols(tok); - if (! DEFECTIVE_CLONE_OF_UNKNOWN_ANNOTATION_TEXT) { - thrown.expect(UnknownSymbolException.class); - thrown.expectMessage("$99"); - } - IonInt actual = system().clone(original); - if (DEFECTIVE_CLONE_OF_UNKNOWN_ANNOTATION_TEXT) { - assertEquals(original, actual); - } + assertEquals(original, actual); } @Test diff --git a/test/software/amazon/ion/HashCodeCorrectnessTest.java b/test/software/amazon/ion/HashCodeCorrectnessTest.java index edd435c721..1a48f634fa 100644 --- a/test/software/amazon/ion/HashCodeCorrectnessTest.java +++ b/test/software/amazon/ion/HashCodeCorrectnessTest.java @@ -543,10 +543,11 @@ public void testIonStructEvenEquivChildValueHashCode() "{a:456, b:456}"); assertIonNotEqImpliesHashNotEq("{a:annot::123, b:123}", "{a:456, b:annot::456}"); - assertIonNotEqImpliesHashNotEq("{$99:123, $98:123}", - "{$99:456, $98:456}"); - assertIonNotEqImpliesHashNotEq("{$99:annot::123, $98:123}", - "{$99:456, $98:annot::456}"); + String sharedSymbolTable = "$ion_symbol_table::{imports:[{name:\"foo\", version: 1, max_id:90}]}"; + assertIonNotEqImpliesHashNotEq(sharedSymbolTable + "{$99:123, $98:123}", + sharedSymbolTable + "{$99:456, $98:456}"); + assertIonNotEqImpliesHashNotEq(sharedSymbolTable + "{$99:annot::123, $98:123}", + sharedSymbolTable + "{$99:456, $98:annot::456}"); } @Test @@ -564,18 +565,20 @@ public void testIonStructSwappedFieldNameAndValueHashCode() "{alpha:a, beta:b}"); assertIonNotEqImpliesHashNotEq("{a:alpha, b:beta, c:charlie}", "{alpha:a, beta:b, charlie:c}"); - assertIonNotEqImpliesHashNotEq("{$99:a}", - "{a:$99}"); - assertIonNotEqImpliesHashNotEq("{$99:a, $999:b}", - "{a:$99, b:$999}"); - assertIonNotEqImpliesHashNotEq("{$99:a, $999:b, $9999:c}", - "{a:$99, b:$999, c:$9999}"); + String sharedSymbolTable = "$ion_symbol_table::{imports:[{name:\"foo\", version: 1, max_id:9990}]} "; + assertIonNotEqImpliesHashNotEq(sharedSymbolTable + "{$99:a}", + sharedSymbolTable + "{a:$99}"); + assertIonNotEqImpliesHashNotEq(sharedSymbolTable + "{$99:a, $999:b}", + sharedSymbolTable + "{a:$99, b:$999}"); + assertIonNotEqImpliesHashNotEq(sharedSymbolTable + "{$99:a, $999:b, $9999:c}", + sharedSymbolTable + "{a:$99, b:$999, c:$9999}"); } protected void testTypeAnnotationHashCode(String text, IonType type) { + String sharedSymbolTable = "$ion_symbol_table::{imports:[{name:\"foo\", version: 1, max_id:90}]}"; checkType(type, oneValue("annot1::" + text)); - checkType(type, oneValue("$99::" + text)); + checkType(type, oneValue(sharedSymbolTable + "$99::" + text)); assertIonEqImpliesHashEq(oneValue("annot1::" + text), oneValue("annot2::" + text)); @@ -584,12 +587,12 @@ protected void testTypeAnnotationHashCode(String text, IonType type) assertIonEqImpliesHashEq(oneValue("annot1::annot2::annot3::" + text), oneValue("annot1::annot2::annot3::" + text)); - assertIonEqImpliesHashEq(oneValue("$99::" + text), - oneValue("$98::" + text)); - assertIonEqImpliesHashEq(oneValue("$99::$98::" + text), - oneValue("$99::$98::" + text)); - assertIonEqImpliesHashEq(oneValue("$99::$98::$97::" + text), - oneValue("$99::$98::$97::" + text)); + assertIonEqImpliesHashEq(oneValue(sharedSymbolTable + "$99::" + text), + oneValue(sharedSymbolTable + "$98::" + text)); + assertIonEqImpliesHashEq(oneValue(sharedSymbolTable + "$99::$98::" + text), + oneValue(sharedSymbolTable + "$99::$98::" + text)); + assertIonEqImpliesHashEq(oneValue(sharedSymbolTable + "$99::$98::$97::" + text), + oneValue(sharedSymbolTable + "$99::$98::$97::" + text)); assertIonNotEqImpliesHashNotEq("annot1::" + text, "annot2::" + text); @@ -598,12 +601,12 @@ protected void testTypeAnnotationHashCode(String text, IonType type) assertIonNotEqImpliesHashNotEq("annot1::annot2::annot3::" + text, "annot3::annot2::annot1::" + text); - assertIonNotEqImpliesHashNotEq("$99::" + text, - "$98::" + text); - assertIonNotEqImpliesHashNotEq("$99::$98::" + text, - "$98::$99::" + text); - assertIonNotEqImpliesHashNotEq("$99::$98::$97::" + text, - "$97::$98::$99::" + text); + assertIonNotEqImpliesHashNotEq(sharedSymbolTable + "$99::" + text, + sharedSymbolTable + "$98::" + text); + assertIonNotEqImpliesHashNotEq(sharedSymbolTable + "$99::$98::" + text, + sharedSymbolTable + "$98::$99::" + text); + assertIonNotEqImpliesHashNotEq(sharedSymbolTable + "$99::$98::$97::" + text, + sharedSymbolTable + "$97::$98::$99::" + text); } @Test diff --git a/test/software/amazon/ion/IonValueTest.java b/test/software/amazon/ion/IonValueTest.java index 7f907d0203..c6e92c4087 100644 --- a/test/software/amazon/ion/IonValueTest.java +++ b/test/software/amazon/ion/IonValueTest.java @@ -272,11 +272,12 @@ public void testRemoveDuplicateAnnotation() @Test public void testDetachWithUnknownAnnotation() { - IonList list = (IonList) system().singleValue("[$99::null]"); + String sharedSymbolTable = "$ion_symbol_table::{imports:[{name:\"foo\", version: 1, max_id: 90}]}"; + IonList list = (IonList) system().singleValue(sharedSymbolTable + "[$99::null]"); IonValue child = list.get(0); child.removeFromContainer(); - IonAssert.assertIonEquals(system().singleValue("$99::null"), child); + IonAssert.assertIonEquals(system().singleValue(sharedSymbolTable + "$99::null"), child); } diff --git a/test/software/amazon/ion/StructTest.java b/test/software/amazon/ion/StructTest.java index 23ee940fc8..15255dbdb2 100644 --- a/test/software/amazon/ion/StructTest.java +++ b/test/software/amazon/ion/StructTest.java @@ -14,7 +14,6 @@ package software.amazon.ion; -import static software.amazon.ion.CloneTest.DEFECTIVE_CLONE_OF_UNKNOWN_ANNOTATION_TEXT; import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; import java.io.PrintWriter; @@ -50,6 +49,10 @@ public class StructTest extends ContainerTestCase { + + private static final String SHARED_SYMBOL_TABLE + = "$ion_symbol_table::{imports:[{name:\"foo\", version:1, max_id:90}]} "; + @Rule public ExpectedException thrown = ExpectedException.none(); @@ -217,7 +220,7 @@ public void modifyStruct(IonStruct value) @Test public void testPutAfterUnknownFieldName() { - IonStruct value = struct("{$99:null}"); + IonStruct value = struct(SHARED_SYMBOL_TABLE + "{$99:null}"); value.put("hi", system().newNull()); } @@ -1120,7 +1123,7 @@ public void testCloneAndRemove() @Test public void testCloneAndRemoveWithSpecialFieldNames() { - IonStruct s1 = struct("{c:1,$99:2,'$19':3,'$20':3}"); + IonStruct s1 = struct(SHARED_SYMBOL_TABLE + "{c:1,$99:2,'$19':3,'$20':3}"); // Unlike cloneAndRetain(), cloneAndRemove() allows null args since it // makes sense to request removal of unknown field names. @@ -1138,7 +1141,7 @@ public void testCloneAndRemoveWithSpecialFieldNames() @Test public void testCloneAndRemoveWithUnknownFieldNameOnRoot() { - IonStruct s1 = struct("{$99:a::{c:1,d:2,d:3}}"); + IonStruct s1 = struct(SHARED_SYMBOL_TABLE + "{$99:a::{c:1,d:2,d:3}}"); IonStruct root = (IonStruct) s1.iterator().next(); @@ -1150,7 +1153,7 @@ public void testCloneAndRemoveWithUnknownFieldNameOnRoot() @Test public void testCloneAndRemoveWithUnknownFieldNameOnField() { - IonStruct s1 = struct("a::{$99:1,d:2,e:3,d:3}"); + IonStruct s1 = struct(SHARED_SYMBOL_TABLE + "a::{$99:1,d:2,e:3,d:3}"); // OK if the unknown symbol is on a removed node. IonStruct actual = s1.cloneAndRemove(null, "e"); @@ -1158,7 +1161,7 @@ public void testCloneAndRemoveWithUnknownFieldNameOnField() assertEquals(expected, actual); // Not OK if the unknown symbol is on a retained node. - s1 = struct("a::{c:1,$99:2,e:3,d:3}"); + s1 = struct(SHARED_SYMBOL_TABLE + "a::{c:1,$99:2,e:3,d:3}"); thrown.expect(UnknownSymbolException.class); thrown.expectMessage("$99"); s1.cloneAndRemove("c", "e"); @@ -1168,7 +1171,7 @@ public void testCloneAndRemoveWithUnknownFieldNameOnField() @Test public void testCloneAndRemoveWithUnknownSymbolTextOnField() { - IonStruct s1 = struct("a::{c:$99,d:2,e:3,d:3}"); + IonStruct s1 = struct(SHARED_SYMBOL_TABLE + "a::{c:$99,d:2,e:3,d:3}"); // OK if the unknown symbol is on a removed node. IonStruct actual = s1.cloneAndRemove("c", "e"); @@ -1176,7 +1179,7 @@ public void testCloneAndRemoveWithUnknownSymbolTextOnField() assertEquals(expected, actual); // Not OK if the unknown symbol is on a retained node. - s1 = struct("a::{c:1,d:$99,e:3,d:3}"); + s1 = struct(SHARED_SYMBOL_TABLE + "a::{c:1,d:$99,e:3,d:3}"); thrown.expect(UnknownSymbolException.class); thrown.expectMessage("$99"); s1.cloneAndRemove("c", "e"); @@ -1185,23 +1188,17 @@ public void testCloneAndRemoveWithUnknownSymbolTextOnField() @Test public void testCloneAndRemoveWithUnknownAnnotationTextOnRoot() { - IonStruct s1 = struct("$99::{c:1,d:2,e:3,d:3}"); - if (! DEFECTIVE_CLONE_OF_UNKNOWN_ANNOTATION_TEXT) { - thrown.expect(UnknownSymbolException.class); - thrown.expectMessage("$99"); - } + IonStruct s1 = struct(SHARED_SYMBOL_TABLE + "$99::{c:1,d:2,e:3,d:3}"); IonStruct actual = s1.cloneAndRemove("c", "e"); - if (DEFECTIVE_CLONE_OF_UNKNOWN_ANNOTATION_TEXT) { - // If we don't fail we should at least retain the SID. - IonStruct expected = struct("$99::{d:2,d:3}"); - assertEquals(expected, actual); - } + // If we don't fail we should at least retain the SID. + IonStruct expected = struct(SHARED_SYMBOL_TABLE + "$99::{d:2,d:3}"); + assertEquals(expected, actual); } @Test public void testCloneAndRemoveWithUnknownAnnotationTextOnField() { - IonStruct s1 = struct("a::{c:$99::1,d:2,e:3,d:3}"); + IonStruct s1 = struct(SHARED_SYMBOL_TABLE + "a::{c:$99::1,d:2,e:3,d:3}"); // OK if the unknown symbol is on a removed node. IonStruct actual = s1.cloneAndRemove("c", "e"); @@ -1209,17 +1206,11 @@ public void testCloneAndRemoveWithUnknownAnnotationTextOnField() assertEquals(expected, actual); // Not OK if the unknown symbol is on a retained node. - s1 = struct("a::{c:1,d:$99::2,e:3}"); - if (! DEFECTIVE_CLONE_OF_UNKNOWN_ANNOTATION_TEXT) { - thrown.expect(UnknownSymbolException.class); - thrown.expectMessage("$99"); - } + s1 = struct(SHARED_SYMBOL_TABLE + "a::{c:1,d:$99::2,e:3}"); actual = s1.cloneAndRemove("c", "e"); - if (DEFECTIVE_CLONE_OF_UNKNOWN_ANNOTATION_TEXT) { - // If we don't fail we should at least retain the SID. - expected = struct("a::{d:$99::2}"); - assertEquals(expected, actual); - } + // If we don't fail we should at least retain the SID. + expected = struct(SHARED_SYMBOL_TABLE + "a::{d:$99::2}"); + assertEquals(expected, actual); } //------------------------------------------------------------------------- @@ -1246,7 +1237,7 @@ public void testCloneAndRetain() @Test public void testCloneAndRetainWithSpecialFieldNames() { - IonStruct s1 = struct("{c:1,$99:2,'$19':3,'$20':3}"); + IonStruct s1 = struct(SHARED_SYMBOL_TABLE + "{c:1,$99:2,'$19':3,'$20':3}"); IonStruct actual = s1.cloneAndRetain("c", "$20"); IonStruct expected = struct("{c:1,'$20':3}"); assertEquals(expected, actual); @@ -1262,7 +1253,7 @@ public void testCloneAndRetainWithSpecialFieldNames() @Test public void testCloneAndRetainWithUnknownFieldNameOnRoot() { - IonStruct s1 = struct("{$99:a::{c:1,d:2,d:3}}"); + IonStruct s1 = struct(SHARED_SYMBOL_TABLE + "{$99:a::{c:1,d:2,d:3}}"); IonStruct root = (IonStruct) s1.iterator().next(); @@ -1274,7 +1265,7 @@ public void testCloneAndRetainWithUnknownFieldNameOnRoot() @Test public void testCloneAndRetainWithUnknownFieldNameOnField() { - IonStruct s1 = struct("a::{$99:1,d:2,e:3,d:3}"); + IonStruct s1 = struct(SHARED_SYMBOL_TABLE + "a::{$99:1,d:2,e:3,d:3}"); // OK if the unknown symbol is on a removed node. IonStruct actual = s1.cloneAndRetain("d", "e"); @@ -1282,7 +1273,7 @@ public void testCloneAndRetainWithUnknownFieldNameOnField() assertEquals(expected, actual); // Not OK if the unknown symbol is on a retained node. - s1 = struct("a::{c:1,$99:2,e:3,d:3}"); + s1 = struct(SHARED_SYMBOL_TABLE + "a::{c:1,$99:2,e:3,d:3}"); thrown.expect(NullPointerException.class); s1.cloneAndRetain(null, "e"); } @@ -1291,7 +1282,7 @@ public void testCloneAndRetainWithUnknownFieldNameOnField() @Test public void testCloneAndRetainWithUnknownSymbolTextOnField() { - IonStruct s1 = struct("a::{c:$99,d:2,e:3,d:3}"); + IonStruct s1 = struct(SHARED_SYMBOL_TABLE + "a::{c:$99,d:2,e:3,d:3}"); // OK if the unknown symbol is on a removed node. IonStruct actual = s1.cloneAndRetain("d"); @@ -1299,7 +1290,7 @@ public void testCloneAndRetainWithUnknownSymbolTextOnField() assertEquals(expected, actual); // Not OK if the unknown symbol is on a retained node. - s1 = struct("a::{c:1,d:$99,e:3,d:3}"); + s1 = struct(SHARED_SYMBOL_TABLE + "a::{c:1,d:$99,e:3,d:3}"); thrown.expect(UnknownSymbolException.class); thrown.expectMessage("$99"); s1.cloneAndRetain("c", "d"); @@ -1308,23 +1299,17 @@ public void testCloneAndRetainWithUnknownSymbolTextOnField() @Test public void testCloneAndRetainWithUnknownAnnotationTextOnRoot() { - IonStruct s1 = struct("$99::{c:1,d:2,e:3,d:3}"); - if (! DEFECTIVE_CLONE_OF_UNKNOWN_ANNOTATION_TEXT) { - thrown.expect(UnknownSymbolException.class); - thrown.expectMessage("$99"); - } + IonStruct s1 = struct(SHARED_SYMBOL_TABLE + "$99::{c:1,d:2,e:3,d:3}"); IonStruct actual = s1.cloneAndRetain("c", "e"); - if (DEFECTIVE_CLONE_OF_UNKNOWN_ANNOTATION_TEXT) { - // If we don't fail we should at least retain the SID. - IonStruct expected = struct("$99::{c:1,e:3}"); - assertEquals(expected, actual); - } + // If we don't fail we should at least retain the SID. + IonStruct expected = struct(SHARED_SYMBOL_TABLE + "$99::{c:1,e:3}"); + assertEquals(expected, actual); } @Test public void testCloneAndRetainWithUnknownAnnotationTextOnField() { - IonStruct s1 = struct("a::{c:$99::1,d:2,e:3,d:3}"); + IonStruct s1 = struct(SHARED_SYMBOL_TABLE + "a::{c:$99::1,d:2,e:3,d:3}"); // OK if the unknown symbol is on a removed node. IonStruct actual = s1.cloneAndRetain("d"); @@ -1332,16 +1317,10 @@ public void testCloneAndRetainWithUnknownAnnotationTextOnField() assertEquals(expected, actual); // Not OK if the unknown symbol is on a retained node. - if (! DEFECTIVE_CLONE_OF_UNKNOWN_ANNOTATION_TEXT) { - thrown.expect(UnknownSymbolException.class); - thrown.expectMessage("$99"); - } actual = s1.cloneAndRetain("c", "e"); - if (DEFECTIVE_CLONE_OF_UNKNOWN_ANNOTATION_TEXT) { - // If we don't fail we should at least retain the SID. - expected = struct("a::{c:$99::1,e:3}"); - assertEquals(expected, actual); - } + // If we don't fail we should at least retain the SID. + expected = struct(SHARED_SYMBOL_TABLE + "a::{c:$99::1,e:3}"); + assertEquals(expected, actual); } diff --git a/test/software/amazon/ion/SymbolTest.java b/test/software/amazon/ion/SymbolTest.java index d968b721c3..853aafd3bb 100644 --- a/test/software/amazon/ion/SymbolTest.java +++ b/test/software/amazon/ion/SymbolTest.java @@ -165,7 +165,7 @@ public void testEmptySymbol() @Test public void testSyntheticSymbols() { - String symText = "$324"; + String symText = "$ion_symbol_table::{imports:[{name:\"foo\", version: 1, max_id: 315}]} $324"; IonSymbol value = (IonSymbol) oneValue(symText); checkUnknownSymbol(324, value); diff --git a/test/software/amazon/ion/SystemProcessingTestCase.java b/test/software/amazon/ion/SystemProcessingTestCase.java index 98fcc99f94..5ec1082e12 100644 --- a/test/software/amazon/ion/SystemProcessingTestCase.java +++ b/test/software/amazon/ion/SystemProcessingTestCase.java @@ -403,7 +403,7 @@ public void testLocalTableWithLesserImport() " max_id:" + Symtabs.FRED_MAX_IDS[2] + "}]," + "}\n" + "local1 local2 fred_1 fred_2 fred_3 $12 " + - "fred_3::$99 [{fred_3:local2}]"; + "fred_3::$10 [{fred_3:local2}]"; // TODO { $12:something } // TODO $12::something // Nesting flushed out a bug at one point @@ -450,7 +450,7 @@ public void testLocalTableWithLesserImport() // TODO checkAbsentSidLiteral("fred_3", fred3id); nextValue(); - checkSymbol(null, 99); + checkSymbol("fred_1", fred1id); checkMissingAnnotation("fred_3", fred3id); nextValue(); @@ -835,6 +835,7 @@ public void testLargeStructWithUnknownFieldNames() throws Exception { startIteration(ION_1_0 + + " $ion_symbol_table::{imports:[{name:\"foo\", version: 1, max_id: 10}]}" + " { $10:10, $11:11, $12:12, $13:13, $14:14," + " $15:15, $16:16, $17:17, $18:18, $19:19 }"); nextValue(); @@ -852,7 +853,7 @@ public void testLargeStructWithUnknownFieldNames() public void testUnknownFieldNames() throws Exception { - startIteration(ION_1_0 + " $10 { $11:$11, $12:$12 } "); + startIteration(ION_1_0 + " $ion_symbol_table::{imports:[{name:\"foo\", version: 1, max_id: 3}]} $10 { $11:$11, $12:$12 } "); nextValue(); checkSymbol(null, 10); @@ -877,7 +878,7 @@ public void testUnknownFieldNames() public void testUnknownAnnotations() throws Exception { - startIteration(ION_1_0 + " $10::$10 $11::[ $12::$12 ]"); + startIteration(ION_1_0 + " $ion_symbol_table::{imports:[{name:\"foo\", version: 1, max_id: 3}]} $10::$10 $11::[ $12::$12 ]"); nextValue(); checkSymbol(null, 10); @@ -1121,12 +1122,12 @@ public void testIvmWithAnnotationText() public void testIvmWithAnnotationSid() throws Exception { - startIteration("$99::" + ION_1_0 + " 123"); + startIteration("$0::" + ION_1_0 + " 123"); - // $99::$ion_1_0 is not an IVM, but an annotated user-value symbol + // $0::$ion_1_0 is not an IVM, but an annotated user-value symbol nextValue(); checkSymbol(ION_1_0); - checkAnnotation(null, 99); + checkAnnotation(null, 0); nextValue(); checkInt(123); diff --git a/test/software/amazon/ion/TestUtils.java b/test/software/amazon/ion/TestUtils.java index fe20f57d60..ba7cad7503 100644 --- a/test/software/amazon/ion/TestUtils.java +++ b/test/software/amazon/ion/TestUtils.java @@ -132,7 +132,6 @@ public boolean accept(File dir, String name) "bad/clobWithNullCharacter.ion" // TODO amzn/ion-java#43 , "bad/clobWithNonAsciiCharacter.ion" // TODO amzn/ion-java#99 , "bad/structOrderedEmpty.10n" // TODO amzn/ion-java#100 - , "bad/symbolIDUnmapped.ion" // TODO amzn/ion-java#101 , "bad/emptyAnnotatedInt.10n" // TODO amzn/ion-java#55 , "bad/timestamp/timestampNegativeFraction.10n" // TODO amzn/ion-java#102 , "bad/utf8/surrogate_5.ion" // TODO amzn/ion-java#60 @@ -147,6 +146,7 @@ public boolean accept(File dir, String name) , "good/whitespace.ion" // TODO amzn/ion-java#104 , "bad/negativeIntZero.10n" // TODO amzn/ion-java#138 , "bad/negativeIntZeroLn.10n" // TODO amzn/ion-java#138 + , "good/item1.10n" // TODO amzn/ion-java#126 (roundtrip symbols with unknown text) ); diff --git a/test/software/amazon/ion/TrBwBrProcessingTest.java b/test/software/amazon/ion/TrBwBrProcessingTest.java index 2c6c5d12f5..42d87d35ac 100644 --- a/test/software/amazon/ion/TrBwBrProcessingTest.java +++ b/test/software/amazon/ion/TrBwBrProcessingTest.java @@ -14,6 +14,8 @@ package software.amazon.ion; +import org.junit.Ignore; +import org.junit.Test; import software.amazon.ion.IonReader; /** @@ -31,4 +33,14 @@ protected void prepare(String text) IonReader textReader = system().newSystemReader(text); myBytes = writeBinaryBytes(textReader); } + + @Override + @Ignore + @Test + public void testLocalSymtabWithMalformedSymbolEntries() throws Exception { + // TODO amzn/ion-java#151 this test exercises null slots in the local symbol table. The reader should collapse + // all local symbol table null slots to $0. Currently, since this doesn't happen, the reader passes $10 to the + // writer, which fails due to an out-of-range symbol ID. + super.testLocalSymtabWithMalformedSymbolEntries(); + } } diff --git a/test/software/amazon/ion/impl/IonWriterTestCase.java b/test/software/amazon/ion/impl/IonWriterTestCase.java index 2c7e435489..6cf30dfbc3 100644 --- a/test/software/amazon/ion/impl/IonWriterTestCase.java +++ b/test/software/amazon/ion/impl/IonWriterTestCase.java @@ -56,6 +56,7 @@ import software.amazon.ion.SymbolToken; import software.amazon.ion.SystemSymbols; import software.amazon.ion.TestUtils; +import software.amazon.ion.UnknownSymbolException; import software.amazon.ion.impl.PrivateIonWriter; import software.amazon.ion.junit.IonAssert; @@ -229,7 +230,7 @@ public void testWritingFieldName() iw = makeWriter(); iw.stepIn(IonType.STRUCT); iw.setFieldName("foo"); - iw.setFieldNameSymbol(newSymbolToken((String) null, 99)); // Replaces "foo" + iw.setFieldNameSymbol(newSymbolToken((String) null, 0)); // Replaces "foo" iw.writeNull(); iw.stepOut(); @@ -237,7 +238,7 @@ public void testWritingFieldName() r.next(); r.stepIn(); r.next(); - check(r).fieldName(null, 99); + check(r).fieldName(null, 0); } @Test @@ -322,11 +323,27 @@ public void testWritingUnknownSymbol() throws Exception { iw = makeWriter(); + thrown.expect(UnknownSymbolException.class); iw.writeSymbolToken(newSymbolToken((String) null, 99)); + } - IonReader in = reread(); - in.next(); - IonAssert.checkSymbol(in, null, 99); + @Test + public void testWritingUnknownFieldName() + throws Exception + { + iw = makeWriter(); + iw.stepIn(IonType.STRUCT); + thrown.expect(UnknownSymbolException.class); + iw.setFieldNameSymbol(newSymbolToken((String) null, 99)); + } + + @Test + public void testWritingUnknownAnnotation() + throws Exception + { + iw = makeWriter(); + thrown.expect(UnknownSymbolException.class); + iw.setTypeAnnotationSymbols(newSymbolToken((String) null, 99)); } @Test @@ -728,10 +745,10 @@ public void testWritingAnnotationWithUnknownText() iw = makeWriter(); IonDatagram expected = system().newDatagram(); - iw.setTypeAnnotationSymbols(newSymbolToken((String) null, 99)); + iw.setTypeAnnotationSymbols(newSymbolToken((String) null, 0)); iw.writeNull(); // expected: the type annotation is written IonValue v = expected.add().newNull(); - v.setTypeAnnotationSymbols(newSymbolToken(99)); + v.setTypeAnnotationSymbols(newSymbolToken(0)); assertEquals(expected, reload()); } @@ -816,7 +833,7 @@ public void testFinishDoesReset() checkSymbol(SystemSymbols.ION_1_0, it.next()); } checkAnnotation(SystemSymbols.ION_SYMBOL_TABLE, it.next()); - // TODO amzn/ion-java#63 + // TODO amzn/ion-java#14 if (myOutputForm != OutputForm.TEXT) { checkSymbol(null, 12, it.next()); @@ -994,7 +1011,7 @@ public void testWriteIVMImplicitly() if (myOutputForm == OutputForm.BINARY) { checkAnnotation(ION_SYMBOL_TABLE, it.next()); } - // TODO amzn/ion-java#63 + // TODO amzn/ion-java#14 if (myOutputForm == OutputForm.BINARY) { checkSymbol(null, 10, it.next()); @@ -1038,7 +1055,7 @@ public void testWritingDatagram() if (myOutputForm != OutputForm.TEXT) { checkAnnotation(SystemSymbols.ION_SYMBOL_TABLE, it.next()); } - // TODO amzn/ion-java#63 + // TODO amzn/ion-java#14 if (myOutputForm != OutputForm.TEXT) { checkSymbol(null, 10, it.next()); diff --git a/test/software/amazon/ion/streaming/SeekableReaderTest.java b/test/software/amazon/ion/streaming/SeekableReaderTest.java index 4b4b289b63..1c73e42b8d 100644 --- a/test/software/amazon/ion/streaming/SeekableReaderTest.java +++ b/test/software/amazon/ion/streaming/SeekableReaderTest.java @@ -112,7 +112,7 @@ public void testWalkingBackwards() @Test public void testHoistingWithinContainers() { - read("{f:v,g:[c, (d), e], /* h */ $99:null} s"); + read("{f:v,g:[c, (d), e], /* h */ $0:null} s"); in.next(); in.stepIn(); From 1a0af9e478004c9473774197fff7b0b157e42dab Mon Sep 17 00:00:00 2001 From: wesboyt <30701972+wesboyt@users.noreply.github.com> Date: Mon, 17 Sep 2018 12:09:55 -0700 Subject: [PATCH 009/490] IonStructLite bugfixes (#155) * Updated ionstructlite so fieldmap will not always result as null on construction, added a boolean hasNullField to throw correctly when a fieldname does not resolve and correctness cannot be determined. --- .../amazon/ion/UnknownSymbolException.java | 13 ++++++++++++- .../amazon/ion/impl/lite/IonStructLite.java | 19 +++++++------------ test/software/amazon/ion/StructTest.java | 17 +++++++++++++++++ 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/software/amazon/ion/UnknownSymbolException.java b/src/software/amazon/ion/UnknownSymbolException.java index becfecd654..e3cba54630 100644 --- a/src/software/amazon/ion/UnknownSymbolException.java +++ b/src/software/amazon/ion/UnknownSymbolException.java @@ -29,10 +29,17 @@ public class UnknownSymbolException private static final long serialVersionUID = 1L; private final int mySid; + private final String myText; public UnknownSymbolException(int sid) { mySid = sid; + myText = null; + } + public UnknownSymbolException(String message) + { + myText = message; + mySid = 0; } public int getSid() @@ -43,6 +50,10 @@ public int getSid() @Override public String getMessage() { - return "Unknown symbol text for $" + mySid; + if(myText == null) { + return "Unknown symbol text for $" + mySid; + } else { + return myText; + } } } diff --git a/src/software/amazon/ion/impl/lite/IonStructLite.java b/src/software/amazon/ion/impl/lite/IonStructLite.java index 9358b273d8..8d7cb0d814 100644 --- a/src/software/amazon/ion/impl/lite/IonStructLite.java +++ b/src/software/amazon/ion/impl/lite/IonStructLite.java @@ -34,6 +34,7 @@ import software.amazon.ion.impl.PrivateCurriedValueFactory; import software.amazon.ion.util.Equivalence; import java.util.Set; +import software.amazon.ion.UnknownSymbolException; final class IonStructLite extends IonContainerLite @@ -41,7 +42,6 @@ final class IonStructLite { private static final int HASH_SIGNATURE = IonType.STRUCT.toString().hashCode(); - // TODO amzn/ion-java#41: add support for _isOrdered IonStructLite(ContainerlessContext context, boolean isNull) @@ -54,14 +54,13 @@ private IonStructLite(IonStructLite existing, IonContext context) super(existing, context, true); // field map can be shallow cloned due to it dealing with String and Integer // values - both of which are immutable constructs and so safe to retain as references - this._field_map = null == _field_map - ? null - : new HashMap(existing._field_map); + this._field_map = null == existing._field_map ? null : new HashMap(existing._field_map); this._field_map_duplicate_count = existing._field_map_duplicate_count; + this.hasNullFieldName = existing.hasNullFieldName; } private Map _field_map; - + private boolean hasNullFieldName = false; public int _field_map_duplicate_count; @@ -402,9 +401,9 @@ public IonValue get(String fieldName) IonValue field; if (field_idx < 0) { + if(hasNullFieldName) throw new UnknownSymbolException("Unable to determine whether the field exists because the struct contains field names with unknown text."); field = null; - } - else { + } else { field = get_child(field_idx); } @@ -450,11 +449,6 @@ public boolean add(IonValue child) { // TODO validate in struct.setFieldName String text = child.getFieldNameSymbol().getText(); - if (text != null) - { - validateFieldName(text); - } - IonValueLite concrete = (IonValueLite) child; _add(text, concrete); @@ -483,6 +477,7 @@ protected void handle(IonValue newValue) */ private void _add(String fieldName, IonValueLite child) { + hasNullFieldName |= fieldName == null; int size = get_child_count(); // add this to the Container child collection diff --git a/test/software/amazon/ion/StructTest.java b/test/software/amazon/ion/StructTest.java index 15255dbdb2..8a06144145 100644 --- a/test/software/amazon/ion/StructTest.java +++ b/test/software/amazon/ion/StructTest.java @@ -22,6 +22,8 @@ import java.util.Iterator; import java.util.ListIterator; import java.util.Random; + +import com.sun.org.apache.bcel.internal.classfile.Unknown; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -738,6 +740,21 @@ public void testMakeNullRemovesChildsContainer() val.getContainer()); } + @Test + public void testUnknownSymbolException() + { + String input = "$ion_symbol_table::{imports:[{name:\"foo\", version:1, max_id:1}]} { $10: \"Unknown symbol\"}"; + IonDatagram dg = loader().load(input); + IonStruct val = (IonStruct) dg.get(0); + try { + val.get("z"); + fail("Should've thrown UnknownSymbolException"); + } catch (UnknownSymbolException e) { } + try { + val = system().clone(val); + } catch (UnknownSymbolException e) { } + } + @Test public void testRemoveAfterClone() { From 7243a57125d341477cc2e82bf84769457cd84172 Mon Sep 17 00:00:00 2001 From: Boyett Date: Mon, 17 Sep 2018 12:16:03 -0700 Subject: [PATCH 010/490] Removed extraneous import. --- test/software/amazon/ion/StructTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/software/amazon/ion/StructTest.java b/test/software/amazon/ion/StructTest.java index 8a06144145..7c1210d33b 100644 --- a/test/software/amazon/ion/StructTest.java +++ b/test/software/amazon/ion/StructTest.java @@ -22,8 +22,6 @@ import java.util.Iterator; import java.util.ListIterator; import java.util.Random; - -import com.sun.org.apache.bcel.internal.classfile.Unknown; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; From 7228e3cf343a67ad64c04a7e9aaf94b7c60baebc Mon Sep 17 00:00:00 2001 From: wesboyt <30701972+wesboyt@users.noreply.github.com> Date: Wed, 26 Sep 2018 16:24:21 -0700 Subject: [PATCH 011/490] Updated number exceptions to indicate what the source of the error was. (#157) Updated binary/hexadecimal parse exceptions to indicate what the source of the error was. --- .../amazon/ion/impl/IonReaderTextSystemX.java | 23 ++++++------------- test/software/amazon/ion/IntTest.java | 5 +++- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/software/amazon/ion/impl/IonReaderTextSystemX.java b/src/software/amazon/ion/impl/IonReaderTextSystemX.java index 459a067f10..55b5a849de 100644 --- a/src/software/amazon/ion/impl/IonReaderTextSystemX.java +++ b/src/software/amazon/ion/impl/IonReaderTextSystemX.java @@ -41,6 +41,7 @@ import software.amazon.ion.impl.IonTokenConstsX.CharacterSequence; import software.amazon.ion.impl.PrivateScalarConversions.AS_TYPE; import software.amazon.ion.impl.PrivateScalarConversions.CantConvertException; +import java.lang.Character; /** * This reader calls the {@link IonReaderTextRawX} for low level events. @@ -235,27 +236,17 @@ private final void load_scalar_value() throws IOException { break; } } - } - else if (token_type == IonTokenConstsX.TOKEN_HEX) { - boolean is_negative = (cs.charAt(0) == '-'); + } else if (token_type == IonTokenConstsX.TOKEN_HEX || token_type == IonTokenConstsX.TOKEN_BINARY) { + boolean isNegative = (cs.charAt(0) == '-'); // prefix = is_negative ? "-0x" : "0x"; - int pos; - if (is_negative) { - pos = 1; - } - else { - pos = 0; + int pos = isNegative ? 1 : 0; + char caseChar = token_type == IonTokenConstsX.TOKEN_HEX ? 'x' : 'b'; + if (cs.length() <= (isNegative ? 3 : 2) || Character.toLowerCase(cs.charAt(pos + 1)) != caseChar) { + parse_error("Invalid " + (caseChar == 'x' ? "hexadecimal" : "binary") + " int value."); } - assert((cs.length() > 2) && (cs.charAt(pos) == '0') && (cs.charAt(pos+1) == 'x' || cs.charAt(pos+1) == 'X')); cs.deleteCharAt(pos); cs.deleteCharAt(pos); } - else if (token_type == IonTokenConstsX.TOKEN_BINARY) { - boolean isNegative = (cs.charAt(0) == '-'); - int position = isNegative ? 1 : 0; - cs.deleteCharAt(position); - cs.deleteCharAt(position); - } int len = cs.length(); diff --git a/test/software/amazon/ion/IntTest.java b/test/software/amazon/ion/IntTest.java index 466952c431..50d18a85bd 100644 --- a/test/software/amazon/ion/IntTest.java +++ b/test/software/amazon/ion/IntTest.java @@ -132,7 +132,6 @@ public void testInts() assertEquals(Integer.MIN_VALUE, value.intValue()); } - @Test public void testPositiveSign() { @@ -243,6 +242,8 @@ public void testHexadecimal() { checkInt(-3, oneValue("-0x3")); checkInt(-3, oneValue("-0x0003")); + badValue("0x"); + badValue("-0x"); } @Test @@ -293,6 +294,8 @@ public void testReadOnlyInt() public void testBinaryInt() { assertEquals(system().newInt(4), oneValue("0b0100")); + badValue("-0b"); + badValue("0b"); } @Test From 88d5d45c8532f34fa7195e5ba8d205de28bd6e04 Mon Sep 17 00:00:00 2001 From: David Murray Date: Wed, 3 Oct 2018 04:52:26 +1000 Subject: [PATCH 012/490] Fixes for compile errors when using Java 11 (#158) * Disambiguate calls to IonSequence.toArray(null) * Remove test dependency on javax.xml.bind.DatatypeConverter --- .../software/amazon/ion/SequenceTestCase.java | 2 +- .../amazon/ion/TrueSequenceTestCase.java | 2 +- test/software/amazon/ion/impl/VarIntTest.java | 33 +++++++++++++++++-- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/test/software/amazon/ion/SequenceTestCase.java b/test/software/amazon/ion/SequenceTestCase.java index 1412fe2c9b..c30e12520d 100644 --- a/test/software/amazon/ion/SequenceTestCase.java +++ b/test/software/amazon/ion/SequenceTestCase.java @@ -679,7 +679,7 @@ public void testToArray() try { - seq.toArray(null); + seq.toArray((Object[]) null); fail("expected exception"); } catch (NullPointerException e) { } diff --git a/test/software/amazon/ion/TrueSequenceTestCase.java b/test/software/amazon/ion/TrueSequenceTestCase.java index 097b952729..b9d1e0bb63 100644 --- a/test/software/amazon/ion/TrueSequenceTestCase.java +++ b/test/software/amazon/ion/TrueSequenceTestCase.java @@ -489,7 +489,7 @@ public void testNullSequenceToArray() try { - seq.toArray(null); + seq.toArray((Object[]) null); fail("expected exception"); } catch (NullPointerException e) { } diff --git a/test/software/amazon/ion/impl/VarIntTest.java b/test/software/amazon/ion/impl/VarIntTest.java index ae0e856cd1..d5ab77d912 100644 --- a/test/software/amazon/ion/impl/VarIntTest.java +++ b/test/software/amazon/ion/impl/VarIntTest.java @@ -5,7 +5,6 @@ import software.amazon.ion.IonTestCase; import software.amazon.ion.system.SimpleCatalog; -import javax.xml.bind.DatatypeConverter; import java.io.ByteArrayInputStream; public class VarIntTest extends IonTestCase { @@ -86,11 +85,41 @@ public void readVarIntegerNegativeZero() throws Exception { } private IonReaderBinaryUserX makeReader(String hex) throws Exception { - ByteArrayInputStream input = new ByteArrayInputStream(DatatypeConverter.parseHexBinary("E00100EA" + hex)); + ByteArrayInputStream input = new ByteArrayInputStream(parseHexBinary("E00100EA" + hex)); UnifiedInputStreamX uis = UnifiedInputStreamX.makeStream(input); uis.skip(4); return new IonReaderBinaryUserX(new SimpleCatalog(), LocalSymbolTable.DEFAULT_LST_FACTORY, uis, 0); } + + private static byte[] parseHexBinary(String str) { + int len = str.length(); + if (len % 2 != 0) { + throw new IllegalArgumentException("str must have even length"); + } + + byte[] result = new byte[len / 2]; + + for (int i = 0; i < len; i += 2) { + int high = parseHexChar(str.charAt(i)); + int low = parseHexChar(str.charAt(i + 1)); + result[i / 2] = (byte) ((high << 4) | low); + } + + return result; + } + + private static int parseHexChar(char c) { + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return (c - 'a') + 10; + } + if (c >= 'A' && c <= 'F') { + return (c - 'A') + 10; + } + throw new IllegalArgumentException("invalid hex character: " + c); + } } \ No newline at end of file From c4750dfbb6072e467919365636af8ca31d0f9f59 Mon Sep 17 00:00:00 2001 From: David Lurton Date: Fri, 12 Oct 2018 13:20:46 -0700 Subject: [PATCH 013/490] Add additonal JDKs to .travis.yml, issue 164 (#166) * Add additional JDKs to .travis.yml (#164) --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.travis.yml b/.travis.yml index 9bcf99945d..0f6968597c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,10 @@ language: java jdk: + - openjdk8 + - openjdk9 + - openjdk10 + - openjdk11 - oraclejdk8 + - oraclejdk9 + - oraclejdk10 + - oraclejdk11 From db930a27f0f37ba3cc0a8cfe5cc172503cefcddd Mon Sep 17 00:00:00 2001 From: David Lurton Date: Fri, 12 Oct 2018 13:55:18 -0700 Subject: [PATCH 014/490] Implements workaround for java.util.Date bug described in issue 160 (#162) --- src/software/amazon/ion/Timestamp.java | 20 ++++++-- test/software/amazon/ion/TimestampTest.java | 52 +++++++++++++++++++++ 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/src/software/amazon/ion/Timestamp.java b/src/software/amazon/ion/Timestamp.java index bd036ae97e..645815488f 100644 --- a/src/software/amazon/ion/Timestamp.java +++ b/src/software/amazon/ion/Timestamp.java @@ -23,6 +23,7 @@ import java.util.Calendar; import java.util.Date; import java.util.GregorianCalendar; +import java.util.TimeZone; import software.amazon.ion.impl.PrivateUtils; import software.amazon.ion.util.IonTextUtils; @@ -276,14 +277,27 @@ private void apply_offset(int offset) @SuppressWarnings("deprecation") private void set_fields_from_millis(long millis) { - if(millis < MINIMUM_ALLOWED_TIMESTAMP_IN_MILLIS){ + if(millis < MINIMUM_ALLOWED_TIMESTAMP_IN_MILLIS) { throw new IllegalArgumentException("year is less than 1"); } Date date = new Date(millis); - // These fields are in the system timezone! - this._year = checkAndCastYear(date.getYear() + 1900); + // https://github.com/amzn/ion-java/issues/160 + // The java.util.Date(long) constructor expects an epoch time in milliseconds, and getYear(), getMonth(), + // getHour() on the resulting Date are supposed to return values adjusted to the default timezone. + // In Pacific Standard Time (offset -08:00), for a Date constructed with an epoch time equivalent to + // 0001-01-01T00:00:00.000Z, the Date.get*() methods should return values for + // 0000-12-31T16:00:00.000Z; however, Date.getYear() incorrectly returns a value for year 1 (-1899) + // in this scenario. The following if/else block compensates for this bug: + int currentRawOffset = TimeZone.getDefault().getRawOffset(); + if(currentRawOffset < 0 && MINIMUM_ALLOWED_TIMESTAMP_IN_MILLIS - currentRawOffset > millis) { + this._year = 0; + } else { + this._year = checkAndCastYear(date.getYear() + 1900); + } + + // Note: date.get*() return values are in the local timezone! this._month = checkAndCastMonth(date.getMonth() + 1); // calendar months are 0 based, timestamp months are 1 based this._day = checkAndCastDay(date.getDate(), _year, _month); this._hour = checkAndCastHour(date.getHours()); diff --git a/test/software/amazon/ion/TimestampTest.java b/test/software/amazon/ion/TimestampTest.java index ec55c0933a..c0017c7569 100644 --- a/test/software/amazon/ion/TimestampTest.java +++ b/test/software/amazon/ion/TimestampTest.java @@ -860,6 +860,58 @@ public void testNewTimestampFromLong() assertEquals("2010-02-01T10:11:12.340Z", ts.toZString()); } + /** + * Regression test for https://github.com/amzn/ion-java/issues/160 + */ + @Test + public void testNewTimestampFromYearOneRegressionBug() + { + // This is the same as the private field Timestamp.MINIMUM_ALLOWED_TIMESTAMP_IN_MILLIS + final long MINIMUM_TIMESTAMP_MILLIS = -62135769600000L; + // This is the minimum timestamp instantiating by parsing a timestamp string, + // which is not effected by the `java.util.Date` bug. + final Timestamp MINIMUM_TIMESTAMP = Timestamp.valueOf("0001-01-01T00:00:00.000Z"); + // Save the original timezone since we modify it below. + final TimeZone originalTimeZone = TimeZone.getDefault(); + + try { + // The `java.util.Date` bug has a different range of times depending on the local offset, so + // let's test against all time zones. + for (final String tzId : TimeZone.getAvailableIDs()) { + final TimeZone tz = TimeZone.getTimeZone(tzId); + TimeZone.setDefault(tz); + + // The Timestamps under test must be instantiated *after* the the call to TimeZone.setDefault() + // since `java.util.Date` references the default TimeZone when calculating the values for its + // date field accessor methods (i.e. Date.get*()). + Timestamp minimumTimestampFromMillis = Timestamp.forMillis(MINIMUM_TIMESTAMP_MILLIS, 0); + assertEquals(MINIMUM_TIMESTAMP, minimumTimestampFromMillis); + + // Only perform further assertions on timezones with negative offsets because positive + // offsets create millisecond values that are less than Timestamp.MINIMUM_ALLOWED_TIMESTAMP_IN_MILLIS + if (tz.getRawOffset() < 0) { + // This will be the latest millisecond in which the bug with `java.util.Date` can happen + Timestamp maximumTimestampWithBug = + Timestamp.forMillis(MINIMUM_TIMESTAMP_MILLIS - tz.getRawOffset(), 0); + assertEquals(1, maximumTimestampWithBug.getYear()); + + // Test one millisecond after, just to be sure. + Timestamp timestampPlusOne = + Timestamp.forMillis(MINIMUM_TIMESTAMP_MILLIS - tz.getRawOffset() + 1, 0); + assertEquals(1, timestampPlusOne.getYear()); + + // Also test one milliscond before, just to be sure. + Timestamp timestampMinusOne = + Timestamp.forMillis(MINIMUM_TIMESTAMP_MILLIS - tz.getRawOffset() - 1L, 0); + assertEquals(1, timestampMinusOne.getYear()); + } + } + } finally { + // Have to set the TimeZone back to its original value so as not to effect other tests. + TimeZone.setDefault(originalTimeZone); + } + } + /** * Test for {@link Timestamp#createFromUtcFields(Precision, int, int, int, int, int, int, BigDecimal, Integer)} * ensuring that varying precisions produce Timestamps as expected as per From c653aad103e87b8be9828cbce368ac3f5aec4a65 Mon Sep 17 00:00:00 2001 From: Fernando Raganhan Barbosa Date: Fri, 19 Oct 2018 13:35:02 -0700 Subject: [PATCH 015/490] Removes oraclejdk10 from build list (#169) Oracle JDK 10 was deprecated and fails travis builds example: https://travis-ci.org/amzn/ion-java/jobs/442398169 --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0f6968597c..cc409f79fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,5 +6,4 @@ jdk: - openjdk11 - oraclejdk8 - oraclejdk9 - - oraclejdk10 - oraclejdk11 From cc146d6e672f8ad0146c1dd1a4bced50bd3640bc Mon Sep 17 00:00:00 2001 From: Hen Sapir Date: Fri, 28 Sep 2018 15:46:18 -0700 Subject: [PATCH 016/490] Fix javadoc for some methods of Timestamp Some Timestamp methods have incorrect javadocs due to copy/pasting. This change fixes those javadocs. --- src/software/amazon/ion/Timestamp.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/software/amazon/ion/Timestamp.java b/src/software/amazon/ion/Timestamp.java index 645815488f..89b14f0b99 100644 --- a/src/software/amazon/ion/Timestamp.java +++ b/src/software/amazon/ion/Timestamp.java @@ -2025,7 +2025,7 @@ private static void print_fractional_digits(Appendable out, BigDecimal value) * Returns a timestamp relative to this one by the given number of * milliseconds. * - * @param amount a (positive or negative) number of milliseconds. + * @param amount a number of milliseconds. */ public final Timestamp addMillis(long amount) { @@ -2052,7 +2052,7 @@ public final Timestamp addMillis(long amount) /** * Returns a timestamp relative to this one by the given number of seconds. * - * @param amount a (positive or negative) number of seconds. + * @param amount a number of seconds. */ public final Timestamp addSecond(int amount) { @@ -2064,7 +2064,7 @@ public final Timestamp addSecond(int amount) /** * Returns a timestamp relative to this one by the given number of minutes. * - * @param amount a (positive or negative) number of minutes. + * @param amount a number of minutes. */ public final Timestamp addMinute(int amount) { @@ -2076,7 +2076,7 @@ public final Timestamp addMinute(int amount) /** * Returns a timestamp relative to this one by the given number of hours. * - * @param amount a (positive or negative) number of hours. + * @param amount a number of hours. */ public final Timestamp addHour(int amount) { @@ -2088,7 +2088,7 @@ public final Timestamp addHour(int amount) /** * Returns a timestamp relative to this one by the given number of days. * - * @param amount a (positive or negative) number of hours. + * @param amount a number of days. */ public final Timestamp addDay(int amount) { @@ -2107,7 +2107,7 @@ public final Timestamp addDay(int amount) * leap days. For example, adding one month to {@code 2011-01-31} * results in {@code 2011-02-28}. * - * @param amount a (positive or negative) number of hours. + * @param amount a number of months. */ public final Timestamp addMonth(int amount) { @@ -2124,7 +2124,7 @@ public final Timestamp addMonth(int amount) * The day field may be adjusted to account for leap days. For example, * adding one year to {@code 2012-02-29} results in {@code 2013-02-28}. * - * @param amount a (positive or negative) number of hours. + * @param amount a number of years. */ public final Timestamp addYear(int amount) { From 173877e7794b9e17795c22de577339401fefe5d0 Mon Sep 17 00:00:00 2001 From: Fernando Raganhan Barbosa Date: Mon, 22 Oct 2018 16:50:17 -0700 Subject: [PATCH 017/490] Bound checking BigDecimal in Timestamp to guard against DoS (#161) * Bound checking BigDecimal in Timestamp to guard against DoS Operations that require inflating the BigDecimal can be expensive for large exponents, examples: * longValue * intValue * setScale https://github.com/amzn/ion-java/issues/159 --- src/software/amazon/ion/Timestamp.java | 47 ++++++++++---- test/software/amazon/ion/TimestampTest.java | 70 ++++++++++++++++++--- 2 files changed, 97 insertions(+), 20 deletions(-) diff --git a/src/software/amazon/ion/Timestamp.java b/src/software/amazon/ion/Timestamp.java index 89b14f0b99..1d668bb97c 100644 --- a/src/software/amazon/ion/Timestamp.java +++ b/src/software/amazon/ion/Timestamp.java @@ -84,6 +84,15 @@ public final class Timestamp private static final int NO_SECONDS = 0; private static final BigDecimal NO_FRACTIONAL_SECONDS = null; + // 0001-01-01T00:00:00.0Z in millis + private static final long MINIMUM_ALLOWED_TIMESTAMP_IN_MILLIS = -62135769600000L; + + // 0001-01-01T00:00:00.0Z in millis + static final BigDecimal MINIMUM_ALLOWED_FRACTIONAL_MILLIS = new BigDecimal(MINIMUM_ALLOWED_TIMESTAMP_IN_MILLIS); + + // 10000T in millis, upper bound exclusive + static final BigDecimal FRACTIONAL_MILLIS_UPPER_BOUND = new BigDecimal(253402300800000L); + /** * Unknown local offset from UTC. */ @@ -266,9 +275,6 @@ private void apply_offset(int offset) } } - // 0001-01-01T00:00:00.0Z in millis - private static final long MINIMUM_ALLOWED_TIMESTAMP_IN_MILLIS = -62135769600000L; - /** * This method uses deprecated methods from {@link java.util.Date} * instead of {@link Calendar} so that this code can be used (more easily) @@ -704,7 +710,22 @@ private Timestamp(BigDecimal millis, Integer localOffset) { if (millis == null) throw new NullPointerException("millis is null"); - long ms = millis.longValue(); + // check bounds to avoid hanging when calling longValue() on decimals with large positive exponents, + // e.g. 1e10000000 + if(millis.compareTo(MINIMUM_ALLOWED_FRACTIONAL_MILLIS) < 0 || + FRACTIONAL_MILLIS_UPPER_BOUND.compareTo(millis) < 0) { + throw new IllegalArgumentException("millis: " + millis + " is outside of valid the range: from " + + MINIMUM_ALLOWED_FRACTIONAL_MILLIS + + " (0001T)" + + ", inclusive, to " + + FRACTIONAL_MILLIS_UPPER_BOUND + + " (10000T)" + + " , exclusive"); + } + + // quick handle integral zero + long ms = isIntegralZero(millis) ? 0 : millis.longValue(); + set_fields_from_millis(ms); this._precision = Precision.SECOND; @@ -714,12 +735,19 @@ private Timestamp(BigDecimal millis, Integer localOffset) } else { BigDecimal secs = millis.movePointLeft(3); - BigDecimal secsDown = secs.setScale(0, RoundingMode.FLOOR); + BigDecimal secsDown = isIntegralZero(secs) ? BigDecimal.ZERO : secs.setScale(0, BigDecimal.ROUND_FLOOR); this._fraction = secs.subtract(secsDown); } this._offset = localOffset; } + private boolean isIntegralZero(final BigDecimal decimal) { + // zero || no low-order bits || < 1.0 + return decimal.signum() == 0 + || decimal.scale() < -63 + || (decimal.precision() - decimal.scale() <= 0); + } + /** * Creates a new Timestamp that represents the point in time that is * {@code millis} milliseconds from the epoch, with a given local offset. @@ -1418,7 +1446,8 @@ public long getMillis() // month is 0 based for Date long millis = Date.UTC(this._year - 1900, this._month - 1, this._day, this._hour, this._minute, this._second); if (this._fraction != null) { - int frac = this._fraction.movePointRight(3).intValue(); + BigDecimal fracAsDecimal = this._fraction.movePointRight(3); + int frac = isIntegralZero(fracAsDecimal) ? 0 : fracAsDecimal.intValue(); millis += frac; } return millis; @@ -1816,7 +1845,6 @@ public String toString() return buffer.toString(); } - /** * Returns the string representation (in Ion format) of this Timestamp * in UTC. @@ -1834,12 +1862,11 @@ public String toZString() catch (IOException e) { throw new RuntimeException("Exception printing to StringBuilder", - e); + e); } return buffer.toString(); } - /** * Prints to an {@code Appendable} the string representation (in Ion format) * of this Timestamp in its local time. @@ -1868,7 +1895,6 @@ public void print(Appendable out) print(out, adjusted); } - /** * Prints to an {@code Appendable} the string representation (in Ion format) * of this Timestamp in UTC. @@ -1906,7 +1932,6 @@ public void printZ(Appendable out) } } - /** * helper for print(out) and printZ(out) so that printZ can create * a zulu time and pass it directly and print can apply the local diff --git a/test/software/amazon/ion/TimestampTest.java b/test/software/amazon/ion/TimestampTest.java index c0017c7569..28bffb87f7 100644 --- a/test/software/amazon/ion/TimestampTest.java +++ b/test/software/amazon/ion/TimestampTest.java @@ -16,6 +16,8 @@ import static software.amazon.ion.Decimal.NEGATIVE_ZERO; import static software.amazon.ion.Decimal.negativeZero; +import static software.amazon.ion.Timestamp.FRACTIONAL_MILLIS_UPPER_BOUND; +import static software.amazon.ion.Timestamp.MINIMUM_ALLOWED_FRACTIONAL_MILLIS; import static software.amazon.ion.Timestamp.UNKNOWN_OFFSET; import static software.amazon.ion.Timestamp.UTC_OFFSET; import static software.amazon.ion.Timestamp.createFromUtcFields; @@ -26,19 +28,17 @@ import static software.amazon.ion.Timestamp.Precision.YEAR; import static software.amazon.ion.impl.PrivateUtils.UTC; +import java.io.IOException; import java.math.BigDecimal; +import java.sql.Time; import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.time.Instant; import java.util.Calendar; import java.util.Date; import java.util.TimeZone; +import org.junit.Ignore; import org.junit.Test; -import software.amazon.ion.Decimal; -import software.amazon.ion.IonTimestamp; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.NullValueException; -import software.amazon.ion.Timestamp; import software.amazon.ion.Timestamp.Precision; /** @@ -781,12 +781,47 @@ public void testNewTimestampFromBigDecimal() assertEquals("2012-01-01T12:12:30.555123Z", ts.toZString()); } + @Test + @Ignore // see https://github.com/amzn/ion-java/issues/160 + public void testNewTimestampFromMinimumAllowedMillis() + { + Timestamp ts = Timestamp.forMillis(MINIMUM_ALLOWED_FRACTIONAL_MILLIS, PST_OFFSET); + assertEquals("0001-01-01T00:00:00.000Z", ts.toZString()); + } + + @Test + public void testNewTimestampFromBigDecimalWithMaximumAllowedMillis() + { + Timestamp ts = Timestamp.forMillis(FRACTIONAL_MILLIS_UPPER_BOUND.add(BigDecimal.ONE.negate()), PST_OFFSET); + checkFields(9999, 12, 31, 15, 59, 59, new BigDecimal("0.999"), PST_OFFSET, SECOND, ts); + assertEquals("9999-12-31T15:59:59.999-08:00", ts.toString()); + assertEquals("9999-12-31T23:59:59.999Z", ts.toZString()); + } - @SuppressWarnings("unused") - @Test (expected = NullPointerException.class) + @Test(expected = NullPointerException.class) public void testNewTimestampFromBigDecimalWithNull() { - Timestamp ts = Timestamp.forMillis(null, PST_OFFSET); + Timestamp.forMillis(null, PST_OFFSET); + } + + @Test(expected = IllegalArgumentException.class) + public void testNewTimestampFromBigDecimalWithMillisTooSmall() + { + // MIN - 1 + Timestamp.forMillis(MINIMUM_ALLOWED_FRACTIONAL_MILLIS.add(BigDecimal.ONE.negate()), PST_OFFSET); + } + + @Test(expected = IllegalArgumentException.class) + public void testNewTimestampFromBigDecimalWithMillisTooBig() + { + // Max + Timestamp.forMillis(FRACTIONAL_MILLIS_UPPER_BOUND, PST_OFFSET); + } + + @Test(expected = IllegalArgumentException.class) + public void testNewTimestampFromBigDecimalWithScaleTooBigPositive() + { + Timestamp.forMillis(new BigDecimal("1e100000"), PST_OFFSET); } /** @@ -2307,4 +2342,21 @@ public void forCalendarRejectsLargeYear() Timestamp t = Timestamp.forCalendar(cal); assertEquals(year, t.getYear()); } + + // High scale timeout tests + + // max scale permitted by BigDecimal from the String constructor + private static BigDecimal LARGE_SCALE_DECIMAL = new BigDecimal("1e-1000000000"); + + @Test(timeout = 50L) + public void testForMillisWithLargeScaleBigDecimal() + { + Timestamp ts = Timestamp.forMillis(LARGE_SCALE_DECIMAL, PST_OFFSET); + } + + @Test(timeout = 50L) + public void testGetMillisWithLargeScaleBigDecimal() + { + Timestamp.forMillis(LARGE_SCALE_DECIMAL, PST_OFFSET).getMillis(); + } } From 7d6156525968cac7bbc965f0db98cf6983ed91a7 Mon Sep 17 00:00:00 2001 From: barbosf Date: Tue, 23 Oct 2018 18:50:18 -0700 Subject: [PATCH 018/490] Fixing fraction calculation logic Correctly floors negative big decimals with a fast path for numbers close to zero Was broken by the DoS prevention fix. --- src/software/amazon/ion/Timestamp.java | 8 +++++++- test/software/amazon/ion/TimestampTest.java | 7 +++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/software/amazon/ion/Timestamp.java b/src/software/amazon/ion/Timestamp.java index 1d668bb97c..cff8412eac 100644 --- a/src/software/amazon/ion/Timestamp.java +++ b/src/software/amazon/ion/Timestamp.java @@ -735,12 +735,18 @@ private Timestamp(BigDecimal millis, Integer localOffset) } else { BigDecimal secs = millis.movePointLeft(3); - BigDecimal secsDown = isIntegralZero(secs) ? BigDecimal.ZERO : secs.setScale(0, BigDecimal.ROUND_FLOOR); + BigDecimal secsDown = fastRoundZeroFloor(secs); this._fraction = secs.subtract(secsDown); } this._offset = localOffset; } + private BigDecimal fastRoundZeroFloor(final BigDecimal decimal) { + BigDecimal fastValue = decimal.signum() < 0 ? BigDecimal.ONE.negate() : BigDecimal.ZERO; + + return isIntegralZero(decimal) ? fastValue : decimal.setScale(0, RoundingMode.FLOOR); + } + private boolean isIntegralZero(final BigDecimal decimal) { // zero || no low-order bits || < 1.0 return decimal.signum() == 0 diff --git a/test/software/amazon/ion/TimestampTest.java b/test/software/amazon/ion/TimestampTest.java index 28bffb87f7..5ca81e707b 100644 --- a/test/software/amazon/ion/TimestampTest.java +++ b/test/software/amazon/ion/TimestampTest.java @@ -781,6 +781,13 @@ public void testNewTimestampFromBigDecimal() assertEquals("2012-01-01T12:12:30.555123Z", ts.toZString()); } + @Test + public void testForMillisWithNegativeMilli() + { + Timestamp ts = Timestamp.forMillis(BigDecimal.valueOf(-1), 0); + assertEquals("1969-12-31T23:59:59.999Z", ts.toZString()); + } + @Test @Ignore // see https://github.com/amzn/ion-java/issues/160 public void testNewTimestampFromMinimumAllowedMillis() From 9a567e2a01c06a0abb02dc038fce88e070749d48 Mon Sep 17 00:00:00 2001 From: Zack Slayton Date: Fri, 26 Oct 2018 12:29:24 -0400 Subject: [PATCH 019/490] WriteBuffer truncation fix (#171) --- src/software/amazon/ion/impl/bin/WriteBuffer.java | 5 +++-- test/software/amazon/ion/impl/bin/WriteBufferTest.java | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/software/amazon/ion/impl/bin/WriteBuffer.java b/src/software/amazon/ion/impl/bin/WriteBuffer.java index 34ef9fbc07..b5ddb05b72 100644 --- a/src/software/amazon/ion/impl/bin/WriteBuffer.java +++ b/src/software/amazon/ion/impl/bin/WriteBuffer.java @@ -1071,9 +1071,10 @@ public void writeUInt8At(final long position, final long value) /** Write the entire buffer to output stream. */ public void writeTo(final OutputStream out) throws IOException { - for (final Block block : blocks) + for (int i = 0; i <= index; i++) { - out.write(block.data, 0, block.limit); + Block block = blocks.get(i); + out.write(block.data, 0, block.limit); } } diff --git a/test/software/amazon/ion/impl/bin/WriteBufferTest.java b/test/software/amazon/ion/impl/bin/WriteBufferTest.java index e5386c9272..8c200e9c04 100644 --- a/test/software/amazon/ion/impl/bin/WriteBufferTest.java +++ b/test/software/amazon/ion/impl/bin/WriteBufferTest.java @@ -915,4 +915,12 @@ public void testBytes() throws IOException buf.writeBytes("DOO".getBytes("UTF-8")); assertBuffer("ARGLEFOOBARGLEDOO".getBytes("UTF-8")); } + + @Test + public void testTruncate() throws IOException + { + buf.writeBytes("ARGLEFOOBARGLEDOO".getBytes("UTF-8")); + buf.truncate(3); + assertBuffer("ARG".getBytes("UTF-8")); + } } From 39a5b56bca48ae63bfd1776443acc6eb69bf46ed Mon Sep 17 00:00:00 2001 From: Peter Cornell Date: Wed, 31 Oct 2018 12:52:45 -0700 Subject: [PATCH 020/490] Avoids calling WriteBuffer.finish() if the writer is closed (#175) --- src/software/amazon/ion/impl/bin/IonRawBinaryWriter.java | 6 +++++- .../amazon/ion/impl/bin/IonRawBinaryWriterTest.java | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/software/amazon/ion/impl/bin/IonRawBinaryWriter.java b/src/software/amazon/ion/impl/bin/IonRawBinaryWriter.java index ae3e7bf4b7..b8e13d68aa 100644 --- a/src/software/amazon/ion/impl/bin/IonRawBinaryWriter.java +++ b/src/software/amazon/ion/impl/bin/IonRawBinaryWriter.java @@ -1432,6 +1432,10 @@ public void flush() throws IOException {} public void finish() throws IOException { + if (closed) + { + return; + } if (!containers.isEmpty()) { throw new IllegalStateException("Cannot finish within container: " + containers); @@ -1478,7 +1482,6 @@ public void close() throws IOException { return; } - closed = true; try { try @@ -1497,6 +1500,7 @@ public void close() throws IOException } finally { + closed = true; if (streamCloseMode == StreamCloseMode.CLOSE) { // release the stream diff --git a/test/software/amazon/ion/impl/bin/IonRawBinaryWriterTest.java b/test/software/amazon/ion/impl/bin/IonRawBinaryWriterTest.java index 909681de44..33745b978e 100644 --- a/test/software/amazon/ion/impl/bin/IonRawBinaryWriterTest.java +++ b/test/software/amazon/ion/impl/bin/IonRawBinaryWriterTest.java @@ -179,6 +179,15 @@ public NullDesc(final IonType type, final String literal) new NullDesc(STRUCT, "null.struct") }; + @Test + public void testFinishOnClose() throws IOException + { + // verify data is flushed if a writer is closed prior to calling finish() + writer.writeNull(); + writer.close(); + assertValue("null.null"); + } + @Test public void testNullNull() throws Exception { From 14a1d37e955b44ce5e0e5a6ecfaff8d358cb23e5 Mon Sep 17 00:00:00 2001 From: Peter Cornell Date: Wed, 31 Oct 2018 13:47:40 -0700 Subject: [PATCH 021/490] miscellaneous javadoc cleanup (#177) --- src/software/amazon/ion/IonLoader.java | 2 +- src/software/amazon/ion/IonSequence.java | 8 ++++---- src/software/amazon/ion/IonValue.java | 8 ++++---- src/software/amazon/ion/Timestamp.java | 10 +++++----- .../amazon/ion/system/IonBinaryWriterBuilder.java | 3 --- src/software/amazon/ion/util/Equivalence.java | 2 -- 6 files changed, 14 insertions(+), 19 deletions(-) diff --git a/src/software/amazon/ion/IonLoader.java b/src/software/amazon/ion/IonLoader.java index 147997f359..9b6e7819c5 100644 --- a/src/software/amazon/ion/IonLoader.java +++ b/src/software/amazon/ion/IonLoader.java @@ -127,7 +127,7 @@ public IonDatagram load(byte[] ionData) /** * Loads an entire stream of Ion data into a single datagram, * detecting whether it's text or binary data. - *

+ *

* The specified stream remains open after this method returns. *

* This method will auto-detect and uncompress GZIPped Ion data. diff --git a/src/software/amazon/ion/IonSequence.java b/src/software/amazon/ion/IonSequence.java index 68816412a1..378d9dc261 100644 --- a/src/software/amazon/ion/IonSequence.java +++ b/src/software/amazon/ion/IonSequence.java @@ -60,7 +60,7 @@ public interface IonSequence * @return the element at the given index; not null. * @throws NullValueException if {@link #isNullValue()}. * @throws IndexOutOfBoundsException if the index is out of range - * (index < 0 || index >= size()). + * (index < 0 || index >= size()). */ public IonValue get(int index) throws NullValueException, IndexOutOfBoundsException; @@ -113,7 +113,7 @@ public boolean add(IonValue child) * @throws IllegalArgumentException * if {@code child} is an {@link IonDatagram}. * @throws IndexOutOfBoundsException if the index is out of range - * (index < 0 || index > size()). + * (index < 0 || index > size()). */ public void add(int index, IonValue child) throws ContainedValueException, NullPointerException; @@ -169,7 +169,7 @@ public void add(int index, IonValue child) * @return the element previously at the specified position. * * @throws IndexOutOfBoundsException if the index is out of range - * (index < 0 || index >= size()). + * (index < 0 || index >= size()). */ public IonValue remove(int index); @@ -357,7 +357,7 @@ public void add(int index, IonValue child) * @throws ContainedValueException * if one of the elements is already contained by an {@link IonContainer}. * @throws IndexOutOfBoundsException - * if the index is out of range (index < 0 || index > size()). + * if the index is out of range (index < 0 || index > size()). */ public boolean addAll(int index, Collection c); diff --git a/src/software/amazon/ion/IonValue.java b/src/software/amazon/ion/IonValue.java index 3159df19a8..d22a58ed43 100644 --- a/src/software/amazon/ion/IonValue.java +++ b/src/software/amazon/ion/IonValue.java @@ -434,10 +434,10 @@ public IonValue clone() * {@link IonString} or {@link IonSymbol}! * Use {@link IonText#stringValue()} for that purpose. *

-     *    ionSystem.newString("Levi's").toString()     =>  "\"Levi's\""
-     *    ionSystem.newString("Levi's").stringValue()  =>  "Levi's"
-     *    ionSystem.newSymbol("Levi's").toString()     =>  "'Levi\\'s'"
-     *    ionSystem.newSymbol("Levi's").stringValue()  =>  "Levi's"
+     *    ionSystem.newString("Levi's").toString()     =>  "\"Levi's\""
+     *    ionSystem.newString("Levi's").stringValue()  =>  "Levi's"
+     *    ionSystem.newSymbol("Levi's").toString()     =>  "'Levi\\'s'"
+     *    ionSystem.newSymbol("Levi's").stringValue()  =>  "Levi's"
      * 
* * @return Ion text data equivalent to this value. diff --git a/src/software/amazon/ion/Timestamp.java b/src/software/amazon/ion/Timestamp.java index cff8412eac..7d504fd47a 100644 --- a/src/software/amazon/ion/Timestamp.java +++ b/src/software/amazon/ion/Timestamp.java @@ -40,7 +40,7 @@ * as the start of the new year/month/day. * * - *

Equality and Comparison

+ *

Equality and Comparison

* * As with {@link IonValue} classes, the {@link #equals equals} methods on this class * perform a strict equivalence that observes the precision and local offset @@ -544,7 +544,7 @@ private Timestamp(Precision p, int zyear, int zmonth, int zday, * * Note: All of these resulting Timestamps have the similar value (in UTC) 2012-02-03T04:05:06.007Z. * - *

Precision "Narrowing"

+ *

Precision "Narrowing"

* *

* Any time component that is more precise @@ -2226,10 +2226,10 @@ public int hashCode() * *

* This method is provided in preference to individual methods for each of - * the six boolean comparison operators (<, ==, >, >=, !=, <=). + * the six boolean comparison operators (<, ==, >, >=, !=, <=). * The suggested idiom for performing these comparisons is: - * {@code (x.compareTo(y)}<op>{@code 0)}, - * where <op> is one of the six comparison operators. + * {@code (x.compareTo(y)}<op>{@code 0)}, + * where <op> is one of the six comparison operators. * *

* For example, the pairs below will return a {@code 0} result: diff --git a/src/software/amazon/ion/system/IonBinaryWriterBuilder.java b/src/software/amazon/ion/system/IonBinaryWriterBuilder.java index 6ffc6a5a0c..eddc25b362 100644 --- a/src/software/amazon/ion/system/IonBinaryWriterBuilder.java +++ b/src/software/amazon/ion/system/IonBinaryWriterBuilder.java @@ -221,9 +221,6 @@ public IvmMinimizing getIvmMinimizing() * regardless of value. This is the legacy behavior for all Ion binary * writers and ensures the boarded compatibility with other Ion consumers. * - * @param enabled {@code true} to enable writing 4-byte floats, - * {@code false} to always write 8-byte floats. - * * @see IonBinaryWriterBuilder#setIsFloatBinary32Enabled(boolean) * @see IonBinaryWriterBuilder#withFloatBinary32Enabled */ diff --git a/src/software/amazon/ion/util/Equivalence.java b/src/software/amazon/ion/util/Equivalence.java index 8e93ac787f..644225ddc0 100644 --- a/src/software/amazon/ion/util/Equivalence.java +++ b/src/software/amazon/ion/util/Equivalence.java @@ -59,8 +59,6 @@ * boolean equivalent = ionEquals( v1, v2 ); * * - *

- * *

*

Ion Equivalence

* In order to make Ion a useful model to describe data, we must first define From 400f681b5d4d553d0c94efb3c53bab6604d66ec4 Mon Sep 17 00:00:00 2001 From: Fernando Raganhan Barbosa Date: Fri, 9 Nov 2018 13:59:53 -0800 Subject: [PATCH 022/490] Fixing javadoc badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 04cb20b202..18c5f78708 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ A Java implementation of the [Ion data notation](http://amzn.github.io/ion-docs) [![Build Status](https://travis-ci.org/amzn/ion-java.svg?branch=master)](https://travis-ci.org/amzn/ion-java) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/software.amazon.ion/ion-java/badge.svg)](https://maven-badges.herokuapp.com/maven-central/software.amazon.ion/ion-java) -[![Javadoc](https://javadoc-emblem.rhcloud.com/doc/software.amazon.ion/ion-java/badge.svg)](http://www.javadoc.io/doc/software.amazon.ion/ion-java) +[![Javadoc](https://javadoc-badge.appspot.com/software.amazon.ion/ion-java.svg?label=javadoc)](http://www.javadoc.io/doc/software.amazon.ion/ion-java) ## Setup This repository contains a [git submodule](https://git-scm.com/docs/git-submodule) From 1db8c657ce88f20c82228d920b6787a27ace8ecc Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Mon, 3 Dec 2018 15:35:28 -0800 Subject: [PATCH 023/490] Removes unnecessary synchronization from IonSystemLite.getLoader. --- src/software/amazon/ion/impl/lite/IonSystemLite.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/software/amazon/ion/impl/lite/IonSystemLite.java b/src/software/amazon/ion/impl/lite/IonSystemLite.java index d7e94b36eb..ff74dd44ee 100644 --- a/src/software/amazon/ion/impl/lite/IonSystemLite.java +++ b/src/software/amazon/ion/impl/lite/IonSystemLite.java @@ -140,7 +140,7 @@ public IonCatalog getCatalog() return _catalog; } - public synchronized IonLoader getLoader() + public IonLoader getLoader() { return _loader; } From 741556070b0518eb8f1a937ff77d4b966c028914 Mon Sep 17 00:00:00 2001 From: Fernando Raganhan Barbosa Date: Mon, 10 Dec 2018 13:10:08 -0800 Subject: [PATCH 024/490] Throwing for negative zero in binary (#182) https://github.com/amzn/ion-java/issues/138 Spec: http://amzn.github.io/ion-docs/docs/binary.html#2-and-3-int --- .../amazon/ion/impl/IonReaderBinarySystemX.java | 15 +++++++++++++-- test/software/amazon/ion/TestUtils.java | 2 -- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/software/amazon/ion/impl/IonReaderBinarySystemX.java b/src/software/amazon/ion/impl/IonReaderBinarySystemX.java index 27b9900d48..f28361e7e8 100644 --- a/src/software/amazon/ion/impl/IonReaderBinarySystemX.java +++ b/src/software/amazon/ion/impl/IonReaderBinarySystemX.java @@ -180,14 +180,19 @@ private final void load_scalar_value() throws IOException _v.setAuthoritativeType(AS_TYPE.boolean_value); break; case INT: + boolean is_negative = _value_tid == PrivateIonConstants.tidNegInt; + if (_value_len == 0) { + if (is_negative) { + throwIllegalNegativeZeroException(); + } + int v = 0; _v.setValue(v); _v.setAuthoritativeType(AS_TYPE.int_value); } else if (_value_len <= Long.BYTES) { long v = readULong(_value_len); - boolean is_negative = _value_tid == PrivateIonConstants.tidNegInt; if (v < 0) { // we probably can't fit this magnitude properly into a Java long @@ -205,6 +210,9 @@ else if (_value_len <= Long.BYTES) { } else { if (is_negative) { + if(v == 0) { + throwIllegalNegativeZeroException(); + } v = -v; } if (v < Integer.MIN_VALUE || v > Integer.MAX_VALUE) { @@ -217,7 +225,6 @@ else if (_value_len <= Long.BYTES) { } } else { - boolean is_negative = (_value_tid == PrivateIonConstants.tidNegInt); BigInteger v = readBigInteger(_value_len, is_negative); _v.setValue(v); _v.setAuthoritativeType(AS_TYPE.bigInteger_value); @@ -482,4 +489,8 @@ public SymbolTable pop_passed_symbol_table() { return null; } + + private void throwIllegalNegativeZeroException() { + throw newErrorAt("negative zero is illegal in the binary format"); + } } diff --git a/test/software/amazon/ion/TestUtils.java b/test/software/amazon/ion/TestUtils.java index ba7cad7503..f76b96d771 100644 --- a/test/software/amazon/ion/TestUtils.java +++ b/test/software/amazon/ion/TestUtils.java @@ -144,8 +144,6 @@ public boolean accept(File dir, String name) , "good/utf16.ion" // TODO amzn/ion-java#61 , "good/utf32.ion" // TODO amzn/ion-java#61 , "good/whitespace.ion" // TODO amzn/ion-java#104 - , "bad/negativeIntZero.10n" // TODO amzn/ion-java#138 - , "bad/negativeIntZeroLn.10n" // TODO amzn/ion-java#138 , "good/item1.10n" // TODO amzn/ion-java#126 (roundtrip symbols with unknown text) ); From b7c13ad308f049eee5414e9e851c4536a1fc16e5 Mon Sep 17 00:00:00 2001 From: Fernando Raganhan Barbosa Date: Mon, 10 Dec 2018 14:22:50 -0800 Subject: [PATCH 025/490] IonSequenceLite#retainAll javadoc fix and tests (#181) * Removes retainAll from IonDatagram interface Interface and implementation are inherited from IonSequence and IonSequenceLite respectively so no need to have it in the IonDatagram interface * IonSequenceLite#retailAll tests Adding tests to IonSequenceLite#retainAll and subclasses --- src/software/amazon/ion/IonDatagram.java | 13 ---- .../impl/lite/BaseIonSequenceLiteTest.java | 67 +++++++++++++++++++ .../ion/impl/lite/IonDatagramLiteTest.java | 10 +++ .../amazon/ion/impl/lite/IonListLiteTest.java | 10 +++ .../amazon/ion/impl/lite/IonSexpLiteTest.java | 10 +++ 5 files changed, 97 insertions(+), 13 deletions(-) create mode 100644 test/software/amazon/ion/impl/lite/BaseIonSequenceLiteTest.java create mode 100644 test/software/amazon/ion/impl/lite/IonDatagramLiteTest.java create mode 100644 test/software/amazon/ion/impl/lite/IonListLiteTest.java create mode 100644 test/software/amazon/ion/impl/lite/IonSexpLiteTest.java diff --git a/src/software/amazon/ion/IonDatagram.java b/src/software/amazon/ion/IonDatagram.java index 747390f443..fad4c6ce62 100644 --- a/src/software/amazon/ion/IonDatagram.java +++ b/src/software/amazon/ion/IonDatagram.java @@ -238,19 +238,6 @@ public int getBytes(OutputStream out) */ public void makeNull(); - - /** - * This inherited method is not yet supported by datagrams. - *

- * Vote for SIM issue amzn/ion-java#49 if you need this. - * - * @throws UnsupportedOperationException at every call. - * - * @see amzn/ion-java#49 - */ - public boolean retainAll(Collection c); - - public IonDatagram clone() throws UnknownSymbolException; } diff --git a/test/software/amazon/ion/impl/lite/BaseIonSequenceLiteTest.java b/test/software/amazon/ion/impl/lite/BaseIonSequenceLiteTest.java new file mode 100644 index 0000000000..e0614b3e37 --- /dev/null +++ b/test/software/amazon/ion/impl/lite/BaseIonSequenceLiteTest.java @@ -0,0 +1,67 @@ +package software.amazon.ion.impl.lite; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Collections; +import org.junit.Test; +import software.amazon.ion.IonSequence; +import software.amazon.ion.IonSystem; +import software.amazon.ion.IonValue; +import software.amazon.ion.ReadOnlyValueException; +import software.amazon.ion.system.IonSystemBuilder; + +public abstract class BaseIonSequenceLiteTest { + + protected static final IonSystem SYSTEM = IonSystemBuilder.standard().build(); + + protected abstract IonSequence newEmptySequence(); + + @Test + public void retainAll() { + final IonSequence sequence = newEmptySequence(); + final ArrayList toRetain = new ArrayList(); + + final IonValue retainedValue = SYSTEM.newInt(1); + sequence.add(retainedValue); + toRetain.add(retainedValue); + + final IonValue toRemoveValue = SYSTEM.newInt(2); + sequence.add(toRemoveValue); + + assertTrue(sequence.retainAll(toRetain)); + + assertEquals(1, sequence.size()); + assertTrue(sequence.contains(retainedValue)); + assertFalse(sequence.contains(toRemoveValue)); + } + + @Test + public void retainAllUsesReferenceEquality() { + final IonSequence sequence = newEmptySequence(); + final ArrayList toRetain = new ArrayList(); + + final IonValue value = SYSTEM.newInt(1); + sequence.add(value); + + final IonValue equalValue = SYSTEM.newInt(1); + toRetain.add(equalValue); + + assertEquals(equalValue, value); + assertNotSame(equalValue, value); + + assertTrue(sequence.retainAll(toRetain)); + assertEquals(0, sequence.size()); + } + + @Test(expected = ReadOnlyValueException.class) + public void retainAllReadOnly() { + final IonSequence sequence = newEmptySequence(); + sequence.makeReadOnly(); + + sequence.retainAll(Collections.emptyList()); + } +} \ No newline at end of file diff --git a/test/software/amazon/ion/impl/lite/IonDatagramLiteTest.java b/test/software/amazon/ion/impl/lite/IonDatagramLiteTest.java new file mode 100644 index 0000000000..45af7a15d0 --- /dev/null +++ b/test/software/amazon/ion/impl/lite/IonDatagramLiteTest.java @@ -0,0 +1,10 @@ +package software.amazon.ion.impl.lite; + +import software.amazon.ion.IonSequence; + +public class IonDatagramLiteTest extends BaseIonSequenceLiteTest { + @Override + protected IonSequence newEmptySequence() { + return SYSTEM.newDatagram(); + } +} diff --git a/test/software/amazon/ion/impl/lite/IonListLiteTest.java b/test/software/amazon/ion/impl/lite/IonListLiteTest.java new file mode 100644 index 0000000000..ee5b48c980 --- /dev/null +++ b/test/software/amazon/ion/impl/lite/IonListLiteTest.java @@ -0,0 +1,10 @@ +package software.amazon.ion.impl.lite; + +import software.amazon.ion.IonSequence; + +public class IonListLiteTest extends BaseIonSequenceLiteTest { + @Override + protected IonSequence newEmptySequence() { + return SYSTEM.newEmptyList(); + } +} diff --git a/test/software/amazon/ion/impl/lite/IonSexpLiteTest.java b/test/software/amazon/ion/impl/lite/IonSexpLiteTest.java new file mode 100644 index 0000000000..56528bcd57 --- /dev/null +++ b/test/software/amazon/ion/impl/lite/IonSexpLiteTest.java @@ -0,0 +1,10 @@ +package software.amazon.ion.impl.lite; + +import software.amazon.ion.IonSequence; + +public class IonSexpLiteTest extends BaseIonSequenceLiteTest { + @Override + protected IonSequence newEmptySequence() { + return SYSTEM.newEmptySexp(); + } +} From d014ffc85b12182a22e808c430080636d093daac Mon Sep 17 00:00:00 2001 From: Therapon Skoteiniotis Date: Tue, 11 Dec 2018 13:46:58 -0800 Subject: [PATCH 026/490] (Build) Adds codecoverage (#183) * (DOCS) remove callouts to empty symbol/field being disallowed * Adds coverage reports * Adds gitignore and removes idea file * Fix JaCoCo breaking Travis --- .gitignore | 257 +++++++++++++++++++++++++++++++++++++++++++++++++++++ pom.xml | 59 ++++++++++-- 2 files changed, 307 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index d20bced0a4..708300db93 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,260 @@ tmp* /target /resources *~ + +# Created by https://www.gitignore.io/api/vim,emacs,maven,gradle,eclipse,intellij +# Edit at https://www.gitignore.io/?templates=vim,emacs,maven,gradle,eclipse,intellij + +### Eclipse ### + +.metadata +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.settings/ +.loadpath +.recommenders + +# External tool builders +.externalToolBuilders/ + +# Locally stored "Eclipse launch configurations" +*.launch + +# PyDev specific (Python IDE for Eclipse) +*.pydevproject + +# CDT-specific (C/C++ Development Tooling) +.cproject + +# CDT- autotools +.autotools + +# Java annotation processor (APT) +.factorypath + +# PDT-specific (PHP Development Tools) +.buildpath + +# sbteclipse plugin +.target + +# Tern plugin +.tern-project + +# TeXlipse plugin +.texlipse + +# STS (Spring Tool Suite) +.springBeans + +# Code Recommenders +.recommenders/ + +# Annotation Processing +.apt_generated/ + +# Scala IDE specific (Scala & Java development for Eclipse) +.cache-main +.scala_dependencies +.worksheet + +### Eclipse Patch ### +# Eclipse Core +.project + +# JDT-specific (Eclipse Java Development Tools) +.classpath + +# Annotation Processing +.apt_generated + +.sts4-cache/ + +### Emacs ### +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile + +# directory configuration +.dir-locals.el + +# network security +/network-security.data + + +### Intellij ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +.idea + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/modules.xml +# .idea/*.iml +# .idea/modules + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### Intellij Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +.idea/sonarlint + +### Maven ### +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties +.mvn/wrapper/maven-wrapper.jar + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim + +# Temporary +.netrwhist +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +### Gradle ### +.gradle +/build/ + +# Ignore Gradle GUI config +gradle-app.setting + +# Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) +!gradle-wrapper.jar + +# Cache of project +.gradletasknamecache + +# # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 +# gradle/wrapper/gradle-wrapper.properties + +### Gradle Patch ### +**/build/ + +# End of https://www.gitignore.io/api/vim,emacs,maven,gradle,eclipse,intellij + diff --git a/pom.xml b/pom.xml index 9c2203e472..defe6f67e9 100644 --- a/pom.xml +++ b/pom.xml @@ -100,7 +100,7 @@ **/*TestCase.java - -ea -Xmx3000M + -ea -Xmx3000M ${argLine} @@ -124,6 +124,43 @@ + + + org.jacoco + jacoco-maven-plugin + 0.8.2 + + + default-prepare-agent + + prepare-agent + + + + default-report + test + + report + + + + default-check + + check + + + + + + + + + + + org.apache.maven.plugins + maven-site-plugin + 3.7.1 + @@ -131,15 +168,19 @@ - - org.apache.maven.plugins - maven-surefire-report-plugin - 2.19.1 - - test-report - false - + org.jacoco + jacoco-maven-plugin + 0.8.2 + + + + + report + + + + org.apache.maven.plugins From 39c9e823db5804345b67bbc216b3d54197031477 Mon Sep 17 00:00:00 2001 From: Peter Cornell Date: Tue, 11 Dec 2018 18:00:59 -0800 Subject: [PATCH 027/490] Ignores pseudo-IVMs as expected by the specification (#184) --- src/software/amazon/ion/impl/IonReaderTextRawX.java | 2 +- src/software/amazon/ion/impl/IonReaderTextUserX.java | 7 +++++-- test/software/amazon/ion/TestUtils.java | 1 - 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/software/amazon/ion/impl/IonReaderTextRawX.java b/src/software/amazon/ion/impl/IonReaderTextRawX.java index 8a4e633755..57715b85cb 100644 --- a/src/software/amazon/ion/impl/IonReaderTextRawX.java +++ b/src/software/amazon/ion/impl/IonReaderTextRawX.java @@ -944,7 +944,7 @@ protected final void parse_to_next_value() throws IOException case ACTION_LOAD_SCALAR: if (t == IonTokenConstsX.TOKEN_SYMBOL_IDENTIFIER) { sb = token_contents_load(t); - int _value_keyword = IonTokenConstsX.keyword(sb, 0, sb.length()); + _value_keyword = IonTokenConstsX.keyword(sb, 0, sb.length()); switch (_value_keyword) { case IonTokenConstsX.KEYWORD_NULL: { diff --git a/src/software/amazon/ion/impl/IonReaderTextUserX.java b/src/software/amazon/ion/impl/IonReaderTextUserX.java index 64fa372ac8..b1dfc8d5a4 100644 --- a/src/software/amazon/ion/impl/IonReaderTextUserX.java +++ b/src/software/amazon/ion/impl/IonReaderTextUserX.java @@ -146,8 +146,11 @@ private final boolean has_next_user_value() { if (ION_1_0.equals(version)) { - symbol_table_reset(); - push_symbol_table(_system_symtab); + if (_value_keyword != IonTokenConstsX.KEYWORD_sid) + { + symbol_table_reset(); + push_symbol_table(_system_symtab); + } _has_next_called = false; } else diff --git a/test/software/amazon/ion/TestUtils.java b/test/software/amazon/ion/TestUtils.java index f76b96d771..187af1a6af 100644 --- a/test/software/amazon/ion/TestUtils.java +++ b/test/software/amazon/ion/TestUtils.java @@ -139,7 +139,6 @@ public boolean accept(File dir, String name) , "bad/utf8/surrogate_2.ion" // TODO amzn/ion-java#105 , "bad/utf8/surrogate_4.ion" // TODO amzn/ion-java#105 , "equivs/paddedInts.10n" // TODO amzn/ion-java#54 - , "equivs/nonIVMNoOps.ion" // TODO amzn/ion-java#114 , "good/subfieldVarUInt32bit.ion" // TODO amzn/ion-java#62 , "good/utf16.ion" // TODO amzn/ion-java#61 , "good/utf32.ion" // TODO amzn/ion-java#61 From 62ebb7424fc438a8682765c7c430da6a551daae6 Mon Sep 17 00:00:00 2001 From: Zack Slayton Date: Wed, 12 Dec 2018 17:16:43 -0500 Subject: [PATCH 028/490] Empty ordered structs now cause an exception. (#185) --- src/software/amazon/ion/impl/IonReaderBinaryRawX.java | 3 +++ test/software/amazon/ion/TestUtils.java | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/software/amazon/ion/impl/IonReaderBinaryRawX.java b/src/software/amazon/ion/impl/IonReaderBinaryRawX.java index ae391f7f32..92470bb5fc 100644 --- a/src/software/amazon/ion/impl/IonReaderBinaryRawX.java +++ b/src/software/amazon/ion/impl/IonReaderBinaryRawX.java @@ -494,6 +494,9 @@ else if (tid == PrivateIonConstants.tidStruct) { // special case of an ordered struct, it gets the // otherwise impossible to have length of 1 len = readVarUInt(); + if (len == 0) { + throwErrorAt("Structs flagged as having ordered keys must contain at least one key/value pair."); + } start_of_value = _input.getPosition(); } } diff --git a/test/software/amazon/ion/TestUtils.java b/test/software/amazon/ion/TestUtils.java index 187af1a6af..4455209899 100644 --- a/test/software/amazon/ion/TestUtils.java +++ b/test/software/amazon/ion/TestUtils.java @@ -131,7 +131,6 @@ public boolean accept(File dir, String name) new FileIsNot( "bad/clobWithNullCharacter.ion" // TODO amzn/ion-java#43 , "bad/clobWithNonAsciiCharacter.ion" // TODO amzn/ion-java#99 - , "bad/structOrderedEmpty.10n" // TODO amzn/ion-java#100 , "bad/emptyAnnotatedInt.10n" // TODO amzn/ion-java#55 , "bad/timestamp/timestampNegativeFraction.10n" // TODO amzn/ion-java#102 , "bad/utf8/surrogate_5.ion" // TODO amzn/ion-java#60 From cbd1ab93393fbf213d6cdffbd4200b00dd261029 Mon Sep 17 00:00:00 2001 From: Peter Cornell Date: Wed, 12 Dec 2018 16:50:01 -0800 Subject: [PATCH 029/490] Produces an error for a struct field that has no value (#187) * Produces an error for a struct field that has no value * updates existing 'lob' tests to actually test lobs, rather than just strings --- ion-tests | 2 +- .../amazon/ion/impl/IonReaderTextRawX.java | 2 +- .../amazon/ion/streaming/ReaderTest.java | 24 +++++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/ion-tests b/ion-tests index 9bbb4ea124..ad489bf2df 160000 --- a/ion-tests +++ b/ion-tests @@ -1 +1 @@ -Subproject commit 9bbb4ea1244c707d77d0d0ff0d35a90d44b4d885 +Subproject commit ad489bf2df23381eedb3df8ccd69931f581dea18 diff --git a/src/software/amazon/ion/impl/IonReaderTextRawX.java b/src/software/amazon/ion/impl/IonReaderTextRawX.java index 57715b85cb..d0b1369713 100644 --- a/src/software/amazon/ion/impl/IonReaderTextRawX.java +++ b/src/software/amazon/ion/impl/IonReaderTextRawX.java @@ -183,7 +183,7 @@ static final int[][] makeTransitionActionArray() // now patch up the differences between these 4 states handling of tokens vs before_annotation_datagram actions[STATE_BEFORE_ANNOTATION_CONTAINED][IonTokenConstsX.TOKEN_EOF] = 0; actions[STATE_BEFORE_ANNOTATION_CONTAINED][IonTokenConstsX.TOKEN_CLOSE_PAREN] = ACTION_FINISH_CONTAINER; - actions[STATE_BEFORE_ANNOTATION_CONTAINED][IonTokenConstsX.TOKEN_CLOSE_BRACE] = ACTION_FINISH_CONTAINER; + actions[STATE_BEFORE_ANNOTATION_CONTAINED][IonTokenConstsX.TOKEN_CLOSE_BRACE] = 0; actions[STATE_BEFORE_ANNOTATION_CONTAINED][IonTokenConstsX.TOKEN_CLOSE_SQUARE] = ACTION_FINISH_CONTAINER; actions[STATE_BEFORE_ANNOTATION_SEXP][IonTokenConstsX.TOKEN_EOF] = 0; diff --git a/test/software/amazon/ion/streaming/ReaderTest.java b/test/software/amazon/ion/streaming/ReaderTest.java index 1a5eb9517b..72e4a83fa0 100644 --- a/test/software/amazon/ion/streaming/ReaderTest.java +++ b/test/software/amazon/ion/streaming/ReaderTest.java @@ -319,18 +319,18 @@ private void skipThroughTopLevelContainer(String data) private String[] LOB_DATA = { - "\"\"", - "\"clob\"", - - "''''''", - "''' '''", - "'''clob'''", - "'''c}ob'''", - "'''c}}b'''", - "'''c\\'''ob'''", - - "", - "Zm9v", + "{{\"\"}}", + "{{\"clob\"}}", + + "{{''''''}}", + "{{''' '''}}", + "{{'''clob'''}}", + "{{'''c}ob'''}}", + "{{'''c}}b'''}}", + "{{'''c\\'''ob'''}}", + + "{{}}", + "{{Zm9v}}", }; private void testSkippingLob(String containerPrefix, From 1bf99b6eb2f3510325366f4c532600d8c92515e8 Mon Sep 17 00:00:00 2001 From: Therapon Skoteiniotis Date: Thu, 13 Dec 2018 13:58:29 -0800 Subject: [PATCH 030/490] (Build) Enable site generation and deployment with Travis (#189) --- .travis.yml | 28 +++++++++++++++++++++------- pom.xml | 23 +++++++++++++++++++++++ 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index cc409f79fe..363db65d7a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,23 @@ +sudo: false language: java jdk: - - openjdk8 - - openjdk9 - - openjdk10 - - openjdk11 - - oraclejdk8 - - oraclejdk9 - - oraclejdk11 +- openjdk8 +- openjdk9 +- openjdk10 +- openjdk11 +- oraclejdk8 +- oraclejdk9 +- oraclejdk11 +after_success: + # enable once we get authorization with codecov.io sorted out + # - bash <(curl -s https://codecov.io/bash) +- mvn site +deploy: + provider: pages + local-dir: "./target/site/" + skip-cleanup: true + github-token: "$GITHUB_TOKEN" + keep-history: true # keeps commit history of gh-pages branch + on: + branch: master + jdk: openjdk11 diff --git a/pom.xml b/pom.xml index defe6f67e9..d23bc97626 100644 --- a/pom.xml +++ b/pom.xml @@ -208,6 +208,29 @@ + + + org.codehaus.mojo + findbugs-maven-plugin + 3.0.5 + + + + org.apache.maven.plugins + maven-pmd-plugin + 3.11.0 + + true + utf-8 + 100 + + + + + org.apache.maven.plugins + maven-jxr-plugin + 3.0.0 + From fcaf7b0555355806aea5fb4894186a9502fd9564 Mon Sep 17 00:00:00 2001 From: Zack Slayton Date: Thu, 13 Dec 2018 17:59:08 -0500 Subject: [PATCH 031/490] Timestamp fractional seconds outside of [0, 1) now cause an exception. (#190) --- src/software/amazon/ion/impl/IonReaderBinaryRawX.java | 10 ++++++++-- test/software/amazon/ion/TestUtils.java | 1 - 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/software/amazon/ion/impl/IonReaderBinaryRawX.java b/src/software/amazon/ion/impl/IonReaderBinaryRawX.java index 92470bb5fc..aca36e5de9 100644 --- a/src/software/amazon/ion/impl/IonReaderBinaryRawX.java +++ b/src/software/amazon/ion/impl/IonReaderBinaryRawX.java @@ -1115,12 +1115,12 @@ protected final Timestamp readTimestamp(int len) throws IOException year = readVarUInt(); Precision p = Precision.YEAR; // our lowest significant option - // now we look for hours and minutes + // now we look for months if (_local_remaining > 0) { month = readVarUInt(); p = Precision.MONTH; - // now we look for hours and minutes + // now we look for days if (_local_remaining > 0) { day = readVarUInt(); p = Precision.DAY; // our lowest significant option @@ -1136,6 +1136,12 @@ protected final Timestamp readTimestamp(int len) throws IOException if (_local_remaining > 0) { // now we read in our actual "milliseconds since the epoch" frac = readDecimal(_local_remaining); + if (frac.compareTo(BigDecimal.ZERO) < 0 || frac.compareTo(BigDecimal.ONE) >= 0) { + throwErrorAt( + "The fractional seconds value in a timestamp must be greater than or " + + "equal to zero and less than one." + ); + } } } } diff --git a/test/software/amazon/ion/TestUtils.java b/test/software/amazon/ion/TestUtils.java index 4455209899..f50ff04276 100644 --- a/test/software/amazon/ion/TestUtils.java +++ b/test/software/amazon/ion/TestUtils.java @@ -132,7 +132,6 @@ public boolean accept(File dir, String name) "bad/clobWithNullCharacter.ion" // TODO amzn/ion-java#43 , "bad/clobWithNonAsciiCharacter.ion" // TODO amzn/ion-java#99 , "bad/emptyAnnotatedInt.10n" // TODO amzn/ion-java#55 - , "bad/timestamp/timestampNegativeFraction.10n" // TODO amzn/ion-java#102 , "bad/utf8/surrogate_5.ion" // TODO amzn/ion-java#60 , "bad/utf8/surrogate_1.ion" // TODO amzn/ion-java#105 , "bad/utf8/surrogate_2.ion" // TODO amzn/ion-java#105 From 2101628f83aee87d0ab6d4a83482397e258bf32d Mon Sep 17 00:00:00 2001 From: Fernando Raganhan Barbosa Date: Fri, 14 Dec 2018 13:26:21 -0800 Subject: [PATCH 032/490] Implements IonSequence#sublist (#188) Implements IonSequence#sublist Implements sublist according to java List spec, i.e. sublist is a view of the parent list so changes to the parent list are reflected on the the sublist. Java's List contract warns that the sublist semantics became undefined after any structural changes, that affect size, to the parent list. Most implementations, see ArrayList and LinkedList, throw a `ConcurrentModificationException` to avoid surprising behavior. This PR takes the same approach being even more conservative by throwing on all sublist operations after a parent structural change. https://github.com/amzn/ion-java/issues/52 --- src/software/amazon/ion/IonSequence.java | 22 +- .../ion/impl/lite/IonContainerLite.java | 4 + .../amazon/ion/impl/lite/IonSequenceLite.java | 330 +++++++++- .../BaseIonSequenceLiteSublistTestCase.java | 597 ++++++++++++++++++ ....java => BaseIonSequenceLiteTestCase.java} | 12 +- .../impl/lite/IonDatagramLiteSublistTest.java | 16 + .../ion/impl/lite/IonDatagramLiteTest.java | 2 +- .../amazon/ion/impl/lite/IonListLiteTest.java | 2 +- .../impl/lite/IonListSexpLiteSublistTest.java | 111 ++++ .../amazon/ion/impl/lite/IonSexpLiteTest.java | 2 +- 10 files changed, 1085 insertions(+), 13 deletions(-) create mode 100644 test/software/amazon/ion/impl/lite/BaseIonSequenceLiteSublistTestCase.java rename test/software/amazon/ion/impl/lite/{BaseIonSequenceLiteTest.java => BaseIonSequenceLiteTestCase.java} (82%) create mode 100644 test/software/amazon/ion/impl/lite/IonDatagramLiteSublistTest.java create mode 100644 test/software/amazon/ion/impl/lite/IonListSexpLiteSublistTest.java diff --git a/src/software/amazon/ion/IonSequence.java b/src/software/amazon/ion/IonSequence.java index 378d9dc261..f42a9c6dfe 100644 --- a/src/software/amazon/ion/IonSequence.java +++ b/src/software/amazon/ion/IonSequence.java @@ -286,10 +286,15 @@ public void add(int index, IonValue child) * Returns the index in the sequence of the specified element, * or -1 if this sequence doesn't contain the element. *

- * Due to the reference-equality-based semantics of Ion sequences, + * + * Due to the reference-equality-based semantics of Ion sequences, * this method does not use {@link Object#equals} as specified by the * contract of {@link java.util.List}. Instead it uses reference - * equality ({@code ==} operator) to find the instance. + * equality ({@code ==} operator) to find the instance. + * + * And since IonSequences do not allow for duplicates this method behaves + * in the same way as {@link IonSequence#indexOf(Object)} + * * * @param o the element to search for. * @return the index in this sequence of the element, @@ -393,13 +398,18 @@ public void add(int index, IonValue child) /** - * This inherited method is not yet supported. *

- * Vote for issue amzn/ion-java#52 if you need this. + * Returns a view of the portion of this list according to {@link List#subList(int, int)} + * contract. + *

* - * @throws UnsupportedOperationException at every call. + *

+ * Sublist methods will throw a {@link java.util.ConcurrentModificationException} if + * its parent list, i.e. this list, had any changes that affect its size the after sublist + * was created. + *

* - * @see amzn/ion-java#52 + * @see List#subList(int, int) */ public List subList(int fromIndex, int toIndex); diff --git a/src/software/amazon/ion/impl/lite/IonContainerLite.java b/src/software/amazon/ion/impl/lite/IonContainerLite.java index e3653a3acd..a77b583431 100644 --- a/src/software/amazon/ion/impl/lite/IonContainerLite.java +++ b/src/software/amazon/ion/impl/lite/IonContainerLite.java @@ -38,6 +38,7 @@ abstract class IonContainerLite protected int _child_count; protected IonValueLite[] _children; + protected int structuralModificationCount; protected IonContainerLite(ContainerlessContext context, boolean isNull) { @@ -100,6 +101,7 @@ else if (!isEmpty()) { detachAllChildren(); _child_count = 0; + structuralModificationCount++; } } @@ -655,6 +657,7 @@ protected int add_child(int idx, IonValueLite child) } _child_count++; _children[idx] = child; + structuralModificationCount++; child._elementid(idx); return idx; @@ -681,6 +684,7 @@ void remove_child(int idx) } _child_count--; _children[_child_count] = null; + structuralModificationCount++; } public final void patch_elements_helper(int lowest_bad_idx) diff --git a/src/software/amazon/ion/impl/lite/IonSequenceLite.java b/src/software/amazon/ion/impl/lite/IonSequenceLite.java index 5585677352..9fb5db65be 100644 --- a/src/software/amazon/ion/impl/lite/IonSequenceLite.java +++ b/src/software/amazon/ion/impl/lite/IonSequenceLite.java @@ -16,10 +16,14 @@ import java.io.IOException; import java.lang.reflect.Array; +import java.util.ArrayList; import java.util.Collection; +import java.util.ConcurrentModificationException; import java.util.IdentityHashMap; import java.util.Iterator; import java.util.List; +import java.util.ListIterator; +import java.util.Map; import software.amazon.ion.ContainedValueException; import software.amazon.ion.IonSequence; import software.amazon.ion.IonType; @@ -334,8 +338,7 @@ public int lastIndexOf(Object o) public List subList(int fromIndex, int toIndex) { - // TODO amzn/ion-java#52 - throw new UnsupportedOperationException("issue amzn/ion-java#52"); + return new SubListView(this, fromIndex, toIndex); } public IonValue[] toArray() @@ -398,4 +401,327 @@ void writeBodyTo(IonWriter writer, SymbolTableProvider symbolTableProvider) writer.stepOut(); } } + + /** + * SubListView throws a {@link ConcurrentModificationException} if the parent list has any + * structural modifications, i.e. any operation that cause its size to change. To determine if + * a parent structural modification happened it keeps track of the structural modification count + * to compare against the parent + * + * Structural modifications from the sublist itself are allowed. + */ + private class SubListView implements List { + + /** + * index in top level IonSequenceLite that marks the start of this sublist view. For nested + * sublists fromIndex will be in relation to the root parent which must be a IonSequenceLite + */ + private final int fromIndex; + private int size; + private int structuralModificationCount; + + private SubListView(final List parent, + final int fromIndex, + final int toIndex) { + this.fromIndex = toTopLevelFromIndex(parent, fromIndex); + this.size = toIndex - fromIndex; + this.structuralModificationCount = IonSequenceLite.this.structuralModificationCount; + } + + public int size() { + checkForParentModification(); + return size; + } + + public boolean isEmpty() { + checkForParentModification(); + return size == 0; + } + + public IonValue get(final int index) { + checkForParentModification(); + rangeCheck(index); + + return IonSequenceLite.this.get(toParentIndex(index)); + } + + public IonValue set(final int index, final IonValue element) { + checkForParentModification(); + rangeCheck(index); + + return IonSequenceLite.this.set(toParentIndex(index), element); + } + + public boolean contains(final Object o) { + checkForParentModification(); + return indexOf(o) != -1; + } + + public boolean containsAll(final Collection c) { + for (Object o : c) { + if (!contains(o)) { + return false; + } + } + + return true; + } + + public IonValue[] toArray() { + checkForParentModification(); + + if (size < 1) { + return EMPTY_VALUE_ARRAY; + } + + return toArray(new IonValue[size]); + } + + @SuppressWarnings("unchecked") + public T[] toArray(T[] array) { + checkForParentModification(); + + if (array.length < size) { + final Class type = array.getClass().getComponentType(); + // generates unchecked warning + array = (T[]) Array.newInstance(type, size); + } + + if (size > 0) { + System.arraycopy(IonSequenceLite.this._children, fromIndex, array, 0, size); + } + + if (size < array.length) { + // See IonSequence#toArray and ArrayList#toArray + array[size] = null; + } + + return array; + } + + public boolean add(final IonValue ionValue) { + checkForParentModification(); + + int parentIndex = toParentIndex(size); + + // adds at end of parent list + if (parentIndex == IonSequenceLite.this.size()) { + IonSequenceLite.this.add(ionValue); + } else { + IonSequenceLite.this.add(parentIndex, ionValue); + } + + this.structuralModificationCount = IonSequenceLite.this.structuralModificationCount; + size++; + + return true; + } + + public void add(final int index, final IonValue ionValue) { + checkForParentModification(); + rangeCheck(index); + + IonSequenceLite.this.add(toParentIndex(index), ionValue); + + this.structuralModificationCount = IonSequenceLite.this.structuralModificationCount; + size++; + } + + public boolean addAll(final Collection c) { + for (IonValue ionValue : c) { + add(ionValue); + } + + return true; + } + + public boolean addAll(final int index, final Collection c) { + int i = index; + + for (IonValue ionValue : c) { + add(i, ionValue); + i++; + } + + return true; + } + + public boolean retainAll(final Collection c) { + if (size < 1) { + return false; + } + + final Map toRetain = new IdentityHashMap(); + for (Object o : c) { + toRetain.put(o, null); + } + + final List toRemove = new ArrayList(size - c.size()); + for (int i = 0; i < size; i++) { + final IonValue ionValue = get(i); + + if (!toRetain.containsKey(ionValue)) { + toRemove.add(ionValue); + } + } + + return removeAll(toRemove); + } + + public void clear() { + checkForParentModification(); + + // remove the first element size times to remove all elements + int parentIndex = toParentIndex(0); + for (int i = 0; i < size; i++) { + IonSequenceLite.this.remove(parentIndex); + } + + size = 0; + this.structuralModificationCount = IonSequenceLite.this.structuralModificationCount; + } + + public IonValue remove(final int index) { + checkForParentModification(); + rangeCheck(index); + + final IonValue removed = IonSequenceLite.this.remove(toParentIndex(index)); + + this.structuralModificationCount = IonSequenceLite.this.structuralModificationCount; + size--; + + return removed; + } + + public boolean remove(final Object o) { + int index = indexOf(o); + if (index < 0) { + return false; + } + + remove(index); + return true; + } + + public boolean removeAll(final Collection c) { + boolean changed = false; + for (Object o : c) { + if (remove(o)) { + changed = true; + } + } + + return changed; + } + + public int indexOf(final Object o) { + checkForParentModification(); + + final int parentIndex = IonSequenceLite.this.indexOf(o); + final int index = fromParentIndex(parentIndex); + + // not found + if (parentIndex < 0 || index < 0 || index >= size) { + return -1; + } + + return index; + } + + public int lastIndexOf(final Object o) { + return indexOf(o); + } + + public Iterator iterator() { + return listIterator(0); + } + + public ListIterator listIterator() { + return listIterator(0); + } + + public ListIterator listIterator(final int index) { + checkForParentModification(); + + return new ListIterator() { + private int lastReturnedIndex = index; + private int nextIndex = index; + + public boolean hasNext() { + return nextIndex < SubListView.this.size(); + } + + public IonValue next() { + lastReturnedIndex = nextIndex++; + return SubListView.this.get(lastReturnedIndex); + } + + public boolean hasPrevious() { + return nextIndex > 0; + } + + public IonValue previous() { + lastReturnedIndex = --nextIndex; + return SubListView.this.get(lastReturnedIndex); + } + + public int nextIndex() { + return nextIndex; + } + + public int previousIndex() { + return nextIndex - 1; + } + + public void remove() { + throw new UnsupportedOperationException(); + } + + public void set(final IonValue ionValue) { + SubListView.this.set(lastReturnedIndex, ionValue); + } + + public void add(final IonValue ionValue) { + SubListView.this.add(lastReturnedIndex, ionValue); + } + }; + } + + public List subList(final int fromIndex, final int toIndex) { + checkForParentModification(); + return new SubListView(this, fromIndex, toIndex); + } + + private void rangeCheck(int index) { + if (index < 0 || index >= size) { + throw new IndexOutOfBoundsException(String.valueOf(index)); + } + } + + private int toParentIndex(int index) { + return index + fromIndex; + } + + private int fromParentIndex(int index) { + return index - fromIndex; + } + + /** + * Calculates fromIndex based on the top level parent which must be an {@link IonSequenceLite}. With this + * nested sublists are able to directly call the top level parent instead of delegating up the parent chain + */ + private int toTopLevelFromIndex(final List parent, int fromIndex) { + if (parent instanceof SubListView) { + return fromIndex + ((SubListView) parent).fromIndex; + } + + return fromIndex; + } + + private void checkForParentModification() { + if (this.structuralModificationCount != IonSequenceLite.this.structuralModificationCount) { + throw new ConcurrentModificationException(); + } + } + } } diff --git a/test/software/amazon/ion/impl/lite/BaseIonSequenceLiteSublistTestCase.java b/test/software/amazon/ion/impl/lite/BaseIonSequenceLiteSublistTestCase.java new file mode 100644 index 0000000000..644ece031f --- /dev/null +++ b/test/software/amazon/ion/impl/lite/BaseIonSequenceLiteSublistTestCase.java @@ -0,0 +1,597 @@ +package software.amazon.ion.impl.lite; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.Collections; +import java.util.ConcurrentModificationException; +import java.util.Iterator; +import java.util.List; +import java.util.ListIterator; +import org.junit.Test; +import software.amazon.ion.ContainedValueException; +import software.amazon.ion.IonInt; +import software.amazon.ion.IonSequence; +import software.amazon.ion.IonSystem; +import software.amazon.ion.IonValue; +import software.amazon.ion.system.IonSystemBuilder; + +/** + * All tests related to {@link IonSequenceLite#subList(int, int)}. Extracted to a separate test due to the amount of + * tests + */ +public abstract class BaseIonSequenceLiteSublistTestCase { + + static final IonSystem SYSTEM = IonSystemBuilder.standard().build(); + static final int[] INTS = new int[]{0, 1, 2, 3, 4, 5, 6}; + + protected abstract IonSequence newSequence(); + + @Test + public void sublistSize() { + final IonSequence sequence = newSequence(); + final List actual = sequence.subList(2, 5); + + assertEquals(3, actual.size()); + } + + @Test + public void sublistIsEmpty() { + IonSequence sequence = newSequence(); + + assertTrue(sequence.subList(0, 0).isEmpty()); + assertFalse(sequence.subList(0, 1).isEmpty()); + } + + @Test + public void sublistGet() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final IonValue element = sublist.get(0); + + assertEquals(2, ((IonInt) element).intValue()); + } + + @Test(expected = IndexOutOfBoundsException.class) + public void sublistGetOutOfRange() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + sublist.get(4); + } + + @Test + public void sublistContains() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final IonValue insideSubList = sequence.get(2); + final IonValue outsideSublist = sequence.get(0); + + assertTrue(sublist.contains(insideSubList)); + assertFalse(sublist.contains(outsideSublist)); + } + + @Test + public void sublistContainsAll() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final List insideSubList = Arrays.asList(sequence.get(2), sequence.get(3)); + final List outsideSublist = Arrays.asList(sequence.get(0), sequence.get(1)); + final List mixed = Arrays.asList(sequence.get(0), sequence.get(3)); + + assertTrue(sublist.containsAll(insideSubList)); + assertFalse(sublist.containsAll(outsideSublist)); + assertFalse(sublist.containsAll(mixed)); + } + + @Test + public void sublistToArray() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final Object[] array = sublist.toArray(); + assertEquals(3, array.length); + assertEquals(2, ((IonInt) array[0]).intValue()); + assertEquals(3, ((IonInt) array[1]).intValue()); + assertEquals(4, ((IonInt) array[2]).intValue()); + } + + @Test + public void sublistToTypedArray() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final IonValue[] array = sublist.toArray(IonValue.EMPTY_ARRAY); + + assertEquals(3, array.length); + assertEquals(2, ((IonInt) array[0]).intValue()); + assertEquals(3, ((IonInt) array[1]).intValue()); + assertEquals(4, ((IonInt) array[2]).intValue()); + } + + @Test + public void sublistAdd() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final IonInt value = SYSTEM.newInt(99); + sublist.add(value); + assertEquals(4, sublist.size()); + assertEquals(value, sublist.get(3)); + } + + @Test(expected = ContainedValueException.class) + public void sublistAddSame() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final IonValue value = sequence.get(0); + sublist.add(value); + } + + @Test + public void sublistAddWithIndex() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final IonInt value = SYSTEM.newInt(99); + sublist.add(0, value); + assertEquals(4, sublist.size()); + assertEquals(value, sublist.get(0)); + assertEquals(2, ((IonInt) sublist.get(1)).intValue()); + } + + @Test(expected = IndexOutOfBoundsException.class) + public void sublistAddWithIndexOutOfRange() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final IonInt value = SYSTEM.newInt(99); + sublist.add(4, value); + } + + @Test + public void sublistAddAll() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final List values = Arrays.asList(SYSTEM.newInt(100), SYSTEM.newInt(101)); + sublist.addAll(values); + assertEquals(5, sublist.size()); + assertEquals(values.get(0), sublist.get(3)); + assertEquals(values.get(1), sublist.get(4)); + } + + @Test + public void sublistAddAllWithIndex() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final List values = Arrays.asList(SYSTEM.newInt(100), SYSTEM.newInt(101)); + sublist.addAll(0, values); + assertEquals(5, sublist.size()); + assertEquals(values.get(0), sublist.get(0)); + assertEquals(values.get(1), sublist.get(1)); + + assertEquals(2, ((IonInt) sublist.get(2)).intValue()); + } + + @Test(expected = IndexOutOfBoundsException.class) + public void sublistAddAllWithIndexOutOfRange() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final List values = Arrays.asList(SYSTEM.newInt(100), SYSTEM.newInt(101)); + sublist.addAll(3, values); + } + + @Test + public void sublistRetainAll() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final List toRetain = Collections.singletonList(sublist.get(0)); + + assertTrue(sublist.retainAll(toRetain)); + + assertEquals(1, sublist.size()); + assertTrue(sublist.contains(toRetain.get(0))); + assertEquals(5, sequence.size()); + } + + @Test + public void sublistRetainAllWithDuplicates() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final List toRetain = Arrays.asList(sublist.get(0), sublist.get(0)); + + assertTrue(sublist.retainAll(toRetain)); + + assertEquals(1, sublist.size()); + assertTrue(sublist.contains(toRetain.get(0))); + assertEquals(5, sequence.size()); + } + + @Test + public void sublistClear() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + sublist.clear(); + + assertEquals(0, sublist.size()); + assertEquals(4, sequence.size()); + } + + @Test + public void sublistRemoveIndex() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final IonInt ionValue = (IonInt) sublist.remove(0); + assertEquals(2, sublist.size()); + assertEquals(2, ionValue.intValue()); + } + + @Test(expected = IndexOutOfBoundsException.class) + public void sublistRemoveIndexOutOfRange() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + sublist.remove(3); + } + + @Test + public void sublistRemoveObject() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final IonInt ionValue = (IonInt) sequence.get(2); + assertTrue(sublist.remove(ionValue)); + assertEquals(2, sublist.size()); + } + + @Test + public void sublistRemoveAll() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final List toRemove = Arrays.asList(sequence.get(2), sequence.get(3)); + assertTrue(sublist.removeAll(toRemove)); + assertEquals(1, sublist.size()); + for (IonValue v : toRemove) { + assertFalse(sublist.contains(v)); + } + } + + @Test + public void sublistRemoveAllNotInList() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final List toRemove = Arrays.asList(sequence.get(0), sequence.get(1)); + assertFalse(sublist.removeAll(toRemove)); + assertEquals(3, sublist.size()); + } + + @Test + public void sublistIndexOf() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + assertEquals(0, sublist.indexOf(sequence.get(2))); + assertEquals(-1, sublist.indexOf(sequence.get(0))); + assertEquals(-1, sublist.indexOf(SYSTEM.newInt(99))); + } + + @Test + public void sublistLastIndexOf() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + assertEquals(0, sublist.lastIndexOf(sequence.get(2))); + assertEquals(-1, sublist.lastIndexOf(sequence.get(0))); + assertEquals(-1, sublist.lastIndexOf(SYSTEM.newInt(99))); + } + + @Test + public void sublistIterator() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final Iterator iterator = sublist.iterator(); + assertTrue(iterator.hasNext()); + assertEquals(sublist.get(0), iterator.next()); + + assertTrue(iterator.hasNext()); + assertEquals(sublist.get(1), iterator.next()); + + assertTrue(iterator.hasNext()); + assertEquals(sublist.get(2), iterator.next()); + + assertFalse(iterator.hasNext()); + } + + @Test + public void sublistListIterator() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final ListIterator iterator = sublist.listIterator(); + + assertFalse(iterator.hasPrevious()); + assertTrue(iterator.hasNext()); + assertEquals(0, iterator.nextIndex()); + assertEquals(-1, iterator.previousIndex()); + assertEquals(sublist.get(0), iterator.next()); + + assertTrue(iterator.hasNext()); + assertTrue(iterator.hasPrevious()); + assertEquals(1, iterator.nextIndex()); + assertEquals(0, iterator.previousIndex()); + assertEquals(sublist.get(1), iterator.next()); + + assertTrue(iterator.hasNext()); + assertTrue(iterator.hasPrevious()); + assertEquals(2, iterator.nextIndex()); + assertEquals(1, iterator.previousIndex()); + assertEquals(sublist.get(2), iterator.next()); + + assertEquals(3, iterator.nextIndex()); + assertEquals(2, iterator.previousIndex()); + assertFalse(iterator.hasNext()); + assertTrue(iterator.hasPrevious()); + + assertEquals(sublist.get(2), iterator.previous()); + final IonInt newAddValue = SYSTEM.newInt(200); + iterator.add(newAddValue); + assertEquals(newAddValue, sublist.get(2)); + assertTrue(iterator.hasNext()); + } + + @Test + public void sublistListIteratorWithIndex() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final ListIterator iterator = sublist.listIterator(1); + + assertTrue(iterator.hasNext()); + assertTrue(iterator.hasPrevious()); + assertEquals(1, iterator.nextIndex()); + assertEquals(0, iterator.previousIndex()); + assertEquals(sublist.get(1), iterator.next()); + + iterator.previous(); // back to initial state + + assertEquals(sublist.get(0), iterator.previous()); + } + + @Test + public void sublistSublist() { + IonSequence sequence = newSequence(); + + final List sublist = sequence.subList(1, 5); // 1,2,3,4 + final List subSublist = sublist.subList(1, 2); // 2 + + assertEquals(sublist.get(1), subSublist.get(0)); + assertEquals(sequence.get(2), subSublist.get(0)); + } + + // Concurrent modification tests --------------------------------------------------------------------------------- + + @Test(expected = ConcurrentModificationException.class) + public void sublistSizeConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + sequence.add(SYSTEM.newInt(99)); + + sublist.size(); + } + + @Test(expected = ConcurrentModificationException.class) + public void sublistIsEmptyConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + sequence.remove(0); + + sublist.isEmpty(); + } + + @Test(expected = ConcurrentModificationException.class) + public void sublistGetConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + sequence.clear(); + + sublist.get(0); + } + + @Test(expected = ConcurrentModificationException.class) + public void sublistSetConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + sequence.remove(0); + + final IonInt element = SYSTEM.newInt(99); + sublist.set(0, element); + } + + @Test(expected = ConcurrentModificationException.class) + public void sublistContainsConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + final IonInt value = SYSTEM.newInt(99); + sequence.add(value); + + sublist.contains(value); + } + + @Test(expected = ConcurrentModificationException.class) + public void sublistContainsAllConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + final IonValue value = sequence.remove(0); + + sublist.containsAll(Collections.singletonList(value)); + } + + @Test(expected = ConcurrentModificationException.class) + public void sublistToArrayConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + sequence.remove(0); + + sublist.toArray(); + } + + @Test(expected = ConcurrentModificationException.class) + public void sublistToTypedArrayConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + sequence.remove(0); + + sublist.toArray(IonValue.EMPTY_ARRAY); + } + + @Test(expected = ConcurrentModificationException.class) + public void sublistAddConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + sequence.remove(0); + + sublist.add(SYSTEM.newInt(0)); + } + + @Test(expected = ConcurrentModificationException.class) + public void sublistAddWithIndexConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + sequence.remove(0); + + sublist.add(0, SYSTEM.newInt(0)); + } + + @Test(expected = ConcurrentModificationException.class) + public void sublistAddAllConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + sequence.remove(0); + + sublist.addAll(Arrays.asList(SYSTEM.newInt(100), SYSTEM.newInt(101))); + } + + @Test(expected = ConcurrentModificationException.class) + public void sublistAddAllWithIndexConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + sequence.remove(0); + + sublist.addAll(0, Arrays.asList(SYSTEM.newInt(100), SYSTEM.newInt(101))); + } + + @Test(expected = ConcurrentModificationException.class) + public void sublistRetainAllConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + sequence.remove(0); + + sublist.retainAll(Arrays.asList(SYSTEM.newInt(100), SYSTEM.newInt(101))); + } + + @Test(expected = ConcurrentModificationException.class) + public void sublistClearConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + sequence.retainAll(Collections.emptyList()); + + sublist.clear(); + } + + @Test(expected = ConcurrentModificationException.class) + public void sublistRemoveIndexConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + sequence.remove(0); + + sublist.remove(0); + } + + @Test(expected = ConcurrentModificationException.class) + public void sublistRemoveObjectConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + sequence.remove(0); + + sublist.remove(sequence.get(2)); + } + + @Test(expected = ConcurrentModificationException.class) + public void sublistRemoveAllConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + sequence.remove(0); + + sublist.removeAll(Collections.singletonList(sequence.get(2))); + } + + @Test(expected = ConcurrentModificationException.class) + public void sublistIndexOfConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + sequence.remove(0); + + sublist.indexOf(sequence.get(2)); + } + + @Test(expected = ConcurrentModificationException.class) + public void sublistLastIndexOfConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + sequence.remove(0); + + sublist.lastIndexOf(sequence.get(2)); + } + + @Test(expected = ConcurrentModificationException.class) + public void sublistIteratorConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + sequence.add(SYSTEM.newInt(99)); + + sublist.iterator(); + } + + @Test(expected = ConcurrentModificationException.class) + public void sublistListIteratorConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + sequence.remove(0); + + sublist.listIterator(); + } + + @Test(expected = ConcurrentModificationException.class) + public void sublistListIteratorWithIndexConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + sequence.add(SYSTEM.newInt(99)); + + sublist.listIterator(1); + } + + @Test(expected = ConcurrentModificationException.class) + public void sublistSubListConcurrentModification() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + sequence.remove(0); + + sublist.subList(0, 1); + } +} diff --git a/test/software/amazon/ion/impl/lite/BaseIonSequenceLiteTest.java b/test/software/amazon/ion/impl/lite/BaseIonSequenceLiteTestCase.java similarity index 82% rename from test/software/amazon/ion/impl/lite/BaseIonSequenceLiteTest.java rename to test/software/amazon/ion/impl/lite/BaseIonSequenceLiteTestCase.java index e0614b3e37..0942a0ae7c 100644 --- a/test/software/amazon/ion/impl/lite/BaseIonSequenceLiteTest.java +++ b/test/software/amazon/ion/impl/lite/BaseIonSequenceLiteTestCase.java @@ -6,17 +6,25 @@ import static org.junit.Assert.assertTrue; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.ConcurrentModificationException; +import java.util.Iterator; +import java.util.List; +import java.util.ListIterator; import org.junit.Test; +import software.amazon.ion.ContainedValueException; +import software.amazon.ion.IonInt; +import software.amazon.ion.IonList; import software.amazon.ion.IonSequence; import software.amazon.ion.IonSystem; import software.amazon.ion.IonValue; import software.amazon.ion.ReadOnlyValueException; import software.amazon.ion.system.IonSystemBuilder; -public abstract class BaseIonSequenceLiteTest { +public abstract class BaseIonSequenceLiteTestCase { - protected static final IonSystem SYSTEM = IonSystemBuilder.standard().build(); + static final IonSystem SYSTEM = IonSystemBuilder.standard().build(); protected abstract IonSequence newEmptySequence(); diff --git a/test/software/amazon/ion/impl/lite/IonDatagramLiteSublistTest.java b/test/software/amazon/ion/impl/lite/IonDatagramLiteSublistTest.java new file mode 100644 index 0000000000..7bb57e17c9 --- /dev/null +++ b/test/software/amazon/ion/impl/lite/IonDatagramLiteSublistTest.java @@ -0,0 +1,16 @@ +package software.amazon.ion.impl.lite; + +import software.amazon.ion.IonDatagram; +import software.amazon.ion.IonSequence; + +public class IonDatagramLiteSublistTest extends BaseIonSequenceLiteSublistTestCase { + + protected IonSequence newSequence() { + final IonDatagram datagram = SYSTEM.newDatagram(); + for(int i : INTS) { + datagram.add(SYSTEM.newInt(INTS[i])); + } + + return datagram; + } +} diff --git a/test/software/amazon/ion/impl/lite/IonDatagramLiteTest.java b/test/software/amazon/ion/impl/lite/IonDatagramLiteTest.java index 45af7a15d0..3637df1eba 100644 --- a/test/software/amazon/ion/impl/lite/IonDatagramLiteTest.java +++ b/test/software/amazon/ion/impl/lite/IonDatagramLiteTest.java @@ -2,7 +2,7 @@ import software.amazon.ion.IonSequence; -public class IonDatagramLiteTest extends BaseIonSequenceLiteTest { +public class IonDatagramLiteTest extends BaseIonSequenceLiteTestCase { @Override protected IonSequence newEmptySequence() { return SYSTEM.newDatagram(); diff --git a/test/software/amazon/ion/impl/lite/IonListLiteTest.java b/test/software/amazon/ion/impl/lite/IonListLiteTest.java index ee5b48c980..2da37f29b7 100644 --- a/test/software/amazon/ion/impl/lite/IonListLiteTest.java +++ b/test/software/amazon/ion/impl/lite/IonListLiteTest.java @@ -2,7 +2,7 @@ import software.amazon.ion.IonSequence; -public class IonListLiteTest extends BaseIonSequenceLiteTest { +public class IonListLiteTest extends BaseIonSequenceLiteTestCase { @Override protected IonSequence newEmptySequence() { return SYSTEM.newEmptyList(); diff --git a/test/software/amazon/ion/impl/lite/IonListSexpLiteSublistTest.java b/test/software/amazon/ion/impl/lite/IonListSexpLiteSublistTest.java new file mode 100644 index 0000000000..6166a5843b --- /dev/null +++ b/test/software/amazon/ion/impl/lite/IonListSexpLiteSublistTest.java @@ -0,0 +1,111 @@ +package software.amazon.ion.impl.lite; + +import static org.junit.Assert.assertEquals; + +import java.util.List; +import java.util.ListIterator; +import org.junit.Test; +import org.junit.runner.RunWith; +import software.amazon.ion.IonInt; +import software.amazon.ion.IonSequence; +import software.amazon.ion.IonValue; +import software.amazon.ion.junit.Injected; +import software.amazon.ion.junit.Injected.Inject; + +/** + * Sublist tests for list and sexp + */ +@RunWith(Injected.class) +public class IonListSexpLiteSublistTest extends BaseIonSequenceLiteSublistTestCase { + + interface IonSequenceProvider { + + IonSequence newSequence(); + } + + @Inject("provider") + public static final IonSequenceProvider[] providers = new IonSequenceProvider[]{ + new IonSequenceProvider() { + public IonSequence newSequence() { + return SYSTEM.newList(INTS); + } + }, + new IonSequenceProvider() { + public IonSequence newSequence() { + return SYSTEM.newSexp(INTS); + } + }, + }; + + private IonSequenceProvider provider; + + public void setProvider(final IonSequenceProvider provider) { + this.provider = provider; + } + + protected IonSequence newSequence() { + return provider.newSequence(); + } + + @Test + public void sublistSet() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final IonInt element = SYSTEM.newInt(99); + final IonValue previous = sublist.set(0, element); + + assertEquals(2, ((IonInt) previous).intValue()); + assertEquals(element, sublist.get(0)); + assertEquals(element, sequence.get(2)); + } + + @Test + public void sublistSetOnParent() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final IonInt element = SYSTEM.newInt(99); + final IonValue previous = sequence.set(2, element); + + assertEquals(2, ((IonInt) previous).intValue()); + assertEquals(element, sublist.get(0)); + assertEquals(element, sequence.get(2)); + } + + @Test(expected = IndexOutOfBoundsException.class) + public void sublistSetOutOfRange() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final IonInt element = SYSTEM.newInt(99); + sublist.set(4, element); + } + + @Test + public void sublistSublistChangingParentValues() { + IonSequence sequence = newSequence(); + + final List sublist = sequence.subList(1, 5); // 1,2,3,4 + final List subSublist = sublist.subList(0, 2); // 1,2 + + final IonInt newValue = SYSTEM.newInt(100); + sequence.set(1, newValue); + assertEquals(newValue, sublist.get(0)); + assertEquals(newValue, subSublist.get(0)); + } + + @Test + public void sublistListIteratorWithSet() { + final IonSequence sequence = newSequence(); + final List sublist = sequence.subList(2, 5); + + final ListIterator iterator = sublist.listIterator(); + + iterator.next(); + + final IonInt newSetValue = SYSTEM.newInt(100); + iterator.set(newSetValue); + assertEquals(newSetValue, sublist.get(0)); + } +} diff --git a/test/software/amazon/ion/impl/lite/IonSexpLiteTest.java b/test/software/amazon/ion/impl/lite/IonSexpLiteTest.java index 56528bcd57..3b3fbfaa96 100644 --- a/test/software/amazon/ion/impl/lite/IonSexpLiteTest.java +++ b/test/software/amazon/ion/impl/lite/IonSexpLiteTest.java @@ -2,7 +2,7 @@ import software.amazon.ion.IonSequence; -public class IonSexpLiteTest extends BaseIonSequenceLiteTest { +public class IonSexpLiteTest extends BaseIonSequenceLiteTestCase { @Override protected IonSequence newEmptySequence() { return SYSTEM.newEmptySexp(); From f302af7a76b100bd18c9803c5dc34615abf5ec49 Mon Sep 17 00:00:00 2001 From: 0xflotus <0xflotus@gmail.com> Date: Sat, 15 Dec 2018 07:13:21 +0100 Subject: [PATCH 033/490] changed some if-statements to ternary expressions (#156) You can use ternary expressions, if you only return a conditional value --- src/software/amazon/ion/Decimal.java | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/src/software/amazon/ion/Decimal.java b/src/software/amazon/ion/Decimal.java index 08b235ba62..a92dc06644 100644 --- a/src/software/amazon/ion/Decimal.java +++ b/src/software/amazon/ion/Decimal.java @@ -68,16 +68,14 @@ private NegativeZero(int scale, MathContext mc) public float floatValue() { float v = super.floatValue(); - if (Float.compare(0f, v) <= 0) v = -1 * v; - return v; + return Float.compare(0f, v) <= 0 ? -1 * v : v; } @Override public double doubleValue() { double v = super.doubleValue(); - if (Double.compare(0d, v) <= 0) v = -1 * v; - return v; + return Double.compare(0d, v) <= 0 ? -1 * v : v; } @@ -159,12 +157,8 @@ public static boolean isNegativeZero(BigDecimal val) */ public static BigDecimal bigDecimalValue(BigDecimal val) { - if (val == null - || val.getClass() == BigDecimal.class) - { - return val; - } - return new BigDecimal(val.unscaledValue(), val.scale()); + return val == null || val.getClass() == BigDecimal.class ? + val : new BigDecimal(val.unscaledValue(), val.scale()); } /** @@ -269,17 +263,12 @@ public static Decimal valueOf(double val) public static Decimal valueOf(double val, MathContext mc) { - if (Double.compare(val, -0d) == 0) - { - return new NegativeZero(1, mc); - } - return new Decimal(Double.toString(val), mc); + return Double.compare(val, -0d) == 0 ? new NegativeZero(1, mc) : new Decimal(Double.toString(val), mc); } public static Decimal valueOf(BigDecimal val) { - if (val == null || val instanceof Decimal) return (Decimal) val; - return new Decimal(val.unscaledValue(), val.scale()); + return val == null || val instanceof Decimal ? (Decimal) val : new Decimal(val.unscaledValue(), val.scale()); } public static Decimal valueOf(BigDecimal val, MathContext mc) @@ -421,7 +410,6 @@ public final boolean isNegativeZero() */ public final BigDecimal bigDecimalValue() { - return new BigDecimal(unscaledValue(), scale()); } } From fcd528e308f9ea585ae627a64d5eda57b416ecd3 Mon Sep 17 00:00:00 2001 From: Therapon Skoteiniotis Date: Sat, 15 Dec 2018 06:16:44 -0800 Subject: [PATCH 034/490] (Build) Enable codecov (#194) Enable pushing coverage to codecov --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 363db65d7a..faced71d63 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,8 +9,7 @@ jdk: - oraclejdk9 - oraclejdk11 after_success: - # enable once we get authorization with codecov.io sorted out - # - bash <(curl -s https://codecov.io/bash) +- bash <(curl -s https://codecov.io/bash) - mvn site deploy: provider: pages From 369b5618a23caaac1c0730200cb21742d749c65f Mon Sep 17 00:00:00 2001 From: Therapon Skoteiniotis Date: Mon, 17 Dec 2018 11:30:48 -0800 Subject: [PATCH 035/490] (Build) Split tests and report generation (#196) * Adds stages for build & test & report generation --- .travis.yml | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index faced71d63..a8b579c681 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,15 +8,21 @@ jdk: - oraclejdk8 - oraclejdk9 - oraclejdk11 -after_success: -- bash <(curl -s https://codecov.io/bash) -- mvn site -deploy: - provider: pages - local-dir: "./target/site/" - skip-cleanup: true - github-token: "$GITHUB_TOKEN" - keep-history: true # keeps commit history of gh-pages branch - on: - branch: master - jdk: openjdk11 + +script: mvn test + +jobs: + include: + stage: report generation + jdk: openjdk11 + script: mvn test site + after_success: + - bash <(curl -s https://codecov.io/bash) + deploy: + provider: pages + local-dir: "./target/site/" + skip-cleanup: true + github-token: "$GITHUB_TOKEN" + keep-history: true # keeps commit history of gh-pages branch + on: + branch: master From b73931ead1c0e8da6dbcd9e0790b9ccfffe2efd6 Mon Sep 17 00:00:00 2001 From: Fernando Raganhan Barbosa Date: Mon, 17 Dec 2018 12:06:25 -0800 Subject: [PATCH 036/490] Clob nonascii error 99 (#192) * Throws error when reading clobs with non ascii characters * Change UTF8 check Relies on actual test file data instead of a naming convention that is not being followed * Updates skip list and ion-tests https://github.com/amzn/ion-java/issues/99 --- ion-tests | 2 +- .../amazon/ion/impl/IonReaderTextRawTokensX.java | 7 +++++++ .../amazon/ion/impl/IonReaderTextSystemX.java | 5 ++++- test/software/amazon/ion/BadIonTest.java | 16 +++++++++++++--- test/software/amazon/ion/TestUtils.java | 8 ++++++-- 5 files changed, 31 insertions(+), 7 deletions(-) diff --git a/ion-tests b/ion-tests index ad489bf2df..0281240a90 160000 --- a/ion-tests +++ b/ion-tests @@ -1 +1 @@ -Subproject commit ad489bf2df23381eedb3df8ccd69931f581dea18 +Subproject commit 0281240a9039a1f609fc8e289721efff489956be diff --git a/src/software/amazon/ion/impl/IonReaderTextRawTokensX.java b/src/software/amazon/ion/impl/IonReaderTextRawTokensX.java index 33757bc3a6..aba793f173 100644 --- a/src/software/amazon/ion/impl/IonReaderTextRawTokensX.java +++ b/src/software/amazon/ion/impl/IonReaderTextRawTokensX.java @@ -2049,6 +2049,9 @@ protected int load_double_quoted_string(StringBuilder sb, boolean is_clob) protected int read_double_quoted_char(boolean is_clob) throws IOException { int c = read_char(); + if(is_clob && c > 127) { + throw new IonReaderTextTokenException("non ASCII character in clob: " + c); + } switch (c) { case '"': @@ -2174,6 +2177,10 @@ protected int load_triple_quoted_string(StringBuilder sb, boolean is_clob) protected int read_triple_quoted_char(boolean is_clob) throws IOException { int c = read_string_char(ProhibitedCharacters.LONG_CHAR); + if(is_clob && c > 127) { + throw new IonReaderTextTokenException("non ASCII character in clob: " + c); + } + switch (c) { case '\'': if (is_2_single_quotes_helper()) { diff --git a/src/software/amazon/ion/impl/IonReaderTextSystemX.java b/src/software/amazon/ion/impl/IonReaderTextSystemX.java index 55b5a849de..07c5668365 100644 --- a/src/software/amazon/ion/impl/IonReaderTextSystemX.java +++ b/src/software/amazon/ion/impl/IonReaderTextSystemX.java @@ -849,6 +849,7 @@ private int readBytes(byte[] buffer, int offset, int len) int c = -1; switch (_lob_token) { + // BLOB case IonTokenConstsX.TOKEN_OPEN_DOUBLE_BRACE: while (len-- > 0) { c = _scanner.read_base64_byte(); @@ -856,6 +857,7 @@ private int readBytes(byte[] buffer, int offset, int len) buffer[offset++] = (byte)c; } break; + // CLOB case IonTokenConstsX.TOKEN_STRING_DOUBLE_QUOTE: while (len-- > 0) { c = _scanner.read_double_quoted_char(true); @@ -868,10 +870,11 @@ private int readBytes(byte[] buffer, int offset, int len) } break; } - assert(c >= 0 && c <= UNSIGNED_BYTE_MAX_VALUE); + assert(c <= UNSIGNED_BYTE_MAX_VALUE); buffer[offset++] = (byte)c; } break; + // CLOB case IonTokenConstsX.TOKEN_STRING_TRIPLE_QUOTE: while (len-- > 0) { c = _scanner.read_triple_quoted_char(true); diff --git a/test/software/amazon/ion/BadIonTest.java b/test/software/amazon/ion/BadIonTest.java index ff02120175..326066842a 100644 --- a/test/software/amazon/ion/BadIonTest.java +++ b/test/software/amazon/ion/BadIonTest.java @@ -18,10 +18,12 @@ import static software.amazon.ion.TestUtils.testdataFiles; import java.io.File; +import java.io.FileReader; import java.io.IOException; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetEncoder; import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonException; import software.amazon.ion.impl.PrivateUtils; import software.amazon.ion.junit.Injected.Inject; @@ -75,7 +77,15 @@ public void testLoadString() } catch (IonException e) { - assert myTestFile.getPath().contains("bad/utf8") || myTestFile.getPath().contains("bad\\utf8"); + // checks that test failed because of bad UTF-8 data + final CharBuffer buffer = CharBuffer.allocate(1024); + final CharsetEncoder utf8Encoder = Charset.forName("UTF-8").newEncoder(); + + final FileReader fileReader = new FileReader(myTestFile); + while(fileReader.read(buffer) != -1){ + assert utf8Encoder.canEncode(buffer); + } + return; } diff --git a/test/software/amazon/ion/TestUtils.java b/test/software/amazon/ion/TestUtils.java index f50ff04276..d7ade58f7c 100644 --- a/test/software/amazon/ion/TestUtils.java +++ b/test/software/amazon/ion/TestUtils.java @@ -130,12 +130,16 @@ public boolean accept(File dir, String name) public static final FilenameFilter GLOBAL_SKIP_LIST = new FileIsNot( "bad/clobWithNullCharacter.ion" // TODO amzn/ion-java#43 - , "bad/clobWithNonAsciiCharacter.ion" // TODO amzn/ion-java#99 , "bad/emptyAnnotatedInt.10n" // TODO amzn/ion-java#55 - , "bad/utf8/surrogate_5.ion" // TODO amzn/ion-java#60 , "bad/utf8/surrogate_1.ion" // TODO amzn/ion-java#105 , "bad/utf8/surrogate_2.ion" // TODO amzn/ion-java#105 , "bad/utf8/surrogate_4.ion" // TODO amzn/ion-java#105 + , "bad/utf8/surrogate_5.ion" // TODO amzn/ion-java#60 + , "bad/utf8/surrogate_6.ion" // TODO amzn/ion-java#105 + , "bad/utf8/surrogate_7.ion" // TODO amzn/ion-java#105 + , "bad/utf8/surrogate_8.ion" // TODO amzn/ion-java#105 + , "bad/utf8/surrogate_9.ion" // TODO amzn/ion-java#105 + , "bad/utf8/surrogate_10.ion" // TODO amzn/ion-java#105 , "equivs/paddedInts.10n" // TODO amzn/ion-java#54 , "good/subfieldVarUInt32bit.ion" // TODO amzn/ion-java#62 , "good/utf16.ion" // TODO amzn/ion-java#61 From f8dad16d1c17deca775dd08b43a3a192c453c90b Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Mon, 17 Dec 2018 21:04:20 +0000 Subject: [PATCH 037/490] Replaces Timestamp internals with Calendar. (#195) * Makes Timestamp use Calendar internally for date arithmetic and leap year calculations. * Removes Timestamp's individual fields in favor of a backing Calendar. * Adds support in Timestamp for receiving lenient Calendars as input. * Creates the Timestamp.addSecond(long seconds) method and enhances some JavaDoc comments. --- src/software/amazon/ion/Timestamp.java | 714 +++++++++----------- test/software/amazon/ion/TimestampTest.java | 525 +++++++++----- 2 files changed, 678 insertions(+), 561 deletions(-) diff --git a/src/software/amazon/ion/Timestamp.java b/src/software/amazon/ion/Timestamp.java index 7d504fd47a..2f5fa7c3f1 100644 --- a/src/software/amazon/ion/Timestamp.java +++ b/src/software/amazon/ion/Timestamp.java @@ -23,7 +23,7 @@ import java.util.Calendar; import java.util.Date; import java.util.GregorianCalendar; -import java.util.TimeZone; + import software.amazon.ion.impl.PrivateUtils; import software.amazon.ion.util.IonTextUtils; @@ -68,6 +68,16 @@ *
  • {@code 2009-01-01T00:00:00.000Z} etc.
  • * * + * + *

    Date Arithmetic and Leap Years

    + * Date arithmetic is performed according to the logic provided by + * {@link Calendar#add(int, int)}. When constructed by {@link Timestamp#forCalendar(Calendar)} + * the new Timestamp and any other Timestamps that spawn from it (e.g. through + * {@link #clone()}, {@link #addDay(int)}, etc.) will use the given Calendar's date arithmetic + * rules, including its rules for determining leap years. When constructed without a + * Calendar, a default GregorianCalendar (as constructed by + * new GregorianCalendar(TimeZone.getTimeZone("UTC")) will be used. + * * @see #equals(Timestamp) * @see #compareTo(Timestamp) */ @@ -84,15 +94,6 @@ public final class Timestamp private static final int NO_SECONDS = 0; private static final BigDecimal NO_FRACTIONAL_SECONDS = null; - // 0001-01-01T00:00:00.0Z in millis - private static final long MINIMUM_ALLOWED_TIMESTAMP_IN_MILLIS = -62135769600000L; - - // 0001-01-01T00:00:00.0Z in millis - static final BigDecimal MINIMUM_ALLOWED_FRACTIONAL_MILLIS = new BigDecimal(MINIMUM_ALLOWED_TIMESTAMP_IN_MILLIS); - - // 10000T in millis, upper bound exclusive - static final BigDecimal FRACTIONAL_MILLIS_UPPER_BOUND = new BigDecimal(253402300800000L); - /** * Unknown local offset from UTC. */ @@ -162,18 +163,16 @@ public boolean includes(Precision isIncluded) */ private Precision _precision; - // These are the time field values for the Timestamp. - // _month and _day are 1-based (0 is an invalid value for - // these in a non-null Timestamp). - // TODO amzn/ion-java#28 - Represent internal time field values in its local time, - // instead of UTC. This makes it much less confusing. - private short _year; - private byte _month = 1; // Initialized to valid default - private byte _day = 1; // Initialized to valid default - private byte _hour; - private byte _minute; - private byte _second; - private BigDecimal _fraction; // fractional seconds, must be within range [0, 1) + /** + * Calendar to hold the Timestamp's year, month, day, hour, minute, second, and calendar system. Fractional + * seconds are left to {@link #_fraction}, while local offset is left to {@link #_offset}. + */ + private final Calendar _calendar; + + /** + * Fractional seconds. Must be within range [0, 1). + */ + private BigDecimal _fraction; /** * Minutes offset from UTC; zero means UTC proper, @@ -181,39 +180,18 @@ public boolean includes(Precision isIncluded) */ private Integer _offset; - // jan, feb, mar, apr, may, jun, jul, aug, sep, oct, nov, dec - // the first 0 is to make these arrays 1 based (since month values are 1-12) - private static final int[] LEAP_DAYS_IN_MONTH = { 0, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; - private static final int[] NORMAL_DAYS_IN_MONTH = { 0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; - - private static int last_day_in_month(int year, int month) { - boolean is_leap; - if ((year % 4) == 0) { - // divisible by 4 (lower 2 bits are zero) - may be a leap year - if ((year % 100) == 0) { - // and divisible by 100 - not a leap year - if ((year % 400) == 0) { - // but divisible by 400 - then it is a leap year - is_leap = true; - } - else { - is_leap = false; - } - } - else { - is_leap = true; - } - } - else { - is_leap = false; - } - return is_leap ? LEAP_DAYS_IN_MONTH[month] : NORMAL_DAYS_IN_MONTH[month]; - } + // Minimum millis under the calendar system provided by the default GregorianCalendar implementation. + private static final long MINIMUM_TIMESTAMP_IN_MILLIS = Timestamp.valueOf("0001-01-01T00:00:00.000Z").getMillis(); + static final BigDecimal MINIMUM_TIMESTAMP_IN_MILLIS_DECIMAL = new BigDecimal(MINIMUM_TIMESTAMP_IN_MILLIS); + + // 10000T in millis, upper bound exclusive, under the calendar system provided by the default GregorianCalendar implementation. + private static final long UPPER_BOUND_TIMESTAMP_IN_MILLIS = Timestamp.valueOf("9999-12-31T23:59:59.999-00:00").getMillis() + 1; + static final BigDecimal UPPER_BOUND_TIMESTAMP_IN_MILLIS_DECIMAL = new BigDecimal(UPPER_BOUND_TIMESTAMP_IN_MILLIS); /** * Applies the local time zone offset from UTC to the applicable time field * values. Depending on the local time zone offset, adjustments - * (i.e. rollover) will be made to the Year, Day, Hour, Minute time field + * (i.e. rollover) will be made to the calendar's year, day, hour, and minute * values. * * @param offset the local offset, in minutes from UTC. @@ -228,164 +206,80 @@ private void apply_offset(int offset) offset = -offset; int hour_offset = offset / 60; int min_offset = offset - (hour_offset * 60); - if (offset < 0) { - _minute += min_offset; // lower the minute value by adding a negative offset - _hour += hour_offset; - if (_minute < 0) { - _minute += 60; - _hour -= 1; - } - if (_hour >= 0) return; // hour is 0-23 - _hour += 24; - _day -= 1; - if (_day >= 1) return; // day is 1-31 - // we can't do this until we've figured out the month and year: _day += last_day_in_month(_year, _month); - _month -= 1; - if (_month >= 1) { - _day += last_day_in_month(_year, _month); // now we know (when the year doesn't change - assert(_day == last_day_in_month(_year, _month)); - return; // 1-12 - } - _month += 12; - _year -= 1; - if (_year < 1) throw new IllegalArgumentException("year is less than 1"); - _day += last_day_in_month(_year, _month); // and now we know, even if the year did change - assert(_day == last_day_in_month(_year, _month)); - } - else { - _minute += min_offset; // lower the minute value by adding a negative offset - _hour += hour_offset; - if (_minute > 59) { - _minute -= 60; - _hour += 1; - } - if (_hour < 24) return; // hour is 0-23 - _hour -= 24; - _day += 1; - if (_day <= last_day_in_month(_year, _month)) return; // day is 1-31 - // we can't do this until we figure out the final month and year: _day -= last_day_in_month(_year, _month); - _day = 1; // this is always the case - _month += 1; - if (_month <= 12) { - return; // 1-12 - } - _month -= 12; - _year += 1; - if (_year > 9999) throw new IllegalArgumentException("year exceeds 9999"); - } + // First, clear the offsets that are already set. Otherwise, the 'add' calls will add them in, which will + // result in a double add. + _calendar.clear(Calendar.ZONE_OFFSET); + _calendar.clear(Calendar.DST_OFFSET); + _calendar.add(Calendar.MINUTE, min_offset); + _calendar.add(Calendar.HOUR_OF_DAY, hour_offset); } /** - * This method uses deprecated methods from {@link java.util.Date} - * instead of {@link Calendar} so that this code can be used (more easily) - * on the mobile Java platform (which has Date but does not have Calendar). + * Create a Calendar from a number of milliseconds and the given local offset. + * @param millis a number of epoch milliseconds. + * @param localOffset a local offset in minutes. + * @return a new Calendar. */ - @SuppressWarnings("deprecation") - private void set_fields_from_millis(long millis) - { - if(millis < MINIMUM_ALLOWED_TIMESTAMP_IN_MILLIS) { - throw new IllegalArgumentException("year is less than 1"); + private static Calendar calendarFromMillis(long millis, Integer localOffset) { + Calendar calendar = new GregorianCalendar(PrivateUtils.UTC); + calendar.clear(); + calendar.setTimeInMillis(millis); + if (localOffset != null) { + calendar.set(Calendar.ZONE_OFFSET, localOffset * 60 * 1000); } - - Date date = new Date(millis); - - // https://github.com/amzn/ion-java/issues/160 - // The java.util.Date(long) constructor expects an epoch time in milliseconds, and getYear(), getMonth(), - // getHour() on the resulting Date are supposed to return values adjusted to the default timezone. - // In Pacific Standard Time (offset -08:00), for a Date constructed with an epoch time equivalent to - // 0001-01-01T00:00:00.000Z, the Date.get*() methods should return values for - // 0000-12-31T16:00:00.000Z; however, Date.getYear() incorrectly returns a value for year 1 (-1899) - // in this scenario. The following if/else block compensates for this bug: - int currentRawOffset = TimeZone.getDefault().getRawOffset(); - if(currentRawOffset < 0 && MINIMUM_ALLOWED_TIMESTAMP_IN_MILLIS - currentRawOffset > millis) { - this._year = 0; - } else { - this._year = checkAndCastYear(date.getYear() + 1900); - } - - // Note: date.get*() return values are in the local timezone! - this._month = checkAndCastMonth(date.getMonth() + 1); // calendar months are 0 based, timestamp months are 1 based - this._day = checkAndCastDay(date.getDate(), _year, _month); - this._hour = checkAndCastHour(date.getHours()); - this._minute = checkAndCastMinute(date.getMinutes()); - this._second = checkAndCastSecond(date.getSeconds()); - - // this is done because the y-m-d values are in the local timezone - // so this adjusts the value back to zulu time (UTC) - // Note that the sign on this is opposite of Ion (and Calendar) offset. - // Example: PST = 480 here but Ion/Calendar use -480 = -08:00 = UTC-8 - int offset = date.getTimezoneOffset(); - this.apply_offset(-offset); + return calendar; } /** * Copies data from a {@link Calendar} into this timestamp. * Must only be called during construction due to timestamp immutabliity. * - * @param cal must have at least one field set. + * @param setLocalOffset if true and the given calendar has its ZONE_OFFSET set, sets the timestamp's _offset field. + * @param applyLocalOffset if true and _offset has been set to a known offset, applies the local offset to the + * timestamp's fields to convert them from local time to UTC. This should be false whenever + * the given calendar is already in UTC (such as when it was constructed from millis). * * @throws IllegalArgumentException if the calendar has no fields set. */ - private void set_fields_from_calendar(Calendar cal, - Precision precision, - boolean setLocalOffset) + private void setFieldsFromCalendar(Precision precision, + boolean setLocalOffset, + boolean applyLocalOffset) { _precision = precision; _offset = UNKNOWN_OFFSET; - boolean dayPrecision = false; - boolean calendarHasMilliseconds = cal.isSet(Calendar.MILLISECOND); + boolean calendarHasMilliseconds = _calendar.isSet(Calendar.MILLISECOND); switch (this._precision) { case SECOND: - this._second = checkAndCastSecond(cal.get(Calendar.SECOND)); if (calendarHasMilliseconds) { - BigDecimal millis = BigDecimal.valueOf(cal.get(Calendar.MILLISECOND)); + BigDecimal millis = BigDecimal.valueOf(_calendar.get(Calendar.MILLISECOND)); this._fraction = millis.movePointLeft(3); // convert to fraction + checkFraction(precision, this._fraction); } case MINUTE: { - this._hour = checkAndCastHour(cal.get(Calendar.HOUR_OF_DAY)); - this._minute = checkAndCastMinute(cal.get(Calendar.MINUTE)); - - // If this test is made before calling get(), it will return - // false even when Calendar.setTimeZone() was called. - if (setLocalOffset && cal.isSet(Calendar.ZONE_OFFSET)) + int offset = _calendar.get(Calendar.ZONE_OFFSET); + if (setLocalOffset) { - int offset = cal.get(Calendar.ZONE_OFFSET); - if (cal.isSet(Calendar.DST_OFFSET)) { - offset += cal.get(Calendar.DST_OFFSET); + if (_calendar.isSet(Calendar.DST_OFFSET)) { + offset += _calendar.get(Calendar.DST_OFFSET); } - // convert ms to minutes _offset = offset / (1000*60); } } case DAY: - dayPrecision = true; case MONTH: - // Calendar months are 0 based, Timestamp months are 1 based - this._month = checkAndCastMonth((cal.get(Calendar.MONTH) + 1)); case YEAR: - int year; - if(cal.get(Calendar.ERA) == GregorianCalendar.AD) { - year = cal.get(Calendar.YEAR); - } - else { - year = -cal.get(Calendar.YEAR); - } - - this._year = checkAndCastYear(year); - } - - if (dayPrecision) - { - this._day = checkAndCastDay(cal.get(Calendar.DAY_OF_MONTH), _year, _month); } - if (_offset != UNKNOWN_OFFSET) { + if (_offset != UNKNOWN_OFFSET && applyLocalOffset) { // Transform our members from local time to Zulu this.apply_offset(_offset); } + // fractional seconds are ONLY tracked by the _fraction field. + _calendar.clear(Calendar.MILLISECOND); + checkCalendarYear(_calendar); } /** @@ -482,6 +376,8 @@ private Timestamp(Precision p, int zyear, int zmonth, int zday, int zhour, int zminute, int zsecond, BigDecimal frac, Integer offset, boolean shouldApplyOffset) { + _calendar = new GregorianCalendar(PrivateUtils.UTC); + _calendar.clear(); boolean dayPrecision = false; switch (p) { @@ -496,22 +392,23 @@ private Timestamp(Precision p, int zyear, int zmonth, int zday, { _fraction = frac.abs(); } - _second = checkAndCastSecond(zsecond); + _calendar.set(Calendar.SECOND, checkAndCastSecond(zsecond)); case MINUTE: - _minute = checkAndCastMinute(zminute); - _hour = checkAndCastHour(zhour); + _calendar.set(Calendar.MINUTE, checkAndCastMinute(zminute)); + _calendar.set(Calendar.HOUR_OF_DAY, checkAndCastHour(zhour)); _offset = offset; // offset must be null for years/months/days case DAY: dayPrecision = true; case MONTH: - _month = checkAndCastMonth(zmonth); + _calendar.set(Calendar.MONTH, checkAndCastMonth(zmonth) - 1); case YEAR: - _year = checkAndCastYear(zyear); + _calendar.set(Calendar.YEAR, checkAndCastYear(zyear)); } if (dayPrecision) { - _day = checkAndCastDay(zday, zyear, zmonth); + checkCalendarDay(zday); + _calendar.set(Calendar.DAY_OF_MONTH, zday); } _precision = checkFraction(p, _fraction); @@ -528,8 +425,10 @@ private Timestamp(Precision p, int zyear, int zmonth, int zday, * applied to the time components. * As such, if the given {@code offset} is non-null or zero, the resulting * Timestamp will have time values that DO NOT match the time - * parameters. This method also has a behavior of precision "narrowing", - * detailed in the sub-section below. + * parameters. A default {@link GregorianCalendar} will be used to + * perform any arithmetic operations on the resulting Timestamp. This + * method also has a behavior of precision "narrowing", detailed in the + * sub-section below. * *

    * For example, the following method calls will return Timestamps with @@ -625,51 +524,62 @@ else if (cal.isSet(Calendar.YEAR)) { else { throw new IllegalArgumentException("Calendar has no fields set"); } - - set_fields_from_calendar(cal, precision, true); + _calendar = (Calendar) cal.clone(); + setFieldsFromCalendar(precision, true, APPLY_OFFSET_YES); } private Timestamp(Calendar cal, Precision precision, BigDecimal fraction, Integer offset) { - set_fields_from_calendar(cal, precision, false); + this._calendar = cal; + setFieldsFromCalendar(precision, false, APPLY_OFFSET_NO); _fraction = fraction; if (offset != null) { _offset = offset; - apply_offset(offset); } } + private static void throwTimestampOutOfRangeError(Number millis) { + throw new IllegalArgumentException("millis: " + millis + " is outside of valid the range: from " + + MINIMUM_TIMESTAMP_IN_MILLIS_DECIMAL + + " (0001T)" + + ", inclusive, to " + + UPPER_BOUND_TIMESTAMP_IN_MILLIS_DECIMAL + + " (10000T)" + + " , exclusive"); + } private Timestamp(BigDecimal millis, Precision precision, Integer localOffset) { - long ms = millis.longValue(); - set_fields_from_millis(ms); + if (millis == null) throw new NullPointerException("millis is null"); - switch (precision) - { - case YEAR: - _month = 1; - case MONTH: - _day = 1; - case DAY: - _hour = 0; - _minute = 0; - case MINUTE: - _second = 0; - _fraction = null; - break; - case SECOND: - BigDecimal secs = millis.movePointLeft(3); - BigDecimal secsDown = secs.setScale(0, RoundingMode.FLOOR); - _fraction = secs.subtract(secsDown); + // check bounds to avoid hanging when calling longValue() on decimals with large positive exponents, + // e.g. 1e10000000 + if(millis.compareTo(MINIMUM_TIMESTAMP_IN_MILLIS_DECIMAL) < 0 || + UPPER_BOUND_TIMESTAMP_IN_MILLIS_DECIMAL.compareTo(millis) <= 0) { + throwTimestampOutOfRangeError(millis); } - _precision = checkFraction(precision, _fraction); + // quick handle integral zero + long ms = isIntegralZero(millis) ? 0 : millis.longValue(); + + _calendar = calendarFromMillis(ms, localOffset); + setFieldsFromCalendar(precision, localOffset != null, APPLY_OFFSET_NO); - _offset = localOffset; + // The given BigDecimal may contain greater than milliseconds precision, which is the maximum precision that + // a Calendar can handle. Set the _fraction here so that extra precision (if any) is not lost. + // However, don't set the fraction if the given BigDecimal does not have precision at least to the tenth of + // a second. + if (precision == Precision.SECOND && millis.scale() > -3) { + BigDecimal secs = millis.movePointLeft(3); + BigDecimal secsDown = fastRoundZeroFloor(secs); + _fraction = secs.subtract(secsDown); + } else { + _fraction = null; + } + checkFraction(precision, _fraction); } @@ -708,37 +618,7 @@ private Timestamp(BigDecimal millis, Precision precision, Integer localOffset) @Deprecated private Timestamp(BigDecimal millis, Integer localOffset) { - if (millis == null) throw new NullPointerException("millis is null"); - - // check bounds to avoid hanging when calling longValue() on decimals with large positive exponents, - // e.g. 1e10000000 - if(millis.compareTo(MINIMUM_ALLOWED_FRACTIONAL_MILLIS) < 0 || - FRACTIONAL_MILLIS_UPPER_BOUND.compareTo(millis) < 0) { - throw new IllegalArgumentException("millis: " + millis + " is outside of valid the range: from " - + MINIMUM_ALLOWED_FRACTIONAL_MILLIS - + " (0001T)" - + ", inclusive, to " - + FRACTIONAL_MILLIS_UPPER_BOUND - + " (10000T)" - + " , exclusive"); - } - - // quick handle integral zero - long ms = isIntegralZero(millis) ? 0 : millis.longValue(); - - set_fields_from_millis(ms); - - this._precision = Precision.SECOND; - int scale = millis.scale(); - if (scale <= -3) { - this._fraction = null; - } - else { - BigDecimal secs = millis.movePointLeft(3); - BigDecimal secsDown = fastRoundZeroFloor(secs); - this._fraction = secs.subtract(secsDown); - } - this._offset = localOffset; + this(millis, Precision.SECOND, localOffset); } private BigDecimal fastRoundZeroFloor(final BigDecimal decimal) { @@ -769,15 +649,11 @@ private boolean isIntegralZero(final BigDecimal decimal) { @Deprecated private Timestamp(long millis, Integer localOffset) { - this.set_fields_from_millis(millis); - - // fractional seconds portion - BigDecimal secs = BigDecimal.valueOf(millis).movePointLeft(3); - BigDecimal secsDown = secs.setScale(0, RoundingMode.FLOOR); - this._fraction = secs.subtract(secsDown); - this._precision = checkFraction(Precision.SECOND, _fraction); - - this._offset = localOffset; + if(millis < MINIMUM_TIMESTAMP_IN_MILLIS || millis >= UPPER_BOUND_TIMESTAMP_IN_MILLIS) { + throwTimestampOutOfRangeError(millis); + } + this._calendar = calendarFromMillis(millis, localOffset); + setFieldsFromCalendar(Precision.SECOND, localOffset != null, APPLY_OFFSET_NO); } @@ -806,6 +682,8 @@ private static IllegalArgumentException fail(CharSequence input) /** * Returns a new Timestamp that represents the point in time, precision * and local offset defined in Ion format by the {@link CharSequence}. + * A default {@link GregorianCalendar} will be used to perform any + * arithmetic operations on the resulting Timestamp. * * @param ionFormattedTimestamp * a sequence of characters that is the Ion representation of a @@ -1069,27 +947,15 @@ private static boolean isValidFollowChar(char c) { /** * Creates a copy of this Timestamp. The resulting Timestamp will - * represent the same point in time and has the same precision and local - * offset. + * represent the same point in time, have the same precision and local + * offset, and use the same calendar system for date arithmetic. *

    * {@inheritDoc} */ @Override public Timestamp clone() { - // The Copy-Constructor we're using here already expects the time field - // values to be in UTC, and that is already what we have for this - // Timestamp -- no adjustment necessary to make it local time. - return new Timestamp(_precision, - _year, - _month, - _day, - _hour, - _minute, - _second, - _fraction, - _offset, - APPLY_OFFSET_NO); + return new Timestamp((Calendar) _calendar.clone(), _precision, _fraction, _offset); } /** @@ -1105,22 +971,7 @@ private Timestamp make_localtime() ? _offset.intValue() : 0; - // We use a Copy-Constructor that expects the time parameters to be in - // UTC, as that's what we're supposed to have. - // As this Copy-Constructor doesn't apply local offset to the time - // field values (it assumes that the local offset is already applied to - // them), we explicitly apply the local offset to the time field values - // after we obtain the new Timestamp instance. - Timestamp localtime = new Timestamp(_precision, - _year, - _month, - _day, - _hour, - _minute, - _second, - _fraction, - _offset, - APPLY_OFFSET_NO); + Timestamp localtime = clone(); // explicitly apply the local offset to the time field values localtime.apply_offset(-offset); @@ -1131,6 +982,8 @@ private Timestamp make_localtime() /** * Returns a Timestamp, precise to the year, with unknown local offset. + * A default {@link GregorianCalendar} will be used to perform any + * arithmetic operations on the resulting Timestamp. *

    * This is equivalent to the corresponding Ion value {@code YYYYT}. */ @@ -1141,6 +994,8 @@ public static Timestamp forYear(int yearZ) /** * Returns a Timestamp, precise to the month, with unknown local offset. + * A default {@link GregorianCalendar} will be used to perform any + * arithmetic operations on the resulting Timestamp. *

    * This is equivalent to the corresponding Ion value {@code YYYY-MMT}. */ @@ -1151,6 +1006,8 @@ public static Timestamp forMonth(int yearZ, int monthZ) /** * Returns a Timestamp, precise to the day, with unknown local offset. + * A default {@link GregorianCalendar} will be used to perform any + * arithmetic operations on the resulting Timestamp. *

    * This is equivalent to the corresponding Ion value {@code YYYY-MM-DD}. * @@ -1163,7 +1020,8 @@ public static Timestamp forDay(int yearZ, int monthZ, int dayZ) /** * Returns a Timestamp, precise to the minute, with a given local - * offset. + * offset. A default {@link GregorianCalendar} will be used to perform any + * arithmetic operations on the resulting Timestamp. *

    * This is equivalent to the corresponding Ion value * {@code YYYY-MM-DDThh:mm+-oo:oo}, where {@code oo:oo} represents the @@ -1184,6 +1042,8 @@ public static Timestamp forMinute(int year, int month, int day, /** * Returns a Timestamp, precise to the second, with a given local offset. + * A default {@link GregorianCalendar} will be used to perform any + * arithmetic operations on the resulting Timestamp. *

    * This is equivalent to the corresponding Ion value * {@code YYYY-MM-DDThh:mm:ss+-oo:oo}, where {@code oo:oo} represents the @@ -1204,6 +1064,8 @@ public static Timestamp forSecond(int year, int month, int day, /** * Returns a Timestamp, precise to the second, with a given local offset. + * A default {@link GregorianCalendar} will be used to perform any + * arithmetic operations on the resulting Timestamp. *

    * This is equivalent to the corresponding Ion value * {@code YYYY-MM-DDThh:mm:ss.sss+-oo:oo}, where {@code oo:oo} represents @@ -1232,6 +1094,14 @@ public static Timestamp forSecond(int year, int month, int day, /** * Returns a Timestamp that represents the point in time that is * {@code millis} milliseconds from the epoch, with a given local offset. + * A default {@link GregorianCalendar} will be used to perform any + * arithmetic operations on the resulting Timestamp. + *

    + * NOTE: this means that providing a number of milliseconds + * that was produced using a different calendar system may result in a Timestamp + * that represents a different point in time than the one that originally + * produced the milliseconds. In this case, {@link #forCalendar(Calendar)} should + * be used instead. *

    * The resulting Timestamp will be precise to the millisecond. * @@ -1252,7 +1122,14 @@ public static Timestamp forMillis(long millis, Integer localOffset) * Returns a Timestamp that represents the point in time that is * {@code millis} milliseconds (including any fractional * milliseconds) from the epoch, with a given local offset. - * + * A default {@link GregorianCalendar} will be used to perform any + * arithmetic operations on the resulting Timestamp. + *

    + * NOTE: this means that providing a number of milliseconds + * that was produced using a different calendar system may result in a Timestamp + * that represents a different point in time than the one that originally + * produced the milliseconds. In this case, {@link #forCalendar(Calendar)} should + * be used instead. *

    * The resulting Timestamp will be precise to the second if {@code millis} * doesn't contain information that is more granular than seconds. @@ -1290,7 +1167,8 @@ public static Timestamp forMillis(BigDecimal millis, Integer localOffset) /** * Converts a {@link Calendar} to a Timestamp, preserving the calendar's * time zone as the equivalent local offset when it has at least minutes - * precision. + * precision. The given Calendar will be used to perform any arithmetic + * operations on the resulting Timestamp. * * @return a Timestamp instance, with precision determined by the smallest * field set in the {@code Calendar}; @@ -1306,7 +1184,8 @@ public static Timestamp forCalendar(Calendar calendar) /** * Converts a {@link Date} to a Timestamp in UTC representing the same - * point in time. + * point in time. A default {@link GregorianCalendar} will be used to perform + * any arithmetic operations on the resulting Timestamp. *

    * The resulting Timestamp will be precise to the millisecond. * @@ -1325,7 +1204,8 @@ public static Timestamp forDateZ(Date date) /** * Converts a {@link java.sql.Timestamp} to a Timestamp in UTC representing - * the same point in time. + * the same point in time. A default {@link GregorianCalendar} will be used to perform + * any arithmetic operations on the resulting Timestamp. *

    * The resulting Timestamp will be precise to the nanosecond. * @@ -1352,7 +1232,8 @@ public static Timestamp forSqlTimestampZ(java.sql.Timestamp sqlTimestamp) /** * Returns a Timestamp representing the current time (based on the JVM - * clock), with an unknown local offset. + * clock), with an unknown local offset. A default {@link GregorianCalendar} + * will be used to perform any arithmetic operations on the resulting Timestamp. *

    * The resulting Timestamp will be precise to the millisecond. * @@ -1367,7 +1248,8 @@ public static Timestamp now() /** * Returns a Timestamp in UTC representing the current time (based on the - * the JVM clock). + * the JVM clock). A default {@link GregorianCalendar} will be used to perform + * any arithmetic operations on the resulting Timestamp. *

    * The resulting Timestamp will be precise to the millisecond. * @@ -1401,10 +1283,11 @@ public Date dateValue() return new Date(millis); } - /** * Converts the value of this Timestamp as a {@link Calendar}, in its - * local time. + * local time. The resulting Calendar will have its fields set up to + * this Timestamp's precision. The maximum fractional precision supported + * by Calendar is milliseconds; any additional precision will be truncated. *

    * Because {@link Calendar} instances are mutable, this method returns a * new instance from each call. @@ -1414,22 +1297,26 @@ public Date dateValue() */ public Calendar calendarValue() { - Calendar cal = new GregorianCalendar(PrivateUtils.UTC); - - long millis = getMillis(); - Integer offset = _offset; - if (offset != null && offset != 0) + Calendar cal = (Calendar) _calendar.clone(); + if (_precision.includes(Precision.SECOND)) { + if (this._fraction != null) + { + int fractionalMillis = this._fraction.movePointRight(3).intValue(); + cal.set(Calendar.MILLISECOND, fractionalMillis); + } + } + if (_precision.includes(Precision.MINUTE) && _offset != null && _offset != 0) { - int offsetMillis = offset * 60 * 1000; - millis += offsetMillis; - cal.setTimeInMillis(millis); // Resets the offset! + int offsetMillis = _offset * 60 * 1000; + cal.add(Calendar.MILLISECOND, offsetMillis); cal.set(Calendar.ZONE_OFFSET, offsetMillis); } - else - { - cal.setTimeInMillis(millis); + if (!_precision.includes(Precision.SECOND)) { + cal.clear(Calendar.SECOND); + } + if (_fraction == null) { + cal.clear(Calendar.MILLISECOND); } - return cal; } @@ -1437,7 +1324,7 @@ public Calendar calendarValue() /** * Returns a number representing the Timestamp's point in time that is * the number of milliseconds (ignoring any fractional milliseconds) - * from the epoch. + * from the epoch, using this Timestamp's configured Calendar. *

    * This method will return the same result for all Timestamps representing * the same point in time, regardless of the local offset. @@ -1446,11 +1333,9 @@ public Calendar calendarValue() * number of milliseconds (ignoring any fractional * milliseconds) from the epoch (1970-01-01T00:00:00.000Z) */ - @SuppressWarnings("deprecation") public long getMillis() { - // month is 0 based for Date - long millis = Date.UTC(this._year - 1900, this._month - 1, this._day, this._hour, this._minute, this._second); + long millis = _calendar.getTimeInMillis(); if (this._fraction != null) { BigDecimal fracAsDecimal = this._fraction.movePointRight(3); int frac = isIntegralZero(fracAsDecimal) ? 0 : fracAsDecimal.intValue(); @@ -1463,7 +1348,7 @@ public long getMillis() /** * Returns a BigDecimal representing the Timestamp's point in time that is * the number of milliseconds (including any fractional milliseconds) - * from the epoch. + * from the epoch, using this Timestamp's configured Calendar. *

    * This method will return the same result for all Timestamps representing * the same point in time, regardless of the local offset. @@ -1472,20 +1357,16 @@ public long getMillis() * number of milliseconds (including any fractional * milliseconds) from the epoch (1970-01-01T00:00:00.000Z) */ - @SuppressWarnings("deprecation") public BigDecimal getDecimalMillis() { - long millis; - BigDecimal dec; - switch (this._precision) { case YEAR: case MONTH: case DAY: case MINUTE: case SECOND: - millis = Date.UTC(this._year - 1900, this._month - 1, this._day, this._hour, this._minute, this._second); - dec = BigDecimal.valueOf(millis); + long millis = _calendar.getTimeInMillis(); + BigDecimal dec = BigDecimal.valueOf(millis); if (_fraction != null) { dec = dec.add(this._fraction.movePointRight(3)); } @@ -1540,7 +1421,7 @@ public int getYear() adjusted = make_localtime(); } } - return adjusted._year; + return adjusted.getZYear(); } @@ -1562,7 +1443,7 @@ public int getMonth() adjusted = make_localtime(); } } - return adjusted._month; + return adjusted.getZMonth(); } @@ -1582,7 +1463,7 @@ public int getDay() adjusted = make_localtime(); } } - return adjusted._day; + return adjusted.getZDay(); } @@ -1602,7 +1483,7 @@ public int getHour() adjusted = make_localtime(); } } - return adjusted._hour; + return adjusted.getZHour(); } @@ -1622,7 +1503,7 @@ public int getMinute() adjusted = make_localtime(); } } - return adjusted._minute; + return adjusted.getZMinute(); } @@ -1640,7 +1521,7 @@ public int getMinute() */ public int getSecond() { - return this._second; + return this.getZSecond(); } @@ -1659,7 +1540,7 @@ public int getSecond() */ public BigDecimal getDecimalSecond() { - BigDecimal sec = BigDecimal.valueOf(_second); + BigDecimal sec = BigDecimal.valueOf(getSecond()); if (_fraction != null) { sec = sec.add(_fraction); @@ -1676,7 +1557,7 @@ public BigDecimal getDecimalSecond() */ public int getZYear() { - return this._year; + return this._calendar.get(Calendar.YEAR); } @@ -1691,7 +1572,7 @@ public int getZYear() */ public int getZMonth() { - return this._month; + return this._calendar.get(Calendar.MONTH) + 1; } @@ -1705,7 +1586,7 @@ public int getZMonth() */ public int getZDay() { - return this._day; + return this._calendar.get(Calendar.DAY_OF_MONTH); } @@ -1719,7 +1600,7 @@ public int getZDay() */ public int getZHour() { - return this._hour; + return this._calendar.get(Calendar.HOUR_OF_DAY); } @@ -1733,7 +1614,7 @@ public int getZHour() */ public int getZMinute() { - return this._minute; + return this._calendar.get(Calendar.MINUTE); } @@ -1751,7 +1632,7 @@ public int getZMinute() */ public int getZSecond() { - return this._second; + return this._calendar.get(Calendar.SECOND); } @@ -1811,17 +1692,7 @@ public Timestamp withLocalOffset(Integer offset) { return this; } - - Timestamp ts = createFromUtcFields(precision, - getZYear(), - getZMonth(), - getZDay(), - getZHour(), - getZMinute(), - getZSecond(), - getZFractionalSecond(), - offset); - return ts; + return new Timestamp((Calendar) _calendar.clone(), precision, _fraction, offset); } @@ -1958,7 +1829,7 @@ private static void print(Appendable out, Timestamp adjusted) // so we have a real value - we'll start with the date portion // which we always have - print_digits(out, adjusted._year, 4); + print_digits(out, adjusted.getZYear(), 4); if (adjusted._precision == Precision.YEAR) { assert adjusted._offset == UNKNOWN_OFFSET; out.append("T"); @@ -1966,7 +1837,7 @@ private static void print(Appendable out, Timestamp adjusted) } out.append("-"); - print_digits(out, adjusted._month, 2); // convert calendar months to a base 1 value + print_digits(out, adjusted.getZMonth(), 2); // convert calendar months to a base 1 value if (adjusted._precision == Precision.MONTH) { assert adjusted._offset == UNKNOWN_OFFSET; out.append("T"); @@ -1974,7 +1845,7 @@ private static void print(Appendable out, Timestamp adjusted) } out.append("-"); - print_digits(out, adjusted._day, 2); + print_digits(out, adjusted.getZDay(), 2); if (adjusted._precision == Precision.DAY) { assert adjusted._offset == UNKNOWN_OFFSET; // out.append("T"); @@ -1982,13 +1853,13 @@ private static void print(Appendable out, Timestamp adjusted) } out.append("T"); - print_digits(out, adjusted._hour, 2); + print_digits(out, adjusted.getZHour(), 2); out.append(":"); - print_digits(out, adjusted._minute, 2); + print_digits(out, adjusted.getZMinute(), 2); // ok, so how much time do we have ? if (adjusted._precision == Precision.SECOND) { out.append(":"); - print_digits(out, adjusted._second, 2); + print_digits(out, adjusted.getZSecond(), 2); if (adjusted._fraction != null) { print_fractional_digits(out, adjusted._fraction); } @@ -2051,119 +1922,155 @@ private static void print_fractional_digits(Appendable out, BigDecimal value) //========================================================================= // Timestamp arithmetic - /** * Returns a timestamp relative to this one by the given number of - * milliseconds. + * milliseconds. Uses this Timestamp's configured Calendar to perform the + * arithmetic. * * @param amount a number of milliseconds. */ - public final Timestamp addMillis(long amount) - { - if (amount == 0) return this; - - // This strips off the local offset, expressing our fields as if they - // were UTC. - BigDecimal millis = make_localtime().getDecimalMillis(); - millis = millis.add(BigDecimal.valueOf(amount)); - - Timestamp ts = new Timestamp(millis, _precision, _offset); - - // Anything with courser-than-millis precision will have been extended - // to 3 decimal places due to use of getDecimalMillis(). Fix that. - ts._fraction = _fraction; - if (_offset != null && _offset != 0) - { - ts.apply_offset(_offset); + public final Timestamp addMillis(long amount) { + if (amount == 0 && _precision == Precision.SECOND && _fraction != null && _fraction.scale() >= 3) { + // Zero milliseconds are to be added, and the precision does not need to be increased. + return this; } + long seconds = amount / 1000; + BigDecimal millis = BigDecimal.valueOf(amount % 1000).movePointLeft(3); + if (_fraction != null) { + millis = _fraction.add(millis); + } + BigDecimal newFraction; + if (BigDecimal.ONE.compareTo(millis) <= 0) { + newFraction = millis.subtract(BigDecimal.ONE); + seconds += 1; + } else if (BigDecimal.ZERO.compareTo(millis) > 0) { + newFraction = BigDecimal.ONE.add(millis); + seconds -= 1; + } else { + newFraction = millis; + } + Timestamp ts = addSecond(seconds); + ts._fraction = newFraction; return ts; } + /** + * Adds the given amount to the given {@link Calendar} field and returns a new Timestamp. + * @param field the field. + * @param amount an amount. + * @param precision the precision corresponding to the given field. + * @return a new Timestamp instance. + */ + private Timestamp calendarAdd(int field, int amount, Precision precision) { + if (amount == 0 && _precision == precision) return this; + Timestamp timestamp = make_localtime(); + timestamp._calendar.add(field, amount); + checkCalendarYear(timestamp._calendar); + if (_offset != null) { + timestamp.apply_offset(_offset); + timestamp._offset = _offset; + } + timestamp._precision = _precision.includes(precision) ? timestamp._precision : precision; + return timestamp; + } /** * Returns a timestamp relative to this one by the given number of seconds. + * Uses this Timestamp's configured Calendar to perform the arithmetic. + * + * @param seconds a number of seconds. + */ + private final Timestamp addSecond(long seconds) { + Timestamp ts = this; + do { + int incrementalSeconds; + if (seconds > Integer.MAX_VALUE) { + incrementalSeconds = Integer.MAX_VALUE; + } else if (seconds < Integer.MIN_VALUE) { + incrementalSeconds = Integer.MIN_VALUE; + } else { + incrementalSeconds = (int)seconds; + } + ts = ts.addSecond(incrementalSeconds); + seconds -= incrementalSeconds; + } while (seconds != 0); + return ts; + } + + /** + * Returns a timestamp relative to this one by the given number of seconds. + * Uses this Timestamp's configured Calendar to perform the arithmetic. * * @param amount a number of seconds. */ public final Timestamp addSecond(int amount) { - long delta = (long) amount * 1000; - return addMillis(delta); + return calendarAdd(Calendar.SECOND, amount, Precision.SECOND); } /** * Returns a timestamp relative to this one by the given number of minutes. + * Uses this Timestamp's configured Calendar to perform the arithmetic. * * @param amount a number of minutes. */ public final Timestamp addMinute(int amount) { - long delta = (long) amount * 60 * 1000; - return addMillis(delta); + return calendarAdd(Calendar.MINUTE, amount, Precision.MINUTE); } /** * Returns a timestamp relative to this one by the given number of hours. + * Uses this Timestamp's configured Calendar to perform the arithmetic. * * @param amount a number of hours. */ public final Timestamp addHour(int amount) { - long delta = (long) amount * 60 * 60 * 1000; - return addMillis(delta); + return calendarAdd(Calendar.HOUR_OF_DAY, amount, Precision.MINUTE); } /** * Returns a timestamp relative to this one by the given number of days. + * Uses this Timestamp's configured Calendar to perform the arithmetic. * * @param amount a number of days. */ public final Timestamp addDay(int amount) { - long delta = (long) amount * 24 * 60 * 60 * 1000; - return addMillis(delta); + return calendarAdd(Calendar.DAY_OF_MONTH, amount, Precision.DAY); } - - // Shifting month and year are more complicated since the length of a month - // varies and we want the day-of-month to stay the same when possible. - // We rely on Calendar for the logic. - /** * Returns a timestamp relative to this one by the given number of months. * The day field may be adjusted to account for different month length and - * leap days. For example, adding one month to {@code 2011-01-31} - * results in {@code 2011-02-28}. + * leap days, as required by the configured Calendar's rules. For example, + * using the default {@link GregorianCalendar}, adding one month to + * {@code 2011-01-31} results in {@code 2011-02-28}. * * @param amount a number of months. */ public final Timestamp addMonth(int amount) { - if (amount == 0) return this; - - Calendar cal = calendarValue(); - cal.add(Calendar.MONTH, amount); - return new Timestamp(cal, _precision, _fraction, _offset); + return calendarAdd(Calendar.MONTH, amount, Precision.MONTH); } /** * Returns a timestamp relative to this one by the given number of years. - * The day field may be adjusted to account for leap days. For example, - * adding one year to {@code 2012-02-29} results in {@code 2013-02-28}. + * The day field may be adjusted to account for leap days, as required by + * the configured Calendar's rules. For example, using the default + * {@link GregorianCalendar}, adding one year to {@code 2012-02-29} results + * in {@code 2013-02-28}. * * @param amount a number of years. */ public final Timestamp addYear(int amount) { - if (amount == 0) return this; - - Calendar cal = calendarValue(); - cal.add(Calendar.YEAR, amount); - return new Timestamp(cal, _precision, _fraction, _offset); + return calendarAdd(Calendar.YEAR, amount, Precision.YEAR); } @@ -2190,12 +2097,12 @@ public int hashCode() result ^= (result << 19) ^ (result >> 13); - result = prime * result + this._year; - result = prime * result + this._month; - result = prime * result + this._day; - result = prime * result + this._hour; - result = prime * result + this._minute; - result = prime * result + this._second; + result = prime * result + this.getZYear(); + result = prime * result + this.getZMonth(); + result = prime * result + this.getZDay(); + result = prime * result + this.getZHour(); + result = prime * result + this.getZMinute(); + result = prime * result + this.getZSecond(); result ^= (result << 19) ^ (result >> 13); @@ -2355,12 +2262,12 @@ public boolean equals(Timestamp t) } // so now we check the actual time value - if (this._year != t._year) return false; - if (this._month != t._month) return false; - if (this._day != t._day) return false; - if (this._hour != t._hour) return false; - if (this._minute != t._minute) return false; - if (this._second != t._second) return false; + if (this.getZYear() != t.getZYear()) return false; + if (this.getZMonth() != t.getZMonth()) return false; + if (this.getZDay() != t.getZDay()) return false; + if (this.getZHour() != t.getZHour()) return false; + if (this.getZMinute() != t.getZMinute()) return false; + if (this.getZSecond() != t.getZSecond()) return false; // and if we have a local offset, check the value here if (this._offset != null) { @@ -2381,6 +2288,14 @@ public boolean equals(Timestamp t) return this._fraction.equals(t._fraction); } + private static void checkCalendarYear(Calendar calendar) { + int year = calendar.get(Calendar.YEAR); + if (calendar.get(Calendar.ERA) == GregorianCalendar.BC) { + year *= -1; + } + checkAndCastYear(year); + } + private static short checkAndCastYear(int year) { if (year < 1 || year > 9999) @@ -2401,14 +2316,11 @@ private static byte checkAndCastMonth(int month) return (byte) month; } - private static byte checkAndCastDay(int day, int year, int month) - { - int lastDayInMonth = last_day_in_month(year, month); - if (day < 1 || day > lastDayInMonth) { - throw new IllegalArgumentException(String.format("Day %s for year %s and month %s must be between 1 and %s inclusive", day, year, month, lastDayInMonth)); + private void checkCalendarDay(int day) { + int lastDayInMonth = _calendar.getActualMaximum(Calendar.DAY_OF_MONTH); + if (day > lastDayInMonth || day < _calendar.getActualMinimum(Calendar.DAY_OF_MONTH)) { + throw new IllegalArgumentException(String.format("Day %s for year %s and month %s must be between 1 and %s inclusive", day, getZYear(), getZMonth(), lastDayInMonth)); } - - return (byte) day; } private static byte checkAndCastHour(int hour) diff --git a/test/software/amazon/ion/TimestampTest.java b/test/software/amazon/ion/TimestampTest.java index 5ca81e707b..491aaa89ac 100644 --- a/test/software/amazon/ion/TimestampTest.java +++ b/test/software/amazon/ion/TimestampTest.java @@ -16,8 +16,8 @@ import static software.amazon.ion.Decimal.NEGATIVE_ZERO; import static software.amazon.ion.Decimal.negativeZero; -import static software.amazon.ion.Timestamp.FRACTIONAL_MILLIS_UPPER_BOUND; -import static software.amazon.ion.Timestamp.MINIMUM_ALLOWED_FRACTIONAL_MILLIS; +import static software.amazon.ion.Timestamp.UPPER_BOUND_TIMESTAMP_IN_MILLIS_DECIMAL; +import static software.amazon.ion.Timestamp.MINIMUM_TIMESTAMP_IN_MILLIS_DECIMAL; import static software.amazon.ion.Timestamp.UNKNOWN_OFFSET; import static software.amazon.ion.Timestamp.UTC_OFFSET; import static software.amazon.ion.Timestamp.createFromUtcFields; @@ -28,16 +28,16 @@ import static software.amazon.ion.Timestamp.Precision.YEAR; import static software.amazon.ion.impl.PrivateUtils.UTC; -import java.io.IOException; import java.math.BigDecimal; -import java.sql.Time; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.time.Instant; +import java.time.ZoneId; import java.util.Calendar; import java.util.Date; +import java.util.GregorianCalendar; import java.util.TimeZone; -import org.junit.Ignore; + import org.junit.Test; import software.amazon.ion.Timestamp.Precision; @@ -789,17 +789,23 @@ public void testForMillisWithNegativeMilli() } @Test - @Ignore // see https://github.com/amzn/ion-java/issues/160 public void testNewTimestampFromMinimumAllowedMillis() { - Timestamp ts = Timestamp.forMillis(MINIMUM_ALLOWED_FRACTIONAL_MILLIS, PST_OFFSET); + Timestamp ts = Timestamp.forMillis(MINIMUM_TIMESTAMP_IN_MILLIS_DECIMAL, PST_OFFSET); assertEquals("0001-01-01T00:00:00.000Z", ts.toZString()); } + @Test + public void testNewMinimumTimestampFromStringAndMillisIsSame() { + Timestamp t1 = Timestamp.valueOf("0001-01-01T00:00:00.000Z"); + Timestamp t2 = Timestamp.forMillis(t1.getMillis(), 0); + assertEquals(t1, t2); + } + @Test public void testNewTimestampFromBigDecimalWithMaximumAllowedMillis() { - Timestamp ts = Timestamp.forMillis(FRACTIONAL_MILLIS_UPPER_BOUND.add(BigDecimal.ONE.negate()), PST_OFFSET); + Timestamp ts = Timestamp.forMillis(UPPER_BOUND_TIMESTAMP_IN_MILLIS_DECIMAL.add(BigDecimal.ONE.negate()), PST_OFFSET); checkFields(9999, 12, 31, 15, 59, 59, new BigDecimal("0.999"), PST_OFFSET, SECOND, ts); assertEquals("9999-12-31T15:59:59.999-08:00", ts.toString()); assertEquals("9999-12-31T23:59:59.999Z", ts.toZString()); @@ -815,14 +821,26 @@ public void testNewTimestampFromBigDecimalWithNull() public void testNewTimestampFromBigDecimalWithMillisTooSmall() { // MIN - 1 - Timestamp.forMillis(MINIMUM_ALLOWED_FRACTIONAL_MILLIS.add(BigDecimal.ONE.negate()), PST_OFFSET); + Timestamp.forMillis(MINIMUM_TIMESTAMP_IN_MILLIS_DECIMAL.add(BigDecimal.ONE.negate()), PST_OFFSET); + } + + @Test(expected = IllegalArgumentException.class) + public void testNewTimestampFromLongWithMillisTooSmall() + { + Timestamp.forMillis(MINIMUM_TIMESTAMP_IN_MILLIS_DECIMAL.longValue() - 1, 0); } @Test(expected = IllegalArgumentException.class) public void testNewTimestampFromBigDecimalWithMillisTooBig() { // Max - Timestamp.forMillis(FRACTIONAL_MILLIS_UPPER_BOUND, PST_OFFSET); + Timestamp.forMillis(UPPER_BOUND_TIMESTAMP_IN_MILLIS_DECIMAL, PST_OFFSET); + } + + @Test(expected = IllegalArgumentException.class) + public void testNewTimestampFromLongWithMillisTooBig() + { + Timestamp.forMillis(UPPER_BOUND_TIMESTAMP_IN_MILLIS_DECIMAL.longValue(), PST_OFFSET); } @Test(expected = IllegalArgumentException.class) @@ -902,58 +920,6 @@ public void testNewTimestampFromLong() assertEquals("2010-02-01T10:11:12.340Z", ts.toZString()); } - /** - * Regression test for https://github.com/amzn/ion-java/issues/160 - */ - @Test - public void testNewTimestampFromYearOneRegressionBug() - { - // This is the same as the private field Timestamp.MINIMUM_ALLOWED_TIMESTAMP_IN_MILLIS - final long MINIMUM_TIMESTAMP_MILLIS = -62135769600000L; - // This is the minimum timestamp instantiating by parsing a timestamp string, - // which is not effected by the `java.util.Date` bug. - final Timestamp MINIMUM_TIMESTAMP = Timestamp.valueOf("0001-01-01T00:00:00.000Z"); - // Save the original timezone since we modify it below. - final TimeZone originalTimeZone = TimeZone.getDefault(); - - try { - // The `java.util.Date` bug has a different range of times depending on the local offset, so - // let's test against all time zones. - for (final String tzId : TimeZone.getAvailableIDs()) { - final TimeZone tz = TimeZone.getTimeZone(tzId); - TimeZone.setDefault(tz); - - // The Timestamps under test must be instantiated *after* the the call to TimeZone.setDefault() - // since `java.util.Date` references the default TimeZone when calculating the values for its - // date field accessor methods (i.e. Date.get*()). - Timestamp minimumTimestampFromMillis = Timestamp.forMillis(MINIMUM_TIMESTAMP_MILLIS, 0); - assertEquals(MINIMUM_TIMESTAMP, minimumTimestampFromMillis); - - // Only perform further assertions on timezones with negative offsets because positive - // offsets create millisecond values that are less than Timestamp.MINIMUM_ALLOWED_TIMESTAMP_IN_MILLIS - if (tz.getRawOffset() < 0) { - // This will be the latest millisecond in which the bug with `java.util.Date` can happen - Timestamp maximumTimestampWithBug = - Timestamp.forMillis(MINIMUM_TIMESTAMP_MILLIS - tz.getRawOffset(), 0); - assertEquals(1, maximumTimestampWithBug.getYear()); - - // Test one millisecond after, just to be sure. - Timestamp timestampPlusOne = - Timestamp.forMillis(MINIMUM_TIMESTAMP_MILLIS - tz.getRawOffset() + 1, 0); - assertEquals(1, timestampPlusOne.getYear()); - - // Also test one milliscond before, just to be sure. - Timestamp timestampMinusOne = - Timestamp.forMillis(MINIMUM_TIMESTAMP_MILLIS - tz.getRawOffset() - 1L, 0); - assertEquals(1, timestampMinusOne.getYear()); - } - } - } finally { - // Have to set the TimeZone back to its original value so as not to effect other tests. - TimeZone.setDefault(originalTimeZone); - } - } - /** * Test for {@link Timestamp#createFromUtcFields(Precision, int, int, int, int, int, int, BigDecimal, Integer)} * ensuring that varying precisions produce Timestamps as expected as per @@ -1768,9 +1734,9 @@ private void addMonthWithMins(String orig, int amount, String expected) @Test public void testAddMonth() { - // TODO Is it reasonable to add months to TSs that aren't that precise? - addMonth("2012T", -1, "2011T"); - addMonth("2012T", 1, "2012T"); + // Adding more precise amounts extends the precision. + addMonth("2012T", -1, "2011-12T"); + addMonth("2012T", 1, "2012-02T"); addMonth("2012-04T", -4, "2011-12T"); addMonth("2012-04T", -1, "2012-03T"); @@ -1851,17 +1817,18 @@ private void addDayWithMins(String orig, int amount, String expected) @Test public void testAddDay() { - // TODO Is it reasonable to add days to TSs that aren't that precise? - addDay("2012T", 1, "2012T"); - addDay("2012T", -1, "2011T"); + // Adding more precise amounts extends the precision. + addDay("2012T", 1, "2012-01-02"); + addDay("2012T", 0, "2012-01-01"); + addDay("2012T", -1, "2011-12-31"); - addDay("2012-04T",-32, "2012-02T"); - addDay("2012-04T",-31, "2012-03T"); - addDay("2012-04T", -1, "2012-03T"); - addDay("2012-04T", 1, "2012-04T"); - addDay("2012-04T", 9, "2012-04T"); - addDay("2012-04T", 30, "2012-05T"); - addDay("2012-04T", 31, "2012-05T"); + addDay("2012-04T",-32, "2012-02-29"); + addDay("2012-04T",-31, "2012-03-01"); + addDay("2012-04T", -1, "2012-03-31"); + addDay("2012-04T", 1, "2012-04-02"); + addDay("2012-04T", 9, "2012-04-10"); + addDay("2012-04T", 30, "2012-05-01"); + addDay("2012-04T", 31, "2012-05-02"); addDayWithMins("2011-01-31",-31, "2010-12-31"); addDayWithMins("2011-01-31",-30, "2011-01-01"); @@ -1931,34 +1898,34 @@ private void addHourWithMins(String orig, int amount, String expected) @Test public void testAddHour() { - // TODO Is it reasonable to add hours to TSs that aren't that precise? - addHour("2012T", -1, "2011T"); - addHour("2012T", 0, "2012T"); - addHour("2012T", 1, "2012T"); - - addHour("2012-04T", -31 * 24 - 1, "2012-02T"); - addHour("2012-04T", -31 * 24 , "2012-03T"); - addHour("2012-04T", -1, "2012-03T"); - addHour("2012-04T", 0, "2012-04T"); - addHour("2012-04T", 1, "2012-04T"); - addHour("2012-04T", 30 * 24 - 1, "2012-04T"); - addHour("2012-04T", 30 * 24 , "2012-05T"); - - addHour("2011-02-28", -1, "2011-02-27"); - addHour("2011-02-28", 0, "2011-02-28"); - addHour("2011-02-28", 1, "2011-02-28"); - addHour("2011-02-28", 24, "2011-03-01"); - addHour("2011-03-01", -24, "2011-02-28"); - addHour("2012-02-28", 24, "2012-02-29"); - addHour("2012-02-29", -24, "2012-02-28"); - addHour("2012-02-29", 24, "2012-03-01"); - - addHour("2012-10-04",-48, "2012-10-02"); - addHour("2012-10-04",-24, "2012-10-03"); - addHour("2012-10-04", -1, "2012-10-03"); - addHour("2012-10-04", 1, "2012-10-04"); - addHour("2012-10-04", 24, "2012-10-05"); - addHour("2012-10-04", 48, "2012-10-06"); + // Adding more precise amounts extends the precision. + addHour("2012T", -1, "2011-12-31T23:00-00:00"); + addHour("2012T", 0, "2012-01-01T00:00-00:00"); + addHour("2012T", 1, "2012-01-01T01:00-00:00"); + + addHour("2012-04T", -31 * 24 - 1, "2012-02-29T23:00-00:00"); + addHour("2012-04T", -31 * 24 , "2012-03-01T00:00-00:00"); + addHour("2012-04T", -1, "2012-03-31T23:00-00:00"); + addHour("2012-04T", 0, "2012-04-01T00:00-00:00"); + addHour("2012-04T", 1, "2012-04-01T01:00-00:00"); + addHour("2012-04T", 30 * 24 - 1, "2012-04-30T23:00-00:00"); + addHour("2012-04T", 30 * 24 , "2012-05-01T00:00-00:00"); + + addHour("2011-02-28", -1, "2011-02-27T23:00-00:00"); + addHour("2011-02-28", 0, "2011-02-28T00:00-00:00"); + addHour("2011-02-28", 1, "2011-02-28T01:00-00:00"); + addHour("2011-02-28", 24, "2011-03-01T00:00-00:00"); + addHour("2011-03-01", -24, "2011-02-28T00:00-00:00"); + addHour("2012-02-28", 24, "2012-02-29T00:00-00:00"); + addHour("2012-02-29", -24, "2012-02-28T00:00-00:00"); + addHour("2012-02-29", 24, "2012-03-01T00:00-00:00"); + + addHour("2012-10-04",-48, "2012-10-02T00:00-00:00"); + addHour("2012-10-04",-24, "2012-10-03T00:00-00:00"); + addHour("2012-10-04", -1, "2012-10-03T23:00-00:00"); + addHour("2012-10-04", 1, "2012-10-04T01:00-00:00"); + addHour("2012-10-04", 24, "2012-10-05T00:00-00:00"); + addHour("2012-10-04", 48, "2012-10-06T00:00-00:00"); addHourWithMins("2011-01-31T12", 0, "2011-01-31T12"); addHourWithMins("2011-01-31T12", 12, "2011-02-01T00"); @@ -2025,34 +1992,34 @@ private void addMinuteWithsSecs(String orig, int amount, String expected) @Test public void testAddMinute() { - // TODO Is it reasonable to add minutes to TSs that aren't that precise? - addMinute("2012T", -1, "2011T"); - addMinute("2012T", 0, "2012T"); - addMinute("2012T", 1, "2012T"); - - addMinute("2012-04T", -31 * 24 * 60 - 1, "2012-02T"); - addMinute("2012-04T", -31 * 24 * 60 , "2012-03T"); - addMinute("2012-04T", -1, "2012-03T"); - addMinute("2012-04T", 0, "2012-04T"); - addMinute("2012-04T", 1, "2012-04T"); - addMinute("2012-04T", 30 * 24 * 60 - 1, "2012-04T"); - addMinute("2012-04T", 30 * 24 * 60 , "2012-05T"); - - addMinute("2011-02-28", -1, "2011-02-27"); - addMinute("2011-02-28", 0, "2011-02-28"); - addMinute("2011-02-28", 1, "2011-02-28"); - addMinute("2011-02-28", 24 * 60, "2011-03-01"); - addMinute("2011-03-01", -24 * 60, "2011-02-28"); - addMinute("2012-02-28", 24 * 60, "2012-02-29"); - addMinute("2012-02-29", -24 * 60, "2012-02-28"); - addMinute("2012-02-29", 24 * 60, "2012-03-01"); - - addMinute("2012-10-04", -48 * 60, "2012-10-02"); - addMinute("2012-10-04", -24 * 60, "2012-10-03"); - addMinute("2012-10-04", -1, "2012-10-03"); - addMinute("2012-10-04", 1, "2012-10-04"); - addMinute("2012-10-04", 24 * 60, "2012-10-05"); - addMinute("2012-10-04", 48 * 60, "2012-10-06"); + // Adding more precise amounts extends the precision. + addMinute("2012T", -1, "2011-12-31T23:59-00:00"); + addMinute("2012T", 0, "2012-01-01T00:00-00:00"); + addMinute("2012T", 1, "2012-01-01T00:01-00:00"); + + addMinute("2012-04T", -31 * 24 * 60 - 1, "2012-02-29T23:59-00:00"); + addMinute("2012-04T", -31 * 24 * 60 , "2012-03-01T00:00-00:00"); + addMinute("2012-04T", -1, "2012-03-31T23:59-00:00"); + addMinute("2012-04T", 0, "2012-04-01T00:00-00:00"); + addMinute("2012-04T", 1, "2012-04-01T00:01-00:00"); + addMinute("2012-04T", 30 * 24 * 60 - 1, "2012-04-30T23:59-00:00"); + addMinute("2012-04T", 30 * 24 * 60 , "2012-05-01T00:00-00:00"); + + addMinute("2011-02-28", -1, "2011-02-27T23:59-00:00"); + addMinute("2011-02-28", 0, "2011-02-28T00:00-00:00"); + addMinute("2011-02-28", 1, "2011-02-28T00:01-00:00"); + addMinute("2011-02-28", 24 * 60, "2011-03-01T00:00-00:00"); + addMinute("2011-03-01", -24 * 60, "2011-02-28T00:00-00:00"); + addMinute("2012-02-28", 24 * 60, "2012-02-29T00:00-00:00"); + addMinute("2012-02-29", -24 * 60, "2012-02-28T00:00-00:00"); + addMinute("2012-02-29", 24 * 60, "2012-03-01T00:00-00:00"); + + addMinute("2012-10-04", -48 * 60, "2012-10-02T00:00-00:00"); + addMinute("2012-10-04", -24 * 60, "2012-10-03T00:00-00:00"); + addMinute("2012-10-04", -1, "2012-10-03T23:59-00:00"); + addMinute("2012-10-04", 1, "2012-10-04T00:01-00:00"); + addMinute("2012-10-04", 24 * 60, "2012-10-05T00:00-00:00"); + addMinute("2012-10-04", 48 * 60, "2012-10-06T00:00-00:00"); addMinuteWithsSecs("2011-01-31T12:03", -60, "2011-01-31T11:03"); addMinuteWithsSecs("2011-01-31T12:03", 0, "2011-01-31T12:03"); @@ -2120,35 +2087,35 @@ private void addSecondWithsFrac(String orig, int amount, String expected) @Test public void testAddSecond() { - // TODO Is it reasonable to add seconds to TSs that aren't that precise? - addSecond("2012T", -1, "2011T"); - addSecond("2012T", 0, "2012T"); - addSecond("2012T", 1, "2012T"); - - addSecond("2012-04T", -31 * 24 * 60 * 60 - 1, "2012-02T"); - addSecond("2012-04T", -31 * 24 * 60 * 60 , "2012-03T"); - addSecond("2012-04T", -1, "2012-03T"); - addSecond("2012-04T", 0, "2012-04T"); - addSecond("2012-04T", 1, "2012-04T"); - addSecond("2012-04T", 30 * 24 * 60 * 60 - 1, "2012-04T"); - addSecond("2012-04T", 30 * 24 * 60 * 60 , "2012-05T"); - - - addSecond("2011-02-28", -1, "2011-02-27"); - addSecond("2011-02-28", 0, "2011-02-28"); - addSecond("2011-02-28", 1, "2011-02-28"); - addSecond("2011-02-28", 24 * 60 * 60, "2011-03-01"); - addSecond("2011-03-01", -24 * 60 * 60, "2011-02-28"); - addSecond("2012-02-28", 24 * 60 * 60, "2012-02-29"); - addSecond("2012-02-29", -24 * 60 * 60, "2012-02-28"); - addSecond("2012-02-29", 24 * 60 * 60, "2012-03-01"); - - addSecond("2012-10-04", -24 * 60 * 60 - 1, "2012-10-02"); - addSecond("2012-10-04", -24 * 60 * 60 , "2012-10-03"); - addSecond("2012-10-04", -1, "2012-10-03"); - addSecond("2012-10-04", 1, "2012-10-04"); - addSecond("2012-10-04", 48 * 60 * 60 - 1, "2012-10-05"); - addSecond("2012-10-04", 48 * 60 * 60 , "2012-10-06"); + // Adding more precise amounts extends the precision. + addSecond("2012T", -1, "2011-12-31T23:59:59-00:00"); + addSecond("2012T", 0, "2012-01-01T00:00:00-00:00"); + addSecond("2012T", 1, "2012-01-01T00:00:01-00:00"); + + addSecond("2012-04T", -31 * 24 * 60 * 60 - 1, "2012-02-29T23:59:59-00:00"); + addSecond("2012-04T", -31 * 24 * 60 * 60 , "2012-03-01T00:00:00-00:00"); + addSecond("2012-04T", -1, "2012-03-31T23:59:59-00:00"); + addSecond("2012-04T", 0, "2012-04-01T00:00:00-00:00"); + addSecond("2012-04T", 1, "2012-04-01T00:00:01-00:00"); + addSecond("2012-04T", 30 * 24 * 60 * 60 - 1, "2012-04-30T23:59:59-00:00"); + addSecond("2012-04T", 30 * 24 * 60 * 60 , "2012-05-01T00:00:00-00:00"); + + + addSecond("2011-02-28", -1, "2011-02-27T23:59:59-00:00"); + addSecond("2011-02-28", 0, "2011-02-28T00:00:00-00:00"); + addSecond("2011-02-28", 1, "2011-02-28T00:00:01-00:00"); + addSecond("2011-02-28", 24 * 60 * 60, "2011-03-01T00:00:00-00:00"); + addSecond("2011-03-01", -24 * 60 * 60, "2011-02-28T00:00:00-00:00"); + addSecond("2012-02-28", 24 * 60 * 60, "2012-02-29T00:00:00-00:00"); + addSecond("2012-02-29", -24 * 60 * 60, "2012-02-28T00:00:00-00:00"); + addSecond("2012-02-29", 24 * 60 * 60, "2012-03-01T00:00:00-00:00"); + + addSecond("2012-10-04", -24 * 60 * 60 - 1, "2012-10-02T23:59:59-00:00"); + addSecond("2012-10-04", -24 * 60 * 60 , "2012-10-03T00:00:00-00:00"); + addSecond("2012-10-04", -1, "2012-10-03T23:59:59-00:00"); + addSecond("2012-10-04", 1, "2012-10-04T00:00:01-00:00"); + addSecond("2012-10-04", 48 * 60 * 60 - 1, "2012-10-05T23:59:59-00:00"); + addSecond("2012-10-04", 48 * 60 * 60 , "2012-10-06T00:00:00-00:00"); addSecondWithsFrac("2011-01-31T12:03:23", -60 * 60, "2011-01-31T11:03:23"); addSecondWithsFrac("2011-01-31T12:03:23", 0, "2011-01-31T12:03:23"); @@ -2162,6 +2129,221 @@ public void testAddSecond() addSecondWithsFrac("2012-03-01T02:35:23", -4 * 60 * 60, "2012-02-29T22:35:23"); } + private void addMillisecond(String orig, long amount, String expected) + { + Timestamp ts1 = Timestamp.valueOf(orig); + Timestamp ts2 = ts1.addMillis(amount); + checkTimestamp(expected, ts2); + } + + private void addMillisecond(String orig, long amount, String expected, String suffix) + { + addMillisecond(orig + suffix, amount, expected + suffix); + } + + private void addMillisecondWithOffsets(String orig, long amount, String expected) { + addMillisecond(orig, amount, expected, "-00:00"); + addMillisecond(orig, amount, expected, "Z"); + addMillisecond(orig, amount, expected, "+01:23"); + addMillisecond(orig, amount, expected, "-01:23"); + addMillisecond(orig, amount, expected, "+23:59"); + addMillisecond(orig, amount, expected, "-23:59"); + } + + private void addMillisecondWithFrac(String orig, long amount, String expected) + { + // If the timestamp has less than milliseconds precision, always expand its precision to milliseconds. + addMillisecondWithOffsets(orig, amount, expected + ".000"); + addMillisecondWithOffsets(orig + ".0", amount, expected + ".000"); + addMillisecondWithOffsets(orig + ".00", amount, expected + ".000"); + addMillisecondWithOffsets(orig + ".000", amount, expected +".000"); + addMillisecondWithOffsets(orig + ".456", amount, expected + ".456"); + // If the Timestamp has greater than milliseconds precision, don't reduce the precision. + addMillisecondWithOffsets(orig + ".0000", amount, expected + ".0000"); + addMillisecondWithOffsets(orig + ".45678", amount, expected +".45678"); + } + + @Test + public void testAddMillisecond() + { + addMillisecond("2012T", -1, "2011-12-31T23:59:59.999-00:00"); + addMillisecond("2012T", 0, "2012-01-01T00:00:00.000-00:00"); + addMillisecond("2012T", 1, "2012-01-01T00:00:00.001-00:00"); + + addMillisecond("2012-04T", -31L * 24 * 60 * 60 * 1000 - 1, "2012-02-29T23:59:59.999-00:00"); + addMillisecond("2012-04T", -31L * 24 * 60 * 60 * 1000 , "2012-03-01T00:00:00.000-00:00"); + addMillisecond("2012-04T", -1, "2012-03-31T23:59:59.999-00:00"); + addMillisecond("2012-04T", 0, "2012-04-01T00:00:00.000-00:00"); + addMillisecond("2012-04T", 1, "2012-04-01T00:00:00.001-00:00"); + addMillisecond("2012-04T", 30L * 24 * 60 * 60 * 1000 - 1, "2012-04-30T23:59:59.999-00:00"); + addMillisecond("2012-04T", 30L * 24 * 60 * 60 * 1000 , "2012-05-01T00:00:00.000-00:00"); + + + addMillisecond("2011-02-28", -1, "2011-02-27T23:59:59.999-00:00"); + addMillisecond("2011-02-28", 0, "2011-02-28T00:00:00.000-00:00"); + addMillisecond("2011-02-28", 1, "2011-02-28T00:00:00.001-00:00"); + addMillisecond("2011-02-28", 24 * 60 * 60 * 1000, "2011-03-01T00:00:00.000-00:00"); + addMillisecond("2011-03-01", -24 * 60 * 60 * 1000, "2011-02-28T00:00:00.000-00:00"); + addMillisecond("2012-02-28", 24 * 60 * 60 * 1000, "2012-02-29T00:00:00.000-00:00"); + addMillisecond("2012-02-29", -24 * 60 * 60 * 1000, "2012-02-28T00:00:00.000-00:00"); + addMillisecond("2012-02-29", 24 * 60 * 60 * 1000, "2012-03-01T00:00:00.000-00:00"); + + addMillisecond("2012-10-04", -24 * 60 * 60 * 1000 - 1, "2012-10-02T23:59:59.999-00:00"); + addMillisecond("2012-10-04", -24 * 60 * 60 * 1000 , "2012-10-03T00:00:00.000-00:00"); + addMillisecond("2012-10-04", -1, "2012-10-03T23:59:59.999-00:00"); + addMillisecond("2012-10-04", 1, "2012-10-04T00:00:00.001-00:00"); + addMillisecond("2012-10-04", 48 * 60 * 60 * 1000 - 1, "2012-10-05T23:59:59.999-00:00"); + addMillisecond("2012-10-04", 48 * 60 * 60 * 1000 , "2012-10-06T00:00:00.000-00:00"); + + addMillisecondWithFrac("2011-01-31T12:03:23", -60 * 60 * 1000, "2011-01-31T11:03:23"); + addMillisecondWithFrac("2011-01-31T12:03:23", 0, "2011-01-31T12:03:23"); + addMillisecondWithFrac("2011-01-31T12:03:23", 60 * 60 * 1000, "2011-01-31T13:03:23"); + addMillisecondWithFrac("2011-01-31T23:03:23", 57 * 60 * 1000, "2011-02-01T00:00:23"); + + addMillisecondWithFrac("2011-02-28T02:03:23", 25 * 60 * 60 * 1000, "2011-03-01T03:03:23"); + addMillisecondWithFrac("2012-02-28T02:03:23", 25 * 60 * 60 * 1000, "2012-02-29T03:03:23"); + + addMillisecondWithFrac("2011-03-01T02:35:23", -4 * 60 * 60 * 1000, "2011-02-28T22:35:23"); + addMillisecondWithFrac("2012-03-01T02:35:23", -4 * 60 * 60 * 1000, "2012-02-29T22:35:23"); + + addMillisecondWithOffsets("2011-01-31T23:59:59.999", 1, "2011-02-01T00:00:00.000"); + addMillisecondWithOffsets("2011-02-01T00:00:00.000", -1, "2011-01-31T23:59:59.999"); + addMillisecondWithOffsets("2011-01-31T23:59:59.999123", 1, "2011-02-01T00:00:00.000123"); + addMillisecondWithOffsets("2011-02-01T00:00:00.000123", -1, "2011-01-31T23:59:59.999123"); + + } + + @Test + public void addLargeNumberOfMilliseconds() { + // Internally, the value is divided by 1000, then added as seconds. Multiply by more than 1000 to make sure + // more seconds than can fit in an integer can be added. + long largeAmountToAdd = (Integer.MAX_VALUE + 1L) * 1000; + Timestamp ts1 = Timestamp.forMillis(0, 0); + Timestamp ts2 = ts1.addMillis(largeAmountToAdd); + assertEquals(Timestamp.forMillis(largeAmountToAdd, 0), ts2); + long largeAmountToSubtract = -(Integer.MIN_VALUE - 1L) * 1000; + Timestamp ts3 = Timestamp.forMillis(largeAmountToSubtract, 0); + assertEquals(ts1, ts3.addMillis(-largeAmountToSubtract)); + } + + @Test + public void testSubtractDayNonLeapYear() + { + // See: ion-java#163. Previously, this failed because the Timestamp arithmetic was performed using + // java.util.Date, which returns its components in the system's local time. + Timestamp timestamp = Timestamp.valueOf("1500-03-02T00:00:00.000Z"); + Timestamp result = timestamp.addDay(-1); + assertEquals(Timestamp.valueOf("1500-03-01T00:00:00.000Z"), result); + } + + @Test + public void testCalendarValueRoundtrip() { + Timestamp timestamp = Timestamp.valueOf("2014T"); + assertEquals(timestamp, Timestamp.forCalendar(timestamp.calendarValue())); + timestamp = Timestamp.valueOf("2014-06T"); + assertEquals(timestamp, Timestamp.forCalendar(timestamp.calendarValue())); + timestamp = Timestamp.valueOf("2014-06-02T"); + assertEquals(timestamp, Timestamp.forCalendar(timestamp.calendarValue())); + timestamp = Timestamp.valueOf("2014-06-02T08:00-07:00"); + assertEquals(timestamp, Timestamp.forCalendar(timestamp.calendarValue())); + timestamp = Timestamp.valueOf("2014-06-02T08:00:01-07:00"); + assertEquals(timestamp, Timestamp.forCalendar(timestamp.calendarValue())); + // Calendar can represent up to milliseconds precision, but not more. + timestamp = Timestamp.valueOf("2014-06-02T08:00:01.987-07:00"); + assertEquals(timestamp, Timestamp.forCalendar(timestamp.calendarValue())); + Timestamp timestampTooPrecise = Timestamp.valueOf("2014-06-02T08:00:01.9876-07:00"); + assertEquals(timestamp, Timestamp.forCalendar(timestampTooPrecise.calendarValue())); + } + + @Test + public void testGregorianCalendarNonLeapYear() { + GregorianCalendar calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC")); + calendar.clear(); + // 1800 is not a leap year under the Gregorian calendar. + calendar.set(1800, Calendar.FEBRUARY, 28); + Timestamp timestamp1 = Timestamp.forCalendar(calendar); + Timestamp timestamp2 = timestamp1.addDay(1); + assertEquals("1800-03-01", timestamp2.toString()); + } + + @Test + public void testConstructCustomCalendarWithValidLeapYear() { + // Create a purely Julian calendar, where leap years were every four years. + GregorianCalendar julianCalendar = new GregorianCalendar(TimeZone.getTimeZone("UTC")); + julianCalendar.setGregorianChange(new Date(Long.MAX_VALUE)); + julianCalendar.clear(); + // 1800 is not a leap year under the Gregorian calendar, but it is under the Julian calendar. + julianCalendar.set(1800, Calendar.FEBRUARY, 29, 0, 0); + Timestamp timestamp = Timestamp.forCalendar(julianCalendar); + assertEquals("1800-02-29T00:00Z", timestamp.toString()); + } + + @Test + public void testCustomCalendarLeapYear() { + // Create a purely Julian calendar, where leap years were every four years. + GregorianCalendar julianCalendar = new GregorianCalendar(TimeZone.getTimeZone("UTC")); + julianCalendar.setGregorianChange(new Date(Long.MAX_VALUE)); + julianCalendar.clear(); + // 1800 is not a leap year under the Gregorian calendar, but it is under the Julian calendar. + julianCalendar.set(1800, Calendar.FEBRUARY, 28); + Timestamp timestamp1 = Timestamp.forCalendar(julianCalendar); + Timestamp timestamp2 = timestamp1.addDay(1); + assertEquals("1800-02-29", timestamp2.toString()); + } + + @Test + public void testCustomCalendarLeapYearAfterClone() { + // Create a purely Julian calendar, where leap years were every four years. + GregorianCalendar julianCalendar = new GregorianCalendar(TimeZone.getTimeZone("UTC")); + julianCalendar.setGregorianChange(new Date(Long.MAX_VALUE)); + julianCalendar.clear(); + // 1800 is not a leap year under the Gregorian calendar, but it is under the Julian calendar. + julianCalendar.set(1800, Calendar.MARCH, 1, 0, 0); + Timestamp timestamp1 = Timestamp.forCalendar(julianCalendar); + Timestamp timestamp2 = timestamp1.clone().addHour(-1); + assertEquals("1800-02-29T23:00Z", timestamp2.toString()); + } + + @Test + public void testCustomCalendarLeapYearAfterCalendarValue() { + // Create a purely Julian calendar, where leap years were every four years. + GregorianCalendar julianCalendar = new GregorianCalendar(TimeZone.getTimeZone("UTC")); + julianCalendar.setGregorianChange(new Date(Long.MAX_VALUE)); + julianCalendar.clear(); + // 1800 is not a leap year under the Gregorian calendar, but it is under the Julian calendar. + julianCalendar.set(1800, Calendar.MARCH, 1, 0, 0); + Timestamp timestamp1 = Timestamp.forCalendar(julianCalendar); + Timestamp timestamp2 = Timestamp.forCalendar(timestamp1.calendarValue()).addMinute(-1); + assertEquals("1800-02-29T23:59Z", timestamp2.toString()); + } + + @Test + public void testCustomCalendarLeapYearAfterWithLocalOffset() { + // Create a purely Julian calendar, where leap years were every four years. + GregorianCalendar julianCalendar = new GregorianCalendar(TimeZone.getTimeZone("UTC")); + julianCalendar.setGregorianChange(new Date(Long.MAX_VALUE)); + julianCalendar.clear(); + // 1800 is not a leap year under the Gregorian calendar, but it is under the Julian calendar. + julianCalendar.set(1800, Calendar.FEBRUARY, 28, 23, 59, 59); + Timestamp timestamp1 = Timestamp.forCalendar(julianCalendar); + Timestamp timestamp2 = timestamp1.withLocalOffset(0).addSecond(1); + assertEquals("1800-02-29T00:00:00Z", timestamp2.toString()); + } + + @Test + public void testCustomCalendarLeapYearWithChainedArithmetic() { + // Create a purely Julian calendar, where leap years were every four years. + GregorianCalendar julianCalendar = new GregorianCalendar(TimeZone.getTimeZone("UTC")); + julianCalendar.setGregorianChange(new Date(Long.MAX_VALUE)); + julianCalendar.clear(); + // 1800 is not a leap year under the Gregorian calendar, but it is under the Julian calendar. + julianCalendar.set(1799, Calendar.JANUARY, 27, 22, 58, 58); + julianCalendar.set(Calendar.MILLISECOND, 999); + Timestamp timestamp1 = Timestamp.forCalendar(julianCalendar); + Timestamp timestamp2 = timestamp1.addYear(1).addMonth(1).addDay(1).addHour(1).addMinute(1).addSecond(1).addMillis(1); + assertEquals("1800-02-29T00:00:00.000Z", timestamp2.toString()); + } + @Test(expected = IllegalArgumentException.class) public void testAddSecondOutsideMaxRange() { @@ -2366,4 +2548,27 @@ public void testGetMillisWithLargeScaleBigDecimal() { Timestamp.forMillis(LARGE_SCALE_DECIMAL, PST_OFFSET).getMillis(); } + + @Test + public void testInstantVsTimestampMillis() { + // addresses: https://github.com/amzn/ion-java/issues/165 + String tsText = "0001-01-01T00:00:00.000Z"; + // Instant extends the Gregorian calendar system all the way back to the beginning, whereas Timestamp uses + // the default GregorianCalendar implementation, which transitions from Julian to Gregorian in 1582. As a + // result, the two map from milliseconds to date differently before 1582, as demonstrated below. + long millisFromInstant = Instant.parse(tsText).toEpochMilli(); + long millisFromTimestamp = Timestamp.valueOf(tsText).getMillis(); + assertNotEquals(millisFromInstant, millisFromTimestamp); + // However, the discrepancy can be avoided by using Timestamp.forCalendar, which always respects the given + // Calendar's method for determining leap years. + Timestamp ts1 = Timestamp.forCalendar(GregorianCalendar.from(Instant.parse(tsText).atZone(ZoneId.of("UTC")))); + Timestamp ts2 = Timestamp.valueOf(tsText); + assertEquals(tsText, ts1.toString()); + assertEquals(tsText, ts2.toString()); + assertEquals(ts1, ts2); + // Because Timestamp always uses the configured Calendar for determining leap years, the original milliseconds + // values are roundtripped unchanged. + assertEquals(millisFromInstant, ts1.getMillis()); + assertEquals(millisFromTimestamp, ts2.getMillis()); + } } From 5beefbdd63984a31c0e43cec5e66ba3c00ed6223 Mon Sep 17 00:00:00 2001 From: Peter Cornell Date: Mon, 17 Dec 2018 16:45:24 -0800 Subject: [PATCH 038/490] Produces an error when an unpaired surrogate is encountered (#193) --- .../ion/impl/IonReaderTextRawTokensX.java | 48 ++++++++++++++++++- .../amazon/ion/SystemProcessingTestCase.java | 3 -- test/software/amazon/ion/TestUtils.java | 9 ---- 3 files changed, 46 insertions(+), 14 deletions(-) diff --git a/src/software/amazon/ion/impl/IonReaderTextRawTokensX.java b/src/software/amazon/ion/impl/IonReaderTextRawTokensX.java index aba793f173..4e815ebb0d 100644 --- a/src/software/amazon/ion/impl/IonReaderTextRawTokensX.java +++ b/src/software/amazon/ion/impl/IonReaderTextRawTokensX.java @@ -1935,6 +1935,7 @@ protected int load_single_quoted_string(StringBuilder sb, boolean is_clob) throws IOException { int c; + boolean expectLowSurrogate = false; for (;;) { c = read_string_char(ProhibitedCharacters.NONE); @@ -1945,6 +1946,9 @@ protected int load_single_quoted_string(StringBuilder sb, boolean is_clob) continue; case -1: case '\'': + if (!is_clob) { + check_for_low_surrogate(c, expectLowSurrogate); + } return c; // new line normalization and counting is handled in read_char case CharacterSequence.CHAR_SEQ_NEWLINE_SEQUENCE_1: @@ -1962,11 +1966,16 @@ protected int load_single_quoted_string(StringBuilder sb, boolean is_clob) c = read_large_char_sequence(c); } } - + // if this isn't a clob we need to decode UTF8 and + // handle surrogate encoding (otherwise we don't care) if (!is_clob) { + expectLowSurrogate = check_for_low_surrogate(c, expectLowSurrogate); + if (IonUTF8.needsSurrogateEncoding(c)) { sb.append(IonUTF8.highSurrogate(c)); c = IonUTF8.lowSurrogate(c); + } else { + expectLowSurrogate = IonUTF8.isHighSurrogate(c); } } else if (IonTokenConstsX.is8bitValue(c)) { @@ -2010,6 +2019,7 @@ protected int load_double_quoted_string(StringBuilder sb, boolean is_clob) throws IOException { int c; + boolean expectLowSurrogate = false; for (;;) { c = read_string_char(ProhibitedCharacters.SHORT_CHAR); @@ -2020,6 +2030,9 @@ protected int load_double_quoted_string(StringBuilder sb, boolean is_clob) continue; case -1: case '"': + if (!is_clob) { + check_for_low_surrogate(c, expectLowSurrogate); + } return c; // new line normalization and counting is handled in read_char case CharacterSequence.CHAR_SEQ_NEWLINE_SEQUENCE_1: @@ -2035,17 +2048,36 @@ protected int load_double_quoted_string(StringBuilder sb, boolean is_clob) } break; } - + // if this isn't a clob we need to decode UTF8 and + // handle surrogate encoding (otherwise we don't care) if (!is_clob) { + expectLowSurrogate = check_for_low_surrogate(c, expectLowSurrogate); + if (IonUTF8.needsSurrogateEncoding(c)) { sb.append(IonUTF8.highSurrogate(c)); c = IonUTF8.lowSurrogate(c); + } else { + expectLowSurrogate = IonUTF8.isHighSurrogate(c); } } sb.append((char)c); } } + private boolean check_for_low_surrogate(int c, boolean expectLowSurrogate) throws IonException + { + if (IonUTF8.isLowSurrogate(c)) { + if (expectLowSurrogate) { + return false; + } else { + error("unexpected low surrogate " + printCodePointAsString(c)); + } + } else if (expectLowSurrogate) { + expected_but_found("a low surrogate", c); + } + return false; + } + protected int read_double_quoted_char(boolean is_clob) throws IOException { int c = read_char(); @@ -2135,12 +2167,16 @@ protected int load_triple_quoted_string(StringBuilder sb, boolean is_clob) throws IOException { int c; + boolean expectLowSurrogate = false; for (;;) { c = read_triple_quoted_char(is_clob); switch(c) { case CharacterSequence.CHAR_SEQ_STRING_TERMINATOR: case CharacterSequence.CHAR_SEQ_EOF: // was EOF + if (!is_clob) { + check_for_low_surrogate(c, expectLowSurrogate); + } return c; // new line normalization and counting is handled in read_char case CharacterSequence.CHAR_SEQ_NEWLINE_SEQUENCE_1: @@ -2157,7 +2193,11 @@ protected int load_triple_quoted_string(StringBuilder sb, boolean is_clob) case CharacterSequence.CHAR_SEQ_ESCAPED_NEWLINE_SEQUENCE_1: case CharacterSequence.CHAR_SEQ_ESCAPED_NEWLINE_SEQUENCE_2: case CharacterSequence.CHAR_SEQ_ESCAPED_NEWLINE_SEQUENCE_3: + continue; case CharacterSequence.CHAR_SEQ_STRING_NON_TERMINATOR: + if (!is_clob) { + expectLowSurrogate = check_for_low_surrogate(c, expectLowSurrogate); + } continue; default: break; @@ -2165,9 +2205,13 @@ protected int load_triple_quoted_string(StringBuilder sb, boolean is_clob) // if this isn't a clob we need to decode UTF8 and // handle surrogate encoding (otherwise we don't care) if (!is_clob) { + expectLowSurrogate = check_for_low_surrogate(c, expectLowSurrogate); + if (IonUTF8.needsSurrogateEncoding(c)) { sb.append(IonUTF8.highSurrogate(c)); c = IonUTF8.lowSurrogate(c); + } else { + expectLowSurrogate = IonUTF8.isHighSurrogate(c); } } sb.append((char)c); diff --git a/test/software/amazon/ion/SystemProcessingTestCase.java b/test/software/amazon/ion/SystemProcessingTestCase.java index 5ec1082e12..c72e8ffa22 100644 --- a/test/software/amazon/ion/SystemProcessingTestCase.java +++ b/test/software/amazon/ion/SystemProcessingTestCase.java @@ -804,9 +804,6 @@ public void testSurrogateGluing() ionData = "'''" + high + low + "'''"; checkString(FERMATA, "\"\\U0001d110\"", ionData); - - ionData = "'''" + high + "''' '''" + low + "'''"; - checkString(FERMATA, "\"\\U0001d110\"", ionData); } } } diff --git a/test/software/amazon/ion/TestUtils.java b/test/software/amazon/ion/TestUtils.java index d7ade58f7c..a2b5cb68ac 100644 --- a/test/software/amazon/ion/TestUtils.java +++ b/test/software/amazon/ion/TestUtils.java @@ -131,15 +131,6 @@ public boolean accept(File dir, String name) new FileIsNot( "bad/clobWithNullCharacter.ion" // TODO amzn/ion-java#43 , "bad/emptyAnnotatedInt.10n" // TODO amzn/ion-java#55 - , "bad/utf8/surrogate_1.ion" // TODO amzn/ion-java#105 - , "bad/utf8/surrogate_2.ion" // TODO amzn/ion-java#105 - , "bad/utf8/surrogate_4.ion" // TODO amzn/ion-java#105 - , "bad/utf8/surrogate_5.ion" // TODO amzn/ion-java#60 - , "bad/utf8/surrogate_6.ion" // TODO amzn/ion-java#105 - , "bad/utf8/surrogate_7.ion" // TODO amzn/ion-java#105 - , "bad/utf8/surrogate_8.ion" // TODO amzn/ion-java#105 - , "bad/utf8/surrogate_9.ion" // TODO amzn/ion-java#105 - , "bad/utf8/surrogate_10.ion" // TODO amzn/ion-java#105 , "equivs/paddedInts.10n" // TODO amzn/ion-java#54 , "good/subfieldVarUInt32bit.ion" // TODO amzn/ion-java#62 , "good/utf16.ion" // TODO amzn/ion-java#61 From 77ba31c570b6cd6c2f35f70a0896b0e0d0e472ac Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Mon, 17 Dec 2018 18:03:23 -0800 Subject: [PATCH 039/490] Uses Precision.includes when testing for SECOND precision. --- src/software/amazon/ion/Timestamp.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/software/amazon/ion/Timestamp.java b/src/software/amazon/ion/Timestamp.java index 2f5fa7c3f1..7349107581 100644 --- a/src/software/amazon/ion/Timestamp.java +++ b/src/software/amazon/ion/Timestamp.java @@ -572,7 +572,7 @@ private Timestamp(BigDecimal millis, Precision precision, Integer localOffset) // a Calendar can handle. Set the _fraction here so that extra precision (if any) is not lost. // However, don't set the fraction if the given BigDecimal does not have precision at least to the tenth of // a second. - if (precision == Precision.SECOND && millis.scale() > -3) { + if (precision.includes(Precision.SECOND) && millis.scale() > -3) { BigDecimal secs = millis.movePointLeft(3); BigDecimal secsDown = fastRoundZeroFloor(secs); _fraction = secs.subtract(secsDown); @@ -1857,7 +1857,7 @@ private static void print(Appendable out, Timestamp adjusted) out.append(":"); print_digits(out, adjusted.getZMinute(), 2); // ok, so how much time do we have ? - if (adjusted._precision == Precision.SECOND) { + if (adjusted._precision.includes(Precision.SECOND)) { out.append(":"); print_digits(out, adjusted.getZSecond(), 2); if (adjusted._fraction != null) { @@ -1930,7 +1930,7 @@ private static void print_fractional_digits(Appendable out, BigDecimal value) * @param amount a number of milliseconds. */ public final Timestamp addMillis(long amount) { - if (amount == 0 && _precision == Precision.SECOND && _fraction != null && _fraction.scale() >= 3) { + if (amount == 0 && _precision.includes(Precision.SECOND) && _fraction != null && _fraction.scale() >= 3) { // Zero milliseconds are to be added, and the precision does not need to be increased. return this; } @@ -2355,7 +2355,7 @@ private static byte checkAndCastSecond(int second) private static Precision checkFraction(Precision precision, BigDecimal fraction) { - if (precision == Precision.SECOND) { + if (precision.includes(Precision.SECOND)) { if (fraction != null && (fraction.signum() == -1 || BigDecimal.ONE.compareTo(fraction) != 1)) { throw new IllegalArgumentException(String.format("Fractional seconds %s must be greater than or equal to 0 and less than 1", fraction)); } From 931d89962d8c023ebdd2d661c64a82b01b6c3c03 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Wed, 19 Dec 2018 12:11:19 -0800 Subject: [PATCH 040/490] Adds Timestamp.adjust* methods, which perform date arithmetic without extending precision. --- src/software/amazon/ion/Timestamp.java | 218 +++- test/software/amazon/ion/TimestampTest.java | 1151 ++++++++++++------- 2 files changed, 952 insertions(+), 417 deletions(-) diff --git a/src/software/amazon/ion/Timestamp.java b/src/software/amazon/ion/Timestamp.java index 7349107581..62984161f7 100644 --- a/src/software/amazon/ion/Timestamp.java +++ b/src/software/amazon/ion/Timestamp.java @@ -1926,6 +1926,48 @@ private static void print_fractional_digits(Appendable out, BigDecimal value) * Returns a timestamp relative to this one by the given number of * milliseconds. Uses this Timestamp's configured Calendar to perform the * arithmetic. + *

    + * This method always returns a Timestamp with the same precision as + * the original. After performing the arithmetic, the resulting Timestamp's + * seconds value will be truncated to the same fractional precision as the + * original. For example, adjusting {@code 2012-04-01T00:00:00Z} by one + * millisecond results in {@code 2012-04-01T00:00:00Z}; adjusting + * {@code 2012-04-01T00:00:00.0010Z} by -1 millisecond results in + * {@code 2012-04-01T00:00:00.0000Z}. To extend the precision when the + * original Timestamp has coarser than SECOND precision and to avoid + * truncation of the seconds value, use {@link #addSecond(int)}. + * + * @param amount a number of milliseconds. + */ + public final Timestamp adjustMillis(long amount) { + if (amount == 0) return this; + Timestamp ts = addMillis(amount); + ts._precision = _precision; + ts.clearUnusedPrecision(); + if (_precision.includes(Precision.SECOND)) { + // Maintain the same amount of fractional precision. + if (_fraction == null) { + ts._fraction = null; + } else { + // Truncate the result only if it exceeds the fractional precision of the original. + if (ts._fraction.scale() > _fraction.scale()) { + ts._fraction = ts._fraction.setScale(_fraction.scale(), RoundingMode.FLOOR); + } + } + } + return ts; + } + + /** + * Returns a timestamp relative to this one by the given number of + * milliseconds. Uses this Timestamp's configured Calendar to perform the + * arithmetic. + *

    + * This method always returns a Timestamp with SECOND precision and a seconds + * value precise at least to the millisecond. For example, adding one millisecond + * to {@code 2011T} results in {@code 2011-01-01T00:00:00.001-00:00}. To receive + * a Timestamp that always maintains the same precision as the original, use + * {@link #adjustMillis(long)}. * * @param amount a number of milliseconds. */ @@ -1950,12 +1992,19 @@ public final Timestamp addMillis(long amount) { newFraction = millis; } Timestamp ts = addSecond(seconds); + if (ts == this) { + // When the precision does not need to be extended and there are no seconds to add, + // addSecond returns the same reference unchanged. Clone it to ensure it is not + // mutated. + ts = clone(); + } ts._fraction = newFraction; return ts; } /** - * Adds the given amount to the given {@link Calendar} field and returns a new Timestamp. + * Adds the given amount to the given {@link Calendar} field and returns a new Timestamp + * with precision set to the maximum of the current precision and the given precision. * @param field the field. * @param amount an amount. * @param precision the precision corresponding to the given field. @@ -1974,13 +2023,46 @@ private Timestamp calendarAdd(int field, int amount, Precision precision) { return timestamp; } + /** + * Clears any fields more precise than this Timestamp's precision supports. + */ + private void clearUnusedPrecision() { + switch (_precision) { + case YEAR: + _calendar.set(Calendar.MONTH, 0); + case MONTH: + _calendar.set(Calendar.DAY_OF_MONTH, 1); + case DAY: + _calendar.set(Calendar.HOUR_OF_DAY, 0); + _calendar.set(Calendar.MINUTE, 0); + case MINUTE: + _calendar.set(Calendar.SECOND, 0); + _fraction = null; + case SECOND: + } + } + + /** + * Adds the given amount to the given {@link Calendar} field and returns a new Timestamp + * with the same precision as the original Timestamp. + * @param field the field. + * @param amount an amount. + * @return a new Timestamp instance. + */ + private Timestamp calendarAdjust(int field, int amount) { + if (amount == 0) return this; + Timestamp ts = calendarAdd(field, amount, _precision); + ts.clearUnusedPrecision(); + return ts; + } + /** * Returns a timestamp relative to this one by the given number of seconds. * Uses this Timestamp's configured Calendar to perform the arithmetic. * * @param seconds a number of seconds. */ - private final Timestamp addSecond(long seconds) { + private Timestamp addSecond(long seconds) { Timestamp ts = this; do { int incrementalSeconds; @@ -2000,6 +2082,29 @@ private final Timestamp addSecond(long seconds) { /** * Returns a timestamp relative to this one by the given number of seconds. * Uses this Timestamp's configured Calendar to perform the arithmetic. + *

    + * This method always returns a Timestamp with the same precision as + * the original. For example, adjusting {@code 2012-04-01T00:00Z} by one + * second results in {@code 2012-04-01T00:00Z}; adjusting + * {@code 2012-04-01T00:00:00Z} by -1 second results in + * {@code 2012-03-31T23:59:59Z}. To extend the precision when the original + * Timestamp has coarser than SECOND precision, use {@link #addSecond(int)}. + * + * @param amount a number of seconds. + */ + public final Timestamp adjustSecond(int amount) + { + return calendarAdjust(Calendar.SECOND, amount); + } + + /** + * Returns a timestamp relative to this one by the given number of seconds. + * Uses this Timestamp's configured Calendar to perform the arithmetic. + *

    + * This method always returns a Timestamp with SECOND precision. + * For example, adding one second to {@code 2011T} results in + * {@code 2011-01-01T00:00:01-00:00}. To receive a Timestamp that always + * maintains the same precision as the original, use {@link #adjustSecond(int)}. * * @param amount a number of seconds. */ @@ -2008,10 +2113,32 @@ public final Timestamp addSecond(int amount) return calendarAdd(Calendar.SECOND, amount, Precision.SECOND); } + /** + * Returns a timestamp relative to this one by the given number of minutes. + * Uses this Timestamp's configured Calendar to perform the arithmetic. + *

    + * This method always returns a Timestamp with the same precision as + * the original. For example, adjusting {@code 2012-04-01T} by one minute + * results in {@code 2012-04-01T}; adjusting {@code 2012-04-01T00:00-00:00} + * by -1 minute results in {@code 2012-03-31T23:59-00:00}. To extend the + * precision when the original Timestamp has coarser than MINUTE precision, + * use {@link #addMinute(int)}. + * + * @param amount a number of minutes. + */ + public final Timestamp adjustMinute(int amount) + { + return calendarAdjust(Calendar.MINUTE, amount); + } /** * Returns a timestamp relative to this one by the given number of minutes. * Uses this Timestamp's configured Calendar to perform the arithmetic. + *

    + * This method always returns a Timestamp with at least MINUTE precision. + * For example, adding one minute to {@code 2011T} results in + * {@code 2011-01-01T00:01-00:00}. To receive a Timestamp that always + * maintains the same precision as the original, use {@link #adjustMinute(int)}. * * @param amount a number of minutes. */ @@ -2020,10 +2147,32 @@ public final Timestamp addMinute(int amount) return calendarAdd(Calendar.MINUTE, amount, Precision.MINUTE); } + /** + * Returns a timestamp relative to this one by the given number of hours. + * Uses this Timestamp's configured Calendar to perform the arithmetic. + *

    + * This method always returns a Timestamp with the same precision as + * the original. For example, adjusting {@code 2012-04-01T} by one hour + * results in {@code 2012-04-01T}; adjusting {@code 2012-04-01T00:00-00:00} + * by -1 hour results in {@code 2012-03-31T23:00-00:00}. To extend the + * precision when the original Timestamp has coarser than MINUTE precision, + * use {@link #addHour(int)}. + * + * @param amount a number of hours. + */ + public final Timestamp adjustHour(int amount) + { + return calendarAdjust(Calendar.HOUR_OF_DAY, amount); + } /** * Returns a timestamp relative to this one by the given number of hours. * Uses this Timestamp's configured Calendar to perform the arithmetic. + *

    + * This method always returns a Timestamp with at least MINUTE precision. + * For example, adding one hour to {@code 2011T} results in + * {@code 2011-01-01T01:00-00:00}. To receive a Timestamp that always + * maintains the same precision as the original, use {@link #adjustHour(int)}. * * @param amount a number of hours. */ @@ -2032,10 +2181,31 @@ public final Timestamp addHour(int amount) return calendarAdd(Calendar.HOUR_OF_DAY, amount, Precision.MINUTE); } + /** + * Returns a timestamp relative to this one by the given number of days. + * Uses this Timestamp's configured Calendar to perform the arithmetic. + *

    + * This method always returns a Timestamp with the same precision as + * the original. For example, adjusting {@code 2012-04T} by one day results + * in {@code 2012-04T}; adjusting {@code 2012-04-01T} by -1 days results in + * {@code 2012-03-31T}. To extend the precision when the original Timestamp + * has coarser than DAY precision, use {@link #addDay(int)}. + * + * @param amount a number of days. + */ + public final Timestamp adjustDay(int amount) + { + return calendarAdjust(Calendar.DAY_OF_MONTH, amount); + } /** * Returns a timestamp relative to this one by the given number of days. * Uses this Timestamp's configured Calendar to perform the arithmetic. + *

    + * This method always returns a Timestamp with at least DAY precision. + * For example, adding one day to {@code 2011T} results in {@code 2011-01-02T}. + * To receive a Timestamp that always maintains the same precision as the + * original, use {@link #adjustDay(int)}. * * @param amount a number of days. */ @@ -2044,12 +2214,37 @@ public final Timestamp addDay(int amount) return calendarAdd(Calendar.DAY_OF_MONTH, amount, Precision.DAY); } + /** + * Returns a timestamp relative to this one by the given number of months. + * The day field may be adjusted to account for different month length and + * leap days, as required by the configured Calendar's rules. For example, + * using the default {@link GregorianCalendar}, adjusting {@code 2011-01-31} + * by one month results in {@code 2011-02-28}. + *

    + * This method always returns a Timestamp with the same precision as the + * original. For example, adjusting {@code 2011T} by one month results in + * {@code 2011T}; adding 12 months to {@code 2011T} results in {@code 2012T}. + * To extend the precision when the original Timestamp has coarser than MONTH + * precision, use {@link #addMonth(int)}. + * + * @param amount a number of months. + */ + public final Timestamp adjustMonth(int amount) + { + return calendarAdjust(Calendar.MONTH, amount); + } + /** * Returns a timestamp relative to this one by the given number of months. * The day field may be adjusted to account for different month length and * leap days, as required by the configured Calendar's rules. For example, * using the default {@link GregorianCalendar}, adding one month to * {@code 2011-01-31} results in {@code 2011-02-28}. + *

    + * This method always returns a Timestamp with at least MONTH precision. + * For example, adding one month to {@code 2011T} results in {@code 2011-02T}. + * To receive a Timestamp that always maintains the same precision as the + * original, use {@link #adjustMonth(int)}. * * @param amount a number of months. */ @@ -2058,6 +2253,22 @@ public final Timestamp addMonth(int amount) return calendarAdd(Calendar.MONTH, amount, Precision.MONTH); } + /** + * Returns a timestamp relative to this one by the given number of years. + * The day field may be adjusted to account for leap days, as required by + * the configured Calendar's rules. For example, using the default + * {@link GregorianCalendar}, adjusting {@code 2012-02-29} by one year + * results in {@code 2013-02-28}. + *

    + * Because YEAR is the coarsest precision possible, this method always + * returns a Timestamp with the same precision as the original and + * behaves identically to {@link #addYear(int)}. + * + * @param amount a number of years. + */ + public final Timestamp adjustYear(int amount) { + return addYear(amount); + } /** * Returns a timestamp relative to this one by the given number of years. @@ -2065,6 +2276,9 @@ public final Timestamp addMonth(int amount) * the configured Calendar's rules. For example, using the default * {@link GregorianCalendar}, adding one year to {@code 2012-02-29} results * in {@code 2013-02-28}. + *

    + * This method always returns a Timestamp with the same precision as + * the original. See also: {@link #adjustYear(int)}. * * @param amount a number of years. */ diff --git a/test/software/amazon/ion/TimestampTest.java b/test/software/amazon/ion/TimestampTest.java index 491aaa89ac..5893f27470 100644 --- a/test/software/amazon/ion/TimestampTest.java +++ b/test/software/amazon/ion/TimestampTest.java @@ -1617,64 +1617,86 @@ public void testWithLocalOffset() //========================================================================= // Timestamp arithmetic + private interface TimestampArithmeticInvoker { + Timestamp invoke(Timestamp input, Number amount); + } - private void addYear(String orig, int amount, String expected) + private void addAndCompare(String orig, Number amount, String expected, TimestampArithmeticInvoker invoker) { Timestamp ts1 = Timestamp.valueOf(orig); - Timestamp ts2 = ts1.addYear(amount); + Timestamp ts2 = invoker.invoke(ts1, amount); checkTimestamp(expected, ts2); + checkTimestamp(orig, ts1); // Make sure the arithmetic does not mutate the original timestamp. } - private void addYear(String orig, int amount, String expected, - String suffix) + private void addAndCompare(String orig, Number amount, String expected, + String suffix, TimestampArithmeticInvoker invoker) { - addYear(orig + suffix, amount, expected + suffix); + addAndCompare(orig + suffix, amount, expected + suffix, invoker); } - private void addYearWithOffsets(String orig, int amount, String expected, - String suffix) + private void addAndCompareWithOffsets(String orig, Number amount, String expected, + String suffix, TimestampArithmeticInvoker invoker) { - addYear(orig, amount, expected, suffix + "-00:00"); - addYear(orig, amount, expected, suffix + "Z"); - addYear(orig, amount, expected, suffix + "+01:23"); - addYear(orig, amount, expected, suffix + "-01:23"); - addYear(orig, amount, expected, suffix + "+23:59"); - addYear(orig, amount, expected, suffix + "-23:59"); + addAndCompare(orig, amount, expected, suffix + "-00:00", invoker); + addAndCompare(orig, amount, expected, suffix + "Z", invoker); + addAndCompare(orig, amount, expected, suffix + "+01:23", invoker); + addAndCompare(orig, amount, expected, suffix + "-01:23", invoker); + addAndCompare(orig, amount, expected, suffix + "+23:59", invoker); + addAndCompare(orig, amount, expected, suffix + "-23:59", invoker); } - private void addYearWithMins(String orig, int amount, String expected) + private void addAndCompareWithFullPrecision(String orig, Number amount, String expected, TimestampArithmeticInvoker invoker) { - addYear(orig, amount, expected); + addAndCompare(orig, amount, expected, invoker); - addYearWithOffsets(orig, amount, expected, "T19:03"); - addYearWithOffsets(orig, amount, expected, "T19:03:23"); - addYearWithOffsets(orig, amount, expected, "T19:03:23.0"); - addYearWithOffsets(orig, amount, expected, "T19:03:23.00"); - addYearWithOffsets(orig, amount, expected, "T19:03:23.000"); - addYearWithOffsets(orig, amount, expected, "T19:03:23.456"); - addYearWithOffsets(orig, amount, expected, "T19:03:23.0000"); - addYearWithOffsets(orig, amount, expected, "T19:03:23.45678"); + addAndCompareWithOffsets(orig, amount, expected, "T19:03", invoker); + addAndCompareWithOffsets(orig, amount, expected, "T19:03:23", invoker); + addAndCompareWithOffsets(orig, amount, expected, "T19:03:23.0", invoker); + addAndCompareWithOffsets(orig, amount, expected, "T19:03:23.00", invoker); + addAndCompareWithOffsets(orig, amount, expected, "T19:03:23.000", invoker); + addAndCompareWithOffsets(orig, amount, expected, "T19:03:23.456", invoker); + addAndCompareWithOffsets(orig, amount, expected, "T19:03:23.0000", invoker); + addAndCompareWithOffsets(orig, amount, expected, "T19:03:23.45678", invoker); } - @Test - public void testAddYear() - { - addYear("2012T", 1, "2013T"); - addYear("2012T", -1, "2011T"); + private void testYearArithmetic(TimestampArithmeticInvoker invoker) { + addAndCompare("2012T", 1, "2013T", invoker); + addAndCompare("2012T", -1, "2011T", invoker); - addYear("2012-04T", 1, "2013-04T"); - addYear("2012-04T", -1, "2011-04T"); + addAndCompare("2012-04T", 1, "2013-04T", invoker); + addAndCompare("2012-04T", -1, "2011-04T", invoker); - addYearWithMins("2012-04-23", 1, "2013-04-23"); - addYearWithMins("2012-04-23", -1, "2011-04-23"); + addAndCompareWithFullPrecision("2012-04-23", 1, "2013-04-23", invoker); + addAndCompareWithFullPrecision("2012-04-23", -1, "2011-04-23", invoker); // Leap-year handling - addYearWithMins("2012-02-29", -5, "2007-02-28"); - addYearWithMins("2012-02-29", -4, "2008-02-29"); - addYearWithMins("2012-02-29", -1, "2011-02-28"); - addYearWithMins("2012-02-29", 1, "2013-02-28"); - addYearWithMins("2012-02-29", 4, "2016-02-29"); - addYearWithMins("2012-02-29", 5, "2017-02-28"); + addAndCompareWithFullPrecision("2012-02-29", -5, "2007-02-28", invoker); + addAndCompareWithFullPrecision("2012-02-29", -4, "2008-02-29", invoker); + addAndCompareWithFullPrecision("2012-02-29", -1, "2011-02-28", invoker); + addAndCompareWithFullPrecision("2012-02-29", 1, "2013-02-28", invoker); + addAndCompareWithFullPrecision("2012-02-29", 4, "2016-02-29", invoker); + addAndCompareWithFullPrecision("2012-02-29", 5, "2017-02-28", invoker); + } + + @Test + public void testAddYear() + { + testYearArithmetic(new TimestampArithmeticInvoker() { + public Timestamp invoke(Timestamp input, Number amount) { + return input.addYear(amount.intValue()); + } + }); + } + + @Test + public void testAdjustYear() + { + testYearArithmetic(new TimestampArithmeticInvoker() { + public Timestamp invoke(Timestamp input, Number amount) { + return input.adjustYear(amount.intValue()); + } + }); } @Test(expected = IllegalArgumentException.class) @@ -1691,156 +1713,187 @@ public void testAddYearOutsideMinRange() ts1.addYear(-2000); } - //------------------------------------------------------------------------- - - private void addMonth(String orig, int amount, String expected) - { - Timestamp ts1 = Timestamp.valueOf(orig); - Timestamp ts2 = ts1.addMonth(amount); - checkTimestamp(expected, ts2); - } - - private void addMonth(String orig, int amount, String expected, - String suffix) + @Test(expected = IllegalArgumentException.class) + public void testAdjustYearOutsideMaxRange() { - addMonth(orig + suffix, amount, expected + suffix); + Timestamp ts1 = Timestamp.valueOf("1000T"); + ts1.adjustYear(10000); } - private void addMonthWithOffsets(String orig, int amount, String expected, - String suffix) + @Test(expected = IllegalArgumentException.class) + public void testAdjustYearOutsideMinRange() { - addMonth(orig, amount, expected, suffix + "-00:00"); - addMonth(orig, amount, expected, suffix + "Z"); - addMonth(orig, amount, expected, suffix + "+01:23"); - addMonth(orig, amount, expected, suffix + "-01:23"); - addMonth(orig, amount, expected, suffix + "+23:59"); - addMonth(orig, amount, expected, suffix + "-23:59"); + Timestamp ts1 = Timestamp.valueOf("1000T"); + ts1.adjustYear(-2000); } - private void addMonthWithMins(String orig, int amount, String expected) - { - addMonth(orig, amount, expected); - - addMonthWithOffsets(orig, amount, expected, "T19:03"); - addMonthWithOffsets(orig, amount, expected, "T19:03:23"); - addMonthWithOffsets(orig, amount, expected, "T19:03:23.0"); - addMonthWithOffsets(orig, amount, expected, "T19:03:23.00"); - addMonthWithOffsets(orig, amount, expected, "T19:03:23.000"); - addMonthWithOffsets(orig, amount, expected, "T19:03:23.456"); - addMonthWithOffsets(orig, amount, expected, "T19:03:23.0000"); - addMonthWithOffsets(orig, amount, expected, "T19:03:23.45678"); - } + //------------------------------------------------------------------------- @Test public void testAddMonth() { + TimestampArithmeticInvoker invoker = new TimestampArithmeticInvoker() { + @Override + public Timestamp invoke(Timestamp input, Number amount) { + return input.addMonth(amount.intValue()); + } + }; // Adding more precise amounts extends the precision. - addMonth("2012T", -1, "2011-12T"); - addMonth("2012T", 1, "2012-02T"); - - addMonth("2012-04T", -4, "2011-12T"); - addMonth("2012-04T", -1, "2012-03T"); - addMonth("2012-04T", 1, "2012-05T"); - addMonth("2012-04T", 9, "2013-01T"); - - addMonthWithMins("2012-04-23", -4, "2011-12-23"); - addMonthWithMins("2012-04-23", -1, "2012-03-23"); - addMonthWithMins("2012-04-23", 1, "2012-05-23"); - addMonthWithMins("2012-04-23", 9, "2013-01-23"); - - addMonthWithMins("2011-01-31", 1, "2011-02-28"); - addMonthWithMins("2011-02-28", 12, "2012-02-28"); - addMonthWithMins("2012-01-31", 1, "2012-02-29"); - addMonthWithMins("2012-01-31", 2, "2012-03-31"); - addMonthWithMins("2012-01-31", 3, "2012-04-30"); - addMonthWithMins("2012-02-29", -1, "2012-01-29"); - addMonthWithMins("2012-02-29", 12, "2013-02-28"); - addMonthWithMins("2012-02-29", 24, "2014-02-28"); - addMonthWithMins("2012-02-29", 48, "2016-02-29"); - addMonthWithMins("2013-01-31",-11, "2012-02-29"); + addAndCompare("2012T", -1, "2011-12T", invoker); + addAndCompare("2012T", 1, "2012-02T", invoker); + + addAndCompare("2012-04T", -4, "2011-12T", invoker); + addAndCompare("2012-04T", -1, "2012-03T", invoker); + addAndCompare("2012-04T", 1, "2012-05T", invoker); + addAndCompare("2012-04T", 9, "2013-01T", invoker); + + addAndCompareWithFullPrecision("2012-04-23", -4, "2011-12-23", invoker); + addAndCompareWithFullPrecision("2012-04-23", -1, "2012-03-23", invoker); + addAndCompareWithFullPrecision("2012-04-23", 1, "2012-05-23", invoker); + addAndCompareWithFullPrecision("2012-04-23", 9, "2013-01-23", invoker); + + addAndCompareWithFullPrecision("2011-01-31", 1, "2011-02-28", invoker); + addAndCompareWithFullPrecision("2011-02-28", 12, "2012-02-28", invoker); + addAndCompareWithFullPrecision("2012-01-31", 1, "2012-02-29", invoker); + addAndCompareWithFullPrecision("2012-01-31", 2, "2012-03-31", invoker); + addAndCompareWithFullPrecision("2012-01-31", 3, "2012-04-30", invoker); + addAndCompareWithFullPrecision("2012-02-29", -1, "2012-01-29", invoker); + addAndCompareWithFullPrecision("2012-02-29", 12, "2013-02-28", invoker); + addAndCompareWithFullPrecision("2012-02-29", 24, "2014-02-28", invoker); + addAndCompareWithFullPrecision("2012-02-29", 48, "2016-02-29", invoker); + addAndCompareWithFullPrecision("2013-01-31",-11, "2012-02-29", invoker); + } + + @Test + public void testAdjustMonth() + { + TimestampArithmeticInvoker invoker = new TimestampArithmeticInvoker() { + @Override + public Timestamp invoke(Timestamp input, Number amount) { + return input.adjustMonth(amount.intValue()); + } + }; + // Adding more precise amounts does not extend the precision. + addAndCompare("2012T", -1, "2011T", invoker); + addAndCompare("2012T", 1, "2012T", invoker); + + addAndCompare("2012-04T", -4, "2011-12T", invoker); + addAndCompare("2012-04T", -1, "2012-03T", invoker); + addAndCompare("2012-04T", 1, "2012-05T", invoker); + addAndCompare("2012-04T", 9, "2013-01T", invoker); + + addAndCompareWithFullPrecision("2012-04-23", -4, "2011-12-23", invoker); + addAndCompareWithFullPrecision("2012-04-23", -1, "2012-03-23", invoker); + addAndCompareWithFullPrecision("2012-04-23", 1, "2012-05-23", invoker); + addAndCompareWithFullPrecision("2012-04-23", 9, "2013-01-23", invoker); + + addAndCompareWithFullPrecision("2011-01-31", 1, "2011-02-28", invoker); + addAndCompareWithFullPrecision("2011-02-28", 12, "2012-02-28", invoker); + addAndCompareWithFullPrecision("2012-01-31", 1, "2012-02-29", invoker); + addAndCompareWithFullPrecision("2012-01-31", 2, "2012-03-31", invoker); + addAndCompareWithFullPrecision("2012-01-31", 3, "2012-04-30", invoker); + addAndCompareWithFullPrecision("2012-02-29", -1, "2012-01-29", invoker); + addAndCompareWithFullPrecision("2012-02-29", 12, "2013-02-28", invoker); + addAndCompareWithFullPrecision("2012-02-29", 24, "2014-02-28", invoker); + addAndCompareWithFullPrecision("2012-02-29", 48, "2016-02-29", invoker); + addAndCompareWithFullPrecision("2013-01-31",-11, "2012-02-29", invoker); } @Test(expected = IllegalArgumentException.class) public void testAddMonthOutsideMaxRange() { Timestamp ts1 = Timestamp.valueOf("1000T"); - ts1.addYear(10000*12); + ts1.addMonth(10000*12); } @Test(expected = IllegalArgumentException.class) public void testAddMonthOutsideMinRange() { Timestamp ts1 = Timestamp.valueOf("1000T"); - ts1.addYear(-2000*12); + ts1.addMonth(-2000*12); } - //------------------------------------------------------------------------- - - private void addDay(String orig, int amount, String expected) - { - Timestamp ts1 = Timestamp.valueOf(orig); - Timestamp ts2 = ts1.addDay(amount); - checkTimestamp(expected, ts2); - } - - private void addDay(String orig, int amount, String expected, - String suffix) + @Test(expected = IllegalArgumentException.class) + public void testAdjustMonthOutsideMaxRange() { - addDay(orig + suffix, amount, expected + suffix); + Timestamp ts1 = Timestamp.valueOf("1000T"); + ts1.adjustMonth(10000*12); } - private void addDayWithOffsets(String orig, int amount, String expected, - String suffix) + @Test(expected = IllegalArgumentException.class) + public void testAdjustMonthOutsideMinRange() { - addDay(orig, amount, expected, suffix + "-00:00"); - addDay(orig, amount, expected, suffix + "Z"); - addDay(orig, amount, expected, suffix + "+01:23"); - addDay(orig, amount, expected, suffix + "-01:23"); - addDay(orig, amount, expected, suffix + "+23:59"); - addDay(orig, amount, expected, suffix + "-23:59"); + Timestamp ts1 = Timestamp.valueOf("1000T"); + ts1.adjustMonth(-2000*12); } - private void addDayWithMins(String orig, int amount, String expected) - { - addDay(orig, amount, expected); - - addDayWithOffsets(orig, amount, expected, "T19:03"); - addDayWithOffsets(orig, amount, expected, "T19:03:23"); - addDayWithOffsets(orig, amount, expected, "T19:03:23.0"); - addDayWithOffsets(orig, amount, expected, "T19:03:23.00"); - addDayWithOffsets(orig, amount, expected, "T19:03:23.000"); - addDayWithOffsets(orig, amount, expected, "T19:03:23.456"); - addDayWithOffsets(orig, amount, expected, "T19:03:23.0000"); - addDayWithOffsets(orig, amount, expected, "T19:03:23.45678"); - } + //------------------------------------------------------------------------- @Test public void testAddDay() { + TimestampArithmeticInvoker invoker = new TimestampArithmeticInvoker() { + public Timestamp invoke(Timestamp input, Number amount) { + return input.addDay(amount.intValue()); + } + }; // Adding more precise amounts extends the precision. - addDay("2012T", 1, "2012-01-02"); - addDay("2012T", 0, "2012-01-01"); - addDay("2012T", -1, "2011-12-31"); - - addDay("2012-04T",-32, "2012-02-29"); - addDay("2012-04T",-31, "2012-03-01"); - addDay("2012-04T", -1, "2012-03-31"); - addDay("2012-04T", 1, "2012-04-02"); - addDay("2012-04T", 9, "2012-04-10"); - addDay("2012-04T", 30, "2012-05-01"); - addDay("2012-04T", 31, "2012-05-02"); - - addDayWithMins("2011-01-31",-31, "2010-12-31"); - addDayWithMins("2011-01-31",-30, "2011-01-01"); - addDayWithMins("2011-01-31", 1, "2011-02-01"); - addDayWithMins("2011-01-31", 28, "2011-02-28"); - addDayWithMins("2011-01-31", 29, "2011-03-01"); - addDayWithMins("2011-01-31", 30, "2011-03-02"); - addDayWithMins("2012-01-31", 1, "2012-02-01"); - addDayWithMins("2012-01-31", 28, "2012-02-28"); - addDayWithMins("2012-01-31", 29, "2012-02-29"); - addDayWithMins("2012-01-31", 30, "2012-03-01"); - addDayWithMins("2012-03-01",-30, "2012-01-31"); + addAndCompare("2012T", 1, "2012-01-02", invoker); + addAndCompare("2012T", 0, "2012-01-01", invoker); + addAndCompare("2012T", -1, "2011-12-31", invoker); + + addAndCompare("2012-04T",-32, "2012-02-29", invoker); + addAndCompare("2012-04T",-31, "2012-03-01", invoker); + addAndCompare("2012-04T", -1, "2012-03-31", invoker); + addAndCompare("2012-04T", 1, "2012-04-02", invoker); + addAndCompare("2012-04T", 9, "2012-04-10", invoker); + addAndCompare("2012-04T", 30, "2012-05-01", invoker); + addAndCompare("2012-04T", 31, "2012-05-02", invoker); + + addAndCompareWithFullPrecision("2011-01-31",-31, "2010-12-31", invoker); + addAndCompareWithFullPrecision("2011-01-31",-30, "2011-01-01", invoker); + addAndCompareWithFullPrecision("2011-01-31", 1, "2011-02-01", invoker); + addAndCompareWithFullPrecision("2011-01-31", 28, "2011-02-28", invoker); + addAndCompareWithFullPrecision("2011-01-31", 29, "2011-03-01", invoker); + addAndCompareWithFullPrecision("2011-01-31", 30, "2011-03-02", invoker); + addAndCompareWithFullPrecision("2012-01-31", 1, "2012-02-01", invoker); + addAndCompareWithFullPrecision("2012-01-31", 28, "2012-02-28", invoker); + addAndCompareWithFullPrecision("2012-01-31", 29, "2012-02-29", invoker); + addAndCompareWithFullPrecision("2012-01-31", 30, "2012-03-01", invoker); + addAndCompareWithFullPrecision("2012-03-01",-30, "2012-01-31", invoker); + } + + @Test + public void testAdjustDay() + { + TimestampArithmeticInvoker invoker = new TimestampArithmeticInvoker() { + public Timestamp invoke(Timestamp input, Number amount) { + return input.adjustDay(amount.intValue()); + } + }; + // Adding more precise amounts does not extend the precision. + addAndCompare("2012T", 1, "2012T", invoker); + addAndCompare("2012T", -1, "2011T", invoker); + + addAndCompare("2012-04T",-32, "2012-02T", invoker); + addAndCompare("2012-04T",-31, "2012-03T", invoker); + addAndCompare("2012-04T", -1, "2012-03T", invoker); + addAndCompare("2012-04T", 1, "2012-04T", invoker); + addAndCompare("2012-04T", 9, "2012-04T", invoker); + addAndCompare("2012-04T", 30, "2012-05T", invoker); + addAndCompare("2012-04T", 31, "2012-05T", invoker); + + addAndCompareWithFullPrecision("2011-01-31",-31, "2010-12-31", invoker); + addAndCompareWithFullPrecision("2011-01-31",-30, "2011-01-01", invoker); + addAndCompareWithFullPrecision("2011-01-31", 1, "2011-02-01", invoker); + addAndCompareWithFullPrecision("2011-01-31", 28, "2011-02-28", invoker); + addAndCompareWithFullPrecision("2011-01-31", 29, "2011-03-01", invoker); + addAndCompareWithFullPrecision("2011-01-31", 30, "2011-03-02", invoker); + addAndCompareWithFullPrecision("2012-01-31", 1, "2012-02-01", invoker); + addAndCompareWithFullPrecision("2012-01-31", 28, "2012-02-28", invoker); + addAndCompareWithFullPrecision("2012-01-31", 29, "2012-02-29", invoker); + addAndCompareWithFullPrecision("2012-01-31", 30, "2012-03-01", invoker); + addAndCompareWithFullPrecision("2012-03-01",-30, "2012-01-31", invoker); } @Test(expected = IllegalArgumentException.class) @@ -1857,84 +1910,126 @@ public void testAddDayOutsideMinRange() ts1.addDay(-2000*12*35); } - //------------------------------------------------------------------------- - - private void addHour(String orig, int amount, String expected) + @Test(expected = IllegalArgumentException.class) + public void testAdjustDayOutsideMaxRange() { - Timestamp ts1 = Timestamp.valueOf(orig); - Timestamp ts2 = ts1.addHour(amount); - checkTimestamp(expected, ts2); + Timestamp ts1 = Timestamp.valueOf("1000T"); + ts1.adjustDay(10000*12*35); } - private void addHour(String orig, int amount, String expected, - String suffix) + @Test(expected = IllegalArgumentException.class) + public void testAdjustDayOutsideMinRange() { - addHour(orig + suffix, amount, expected + suffix); + Timestamp ts1 = Timestamp.valueOf("1000T"); + ts1.adjustDay(-2000*12*35); } - private void addHourWithOffsets(String orig, int amount, String expected, - String suffix) - { - addHour(orig, amount, expected, suffix + "-00:00"); - addHour(orig, amount, expected, suffix + "Z"); - addHour(orig, amount, expected, suffix + "+01:23"); - addHour(orig, amount, expected, suffix + "-01:23"); - addHour(orig, amount, expected, suffix + "+23:59"); - addHour(orig, amount, expected, suffix + "-23:59"); - } + //------------------------------------------------------------------------- - private void addHourWithMins(String orig, int amount, String expected) + private void addAndCompareHourWithFullPrecision(String orig, int amount, String expected, TimestampArithmeticInvoker invoker) { - addHourWithOffsets(orig, amount, expected, ":03"); - addHourWithOffsets(orig, amount, expected, ":03:23"); - addHourWithOffsets(orig, amount, expected, ":03:23.0"); - addHourWithOffsets(orig, amount, expected, ":03:23.00"); - addHourWithOffsets(orig, amount, expected, ":03:23.000"); - addHourWithOffsets(orig, amount, expected, ":03:23.456"); - addHourWithOffsets(orig, amount, expected, ":03:23.0000"); - addHourWithOffsets(orig, amount, expected, ":03:23.45678"); + addAndCompareWithOffsets(orig, amount, expected, ":03", invoker); + addAndCompareWithOffsets(orig, amount, expected, ":03:23", invoker); + addAndCompareWithOffsets(orig, amount, expected, ":03:23.0", invoker); + addAndCompareWithOffsets(orig, amount, expected, ":03:23.00", invoker); + addAndCompareWithOffsets(orig, amount, expected, ":03:23.000", invoker); + addAndCompareWithOffsets(orig, amount, expected, ":03:23.456", invoker); + addAndCompareWithOffsets(orig, amount, expected, ":03:23.0000", invoker); + addAndCompareWithOffsets(orig, amount, expected, ":03:23.45678", invoker); } @Test public void testAddHour() { + TimestampArithmeticInvoker invoker = new TimestampArithmeticInvoker() { + public Timestamp invoke(Timestamp input, Number amount) { + return input.addHour(amount.intValue()); + } + }; // Adding more precise amounts extends the precision. - addHour("2012T", -1, "2011-12-31T23:00-00:00"); - addHour("2012T", 0, "2012-01-01T00:00-00:00"); - addHour("2012T", 1, "2012-01-01T01:00-00:00"); - - addHour("2012-04T", -31 * 24 - 1, "2012-02-29T23:00-00:00"); - addHour("2012-04T", -31 * 24 , "2012-03-01T00:00-00:00"); - addHour("2012-04T", -1, "2012-03-31T23:00-00:00"); - addHour("2012-04T", 0, "2012-04-01T00:00-00:00"); - addHour("2012-04T", 1, "2012-04-01T01:00-00:00"); - addHour("2012-04T", 30 * 24 - 1, "2012-04-30T23:00-00:00"); - addHour("2012-04T", 30 * 24 , "2012-05-01T00:00-00:00"); - - addHour("2011-02-28", -1, "2011-02-27T23:00-00:00"); - addHour("2011-02-28", 0, "2011-02-28T00:00-00:00"); - addHour("2011-02-28", 1, "2011-02-28T01:00-00:00"); - addHour("2011-02-28", 24, "2011-03-01T00:00-00:00"); - addHour("2011-03-01", -24, "2011-02-28T00:00-00:00"); - addHour("2012-02-28", 24, "2012-02-29T00:00-00:00"); - addHour("2012-02-29", -24, "2012-02-28T00:00-00:00"); - addHour("2012-02-29", 24, "2012-03-01T00:00-00:00"); - - addHour("2012-10-04",-48, "2012-10-02T00:00-00:00"); - addHour("2012-10-04",-24, "2012-10-03T00:00-00:00"); - addHour("2012-10-04", -1, "2012-10-03T23:00-00:00"); - addHour("2012-10-04", 1, "2012-10-04T01:00-00:00"); - addHour("2012-10-04", 24, "2012-10-05T00:00-00:00"); - addHour("2012-10-04", 48, "2012-10-06T00:00-00:00"); - - addHourWithMins("2011-01-31T12", 0, "2011-01-31T12"); - addHourWithMins("2011-01-31T12", 12, "2011-02-01T00"); - - addHourWithMins("2011-02-28T02", 25, "2011-03-01T03"); - addHourWithMins("2012-02-28T02", 25, "2012-02-29T03"); - - addHourWithMins("2011-03-01T02", -4, "2011-02-28T22"); - addHourWithMins("2012-03-01T02", -4, "2012-02-29T22"); + addAndCompare("2012T", -1, "2011-12-31T23:00-00:00", invoker); + addAndCompare("2012T", 0, "2012-01-01T00:00-00:00", invoker); + addAndCompare("2012T", 1, "2012-01-01T01:00-00:00", invoker); + + addAndCompare("2012-04T", -31 * 24 - 1, "2012-02-29T23:00-00:00", invoker); + addAndCompare("2012-04T", -31 * 24 , "2012-03-01T00:00-00:00", invoker); + addAndCompare("2012-04T", -1, "2012-03-31T23:00-00:00", invoker); + addAndCompare("2012-04T", 0, "2012-04-01T00:00-00:00", invoker); + addAndCompare("2012-04T", 1, "2012-04-01T01:00-00:00", invoker); + addAndCompare("2012-04T", 30 * 24 - 1, "2012-04-30T23:00-00:00", invoker); + addAndCompare("2012-04T", 30 * 24 , "2012-05-01T00:00-00:00", invoker); + + addAndCompare("2011-02-28", -1, "2011-02-27T23:00-00:00", invoker); + addAndCompare("2011-02-28", 0, "2011-02-28T00:00-00:00", invoker); + addAndCompare("2011-02-28", 1, "2011-02-28T01:00-00:00", invoker); + addAndCompare("2011-02-28", 24, "2011-03-01T00:00-00:00", invoker); + addAndCompare("2011-03-01", -24, "2011-02-28T00:00-00:00", invoker); + addAndCompare("2012-02-28", 24, "2012-02-29T00:00-00:00", invoker); + addAndCompare("2012-02-29", -24, "2012-02-28T00:00-00:00", invoker); + addAndCompare("2012-02-29", 24, "2012-03-01T00:00-00:00", invoker); + + addAndCompare("2012-10-04",-48, "2012-10-02T00:00-00:00", invoker); + addAndCompare("2012-10-04",-24, "2012-10-03T00:00-00:00", invoker); + addAndCompare("2012-10-04", -1, "2012-10-03T23:00-00:00", invoker); + addAndCompare("2012-10-04", 1, "2012-10-04T01:00-00:00", invoker); + addAndCompare("2012-10-04", 24, "2012-10-05T00:00-00:00", invoker); + addAndCompare("2012-10-04", 48, "2012-10-06T00:00-00:00", invoker); + + addAndCompareHourWithFullPrecision("2011-01-31T12", 0, "2011-01-31T12", invoker); + addAndCompareHourWithFullPrecision("2011-01-31T12", 12, "2011-02-01T00", invoker); + + addAndCompareHourWithFullPrecision("2011-02-28T02", 25, "2011-03-01T03", invoker); + addAndCompareHourWithFullPrecision("2012-02-28T02", 25, "2012-02-29T03", invoker); + + addAndCompareHourWithFullPrecision("2011-03-01T02", -4, "2011-02-28T22", invoker); + addAndCompareHourWithFullPrecision("2012-03-01T02", -4, "2012-02-29T22", invoker); + } + + @Test + public void testAdjustHour() + { + TimestampArithmeticInvoker invoker = new TimestampArithmeticInvoker() { + public Timestamp invoke(Timestamp input, Number amount) { + return input.adjustHour(amount.intValue()); + } + }; + // Adding more precise amounts does not extend the precision. + addAndCompare("2012T", -1, "2011T", invoker); + addAndCompare("2012T", 0, "2012T", invoker); + addAndCompare("2012T", 1, "2012T", invoker); + + addAndCompare("2012-04T", -31 * 24 - 1, "2012-02T", invoker); + addAndCompare("2012-04T", -31 * 24 , "2012-03T", invoker); + addAndCompare("2012-04T", -1, "2012-03T", invoker); + addAndCompare("2012-04T", 0, "2012-04T", invoker); + addAndCompare("2012-04T", 1, "2012-04T", invoker); + addAndCompare("2012-04T", 30 * 24 - 1, "2012-04T", invoker); + addAndCompare("2012-04T", 30 * 24 , "2012-05T", invoker); + + addAndCompare("2011-02-28", -1, "2011-02-27", invoker); + addAndCompare("2011-02-28", 0, "2011-02-28", invoker); + addAndCompare("2011-02-28", 1, "2011-02-28", invoker); + addAndCompare("2011-02-28", 24, "2011-03-01", invoker); + addAndCompare("2011-03-01", -24, "2011-02-28", invoker); + addAndCompare("2012-02-28", 24, "2012-02-29", invoker); + addAndCompare("2012-02-29", -24, "2012-02-28", invoker); + addAndCompare("2012-02-29", 24, "2012-03-01", invoker); + + addAndCompare("2012-10-04",-48, "2012-10-02", invoker); + addAndCompare("2012-10-04",-24, "2012-10-03", invoker); + addAndCompare("2012-10-04", -1, "2012-10-03", invoker); + addAndCompare("2012-10-04", 1, "2012-10-04", invoker); + addAndCompare("2012-10-04", 24, "2012-10-05", invoker); + addAndCompare("2012-10-04", 48, "2012-10-06", invoker); + + addAndCompareHourWithFullPrecision("2011-01-31T12", 0, "2011-01-31T12", invoker); + addAndCompareHourWithFullPrecision("2011-01-31T12", 12, "2011-02-01T00", invoker); + + addAndCompareHourWithFullPrecision("2011-02-28T02", 25, "2011-03-01T03", invoker); + addAndCompareHourWithFullPrecision("2012-02-28T02", 25, "2012-02-29T03", invoker); + + addAndCompareHourWithFullPrecision("2011-03-01T02", -4, "2011-02-28T22", invoker); + addAndCompareHourWithFullPrecision("2012-03-01T02", -4, "2012-02-29T22", invoker); } @Test(expected = IllegalArgumentException.class) @@ -1951,86 +2046,130 @@ public void testAddHourOutsideMinRange() ts1.addHour(-2000*12*35*24); } - //------------------------------------------------------------------------- - - private void addMinute(String orig, int amount, String expected) + @Test(expected = IllegalArgumentException.class) + public void testAdjustHourOutsideMaxRange() { - Timestamp ts1 = Timestamp.valueOf(orig); - Timestamp ts2 = ts1.addMinute(amount); - checkTimestamp(expected, ts2); + Timestamp ts1 = Timestamp.valueOf("1000T"); + ts1.adjustHour(10000*12*35*24); } - private void addMinute(String orig, int amount, String expected, - String suffix) + @Test(expected = IllegalArgumentException.class) + public void testAdjustHourOutsideMinRange() { - addMinute(orig + suffix, amount, expected + suffix); + Timestamp ts1 = Timestamp.valueOf("1000T"); + ts1.adjustHour(-2000*12*35*24); } - private void addMinuteWithOffsets(String orig, int amount, String expected, - String suffix) - { - addMinute(orig, amount, expected, suffix + "-00:00"); - addMinute(orig, amount, expected, suffix + "Z"); - addMinute(orig, amount, expected, suffix + "+01:23"); - addMinute(orig, amount, expected, suffix + "-01:23"); - addMinute(orig, amount, expected, suffix + "+23:59"); - addMinute(orig, amount, expected, suffix + "-23:59"); - } + //------------------------------------------------------------------------- - private void addMinuteWithsSecs(String orig, int amount, String expected) + private void addAndCompareMinuteWithFullPrecision(String orig, int amount, String expected, TimestampArithmeticInvoker invoker) { - addMinuteWithOffsets(orig, amount, expected, ""); - addMinuteWithOffsets(orig, amount, expected, ":23"); - addMinuteWithOffsets(orig, amount, expected, ":23.0"); - addMinuteWithOffsets(orig, amount, expected, ":23.00"); - addMinuteWithOffsets(orig, amount, expected, ":23.000"); - addMinuteWithOffsets(orig, amount, expected, ":23.456"); - addMinuteWithOffsets(orig, amount, expected, ":23.0000"); - addMinuteWithOffsets(orig, amount, expected, ":23.45678"); + addAndCompareWithOffsets(orig, amount, expected, "", invoker); + addAndCompareWithOffsets(orig, amount, expected, ":23", invoker); + addAndCompareWithOffsets(orig, amount, expected, ":23.0", invoker); + addAndCompareWithOffsets(orig, amount, expected, ":23.00", invoker); + addAndCompareWithOffsets(orig, amount, expected, ":23.000", invoker); + addAndCompareWithOffsets(orig, amount, expected, ":23.456", invoker); + addAndCompareWithOffsets(orig, amount, expected, ":23.0000", invoker); + addAndCompareWithOffsets(orig, amount, expected, ":23.45678", invoker); } @Test public void testAddMinute() { + TimestampArithmeticInvoker invoker = new TimestampArithmeticInvoker() { + public Timestamp invoke(Timestamp input, Number amount) { + return input.addMinute(amount.intValue()); + } + }; // Adding more precise amounts extends the precision. - addMinute("2012T", -1, "2011-12-31T23:59-00:00"); - addMinute("2012T", 0, "2012-01-01T00:00-00:00"); - addMinute("2012T", 1, "2012-01-01T00:01-00:00"); - - addMinute("2012-04T", -31 * 24 * 60 - 1, "2012-02-29T23:59-00:00"); - addMinute("2012-04T", -31 * 24 * 60 , "2012-03-01T00:00-00:00"); - addMinute("2012-04T", -1, "2012-03-31T23:59-00:00"); - addMinute("2012-04T", 0, "2012-04-01T00:00-00:00"); - addMinute("2012-04T", 1, "2012-04-01T00:01-00:00"); - addMinute("2012-04T", 30 * 24 * 60 - 1, "2012-04-30T23:59-00:00"); - addMinute("2012-04T", 30 * 24 * 60 , "2012-05-01T00:00-00:00"); - - addMinute("2011-02-28", -1, "2011-02-27T23:59-00:00"); - addMinute("2011-02-28", 0, "2011-02-28T00:00-00:00"); - addMinute("2011-02-28", 1, "2011-02-28T00:01-00:00"); - addMinute("2011-02-28", 24 * 60, "2011-03-01T00:00-00:00"); - addMinute("2011-03-01", -24 * 60, "2011-02-28T00:00-00:00"); - addMinute("2012-02-28", 24 * 60, "2012-02-29T00:00-00:00"); - addMinute("2012-02-29", -24 * 60, "2012-02-28T00:00-00:00"); - addMinute("2012-02-29", 24 * 60, "2012-03-01T00:00-00:00"); - - addMinute("2012-10-04", -48 * 60, "2012-10-02T00:00-00:00"); - addMinute("2012-10-04", -24 * 60, "2012-10-03T00:00-00:00"); - addMinute("2012-10-04", -1, "2012-10-03T23:59-00:00"); - addMinute("2012-10-04", 1, "2012-10-04T00:01-00:00"); - addMinute("2012-10-04", 24 * 60, "2012-10-05T00:00-00:00"); - addMinute("2012-10-04", 48 * 60, "2012-10-06T00:00-00:00"); - - addMinuteWithsSecs("2011-01-31T12:03", -60, "2011-01-31T11:03"); - addMinuteWithsSecs("2011-01-31T12:03", 0, "2011-01-31T12:03"); - addMinuteWithsSecs("2011-01-31T12:03", 60, "2011-01-31T13:03"); - addMinuteWithsSecs("2011-01-31T23:03", 57, "2011-02-01T00:00"); - - addMinuteWithsSecs("2011-02-28T02:03", 25 * 60, "2011-03-01T03:03"); - addMinuteWithsSecs("2012-02-28T02:03", 25 * 60, "2012-02-29T03:03"); - - addMinuteWithsSecs("2011-03-01T02:35", -4 * 60, "2011-02-28T22:35"); - addMinuteWithsSecs("2012-03-01T02:35", -4 * 60, "2012-02-29T22:35"); + addAndCompare("2012T", -1, "2011-12-31T23:59-00:00", invoker); + addAndCompare("2012T", 0, "2012-01-01T00:00-00:00", invoker); + addAndCompare("2012T", 1, "2012-01-01T00:01-00:00", invoker); + + addAndCompare("2012-04T", -31 * 24 * 60 - 1, "2012-02-29T23:59-00:00", invoker); + addAndCompare("2012-04T", -31 * 24 * 60 , "2012-03-01T00:00-00:00", invoker); + addAndCompare("2012-04T", -1, "2012-03-31T23:59-00:00", invoker); + addAndCompare("2012-04T", 0, "2012-04-01T00:00-00:00", invoker); + addAndCompare("2012-04T", 1, "2012-04-01T00:01-00:00", invoker); + addAndCompare("2012-04T", 30 * 24 * 60 - 1, "2012-04-30T23:59-00:00", invoker); + addAndCompare("2012-04T", 30 * 24 * 60 , "2012-05-01T00:00-00:00", invoker); + + addAndCompare("2011-02-28", -1, "2011-02-27T23:59-00:00", invoker); + addAndCompare("2011-02-28", 0, "2011-02-28T00:00-00:00", invoker); + addAndCompare("2011-02-28", 1, "2011-02-28T00:01-00:00", invoker); + addAndCompare("2011-02-28", 24 * 60, "2011-03-01T00:00-00:00", invoker); + addAndCompare("2011-03-01", -24 * 60, "2011-02-28T00:00-00:00", invoker); + addAndCompare("2012-02-28", 24 * 60, "2012-02-29T00:00-00:00", invoker); + addAndCompare("2012-02-29", -24 * 60, "2012-02-28T00:00-00:00", invoker); + addAndCompare("2012-02-29", 24 * 60, "2012-03-01T00:00-00:00", invoker); + + addAndCompare("2012-10-04", -48 * 60, "2012-10-02T00:00-00:00", invoker); + addAndCompare("2012-10-04", -24 * 60, "2012-10-03T00:00-00:00", invoker); + addAndCompare("2012-10-04", -1, "2012-10-03T23:59-00:00", invoker); + addAndCompare("2012-10-04", 1, "2012-10-04T00:01-00:00", invoker); + addAndCompare("2012-10-04", 24 * 60, "2012-10-05T00:00-00:00", invoker); + addAndCompare("2012-10-04", 48 * 60, "2012-10-06T00:00-00:00", invoker); + + addAndCompareMinuteWithFullPrecision("2011-01-31T12:03", -60, "2011-01-31T11:03", invoker); + addAndCompareMinuteWithFullPrecision("2011-01-31T12:03", 0, "2011-01-31T12:03", invoker); + addAndCompareMinuteWithFullPrecision("2011-01-31T12:03", 60, "2011-01-31T13:03", invoker); + addAndCompareMinuteWithFullPrecision("2011-01-31T23:03", 57, "2011-02-01T00:00", invoker); + + addAndCompareMinuteWithFullPrecision("2011-02-28T02:03", 25 * 60, "2011-03-01T03:03", invoker); + addAndCompareMinuteWithFullPrecision("2012-02-28T02:03", 25 * 60, "2012-02-29T03:03", invoker); + + addAndCompareMinuteWithFullPrecision("2011-03-01T02:35", -4 * 60, "2011-02-28T22:35", invoker); + addAndCompareMinuteWithFullPrecision("2012-03-01T02:35", -4 * 60, "2012-02-29T22:35", invoker); + } + + @Test + public void testAdjustMinute() + { + TimestampArithmeticInvoker invoker = new TimestampArithmeticInvoker() { + public Timestamp invoke(Timestamp input, Number amount) { + return input.adjustMinute(amount.intValue()); + } + }; + // Adding more precise amounts does not extend the precision. + addAndCompare("2012T", -1, "2011T", invoker); + addAndCompare("2012T", 0, "2012T", invoker); + addAndCompare("2012T", 1, "2012T", invoker); + + addAndCompare("2012-04T", -31 * 24 * 60 - 1, "2012-02T", invoker); + addAndCompare("2012-04T", -31 * 24 * 60 , "2012-03T", invoker); + addAndCompare("2012-04T", -1, "2012-03T", invoker); + addAndCompare("2012-04T", 0, "2012-04T", invoker); + addAndCompare("2012-04T", 1, "2012-04T", invoker); + addAndCompare("2012-04T", 30 * 24 * 60 - 1, "2012-04T", invoker); + addAndCompare("2012-04T", 30 * 24 * 60 , "2012-05T", invoker); + + addAndCompare("2011-02-28", -1, "2011-02-27", invoker); + addAndCompare("2011-02-28", 0, "2011-02-28", invoker); + addAndCompare("2011-02-28", 1, "2011-02-28", invoker); + addAndCompare("2011-02-28", 24 * 60, "2011-03-01", invoker); + addAndCompare("2011-03-01", -24 * 60, "2011-02-28", invoker); + addAndCompare("2012-02-28", 24 * 60, "2012-02-29", invoker); + addAndCompare("2012-02-29", -24 * 60, "2012-02-28", invoker); + addAndCompare("2012-02-29", 24 * 60, "2012-03-01", invoker); + + addAndCompare("2012-10-04", -48 * 60, "2012-10-02", invoker); + addAndCompare("2012-10-04", -24 * 60, "2012-10-03", invoker); + addAndCompare("2012-10-04", -1, "2012-10-03", invoker); + addAndCompare("2012-10-04", 1, "2012-10-04", invoker); + addAndCompare("2012-10-04", 24 * 60, "2012-10-05", invoker); + addAndCompare("2012-10-04", 48 * 60, "2012-10-06", invoker); + + addAndCompareMinuteWithFullPrecision("2011-01-31T12:03", -60, "2011-01-31T11:03", invoker); + addAndCompareMinuteWithFullPrecision("2011-01-31T12:03", 0, "2011-01-31T12:03", invoker); + addAndCompareMinuteWithFullPrecision("2011-01-31T12:03", 60, "2011-01-31T13:03", invoker); + addAndCompareMinuteWithFullPrecision("2011-01-31T23:03", 57, "2011-02-01T00:00", invoker); + + addAndCompareMinuteWithFullPrecision("2011-02-28T02:03", 25 * 60, "2011-03-01T03:03", invoker); + addAndCompareMinuteWithFullPrecision("2012-02-28T02:03", 25 * 60, "2012-02-29T03:03", invoker); + + addAndCompareMinuteWithFullPrecision("2011-03-01T02:35", -4 * 60, "2011-02-28T22:35", invoker); + addAndCompareMinuteWithFullPrecision("2012-03-01T02:35", -4 * 60, "2012-02-29T22:35", invoker); } @Test(expected = IllegalArgumentException.class) @@ -2047,170 +2186,340 @@ public void testAddMinuteOutsideMinRange() ts1.addMinute(-2*12*35*24*60); } - //------------------------------------------------------------------------- - - private void addSecond(String orig, int amount, String expected) + @Test(expected = IllegalArgumentException.class) + public void testAdjustMinuteOutsideMaxRange() { - Timestamp ts1 = Timestamp.valueOf(orig); - Timestamp ts2 = ts1.addSecond(amount); - checkTimestamp(expected, ts2); + Timestamp ts1 = Timestamp.valueOf("9998T"); + ts1.adjustMinute(2*12*35*24*60); } - private void addSecond(String orig, int amount, String expected, - String suffix) + @Test(expected = IllegalArgumentException.class) + public void testAdjustMinuteOutsideMinRange() { - addSecond(orig + suffix, amount, expected + suffix); + Timestamp ts1 = Timestamp.valueOf("0001T"); + ts1.adjustMinute(-2*12*35*24*60); } - private void addSecondWithOffsets(String orig, int amount, String expected, - String suffix) - { - addSecond(orig, amount, expected, suffix + "-00:00"); - addSecond(orig, amount, expected, suffix + "Z"); - addSecond(orig, amount, expected, suffix + "+01:23"); - addSecond(orig, amount, expected, suffix + "-01:23"); - addSecond(orig, amount, expected, suffix + "+23:59"); - addSecond(orig, amount, expected, suffix + "-23:59"); - } + //------------------------------------------------------------------------- - private void addSecondWithsFrac(String orig, int amount, String expected) + private void addAndCompareSecondWithFullPrecision(String orig, int amount, String expected, TimestampArithmeticInvoker invoker) { - addSecondWithOffsets(orig, amount, expected, ""); - addSecondWithOffsets(orig, amount, expected, ".0"); - addSecondWithOffsets(orig, amount, expected, ".00"); - addSecondWithOffsets(orig, amount, expected, ".000"); - addSecondWithOffsets(orig, amount, expected, ".456"); - addSecondWithOffsets(orig, amount, expected, ".0000"); - addSecondWithOffsets(orig, amount, expected, ".45678"); + addAndCompareWithOffsets(orig, amount, expected, "", invoker); + addAndCompareWithOffsets(orig, amount, expected, ".0", invoker); + addAndCompareWithOffsets(orig, amount, expected, ".00", invoker); + addAndCompareWithOffsets(orig, amount, expected, ".000", invoker); + addAndCompareWithOffsets(orig, amount, expected, ".456", invoker); + addAndCompareWithOffsets(orig, amount, expected, ".0000", invoker); + addAndCompareWithOffsets(orig, amount, expected, ".45678", invoker); } @Test public void testAddSecond() { + TimestampArithmeticInvoker invoker = new TimestampArithmeticInvoker() { + public Timestamp invoke(Timestamp input, Number amount) { + return input.addSecond(amount.intValue()); + } + }; // Adding more precise amounts extends the precision. - addSecond("2012T", -1, "2011-12-31T23:59:59-00:00"); - addSecond("2012T", 0, "2012-01-01T00:00:00-00:00"); - addSecond("2012T", 1, "2012-01-01T00:00:01-00:00"); - - addSecond("2012-04T", -31 * 24 * 60 * 60 - 1, "2012-02-29T23:59:59-00:00"); - addSecond("2012-04T", -31 * 24 * 60 * 60 , "2012-03-01T00:00:00-00:00"); - addSecond("2012-04T", -1, "2012-03-31T23:59:59-00:00"); - addSecond("2012-04T", 0, "2012-04-01T00:00:00-00:00"); - addSecond("2012-04T", 1, "2012-04-01T00:00:01-00:00"); - addSecond("2012-04T", 30 * 24 * 60 * 60 - 1, "2012-04-30T23:59:59-00:00"); - addSecond("2012-04T", 30 * 24 * 60 * 60 , "2012-05-01T00:00:00-00:00"); - - - addSecond("2011-02-28", -1, "2011-02-27T23:59:59-00:00"); - addSecond("2011-02-28", 0, "2011-02-28T00:00:00-00:00"); - addSecond("2011-02-28", 1, "2011-02-28T00:00:01-00:00"); - addSecond("2011-02-28", 24 * 60 * 60, "2011-03-01T00:00:00-00:00"); - addSecond("2011-03-01", -24 * 60 * 60, "2011-02-28T00:00:00-00:00"); - addSecond("2012-02-28", 24 * 60 * 60, "2012-02-29T00:00:00-00:00"); - addSecond("2012-02-29", -24 * 60 * 60, "2012-02-28T00:00:00-00:00"); - addSecond("2012-02-29", 24 * 60 * 60, "2012-03-01T00:00:00-00:00"); - - addSecond("2012-10-04", -24 * 60 * 60 - 1, "2012-10-02T23:59:59-00:00"); - addSecond("2012-10-04", -24 * 60 * 60 , "2012-10-03T00:00:00-00:00"); - addSecond("2012-10-04", -1, "2012-10-03T23:59:59-00:00"); - addSecond("2012-10-04", 1, "2012-10-04T00:00:01-00:00"); - addSecond("2012-10-04", 48 * 60 * 60 - 1, "2012-10-05T23:59:59-00:00"); - addSecond("2012-10-04", 48 * 60 * 60 , "2012-10-06T00:00:00-00:00"); - - addSecondWithsFrac("2011-01-31T12:03:23", -60 * 60, "2011-01-31T11:03:23"); - addSecondWithsFrac("2011-01-31T12:03:23", 0, "2011-01-31T12:03:23"); - addSecondWithsFrac("2011-01-31T12:03:23", 60 * 60, "2011-01-31T13:03:23"); - addSecondWithsFrac("2011-01-31T23:03:23", 57 * 60, "2011-02-01T00:00:23"); + addAndCompare("2012T", -1, "2011-12-31T23:59:59-00:00", invoker); + addAndCompare("2012T", 0, "2012-01-01T00:00:00-00:00", invoker); + addAndCompare("2012T", 1, "2012-01-01T00:00:01-00:00", invoker); + + addAndCompare("2012-04T", -31 * 24 * 60 * 60 - 1, "2012-02-29T23:59:59-00:00", invoker); + addAndCompare("2012-04T", -31 * 24 * 60 * 60 , "2012-03-01T00:00:00-00:00", invoker); + addAndCompare("2012-04T", -1, "2012-03-31T23:59:59-00:00", invoker); + addAndCompare("2012-04T", 0, "2012-04-01T00:00:00-00:00", invoker); + addAndCompare("2012-04T", 1, "2012-04-01T00:00:01-00:00", invoker); + addAndCompare("2012-04T", 30 * 24 * 60 * 60 - 1, "2012-04-30T23:59:59-00:00", invoker); + addAndCompare("2012-04T", 30 * 24 * 60 * 60 , "2012-05-01T00:00:00-00:00", invoker); + + + addAndCompare("2011-02-28", -1, "2011-02-27T23:59:59-00:00", invoker); + addAndCompare("2011-02-28", 0, "2011-02-28T00:00:00-00:00", invoker); + addAndCompare("2011-02-28", 1, "2011-02-28T00:00:01-00:00", invoker); + addAndCompare("2011-02-28", 24 * 60 * 60, "2011-03-01T00:00:00-00:00", invoker); + addAndCompare("2011-03-01", -24 * 60 * 60, "2011-02-28T00:00:00-00:00", invoker); + addAndCompare("2012-02-28", 24 * 60 * 60, "2012-02-29T00:00:00-00:00", invoker); + addAndCompare("2012-02-29", -24 * 60 * 60, "2012-02-28T00:00:00-00:00", invoker); + addAndCompare("2012-02-29", 24 * 60 * 60, "2012-03-01T00:00:00-00:00", invoker); + + addAndCompare("2012-10-04", -24 * 60 * 60 - 1, "2012-10-02T23:59:59-00:00", invoker); + addAndCompare("2012-10-04", -24 * 60 * 60 , "2012-10-03T00:00:00-00:00", invoker); + addAndCompare("2012-10-04", -1, "2012-10-03T23:59:59-00:00", invoker); + addAndCompare("2012-10-04", 1, "2012-10-04T00:00:01-00:00", invoker); + addAndCompare("2012-10-04", 48 * 60 * 60 - 1, "2012-10-05T23:59:59-00:00", invoker); + addAndCompare("2012-10-04", 48 * 60 * 60 , "2012-10-06T00:00:00-00:00", invoker); + + addAndCompareSecondWithFullPrecision("2011-01-31T12:03:23", -60 * 60, "2011-01-31T11:03:23", invoker); + addAndCompareSecondWithFullPrecision("2011-01-31T12:03:23", 0, "2011-01-31T12:03:23", invoker); + addAndCompareSecondWithFullPrecision("2011-01-31T12:03:23", 60 * 60, "2011-01-31T13:03:23", invoker); + addAndCompareSecondWithFullPrecision("2011-01-31T23:03:23", 57 * 60, "2011-02-01T00:00:23", invoker); + + addAndCompareSecondWithFullPrecision("2011-02-28T02:03:23", 25 * 60 * 60, "2011-03-01T03:03:23", invoker); + addAndCompareSecondWithFullPrecision("2012-02-28T02:03:23", 25 * 60 * 60, "2012-02-29T03:03:23", invoker); + + addAndCompareSecondWithFullPrecision("2011-03-01T02:35:23", -4 * 60 * 60, "2011-02-28T22:35:23", invoker); + addAndCompareSecondWithFullPrecision("2012-03-01T02:35:23", -4 * 60 * 60, "2012-02-29T22:35:23", invoker); + } + + @Test + public void testAdjustSecond() + { + TimestampArithmeticInvoker invoker = new TimestampArithmeticInvoker() { + public Timestamp invoke(Timestamp input, Number amount) { + return input.adjustSecond(amount.intValue()); + } + }; + // Adding more precise amounts does not extend the precision. + addAndCompare("2012T", -1, "2011T", invoker); + addAndCompare("2012T", 0, "2012T", invoker); + addAndCompare("2012T", 1, "2012T", invoker); + + addAndCompare("2012-04T", -31 * 24 * 60 * 60 - 1, "2012-02T", invoker); + addAndCompare("2012-04T", -31 * 24 * 60 * 60 , "2012-03T", invoker); + addAndCompare("2012-04T", -1, "2012-03T", invoker); + addAndCompare("2012-04T", 0, "2012-04T", invoker); + addAndCompare("2012-04T", 1, "2012-04T", invoker); + addAndCompare("2012-04T", 30 * 24 * 60 * 60 - 1, "2012-04T", invoker); + addAndCompare("2012-04T", 30 * 24 * 60 * 60 , "2012-05T", invoker); + + + addAndCompare("2011-02-28", -1, "2011-02-27", invoker); + addAndCompare("2011-02-28", 0, "2011-02-28", invoker); + addAndCompare("2011-02-28", 1, "2011-02-28", invoker); + addAndCompare("2011-02-28", 24 * 60 * 60, "2011-03-01", invoker); + addAndCompare("2011-03-01", -24 * 60 * 60, "2011-02-28", invoker); + addAndCompare("2012-02-28", 24 * 60 * 60, "2012-02-29", invoker); + addAndCompare("2012-02-29", -24 * 60 * 60, "2012-02-28", invoker); + addAndCompare("2012-02-29", 24 * 60 * 60, "2012-03-01", invoker); + + addAndCompare("2012-10-04", -24 * 60 * 60 - 1, "2012-10-02", invoker); + addAndCompare("2012-10-04", -24 * 60 * 60 , "2012-10-03", invoker); + addAndCompare("2012-10-04", -1, "2012-10-03", invoker); + addAndCompare("2012-10-04", 1, "2012-10-04", invoker); + addAndCompare("2012-10-04", 48 * 60 * 60 - 1, "2012-10-05", invoker); + addAndCompare("2012-10-04", 48 * 60 * 60 , "2012-10-06", invoker); + + addAndCompareSecondWithFullPrecision("2011-01-31T12:03:23", -60 * 60, "2011-01-31T11:03:23", invoker); + addAndCompareSecondWithFullPrecision("2011-01-31T12:03:23", 0, "2011-01-31T12:03:23", invoker); + addAndCompareSecondWithFullPrecision("2011-01-31T12:03:23", 60 * 60, "2011-01-31T13:03:23", invoker); + addAndCompareSecondWithFullPrecision("2011-01-31T23:03:23", 57 * 60, "2011-02-01T00:00:23", invoker); + + addAndCompareSecondWithFullPrecision("2011-02-28T02:03:23", 25 * 60 * 60, "2011-03-01T03:03:23", invoker); + addAndCompareSecondWithFullPrecision("2012-02-28T02:03:23", 25 * 60 * 60, "2012-02-29T03:03:23", invoker); + + addAndCompareSecondWithFullPrecision("2011-03-01T02:35:23", -4 * 60 * 60, "2011-02-28T22:35:23", invoker); + addAndCompareSecondWithFullPrecision("2012-03-01T02:35:23", -4 * 60 * 60, "2012-02-29T22:35:23", invoker); + } - addSecondWithsFrac("2011-02-28T02:03:23", 25 * 60 * 60, "2011-03-01T03:03:23"); - addSecondWithsFrac("2012-02-28T02:03:23", 25 * 60 * 60, "2012-02-29T03:03:23"); + @Test(expected = IllegalArgumentException.class) + public void testAddSecondOutsideMaxRange() + { + Timestamp ts1 = Timestamp.valueOf("9998T"); + ts1.addSecond(2*12*35*24*60*60); + } - addSecondWithsFrac("2011-03-01T02:35:23", -4 * 60 * 60, "2011-02-28T22:35:23"); - addSecondWithsFrac("2012-03-01T02:35:23", -4 * 60 * 60, "2012-02-29T22:35:23"); + @Test(expected = IllegalArgumentException.class) + public void testAddSecondOutsideMinRange() + { + Timestamp ts1 = Timestamp.valueOf("0001T"); + ts1.addSecond(-1); } - private void addMillisecond(String orig, long amount, String expected) + @Test(expected = IllegalArgumentException.class) + public void testAdjustSecondOutsideMaxRange() { - Timestamp ts1 = Timestamp.valueOf(orig); - Timestamp ts2 = ts1.addMillis(amount); - checkTimestamp(expected, ts2); + Timestamp ts1 = Timestamp.valueOf("9998T"); + ts1.adjustSecond(2*12*35*24*60*60); } - private void addMillisecond(String orig, long amount, String expected, String suffix) + @Test(expected = IllegalArgumentException.class) + public void testAdjustSecondOutsideMinRange() { - addMillisecond(orig + suffix, amount, expected + suffix); + Timestamp ts1 = Timestamp.valueOf("0001T"); + ts1.adjustSecond(-1); } - private void addMillisecondWithOffsets(String orig, long amount, String expected) { - addMillisecond(orig, amount, expected, "-00:00"); - addMillisecond(orig, amount, expected, "Z"); - addMillisecond(orig, amount, expected, "+01:23"); - addMillisecond(orig, amount, expected, "-01:23"); - addMillisecond(orig, amount, expected, "+23:59"); - addMillisecond(orig, amount, expected, "-23:59"); + //------------------------------------------------------------------------- + + private void addAndCompareMillisWithOffsets(String orig, long amount, String expected, TimestampArithmeticInvoker invoker) { + addAndCompare(orig, amount, expected, "-00:00", invoker); + addAndCompare(orig, amount, expected, "Z", invoker); + addAndCompare(orig, amount, expected, "+01:23", invoker); + addAndCompare(orig, amount, expected, "-01:23", invoker); + addAndCompare(orig, amount, expected, "+23:59", invoker); + addAndCompare(orig, amount, expected, "-23:59", invoker); } - private void addMillisecondWithFrac(String orig, long amount, String expected) + private void addAndCompareMillisWithFullPrecision(String orig, long amount, String expected, TimestampArithmeticInvoker invoker) { // If the timestamp has less than milliseconds precision, always expand its precision to milliseconds. - addMillisecondWithOffsets(orig, amount, expected + ".000"); - addMillisecondWithOffsets(orig + ".0", amount, expected + ".000"); - addMillisecondWithOffsets(orig + ".00", amount, expected + ".000"); - addMillisecondWithOffsets(orig + ".000", amount, expected +".000"); - addMillisecondWithOffsets(orig + ".456", amount, expected + ".456"); + addAndCompareMillisWithOffsets(orig, amount, expected + ".000", invoker); + addAndCompareMillisWithOffsets(orig + ".0", amount, expected + ".000", invoker); + addAndCompareMillisWithOffsets(orig + ".00", amount, expected + ".000", invoker); + addAndCompareMillisWithOffsets(orig + ".000", amount, expected +".000", invoker); + addAndCompareMillisWithOffsets(orig + ".456", amount, expected + ".456", invoker); + // If the Timestamp has greater than milliseconds precision, don't reduce the precision. + addAndCompareMillisWithOffsets(orig + ".0000", amount, expected + ".0000", invoker); + addAndCompareMillisWithOffsets(orig + ".45678", amount, expected +".45678", invoker); + } + + @Test + public void testAddMillis() + { + TimestampArithmeticInvoker invoker = new TimestampArithmeticInvoker() { + public Timestamp invoke(Timestamp input, Number amount) { + return input.addMillis(amount.longValue()); + } + }; + // Adding more precise amounts extends the precision. + addAndCompare("2012T", -1, "2011-12-31T23:59:59.999-00:00", invoker); + addAndCompare("2012T", 0, "2012-01-01T00:00:00.000-00:00", invoker); + addAndCompare("2012T", 1, "2012-01-01T00:00:00.001-00:00", invoker); + + addAndCompare("2012-04T", -31L * 24 * 60 * 60 * 1000 - 1, "2012-02-29T23:59:59.999-00:00", invoker); + addAndCompare("2012-04T", -31L * 24 * 60 * 60 * 1000 , "2012-03-01T00:00:00.000-00:00", invoker); + addAndCompare("2012-04T", -1, "2012-03-31T23:59:59.999-00:00", invoker); + addAndCompare("2012-04T", 0, "2012-04-01T00:00:00.000-00:00", invoker); + addAndCompare("2012-04T", 1, "2012-04-01T00:00:00.001-00:00", invoker); + addAndCompare("2012-04T", 30L * 24 * 60 * 60 * 1000 - 1, "2012-04-30T23:59:59.999-00:00", invoker); + addAndCompare("2012-04T", 30L * 24 * 60 * 60 * 1000 , "2012-05-01T00:00:00.000-00:00", invoker); + + + addAndCompare("2011-02-28", -1, "2011-02-27T23:59:59.999-00:00", invoker); + addAndCompare("2011-02-28", 0, "2011-02-28T00:00:00.000-00:00", invoker); + addAndCompare("2011-02-28", 1, "2011-02-28T00:00:00.001-00:00", invoker); + addAndCompare("2011-02-28", 24 * 60 * 60 * 1000, "2011-03-01T00:00:00.000-00:00", invoker); + addAndCompare("2011-03-01", -24 * 60 * 60 * 1000, "2011-02-28T00:00:00.000-00:00", invoker); + addAndCompare("2012-02-28", 24 * 60 * 60 * 1000, "2012-02-29T00:00:00.000-00:00", invoker); + addAndCompare("2012-02-29", -24 * 60 * 60 * 1000, "2012-02-28T00:00:00.000-00:00", invoker); + addAndCompare("2012-02-29", 24 * 60 * 60 * 1000, "2012-03-01T00:00:00.000-00:00", invoker); + + addAndCompare("2012-10-04", -24 * 60 * 60 * 1000 - 1, "2012-10-02T23:59:59.999-00:00", invoker); + addAndCompare("2012-10-04", -24 * 60 * 60 * 1000 , "2012-10-03T00:00:00.000-00:00", invoker); + addAndCompare("2012-10-04", -1, "2012-10-03T23:59:59.999-00:00", invoker); + addAndCompare("2012-10-04", 1, "2012-10-04T00:00:00.001-00:00", invoker); + addAndCompare("2012-10-04", 48 * 60 * 60 * 1000 - 1, "2012-10-05T23:59:59.999-00:00", invoker); + addAndCompare("2012-10-04", 48 * 60 * 60 * 1000 , "2012-10-06T00:00:00.000-00:00", invoker); + + addAndCompareMillisWithFullPrecision("2011-01-31T12:03:23", -60 * 60 * 1000, "2011-01-31T11:03:23", invoker); + addAndCompareMillisWithFullPrecision("2011-01-31T12:03:23", 0, "2011-01-31T12:03:23", invoker); + addAndCompareMillisWithFullPrecision("2011-01-31T12:03:23", 60 * 60 * 1000, "2011-01-31T13:03:23", invoker); + addAndCompareMillisWithFullPrecision("2011-01-31T23:03:23", 57 * 60 * 1000, "2011-02-01T00:00:23", invoker); + + addAndCompareMillisWithFullPrecision("2011-02-28T02:03:23", 25 * 60 * 60 * 1000, "2011-03-01T03:03:23", invoker); + addAndCompareMillisWithFullPrecision("2012-02-28T02:03:23", 25 * 60 * 60 * 1000, "2012-02-29T03:03:23", invoker); + + addAndCompareMillisWithFullPrecision("2011-03-01T02:35:23", -4 * 60 * 60 * 1000, "2011-02-28T22:35:23", invoker); + addAndCompareMillisWithFullPrecision("2012-03-01T02:35:23", -4 * 60 * 60 * 1000, "2012-02-29T22:35:23", invoker); + + addAndCompareMillisWithOffsets("2011-01-31T23:59:59.999", 1, "2011-02-01T00:00:00.000", invoker); + addAndCompareMillisWithOffsets("2011-02-01T00:00:00.000", -1, "2011-01-31T23:59:59.999", invoker); + addAndCompareMillisWithOffsets("2011-01-31T23:59:59.999123", 1, "2011-02-01T00:00:00.000123", invoker); + addAndCompareMillisWithOffsets("2011-02-01T00:00:00.000123", -1, "2011-01-31T23:59:59.999123", invoker); + + } + + private void adjustMillisecondWithFrac(String orig, long amount, String expected, TimestampArithmeticInvoker invoker) + { + // If the timestamp has less than milliseconds precision, maintain the original precision. + addAndCompareMillisWithOffsets(orig, amount, expected, invoker); + addAndCompareMillisWithOffsets(orig + ".0", amount, expected + ".0", invoker); + addAndCompareMillisWithOffsets(orig + ".00", amount, expected + ".00", invoker); + addAndCompareMillisWithOffsets(orig + ".000", amount, expected +".000", invoker); + addAndCompareMillisWithOffsets(orig + ".456", amount, expected + ".456", invoker); // If the Timestamp has greater than milliseconds precision, don't reduce the precision. - addMillisecondWithOffsets(orig + ".0000", amount, expected + ".0000"); - addMillisecondWithOffsets(orig + ".45678", amount, expected +".45678"); + addAndCompareMillisWithOffsets(orig + ".0000", amount, expected + ".0000", invoker); + addAndCompareMillisWithOffsets(orig + ".45678", amount, expected +".45678", invoker); } @Test - public void testAddMillisecond() + public void testAdjustMillis() { - addMillisecond("2012T", -1, "2011-12-31T23:59:59.999-00:00"); - addMillisecond("2012T", 0, "2012-01-01T00:00:00.000-00:00"); - addMillisecond("2012T", 1, "2012-01-01T00:00:00.001-00:00"); + TimestampArithmeticInvoker invoker = new TimestampArithmeticInvoker() { + public Timestamp invoke(Timestamp input, Number amount) { + return input.adjustMillis(amount.longValue()); + } + }; + // Adding more precise amounts extends the precision. + addAndCompare("2012T", -1, "2011T", invoker); + addAndCompare("2012T", 0, "2012T", invoker); + addAndCompare("2012T", 1, "2012T", invoker); - addMillisecond("2012-04T", -31L * 24 * 60 * 60 * 1000 - 1, "2012-02-29T23:59:59.999-00:00"); - addMillisecond("2012-04T", -31L * 24 * 60 * 60 * 1000 , "2012-03-01T00:00:00.000-00:00"); - addMillisecond("2012-04T", -1, "2012-03-31T23:59:59.999-00:00"); - addMillisecond("2012-04T", 0, "2012-04-01T00:00:00.000-00:00"); - addMillisecond("2012-04T", 1, "2012-04-01T00:00:00.001-00:00"); - addMillisecond("2012-04T", 30L * 24 * 60 * 60 * 1000 - 1, "2012-04-30T23:59:59.999-00:00"); - addMillisecond("2012-04T", 30L * 24 * 60 * 60 * 1000 , "2012-05-01T00:00:00.000-00:00"); + addAndCompare("2012-04T", -31L * 24 * 60 * 60 * 1000 - 1, "2012-02T", invoker); + addAndCompare("2012-04T", -31L * 24 * 60 * 60 * 1000 , "2012-03T", invoker); + addAndCompare("2012-04T", -1, "2012-03T", invoker); + addAndCompare("2012-04T", 0, "2012-04T", invoker); + addAndCompare("2012-04T", 1, "2012-04T", invoker); + addAndCompare("2012-04T", 30L * 24 * 60 * 60 * 1000 - 1, "2012-04T", invoker); + addAndCompare("2012-04T", 30L * 24 * 60 * 60 * 1000 , "2012-05T", invoker); - addMillisecond("2011-02-28", -1, "2011-02-27T23:59:59.999-00:00"); - addMillisecond("2011-02-28", 0, "2011-02-28T00:00:00.000-00:00"); - addMillisecond("2011-02-28", 1, "2011-02-28T00:00:00.001-00:00"); - addMillisecond("2011-02-28", 24 * 60 * 60 * 1000, "2011-03-01T00:00:00.000-00:00"); - addMillisecond("2011-03-01", -24 * 60 * 60 * 1000, "2011-02-28T00:00:00.000-00:00"); - addMillisecond("2012-02-28", 24 * 60 * 60 * 1000, "2012-02-29T00:00:00.000-00:00"); - addMillisecond("2012-02-29", -24 * 60 * 60 * 1000, "2012-02-28T00:00:00.000-00:00"); - addMillisecond("2012-02-29", 24 * 60 * 60 * 1000, "2012-03-01T00:00:00.000-00:00"); + addAndCompare("2011-02-28", -1, "2011-02-27", invoker); + addAndCompare("2011-02-28", 0, "2011-02-28", invoker); + addAndCompare("2011-02-28", 1, "2011-02-28", invoker); + addAndCompare("2011-02-28", 24 * 60 * 60 * 1000, "2011-03-01", invoker); + addAndCompare("2011-03-01", -24 * 60 * 60 * 1000, "2011-02-28", invoker); + addAndCompare("2012-02-28", 24 * 60 * 60 * 1000, "2012-02-29", invoker); + addAndCompare("2012-02-29", -24 * 60 * 60 * 1000, "2012-02-28", invoker); + addAndCompare("2012-02-29", 24 * 60 * 60 * 1000, "2012-03-01", invoker); - addMillisecond("2012-10-04", -24 * 60 * 60 * 1000 - 1, "2012-10-02T23:59:59.999-00:00"); - addMillisecond("2012-10-04", -24 * 60 * 60 * 1000 , "2012-10-03T00:00:00.000-00:00"); - addMillisecond("2012-10-04", -1, "2012-10-03T23:59:59.999-00:00"); - addMillisecond("2012-10-04", 1, "2012-10-04T00:00:00.001-00:00"); - addMillisecond("2012-10-04", 48 * 60 * 60 * 1000 - 1, "2012-10-05T23:59:59.999-00:00"); - addMillisecond("2012-10-04", 48 * 60 * 60 * 1000 , "2012-10-06T00:00:00.000-00:00"); + addAndCompare("2012-10-04", -24 * 60 * 60 * 1000 - 1, "2012-10-02", invoker); + addAndCompare("2012-10-04", -24 * 60 * 60 * 1000 , "2012-10-03", invoker); + addAndCompare("2012-10-04", -1, "2012-10-03", invoker); + addAndCompare("2012-10-04", 1, "2012-10-04", invoker); + addAndCompare("2012-10-04", 48 * 60 * 60 * 1000 - 1, "2012-10-05", invoker); + addAndCompare("2012-10-04", 48 * 60 * 60 * 1000 , "2012-10-06", invoker); - addMillisecondWithFrac("2011-01-31T12:03:23", -60 * 60 * 1000, "2011-01-31T11:03:23"); - addMillisecondWithFrac("2011-01-31T12:03:23", 0, "2011-01-31T12:03:23"); - addMillisecondWithFrac("2011-01-31T12:03:23", 60 * 60 * 1000, "2011-01-31T13:03:23"); - addMillisecondWithFrac("2011-01-31T23:03:23", 57 * 60 * 1000, "2011-02-01T00:00:23"); + adjustMillisecondWithFrac("2011-01-31T12:03:23", -60 * 60 * 1000, "2011-01-31T11:03:23", invoker); + adjustMillisecondWithFrac("2011-01-31T12:03:23", 0, "2011-01-31T12:03:23", invoker); + adjustMillisecondWithFrac("2011-01-31T12:03:23", 60 * 60 * 1000, "2011-01-31T13:03:23", invoker); + adjustMillisecondWithFrac("2011-01-31T23:03:23", 57 * 60 * 1000, "2011-02-01T00:00:23", invoker); - addMillisecondWithFrac("2011-02-28T02:03:23", 25 * 60 * 60 * 1000, "2011-03-01T03:03:23"); - addMillisecondWithFrac("2012-02-28T02:03:23", 25 * 60 * 60 * 1000, "2012-02-29T03:03:23"); + adjustMillisecondWithFrac("2011-02-28T02:03:23", 25 * 60 * 60 * 1000, "2011-03-01T03:03:23", invoker); + adjustMillisecondWithFrac("2012-02-28T02:03:23", 25 * 60 * 60 * 1000, "2012-02-29T03:03:23", invoker); + + adjustMillisecondWithFrac("2011-03-01T02:35:23", -4 * 60 * 60 * 1000, "2011-02-28T22:35:23", invoker); + adjustMillisecondWithFrac("2012-03-01T02:35:23", -4 * 60 * 60 * 1000, "2012-02-29T22:35:23", invoker); + + addAndCompareMillisWithOffsets("2011-01-31T23:59:59.999", 1, "2011-02-01T00:00:00.000", invoker); + addAndCompareMillisWithOffsets("2011-02-01T00:00:00.000", -1, "2011-01-31T23:59:59.999", invoker); + addAndCompareMillisWithOffsets("2011-01-31T23:59:59.999123", 1, "2011-02-01T00:00:00.000123", invoker); + addAndCompareMillisWithOffsets("2011-02-01T00:00:00.000123", -1, "2011-01-31T23:59:59.999123", invoker); + + // Additional fractional precision is truncated. + addAndCompareMillisWithOffsets("2011-01-31T23:59:59.99", 9, "2011-01-31T23:59:59.99", invoker); + addAndCompareMillisWithOffsets("2011-02-01T00:00:00.00", -1, "2011-01-31T23:59:59.99", invoker); + + } + + @Test(expected = IllegalArgumentException.class) + public void testAddMillisOutsideMaxRange() + { + Timestamp ts1 = Timestamp.valueOf("9998T"); + ts1.addMillis(2*12*35*24*60*60*1000L); + } - addMillisecondWithFrac("2011-03-01T02:35:23", -4 * 60 * 60 * 1000, "2011-02-28T22:35:23"); - addMillisecondWithFrac("2012-03-01T02:35:23", -4 * 60 * 60 * 1000, "2012-02-29T22:35:23"); + @Test(expected = IllegalArgumentException.class) + public void testAddMillisOutsideMinRange() + { + Timestamp ts1 = Timestamp.valueOf("0001T"); + ts1.addMillis(-1); + } - addMillisecondWithOffsets("2011-01-31T23:59:59.999", 1, "2011-02-01T00:00:00.000"); - addMillisecondWithOffsets("2011-02-01T00:00:00.000", -1, "2011-01-31T23:59:59.999"); - addMillisecondWithOffsets("2011-01-31T23:59:59.999123", 1, "2011-02-01T00:00:00.000123"); - addMillisecondWithOffsets("2011-02-01T00:00:00.000123", -1, "2011-01-31T23:59:59.999123"); + @Test(expected = IllegalArgumentException.class) + public void testAdjustMillisOutsideMaxRange() + { + Timestamp ts1 = Timestamp.valueOf("9998T"); + ts1.adjustMillis(2*12*35*24*60*60*1000L); + } + @Test(expected = IllegalArgumentException.class) + public void testAdjustMillisOutsideMinRange() + { + Timestamp ts1 = Timestamp.valueOf("0001T"); + ts1.adjustMillis(-1); } @Test @@ -2331,7 +2640,7 @@ public void testCustomCalendarLeapYearAfterWithLocalOffset() { } @Test - public void testCustomCalendarLeapYearWithChainedArithmetic() { + public void testCustomCalendarLeapYearWithChainedAdd() { // Create a purely Julian calendar, where leap years were every four years. GregorianCalendar julianCalendar = new GregorianCalendar(TimeZone.getTimeZone("UTC")); julianCalendar.setGregorianChange(new Date(Long.MAX_VALUE)); @@ -2344,18 +2653,30 @@ public void testCustomCalendarLeapYearWithChainedArithmetic() { assertEquals("1800-02-29T00:00:00.000Z", timestamp2.toString()); } - @Test(expected = IllegalArgumentException.class) - public void testAddSecondOutsideMaxRange() - { - Timestamp ts1 = Timestamp.valueOf("9998T"); - ts1.addSecond(2*12*35*24*60*60); + @Test + public void testCustomCalendarLeapYearWithChainedAdjust() { + // Create a purely Julian calendar, where leap years were every four years. + GregorianCalendar julianCalendar = new GregorianCalendar(TimeZone.getTimeZone("UTC")); + julianCalendar.setGregorianChange(new Date(Long.MAX_VALUE)); + julianCalendar.clear(); + // 1800 is not a leap year under the Gregorian calendar, but it is under the Julian calendar. + julianCalendar.set(1799, Calendar.JANUARY, 27, 22, 58, 58); + julianCalendar.set(Calendar.MILLISECOND, 999); + Timestamp timestamp1 = Timestamp.forCalendar(julianCalendar); + Timestamp timestamp2 = timestamp1.adjustYear(1).adjustMonth(1).adjustDay(1).adjustHour(1).adjustMinute(1).adjustSecond(1).adjustMillis(1); + assertEquals("1800-02-29T00:00:00.000Z", timestamp2.toString()); } - @Test(expected = IllegalArgumentException.class) - public void testAddSecondOutsideMinRange() - { - Timestamp ts1 = Timestamp.valueOf("0001T"); - ts1.addSecond(-1); + @Test + public void testCustomCalendarLeapYearWithChainedAdjustLessPrecise() { + // Create a purely Julian calendar, where leap years were every four years. + GregorianCalendar julianCalendar = new GregorianCalendar(TimeZone.getTimeZone("UTC")); + julianCalendar.setGregorianChange(new Date(Long.MAX_VALUE)); + julianCalendar.clear(); + julianCalendar.set(1799, Calendar.JANUARY, 28); + Timestamp timestamp1 = Timestamp.forCalendar(julianCalendar); + Timestamp timestamp2 = timestamp1.adjustYear(1).adjustMonth(1).adjustDay(1).adjustHour(1).adjustMinute(1).adjustSecond(1).adjustMillis(1); + assertEquals("1800-02-29", timestamp2.toString()); } @Test From 10d467a6a4086819900c9aa636e331ff6fb227db Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Wed, 26 Dec 2018 22:14:21 +0000 Subject: [PATCH 041/490] Reworks the local symbol table append reading implementation for better performance. (#201) --- .../ion/impl/BaseSymbolTableWrapper.java | 124 -------- .../amazon/ion/impl/LocalSymbolTable.java | 16 +- .../impl/LocalSymbolTableImportAdapter.java | 115 ------- .../ion/impl/LocalSymbolTableImports.java | 70 +---- .../amazon/ion/impl/lite/IonDatagramLite.java | 17 +- .../ion/impl/BaseSymbolTableWrapperTest.java | 134 --------- .../LocalSymbolTableImportAdapterTest.java | 280 ------------------ .../amazon/ion/impl/SymbolTableTest.java | 75 +++-- 8 files changed, 85 insertions(+), 746 deletions(-) delete mode 100644 src/software/amazon/ion/impl/BaseSymbolTableWrapper.java delete mode 100644 src/software/amazon/ion/impl/LocalSymbolTableImportAdapter.java delete mode 100644 test/software/amazon/ion/impl/BaseSymbolTableWrapperTest.java delete mode 100644 test/software/amazon/ion/impl/LocalSymbolTableImportAdapterTest.java diff --git a/src/software/amazon/ion/impl/BaseSymbolTableWrapper.java b/src/software/amazon/ion/impl/BaseSymbolTableWrapper.java deleted file mode 100644 index 7e16d6e628..0000000000 --- a/src/software/amazon/ion/impl/BaseSymbolTableWrapper.java +++ /dev/null @@ -1,124 +0,0 @@ -package software.amazon.ion.impl; - -import java.io.IOException; -import java.util.Iterator; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; - -/** - * Base class for {@link SymbolTable} wrapper. Subclasses can add and/or override methods as needed - *

    - * Delegates all {@link SymbolTable} methods to internal {@link SymbolTable} instance - *

    - */ -abstract class BaseSymbolTableWrapper implements SymbolTable -{ - private final SymbolTable delegate; - - protected BaseSymbolTableWrapper(SymbolTable delegate) - { - this.delegate = delegate; - } - - protected final SymbolTable getDelegate() - { - return delegate; - } - - public String getName() - { - return delegate.getName(); - } - - public int getVersion() - { - return delegate.getVersion(); - } - - public boolean isLocalTable() - { - return delegate.isLocalTable(); - } - - public boolean isSharedTable() - { - return delegate.isSharedTable(); - } - - public boolean isSubstitute() - { - return delegate.isSubstitute(); - } - - public boolean isSystemTable() - { - return delegate.isSystemTable(); - } - - public boolean isReadOnly() - { - return delegate.isReadOnly(); - } - - public void makeReadOnly() - { - delegate.makeReadOnly(); - } - - public SymbolTable getSystemSymbolTable() - { - return delegate.getSystemSymbolTable(); - } - - public String getIonVersionId() - { - return delegate.getIonVersionId(); - } - - public SymbolTable[] getImportedTables() - { - return delegate.getImportedTables(); - } - - public int getImportedMaxId() - { - return delegate.getImportedMaxId(); - } - - public int getMaxId() - { - return delegate.getMaxId(); - } - - public SymbolToken intern(String text) - { - return delegate.intern(text); - } - - public SymbolToken find(String text) - { - return delegate.find(text); - } - - public int findSymbol(String name) - { - return delegate.findSymbol(name); - } - - public String findKnownSymbol(int id) - { - return delegate.findKnownSymbol(id); - } - - public Iterator iterateDeclaredSymbolNames() - { - return delegate.iterateDeclaredSymbolNames(); - } - - public void writeTo(IonWriter writer) throws IOException - { - delegate.writeTo(writer); - } -} - diff --git a/src/software/amazon/ion/impl/LocalSymbolTable.java b/src/software/amazon/ion/impl/LocalSymbolTable.java index 0367184683..d221661400 100644 --- a/src/software/amazon/ion/impl/LocalSymbolTable.java +++ b/src/software/amazon/ion/impl/LocalSymbolTable.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -219,6 +220,10 @@ protected static LocalSymbolTableImports readLocalSymbolTable(IonReader reader, List importsList = new ArrayList(); importsList.add(reader.getSymbolTable().getSystemSymbolTable()); + // Isolate the newly-declared symbols because they must be added after any symbols declared + // by the previous symbol table if this is an appending local symbol table, but the 'imports' and + // 'symbols' declarations can occur in any order. + List newSymbols = new ArrayList(); IonType fieldType; boolean foundImportList = false; @@ -267,7 +272,7 @@ protected static LocalSymbolTableImports readLocalSymbolTable(IonReader reader, text = null; } - symbolsListOut.add(text); + newSymbols.add(text); } reader.stepOut(); } @@ -288,7 +293,12 @@ else if (fieldType == IonType.SYMBOL) // trying to import the current table if(reader.getSymbolTable().isLocalTable() && ION_SYMBOL_TABLE.equals(reader.stringValue())) { - importsList.add(reader.getSymbolTable()); + SymbolTable currentSymbolTable = reader.getSymbolTable(); + importsList.addAll(Arrays.asList(currentSymbolTable.getImportedTables())); + Iterator currentSymbols = currentSymbolTable.iterateDeclaredSymbolNames(); + while (currentSymbols.hasNext()) { + symbolsListOut.add(currentSymbols.next()); + } } } break; @@ -302,7 +312,7 @@ else if (fieldType == IonType.SYMBOL) } reader.stepOut(); - + symbolsListOut.addAll(newSymbols); return new LocalSymbolTableImports(importsList); } diff --git a/src/software/amazon/ion/impl/LocalSymbolTableImportAdapter.java b/src/software/amazon/ion/impl/LocalSymbolTableImportAdapter.java deleted file mode 100644 index 1d4bb82db8..0000000000 --- a/src/software/amazon/ion/impl/LocalSymbolTableImportAdapter.java +++ /dev/null @@ -1,115 +0,0 @@ -package software.amazon.ion.impl; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import software.amazon.ion.ReadOnlyValueException; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; - -/** - * Adapter for {@link LocalSymbolTable} to make it importable by ignoring any potentially imported - * system table. - *

    - * Important: All adapted instances are read only - *

    - */ -final class LocalSymbolTableImportAdapter - extends BaseSymbolTableWrapper -{ - - private final int systemSymbolCount; - - static LocalSymbolTableImportAdapter of(final LocalSymbolTable delegate) - { - /* - * There is a hard boundary between the data regions from the imported and derived table. Example: - * - * LST A that contains symbol A1, LST B is created importing LST A and after B is created symbol A2 is added - * to LST A. LST B should have symbol A1, through the import, but it should not have symbol A2 as it was included - * after LST B creation - * - * To maintain this boundary we make a read only copy for mutable LSTs. Copying is not necessary for read only LSTs - * as no extra items can be added into them - */ - final LocalSymbolTable readOnlyDelegate; - if(delegate.isReadOnly()) - { - readOnlyDelegate = delegate; - } - else { - readOnlyDelegate = delegate.makeCopy(); - readOnlyDelegate.makeReadOnly(); - } - - return new LocalSymbolTableImportAdapter(readOnlyDelegate); - } - - private LocalSymbolTableImportAdapter(final LocalSymbolTable delegate) - { - super(delegate); - - systemSymbolCount = countSystemSymbols(getDelegate()); - } - - @Override - public SymbolTable getSystemSymbolTable() - { - return null; - } - - @Override - public int getImportedMaxId() - { - return getDelegate().getImportedMaxId() - systemSymbolCount; - } - - @Override - public int getMaxId() - { - return getDelegate().getMaxId() - systemSymbolCount; - } - - @Override - public SymbolToken find(String text) - { - SymbolToken symbolToken = getDelegate().find(text); - if(symbolToken != null) - { - symbolToken = new SymbolTokenImpl(symbolToken.getText(), symbolToken.getSid() - systemSymbolCount); - } - - return symbolToken; - } - - @Override - public int findSymbol(String name) - { - int sid = getDelegate().findSymbol(name); - if(sid != UNKNOWN_SYMBOL_ID){ - sid = sid - systemSymbolCount; - } - - return sid; - } - - @Override - public String findKnownSymbol(int id) - { - return getDelegate().findKnownSymbol(id + systemSymbolCount); - } - - private int countSystemSymbols(SymbolTable symbolTable) - { - SymbolTable systemSymbolTable = symbolTable.getSystemSymbolTable(); - - if(systemSymbolTable != null) - { - return systemSymbolTable.getMaxId(); - } - - return 0; - } -} diff --git a/src/software/amazon/ion/impl/LocalSymbolTableImports.java b/src/software/amazon/ion/impl/LocalSymbolTableImports.java index fbb62b944c..ca78b9c322 100644 --- a/src/software/amazon/ion/impl/LocalSymbolTableImports.java +++ b/src/software/amazon/ion/impl/LocalSymbolTableImports.java @@ -74,49 +74,14 @@ final class LocalSymbolTableImports */ LocalSymbolTableImports(List importTables) { - validateImports(importTables); - int importTablesSize = importTables.size(); - // detects and adapts local tables so they are importable - myImports = new SymbolTable[importTablesSize]; - for(int i = 0; i < importTables.size(); i++) - { - SymbolTable symbolTable = importTables.get(i); - if(symbolTable.isLocalTable()) - { - myImports[i] = LocalSymbolTableImportAdapter.of((LocalSymbolTable) symbolTable); - } - else - { - myImports[i] = symbolTable; - } - } + myImports = importTables.toArray(new SymbolTable[importTablesSize]); myBaseSids = new int[importTablesSize]; myMaxId = prepBaseSids(myBaseSids, myImports); } - /** - * Validates the import list to ensure that if there is a {@link LocalSymbolTable} in it then it's a single import - * apart from the system table - */ - private void validateImports(final List importTables) - { - int sizeWithoutSystemTables = importTables.size(); - int numberOfLocalTables = 0; - - for(SymbolTable table : importTables) - { - if(table.isLocalTable()) numberOfLocalTables++; - if(table.isSystemTable()) sizeWithoutSystemTables--; - } - - if(numberOfLocalTables > 0 && sizeWithoutSystemTables != 1){ - throw new IllegalArgumentException("when importing LocalSymbolTables it needs to be the only import"); - } - } - /** * @param defaultSystemSymtab * the default system symtab, which will be used if the first @@ -140,36 +105,17 @@ private void validateImports(final List importTables) if (imports != null && imports.length > 0) { - final int offset; - final SymbolTable systemTable; if (imports[0].isSystemTable()) { - offset = 0; - systemTable = imports[0]; + // copy imports as-is + myImports = imports.clone(); } else { - offset = 1; - systemTable = defaultSystemSymtab; - } - - myImports = new SymbolTable[imports.length + offset]; - myImports[0] = systemTable; - - // no need to consider the offset here as it only comes in play when there is a system table - validateImports(Arrays.asList(imports)); - - for(int i = 1 - offset; i < imports.length; i++) - { - SymbolTable symbolTable = imports[i]; - if(symbolTable instanceof LocalSymbolTable) - { - myImports[i + offset] = LocalSymbolTableImportAdapter.of((LocalSymbolTable) symbolTable); - } - else - { - myImports[i + offset] = symbolTable; - } + // use defaultSystemSymtab and append imports + myImports = new SymbolTable[imports.length + 1]; + myImports[0] = defaultSystemSymtab; + System.arraycopy(imports, 0, myImports, 1, imports.length); } } else @@ -205,7 +151,7 @@ private static int prepBaseSids(int[] baseSids, SymbolTable[] imports) { SymbolTable importedTable = imports[i]; - if (importedTable.isSystemTable()) + if (importedTable.isLocalTable() || importedTable.isSystemTable()) { String message = "only non-system shared tables can be imported"; throw new IllegalArgumentException(message); diff --git a/src/software/amazon/ion/impl/lite/IonDatagramLite.java b/src/software/amazon/ion/impl/lite/IonDatagramLite.java index d8158b97a9..cdc6c62ebd 100644 --- a/src/software/amazon/ion/impl/lite/IonDatagramLite.java +++ b/src/software/amazon/ion/impl/lite/IonDatagramLite.java @@ -410,11 +410,18 @@ public IonType getType() @Override public final void writeTo(IonWriter writer) { - try - { - writer.writeSymbol(SystemSymbols.ION_1_0); // TODO amzn/ion-java#8 ??? - } catch (IOException ioe) { - throw new IonException(ioe); + if (writer.getSymbolTable().isSystemTable()) { + // If the writer was configured with a non-default symbol table, writing an IVM here will + // reset that symbol table. If the datagram contains any symbols with unknown text that + // refer to slots in shared symbol table imports declared by the discarded table, an + // error will be raised unnecessarily. To avoid that, only write an IVM when the writer's + // symbol table is already the system symbol table. + // TODO evaluate whether an IVM should ever be written here. amzn/ion-java#200 + try { + writer.writeSymbol(SystemSymbols.ION_1_0); + } catch (IOException ioe) { + throw new IonException(ioe); + } } for (IonValue iv : this) { iv.writeTo(writer); diff --git a/test/software/amazon/ion/impl/BaseSymbolTableWrapperTest.java b/test/software/amazon/ion/impl/BaseSymbolTableWrapperTest.java deleted file mode 100644 index 0d2fba459b..0000000000 --- a/test/software/amazon/ion/impl/BaseSymbolTableWrapperTest.java +++ /dev/null @@ -1,134 +0,0 @@ -package software.amazon.ion.impl; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import org.junit.Test; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.SymbolTable; - -/** - * Parent test class for classes that extend {@link BaseSymbolTableWrapper}. Provides simple equality tests for non void - * methods with no arguments by comparing delegate and subject returns. Other methods and overridden methods need to be - * handled by the subclasses - */ -public abstract class BaseSymbolTableWrapperTest extends IonTestCase -{ - protected abstract BaseSymbolTableWrapper getSubject(); - - @Test - public void testGetName() - { - assertEquals(getDelegate().getName(), getSubject().getName()); - } - - @Test - public void testGetVersion() - { - assertEquals(getDelegate().getVersion(), getSubject().getVersion()); - } - - @Test - public void testIsLocalTable() - { - assertEquals(getDelegate().isLocalTable(), getSubject().isLocalTable()); - } - - @Test - public void testIsSharedTable() - { - assertEquals(getDelegate().isSharedTable(), getSubject().isSharedTable()); - } - - @Test - public void testIsSubstitute() - { - assertEquals(getDelegate().isSubstitute(), getSubject().isSubstitute()); - } - - @Test - public void testIsSystemTable() - { - assertEquals(getDelegate().isSystemTable(), getSubject().isSystemTable()); - } - - @Test - public void testIsReadOnly() - { - assertEquals(getDelegate().isReadOnly(), getSubject().isReadOnly()); - } - - @Test - public abstract void testMakeReadOnly(); - - @Test - public void testGetSystemSymbolTable() - { - assertEquals(getDelegate().getSystemSymbolTable(), getSubject().getSystemSymbolTable()); - } - - @Test - public void testGetIonVersionId() - { - assertEquals(getDelegate().getIonVersionId(), getSubject().getIonVersionId()); - } - - @Test - public void testGetImportedTables() - { - assertEquals(getDelegate().getImportedTables(), getSubject().getImportedTables()); - } - - @Test - public void testGetImportedMaxId() - { - assertEquals(getDelegate().getImportedMaxId(), getSubject().getImportedMaxId()); - } - - @Test - public void testGetMaxId() - { - assertEquals(getDelegate().getMaxId(), getSubject().getMaxId()); - } - - @Test - public abstract void testIntern(); - - @Test - public abstract void testFind(); - - @Test - public abstract void testFindSymbol(); - - @Test - public abstract void testFindKnownSymbol(); - - @Test - public void testIterateDeclaredSymbolNames() - { - Iterator expected = getDelegate().iterateDeclaredSymbolNames(); - Iterator actual = getSubject().iterateDeclaredSymbolNames(); - - assertEquals(collect(expected), collect(actual)); - } - - private List collect(Iterator iterator) { - List list = new ArrayList(); - - for(T t = null; iterator.hasNext(); t = iterator.next()) - { - list.add(t); - } - - return list; - } - - @Test - public abstract void testWriteTo() throws IOException; - - private SymbolTable getDelegate() - { - return getSubject().getDelegate(); - } -} diff --git a/test/software/amazon/ion/impl/LocalSymbolTableImportAdapterTest.java b/test/software/amazon/ion/impl/LocalSymbolTableImportAdapterTest.java deleted file mode 100644 index b2dd6fce4f..0000000000 --- a/test/software/amazon/ion/impl/LocalSymbolTableImportAdapterTest.java +++ /dev/null @@ -1,280 +0,0 @@ -package software.amazon.ion.impl; - -import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import org.junit.Assert; -import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonWriter; -import software.amazon.ion.ReadOnlyValueException; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; - -public class LocalSymbolTableImportAdapterTest extends BaseSymbolTableWrapperTest -{ - @Test - public void testIsReadOnlyByDefault() - { - LocalSymbolTable lst = lstBuilder().build(); - assertEquals(false, lst.isReadOnly()); - - LocalSymbolTableImportAdapter subject = LocalSymbolTableImportAdapter.of(lst); - - assertEquals(true, subject.isReadOnly()); - } - - @Override - public void testGetSystemSymbolTable() - { - LocalSymbolTableImportAdapter subject = LocalSymbolTableImportAdapter.of(lstBuilder().build()); - - assertEquals(null, subject.getSystemSymbolTable()); - } - - @Test(expected = IllegalArgumentException.class) - public void testImportedTables() - { - LocalSymbolTable imported1 = lstBuilder().build(); - LocalSymbolTable imported2 = lstBuilder().build(); - - lstBuilder().setImportedTables(imported1, imported2).build(); - } - - @Test - public void testMaxIdNoImports() - { - LocalSymbolTable delegate = lstBuilder() - .setSymbols("one", "two") - .build(); - - LocalSymbolTableImportAdapter subject = LocalSymbolTableImportAdapter.of(delegate); - - assertEquals(2, subject.getMaxId()); - } - - @Override - public void testGetMaxId() - { - SymbolTable imported = lstBuilder().setSymbols("1", "2", "3").build(); - - LocalSymbolTable delegate = lstBuilder() - .setSymbols("one", "two") - .setImportedTables(imported) - .build(); - - LocalSymbolTableImportAdapter subject = LocalSymbolTableImportAdapter.of(delegate); - - assertEquals(5, subject.getMaxId()); - } - - @Test - public void testImportedMaxIdNoImports() - { - LocalSymbolTable delegate = lstBuilder() - .setSymbols("one", "two") - .build(); - - LocalSymbolTableImportAdapter subject = LocalSymbolTableImportAdapter.of(delegate); - - assertEquals(0, subject.getImportedMaxId()); - } - - @Override - public void testGetImportedMaxId() - { - SymbolTable imported = lstBuilder().setSymbols("1", "2", "3").build(); - - LocalSymbolTable delegate = lstBuilder() - .setSymbols("one", "two") - .setImportedTables(imported) - .build(); - - LocalSymbolTableImportAdapter subject = LocalSymbolTableImportAdapter.of(delegate); - - assertEquals(3, subject.getImportedMaxId()); - } - - @Test(expected = ReadOnlyValueException.class) - @Override - public void testIntern() - { - LocalSymbolTableImportAdapter subject = LocalSymbolTableImportAdapter.of(lstBuilder().build()); - - subject.intern(""); - } - - @Override - public void testFind() - { - SymbolTable imported = lstBuilder().setSymbols("onImport").build(); - - LocalSymbolTable delegate = lstBuilder() - .setImportedTables(imported) - .setSymbols("onDelegate") - .build(); - - LocalSymbolTableImportAdapter subject = LocalSymbolTableImportAdapter.of(delegate); - - SymbolToken onDelegateSymbol = subject.find("onDelegate"); - assertNotNull(onDelegateSymbol); - assertEquals("onDelegate", onDelegateSymbol.getText()); - assertEquals(2, onDelegateSymbol.getSid()); - - SymbolToken onImportSymbol = subject.find("onImport"); - assertNotNull(onImportSymbol); - assertEquals("onImport", onImportSymbol.getText()); - assertEquals(1, onImportSymbol.getSid()); - - SymbolToken unknown = subject.find("unknown"); - assertNull(unknown); - } - - @Override - protected BaseSymbolTableWrapper getSubject() - { - return LocalSymbolTableImportAdapter.of(lstBuilder().build()); - } - - @Override - public void testMakeReadOnly() - { - LocalSymbolTableImportAdapter subject = LocalSymbolTableImportAdapter.of(lstBuilder().build()); - subject.makeReadOnly(); - assertEquals(true, subject.isReadOnly()); - - try - { - subject.intern("asd"); - Assert.fail("could modify read only"); - } catch (ReadOnlyValueException ignored) - { - - } - } - - @Override - public void testFindSymbol() - { - SymbolTable imported = lstBuilder().setSymbols("onImport").build(); - - LocalSymbolTable delegate = lstBuilder() - .setImportedTables(imported) - .setSymbols("onDelegate") - .build(); - - LocalSymbolTableImportAdapter subject = LocalSymbolTableImportAdapter.of(delegate); - - int onDelegateSid = subject.findSymbol("onDelegate"); - assertEquals(2, onDelegateSid); - - int onImportSid = subject.findSymbol("onImport"); - assertEquals(1, onImportSid); - - int unknown = subject.findSymbol("unknown"); - assertEquals(UNKNOWN_SYMBOL_ID, unknown); - } - - @Override - public void testFindKnownSymbol() - { - SymbolTable imported = lstBuilder().setSymbols("onImport").build(); - - LocalSymbolTable delegate = lstBuilder() - .setImportedTables(imported) - .setSymbols("onDelegate") - .build(); - - LocalSymbolTableImportAdapter subject = LocalSymbolTableImportAdapter.of(delegate); - - String onDelegate = subject.findKnownSymbol(2); - assertEquals("onDelegate", onDelegate); - - String onImport = subject.findKnownSymbol(1); - assertEquals("onImport", onImport); - - String unknown = subject.findKnownSymbol(99); - assertNull(unknown); - } - - @Override - public void testWriteTo() throws IOException - { - LocalSymbolTable delegate = lstBuilder().setSymbols("mySymbol").build(); - LocalSymbolTableImportAdapter subject = LocalSymbolTableImportAdapter.of(delegate); - - StringBuilder out = new StringBuilder(); - - IonWriter ionWriter = system().newTextWriter(out); - subject.writeTo(ionWriter); - ionWriter.close(); - - assertEquals(out.toString(), "$ion_symbol_table::{symbols:[\"mySymbol\"]}"); - } - - @Test - public void testWriteToBinary() throws IOException - { - LocalSymbolTable delegate = lstBuilder().setSymbols("mySymbol").build(); - LocalSymbolTableImportAdapter subject = LocalSymbolTableImportAdapter.of(delegate); - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - - IonWriter ionWriter = system().newBinaryWriter(out); - subject.writeTo(ionWriter); - system().newInt(1).writeTo(ionWriter); - ionWriter.close(); - - IonDatagram datagram = loader().load(out.toByteArray()); - SymbolTable symbolTable = datagram.get(0).getSymbolTable(); - - assertEquals(10, symbolTable.getMaxId()); - assertEquals(10, symbolTable.findSymbol("mySymbol")); - } - - private LocalSymbolTableBuilder lstBuilder() - { - return new LocalSymbolTableBuilder(system()); - } - - private static class LocalSymbolTableBuilder - { - private final PrivateIonSystem system; - private String[] symbols = new String[0]; - private SymbolTable[] importedTables = new SymbolTable[0]; - - private LocalSymbolTableBuilder(PrivateIonSystem system) - { - this.system = system; - } - - public LocalSymbolTable build() - { - LocalSymbolTable lst = (LocalSymbolTable) LocalSymbolTable.DEFAULT_LST_FACTORY.newLocalSymtab( - system.getSystemSymbolTable(), - importedTables - ); - - for(String symbol : symbols) { - lst.intern(symbol); - } - - return lst; - } - - public LocalSymbolTableBuilder setSymbols(final String... symbols) - { - this.symbols = symbols; - - return this; - } - - public LocalSymbolTableBuilder setImportedTables(final SymbolTable... importedTables) - { - this.importedTables = importedTables; - - return this; - } - } -} diff --git a/test/software/amazon/ion/impl/SymbolTableTest.java b/test/software/amazon/ion/impl/SymbolTableTest.java index 32e08bdd8c..d4ea57f8d1 100644 --- a/test/software/amazon/ion/impl/SymbolTableTest.java +++ b/test/software/amazon/ion/impl/SymbolTableTest.java @@ -32,11 +32,14 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Set; import org.junit.Ignore; import org.junit.Test; +import software.amazon.ion.IonCatalog; import software.amazon.ion.IonDatagram; import software.amazon.ion.IonException; import software.amazon.ion.IonInt; @@ -56,6 +59,9 @@ import software.amazon.ion.SymbolToken; import software.amazon.ion.SystemSymbols; import software.amazon.ion.Timestamp; +import software.amazon.ion.system.IonBinaryWriterBuilder; +import software.amazon.ion.system.IonSystemBuilder; +import software.amazon.ion.system.IonTextWriterBuilder; import software.amazon.ion.system.SimpleCatalog; /** @@ -175,26 +181,6 @@ public void testLocalSymbolTableAppend() checkUnknownSymbol(33, symbolTable); } - @Test - public void testLocalSymbolTableAppendThroughDom() - { - SymbolTable base = system().newLocalSymbolTable(); - base.intern("beforeDerivedCreation"); - - SymbolTable derived = system().newLocalSymbolTable(base); - derived.intern("onDerived"); - - base.intern("afterDerivedCreation"); - - checkSymbol("beforeDerivedCreation", systemMaxId() + 1, derived); - checkSymbol("onDerived", systemMaxId() + 2, derived); - - // derived only see symbols that were added in base before derived creation - // so "afterDerivedCreation" is unknown to derived - checkUnknownSymbol("afterDerivedCreation", UNKNOWN_SYMBOL_ID, derived); - checkSymbol("afterDerivedCreation", systemMaxId() + 2, base); - } - @Test public void testLocalSymbolTableMultiAppend() { @@ -297,6 +283,50 @@ public void testLocalSymbolTableAppendImportBoundary() assertNull(appended.find("o1")); } + @Test + public void testMutateDomAfterLocalSymbolTableAppend() throws IOException { + String text = + LocalSymbolTablePrefix + + "{" + + " imports:[{name:\"foo\", version:1, max_id:1}], " + + " symbols:[\"s1\", \"s2\"]" + + "}\n" + + "$10\n" + // Symbol with unknown text from "foo" + "$11\n" + // s1 + LocalSymbolTablePrefix + + "{" + + " imports:" + ION_SYMBOL_TABLE + "," + + " symbols:[\"s3\"]" + + "}\n" + + "$12"; // s2 + IonDatagram datagram = loader().load(text); + datagram.add(1, system().newSymbol("abc")); + datagram.add(system().newSymbol(new SymbolTokenImpl(null, 13))); // s3 + datagram.add(system().newSymbol(new SymbolTokenImpl(null, 10))); // Symbol with unknown text from "foo" + ByteArrayOutputStream textOutput = new ByteArrayOutputStream(); + ByteArrayOutputStream binaryOutput = new ByteArrayOutputStream(); + List fooSymbols = Collections.singletonList("bar"); + SymbolTable fooTable = system().newSharedSymbolTable("foo", 1, fooSymbols.iterator()); + IonWriter textWriter = IonTextWriterBuilder.standard().withImports(fooTable).build(textOutput); + datagram.writeTo(textWriter); + textWriter.close(); + IonWriter binaryWriter = IonBinaryWriterBuilder.standard().withImports(fooTable).build(binaryOutput); + datagram.writeTo(binaryWriter); + binaryWriter.close(); + SimpleCatalog catalog = new SimpleCatalog(); + catalog.putTable(fooTable); + IonSystem system = IonSystemBuilder.standard().withCatalog(catalog).build(); + IonDatagram textRoundtrip = system.getLoader().load(textOutput.toByteArray()); + IonDatagram binaryRoundtrip = system.getLoader().load(binaryOutput.toByteArray()); + assertEquals(textRoundtrip, binaryRoundtrip); + assertEquals("bar", ((IonSymbol)textRoundtrip.get(0)).stringValue()); + assertEquals("abc", ((IonSymbol)textRoundtrip.get(1)).stringValue()); + assertEquals("s1", ((IonSymbol)textRoundtrip.get(2)).stringValue()); + assertEquals("s2", ((IonSymbol)textRoundtrip.get(3)).stringValue()); + assertEquals("s3", ((IonSymbol)textRoundtrip.get(4)).stringValue()); + assertEquals("bar", ((IonSymbol)textRoundtrip.get(0)).stringValue()); + } + @Test public void testSymtabsPrintLocalSymtabWithGaps() throws Exception @@ -491,7 +521,7 @@ public void testOverridingImportedSymbolId() } - @Test @Ignore + @Test public void testInjectingMaxIdIntoImport() // TODO implement { SymbolTable importedTable = registerImportedV1(); @@ -713,8 +743,7 @@ public void testRepeatedImport() checkSymbol("imported 2", 13, /* dupe */ true, symbolTable); } - // amzn/ion-java#46 - @Test @Ignore + @Test public void testDupLocalSymbolOnDatagram() throws Exception { final IonSystem ion1 = system(); final SymbolTable st = ion1.newSharedSymbolTable("foobar", 1, Arrays.asList("s1").iterator()); From 9dc90d41f43b23c8fa9382d9e696665b0054810b Mon Sep 17 00:00:00 2001 From: Fernando Raganhan Barbosa Date: Tue, 8 Jan 2019 20:05:40 -0800 Subject: [PATCH 042/490] Memleak gzip (#204) * Fixing memleak when auto detecting gzipped bytes Correctly closes the created IonReader in order to free the memory allocated outside the heap by GzipInputStream. For methods that return an iterator changed the signature to return a CloseableIterator which is closeable permiting users to close the iterator. This iterator also closes itself when there are no more values next to return. The gzip auto detect functionality should be removed from this package as it is an orthogonal concern to reading/writing Ion data. Right now our API has to close streams created by users to guaranteed that the internal wrapped Gzip stream is correctly closed. https://github.com/amzn/ion-java/issues/198 --- .../amazon/ion/CloseableIterator.java | 14 +++ src/software/amazon/ion/IonLoader.java | 24 +++- src/software/amazon/ion/IonSystem.java | 113 +++++++++++++++--- .../amazon/ion/impl/lite/IonLoaderLite.java | 22 +++- .../amazon/ion/impl/lite/IonSystemLite.java | 52 +++++--- .../amazon/ion/system/IonReaderBuilder.java | 19 ++- 6 files changed, 204 insertions(+), 40 deletions(-) create mode 100644 src/software/amazon/ion/CloseableIterator.java diff --git a/src/software/amazon/ion/CloseableIterator.java b/src/software/amazon/ion/CloseableIterator.java new file mode 100644 index 0000000000..fde07b8de2 --- /dev/null +++ b/src/software/amazon/ion/CloseableIterator.java @@ -0,0 +1,14 @@ +package software.amazon.ion; + +import java.io.Closeable; +import java.util.Iterator; + +/** + * Iterator that can close its source of data. + * + * @param the type of elements returned by this iterator. + */ +public interface CloseableIterator extends Iterator, Closeable +{ + // empty +} diff --git a/src/software/amazon/ion/IonLoader.java b/src/software/amazon/ion/IonLoader.java index 9b6e7819c5..ad2fa9e590 100644 --- a/src/software/amazon/ion/IonLoader.java +++ b/src/software/amazon/ion/IonLoader.java @@ -105,10 +105,16 @@ public IonDatagram load(Reader ionText) /** + *

    * Loads a block of Ion data into a single datagram, * detecting whether it's text or binary data. + *

    + * *

    - * This method will auto-detect and uncompress GZIPped Ion data. + * This method will auto-detect and decompress GZIPped Ion data. + * WARNING: this feature is deprecated and will + * be removed in subsequent releases. + *

    * * @param ionData may be either Ion binary data, or UTF-8 Ion text. * This method assumes ownership of the array and may modify it at @@ -119,6 +125,9 @@ public IonDatagram load(Reader ionText) * * @throws NullPointerException if ionData is null. * @throws IonException if there's a syntax error in the Ion content. + * + * @deprecated auto-detecting of and decompression GZIPped Ion data will + * be removed in subsequent releases. */ public IonDatagram load(byte[] ionData) throws IonException; @@ -129,11 +138,18 @@ public IonDatagram load(byte[] ionData) * detecting whether it's text or binary data. *

    * The specified stream remains open after this method returns. + *

    + * *

    - * This method will auto-detect and uncompress GZIPped Ion data. + * This method will auto-detect and decompress GZIPped Ion data. + * WARNING: this feature is deprecated and will be + * removed in subsequent releases. + *

    + * *

    * Because this library performs its own buffering, it's recommended that * you avoid adding additional buffering to the given stream. + *

    * * @param ionData the stream from which to read Ion data. * @@ -144,6 +160,10 @@ public IonDatagram load(byte[] ionData) * @throws IonException if there's a syntax error in the Ion content. * @throws IOException if reading from the specified input stream results * in an IOException. + * + * @deprecated auto-detecting of and decompression GZIPped Ion data will + * be removed in subsequent releases. Use a {@link java.util.zip.GZIPInputStream} + * to process GZIPped Ion data. */ public IonDatagram load(InputStream ionData) throws IonException, IOException; diff --git a/src/software/amazon/ion/IonSystem.java b/src/software/amazon/ion/IonSystem.java index 56b4849b95..0115ffc251 100644 --- a/src/software/amazon/ion/IonSystem.java +++ b/src/software/amazon/ion/IonSystem.java @@ -204,66 +204,99 @@ public SymbolTable newSharedSymbolTable(String name, /** + *

    * Creates an iterator over a stream of Ion text data. * Values returned by the iterator have no container. + *

    + * *

    * The iterator will automatically consume Ion system IDs and local symbol * tables; they will not be returned by the iterator. + *

    + * *

    * If the input source throws an {@link IOException} during iteration, it * will be wrapped in an {@link IonException}. See documentation there for * tips on how to recover the cause. + *

    + * *

    * This method is suitable for use over unbounded streams with a reasonable * schema. + *

    + * *

    * Applications should generally use {@link #iterate(InputStream)} * whenever possible, since this library has much faster UTF-8 decoding * than the Java IO framework. + *

    + * *

    * Because this library performs its own buffering, it's recommended that * you avoid adding additional buffering to the given stream. + *

    * - * @param ionText a stream of Ion text data. The caller is responsible for + * @param ionText a stream of Ion text data. The caller is responsible for * closing the Reader after iteration is complete. * - * @return a new iterator instance. + * @return a new iterator instance. Closing this iterator + * will close ionText Reader. The iterator automatically closes when + * there are no more values to return. * * @throws NullPointerException if ionText is null. * @throws IonException if the source throws {@link IOException}. */ - public Iterator iterate(Reader ionText); - + public CloseableIterator iterate(Reader ionText); /** + *

    * Creates an iterator over a stream of Ion data, * detecting whether it's text or binary data. * Values returned by the iterator have no container. + *

    + * *

    * The iterator will automatically consume Ion system IDs and local symbol * tables; they will not be returned by the iterator. + *

    + * *

    * If the input source throws an {@link IOException} during iteration, it * will be wrapped in an {@link IonException}. See documentation there for * tips on how to recover the cause. + *

    + * *

    * This method is suitable for use over unbounded streams with a reasonable * schema. + *

    + * *

    - * This method will auto-detect and uncompress GZIPped Ion data. + * This method will auto-detect and decompress GZIPped Ion data. WARNING: + * this feature is deprecated and will be removed in subsequent releases. + *

    + * *

    * Because this library performs its own buffering, it's recommended that * you avoid adding additional buffering to the given stream. + *

    * * @param ionData a stream of Ion data. The caller is responsible for * closing the InputStream after iteration is complete. * - * @return a new iterator instance. + * @return a new CloseableIterator instance. Closing this iterator + * will close ionData InputStream. The iterator automatically closes when + * there are no more values to return. WARNING: This instance + * must be closed by the client to avoid memory leaks when dealing with + * GZIPped Ion data. * * @throws NullPointerException if ionData is null. * @throws IonException if the source throws {@link IOException}. + * + * @deprecated auto-detecting of and decompression GZIPped Ion data + * will be removed in subsequent releases. */ - public Iterator iterate(InputStream ionData); + public CloseableIterator iterate(InputStream ionData); /** @@ -275,32 +308,47 @@ public SymbolTable newSharedSymbolTable(String name, * * @param ionText must not be null. * - * @return a new iterator instance. + * @return a new iterator instance. Closing this iterator will + * close the underlying {@link IonReader} created to read ionText. + * The iterator automatically closes when there are no more values + * to return. * * @throws NullPointerException if ionText is null. */ - public Iterator iterate(String ionText); + public CloseableIterator iterate(String ionText); /** + *

    * Creates an iterator over Ion data. * Values returned by the iterator have no container. + *

    *

    * The iterator will automatically consume Ion system IDs and local symbol * tables; they will not be returned by the iterator. + *

    *

    - * This method will auto-detect and uncompress GZIPped Ion data. + * This method will auto-detect and decompress GZIPped Ion data. WARNING: + * this feature is deprecated and will be removed in subsequent releases. + *

    * * @param ionData may be either Ion binary data or (UTF-8) Ion text, or * GZIPped Ion data. * This method assumes ownership of the array and may modify it at * will. * - * @return a new iterator instance. + * @return a new CloseableIterator instance. Closing this iterator will + * close the underlying {@link IonReader} created to read ionData. The iterator + * automatically closes when there are no more values to return. + * WARNING: This instance must be closed by the client + * to avoid memory leaks when dealing with GZIPped Ion data. * * @throws NullPointerException if ionData is null. + * + * @deprecated auto-detecting of and decompression GZIPped Ion data + * will be removed in subsequent releases. */ - public Iterator iterate(byte[] ionData); + public CloseableIterator iterate(byte[] ionData); /** @@ -320,9 +368,14 @@ public SymbolTable newSharedSymbolTable(String name, /** + *

    * Extracts a single value from Ion text or binary data. + *

    + * *

    - * This method will auto-detect and uncompress GZIPped Ion data. + * This method will auto-detect and decompress GZIPped Ion data. WARNING: + * this feature is deprecated and will be removed in subsequent releases. + *

    * * @param ionData may be either Ion binary data or (UTF-8) Ion text, or * GZIPped Ion data. @@ -336,6 +389,9 @@ public SymbolTable newSharedSymbolTable(String name, * values. * @throws IonException if the data does not contain exactly one user * value. + * + * @deprecated auto-detecting of and decompression GZIPped Ion data + * will be removed in subsequent releases. */ public IonValue singleValue(byte[] ionData); @@ -364,22 +420,35 @@ public SymbolTable newSharedSymbolTable(String name, public IonReader newReader(String ionText); /** + *

    * Creates an new {@link IonReader} instance over a block of Ion data, * detecting whether it's text or binary data. + *

    + * *

    - * This method will auto-detect and uncompress GZIPped Ion data. + * This method will auto-detect and decompress GZIPped Ion data. WARNING: + * this feature is deprecated and will be removed in subsequent releases. + *

    * * @param ionData may be either Ion binary data, or UTF-8 Ion text. * The reader retains a reference to the array, so its data must not be * modified while the reader is active. + * + * @deprecated auto-detecting of and decompression GZIPped Ion data + * will be removed in subsequent releases. */ public IonReader newReader(byte[] ionData); /** + *

    * Creates an new {@link IonReader} instance over a block of Ion data, * detecting whether it's text or binary data. + *

    + * *

    - * This method will auto-detect and uncompress GZIPped Ion data. + * This method will auto-detect and decompress GZIPped Ion data. WARNING: + * this feature is deprecated and will be removed in subsequent releases. + *

    * * @param ionData is used only within the range of bytes starting at * {@code offset} for {@code len} bytes. @@ -389,17 +458,26 @@ public SymbolTable newSharedSymbolTable(String name, * @param offset must be non-negative and less than {@code ionData.length}. * @param len must be non-negative and {@code offset+len} must not exceed * {@code ionData.length}. + * + * @deprecated auto-detecting of and decompression GZIPped Ion data + * will be removed in subsequent releases. */ public IonReader newReader(byte[] ionData, int offset, int len); /** + *

    * Creates a new {@link IonReader} instance over a stream of Ion data, * detecting whether it's text or binary data. + * *

    - * This method will auto-detect and uncompress GZIPped Ion data. + * This method will auto-detect and decompress GZIPped Ion data. WARNING: + * this feature is deprecated and will be removed in subsequent releases. + *

    + * *

    * Because this library performs its own buffering, it's recommended that * you avoid adding additional buffering to the given stream. + *

    * * @param ionData must not be null. * @@ -407,6 +485,9 @@ public SymbolTable newSharedSymbolTable(String name, * Callers must call {@link IonReader#close()} when finished with it. * * @throws IonException if the source throws {@link IOException}. + * + * @deprecated auto-detecting of and decompression GZIPped Ion data + * will be removed in subsequent releases. */ public IonReader newReader(InputStream ionData); diff --git a/src/software/amazon/ion/impl/lite/IonLoaderLite.java b/src/software/amazon/ion/impl/lite/IonLoaderLite.java index 1c50234e65..2b96c923bb 100644 --- a/src/software/amazon/ion/impl/lite/IonLoaderLite.java +++ b/src/software/amazon/ion/impl/lite/IonLoaderLite.java @@ -122,21 +122,34 @@ public IonDatagram load(Reader ionText) throws IonException, IOException public IonDatagram load(byte[] ionData) throws IonException { + IonReader reader = null; try { - IonReader reader = makeReader(_catalog, ionData, 0, ionData.length, _lstFactory); + reader = makeReader(_catalog, ionData, 0, ionData.length, _lstFactory); IonDatagramLite datagram = load_helper(reader); return datagram; } catch (IOException e) { throw new IonException(e); } + finally + { + if (reader != null) { + try { + reader.close(); + } + catch (IOException e) { + throw new IonException(e); + } + } + } } public IonDatagram load(InputStream ionData) throws IonException, IOException { + IonReader reader = null; try { - IonReader reader = makeReader(_catalog, ionData, _lstFactory); + reader = makeReader(_catalog, ionData, _lstFactory); IonDatagramLite datagram = load_helper(reader); return datagram; } @@ -145,6 +158,11 @@ public IonDatagram load(InputStream ionData) if (io != null) throw io; throw e; } + finally { + if (reader != null) { + reader.close(); + } + } } } diff --git a/src/software/amazon/ion/impl/lite/IonSystemLite.java b/src/software/amazon/ion/impl/lite/IonSystemLite.java index ff74dd44ee..601d5ca0e7 100644 --- a/src/software/amazon/ion/impl/lite/IonSystemLite.java +++ b/src/software/amazon/ion/impl/lite/IonSystemLite.java @@ -14,6 +14,7 @@ package software.amazon.ion.impl.lite; +import software.amazon.ion.CloseableIterator; import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; import static software.amazon.ion.SystemSymbols.ION_1_0; import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; @@ -170,28 +171,28 @@ public SymbolTable getSystemSymbolTable(String ionVersionId) return getSystemSymbolTable(); } - public Iterator iterate(Reader ionText) + public CloseableIterator iterate(Reader ionText) { IonReader reader = makeReader(_catalog, ionText, _lstFactory); ReaderIterator iterator = new ReaderIterator(this, reader); return iterator; } - public Iterator iterate(InputStream ionData) + public CloseableIterator iterate(InputStream ionData) { IonReader reader = makeReader(_catalog, ionData, _lstFactory); ReaderIterator iterator = new ReaderIterator(this, reader); return iterator; } - public Iterator iterate(String ionText) + public CloseableIterator iterate(String ionText) { IonReader reader = makeReader(_catalog, ionText, _lstFactory); ReaderIterator iterator = new ReaderIterator(this, reader); return iterator; } - public Iterator iterate(byte[] ionData) + public CloseableIterator iterate(byte[] ionData) { IonReader reader = makeReader(_catalog, ionData, _lstFactory); ReaderIterator iterator = new ReaderIterator(this, reader); @@ -471,31 +472,38 @@ public IonWriter newWriter(IonContainer container) return writer; } - private IonValue singleValue(Iterator it) + private IonValue singleValue(CloseableIterator it) { - IonValue value; try { - value = it.next(); + IonValue value = it.next(); + if (it.hasNext()) { + throw new IonException("not a single value"); + } + return value; } catch (NoSuchElementException e) { throw new UnexpectedEofException("no value found on input stream"); } - if (it.hasNext()) { - throw new IonException("not a single value"); + finally { + try { + it.close(); + } + catch (IOException e) { + throw new IonException(e); + } } - return value; } public IonValue singleValue(String ionText) { - Iterator it = iterate(ionText); + CloseableIterator it = iterate(ionText); return singleValue(it); } public IonValue singleValue(byte[] ionData) { - Iterator it = iterate(ionData); - return singleValue(it); + CloseableIterator iterator = iterate(ionData); + return singleValue(iterator); } protected IonSymbolLite newSystemIdSymbol(String ionVersionMarker) @@ -510,7 +518,7 @@ protected IonSymbolLite newSystemIdSymbol(String ionVersionMarker) } static class ReaderIterator - implements Iterator, Closeable + implements CloseableIterator { private final IonReader _reader; private final IonSystemLite _system; @@ -531,7 +539,18 @@ public boolean hasNext() if (_next == null) { _next = _reader.next(); } - return (_next != null); + boolean hasNext = _next != null; + + if (!hasNext) { + try { + close(); + } + catch (IOException e) { + throw new IonException(e); + } + } + + return hasNext; } public IonValue next() @@ -567,11 +586,10 @@ public void remove() public void close() throws IOException { - // TODO _reader.close(); + _reader.close(); } } - public IonTimestamp newUtcTimestampFromMillis(long millis) { IonTimestamp result = newNullTimestamp(); diff --git a/src/software/amazon/ion/system/IonReaderBuilder.java b/src/software/amazon/ion/system/IonReaderBuilder.java index 7366ba52d9..10b45d4004 100644 --- a/src/software/amazon/ion/system/IonReaderBuilder.java +++ b/src/software/amazon/ion/system/IonReaderBuilder.java @@ -160,7 +160,8 @@ private IonCatalog validateCatalog() * instance over the given block of Ion data, detecting whether it's text or * binary data. *

    - * This method will auto-detect and uncompress GZIPped Ion data. + * This method will auto-detect and uncompress GZIPped Ion data. WARNING: + * * this feature is deprecated and will be removed in subsequent releases. * * @param ionData the source of the Ion data, which may be either Ion binary * data or UTF-8 Ion text. The reader retains a reference to the array, so @@ -170,6 +171,9 @@ private IonCatalog validateCatalog() * @return a new {@link IonReader} instance; not {@code null}. * * @see IonSystem#newReader(byte[]) + * + * @deprecated auto-detecting of and decompression GZIPped Ion data will be + * removed in subsequent releases. */ public IonReader build(byte[] ionData) { @@ -181,7 +185,8 @@ public IonReader build(byte[] ionData) * instance over the given block of Ion data, detecting whether it's text or * binary data. *

    - * This method will auto-detect and uncompress GZIPped Ion data. + * This method will auto-detect and uncompress GZIPped Ion data. WARNING: + * this feature is deprecated and will be removed in subsequent releases. * * @param ionData the source of the Ion data, which is used only within the * range of bytes starting at {@code offset} for {@code len} bytes. @@ -193,6 +198,9 @@ public IonReader build(byte[] ionData) * exceed {@code ionData.length}. * * @see IonSystem#newReader(byte[], int, int) + * + * @deprecated auto-detecting of and decompression GZIPped Ion data will be + * removed in subsequent releases. */ public IonReader build(byte[] ionData, int offset, int length) { @@ -204,7 +212,8 @@ public IonReader build(byte[] ionData, int offset, int length) * instance over the given stream of Ion data, detecting whether it's text or * binary data. *

    - * This method will auto-detect and uncompress GZIPped Ion data. + * This method will auto-detect and uncompress GZIPped Ion data. WARNING: + * this feature is deprecated and will be removed in subsequent releases. *

    * Because this library performs its own buffering, it's recommended that * users avoid adding additional buffering to the given stream. @@ -218,6 +227,10 @@ public IonReader build(byte[] ionData, int offset, int length) * @throws IonException if the source throws {@link IOException}. * * @see IonSystem#newReader(InputStream) + * + * @deprecated auto-detecting of and decompression GZIPped Ion data will be + * removed in subsequent releases. Use a {@link java.util.zip.GZIPInputStream} + * to process GZIPped Ion data. */ public IonReader build(InputStream ionData) { From 135139e58d5e5cdb350e10b4cd784ad59a586f5f Mon Sep 17 00:00:00 2001 From: David Lurton Date: Mon, 4 Feb 2019 12:16:17 -0800 Subject: [PATCH 043/490] OpenJDK11 javadoc fix (#208) #207 fix OpenJDK11 build failing during javadoc --- pom.xml | 3 +++ src/software/amazon/ion/IonSystem.java | 4 ++-- src/software/amazon/ion/ValueFactory.java | 8 ++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index d23bc97626..0747f87d48 100644 --- a/pom.xml +++ b/pom.xml @@ -186,6 +186,9 @@ org.apache.maven.plugins maven-javadoc-plugin 2.10.3 + + 8 + diff --git a/src/software/amazon/ion/IonSystem.java b/src/software/amazon/ion/IonSystem.java index 0115ffc251..25ed5a3b0d 100644 --- a/src/software/amazon/ion/IonSystem.java +++ b/src/software/amazon/ion/IonSystem.java @@ -650,8 +650,8 @@ public IonWriter newTextWriter(Appendable out, SymbolTable... imports) /** * Creates a new datagram, bootstrapped with imported symbol tables. - * Generally an application will use this to aquire a datagram, then adds - * values to it, then calls {@link IonDatagram#getBytes(byte[])} + * Generally an application will use this to acquire a datagram, then adds + * values to it, then calls {@link IonDatagram#getBytes()} * (or similar) to extract binary data. * * @param imports the set of shared symbol tables to import. diff --git a/src/software/amazon/ion/ValueFactory.java b/src/software/amazon/ion/ValueFactory.java index 8767ae5832..7b9f5d9b07 100644 --- a/src/software/amazon/ion/ValueFactory.java +++ b/src/software/amazon/ion/ValueFactory.java @@ -252,14 +252,14 @@ public interface ValueFactory /** * Constructs a new {@code list} with the given child. + * + * @param child the initial child of the new list. *

    - * This method is temporary until {@link #newList(Collection)} is + * This method is temporary until "newList(Collection)" is * removed. It's sole purpose is to avoid the doomed attempt to add all * of the parameter's children to the new list; that will always throw * {@link ContainedValueException}. * - * @param child the initial child of the new list. - * * @throws NullPointerException if {@code child} is null. * @throws IllegalArgumentException if {@code child} is an {@link IonDatagram}. * @throws ContainedValueException @@ -360,7 +360,7 @@ public IonList newList(IonValue... children) /** * Constructs a new {@code sexp} with the given child. *

    - * This method is temporary until {@link #newSexp(Collection)} is + * This method is temporary until "newSexp(Collection)" is * removed. It's sole purpose is to avoid the doomed attempt to add all * of the parameter's children to the new sequence; that will always throw * {@link ContainedValueException}. From 151492d7bf1bb8785a35b275cf431b39d5d80829 Mon Sep 17 00:00:00 2001 From: Peter Cornell Date: Thu, 7 Feb 2019 17:39:16 -0800 Subject: [PATCH 044/490] Revert "Memleak gzip (#204)" (#211) This reverts commit 9dc90d41f43b23c8fa9382d9e696665b0054810b. --- .../amazon/ion/CloseableIterator.java | 14 --- src/software/amazon/ion/IonLoader.java | 24 +--- src/software/amazon/ion/IonSystem.java | 113 +++--------------- .../amazon/ion/impl/lite/IonLoaderLite.java | 22 +--- .../amazon/ion/impl/lite/IonSystemLite.java | 52 +++----- .../amazon/ion/system/IonReaderBuilder.java | 19 +-- 6 files changed, 40 insertions(+), 204 deletions(-) delete mode 100644 src/software/amazon/ion/CloseableIterator.java diff --git a/src/software/amazon/ion/CloseableIterator.java b/src/software/amazon/ion/CloseableIterator.java deleted file mode 100644 index fde07b8de2..0000000000 --- a/src/software/amazon/ion/CloseableIterator.java +++ /dev/null @@ -1,14 +0,0 @@ -package software.amazon.ion; - -import java.io.Closeable; -import java.util.Iterator; - -/** - * Iterator that can close its source of data. - * - * @param the type of elements returned by this iterator. - */ -public interface CloseableIterator extends Iterator, Closeable -{ - // empty -} diff --git a/src/software/amazon/ion/IonLoader.java b/src/software/amazon/ion/IonLoader.java index ad2fa9e590..9b6e7819c5 100644 --- a/src/software/amazon/ion/IonLoader.java +++ b/src/software/amazon/ion/IonLoader.java @@ -105,16 +105,10 @@ public IonDatagram load(Reader ionText) /** - *

    * Loads a block of Ion data into a single datagram, * detecting whether it's text or binary data. - *

    - * *

    - * This method will auto-detect and decompress GZIPped Ion data. - * WARNING: this feature is deprecated and will - * be removed in subsequent releases. - *

    + * This method will auto-detect and uncompress GZIPped Ion data. * * @param ionData may be either Ion binary data, or UTF-8 Ion text. * This method assumes ownership of the array and may modify it at @@ -125,9 +119,6 @@ public IonDatagram load(Reader ionText) * * @throws NullPointerException if ionData is null. * @throws IonException if there's a syntax error in the Ion content. - * - * @deprecated auto-detecting of and decompression GZIPped Ion data will - * be removed in subsequent releases. */ public IonDatagram load(byte[] ionData) throws IonException; @@ -138,18 +129,11 @@ public IonDatagram load(byte[] ionData) * detecting whether it's text or binary data. *

    * The specified stream remains open after this method returns. - *

    - * *

    - * This method will auto-detect and decompress GZIPped Ion data. - * WARNING: this feature is deprecated and will be - * removed in subsequent releases. - *

    - * + * This method will auto-detect and uncompress GZIPped Ion data. *

    * Because this library performs its own buffering, it's recommended that * you avoid adding additional buffering to the given stream. - *

    * * @param ionData the stream from which to read Ion data. * @@ -160,10 +144,6 @@ public IonDatagram load(byte[] ionData) * @throws IonException if there's a syntax error in the Ion content. * @throws IOException if reading from the specified input stream results * in an IOException. - * - * @deprecated auto-detecting of and decompression GZIPped Ion data will - * be removed in subsequent releases. Use a {@link java.util.zip.GZIPInputStream} - * to process GZIPped Ion data. */ public IonDatagram load(InputStream ionData) throws IonException, IOException; diff --git a/src/software/amazon/ion/IonSystem.java b/src/software/amazon/ion/IonSystem.java index 25ed5a3b0d..5c6da86b36 100644 --- a/src/software/amazon/ion/IonSystem.java +++ b/src/software/amazon/ion/IonSystem.java @@ -204,99 +204,66 @@ public SymbolTable newSharedSymbolTable(String name, /** - *

    * Creates an iterator over a stream of Ion text data. * Values returned by the iterator have no container. - *

    - * *

    * The iterator will automatically consume Ion system IDs and local symbol * tables; they will not be returned by the iterator. - *

    - * *

    * If the input source throws an {@link IOException} during iteration, it * will be wrapped in an {@link IonException}. See documentation there for * tips on how to recover the cause. - *

    - * *

    * This method is suitable for use over unbounded streams with a reasonable * schema. - *

    - * *

    * Applications should generally use {@link #iterate(InputStream)} * whenever possible, since this library has much faster UTF-8 decoding * than the Java IO framework. - *

    - * *

    * Because this library performs its own buffering, it's recommended that * you avoid adding additional buffering to the given stream. - *

    * - * @param ionText a stream of Ion text data. The caller is responsible for + * @param ionText a stream of Ion text data. The caller is responsible for * closing the Reader after iteration is complete. * - * @return a new iterator instance. Closing this iterator - * will close ionText Reader. The iterator automatically closes when - * there are no more values to return. + * @return a new iterator instance. * * @throws NullPointerException if ionText is null. * @throws IonException if the source throws {@link IOException}. */ - public CloseableIterator iterate(Reader ionText); + public Iterator iterate(Reader ionText); + /** - *

    * Creates an iterator over a stream of Ion data, * detecting whether it's text or binary data. * Values returned by the iterator have no container. - *

    - * *

    * The iterator will automatically consume Ion system IDs and local symbol * tables; they will not be returned by the iterator. - *

    - * *

    * If the input source throws an {@link IOException} during iteration, it * will be wrapped in an {@link IonException}. See documentation there for * tips on how to recover the cause. - *

    - * *

    * This method is suitable for use over unbounded streams with a reasonable * schema. - *

    - * *

    - * This method will auto-detect and decompress GZIPped Ion data. WARNING: - * this feature is deprecated and will be removed in subsequent releases. - *

    - * + * This method will auto-detect and uncompress GZIPped Ion data. *

    * Because this library performs its own buffering, it's recommended that * you avoid adding additional buffering to the given stream. - *

    * * @param ionData a stream of Ion data. The caller is responsible for * closing the InputStream after iteration is complete. * - * @return a new CloseableIterator instance. Closing this iterator - * will close ionData InputStream. The iterator automatically closes when - * there are no more values to return. WARNING: This instance - * must be closed by the client to avoid memory leaks when dealing with - * GZIPped Ion data. + * @return a new iterator instance. * * @throws NullPointerException if ionData is null. * @throws IonException if the source throws {@link IOException}. - * - * @deprecated auto-detecting of and decompression GZIPped Ion data - * will be removed in subsequent releases. */ - public CloseableIterator iterate(InputStream ionData); + public Iterator iterate(InputStream ionData); /** @@ -308,47 +275,32 @@ public SymbolTable newSharedSymbolTable(String name, * * @param ionText must not be null. * - * @return a new iterator instance. Closing this iterator will - * close the underlying {@link IonReader} created to read ionText. - * The iterator automatically closes when there are no more values - * to return. + * @return a new iterator instance. * * @throws NullPointerException if ionText is null. */ - public CloseableIterator iterate(String ionText); + public Iterator iterate(String ionText); /** - *

    * Creates an iterator over Ion data. * Values returned by the iterator have no container. - *

    *

    * The iterator will automatically consume Ion system IDs and local symbol * tables; they will not be returned by the iterator. - *

    *

    - * This method will auto-detect and decompress GZIPped Ion data. WARNING: - * this feature is deprecated and will be removed in subsequent releases. - *

    + * This method will auto-detect and uncompress GZIPped Ion data. * * @param ionData may be either Ion binary data or (UTF-8) Ion text, or * GZIPped Ion data. * This method assumes ownership of the array and may modify it at * will. * - * @return a new CloseableIterator instance. Closing this iterator will - * close the underlying {@link IonReader} created to read ionData. The iterator - * automatically closes when there are no more values to return. - * WARNING: This instance must be closed by the client - * to avoid memory leaks when dealing with GZIPped Ion data. + * @return a new iterator instance. * * @throws NullPointerException if ionData is null. - * - * @deprecated auto-detecting of and decompression GZIPped Ion data - * will be removed in subsequent releases. */ - public CloseableIterator iterate(byte[] ionData); + public Iterator iterate(byte[] ionData); /** @@ -368,14 +320,9 @@ public SymbolTable newSharedSymbolTable(String name, /** - *

    * Extracts a single value from Ion text or binary data. - *

    - * *

    - * This method will auto-detect and decompress GZIPped Ion data. WARNING: - * this feature is deprecated and will be removed in subsequent releases. - *

    + * This method will auto-detect and uncompress GZIPped Ion data. * * @param ionData may be either Ion binary data or (UTF-8) Ion text, or * GZIPped Ion data. @@ -389,9 +336,6 @@ public SymbolTable newSharedSymbolTable(String name, * values. * @throws IonException if the data does not contain exactly one user * value. - * - * @deprecated auto-detecting of and decompression GZIPped Ion data - * will be removed in subsequent releases. */ public IonValue singleValue(byte[] ionData); @@ -420,35 +364,22 @@ public SymbolTable newSharedSymbolTable(String name, public IonReader newReader(String ionText); /** - *

    * Creates an new {@link IonReader} instance over a block of Ion data, * detecting whether it's text or binary data. - *

    - * *

    - * This method will auto-detect and decompress GZIPped Ion data. WARNING: - * this feature is deprecated and will be removed in subsequent releases. - *

    + * This method will auto-detect and uncompress GZIPped Ion data. * * @param ionData may be either Ion binary data, or UTF-8 Ion text. * The reader retains a reference to the array, so its data must not be * modified while the reader is active. - * - * @deprecated auto-detecting of and decompression GZIPped Ion data - * will be removed in subsequent releases. */ public IonReader newReader(byte[] ionData); /** - *

    * Creates an new {@link IonReader} instance over a block of Ion data, * detecting whether it's text or binary data. - *

    - * *

    - * This method will auto-detect and decompress GZIPped Ion data. WARNING: - * this feature is deprecated and will be removed in subsequent releases. - *

    + * This method will auto-detect and uncompress GZIPped Ion data. * * @param ionData is used only within the range of bytes starting at * {@code offset} for {@code len} bytes. @@ -458,26 +389,17 @@ public SymbolTable newSharedSymbolTable(String name, * @param offset must be non-negative and less than {@code ionData.length}. * @param len must be non-negative and {@code offset+len} must not exceed * {@code ionData.length}. - * - * @deprecated auto-detecting of and decompression GZIPped Ion data - * will be removed in subsequent releases. */ public IonReader newReader(byte[] ionData, int offset, int len); /** - *

    * Creates a new {@link IonReader} instance over a stream of Ion data, * detecting whether it's text or binary data. - * *

    - * This method will auto-detect and decompress GZIPped Ion data. WARNING: - * this feature is deprecated and will be removed in subsequent releases. - *

    - * + * This method will auto-detect and uncompress GZIPped Ion data. *

    * Because this library performs its own buffering, it's recommended that * you avoid adding additional buffering to the given stream. - *

    * * @param ionData must not be null. * @@ -485,9 +407,6 @@ public SymbolTable newSharedSymbolTable(String name, * Callers must call {@link IonReader#close()} when finished with it. * * @throws IonException if the source throws {@link IOException}. - * - * @deprecated auto-detecting of and decompression GZIPped Ion data - * will be removed in subsequent releases. */ public IonReader newReader(InputStream ionData); diff --git a/src/software/amazon/ion/impl/lite/IonLoaderLite.java b/src/software/amazon/ion/impl/lite/IonLoaderLite.java index 2b96c923bb..1c50234e65 100644 --- a/src/software/amazon/ion/impl/lite/IonLoaderLite.java +++ b/src/software/amazon/ion/impl/lite/IonLoaderLite.java @@ -122,34 +122,21 @@ public IonDatagram load(Reader ionText) throws IonException, IOException public IonDatagram load(byte[] ionData) throws IonException { - IonReader reader = null; try { - reader = makeReader(_catalog, ionData, 0, ionData.length, _lstFactory); + IonReader reader = makeReader(_catalog, ionData, 0, ionData.length, _lstFactory); IonDatagramLite datagram = load_helper(reader); return datagram; } catch (IOException e) { throw new IonException(e); } - finally - { - if (reader != null) { - try { - reader.close(); - } - catch (IOException e) { - throw new IonException(e); - } - } - } } public IonDatagram load(InputStream ionData) throws IonException, IOException { - IonReader reader = null; try { - reader = makeReader(_catalog, ionData, _lstFactory); + IonReader reader = makeReader(_catalog, ionData, _lstFactory); IonDatagramLite datagram = load_helper(reader); return datagram; } @@ -158,11 +145,6 @@ public IonDatagram load(InputStream ionData) if (io != null) throw io; throw e; } - finally { - if (reader != null) { - reader.close(); - } - } } } diff --git a/src/software/amazon/ion/impl/lite/IonSystemLite.java b/src/software/amazon/ion/impl/lite/IonSystemLite.java index 601d5ca0e7..ff74dd44ee 100644 --- a/src/software/amazon/ion/impl/lite/IonSystemLite.java +++ b/src/software/amazon/ion/impl/lite/IonSystemLite.java @@ -14,7 +14,6 @@ package software.amazon.ion.impl.lite; -import software.amazon.ion.CloseableIterator; import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; import static software.amazon.ion.SystemSymbols.ION_1_0; import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; @@ -171,28 +170,28 @@ public SymbolTable getSystemSymbolTable(String ionVersionId) return getSystemSymbolTable(); } - public CloseableIterator iterate(Reader ionText) + public Iterator iterate(Reader ionText) { IonReader reader = makeReader(_catalog, ionText, _lstFactory); ReaderIterator iterator = new ReaderIterator(this, reader); return iterator; } - public CloseableIterator iterate(InputStream ionData) + public Iterator iterate(InputStream ionData) { IonReader reader = makeReader(_catalog, ionData, _lstFactory); ReaderIterator iterator = new ReaderIterator(this, reader); return iterator; } - public CloseableIterator iterate(String ionText) + public Iterator iterate(String ionText) { IonReader reader = makeReader(_catalog, ionText, _lstFactory); ReaderIterator iterator = new ReaderIterator(this, reader); return iterator; } - public CloseableIterator iterate(byte[] ionData) + public Iterator iterate(byte[] ionData) { IonReader reader = makeReader(_catalog, ionData, _lstFactory); ReaderIterator iterator = new ReaderIterator(this, reader); @@ -472,38 +471,31 @@ public IonWriter newWriter(IonContainer container) return writer; } - private IonValue singleValue(CloseableIterator it) + private IonValue singleValue(Iterator it) { + IonValue value; try { - IonValue value = it.next(); - if (it.hasNext()) { - throw new IonException("not a single value"); - } - return value; + value = it.next(); } catch (NoSuchElementException e) { throw new UnexpectedEofException("no value found on input stream"); } - finally { - try { - it.close(); - } - catch (IOException e) { - throw new IonException(e); - } + if (it.hasNext()) { + throw new IonException("not a single value"); } + return value; } public IonValue singleValue(String ionText) { - CloseableIterator it = iterate(ionText); + Iterator it = iterate(ionText); return singleValue(it); } public IonValue singleValue(byte[] ionData) { - CloseableIterator iterator = iterate(ionData); - return singleValue(iterator); + Iterator it = iterate(ionData); + return singleValue(it); } protected IonSymbolLite newSystemIdSymbol(String ionVersionMarker) @@ -518,7 +510,7 @@ protected IonSymbolLite newSystemIdSymbol(String ionVersionMarker) } static class ReaderIterator - implements CloseableIterator + implements Iterator, Closeable { private final IonReader _reader; private final IonSystemLite _system; @@ -539,18 +531,7 @@ public boolean hasNext() if (_next == null) { _next = _reader.next(); } - boolean hasNext = _next != null; - - if (!hasNext) { - try { - close(); - } - catch (IOException e) { - throw new IonException(e); - } - } - - return hasNext; + return (_next != null); } public IonValue next() @@ -586,10 +567,11 @@ public void remove() public void close() throws IOException { - _reader.close(); + // TODO _reader.close(); } } + public IonTimestamp newUtcTimestampFromMillis(long millis) { IonTimestamp result = newNullTimestamp(); diff --git a/src/software/amazon/ion/system/IonReaderBuilder.java b/src/software/amazon/ion/system/IonReaderBuilder.java index 10b45d4004..7366ba52d9 100644 --- a/src/software/amazon/ion/system/IonReaderBuilder.java +++ b/src/software/amazon/ion/system/IonReaderBuilder.java @@ -160,8 +160,7 @@ private IonCatalog validateCatalog() * instance over the given block of Ion data, detecting whether it's text or * binary data. *

    - * This method will auto-detect and uncompress GZIPped Ion data. WARNING: - * * this feature is deprecated and will be removed in subsequent releases. + * This method will auto-detect and uncompress GZIPped Ion data. * * @param ionData the source of the Ion data, which may be either Ion binary * data or UTF-8 Ion text. The reader retains a reference to the array, so @@ -171,9 +170,6 @@ private IonCatalog validateCatalog() * @return a new {@link IonReader} instance; not {@code null}. * * @see IonSystem#newReader(byte[]) - * - * @deprecated auto-detecting of and decompression GZIPped Ion data will be - * removed in subsequent releases. */ public IonReader build(byte[] ionData) { @@ -185,8 +181,7 @@ public IonReader build(byte[] ionData) * instance over the given block of Ion data, detecting whether it's text or * binary data. *

    - * This method will auto-detect and uncompress GZIPped Ion data. WARNING: - * this feature is deprecated and will be removed in subsequent releases. + * This method will auto-detect and uncompress GZIPped Ion data. * * @param ionData the source of the Ion data, which is used only within the * range of bytes starting at {@code offset} for {@code len} bytes. @@ -198,9 +193,6 @@ public IonReader build(byte[] ionData) * exceed {@code ionData.length}. * * @see IonSystem#newReader(byte[], int, int) - * - * @deprecated auto-detecting of and decompression GZIPped Ion data will be - * removed in subsequent releases. */ public IonReader build(byte[] ionData, int offset, int length) { @@ -212,8 +204,7 @@ public IonReader build(byte[] ionData, int offset, int length) * instance over the given stream of Ion data, detecting whether it's text or * binary data. *

    - * This method will auto-detect and uncompress GZIPped Ion data. WARNING: - * this feature is deprecated and will be removed in subsequent releases. + * This method will auto-detect and uncompress GZIPped Ion data. *

    * Because this library performs its own buffering, it's recommended that * users avoid adding additional buffering to the given stream. @@ -227,10 +218,6 @@ public IonReader build(byte[] ionData, int offset, int length) * @throws IonException if the source throws {@link IOException}. * * @see IonSystem#newReader(InputStream) - * - * @deprecated auto-detecting of and decompression GZIPped Ion data will be - * removed in subsequent releases. Use a {@link java.util.zip.GZIPInputStream} - * to process GZIPped Ion data. */ public IonReader build(InputStream ionData) { From 4f2bed5ddaa51f2ec7bd3ba33b1d89f3e7aa6c6d Mon Sep 17 00:00:00 2001 From: Peter Cornell Date: Fri, 8 Feb 2019 14:00:50 -0800 Subject: [PATCH 045/490] fixes getMillis() and getDecimalMillis() behavior for Timestamps constructed via forCalendar() (#213) --- src/software/amazon/ion/Timestamp.java | 23 +++++++++++++++--- test/software/amazon/ion/TimestampTest.java | 26 +++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/software/amazon/ion/Timestamp.java b/src/software/amazon/ion/Timestamp.java index 62984161f7..eb1bc13bbc 100644 --- a/src/software/amazon/ion/Timestamp.java +++ b/src/software/amazon/ion/Timestamp.java @@ -166,6 +166,9 @@ public boolean includes(Precision isIncluded) /** * Calendar to hold the Timestamp's year, month, day, hour, minute, second, and calendar system. Fractional * seconds are left to {@link #_fraction}, while local offset is left to {@link #_offset}. + *

    + * This calendar instance is adjusted such that it is always in UTC; see _calendarCompensationOffsetMs + * if the original "instant" millis is needed. */ private final Calendar _calendar; @@ -180,6 +183,12 @@ public boolean includes(Precision isIncluded) */ private Integer _offset; + /** + * Number of milliseconds _calendar has been adjusted so that it is in UTC. + * Allows getMillis() and getDecimalMillis() to return the correct response. + */ + private int _calendarCompensationOffsetMs; + // Minimum millis under the calendar system provided by the default GregorianCalendar implementation. private static final long MINIMUM_TIMESTAMP_IN_MILLIS = Timestamp.valueOf("0001-01-01T00:00:00.000Z").getMillis(); static final BigDecimal MINIMUM_TIMESTAMP_IN_MILLIS_DECIMAL = new BigDecimal(MINIMUM_TIMESTAMP_IN_MILLIS); @@ -206,12 +215,20 @@ private void apply_offset(int offset) offset = -offset; int hour_offset = offset / 60; int min_offset = offset - (hour_offset * 60); + + // When we add hour_offset and min_offset to _calendar below, the underlying millis "instant" + // is changed. To compensate, we set _calendarCompensationOffsetMs here and use it to + // recalculate the correct millis if/when needed + if (_calendar.isSet(Calendar.ZONE_OFFSET) || _calendar.isSet(Calendar.DST_OFFSET)) { + _calendarCompensationOffsetMs = -offset * 60 * 1000; + } + // First, clear the offsets that are already set. Otherwise, the 'add' calls will add them in, which will // result in a double add. _calendar.clear(Calendar.ZONE_OFFSET); _calendar.clear(Calendar.DST_OFFSET); - _calendar.add(Calendar.MINUTE, min_offset); _calendar.add(Calendar.HOUR_OF_DAY, hour_offset); + _calendar.add(Calendar.MINUTE, min_offset); } /** @@ -1341,7 +1358,7 @@ public long getMillis() int frac = isIntegralZero(fracAsDecimal) ? 0 : fracAsDecimal.intValue(); millis += frac; } - return millis; + return millis + _calendarCompensationOffsetMs; } @@ -1365,7 +1382,7 @@ public BigDecimal getDecimalMillis() case DAY: case MINUTE: case SECOND: - long millis = _calendar.getTimeInMillis(); + long millis = _calendar.getTimeInMillis() + _calendarCompensationOffsetMs; BigDecimal dec = BigDecimal.valueOf(millis); if (_fraction != null) { dec = dec.add(this._fraction.movePointRight(3)); diff --git a/test/software/amazon/ion/TimestampTest.java b/test/software/amazon/ion/TimestampTest.java index 5893f27470..ecf86dff1c 100644 --- a/test/software/amazon/ion/TimestampTest.java +++ b/test/software/amazon/ion/TimestampTest.java @@ -30,6 +30,7 @@ import java.math.BigDecimal; import java.text.DateFormat; +import java.text.ParseException; import java.text.SimpleDateFormat; import java.time.Instant; import java.time.ZoneId; @@ -2870,6 +2871,31 @@ public void testGetMillisWithLargeScaleBigDecimal() Timestamp.forMillis(LARGE_SCALE_DECIMAL, PST_OFFSET).getMillis(); } + @Test + public void testMillisIsIndepedentOfOffset() throws ParseException { + DateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); + Date date = format.parse("2010-06-29T23:20:00+0000"); + + Calendar cal1 = Calendar.getInstance(); + cal1.setTimeZone(TimeZone.getTimeZone("GMT+00:00")); + cal1.setTime(date); + Timestamp t1 = Timestamp.forCalendar(cal1); + + Calendar cal2 = Calendar.getInstance(); + cal2.setTimeZone(TimeZone.getTimeZone("GMT+02:00")); + cal2.setTime(date); + Timestamp t2 = Timestamp.forCalendar(cal2); + + assertEquals(t1.getMillis(), date.getTime()); + assertEquals(t2.getMillis(), date.getTime()); + + assertEquals(BigDecimal.valueOf(t1.getMillis()), t1.getDecimalMillis()); + assertEquals(BigDecimal.valueOf(t2.getMillis()), t2.getDecimalMillis()); + + assertEquals(t1.getMillis(), t2.getMillis()); + assertEquals(t1.getDecimalMillis(), t2.getDecimalMillis()); + } + @Test public void testInstantVsTimestampMillis() { // addresses: https://github.com/amzn/ion-java/issues/165 From 295cda36744d370a0556803408beefe8b40587dc Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Tue, 26 Feb 2019 13:54:48 -0800 Subject: [PATCH 046/490] Bumps version to 1.3.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0747f87d48..8e3b3b2930 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 software.amazon.ion ion-java - 1.2.1-SNAPSHOT + 1.3.0 bundle ${project.groupId}:${project.artifactId} From 3aa8d03cb78c38be3bb9ac2233f6a41fd65d5ef1 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Tue, 26 Feb 2019 14:27:31 -0800 Subject: [PATCH 047/490] Updates README.md to refer to 1.3.0. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 18c5f78708..df313a313d 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ dependency into your project's `pom.xml`: software.amazon.ion ion-java - 1.2.0 + 1.3.0 ``` From 9bf1cf895cb1e9866c1881c608f049cfff67cf5b Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Tue, 26 Feb 2019 16:23:07 -0800 Subject: [PATCH 048/490] Bumps version to 1.3.1-SNAPSHOT. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8e3b3b2930..9038a710db 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 software.amazon.ion ion-java - 1.3.0 + 1.3.1-SNAPSHOT bundle ${project.groupId}:${project.artifactId} From 61fae4297abad87f952a02f56e2b6fcce8dbc353 Mon Sep 17 00:00:00 2001 From: sullis Date: Thu, 28 Feb 2019 11:58:16 -0800 Subject: [PATCH 049/490] Updates to maven-bundle-plugin 4.1.0 (#205) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9038a710db..8fa2bfbb8b 100644 --- a/pom.xml +++ b/pom.xml @@ -106,7 +106,7 @@ org.apache.felix maven-bundle-plugin - 3.2.0 + 4.1.0 true From 875df4e565d5a52089ef21fb1113b670a3115c8c Mon Sep 17 00:00:00 2001 From: freddytuxworth Date: Thu, 28 Feb 2019 23:41:39 +0000 Subject: [PATCH 050/490] Added tests for software.amazon.ion.impl.IonUTF8 (#206) These tests were written using Diffblue Cover. --- .../software/amazon/ion/impl/IonUTF8Test.java | 205 ++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 test/software/amazon/ion/impl/IonUTF8Test.java diff --git a/test/software/amazon/ion/impl/IonUTF8Test.java b/test/software/amazon/ion/impl/IonUTF8Test.java new file mode 100644 index 0000000000..15b09cd9e8 --- /dev/null +++ b/test/software/amazon/ion/impl/IonUTF8Test.java @@ -0,0 +1,205 @@ +package software.amazon.ion.impl; + +import org.junit.Assert; +import org.junit.Test; + +public class IonUTF8Test { + + @Test + public void testGetScalarReadLengthFromBytes() { + Assert.assertEquals(-1, IonUTF8.getScalarReadLengthFromBytes(new byte[] {(byte) -8}, 0, 10)); + Assert.assertEquals(2, IonUTF8.getScalarReadLengthFromBytes(new byte[] {(byte) -64}, 0, 20)); + Assert.assertEquals(1, IonUTF8.getScalarReadLengthFromBytes(new byte[] {(byte) 120}, 0, 20)); + Assert.assertEquals(3, IonUTF8.getScalarReadLengthFromBytes(new byte[] {(byte) -32}, 0, 20)); + Assert.assertEquals(4, IonUTF8.getScalarReadLengthFromBytes(new byte[] {(byte) -16}, 0, 20)); + Assert.assertEquals(-1, IonUTF8.getScalarReadLengthFromBytes(new byte[] {(byte) -12, (byte) -128}, 1, 20)); + Assert.assertEquals(1, IonUTF8.getScalarReadLengthFromBytes(new byte[] {(byte) 112, (byte) 112}, 1, 20)); + Assert.assertEquals(4, IonUTF8.getScalarReadLengthFromBytes(new byte[] {(byte) -16, (byte) -16}, 1, 20)); + Assert.assertEquals(3, IonUTF8.getScalarReadLengthFromBytes(new byte[] {(byte) -32, (byte) -32}, 1, 20)); + Assert.assertEquals(2, IonUTF8.getScalarReadLengthFromBytes(new byte[] {(byte) -64, (byte) -63}, 0, 20)); + } + + @Test + public void testGetScalarFromBytes() { + Assert.assertEquals(112, IonUTF8.getScalarFromBytes(new byte[] {112}, 0, 33)); + Assert.assertEquals(9, IonUTF8.getScalarFromBytes(new byte[] {-63, -120, 20, 30}, 0, 3)); + Assert.assertEquals(9, IonUTF8.getScalarFromBytes(new byte[] {-31, -120, -64, 50} , 0, 8)); + Assert.assertEquals(17, IonUTF8.getScalarFromBytes(new byte[] {-15, -112, -128, -128}, 0, 8)); + } + + @Test + public void testIsContinueByteUTF8() { + Assert.assertEquals(true, IonUTF8.isContinueByteUTF8(128)); + Assert.assertEquals(false, IonUTF8.isContinueByteUTF8(0)); + } + + @Test + public void testIsOneByteScalar() { + Assert.assertEquals(true, IonUTF8.isOneByteScalar(-2147450752)); + Assert.assertEquals(false, IonUTF8.isOneByteScalar(128)); + } + + @Test + public void testIsHighSurrogate() { + Assert.assertEquals(true, IonUTF8.isHighSurrogate(55296)); + Assert.assertEquals(false, IonUTF8.isHighSurrogate(0)); + } + + @Test + public void testGetScalarFrom4BytesReversed() { + Assert.assertEquals(0, IonUTF8.getScalarFrom4BytesReversed(192)); + Assert.assertEquals(0, IonUTF8.getScalarFrom4BytesReversed(224)); + Assert.assertEquals(63, IonUTF8.getScalarFrom4BytesReversed((240<<24) - 64)); + Assert.assertEquals(0, IonUTF8.getScalarFrom4BytesReversed(192)); + Assert.assertEquals(0, IonUTF8.getScalarFrom4BytesReversed(240)); + Assert.assertEquals(0, IonUTF8.getScalarFrom4BytesReversed(0)); + } + + @Test + public void testGetUnicodeScalarFromSurrogates() { + Assert.assertEquals(458752, IonUTF8.getUnicodeScalarFromSurrogates(55680, 56320)); + } + + @Test + public void testIsFourByteScalar() { + Assert.assertEquals(true, IonUTF8.isFourByteScalar(-2146369535)); + Assert.assertEquals(false, IonUTF8.isFourByteScalar(1114113)); + } + + @Test + public void testConvertToUTF8Bytes() { + byte[] outBytes = new byte[] {13, 13, 13, 13}; + Assert.assertEquals(1, IonUTF8.convertToUTF8Bytes(32, outBytes, 0, 257)); + Assert.assertArrayEquals(new byte[] {32, 13, 13, 13}, outBytes); + Assert.assertEquals(2, IonUTF8.convertToUTF8Bytes(1026, outBytes, 0, 257)); + Assert.assertArrayEquals(new byte[] {-48, -126, 13, 13}, outBytes); + Assert.assertEquals(3, IonUTF8.convertToUTF8Bytes(3000, outBytes, 0, 257)); + Assert.assertArrayEquals(new byte[] {-32, -82, -72, 13}, outBytes); + Assert.assertEquals(4, IonUTF8.convertToUTF8Bytes(65536, outBytes, 0, 257)); + Assert.assertArrayEquals(new byte[] {-16, -112, -128, -128}, outBytes); + } + + @Test + public void testIsFourByteUTF8() { + Assert.assertEquals(true, IonUTF8.isFourByteUTF8(240)); + Assert.assertEquals(false, IonUTF8.isFourByteUTF8(0)); + } + + @Test + public void testIsThreeByteScalar() { + Assert.assertEquals(true, IonUTF8.isThreeByteScalar(-2147401728)); + Assert.assertEquals(false, IonUTF8.isThreeByteScalar(81920)); + } + + @Test + public void testIsSurrogate() { + Assert.assertEquals(false, IonUTF8.isSurrogate(57344)); + Assert.assertEquals(true, IonUTF8.isSurrogate(55300)); + Assert.assertEquals(false, IonUTF8.isSurrogate(0)); + } + + @Test + public void testIsLowSurrogate() { + Assert.assertEquals(true, IonUTF8.isLowSurrogate(56320)); + Assert.assertEquals(false, IonUTF8.isLowSurrogate(0)); + } + + @Test + public void testGetAs4BytesReversed() { + Assert.assertEquals(-48, IonUTF8.getAs4BytesReversed(1024)); + Assert.assertEquals(-24, IonUTF8.getAs4BytesReversed(32768)); + Assert.assertEquals(-16, IonUTF8.getAs4BytesReversed(66560)); + Assert.assertEquals(0, IonUTF8.getAs4BytesReversed(0)); + } + + @Test + public void testTwoByteScalar() { + Assert.assertEquals('\u0000', IonUTF8.twoByteScalar(0, 0)); + } + + @Test + public void testIsTwoByteScalar() { + Assert.assertEquals(true, IonUTF8.isTwoByteScalar(-2147481600)); + Assert.assertEquals(false, IonUTF8.isTwoByteScalar(2048)); + } + + @Test + public void testIsTwoByteUTF8() { + Assert.assertEquals(true, IonUTF8.isTwoByteUTF8(192)); + Assert.assertEquals(false, IonUTF8.isTwoByteUTF8(0)); + } + + @Test + public void testGetUTF8ByteCount() { + Assert.assertEquals(3, IonUTF8.getUTF8ByteCount(2048)); + Assert.assertEquals(4, IonUTF8.getUTF8ByteCount(67584)); + Assert.assertEquals(2, IonUTF8.getUTF8ByteCount(1024)); + Assert.assertEquals(1, IonUTF8.getUTF8ByteCount(0)); + } + + @Test + public void testPackBytesAfter1() { + Assert.assertEquals(-128, IonUTF8.packBytesAfter1(0, 2)); + Assert.assertEquals(-128, IonUTF8.packBytesAfter1(0, 3)); + Assert.assertEquals(-128, IonUTF8.packBytesAfter1(0, 4)); + Assert.assertEquals(-128, IonUTF8.packBytesAfter1(0, 3)); + Assert.assertEquals(-128, IonUTF8.packBytesAfter1(0, 4)); + } + + @Test + public void testGetUTF8LengthFromFirstByte() { + Assert.assertEquals(-1, IonUTF8.getUTF8LengthFromFirstByte(128)); + Assert.assertEquals(2, IonUTF8.getUTF8LengthFromFirstByte(192)); + Assert.assertEquals(4, IonUTF8.getUTF8LengthFromFirstByte(240)); + Assert.assertEquals(3, IonUTF8.getUTF8LengthFromFirstByte(224)); + Assert.assertEquals(1, IonUTF8.getUTF8LengthFromFirstByte(0)); + } + + @Test + public void testIsOneByteUTF8() { + Assert.assertEquals(false, IonUTF8.isOneByteUTF8(128)); + Assert.assertEquals(true, IonUTF8.isOneByteUTF8(0)); + } + + @Test + public void testIsThreeByteUTF8() { + Assert.assertEquals(true, IonUTF8.isThreeByteUTF8(224)); + Assert.assertEquals(false, IonUTF8.isThreeByteUTF8(0)); + } + + @Test + public void testLowSurrogate() { + Assert.assertEquals('\udc00', IonUTF8.lowSurrogate(73728)); + Assert.assertEquals('\udfc4', IonUTF8.lowSurrogate(131012)); + } + + @Test + public void testHighSurrogate() { + Assert.assertEquals('\ud808', IonUTF8.highSurrogate(73728)); + Assert.assertEquals('\ud83f', IonUTF8.highSurrogate(131012)); + } + + @Test + public void testNeedsSurrogateEncoding() { + Assert.assertEquals(true, IonUTF8.needsSurrogateEncoding(65536)); + Assert.assertEquals(false, IonUTF8.needsSurrogateEncoding(0)); + } + + @Test + public void testIsStartByte() { + Assert.assertEquals(true, IonUTF8.isStartByte(-2147483648)); + Assert.assertEquals(true, IonUTF8.isStartByte(-2147483520)); + Assert.assertEquals(false, IonUTF8.isStartByte(128)); + Assert.assertEquals(true, IonUTF8.isStartByte(0)); + } + + @Test + public void testThreeByteScalar() { + Assert.assertEquals(4227, IonUTF8.threeByteScalar(1, 2, 3)); + } + + @Test + public void testFourByteScalar() { + Assert.assertEquals(270532, IonUTF8.fourByteScalar(1, 2, 3, 4)); + } +} From ef74f881423aea237a751b7d19460a3b805e6d16 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Wed, 27 Feb 2019 18:05:55 -0800 Subject: [PATCH 051/490] Reverts replacing Timestamp internals with Calendar. The additional memory pressure caused by backing Timestamp by Calendar was noticeable by heavy users of Timestamp. Keeps the other bug fixes from that commit (see #160, #163, and most of #165). --- src/software/amazon/ion/Timestamp.java | 862 +++++++++++--------- test/software/amazon/ion/TimestampTest.java | 25 +- 2 files changed, 510 insertions(+), 377 deletions(-) diff --git a/src/software/amazon/ion/Timestamp.java b/src/software/amazon/ion/Timestamp.java index eb1bc13bbc..1dcd2587f2 100644 --- a/src/software/amazon/ion/Timestamp.java +++ b/src/software/amazon/ion/Timestamp.java @@ -68,16 +68,6 @@ *

  • {@code 2009-01-01T00:00:00.000Z} etc.
  • * * - * - *

    Date Arithmetic and Leap Years

    - * Date arithmetic is performed according to the logic provided by - * {@link Calendar#add(int, int)}. When constructed by {@link Timestamp#forCalendar(Calendar)} - * the new Timestamp and any other Timestamps that spawn from it (e.g. through - * {@link #clone()}, {@link #addDay(int)}, etc.) will use the given Calendar's date arithmetic - * rules, including its rules for determining leap years. When constructed without a - * Calendar, a default GregorianCalendar (as constructed by - * new GregorianCalendar(TimeZone.getTimeZone("UTC")) will be used. - * * @see #equals(Timestamp) * @see #compareTo(Timestamp) */ @@ -94,6 +84,18 @@ public final class Timestamp private static final int NO_SECONDS = 0; private static final BigDecimal NO_FRACTIONAL_SECONDS = null; + // 0001-01-01T00:00:00.0Z in millis + static final long MINIMUM_TIMESTAMP_IN_MILLIS = -62135769600000L; + + // 0001-01-01T00:00:00.0Z in millis + static final BigDecimal MINIMUM_TIMESTAMP_IN_MILLIS_DECIMAL = new BigDecimal(MINIMUM_TIMESTAMP_IN_MILLIS); + + // 10000T in millis, upper bound exclusive + static final long MAXIMUM_TIMESTAMP_IN_MILLIS = 253402300800000L; + + // 10000T in millis, upper bound exclusive + static final BigDecimal MAXIMUM_ALLOWED_TIMESTAMP_IN_MILLIS_DECIMAL = new BigDecimal(MAXIMUM_TIMESTAMP_IN_MILLIS); + /** * Unknown local offset from UTC. */ @@ -163,19 +165,16 @@ public boolean includes(Precision isIncluded) */ private Precision _precision; - /** - * Calendar to hold the Timestamp's year, month, day, hour, minute, second, and calendar system. Fractional - * seconds are left to {@link #_fraction}, while local offset is left to {@link #_offset}. - *

    - * This calendar instance is adjusted such that it is always in UTC; see _calendarCompensationOffsetMs - * if the original "instant" millis is needed. - */ - private final Calendar _calendar; - - /** - * Fractional seconds. Must be within range [0, 1). - */ - private BigDecimal _fraction; + // These are the time field values for the Timestamp. + // _month and _day are 1-based (0 is an invalid value for + // these in a non-null Timestamp). + private short _year; + private byte _month = 1; // Initialized to valid default + private byte _day = 1; // Initialized to valid default + private byte _hour; + private byte _minute; + private byte _second; + private BigDecimal _fraction; // fractional seconds, must be within range [0, 1) /** * Minutes offset from UTC; zero means UTC proper, @@ -183,24 +182,39 @@ public boolean includes(Precision isIncluded) */ private Integer _offset; - /** - * Number of milliseconds _calendar has been adjusted so that it is in UTC. - * Allows getMillis() and getDecimalMillis() to return the correct response. - */ - private int _calendarCompensationOffsetMs; - - // Minimum millis under the calendar system provided by the default GregorianCalendar implementation. - private static final long MINIMUM_TIMESTAMP_IN_MILLIS = Timestamp.valueOf("0001-01-01T00:00:00.000Z").getMillis(); - static final BigDecimal MINIMUM_TIMESTAMP_IN_MILLIS_DECIMAL = new BigDecimal(MINIMUM_TIMESTAMP_IN_MILLIS); - - // 10000T in millis, upper bound exclusive, under the calendar system provided by the default GregorianCalendar implementation. - private static final long UPPER_BOUND_TIMESTAMP_IN_MILLIS = Timestamp.valueOf("9999-12-31T23:59:59.999-00:00").getMillis() + 1; - static final BigDecimal UPPER_BOUND_TIMESTAMP_IN_MILLIS_DECIMAL = new BigDecimal(UPPER_BOUND_TIMESTAMP_IN_MILLIS); + // jan, feb, mar, apr, may, jun, jul, aug, sep, oct, nov, dec + // the first 0 is to make these arrays 1 based (since month values are 1-12) + private static final int[] LEAP_DAYS_IN_MONTH = { 0, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; + private static final int[] NORMAL_DAYS_IN_MONTH = { 0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; + + private static int last_day_in_month(int year, int month) { + boolean is_leap; + if ((year % 4) == 0) { + // divisible by 4 (lower 2 bits are zero) - may be a leap year + if ((year % 100) == 0) { + // and divisible by 100 - not a leap year + if ((year % 400) == 0) { + // but divisible by 400 - then it is a leap year + is_leap = true; + } + else { + is_leap = false; + } + } + else { + is_leap = true; + } + } + else { + is_leap = false; + } + return is_leap ? LEAP_DAYS_IN_MONTH[month] : NORMAL_DAYS_IN_MONTH[month]; + } /** * Applies the local time zone offset from UTC to the applicable time field * values. Depending on the local time zone offset, adjustments - * (i.e. rollover) will be made to the calendar's year, day, hour, and minute + * (i.e. rollover) will be made to the Year, Day, Hour, Minute time field * values. * * @param offset the local offset, in minutes from UTC. @@ -216,87 +230,181 @@ private void apply_offset(int offset) int hour_offset = offset / 60; int min_offset = offset - (hour_offset * 60); - // When we add hour_offset and min_offset to _calendar below, the underlying millis "instant" - // is changed. To compensate, we set _calendarCompensationOffsetMs here and use it to - // recalculate the correct millis if/when needed - if (_calendar.isSet(Calendar.ZONE_OFFSET) || _calendar.isSet(Calendar.DST_OFFSET)) { - _calendarCompensationOffsetMs = -offset * 60 * 1000; + if (offset < 0) { + _minute += min_offset; // lower the minute value by adding a negative offset + _hour += hour_offset; + if (_minute < 0) { + _minute += 60; + _hour -= 1; + } + if (_hour >= 0) return; // hour is 0-23 + _hour += 24; + _day -= 1; + if (_day >= 1) return; // day is 1-31 + // we can't do this until we've figured out the month and year: _day += last_day_in_month(_year, _month); + _month -= 1; + if (_month >= 1) { + _day += last_day_in_month(_year, _month); // now we know (when the year doesn't change + assert(_day == last_day_in_month(_year, _month)); + return; // 1-12 + } + _month += 12; + _year -= 1; + if (_year < 1) throw new IllegalArgumentException("year is less than 1"); + _day += last_day_in_month(_year, _month); // and now we know, even if the year did change + assert(_day == last_day_in_month(_year, _month)); + } + else { + _minute += min_offset; // lower the minute value by adding a negative offset + _hour += hour_offset; + if (_minute > 59) { + _minute -= 60; + _hour += 1; + } + if (_hour < 24) return; // hour is 0-23 + _hour -= 24; + _day += 1; + if (_day <= last_day_in_month(_year, _month)) return; // day is 1-31 + // we can't do this until we figure out the final month and year: _day -= last_day_in_month(_year, _month); + _day = 1; // this is always the case + _month += 1; + if (_month <= 12) { + return; // 1-12 + } + _month -= 12; + _year += 1; + if (_year > 9999) throw new IllegalArgumentException("year exceeds 9999"); + } + } + + private static byte requireByte(int value, String location) { + if (value > Byte.MAX_VALUE || value < Byte.MIN_VALUE) { + throw new IllegalArgumentException(String.format("%s of %d is out of range.", location, value)); } + return (byte) value; + } - // First, clear the offsets that are already set. Otherwise, the 'add' calls will add them in, which will - // result in a double add. - _calendar.clear(Calendar.ZONE_OFFSET); - _calendar.clear(Calendar.DST_OFFSET); - _calendar.add(Calendar.HOUR_OF_DAY, hour_offset); - _calendar.add(Calendar.MINUTE, min_offset); + private static short requireShort(int value, String location) { + if (value > Short.MAX_VALUE || value < Short.MIN_VALUE) { + throw new IllegalArgumentException(String.format("%s of %d is out of range.", location, value)); + } + return (short) value; } /** - * Create a Calendar from a number of milliseconds and the given local offset. - * @param millis a number of epoch milliseconds. - * @param localOffset a local offset in minutes. - * @return a new Calendar. + * This method uses deprecated methods from {@link java.util.Date} + * instead of {@link Calendar} so that this code can be used (more easily) + * on the mobile Java platform (which has Date but does not have Calendar). */ - private static Calendar calendarFromMillis(long millis, Integer localOffset) { - Calendar calendar = new GregorianCalendar(PrivateUtils.UTC); - calendar.clear(); - calendar.setTimeInMillis(millis); - if (localOffset != null) { - calendar.set(Calendar.ZONE_OFFSET, localOffset * 60 * 1000); + @SuppressWarnings("deprecation") + private void set_fields_from_millis(long millis) + { + if(millis < MINIMUM_TIMESTAMP_IN_MILLIS){ + throw new IllegalArgumentException("year is less than 1"); } - return calendar; + + Date date = new Date(millis); + + // The Date getters return values in the Date's time zone (i.e. the system time zone). + // The components need to be converted to UTC before being validated for exact ranges, because the offset + // conversion can affect which values are considered valid. Simply verify that the values can fit in the + // destination type here. If they do not, they would be out of range no matter the offset. + _minute = requireByte(date.getMinutes(), "Minute"); + _second = requireByte(date.getSeconds(), "Second"); + _hour = requireByte(date.getHours(), "Hour"); + _day = requireByte(date.getDate(), "Day"); + _month = requireByte(date.getMonth() + 1, "Month"); + + // Date does not correctly handle year values that represent year 0 or earlier in the system time zone through + // getYear(). This case is detected and forced to zero. + int offset = -date.getTimezoneOffset(); + if(offset < 0 && MINIMUM_TIMESTAMP_IN_MILLIS - offset > millis) { + _year = 0; + } else { + _year = requireShort(date.getYear() + 1900, "Year"); + } + + // Now apply the offset to convert the components to UTC. + apply_offset(offset); + + // Now that all components are in UTC, they may be validated for exact ranges. + this._year = checkAndCastYear(_year); + this._month = checkAndCastMonth(_month); + this._day = checkAndCastDay(_day, _year, _month); + this._hour = checkAndCastHour(_hour); + this._minute = checkAndCastMinute(_minute); + this._second = checkAndCastSecond(_second); } /** * Copies data from a {@link Calendar} into this timestamp. * Must only be called during construction due to timestamp immutabliity. * - * @param setLocalOffset if true and the given calendar has its ZONE_OFFSET set, sets the timestamp's _offset field. - * @param applyLocalOffset if true and _offset has been set to a known offset, applies the local offset to the - * timestamp's fields to convert them from local time to UTC. This should be false whenever - * the given calendar is already in UTC (such as when it was constructed from millis). + * @param cal must have at least one field set. * * @throws IllegalArgumentException if the calendar has no fields set. */ - private void setFieldsFromCalendar(Precision precision, - boolean setLocalOffset, - boolean applyLocalOffset) + private void set_fields_from_calendar(Calendar cal, + Precision precision, + boolean setLocalOffset) { _precision = precision; _offset = UNKNOWN_OFFSET; - boolean calendarHasMilliseconds = _calendar.isSet(Calendar.MILLISECOND); + boolean dayPrecision = false; + boolean calendarHasMilliseconds = cal.isSet(Calendar.MILLISECOND); switch (this._precision) { case SECOND: + this._second = checkAndCastSecond(cal.get(Calendar.SECOND)); if (calendarHasMilliseconds) { - BigDecimal millis = BigDecimal.valueOf(_calendar.get(Calendar.MILLISECOND)); + BigDecimal millis = BigDecimal.valueOf(cal.get(Calendar.MILLISECOND)); this._fraction = millis.movePointLeft(3); // convert to fraction checkFraction(precision, this._fraction); } case MINUTE: { - int offset = _calendar.get(Calendar.ZONE_OFFSET); - if (setLocalOffset) + this._hour = checkAndCastHour(cal.get(Calendar.HOUR_OF_DAY)); + this._minute = checkAndCastMinute(cal.get(Calendar.MINUTE)); + + // If this test is made before calling get(), it will return + // false even when Calendar.setTimeZone() was called. + if (setLocalOffset && cal.isSet(Calendar.ZONE_OFFSET)) { - if (_calendar.isSet(Calendar.DST_OFFSET)) { - offset += _calendar.get(Calendar.DST_OFFSET); + int offset = cal.get(Calendar.ZONE_OFFSET); + if (cal.isSet(Calendar.DST_OFFSET)) { + offset += cal.get(Calendar.DST_OFFSET); } + // convert ms to minutes _offset = offset / (1000*60); } } case DAY: + dayPrecision = true; case MONTH: + // Calendar months are 0 based, Timestamp months are 1 based + this._month = checkAndCastMonth((cal.get(Calendar.MONTH) + 1)); case YEAR: + int year; + if(cal.get(Calendar.ERA) == GregorianCalendar.AD) { + year = cal.get(Calendar.YEAR); + } + else { + year = -cal.get(Calendar.YEAR); + } + + this._year = checkAndCastYear(year); + } + + if (dayPrecision) + { + this._day = checkAndCastDay(cal.get(Calendar.DAY_OF_MONTH), _year, _month); } - if (_offset != UNKNOWN_OFFSET && applyLocalOffset) { + if (_offset != UNKNOWN_OFFSET) { // Transform our members from local time to Zulu this.apply_offset(_offset); } - // fractional seconds are ONLY tracked by the _fraction field. - _calendar.clear(Calendar.MILLISECOND); - checkCalendarYear(_calendar); } /** @@ -393,8 +501,6 @@ private Timestamp(Precision p, int zyear, int zmonth, int zday, int zhour, int zminute, int zsecond, BigDecimal frac, Integer offset, boolean shouldApplyOffset) { - _calendar = new GregorianCalendar(PrivateUtils.UTC); - _calendar.clear(); boolean dayPrecision = false; switch (p) { @@ -409,23 +515,22 @@ private Timestamp(Precision p, int zyear, int zmonth, int zday, { _fraction = frac.abs(); } - _calendar.set(Calendar.SECOND, checkAndCastSecond(zsecond)); + _second = checkAndCastSecond(zsecond); case MINUTE: - _calendar.set(Calendar.MINUTE, checkAndCastMinute(zminute)); - _calendar.set(Calendar.HOUR_OF_DAY, checkAndCastHour(zhour)); + _minute = checkAndCastMinute(zminute); + _hour = checkAndCastHour(zhour); _offset = offset; // offset must be null for years/months/days case DAY: dayPrecision = true; case MONTH: - _calendar.set(Calendar.MONTH, checkAndCastMonth(zmonth) - 1); + _month = checkAndCastMonth(zmonth); case YEAR: - _calendar.set(Calendar.YEAR, checkAndCastYear(zyear)); + _year = checkAndCastYear(zyear); } if (dayPrecision) { - checkCalendarDay(zday); - _calendar.set(Calendar.DAY_OF_MONTH, zday); + _day = checkAndCastDay(zday, zyear, zmonth); } _precision = checkFraction(p, _fraction); @@ -442,21 +547,19 @@ private Timestamp(Precision p, int zyear, int zmonth, int zday, * applied to the time components. * As such, if the given {@code offset} is non-null or zero, the resulting * Timestamp will have time values that DO NOT match the time - * parameters. A default {@link GregorianCalendar} will be used to - * perform any arithmetic operations on the resulting Timestamp. This - * method also has a behavior of precision "narrowing", detailed in the - * sub-section below. + * parameters. This method also has a behavior of precision "narrowing", + * detailed in the sub-section below. * *

    * For example, the following method calls will return Timestamps with * values (in its local time) respectively: *

    -     * createFromUtcFields(Precision.FRACTION, 2012, 2, 3, 4, 5, 6, 0.007, null)    will return 2012-02-03T04:05:06.007-00:00 (match)
    -     * createFromUtcFields(Precision.FRACTION, 2012, 2, 3, 4, 5, 6, 0.007, 0)       will return 2012-02-03T04:05:06.007+00:00 (match)
    -     * createFromUtcFields(Precision.FRACTION, 2012, 2, 3, 4, 5, 6, 0.007, 480)     will return 2012-02-03T12:05:06.007+08:00 (do not match)
    -     * createFromUtcFields(Precision.FRACTION, 2012, 2, 3, 4, 5, 6, 0.007, -480)    will return 2012-02-02T20:05:06.007-08:00 (do not match)
    -     * createFromUtcFields(Precision.FRACTION, 2012, 2, 3, 4, 5, 6, 0.007, 720)     will return 2012-02-03T16:05:06.007+12:00 (do not match)
    -     * createFromUtcFields(Precision.FRACTION, 2012, 2, 3, 4, 5, 6, 0.007, -720)    will return 2012-02-02T16:05:06.007-12:00 (do not match)
    +     * createFromUtcFields(Precision.SECOND, 2012, 2, 3, 4, 5, 6, 0.007, null)    will return 2012-02-03T04:05:06.007-00:00 (match)
    +     * createFromUtcFields(Precision.SECOND, 2012, 2, 3, 4, 5, 6, 0.007, 0)       will return 2012-02-03T04:05:06.007+00:00 (match)
    +     * createFromUtcFields(Precision.SECOND, 2012, 2, 3, 4, 5, 6, 0.007, 480)     will return 2012-02-03T12:05:06.007+08:00 (do not match)
    +     * createFromUtcFields(Precision.SECOND, 2012, 2, 3, 4, 5, 6, 0.007, -480)    will return 2012-02-02T20:05:06.007-08:00 (do not match)
    +     * createFromUtcFields(Precision.SECOND, 2012, 2, 3, 4, 5, 6, 0.007, 720)     will return 2012-02-03T16:05:06.007+12:00 (do not match)
    +     * createFromUtcFields(Precision.SECOND, 2012, 2, 3, 4, 5, 6, 0.007, -720)    will return 2012-02-02T16:05:06.007-12:00 (do not match)
          *
    * Note: All of these resulting Timestamps have the similar value (in UTC) 2012-02-03T04:05:06.007Z. * @@ -474,8 +577,8 @@ private Timestamp(Precision p, int zyear, int zmonth, int zday, * createFromUtcFields(Precision.MONTH , 2012, 2, 3, 4, 5, 6, 0.007, 0) will return 2012-02T * createFromUtcFields(Precision.DAY , 2012, 2, 3, 4, 5, 6, 0.007, 0) will return 2012-02-03T * createFromUtcFields(Precision.MINUTE , 2012, 2, 3, 4, 5, 6, 0.007, 0) will return 2012-02-03T04:05Z - * createFromUtcFields(Precision.SECOND , 2012, 2, 3, 4, 5, 6, 0.007, 0) will return 2012-02-03T04:05:06Z - * createFromUtcFields(Precision.FRACTION, 2012, 2, 3, 4, 5, 6, 0.007, 0) will return 2012-02-03T04:05:06.007Z + * createFromUtcFields(Precision.SECOND , 2012, 2, 3, 4, 5, 6, null, 0) will return 2012-02-03T04:05:06Z + * createFromUtcFields(Precision.SECOND , 2012, 2, 3, 4, 5, 6, 0.007, 0) will return 2012-02-03T04:05:06.007Z * * * @param p the desired timestamp precision. The result may have a @@ -541,50 +644,51 @@ else if (cal.isSet(Calendar.YEAR)) { else { throw new IllegalArgumentException("Calendar has no fields set"); } - _calendar = (Calendar) cal.clone(); - setFieldsFromCalendar(precision, true, APPLY_OFFSET_YES); + + set_fields_from_calendar(cal, precision, true); } private Timestamp(Calendar cal, Precision precision, BigDecimal fraction, Integer offset) { - this._calendar = cal; - setFieldsFromCalendar(precision, false, APPLY_OFFSET_NO); + set_fields_from_calendar(cal, precision, false); _fraction = fraction; if (offset != null) { _offset = offset; + apply_offset(offset); } } - private static void throwTimestampOutOfRangeError(Number millis) { - throw new IllegalArgumentException("millis: " + millis + " is outside of valid the range: from " - + MINIMUM_TIMESTAMP_IN_MILLIS_DECIMAL - + " (0001T)" - + ", inclusive, to " - + UPPER_BOUND_TIMESTAMP_IN_MILLIS_DECIMAL - + " (10000T)" - + " , exclusive"); - } private Timestamp(BigDecimal millis, Precision precision, Integer localOffset) { - if (millis == null) throw new NullPointerException("millis is null"); - // check bounds to avoid hanging when calling longValue() on decimals with large positive exponents, // e.g. 1e10000000 if(millis.compareTo(MINIMUM_TIMESTAMP_IN_MILLIS_DECIMAL) < 0 || - UPPER_BOUND_TIMESTAMP_IN_MILLIS_DECIMAL.compareTo(millis) <= 0) { + MAXIMUM_ALLOWED_TIMESTAMP_IN_MILLIS_DECIMAL.compareTo(millis) <= 0) { throwTimestampOutOfRangeError(millis); } - // quick handle integral zero long ms = isIntegralZero(millis) ? 0 : millis.longValue(); + set_fields_from_millis(ms); - _calendar = calendarFromMillis(ms, localOffset); - setFieldsFromCalendar(precision, localOffset != null, APPLY_OFFSET_NO); + switch (precision) + { + case YEAR: + _month = 1; + case MONTH: + _day = 1; + case DAY: + _hour = 0; + _minute = 0; + case MINUTE: + _second = 0; + case SECOND: + } + _offset = localOffset; // The given BigDecimal may contain greater than milliseconds precision, which is the maximum precision that // a Calendar can handle. Set the _fraction here so that extra precision (if any) is not lost. // However, don't set the fraction if the given BigDecimal does not have precision at least to the tenth of @@ -596,7 +700,7 @@ private Timestamp(BigDecimal millis, Precision precision, Integer localOffset) } else { _fraction = null; } - checkFraction(precision, _fraction); + _precision = checkFraction(precision, _fraction); } @@ -635,7 +739,32 @@ private Timestamp(BigDecimal millis, Precision precision, Integer localOffset) @Deprecated private Timestamp(BigDecimal millis, Integer localOffset) { - this(millis, Precision.SECOND, localOffset); + if (millis == null) throw new NullPointerException("millis is null"); + + // check bounds to avoid hanging when calling longValue() on decimals with large positive exponents, + // e.g. 1e10000000 + if(millis.compareTo(MINIMUM_TIMESTAMP_IN_MILLIS_DECIMAL) < 0 || + MAXIMUM_ALLOWED_TIMESTAMP_IN_MILLIS_DECIMAL.compareTo(millis) < 0) { + throwTimestampOutOfRangeError(millis); + } + + // quick handle integral zero + long ms = isIntegralZero(millis) ? 0 : millis.longValue(); + + set_fields_from_millis(ms); + + int scale = millis.scale(); + if (scale <= -3) { + this._precision = Precision.SECOND; + this._fraction = null; + } + else { + BigDecimal secs = millis.movePointLeft(3); + BigDecimal secsDown = fastRoundZeroFloor(secs); + this._fraction = secs.subtract(secsDown); + this._precision = checkFraction(Precision.SECOND, _fraction); + } + this._offset = localOffset; } private BigDecimal fastRoundZeroFloor(final BigDecimal decimal) { @@ -651,6 +780,16 @@ private boolean isIntegralZero(final BigDecimal decimal) { || (decimal.precision() - decimal.scale() <= 0); } + private static void throwTimestampOutOfRangeError(Number millis) { + throw new IllegalArgumentException("millis: " + millis + " is outside of valid the range: from " + + MINIMUM_TIMESTAMP_IN_MILLIS + + " (0001T)" + + ", inclusive, to " + + MAXIMUM_TIMESTAMP_IN_MILLIS + + " (10000T)" + + " , exclusive"); + } + /** * Creates a new Timestamp that represents the point in time that is * {@code millis} milliseconds from the epoch, with a given local offset. @@ -666,11 +805,18 @@ private boolean isIntegralZero(final BigDecimal decimal) { @Deprecated private Timestamp(long millis, Integer localOffset) { - if(millis < MINIMUM_TIMESTAMP_IN_MILLIS || millis >= UPPER_BOUND_TIMESTAMP_IN_MILLIS) { + if(millis < MINIMUM_TIMESTAMP_IN_MILLIS || millis >= MAXIMUM_TIMESTAMP_IN_MILLIS) { throwTimestampOutOfRangeError(millis); } - this._calendar = calendarFromMillis(millis, localOffset); - setFieldsFromCalendar(Precision.SECOND, localOffset != null, APPLY_OFFSET_NO); + this.set_fields_from_millis(millis); + + // fractional seconds portion + BigDecimal secs = BigDecimal.valueOf(millis).movePointLeft(3); + BigDecimal secsDown = secs.setScale(0, RoundingMode.FLOOR); + this._fraction = secs.subtract(secsDown); + this._precision = checkFraction(Precision.SECOND, _fraction); + + this._offset = localOffset; } @@ -699,8 +845,6 @@ private static IllegalArgumentException fail(CharSequence input) /** * Returns a new Timestamp that represents the point in time, precision * and local offset defined in Ion format by the {@link CharSequence}. - * A default {@link GregorianCalendar} will be used to perform any - * arithmetic operations on the resulting Timestamp. * * @param ionFormattedTimestamp * a sequence of characters that is the Ion representation of a @@ -964,15 +1108,27 @@ private static boolean isValidFollowChar(char c) { /** * Creates a copy of this Timestamp. The resulting Timestamp will - * represent the same point in time, have the same precision and local - * offset, and use the same calendar system for date arithmetic. + * represent the same point in time and has the same precision and local + * offset. *

    * {@inheritDoc} */ @Override public Timestamp clone() { - return new Timestamp((Calendar) _calendar.clone(), _precision, _fraction, _offset); + // The Copy-Constructor we're using here already expects the time field + // values to be in UTC, and that is already what we have for this + // Timestamp -- no adjustment necessary to make it local time. + return new Timestamp(_precision, + _year, + _month, + _day, + _hour, + _minute, + _second, + _fraction, + _offset, + APPLY_OFFSET_NO); } /** @@ -988,7 +1144,22 @@ private Timestamp make_localtime() ? _offset.intValue() : 0; - Timestamp localtime = clone(); + // We use a Copy-Constructor that expects the time parameters to be in + // UTC, as that's what we're supposed to have. + // As this Copy-Constructor doesn't apply local offset to the time + // field values (it assumes that the local offset is already applied to + // them), we explicitly apply the local offset to the time field values + // after we obtain the new Timestamp instance. + Timestamp localtime = new Timestamp(_precision, + _year, + _month, + _day, + _hour, + _minute, + _second, + _fraction, + _offset, + APPLY_OFFSET_NO); // explicitly apply the local offset to the time field values localtime.apply_offset(-offset); @@ -999,8 +1170,6 @@ private Timestamp make_localtime() /** * Returns a Timestamp, precise to the year, with unknown local offset. - * A default {@link GregorianCalendar} will be used to perform any - * arithmetic operations on the resulting Timestamp. *

    * This is equivalent to the corresponding Ion value {@code YYYYT}. */ @@ -1011,8 +1180,6 @@ public static Timestamp forYear(int yearZ) /** * Returns a Timestamp, precise to the month, with unknown local offset. - * A default {@link GregorianCalendar} will be used to perform any - * arithmetic operations on the resulting Timestamp. *

    * This is equivalent to the corresponding Ion value {@code YYYY-MMT}. */ @@ -1023,8 +1190,6 @@ public static Timestamp forMonth(int yearZ, int monthZ) /** * Returns a Timestamp, precise to the day, with unknown local offset. - * A default {@link GregorianCalendar} will be used to perform any - * arithmetic operations on the resulting Timestamp. *

    * This is equivalent to the corresponding Ion value {@code YYYY-MM-DD}. * @@ -1037,8 +1202,7 @@ public static Timestamp forDay(int yearZ, int monthZ, int dayZ) /** * Returns a Timestamp, precise to the minute, with a given local - * offset. A default {@link GregorianCalendar} will be used to perform any - * arithmetic operations on the resulting Timestamp. + * offset. *

    * This is equivalent to the corresponding Ion value * {@code YYYY-MM-DDThh:mm+-oo:oo}, where {@code oo:oo} represents the @@ -1059,8 +1223,6 @@ public static Timestamp forMinute(int year, int month, int day, /** * Returns a Timestamp, precise to the second, with a given local offset. - * A default {@link GregorianCalendar} will be used to perform any - * arithmetic operations on the resulting Timestamp. *

    * This is equivalent to the corresponding Ion value * {@code YYYY-MM-DDThh:mm:ss+-oo:oo}, where {@code oo:oo} represents the @@ -1081,8 +1243,6 @@ public static Timestamp forSecond(int year, int month, int day, /** * Returns a Timestamp, precise to the second, with a given local offset. - * A default {@link GregorianCalendar} will be used to perform any - * arithmetic operations on the resulting Timestamp. *

    * This is equivalent to the corresponding Ion value * {@code YYYY-MM-DDThh:mm:ss.sss+-oo:oo}, where {@code oo:oo} represents @@ -1111,14 +1271,6 @@ public static Timestamp forSecond(int year, int month, int day, /** * Returns a Timestamp that represents the point in time that is * {@code millis} milliseconds from the epoch, with a given local offset. - * A default {@link GregorianCalendar} will be used to perform any - * arithmetic operations on the resulting Timestamp. - *

    - * NOTE: this means that providing a number of milliseconds - * that was produced using a different calendar system may result in a Timestamp - * that represents a different point in time than the one that originally - * produced the milliseconds. In this case, {@link #forCalendar(Calendar)} should - * be used instead. *

    * The resulting Timestamp will be precise to the millisecond. * @@ -1139,14 +1291,7 @@ public static Timestamp forMillis(long millis, Integer localOffset) * Returns a Timestamp that represents the point in time that is * {@code millis} milliseconds (including any fractional * milliseconds) from the epoch, with a given local offset. - * A default {@link GregorianCalendar} will be used to perform any - * arithmetic operations on the resulting Timestamp. - *

    - * NOTE: this means that providing a number of milliseconds - * that was produced using a different calendar system may result in a Timestamp - * that represents a different point in time than the one that originally - * produced the milliseconds. In this case, {@link #forCalendar(Calendar)} should - * be used instead. + * *

    * The resulting Timestamp will be precise to the second if {@code millis} * doesn't contain information that is more granular than seconds. @@ -1184,8 +1329,7 @@ public static Timestamp forMillis(BigDecimal millis, Integer localOffset) /** * Converts a {@link Calendar} to a Timestamp, preserving the calendar's * time zone as the equivalent local offset when it has at least minutes - * precision. The given Calendar will be used to perform any arithmetic - * operations on the resulting Timestamp. + * precision. * * @return a Timestamp instance, with precision determined by the smallest * field set in the {@code Calendar}; @@ -1201,8 +1345,7 @@ public static Timestamp forCalendar(Calendar calendar) /** * Converts a {@link Date} to a Timestamp in UTC representing the same - * point in time. A default {@link GregorianCalendar} will be used to perform - * any arithmetic operations on the resulting Timestamp. + * point in time. *

    * The resulting Timestamp will be precise to the millisecond. * @@ -1221,8 +1364,7 @@ public static Timestamp forDateZ(Date date) /** * Converts a {@link java.sql.Timestamp} to a Timestamp in UTC representing - * the same point in time. A default {@link GregorianCalendar} will be used to perform - * any arithmetic operations on the resulting Timestamp. + * the same point in time. *

    * The resulting Timestamp will be precise to the nanosecond. * @@ -1249,8 +1391,7 @@ public static Timestamp forSqlTimestampZ(java.sql.Timestamp sqlTimestamp) /** * Returns a Timestamp representing the current time (based on the JVM - * clock), with an unknown local offset. A default {@link GregorianCalendar} - * will be used to perform any arithmetic operations on the resulting Timestamp. + * clock), with an unknown local offset. *

    * The resulting Timestamp will be precise to the millisecond. * @@ -1265,8 +1406,7 @@ public static Timestamp now() /** * Returns a Timestamp in UTC representing the current time (based on the - * the JVM clock). A default {@link GregorianCalendar} will be used to perform - * any arithmetic operations on the resulting Timestamp. + * the JVM clock). *

    * The resulting Timestamp will be precise to the millisecond. * @@ -1300,11 +1440,10 @@ public Date dateValue() return new Date(millis); } + /** * Converts the value of this Timestamp as a {@link Calendar}, in its - * local time. The resulting Calendar will have its fields set up to - * this Timestamp's precision. The maximum fractional precision supported - * by Calendar is milliseconds; any additional precision will be truncated. + * local time. *

    * Because {@link Calendar} instances are mutable, this method returns a * new instance from each call. @@ -1314,26 +1453,39 @@ public Date dateValue() */ public Calendar calendarValue() { - Calendar cal = (Calendar) _calendar.clone(); - if (_precision.includes(Precision.SECOND)) { - if (this._fraction != null) - { - int fractionalMillis = this._fraction.movePointRight(3).intValue(); - cal.set(Calendar.MILLISECOND, fractionalMillis); - } - } - if (_precision.includes(Precision.MINUTE) && _offset != null && _offset != 0) + Calendar cal = new GregorianCalendar(PrivateUtils.UTC); + + long millis = getMillis(); + Integer offset = _offset; + if (offset != null && offset != 0) { - int offsetMillis = _offset * 60 * 1000; - cal.add(Calendar.MILLISECOND, offsetMillis); + int offsetMillis = offset * 60 * 1000; + millis += offsetMillis; + cal.setTimeInMillis(millis); // Resets the offset! cal.set(Calendar.ZONE_OFFSET, offsetMillis); } - if (!_precision.includes(Precision.SECOND)) { - cal.clear(Calendar.SECOND); + else + { + cal.setTimeInMillis(millis); } - if (_fraction == null) { - cal.clear(Calendar.MILLISECOND); + + switch (_precision) { + case YEAR: + cal.clear(Calendar.MONTH); + case MONTH: + cal.clear(Calendar.DAY_OF_MONTH); + case DAY: + cal.clear(Calendar.HOUR_OF_DAY); + cal.clear(Calendar.MINUTE); + case MINUTE: + cal.clear(Calendar.SECOND); + cal.clear(Calendar.MILLISECOND); + case SECOND: + if (_fraction == null) { + cal.clear(Calendar.MILLISECOND); + } } + return cal; } @@ -1341,7 +1493,7 @@ public Calendar calendarValue() /** * Returns a number representing the Timestamp's point in time that is * the number of milliseconds (ignoring any fractional milliseconds) - * from the epoch, using this Timestamp's configured Calendar. + * from the epoch. *

    * This method will return the same result for all Timestamps representing * the same point in time, regardless of the local offset. @@ -1350,22 +1502,24 @@ public Calendar calendarValue() * number of milliseconds (ignoring any fractional * milliseconds) from the epoch (1970-01-01T00:00:00.000Z) */ + @SuppressWarnings("deprecation") public long getMillis() { - long millis = _calendar.getTimeInMillis(); + // month is 0 based for Date + long millis = Date.UTC(this._year - 1900, this._month - 1, this._day, this._hour, this._minute, this._second); if (this._fraction != null) { BigDecimal fracAsDecimal = this._fraction.movePointRight(3); int frac = isIntegralZero(fracAsDecimal) ? 0 : fracAsDecimal.intValue(); millis += frac; } - return millis + _calendarCompensationOffsetMs; + return millis; } /** * Returns a BigDecimal representing the Timestamp's point in time that is * the number of milliseconds (including any fractional milliseconds) - * from the epoch, using this Timestamp's configured Calendar. + * from the epoch. *

    * This method will return the same result for all Timestamps representing * the same point in time, regardless of the local offset. @@ -1374,6 +1528,7 @@ public long getMillis() * number of milliseconds (including any fractional * milliseconds) from the epoch (1970-01-01T00:00:00.000Z) */ + @SuppressWarnings("deprecation") public BigDecimal getDecimalMillis() { switch (this._precision) { @@ -1382,7 +1537,7 @@ public BigDecimal getDecimalMillis() case DAY: case MINUTE: case SECOND: - long millis = _calendar.getTimeInMillis() + _calendarCompensationOffsetMs; + long millis = Date.UTC(this._year - 1900, this._month - 1, this._day, this._hour, this._minute, this._second); BigDecimal dec = BigDecimal.valueOf(millis); if (_fraction != null) { dec = dec.add(this._fraction.movePointRight(3)); @@ -1438,7 +1593,7 @@ public int getYear() adjusted = make_localtime(); } } - return adjusted.getZYear(); + return adjusted._year; } @@ -1460,7 +1615,7 @@ public int getMonth() adjusted = make_localtime(); } } - return adjusted.getZMonth(); + return adjusted._month; } @@ -1480,7 +1635,7 @@ public int getDay() adjusted = make_localtime(); } } - return adjusted.getZDay(); + return adjusted._day; } @@ -1500,7 +1655,7 @@ public int getHour() adjusted = make_localtime(); } } - return adjusted.getZHour(); + return adjusted._hour; } @@ -1520,7 +1675,7 @@ public int getMinute() adjusted = make_localtime(); } } - return adjusted.getZMinute(); + return adjusted._minute; } @@ -1538,7 +1693,7 @@ public int getMinute() */ public int getSecond() { - return this.getZSecond(); + return this._second; } @@ -1557,7 +1712,7 @@ public int getSecond() */ public BigDecimal getDecimalSecond() { - BigDecimal sec = BigDecimal.valueOf(getSecond()); + BigDecimal sec = BigDecimal.valueOf(_second); if (_fraction != null) { sec = sec.add(_fraction); @@ -1574,7 +1729,7 @@ public BigDecimal getDecimalSecond() */ public int getZYear() { - return this._calendar.get(Calendar.YEAR); + return this._year; } @@ -1589,7 +1744,7 @@ public int getZYear() */ public int getZMonth() { - return this._calendar.get(Calendar.MONTH) + 1; + return this._month; } @@ -1603,7 +1758,7 @@ public int getZMonth() */ public int getZDay() { - return this._calendar.get(Calendar.DAY_OF_MONTH); + return this._day; } @@ -1617,7 +1772,7 @@ public int getZDay() */ public int getZHour() { - return this._calendar.get(Calendar.HOUR_OF_DAY); + return this._hour; } @@ -1631,7 +1786,7 @@ public int getZHour() */ public int getZMinute() { - return this._calendar.get(Calendar.MINUTE); + return this._minute; } @@ -1649,7 +1804,7 @@ public int getZMinute() */ public int getZSecond() { - return this._calendar.get(Calendar.SECOND); + return this._second; } @@ -1709,7 +1864,17 @@ public Timestamp withLocalOffset(Integer offset) { return this; } - return new Timestamp((Calendar) _calendar.clone(), precision, _fraction, offset); + + Timestamp ts = createFromUtcFields(precision, + getZYear(), + getZMonth(), + getZDay(), + getZHour(), + getZMinute(), + getZSecond(), + getZFractionalSecond(), + offset); + return ts; } @@ -1846,7 +2011,7 @@ private static void print(Appendable out, Timestamp adjusted) // so we have a real value - we'll start with the date portion // which we always have - print_digits(out, adjusted.getZYear(), 4); + print_digits(out, adjusted._year, 4); if (adjusted._precision == Precision.YEAR) { assert adjusted._offset == UNKNOWN_OFFSET; out.append("T"); @@ -1854,7 +2019,7 @@ private static void print(Appendable out, Timestamp adjusted) } out.append("-"); - print_digits(out, adjusted.getZMonth(), 2); // convert calendar months to a base 1 value + print_digits(out, adjusted._month, 2); // convert calendar months to a base 1 value if (adjusted._precision == Precision.MONTH) { assert adjusted._offset == UNKNOWN_OFFSET; out.append("T"); @@ -1862,7 +2027,7 @@ private static void print(Appendable out, Timestamp adjusted) } out.append("-"); - print_digits(out, adjusted.getZDay(), 2); + print_digits(out, adjusted._day, 2); if (adjusted._precision == Precision.DAY) { assert adjusted._offset == UNKNOWN_OFFSET; // out.append("T"); @@ -1870,13 +2035,13 @@ private static void print(Appendable out, Timestamp adjusted) } out.append("T"); - print_digits(out, adjusted.getZHour(), 2); + print_digits(out, adjusted._hour, 2); out.append(":"); - print_digits(out, adjusted.getZMinute(), 2); + print_digits(out, adjusted._minute, 2); // ok, so how much time do we have ? - if (adjusted._precision.includes(Precision.SECOND)) { + if (adjusted._precision == Precision.SECOND) { out.append(":"); - print_digits(out, adjusted.getZSecond(), 2); + print_digits(out, adjusted._second, 2); if (adjusted._fraction != null) { print_fractional_digits(out, adjusted._fraction); } @@ -1939,10 +2104,10 @@ private static void print_fractional_digits(Appendable out, BigDecimal value) //========================================================================= // Timestamp arithmetic + /** * Returns a timestamp relative to this one by the given number of - * milliseconds. Uses this Timestamp's configured Calendar to perform the - * arithmetic. + * milliseconds. *

    * This method always returns a Timestamp with the same precision as * the original. After performing the arithmetic, the resulting Timestamp's @@ -1958,10 +2123,9 @@ private static void print_fractional_digits(Appendable out, BigDecimal value) */ public final Timestamp adjustMillis(long amount) { if (amount == 0) return this; - Timestamp ts = addMillis(amount); - ts._precision = _precision; + Timestamp ts = addMillisForPrecision(amount, _precision, false); ts.clearUnusedPrecision(); - if (_precision.includes(Precision.SECOND)) { + if (ts._precision.includes(Precision.SECOND)) { // Maintain the same amount of fractional precision. if (_fraction == null) { ts._fraction = null; @@ -1977,14 +2141,14 @@ public final Timestamp adjustMillis(long amount) { /** * Returns a timestamp relative to this one by the given number of - * milliseconds. Uses this Timestamp's configured Calendar to perform the - * arithmetic. + * milliseconds. *

    * This method always returns a Timestamp with SECOND precision and a seconds * value precise at least to the millisecond. For example, adding one millisecond * to {@code 2011T} results in {@code 2011-01-01T00:00:00.001-00:00}. To receive * a Timestamp that always maintains the same precision as the original, use * {@link #adjustMillis(long)}. + * milliseconds. * * @param amount a number of milliseconds. */ @@ -1993,51 +2157,48 @@ public final Timestamp addMillis(long amount) { // Zero milliseconds are to be added, and the precision does not need to be increased. return this; } - long seconds = amount / 1000; - BigDecimal millis = BigDecimal.valueOf(amount % 1000).movePointLeft(3); + return addMillisForPrecision(amount, Precision.SECOND, true); + } + + /** + * Adds the given number of milliseconds, extending (if necessary) the resulting Timestamp to the given + * precision. + * @param amount the number of milliseconds to add. + * @param precision the precision that the Timestamp will be extended to, if it does not already include that + * precision. + * @param millisecondsPrecision true if and only if the `amount` includes milliseconds precision. If true, the + * resulting timestamp's fraction will have precision at least to the millisecond. + * @return a new Timestamp. + */ + private Timestamp addMillisForPrecision(long amount, Precision precision, boolean millisecondsPrecision) { + // When millisecondsPrecision is true, the caller must do its own short-circuiting because it must + // check the fractional precision. + if (!millisecondsPrecision && amount == 0 && _precision == precision) return this; + // This strips off the local offset, expressing our fields as if they + // were UTC. + BigDecimal millis = make_localtime().getDecimalMillis(); + millis = millis.add(BigDecimal.valueOf(amount)); + Precision newPrecision = _precision.includes(precision) ? _precision : precision; + + + Timestamp ts = new Timestamp(millis, newPrecision, _offset); + // Anything with courser-than-millis precision will have been extended + // to 3 decimal places due to use of getDecimalMillis(). Compensate for + // that by setting the scale such that it is never extended unless + // milliseconds precision is being added and the fraction does not yet + // have milliseconds precision. + int newScale = millisecondsPrecision ? 3 : 0; if (_fraction != null) { - millis = _fraction.add(millis); - } - BigDecimal newFraction; - if (BigDecimal.ONE.compareTo(millis) <= 0) { - newFraction = millis.subtract(BigDecimal.ONE); - seconds += 1; - } else if (BigDecimal.ZERO.compareTo(millis) > 0) { - newFraction = BigDecimal.ONE.add(millis); - seconds -= 1; - } else { - newFraction = millis; + newScale = Math.max(newScale, _fraction.scale()); } - Timestamp ts = addSecond(seconds); - if (ts == this) { - // When the precision does not need to be extended and there are no seconds to add, - // addSecond returns the same reference unchanged. Clone it to ensure it is not - // mutated. - ts = clone(); + if (ts._fraction != null) { + ts._fraction = newScale == 0 ? null : ts._fraction.setScale(newScale, RoundingMode.FLOOR); } - ts._fraction = newFraction; - return ts; - } - - /** - * Adds the given amount to the given {@link Calendar} field and returns a new Timestamp - * with precision set to the maximum of the current precision and the given precision. - * @param field the field. - * @param amount an amount. - * @param precision the precision corresponding to the given field. - * @return a new Timestamp instance. - */ - private Timestamp calendarAdd(int field, int amount, Precision precision) { - if (amount == 0 && _precision == precision) return this; - Timestamp timestamp = make_localtime(); - timestamp._calendar.add(field, amount); - checkCalendarYear(timestamp._calendar); - if (_offset != null) { - timestamp.apply_offset(_offset); - timestamp._offset = _offset; + if (_offset != null && _offset != 0) + { + ts.apply_offset(_offset); } - timestamp._precision = _precision.includes(precision) ? timestamp._precision : precision; - return timestamp; + return ts; } /** @@ -2046,59 +2207,21 @@ private Timestamp calendarAdd(int field, int amount, Precision precision) { private void clearUnusedPrecision() { switch (_precision) { case YEAR: - _calendar.set(Calendar.MONTH, 0); + _month = 1; case MONTH: - _calendar.set(Calendar.DAY_OF_MONTH, 1); + _day = 1; case DAY: - _calendar.set(Calendar.HOUR_OF_DAY, 0); - _calendar.set(Calendar.MINUTE, 0); + _hour = 0; + _minute = 0; case MINUTE: - _calendar.set(Calendar.SECOND, 0); + _second = 0; _fraction = null; case SECOND: } } - /** - * Adds the given amount to the given {@link Calendar} field and returns a new Timestamp - * with the same precision as the original Timestamp. - * @param field the field. - * @param amount an amount. - * @return a new Timestamp instance. - */ - private Timestamp calendarAdjust(int field, int amount) { - if (amount == 0) return this; - Timestamp ts = calendarAdd(field, amount, _precision); - ts.clearUnusedPrecision(); - return ts; - } - /** * Returns a timestamp relative to this one by the given number of seconds. - * Uses this Timestamp's configured Calendar to perform the arithmetic. - * - * @param seconds a number of seconds. - */ - private Timestamp addSecond(long seconds) { - Timestamp ts = this; - do { - int incrementalSeconds; - if (seconds > Integer.MAX_VALUE) { - incrementalSeconds = Integer.MAX_VALUE; - } else if (seconds < Integer.MIN_VALUE) { - incrementalSeconds = Integer.MIN_VALUE; - } else { - incrementalSeconds = (int)seconds; - } - ts = ts.addSecond(incrementalSeconds); - seconds -= incrementalSeconds; - } while (seconds != 0); - return ts; - } - - /** - * Returns a timestamp relative to this one by the given number of seconds. - * Uses this Timestamp's configured Calendar to perform the arithmetic. *

    * This method always returns a Timestamp with the same precision as * the original. For example, adjusting {@code 2012-04-01T00:00Z} by one @@ -2111,12 +2234,12 @@ private Timestamp addSecond(long seconds) { */ public final Timestamp adjustSecond(int amount) { - return calendarAdjust(Calendar.SECOND, amount); + long delta = (long) amount * 1000; + return adjustMillis(delta); } /** * Returns a timestamp relative to this one by the given number of seconds. - * Uses this Timestamp's configured Calendar to perform the arithmetic. *

    * This method always returns a Timestamp with SECOND precision. * For example, adding one second to {@code 2011T} results in @@ -2127,12 +2250,12 @@ public final Timestamp adjustSecond(int amount) */ public final Timestamp addSecond(int amount) { - return calendarAdd(Calendar.SECOND, amount, Precision.SECOND); + long delta = (long) amount * 1000; + return addMillisForPrecision(delta, Precision.SECOND, false); } /** * Returns a timestamp relative to this one by the given number of minutes. - * Uses this Timestamp's configured Calendar to perform the arithmetic. *

    * This method always returns a Timestamp with the same precision as * the original. For example, adjusting {@code 2012-04-01T} by one minute @@ -2145,12 +2268,12 @@ public final Timestamp addSecond(int amount) */ public final Timestamp adjustMinute(int amount) { - return calendarAdjust(Calendar.MINUTE, amount); + long delta = (long) amount * 60 * 1000; + return adjustMillis(delta); } /** * Returns a timestamp relative to this one by the given number of minutes. - * Uses this Timestamp's configured Calendar to perform the arithmetic. *

    * This method always returns a Timestamp with at least MINUTE precision. * For example, adding one minute to {@code 2011T} results in @@ -2161,12 +2284,12 @@ public final Timestamp adjustMinute(int amount) */ public final Timestamp addMinute(int amount) { - return calendarAdd(Calendar.MINUTE, amount, Precision.MINUTE); + long delta = (long) amount * 60 * 1000; + return addMillisForPrecision(delta, Precision.MINUTE, false); } /** * Returns a timestamp relative to this one by the given number of hours. - * Uses this Timestamp's configured Calendar to perform the arithmetic. *

    * This method always returns a Timestamp with the same precision as * the original. For example, adjusting {@code 2012-04-01T} by one hour @@ -2179,12 +2302,12 @@ public final Timestamp addMinute(int amount) */ public final Timestamp adjustHour(int amount) { - return calendarAdjust(Calendar.HOUR_OF_DAY, amount); + long delta = (long) amount * 60 * 60 * 1000; + return adjustMillis(delta); } /** * Returns a timestamp relative to this one by the given number of hours. - * Uses this Timestamp's configured Calendar to perform the arithmetic. *

    * This method always returns a Timestamp with at least MINUTE precision. * For example, adding one hour to {@code 2011T} results in @@ -2195,12 +2318,12 @@ public final Timestamp adjustHour(int amount) */ public final Timestamp addHour(int amount) { - return calendarAdd(Calendar.HOUR_OF_DAY, amount, Precision.MINUTE); + long delta = (long) amount * 60 * 60 * 1000; + return addMillisForPrecision(delta, Precision.MINUTE, false); } /** * Returns a timestamp relative to this one by the given number of days. - * Uses this Timestamp's configured Calendar to perform the arithmetic. *

    * This method always returns a Timestamp with the same precision as * the original. For example, adjusting {@code 2012-04T} by one day results @@ -2212,31 +2335,29 @@ public final Timestamp addHour(int amount) */ public final Timestamp adjustDay(int amount) { - return calendarAdjust(Calendar.DAY_OF_MONTH, amount); + long delta = (long) amount * 24 * 60 * 60 * 1000; + return adjustMillis(delta); } /** * Returns a timestamp relative to this one by the given number of days. - * Uses this Timestamp's configured Calendar to perform the arithmetic. - *

    - * This method always returns a Timestamp with at least DAY precision. - * For example, adding one day to {@code 2011T} results in {@code 2011-01-02T}. - * To receive a Timestamp that always maintains the same precision as the - * original, use {@link #adjustDay(int)}. * * @param amount a number of days. */ public final Timestamp addDay(int amount) { - return calendarAdd(Calendar.DAY_OF_MONTH, amount, Precision.DAY); + long delta = (long) amount * 24 * 60 * 60 * 1000; + return addMillisForPrecision(delta, Precision.DAY, false); } + // Shifting month and year are more complicated since the length of a month + // varies and we want the day-of-month to stay the same when possible. + // We rely on Calendar for the logic. + /** * Returns a timestamp relative to this one by the given number of months. * The day field may be adjusted to account for different month length and - * leap days, as required by the configured Calendar's rules. For example, - * using the default {@link GregorianCalendar}, adjusting {@code 2011-01-31} - * by one month results in {@code 2011-02-28}. + * leap days. *

    * This method always returns a Timestamp with the same precision as the * original. For example, adjusting {@code 2011T} by one month results in @@ -2248,34 +2369,42 @@ public final Timestamp addDay(int amount) */ public final Timestamp adjustMonth(int amount) { - return calendarAdjust(Calendar.MONTH, amount); + if (amount == 0) return this; + return addMonthForPrecision(amount, _precision); + } + + /** + * Adds the given number of months, extending (if necessary) the resulting Timestamp to the given + * precision. + * @param amount the number of months to add. + * @param precision the precision that the Timestamp will be extended to, if it does not already include that + * precision. + * @return a new Timestamp. + */ + private Timestamp addMonthForPrecision(int amount, Precision precision) { + Calendar cal = calendarValue(); + cal.add(Calendar.MONTH, amount); + return new Timestamp(cal, precision, _fraction, _offset); } /** * Returns a timestamp relative to this one by the given number of months. * The day field may be adjusted to account for different month length and - * leap days, as required by the configured Calendar's rules. For example, - * using the default {@link GregorianCalendar}, adding one month to - * {@code 2011-01-31} results in {@code 2011-02-28}. - *

    - * This method always returns a Timestamp with at least MONTH precision. - * For example, adding one month to {@code 2011T} results in {@code 2011-02T}. - * To receive a Timestamp that always maintains the same precision as the - * original, use {@link #adjustMonth(int)}. + * leap days. For example, adding one month to {@code 2011-01-31} + * results in {@code 2011-02-28}. * * @param amount a number of months. */ public final Timestamp addMonth(int amount) { - return calendarAdd(Calendar.MONTH, amount, Precision.MONTH); + if (amount == 0 && _precision.includes(Precision.MONTH)) return this; + return addMonthForPrecision(amount, _precision.includes(Precision.MONTH) ? _precision : Precision.MONTH); } /** * Returns a timestamp relative to this one by the given number of years. - * The day field may be adjusted to account for leap days, as required by - * the configured Calendar's rules. For example, using the default - * {@link GregorianCalendar}, adjusting {@code 2012-02-29} by one year - * results in {@code 2013-02-28}. + * The day field may be adjusted to account for leap days. For example, + * adjusting {@code 2012-02-29} by one year results in {@code 2013-02-28}. *

    * Because YEAR is the coarsest precision possible, this method always * returns a Timestamp with the same precision as the original and @@ -2289,19 +2418,18 @@ public final Timestamp adjustYear(int amount) { /** * Returns a timestamp relative to this one by the given number of years. - * The day field may be adjusted to account for leap days, as required by - * the configured Calendar's rules. For example, using the default - * {@link GregorianCalendar}, adding one year to {@code 2012-02-29} results - * in {@code 2013-02-28}. - *

    - * This method always returns a Timestamp with the same precision as - * the original. See also: {@link #adjustYear(int)}. + * The day field may be adjusted to account for leap days. For example, + * adding one year to {@code 2012-02-29} results in {@code 2013-02-28}. * * @param amount a number of years. */ public final Timestamp addYear(int amount) { - return calendarAdd(Calendar.YEAR, amount, Precision.YEAR); + if (amount == 0) return this; + + Calendar cal = calendarValue(); + cal.add(Calendar.YEAR, amount); + return new Timestamp(cal, _precision, _fraction, _offset); } @@ -2328,12 +2456,12 @@ public int hashCode() result ^= (result << 19) ^ (result >> 13); - result = prime * result + this.getZYear(); - result = prime * result + this.getZMonth(); - result = prime * result + this.getZDay(); - result = prime * result + this.getZHour(); - result = prime * result + this.getZMinute(); - result = prime * result + this.getZSecond(); + result = prime * result + this._year; + result = prime * result + this._month; + result = prime * result + this._day; + result = prime * result + this._hour; + result = prime * result + this._minute; + result = prime * result + this._second; result ^= (result << 19) ^ (result >> 13); @@ -2493,12 +2621,12 @@ public boolean equals(Timestamp t) } // so now we check the actual time value - if (this.getZYear() != t.getZYear()) return false; - if (this.getZMonth() != t.getZMonth()) return false; - if (this.getZDay() != t.getZDay()) return false; - if (this.getZHour() != t.getZHour()) return false; - if (this.getZMinute() != t.getZMinute()) return false; - if (this.getZSecond() != t.getZSecond()) return false; + if (this._year != t._year) return false; + if (this._month != t._month) return false; + if (this._day != t._day) return false; + if (this._hour != t._hour) return false; + if (this._minute != t._minute) return false; + if (this._second != t._second) return false; // and if we have a local offset, check the value here if (this._offset != null) { @@ -2506,7 +2634,6 @@ public boolean equals(Timestamp t) } // we only look at the fraction if we know that it's actually there - if ((this._fraction != null && t._fraction == null) || (this._fraction == null && t._fraction != null)) { // one of the fractions are null @@ -2519,14 +2646,6 @@ public boolean equals(Timestamp t) return this._fraction.equals(t._fraction); } - private static void checkCalendarYear(Calendar calendar) { - int year = calendar.get(Calendar.YEAR); - if (calendar.get(Calendar.ERA) == GregorianCalendar.BC) { - year *= -1; - } - checkAndCastYear(year); - } - private static short checkAndCastYear(int year) { if (year < 1 || year > 9999) @@ -2547,11 +2666,14 @@ private static byte checkAndCastMonth(int month) return (byte) month; } - private void checkCalendarDay(int day) { - int lastDayInMonth = _calendar.getActualMaximum(Calendar.DAY_OF_MONTH); - if (day > lastDayInMonth || day < _calendar.getActualMinimum(Calendar.DAY_OF_MONTH)) { - throw new IllegalArgumentException(String.format("Day %s for year %s and month %s must be between 1 and %s inclusive", day, getZYear(), getZMonth(), lastDayInMonth)); + private static byte checkAndCastDay(int day, int year, int month) + { + int lastDayInMonth = last_day_in_month(year, month); + if (day < 1 || day > lastDayInMonth) { + throw new IllegalArgumentException(String.format("Day %s for year %s and month %s must be between 1 and %s inclusive", day, year, month, lastDayInMonth)); } + + return (byte) day; } private static byte checkAndCastHour(int hour) diff --git a/test/software/amazon/ion/TimestampTest.java b/test/software/amazon/ion/TimestampTest.java index ecf86dff1c..322898aef8 100644 --- a/test/software/amazon/ion/TimestampTest.java +++ b/test/software/amazon/ion/TimestampTest.java @@ -16,7 +16,8 @@ import static software.amazon.ion.Decimal.NEGATIVE_ZERO; import static software.amazon.ion.Decimal.negativeZero; -import static software.amazon.ion.Timestamp.UPPER_BOUND_TIMESTAMP_IN_MILLIS_DECIMAL; +import static software.amazon.ion.Timestamp.MAXIMUM_ALLOWED_TIMESTAMP_IN_MILLIS_DECIMAL; +import static software.amazon.ion.Timestamp.MINIMUM_TIMESTAMP_IN_MILLIS; import static software.amazon.ion.Timestamp.MINIMUM_TIMESTAMP_IN_MILLIS_DECIMAL; import static software.amazon.ion.Timestamp.UNKNOWN_OFFSET; import static software.amazon.ion.Timestamp.UTC_OFFSET; @@ -39,6 +40,7 @@ import java.util.GregorianCalendar; import java.util.TimeZone; +import org.junit.Ignore; import org.junit.Test; import software.amazon.ion.Timestamp.Precision; @@ -792,7 +794,7 @@ public void testForMillisWithNegativeMilli() @Test public void testNewTimestampFromMinimumAllowedMillis() { - Timestamp ts = Timestamp.forMillis(MINIMUM_TIMESTAMP_IN_MILLIS_DECIMAL, PST_OFFSET); + Timestamp ts = Timestamp.forMillis(MINIMUM_TIMESTAMP_IN_MILLIS, PST_OFFSET); assertEquals("0001-01-01T00:00:00.000Z", ts.toZString()); } @@ -806,7 +808,7 @@ public void testNewMinimumTimestampFromStringAndMillisIsSame() { @Test public void testNewTimestampFromBigDecimalWithMaximumAllowedMillis() { - Timestamp ts = Timestamp.forMillis(UPPER_BOUND_TIMESTAMP_IN_MILLIS_DECIMAL.add(BigDecimal.ONE.negate()), PST_OFFSET); + Timestamp ts = Timestamp.forMillis(MAXIMUM_ALLOWED_TIMESTAMP_IN_MILLIS_DECIMAL.add(BigDecimal.ONE.negate()), PST_OFFSET); checkFields(9999, 12, 31, 15, 59, 59, new BigDecimal("0.999"), PST_OFFSET, SECOND, ts); assertEquals("9999-12-31T15:59:59.999-08:00", ts.toString()); assertEquals("9999-12-31T23:59:59.999Z", ts.toZString()); @@ -828,20 +830,20 @@ public void testNewTimestampFromBigDecimalWithMillisTooSmall() @Test(expected = IllegalArgumentException.class) public void testNewTimestampFromLongWithMillisTooSmall() { - Timestamp.forMillis(MINIMUM_TIMESTAMP_IN_MILLIS_DECIMAL.longValue() - 1, 0); + Timestamp.forMillis(MINIMUM_TIMESTAMP_IN_MILLIS - 1, 0); } @Test(expected = IllegalArgumentException.class) public void testNewTimestampFromBigDecimalWithMillisTooBig() { // Max - Timestamp.forMillis(UPPER_BOUND_TIMESTAMP_IN_MILLIS_DECIMAL, PST_OFFSET); + Timestamp.forMillis(MAXIMUM_ALLOWED_TIMESTAMP_IN_MILLIS_DECIMAL, PST_OFFSET); } @Test(expected = IllegalArgumentException.class) public void testNewTimestampFromLongWithMillisTooBig() { - Timestamp.forMillis(UPPER_BOUND_TIMESTAMP_IN_MILLIS_DECIMAL.longValue(), PST_OFFSET); + Timestamp.forMillis(MAXIMUM_ALLOWED_TIMESTAMP_IN_MILLIS_DECIMAL.longValue(), PST_OFFSET); } @Test(expected = IllegalArgumentException.class) @@ -2576,6 +2578,7 @@ public void testGregorianCalendarNonLeapYear() { assertEquals("1800-03-01", timestamp2.toString()); } + @Ignore // TODO ion-java#165 @Test public void testConstructCustomCalendarWithValidLeapYear() { // Create a purely Julian calendar, where leap years were every four years. @@ -2588,6 +2591,7 @@ public void testConstructCustomCalendarWithValidLeapYear() { assertEquals("1800-02-29T00:00Z", timestamp.toString()); } + @Ignore // TODO ion-java#165 @Test public void testCustomCalendarLeapYear() { // Create a purely Julian calendar, where leap years were every four years. @@ -2601,6 +2605,7 @@ public void testCustomCalendarLeapYear() { assertEquals("1800-02-29", timestamp2.toString()); } + @Ignore // TODO ion-java#165 @Test public void testCustomCalendarLeapYearAfterClone() { // Create a purely Julian calendar, where leap years were every four years. @@ -2614,6 +2619,7 @@ public void testCustomCalendarLeapYearAfterClone() { assertEquals("1800-02-29T23:00Z", timestamp2.toString()); } + @Ignore // TODO ion-java#165 @Test public void testCustomCalendarLeapYearAfterCalendarValue() { // Create a purely Julian calendar, where leap years were every four years. @@ -2627,6 +2633,7 @@ public void testCustomCalendarLeapYearAfterCalendarValue() { assertEquals("1800-02-29T23:59Z", timestamp2.toString()); } + @Ignore // TODO ion-java#165 @Test public void testCustomCalendarLeapYearAfterWithLocalOffset() { // Create a purely Julian calendar, where leap years were every four years. @@ -2640,6 +2647,7 @@ public void testCustomCalendarLeapYearAfterWithLocalOffset() { assertEquals("1800-02-29T00:00:00Z", timestamp2.toString()); } + @Ignore // TODO ion-java#165 @Test public void testCustomCalendarLeapYearWithChainedAdd() { // Create a purely Julian calendar, where leap years were every four years. @@ -2654,6 +2662,7 @@ public void testCustomCalendarLeapYearWithChainedAdd() { assertEquals("1800-02-29T00:00:00.000Z", timestamp2.toString()); } + @Ignore // TODO ion-java#165 @Test public void testCustomCalendarLeapYearWithChainedAdjust() { // Create a purely Julian calendar, where leap years were every four years. @@ -2668,6 +2677,7 @@ public void testCustomCalendarLeapYearWithChainedAdjust() { assertEquals("1800-02-29T00:00:00.000Z", timestamp2.toString()); } + @Ignore // TODO ion-java#165 @Test public void testCustomCalendarLeapYearWithChainedAdjustLessPrecise() { // Create a purely Julian calendar, where leap years were every four years. @@ -2896,6 +2906,7 @@ public void testMillisIsIndepedentOfOffset() throws ParseException { assertEquals(t1.getDecimalMillis(), t2.getDecimalMillis()); } + @Ignore // TODO ion-java#165 @Test public void testInstantVsTimestampMillis() { // addresses: https://github.com/amzn/ion-java/issues/165 @@ -2905,7 +2916,7 @@ public void testInstantVsTimestampMillis() { // result, the two map from milliseconds to date differently before 1582, as demonstrated below. long millisFromInstant = Instant.parse(tsText).toEpochMilli(); long millisFromTimestamp = Timestamp.valueOf(tsText).getMillis(); - assertNotEquals(millisFromInstant, millisFromTimestamp); + assertTrue(millisFromInstant != millisFromTimestamp); // However, the discrepancy can be avoided by using Timestamp.forCalendar, which always respects the given // Calendar's method for determining leap years. Timestamp ts1 = Timestamp.forCalendar(GregorianCalendar.from(Instant.parse(tsText).atZone(ZoneId.of("UTC")))); From 36c370bf658dc697ad8fbdd6fbfe7eaac911a38d Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Thu, 28 Feb 2019 17:17:41 -0800 Subject: [PATCH 052/490] Bumps version to 1.3.1. --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index df313a313d..8a9da762ad 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ dependency into your project's `pom.xml`: software.amazon.ion ion-java - 1.3.0 + 1.3.1 ``` diff --git a/pom.xml b/pom.xml index 8fa2bfbb8b..9550562224 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 software.amazon.ion ion-java - 1.3.1-SNAPSHOT + 1.3.1 bundle ${project.groupId}:${project.artifactId} From a19302d1ebcc5e8ca41d19342581364f89d09c1a Mon Sep 17 00:00:00 2001 From: David Lurton Date: Mon, 18 Mar 2019 12:08:39 -0700 Subject: [PATCH 053/490] IonValueLite.clearSymbolIdValues() performance fix (#221) * improve clearSymbolIDValues() performance for IonValueLite DOM by keeping SID presence lifecycle as a flag bit --- .../ion/impl/lite/IonContainerLite.java | 27 ++- .../amazon/ion/impl/lite/IonSymbolLite.java | 60 +++-- .../amazon/ion/impl/lite/IonValueLite.java | 94 +++++++- test/AllTests.java | 5 + .../impl/lite/SIDPresentLifecycleTest.java | 224 ++++++++++++++++++ 5 files changed, 385 insertions(+), 25 deletions(-) create mode 100644 test/software/amazon/ion/impl/lite/SIDPresentLifecycleTest.java diff --git a/src/software/amazon/ion/impl/lite/IonContainerLite.java b/src/software/amazon/ion/impl/lite/IonContainerLite.java index a77b583431..d8fe748df1 100644 --- a/src/software/amazon/ion/impl/lite/IonContainerLite.java +++ b/src/software/amazon/ion/impl/lite/IonContainerLite.java @@ -49,6 +49,7 @@ protected IonContainerLite(ContainerlessContext context, boolean isNull) IonContainerLite(IonContainerLite existing, IonContext context, boolean isStruct) { super(existing, context); + boolean retainingSIDs = false; int childCount = existing._child_count; this._child_count = childCount; // when cloning the children we establish 'this' the cloned outer container as the context @@ -76,7 +77,13 @@ protected IonContainerLite(ContainerlessContext context, boolean isNull) } } this._children[i] = copy; + retainingSIDs |= copy._isSymbolIdPresent(); } + // unfortunately due to the existing behavior in IonValueLite copy-constructor where annotation SID's are + // preserved across the copy-constructor IF they have no resolved text it means that encodings could have + // been preserved on the child - therefore the cloned children each have to be re-interrogated and the + // setting updated IF such a change has occurred. + _isSymbolIdPresent(retainingSIDs); } } @@ -432,13 +439,19 @@ public final SymbolTable getContextSymbolTable() } @Override - void clearSymbolIDValues() + boolean attemptClearSymbolIDValues() { - super.clearSymbolIDValues(); - for (int ii=0; ii 0 || isReadOnly()) { return _sid; } @@ -196,12 +199,21 @@ private int resolveSymbolId(SymbolTableProvider symbolTableProvider) SymbolToken tok = symtab.find(name); if (tok != null) { - _sid = tok.getSid(); + setSID(tok.getSid()); _set_value(tok.getText()); // Use the interned instance of the text } return _sid; } + private void setSID(int sid) + { + _sid = sid; + if (_sid > 0) + { + cascadeSIDPresentToContextRoot(); + } + } + public SymbolToken symbolValue(SymbolTableProvider symbolTableProvider) { if (isNullValue()) return null; @@ -224,15 +236,28 @@ protected boolean isIonVersionMarker() { } @Override - void clearSymbolIDValues() + boolean attemptClearSymbolIDValues() { - super.clearSymbolIDValues(); + boolean allSymbolIDsCleared = super.attemptClearSymbolIDValues(); - // Don't lose the sid if that's all we have! - if (! isNullValue() && _stringValue() != null) + // if there is no value, or there is already no SID - there is no clear action + if (isNullValue() || _sid == UNKNOWN_SYMBOL_ID) + { + // no behavior required - value has no SID value to clear + } + // if there is text, clear the SID + else if (_stringValue() != null) { _sid = UNKNOWN_SYMBOL_ID; } + else + { + // TODO [IONJAVA-579] - needs consistent handling, resolution against context symbol table + // there is not text, so we can't clear the SID. + allSymbolIDsCleared = false; + } + + return allSymbolIDsCleared; } protected void setIsIonVersionMarker(boolean isIVM) @@ -246,7 +271,6 @@ protected void setIsIonVersionMarker(boolean isIVM) _sid = ION_1_0_SID; } - @Override final void writeBodyTo(IonWriter writer, SymbolTableProvider symbolTableProvider) throws IOException diff --git a/src/software/amazon/ion/impl/lite/IonValueLite.java b/src/software/amazon/ion/impl/lite/IonValueLite.java index cc2e80de65..c0245cbcf9 100644 --- a/src/software/amazon/ion/impl/lite/IonValueLite.java +++ b/src/software/amazon/ion/impl/lite/IonValueLite.java @@ -71,6 +71,16 @@ abstract class IonValueLite protected static final int IS_IVM = 0x10; protected static final int IS_AUTO_CREATED = 0x20; protected static final int IS_SYMBOL_PRESENT = 0x40; + /** + * Symbol ID present refers to there being the possibility that the IonValueLite, or it's contained sub graph + * (e.g. if it is a {@link IonContainerLite} based sub type) contains one or more defined + * Symbol ID's (SID)'s. This flag is used to track lifecycle, such that operations that require SID's are purged + * from the IonValueLite and it's contained sub-DOM can conduct a fast evaluation rather than having to do a full + * contained graph traversal on each and every invocation. For more detail on impact see + * IONPT-8 + */ + protected static final int IS_SYMBOL_ID_PRESENT = 0x80; + private static final int ELEMENT_MASK = 0xff; protected static final int ELEMENT_SHIFT = 8; // low 8 bits is flag, upper 24 (or 48 is element id) @@ -200,6 +210,17 @@ protected final boolean _isSymbolPresent(boolean flag) { return flag; } + protected final boolean _isSymbolIdPresent() { return is_true(IS_SYMBOL_ID_PRESENT); } + protected final boolean _isSymbolIdPresent(boolean flag) { + if (flag) { + set_flag(IS_SYMBOL_ID_PRESENT); + } + else { + clear_flag(IS_SYMBOL_ID_PRESENT); + } + return flag; + } + /** * Lazy memoized symtab provider. Should be used when a call path * conditionally needs access to a value's symbol table. This provider @@ -283,6 +304,7 @@ public SymbolTable getSymbolTable() */ IonValueLite(IonValueLite existing, IonContext context) { // Symbols are *immutable* therefore a shallow copy is sufficient + boolean hasSIDsRetained = false; if (null == existing._annotations) { this._annotations = null; } else { @@ -296,9 +318,9 @@ public SymbolTable getSymbolTable() this._annotations[i] = PrivateUtils.newSymbolToken(text, UNKNOWN_SYMBOL_ID); } else { - // TODO - this is clearly wrong; however was the existing behavior as - // existing under #getAnnotationTypeSymbols(); + // TODO - IONJAVA-579 needs consistent handling, should attempt to resolve and if it cant; fail this._annotations[i] = existing._annotations[i]; + hasSIDsRetained |= this._annotations[i].getSid() > UNKNOWN_SYMBOL_ID; } } } @@ -310,6 +332,10 @@ public SymbolTable getSymbolTable() // as IonValue.clone() mandates that the returned value is mutable, regardless of the // existing 'read only' flag - we force the deep-copy back to being mutable clear_flag(IS_LOCKED); + // whilst the clone *should* guarantee symbol context is purged, the annotation behavior existing above + // under the TO DO for IONJAVA-579 does mean that SID context can be propogated through a clone, therefore + // the encoding flag has to reflect this reality + _isSymbolIdPresent(hasSIDsRetained); } public abstract void accept(ValueVisitor visitor) throws Exception; @@ -359,6 +385,10 @@ public SymbolTable getSymbolTable() _annotations[ii] = null; } } + + // although annotations have been removed from the node, which *may* mean the sub-graph from this Node is + // now without encodings... the check is expensive for container types (need to check all immediate children) + // and so we will opt to clear down encoding present in a lazy fashion (e.g. when something actually needs it) } /** @@ -475,15 +505,41 @@ public final SymbolToken getKnownFieldNameSymbol() return token; } + final boolean clearSymbolIDValues() + { + // short circuit exit - no SID's present to remove - so can exit immediately + if (!_isSymbolIdPresent()) + { + return true; + } + + boolean allSIDsRemoved = attemptClearSymbolIDValues(); + // if all the SID's have been successfully removed - the SID Present flag can be set to false + if (allSIDsRemoved) + { + // clear the symbolID status flag + _isSymbolIdPresent(false); + } + return allSIDsRemoved; + } + /** * Sets this value's symbol table to null, and erases any SIDs here and * recursively. + * + * @return true if all SID's have been successfully removed, otherwise false */ - void clearSymbolIDValues() + boolean attemptClearSymbolIDValues() { + boolean sidsRemain = false; if (_fieldName != null) { _fieldId = UNKNOWN_SYMBOL_ID; + } else if (_fieldId > UNKNOWN_SYMBOL_ID) + { + // retaining the field SID, as it couldn't be cleared due to loss of context + // TODO - for SID handling consistency; this should attempt resolution first + sidsRemain = true; } if (_annotations != null) @@ -496,6 +552,7 @@ void clearSymbolIDValues() if (annotation == null) break; String text = annotation.getText(); + // TODO - for SID handling consistency; this should attempt resolution first, not just drop entry if (text != null && annotation.getSid() != UNKNOWN_SYMBOL_ID) { _annotations[i] = @@ -503,8 +560,20 @@ void clearSymbolIDValues() } } } + + return !sidsRemain; } + protected void cascadeSIDPresentToContextRoot() { + // start with self + IonValueLite node = this; + // iterate from leaf to context root setting encoding present until either context root is hit OR a node is + // encountered that already has encodings present (and so no further propogation is required). + while (null != node && !node._isSymbolIdPresent()) { + node._isSymbolIdPresent(true); + node = node.getContainer(); + } + } final void setFieldName(String name) { @@ -525,6 +594,12 @@ final void setFieldNameSymbol(SymbolToken name) assert _fieldId == UNKNOWN_SYMBOL_ID && _fieldName == null; _fieldName = name.getText(); _fieldId = name.getSid(); + + // if a SID has been added by this operation to a previously SID-less node we have to mark upwards + // towards the context root that a SID is present + if (UNKNOWN_SYMBOL_ID != _fieldId && !_isSymbolIdPresent()) { + cascadeSIDPresentToContextRoot(); + } } public final String getFieldName() @@ -625,6 +700,19 @@ public void setTypeAnnotationSymbols(SymbolToken... annotations) else { _annotations = annotations.clone(); + + // new annotations could have SID's - so if this node is not currently marked as SID + // present then the new annotations have to be interrogated to see if they contain SID's and if they + // do - the SID Present flag must be cascaded. + if (!_isSymbolIdPresent()) { + for (SymbolToken token : _annotations) { + // upon finding first match of a symbol token containing a SID - cascade upwards and exit + if (null != token && UNKNOWN_SYMBOL_ID != token.getSid()) { + cascadeSIDPresentToContextRoot(); + break; + } + } + } } } diff --git a/test/AllTests.java b/test/AllTests.java index b5b44ef293..609b86e896 100644 --- a/test/AllTests.java +++ b/test/AllTests.java @@ -71,6 +71,7 @@ import software.amazon.ion.impl.bin.PooledBlockAllocatorProviderTest; import software.amazon.ion.impl.bin.WriteBufferTest; import software.amazon.ion.impl.lite.IonContextTest; +import software.amazon.ion.impl.lite.SIDPresentLifecycleTest; import software.amazon.ion.streaming.BadIonStreamingTest; import software.amazon.ion.streaming.BinaryStreamingTest; import software.amazon.ion.streaming.GoodIonStreamingTest; @@ -198,6 +199,10 @@ // Hash code tests HashCodeCorrectnessTest.class, HashCodeDistributionTest.class, + HashCodeDeltaCollisionTest.class, + + // DOM Lifecycle / mode tests + SIDPresentLifecycleTest.class, HashCodeDeltaCollisionTest.class }) public class AllTests diff --git a/test/software/amazon/ion/impl/lite/SIDPresentLifecycleTest.java b/test/software/amazon/ion/impl/lite/SIDPresentLifecycleTest.java new file mode 100644 index 0000000000..d62e8f89fd --- /dev/null +++ b/test/software/amazon/ion/impl/lite/SIDPresentLifecycleTest.java @@ -0,0 +1,224 @@ +package software.amazon.ion.impl.lite; + +import software.amazon.ion.IonTestCase; +import software.amazon.ion.IonValue; +import software.amazon.ion.SymbolTable; +import software.amazon.ion.UnknownSymbolException; +import software.amazon.ion.impl.PrivateUtils; +import org.junit.Test; + +/** + * Tests to ensure that the current emerged retention behaviors for SID's across the IonValueLite DOM are correctly + * preserved with the symbol ID flag / state management. As of time of authoring the observed behavior for SID + * management is as follows: + * + * ------------------------------------------------------------------------------------------------------- + * | | fieldName | annotations | value (IonSymbolLite) | + * | | text + SID | SID only | text + SID | SID only | text + SID | SID only | + * ------------------------------------------------------------------------------------------------------- + * | | N/A | N/A | N/A | N/A | clear | retain | + * | clone | clear | resolve/fail | clear | retain | clear | clear (?)| + * | clearSymbolIDValues | clear | retain | clear | clear [null] | clear | retain | + * | set | retain | retain | retain | retain | N/A | N/A | + * | get | ephemeral | ephemeral | ephemeral | N/A | ephemeral | memoizes | + * ------------------------------------------------------------------------------------------------------- + * + */ +public class SIDPresentLifecycleTest extends IonTestCase { + + @Test + public void testFieldsWithSID() { + IonStringLite value = (IonStringLite) system().newString("Test Value"); + // a new value should create without SID presence as it has no annotations + assertFalse(value._isSymbolIdPresent()); + value.setFieldNameSymbol(PrivateUtils.newSymbolToken("Symbol-With-Sid", 12)); + // setting a field with a SID means that SID Present now must be true + assertTrue(value._isSymbolIdPresent()); + + // test clone of the value results in no SID present + assertFalse(value.clone()._isSymbolIdPresent()); + + // verify detaching from the container clears the SID present flag + value.detachFromContainer(); + assertFalse(value._isSymbolIdPresent()); + + //verify that detaching still retains symbols - as SID is still present + value.setFieldNameSymbol(PrivateUtils.newSymbolToken(13)); + assertTrue(value._isSymbolIdPresent()); + value.clearSymbolIDValues(); + // NOTE: SID is still present as clearSymbolIDValues doesn't clear SID-only tokens + assertEquals(13, value.getFieldNameSymbol().getSid()); + assertTrue(value._isSymbolIdPresent()); + } + + @Test + public void testAnnotationsWithSIDs() { + IonStringLite value = (IonStringLite) system().newString("Test Value 2"); + // a new value should create without any SID's as it has no annotations + assertFalse(value._isSymbolIdPresent()); + value.setTypeAnnotationSymbols( + PrivateUtils.newSymbolToken("Symbol-Without-SID", -1), + PrivateUtils.newSymbolToken("Symbol-With-SID", 99)); + // setting an annotation with a SID means that SID present now must be true + assertTrue(value._isSymbolIdPresent()); + + // test clone of the value (given no *unresolvable* SID's) results in no SID's present + assertFalse(value.clone()._isSymbolIdPresent()); + + // ensure that setting annotations that don't have SIDs doesn't change the SID present status (need to wait for + // an explicit action such as detatch from container or clone). + value.setTypeAnnotationSymbols(PrivateUtils.newSymbolToken("Symbol-Without-Sid", -1)); + assertTrue(value._isSymbolIdPresent()); + + // verify detaching from the container clears the SID Present flag + value.detachFromContainer(); + assertFalse(value._isSymbolIdPresent()); + + // verify that where SID context preserves through the clone due to SID context not being known (believed a bug) + // that the SID present flag is *retained* + value.setTypeAnnotationSymbols(PrivateUtils.newSymbolToken(101)); + assertTrue(value.clone()._isSymbolIdPresent()); + } + + @Test + public void testPropagationAcrossGraph() { + IonStructLite struct = (IonStructLite) system().newEmptyStruct(); + assertFalse(struct._isSymbolIdPresent()); + + // 1. test addition of child by field WITHOUT SID causes no propagation + IonStringLite keyValue1 = (IonStringLite) system().newString("Foo"); + struct.add(PrivateUtils.newSymbolToken("field_1", -1), keyValue1); + assertFalse(struct._isSymbolIdPresent()); + assertFalse(keyValue1._isSymbolIdPresent()); + + // 2. test addition of child by field WITH SID but also with field text causes no propagation + // (struct only takes field name if present and strips SID) + IonStringLite keyValue2 = (IonStringLite) system().newString("Bar"); + struct.add(PrivateUtils.newSymbolToken("field_2", 87), keyValue2); + assertFalse(struct._isSymbolIdPresent()); + assertFalse(keyValue1._isSymbolIdPresent()); + assertFalse(keyValue2._isSymbolIdPresent()); + + // 3. test addition of child by field with SID ONLY causes propagation due to retention of field SID + IonStringLite keyValue3 = (IonStringLite) system().newString("Car"); + struct.add(PrivateUtils.newSymbolToken(76), keyValue3); + assertTrue(struct._isSymbolIdPresent()); + assertFalse(keyValue1._isSymbolIdPresent()); + assertFalse(keyValue2._isSymbolIdPresent()); + assertTrue(keyValue3._isSymbolIdPresent()); + + // 4. test explicitly annotating a field propagates + IonStructLite struct2 = (IonStructLite) system().newEmptyStruct(); + IonStringLite keyValue4 = (IonStringLite) system().newString("But"); + struct2.add("field_1", keyValue4); + keyValue4.setTypeAnnotationSymbols(PrivateUtils.newSymbolToken("Lah", 13)); + assertTrue(struct2._isSymbolIdPresent()); + + // 5. test clone propagates clearing of status. NOTE - clone with an unresolvable SID for a field ID fails + try { + struct.clone(); + fail("clone was expected NOT to succeed due to behavior at time of writing where unresolvable SID's fail"); + } catch (UnknownSymbolException use) { + // this is expected??! until someone fixed the TO DO in IonValueLite#getFieldName + } + + // remove the field without SID such that the clone can be enacted. + struct.remove_child(2); + + IonStructLite clonedStruct = struct.clone(); + assertFalse(clonedStruct._isSymbolIdPresent()); + for (IonValue value : clonedStruct) { + assertFalse(((IonValueLite) value)._isSymbolIdPresent()); + } + + // 6. ensure clearing symbols propogates from root to impacted leaves + // add back in field ID (SID) only child before clearing to test SID retained + struct.add(PrivateUtils.newSymbolToken(76), keyValue3); + struct.clearSymbolIDValues(); + assertTrue(struct._isSymbolIdPresent()); + assertFalse(keyValue1._isSymbolIdPresent()); + assertFalse(keyValue2._isSymbolIdPresent()); + assertTrue(keyValue3._isSymbolIdPresent()); + + // 7. ensure that nested SID-only annotation clone behavior propogates to cloned root. + IonStructLite outer = (IonStructLite) system().newEmptyStruct(); + IonListLite middle = (IonListLite) system().newEmptyList(); + IonIntLite inner = (IonIntLite) system().newInt(10); + middle.add(inner); + outer.put("foo", middle); + // now add a SID only annotation to inner + inner.setTypeAnnotationSymbols(PrivateUtils.newSymbolToken(99)); + // verify that all components are signaling SID present + assertTrue(outer._isSymbolIdPresent()); + assertTrue(middle._isSymbolIdPresent()); + assertTrue(inner._isSymbolIdPresent()); + // conduct a clone and verify all cloned components have retained SID + IonStructLite clonedOuter = outer.clone(); + assertTrue(clonedOuter._isSymbolIdPresent()); + IonListLite clonedMiddle = (IonListLite) outer.get("foo"); + assertTrue(clonedMiddle._isSymbolIdPresent()); + assertTrue(clonedMiddle.get_child(0)._isSymbolIdPresent()); + } + + @Test + public void testSymbolValueLifecyle() { + // test creation from SymbolToken + IonSymbolLite symbolLite = (IonSymbolLite) system().newSymbol(PrivateUtils.newSymbolToken("version", 5)); + // SID isn't present as it is stripped if there is text present + assertFalse(symbolLite._isSymbolIdPresent()); + + IonListLite container1 = (IonListLite) system().newEmptyList(); + container1.add(symbolLite); + // test propagation of context when symbol context is added + assertEquals(5, symbolLite.symbolValue().getSid()); + // as the getSymbolId memoizes SID into the entity SID present must be set + assertTrue(symbolLite._isSymbolIdPresent()); + // check memoization propagates to container + assertTrue(container1._isSymbolIdPresent()); + container1.clearSymbolIDValues(); + assertFalse(container1._isSymbolIdPresent()); + assertFalse(symbolLite._isSymbolIdPresent()); + + IonSymbolLite symbolSIDOnly = (IonSymbolLite) system().newSymbol(PrivateUtils.newSymbolToken(321)); + // SID Present is preserved when only SID is present + assertTrue(symbolSIDOnly._isSymbolIdPresent()); + IonListLite container2 = (IonListLite) system().newEmptyList(); + container2.add(symbolSIDOnly); + assertTrue(container2._isSymbolIdPresent()); + // clear symbol ID values won't clear the symbolSIDOnly SID value - therefore container2 must still remain + // marked as having a SID present + container2.clearSymbolIDValues(); + assertTrue(container2._isSymbolIdPresent()); + assertTrue(symbolSIDOnly._isSymbolIdPresent()); + } + + @Test + public void testSIDRetentionAcrossObjectHierarchy() { + // field ID's can be retained on Container - so ensure these are retained. + IonStructLite outer = (IonStructLite) system().newEmptyStruct(); + IonListLite middle = (IonListLite) system().newEmptyList(); + IonStringLite inner = (IonStringLite) system().newString("A"); + outer.add(PrivateUtils.newSymbolToken(321), middle); + middle.add(inner); + inner.setTypeAnnotationSymbols(PrivateUtils.newSymbolToken("B", 123)); + assertTrue(outer._isSymbolIdPresent()); + assertTrue(middle._isSymbolIdPresent()); + assertTrue(inner._isSymbolIdPresent()); + outer.clearSymbolIDValues(); + assertTrue(outer._isSymbolIdPresent()); + assertTrue(middle._isSymbolIdPresent()); + assertFalse(inner._isSymbolIdPresent()); + + // field ID's can be retained on IonSymbolLite - so ensure these are retained as well. + IonStructLite container = (IonStructLite) system().newEmptyStruct(); + IonSymbolLite symbol = (IonSymbolLite) system().newSymbol(PrivateUtils.newSymbolToken("Foo", 123)); + container.add(PrivateUtils.newSymbolToken(321), symbol); + container.clearSymbolIDValues(); + assertEquals(SymbolTable.UNKNOWN_SYMBOL_ID, symbol.symbolValue().getSid()); + assertEquals("Foo", symbol.stringValue()); + assertEquals(321, symbol.getFieldNameSymbol().getSid()); + assertTrue(symbol._isSymbolIdPresent()); + + assertTrue(container._isSymbolIdPresent()); + } +} From 40e7a3c946e47a6e0f9d1749fa404fb01a435bef Mon Sep 17 00:00:00 2001 From: Fernando Raganhan Barbosa Date: Thu, 28 Mar 2019 13:57:14 -0700 Subject: [PATCH 054/490] Gzip memleak fix (#224) Gzip memleak fix This commit addresses the memleak caused by auto detecting and uncompressing gzipped ion data by either: * Fixing the issue when possible, example `IonSystem#singleValue` * Deprecating methods where a fix is not feasible providing a safe alternative, example `IonSystem#iterate(InputStream)` The alternative method takes in an `IonReader` which the user is responsible for closing. `IonReader` will close all used resources including the Gzip stream used to decompress which is hidden from users in the unsafe deprecated methods. Issues: * https://github.com/amzn/ion-java/issues/210 * https://github.com/amzn/ion-java/issues/198 --- src/software/amazon/ion/IonLoader.java | 26 ++++++ src/software/amazon/ion/IonSystem.java | 37 ++++++++- .../amazon/ion/impl/PrivateIonSystem.java | 7 +- .../amazon/ion/impl/lite/IonLoaderLite.java | 28 +++++-- .../amazon/ion/impl/lite/IonSystemLite.java | 36 +++++--- ...ByteArrayIteratorSystemProcessingTest.java | 2 +- ...aryStreamIteratorSystemProcessingTest.java | 2 +- test/software/amazon/ion/IonSystemTest.java | 83 +++++++++++++++++++ ...ByteArrayIteratorSystemProcessingTest.java | 2 +- ...extStreamIteratorSystemProcessingTest.java | 2 +- .../amazon/ion/impl/IonWriterTestCase.java | 2 +- 11 files changed, 194 insertions(+), 33 deletions(-) diff --git a/src/software/amazon/ion/IonLoader.java b/src/software/amazon/ion/IonLoader.java index 9b6e7819c5..fe91b340ef 100644 --- a/src/software/amazon/ion/IonLoader.java +++ b/src/software/amazon/ion/IonLoader.java @@ -125,6 +125,11 @@ public IonDatagram load(byte[] ionData) /** + *

    + * WARNING: Will cause a memory leak when reading a gzipped stream, use + * {@link IonLoader#load(IonReader)} instead. + *

    + * * Loads an entire stream of Ion data into a single datagram, * detecting whether it's text or binary data. *

    @@ -144,7 +149,28 @@ public IonDatagram load(byte[] ionData) * @throws IonException if there's a syntax error in the Ion content. * @throws IOException if reading from the specified input stream results * in an IOException. + * + * @deprecated Will cause a memory leak when reading a gzipped stream. + * Use {@link IonLoader#load(IonReader)} instead. */ + @Deprecated public IonDatagram load(InputStream ionData) throws IonException, IOException; + + + /** + * Loads an entire stream of Ion data into a single datagram, + * detecting whether it's text or binary data. + *

    + * The specified reader remains open after this method returns. + *

    + * + * @param reader @param reader source of the Ion data to load. + * + * @return a datagram containing all the values on the reader; not null. + * + * @throws NullPointerException if reader is null. + * @throws IonException if there's a syntax error in the Ion content. + */ + public IonDatagram load(IonReader reader) throws IonException; } diff --git a/src/software/amazon/ion/IonSystem.java b/src/software/amazon/ion/IonSystem.java index 5c6da86b36..57c2e92987 100644 --- a/src/software/amazon/ion/IonSystem.java +++ b/src/software/amazon/ion/IonSystem.java @@ -236,6 +236,11 @@ public SymbolTable newSharedSymbolTable(String name, /** + *

    + * WARNING: Will cause a memory leak when reading a gzipped stream, use + * {@link IonSystem#iterate(IonReader)} instead. + *

    + * * Creates an iterator over a stream of Ion data, * detecting whether it's text or binary data. * Values returned by the iterator have no container. @@ -262,10 +267,13 @@ public SymbolTable newSharedSymbolTable(String name, * * @throws NullPointerException if ionData is null. * @throws IonException if the source throws {@link IOException}. + * + * @deprecated Will cause a memory leak when reading a gzipped stream. + * Use {@link IonSystem#iterate(IonReader)} instead. */ + @Deprecated public Iterator iterate(InputStream ionData); - /** * Creates an iterator over a string containing Ion text data. * Values returned by the iterator have no container. @@ -281,8 +289,12 @@ public SymbolTable newSharedSymbolTable(String name, */ public Iterator iterate(String ionText); - /** + *

    + * WARNING: Will cause a memory leak when reading a gzipped byte[], use + * {@link IonSystem#iterate(IonReader)} instead. + *

    + * * Creates an iterator over Ion data. * Values returned by the iterator have no container. *

    @@ -299,9 +311,30 @@ public SymbolTable newSharedSymbolTable(String name, * @return a new iterator instance. * * @throws NullPointerException if ionData is null. + * + * @deprecated Will cause a memory leak when reading a gzipped byte[]. + * Use {@link IonSystem#iterate(IonReader)} instead. */ + @Deprecated public Iterator iterate(byte[] ionData); + /** + *

    + * Creates an iterator over Ion data. + * Values returned by the iterator have no container. + *

    + *

    + * The iterator will automatically consume Ion system IDs and local symbol + * tables; they will not be returned by the iterator. + *

    + * + * @param reader source of the Ion data to iterate over. + * + * @return a new iterator instance. + * + * @throws NullPointerException if reader is null. + */ + public Iterator iterate(IonReader reader); /** * Extracts a single value from Ion text data. diff --git a/src/software/amazon/ion/impl/PrivateIonSystem.java b/src/software/amazon/ion/impl/PrivateIonSystem.java index 640d0ae6f8..f2d55b2f60 100644 --- a/src/software/amazon/ion/impl/PrivateIonSystem.java +++ b/src/software/amazon/ion/impl/PrivateIonSystem.java @@ -45,12 +45,7 @@ public interface PrivateIonSystem */ public Iterator systemIterate(Reader ionText); - public Iterator systemIterate(byte[] ionData); - - /** - * TODO Must correct amzn/ion-java#14 before exposing this or using from public API. - */ - public Iterator systemIterate(InputStream ionData); + public Iterator systemIterate(IonReader reader); public IonReader newSystemReader(Reader ionText); diff --git a/src/software/amazon/ion/impl/lite/IonLoaderLite.java b/src/software/amazon/ion/impl/lite/IonLoaderLite.java index 1c50234e65..8dd6d5821e 100644 --- a/src/software/amazon/ion/impl/lite/IonLoaderLite.java +++ b/src/software/amazon/ion/impl/lite/IonLoaderLite.java @@ -122,14 +122,19 @@ public IonDatagram load(Reader ionText) throws IonException, IOException public IonDatagram load(byte[] ionData) throws IonException { + IonReader reader = makeReader(_catalog, ionData, 0, ionData.length, _lstFactory); try { - IonReader reader = makeReader(_catalog, ionData, 0, ionData.length, _lstFactory); - IonDatagramLite datagram = load_helper(reader); - return datagram; + return load(reader); } - catch (IOException e) { - throw new IonException(e); + finally { + try { + reader.close(); + } + catch (IOException e) { + throw new IonException(e); + } } + } public IonDatagram load(InputStream ionData) @@ -137,8 +142,7 @@ public IonDatagram load(InputStream ionData) { try { IonReader reader = makeReader(_catalog, ionData, _lstFactory); - IonDatagramLite datagram = load_helper(reader); - return datagram; + return load(reader); } catch (IonException e) { IOException io = e.causeOfType(IOException.class); @@ -147,4 +151,14 @@ public IonDatagram load(InputStream ionData) } } + public IonDatagram load(IonReader reader) throws IonException + { + try { + IonDatagramLite datagram = load_helper(reader); + return datagram; + } + catch (IOException e) { + throw new IonException(e); + } + } } diff --git a/src/software/amazon/ion/impl/lite/IonSystemLite.java b/src/software/amazon/ion/impl/lite/IonSystemLite.java index ff74dd44ee..db49fd6688 100644 --- a/src/software/amazon/ion/impl/lite/IonSystemLite.java +++ b/src/software/amazon/ion/impl/lite/IonSystemLite.java @@ -179,9 +179,9 @@ public Iterator iterate(Reader ionText) public Iterator iterate(InputStream ionData) { + // This method causes a memory leak when reading a gzipped stream, see deprecation notice. IonReader reader = makeReader(_catalog, ionData, _lstFactory); - ReaderIterator iterator = new ReaderIterator(this, reader); - return iterator; + return iterate(reader); } public Iterator iterate(String ionText) @@ -193,7 +193,13 @@ public Iterator iterate(String ionText) public Iterator iterate(byte[] ionData) { + // This method causes a memory leak when reading a gzipped stream, see deprecation notice. IonReader reader = makeReader(_catalog, ionData, _lstFactory); + return iterate(reader); + } + + public Iterator iterate(IonReader reader) + { ReaderIterator iterator = new ReaderIterator(this, reader); return iterator; } @@ -494,8 +500,19 @@ public IonValue singleValue(String ionText) public IonValue singleValue(byte[] ionData) { - Iterator it = iterate(ionData); - return singleValue(it); + IonReader reader = newReader(ionData); + try { + Iterator it = iterate(reader); + return singleValue(it); + } + finally { + try { + reader.close(); + } + catch (IOException e) { + throw new IonException(e); + } + } } protected IonSymbolLite newSystemIdSymbol(String ionVersionMarker) @@ -756,16 +773,9 @@ public Iterator systemIterate(String ionText) return PrivateUtils.iterate(this, ir); } - public Iterator systemIterate(InputStream ionData) + public Iterator systemIterate(IonReader reader) { - IonReader ir = newSystemReader(ionData); - return PrivateUtils.iterate(this, ir); - } - - public Iterator systemIterate(byte[] ionData) - { - IonReader ir = newSystemReader(ionData); - return PrivateUtils.iterate(this, ir); + return PrivateUtils.iterate(this, reader); } diff --git a/test/software/amazon/ion/BinaryByteArrayIteratorSystemProcessingTest.java b/test/software/amazon/ion/BinaryByteArrayIteratorSystemProcessingTest.java index b85cba9ac1..7602be6b37 100644 --- a/test/software/amazon/ion/BinaryByteArrayIteratorSystemProcessingTest.java +++ b/test/software/amazon/ion/BinaryByteArrayIteratorSystemProcessingTest.java @@ -41,6 +41,6 @@ protected Iterator iterate() @Override protected Iterator systemIterate() { - return system().systemIterate(myBytes); + return system().systemIterate(system().newSystemReader(myBytes)); } } diff --git a/test/software/amazon/ion/BinaryStreamIteratorSystemProcessingTest.java b/test/software/amazon/ion/BinaryStreamIteratorSystemProcessingTest.java index 381a01a277..9188d92388 100644 --- a/test/software/amazon/ion/BinaryStreamIteratorSystemProcessingTest.java +++ b/test/software/amazon/ion/BinaryStreamIteratorSystemProcessingTest.java @@ -45,6 +45,6 @@ protected Iterator iterate() @Override protected Iterator systemIterate() { - return system().systemIterate(myStream); + return system().systemIterate(system().newSystemReader(myStream)); } } diff --git a/test/software/amazon/ion/IonSystemTest.java b/test/software/amazon/ion/IonSystemTest.java index c1ca0748e3..f0170c82c9 100644 --- a/test/software/amazon/ion/IonSystemTest.java +++ b/test/software/amazon/ion/IonSystemTest.java @@ -341,4 +341,87 @@ public void testGzipDetection() checkGzipDetection(binaryBytes); checkGzipDetection(gzipBinaryBytes); } + + private byte[] toBinaryIon(String text) throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IonReader ionReader = system().newReader(text); + IonWriter ionWriter = system().newBinaryWriter(out); + ionWriter.writeValues(ionReader); + ionWriter.close(); + ionReader.close(); + return out.toByteArray(); + } + + @Test + public void singleValueTextGzip() throws IOException + { + String ionText = "1234"; + byte[] textBytes = PrivateUtils.utf8(ionText); + byte[] gzipTextBytes = gzip(textBytes); + IonInt ionValue = (IonInt) system().singleValue(gzipTextBytes); + + assertEquals(1234, ionValue.intValue()); + } + + @Test + public void singleValueBinaryGzip() throws IOException + { + String ionText = "1234"; + byte[] binaryIon = toBinaryIon(ionText); + byte[] gzipBytes = gzip(binaryIon); + IonInt ionValue = (IonInt) system().singleValue(gzipBytes); + + assertEquals(1234, ionValue.intValue()); + } + + @Test + public void iterateReader() throws IOException + { + String ionText = "1 2 3 4"; + IonReader ionReader = system().newReader(ionText); + + Iterator iterate = system().iterate(ionReader); + + assertTrue(iterate.hasNext()); + assertEquals(1, ((IonInt) iterate.next()).intValue()); + + assertTrue(iterate.hasNext()); + assertEquals(2, ((IonInt) iterate.next()).intValue()); + + assertTrue(iterate.hasNext()); + assertEquals(3, ((IonInt) iterate.next()).intValue()); + + assertTrue(iterate.hasNext()); + assertEquals(4, ((IonInt) iterate.next()).intValue()); + + assertFalse(iterate.hasNext()); + + ionReader.close(); + } + + @Test + public void systemIterateReader() throws IOException + { + String ionText = "1 2 3 4"; + IonReader ionReader = system().newSystemReader(ionText); + + Iterator iterate = system().systemIterate(ionReader); + + assertTrue(iterate.hasNext()); + assertEquals(1, ((IonInt) iterate.next()).intValue()); + + assertTrue(iterate.hasNext()); + assertEquals(2, ((IonInt) iterate.next()).intValue()); + + assertTrue(iterate.hasNext()); + assertEquals(3, ((IonInt) iterate.next()).intValue()); + + assertTrue(iterate.hasNext()); + assertEquals(4, ((IonInt) iterate.next()).intValue()); + + assertFalse(iterate.hasNext()); + + ionReader.close(); + } } diff --git a/test/software/amazon/ion/TextByteArrayIteratorSystemProcessingTest.java b/test/software/amazon/ion/TextByteArrayIteratorSystemProcessingTest.java index 5cc4ed4b98..2115881ff4 100644 --- a/test/software/amazon/ion/TextByteArrayIteratorSystemProcessingTest.java +++ b/test/software/amazon/ion/TextByteArrayIteratorSystemProcessingTest.java @@ -40,6 +40,6 @@ protected Iterator iterate() @Override protected Iterator systemIterate() { - return system().systemIterate(myBytes); + return system().systemIterate(system().newSystemReader(myBytes)); } } diff --git a/test/software/amazon/ion/TextStreamIteratorSystemProcessingTest.java b/test/software/amazon/ion/TextStreamIteratorSystemProcessingTest.java index 50a189e8ac..a716a80ef7 100644 --- a/test/software/amazon/ion/TextStreamIteratorSystemProcessingTest.java +++ b/test/software/amazon/ion/TextStreamIteratorSystemProcessingTest.java @@ -45,6 +45,6 @@ protected Iterator iterate() @Override protected Iterator systemIterate() { - return system().systemIterate(myStream); + return system().systemIterate(system().newSystemReader(myStream)); } } diff --git a/test/software/amazon/ion/impl/IonWriterTestCase.java b/test/software/amazon/ion/impl/IonWriterTestCase.java index 6cf30dfbc3..79690fbfb4 100644 --- a/test/software/amazon/ion/impl/IonWriterTestCase.java +++ b/test/software/amazon/ion/impl/IonWriterTestCase.java @@ -810,7 +810,7 @@ Iterator systemIterateOutput() throws Exception { byte[] data = outputByteArray(); - Iterator it = system().systemIterate(data); + Iterator it = system().systemIterate(system().newSystemReader(data)); return it; } From d5939190808b815ec940ef5d45db0d39e9c3fb63 Mon Sep 17 00:00:00 2001 From: Fernando Raganhan Barbosa Date: Thu, 11 Apr 2019 11:37:21 -0700 Subject: [PATCH 055/490] Import of Amazon IonJava with com.amazon.ion Re-importing ion-java to keep the original com.amazon.ion groupId and java package name to maintain naming consistency with other Amazon open source projects. Also updates all copyright notices --- .classpath | 8 - .gitignore | 3 +- .gitmodules | 1 - .project | 15 - .settings/AllTests.launch | 18 - .../de.loskutov.anyedit.AnyEditTools.prefs | 13 - .settings/org.eclipse.jdt.core.prefs | 354 -- .settings/org.eclipse.jdt.ui.prefs | 116 - .settings/org.eclipse.wst.validation.prefs | 6 - .travis.yml | 32 +- CODE_OF_CONDUCT.md | 6 +- CONTRIBUTING.md | 16 +- LICENSE | 2 +- NOTICE | 2 +- README.md | 10 +- ion-tests | 2 +- pom.xml | 51 +- .../amazon/ion/ContainedValueException.java | 40 + src/{software => com}/amazon/ion/Decimal.java | 39 +- src/com/amazon/ion/EmptySymbolException.java | 35 + .../amazon/ion/IntegerSize.java | 17 +- .../ion/InvalidSystemSymbolException.java | 42 + src/com/amazon/ion/IonBinaryWriter.java | 85 + src/{software => com}/amazon/ion/IonBlob.java | 15 +- src/{software => com}/amazon/ion/IonBool.java | 15 +- .../amazon/ion/IonCatalog.java | 21 +- src/{software => com}/amazon/ion/IonClob.java | 15 +- .../amazon/ion/IonContainer.java | 15 +- .../amazon/ion/IonDatagram.java | 92 +- .../amazon/ion/IonDecimal.java | 15 +- .../amazon/ion/IonException.java | 15 +- .../amazon/ion/IonFloat.java | 15 +- src/{software => com}/amazon/ion/IonInt.java | 15 +- src/{software => com}/amazon/ion/IonList.java | 15 +- .../amazon/ion/IonLoader.java | 15 +- src/{software => com}/amazon/ion/IonLob.java | 15 +- .../amazon/ion/IonMutableCatalog.java | 15 +- src/{software => com}/amazon/ion/IonNull.java | 15 +- .../amazon/ion/IonNumber.java | 17 +- .../amazon/ion/IonReader.java | 59 +- .../amazon/ion/IonSequence.java | 15 +- src/{software => com}/amazon/ion/IonSexp.java | 15 +- .../amazon/ion/IonString.java | 15 +- .../amazon/ion/IonStruct.java | 15 +- .../amazon/ion/IonSymbol.java | 30 +- .../amazon/ion/IonSystem.java | 60 +- src/{software => com}/amazon/ion/IonText.java | 15 +- src/com/amazon/ion/IonTextReader.java | 35 + .../amazon/ion/IonTimestamp.java | 17 +- src/{software => com}/amazon/ion/IonType.java | 17 +- .../amazon/ion/IonValue.java | 43 +- .../amazon/ion/IonWriter.java | 73 +- .../amazon/ion/NullValueException.java | 16 +- .../amazon/ion/OffsetSpan.java | 19 +- src/com/amazon/ion/RawValueSpanProvider.java | 59 + .../amazon/ion/ReadOnlyValueException.java | 15 +- .../amazon/ion/SeekableReader.java | 15 +- src/{software => com}/amazon/ion/Span.java | 19 +- .../amazon/ion/SpanProvider.java | 15 +- .../ion/SubstituteSymbolTableException.java | 17 +- .../amazon/ion/SymbolTable.java | 21 +- .../amazon/ion/SymbolToken.java | 15 +- .../amazon/ion/SystemSymbols.java | 15 +- .../amazon/ion/TextSpan.java | 19 +- .../amazon/ion/Timestamp.java | 173 +- .../amazon/ion/UnexpectedEofException.java | 16 +- .../amazon/ion/UnknownSymbolException.java | 15 +- .../ion/UnsupportedIonVersionException.java | 15 +- .../amazon/ion/ValueFactory.java | 74 +- .../amazon/ion/ValueVisitor.java | 17 +- .../amazon/ion/apps/BaseApp.java | 29 +- src/com/amazon/ion/apps/EncodeApp.java | 175 + .../amazon/ion/apps/PrintApp.java | 23 +- .../amazon/ion/apps/SymtabApp.java | 40 +- .../amazon/ion/facet/Faceted.java | 15 +- .../amazon/ion/facet/Facets.java | 15 +- .../ion/facet/UnsupportedFacetException.java | 15 +- .../amazon/ion/facet/package-info.java | 21 +- .../ion/impl/AppendableFastAppendable.java | 21 +- .../amazon/ion/impl/Base64Encoder.java | 36 +- src/com/amazon/ion/impl/BlockedBuffer.java | 1778 ++++++++++ src/com/amazon/ion/impl/ByteBuffer.java | 72 + src/com/amazon/ion/impl/ByteReader.java | 185 ++ src/com/amazon/ion/impl/ByteWriter.java | 53 + .../amazon/ion/impl/DowncastingFaceted.java | 19 +- src/com/amazon/ion/impl/IonBinary.java | 2942 +++++++++++++++++ .../amazon/ion/impl/IonCharacterReader.java | 284 ++ .../amazon/ion/impl/IonIteratorImpl.java | 39 +- src/com/amazon/ion/impl/IonMessages.java | 24 + .../amazon/ion/impl/IonReaderBinaryRawX.java | 161 +- .../ion/impl/IonReaderBinarySystemX.java | 107 +- .../amazon/ion/impl/IonReaderBinaryUserX.java | 173 +- .../ion/impl/IonReaderTextRawTokensX.java | 51 +- .../amazon/ion/impl/IonReaderTextRawX.java | 61 +- .../amazon/ion/impl/IonReaderTextSystemX.java | 78 +- .../amazon/ion/impl/IonReaderTextUserX.java | 56 +- .../amazon/ion/impl/IonReaderTreeSystem.java | 103 +- .../amazon/ion/impl/IonReaderTreeUserX.java | 60 +- .../ion/impl/IonTextBufferedStream.java | 311 ++ .../amazon/ion/impl/IonTokenConstsX.java | 21 +- src/com/amazon/ion/impl/IonTokenReader.java | 1623 +++++++++ .../amazon/ion/impl/IonUTF8.java | 17 +- .../amazon/ion/impl/IonWriterSystem.java | 128 +- .../ion/impl/IonWriterSystemBinary.java | 1050 ++++++ .../amazon/ion/impl/IonWriterSystemText.java | 82 +- .../ion/impl/IonWriterSystemTextMarkup.java | 266 ++ .../amazon/ion/impl/IonWriterSystemTree.java | 78 +- .../amazon/ion/impl/IonWriterUser.java | 53 +- .../amazon/ion/impl/IonWriterUserBinary.java | 158 + .../amazon/ion/impl/LocalSymbolTable.java | 79 +- .../ion/impl/LocalSymbolTableAsStruct.java | 57 +- .../ion/impl/LocalSymbolTableImports.java | 24 +- .../ion/impl/OutputStreamFastAppendable.java | 23 +- .../amazon/ion/impl/SharedSymbolTable.java | 59 +- src/com/amazon/ion/impl/SimpleByteBuffer.java | 1386 ++++++++ .../ion/impl/SubstituteSymbolTable.java | 25 +- .../amazon/ion/impl/SymbolTableReader.java | 126 +- src/com/amazon/ion/impl/SymbolTokenImpl.java | 93 + .../amazon/ion/impl/SystemValueIterator.java | 88 + .../amazon/ion/impl/UnifiedDataPageX.java | 15 +- .../amazon/ion/impl/UnifiedInputBufferX.java | 15 +- .../amazon/ion/impl/UnifiedInputStreamX.java | 19 +- .../ion/impl/UnifiedSavePointManagerX.java | 15 +- .../ion/impl/_Private_ByteTransferReader.java | 29 + .../ion/impl/_Private_ByteTransferSink.java} | 22 +- .../ion/impl/_Private_CallbackBuilder.java | 30 + .../ion/impl/_Private_CommandLine.java} | 49 +- .../impl/_Private_CurriedValueFactory.java} | 87 +- .../_Private_FastAppendableDecorator.java} | 28 +- .../_Private_FastAppendableTrampoline.java | 38 + .../_Private_IonBinaryWriterBuilder.java} | 163 +- .../impl/_Private_IonBinaryWriterImpl.java | 84 + .../ion/impl/_Private_IonConstants.java} | 34 +- .../ion/impl/_Private_IonContainer.java | 32 + .../amazon/ion/impl/_Private_IonDatagram.java | 28 + .../ion/impl/_Private_IonReaderFactory.java} | 105 +- .../amazon/ion/impl/_Private_IonSymbol.java} | 26 +- .../amazon/ion/impl/_Private_IonSystem.java} | 40 +- .../ion/impl/_Private_IonTextAppender.java} | 70 +- .../impl/_Private_IonTextWriterBuilder.java | 305 ++ .../amazon/ion/impl/_Private_IonValue.java} | 28 +- .../amazon/ion/impl/_Private_IonWriter.java} | 32 +- .../ion/impl/_Private_IonWriterBase.java} | 76 +- .../ion/impl/_Private_IonWriterFactory.java} | 34 +- .../amazon/ion/impl/_Private_ListWriter.java} | 24 +- .../_Private_LocalSymbolTableFactory.java} | 27 +- .../ion/impl/_Private_MarkupCallback.java | 352 ++ .../ion/impl/_Private_ReaderWriter.java | 27 + .../ion/impl/_Private_ScalarConversions.java} | 34 +- .../amazon/ion/impl/_Private_SymbolToken.java | 31 + .../impl/_Private_SymtabExtendsCache.java} | 25 +- .../amazon/ion/impl/_Private_Utils.java} | 156 +- .../ion/impl/_Private_ValueFactory.java} | 23 +- .../ion/impl/bin/AbstractIonWriter.java | 95 +- .../ion/impl/bin/AbstractSymbolTable.java | 44 +- .../amazon/ion/impl/bin/Block.java | 15 +- .../amazon/ion/impl/bin/BlockAllocator.java | 15 +- .../ion/impl/bin/BlockAllocatorProvider.java | 15 +- .../ion/impl/bin/BlockAllocatorProviders.java | 15 +- .../ion/impl/bin/IonBinaryWriterAdapter.java | 268 ++ .../ion/impl/bin/IonManagedBinaryWriter.java | 124 +- .../ion/impl/bin/IonRawBinaryWriter.java | 206 +- .../bin/PooledBlockAllocatorProvider.java | 15 +- .../amazon/ion/impl/bin/Symbols.java | 61 +- .../amazon/ion/impl/bin/WriteBuffer.java | 19 +- .../impl/bin/_PrivateIon_HashTrampoline.java | 44 + ...rivate_IonManagedBinaryWriterBuilder.java} | 106 +- .../impl/bin/_Private_IonManagedWriter.java | 64 + .../ion/impl/bin/_Private_IonRawWriter.java | 84 + src/com/amazon/ion/impl/bin/package-info.java | 51 + .../ion/impl/lite/ContainerlessContext.java | 17 +- .../amazon/ion/impl/lite/IonBlobLite.java | 28 +- .../amazon/ion/impl/lite/IonBoolLite.java | 26 +- .../amazon/ion/impl/lite/IonClobLite.java | 28 +- .../ion/impl/lite/IonContainerLite.java | 83 +- .../amazon/ion/impl/lite/IonContext.java | 19 +- .../amazon/ion/impl/lite/IonDatagramLite.java | 87 +- .../amazon/ion/impl/lite/IonDecimalLite.java | 28 +- .../amazon/ion/impl/lite/IonFloatLite.java | 28 +- .../amazon/ion/impl/lite/IonIntLite.java | 29 +- .../amazon/ion/impl/lite/IonListLite.java | 28 +- .../amazon/ion/impl/lite/IonLoaderLite.java | 42 +- .../amazon/ion/impl/lite/IonLobLite.java | 18 +- .../amazon/ion/impl/lite/IonNullLite.java | 24 +- .../amazon/ion/impl/lite/IonSequenceLite.java | 38 +- .../amazon/ion/impl/lite/IonSexpLite.java | 28 +- .../amazon/ion/impl/lite/IonStringLite.java | 24 +- .../amazon/ion/impl/lite/IonStructLite.java | 46 +- .../amazon/ion/impl/lite/IonSymbolLite.java | 159 +- .../amazon/ion/impl/lite/IonSystemLite.java | 152 +- .../amazon/ion/impl/lite/IonTextLite.java | 18 +- .../ion/impl/lite/IonTimestampLite.java | 32 +- .../amazon/ion/impl/lite/IonValueLite.java | 127 +- .../ion/impl/lite/ReverseBinaryEncoder.java | 177 +- .../amazon/ion/impl/lite/TopLevelContext.java | 19 +- .../ion/impl/lite/ValueFactoryLite.java | 48 +- .../lite/_Private_LiteDomTrampoline.java} | 36 +- .../amazon/ion/overview.html | 85 +- src/com/amazon/ion/package-info.java | 19 + .../ion/system/IonBinaryWriterBuilder.java | 30 +- .../amazon/ion/system/IonReaderBuilder.java | 32 +- .../amazon/ion/system/IonSystemBuilder.java | 56 +- .../ion/system/IonTextWriterBuilder.java | 56 +- .../amazon/ion/system/IonWriterBuilder.java | 25 +- .../ion/system/IonWriterBuilderBase.java | 22 +- .../amazon/ion/system/SimpleCatalog.java | 21 +- src/com/amazon/ion/system/SystemFactory.java | 66 + src/com/amazon/ion/system/package-info.java | 19 + .../amazon/ion/util/AbstractValueVisitor.java | 49 +- .../amazon/ion/util/Equivalence.java | 68 +- .../amazon/ion/util/GzipOrRawInputStream.java | 15 +- .../amazon/ion/util/IonStreamUtils.java | 61 +- .../amazon/ion/util/IonTextUtils.java | 68 +- .../amazon/ion/util/IonValueUtils.java | 17 +- .../amazon/ion/util/JarInfo.java | 21 +- src/com/amazon/ion/util/Printer.java | 1160 +++++++ .../amazon/ion/util/Spans.java | 21 +- .../ion/util/_Private_FastAppendable.java} | 22 +- src/com/amazon/ion/util/package-info.java | 19 + .../amazon/ion/ContainedValueException.java | 28 - .../amazon/ion/EmptySymbolException.java | 24 - .../ion/InvalidSystemSymbolException.java | 31 - src/software/amazon/ion/impl/IonMessages.java | 22 - .../ion/impl/PrivateByteTransferReader.java | 31 - .../impl/PrivateFastAppendableTrampoline.java | 38 - .../amazon/ion/impl/PrivateIonContainer.java | 32 - .../amazon/ion/impl/PrivateIonDatagram.java | 28 - .../ion/impl/PrivateIonTextWriterBuilder.java | 250 -- .../amazon/ion/impl/PrivateReaderWriter.java | 27 - .../amazon/ion/impl/SymbolTokenImpl.java | 68 - .../impl/bin/PrivateIonHashTrampoline.java | 29 - .../amazon/ion/impl/bin/package-info.java | 50 - src/software/amazon/ion/package-info.java | 18 - .../amazon/ion/system/package-info.java | 18 - .../amazon/ion/util/package-info.java | 18 - test/AllTests.java | 193 +- .../amazon/ion/AnnotationEscapesTest.java | 16 +- .../amazon/ion/AssertionsEnabledTest.java | 16 +- .../amazon/ion/BadIonTest.java | 28 +- ...ByteArrayIteratorSystemProcessingTest.java | 16 +- .../ion/BinaryReaderSystemProcessingTest.java | 20 +- .../BinaryReaderWrappedValueLengthTest.java | 20 +- ...aryStreamIteratorSystemProcessingTest.java | 16 +- .../amazon/ion/BinaryTest.java | 21 +- .../amazon/ion/BitUtils.java | 15 +- .../amazon/ion/BlobTest.java | 28 +- .../amazon/ion/BoolTest.java | 20 +- .../{software => com}/amazon/ion/Checker.java | 16 +- .../amazon/ion/ClobTest.java | 26 +- .../amazon/ion/CloneTest.java | 32 +- .../amazon/ion/ContainerTestCase.java | 35 +- .../DatagramIteratorSystemProcessingTest.java | 19 +- .../amazon/ion/DatagramMaker.java | 26 +- .../amazon/ion/DatagramTest.java | 101 +- ...atagramTreeReaderSystemProcessingTest.java | 22 +- .../amazon/ion/DecimalTest.java | 33 +- .../amazon/ion/EquivTimelineTest.java | 27 +- test/com/amazon/ion/EquivsTest.java | 34 + .../amazon/ion/EquivsTestCase.java | 40 +- .../amazon/ion/ExtendedDecimalTest.java | 18 +- .../amazon/ion/FakeSymbolToken.java | 18 +- .../amazon/ion/FieldNameEscapesTest.java | 17 +- .../amazon/ion/FloatTest.java | 19 +- .../amazon/ion/GoodIonTest.java | 52 +- .../amazon/ion/HashCodeCorrectnessTest.java | 26 +- .../ion/HashCodeDeltaCollisionTest.java | 35 +- .../amazon/ion/HashCodeDistributionTest.java | 29 +- .../amazon/ion/InputStreamWrapper.java | 15 +- .../{software => com}/amazon/ion/IntTest.java | 16 +- .../amazon/ion/IonExceptionTest.java | 17 +- .../com/amazon/ion/IonRawWriterBasicTest.java | 255 ++ .../amazon/ion/IonRawWriterSymbolsTest.java | 477 +++ .../amazon/ion/IonReaderToIonValueTest.java | 19 +- .../amazon/ion/IonSystemTest.java | 44 +- .../amazon/ion/IonTestCase.java | 200 +- .../amazon/ion/IonValueChecker.java | 26 +- .../amazon/ion/IonValueDeltaGenerator.java | 54 +- .../amazon/ion/IonValueTest.java | 37 +- .../ion/IteratorSystemProcessingTestCase.java | 21 +- .../amazon/ion/JavaNumericsTest.java | 15 +- ...avaReaderIteratorSystemProcessingTest.java | 16 +- .../amazon/ion/ListTest.java | 41 +- .../LoadBinaryBytesSystemProcessingTest.java | 19 +- .../LoadBinaryStreamSystemProcessingTest.java | 18 +- .../LoadTextBytesSystemProcessingTest.java | 22 +- .../LoadTextStreamSystemProcessingTest.java | 22 +- .../amazon/ion/LoaderTest.java | 38 +- .../amazon/ion/LongStringTest.java | 16 +- ...wDatagramIteratorSystemProcessingTest.java | 19 +- test/com/amazon/ion/NonEquivsTest.java | 35 + .../amazon/ion/NopPaddingTest.java | 15 +- .../amazon/ion/NullTest.java | 21 +- .../ion/RawValueSpanReaderBasicTest.java | 154 + .../amazon/ion/ReaderChecker.java | 35 +- .../amazon/ion/ReaderMaker.java | 34 +- .../ion/ReaderSystemProcessingTestCase.java | 41 +- .../amazon/ion/RoundTripTest.java | 80 +- .../amazon/ion/SequenceTestCase.java | 33 +- .../amazon/ion/SexpTest.java | 44 +- .../amazon/ion/SharedSymtabMaker.java | 38 +- .../ion/StringFieldNameEscapesTest.java | 17 +- .../amazon/ion/StringTest.java | 19 +- .../amazon/ion/StructTest.java | 52 +- .../amazon/ion/SurrogateEscapeTest.java | 28 +- .../amazon/ion/SymbolTest.java | 35 +- .../amazon/ion/SystemProcessingTestCase.java | 44 +- .../amazon/ion/SystemProcessingTests.java | 16 +- .../amazon/ion/TestUtils.java | 56 +- ...ByteArrayIteratorSystemProcessingTest.java | 20 +- .../ion/TextIteratorSystemProcessingTest.java | 20 +- .../ion/TextReaderSystemProcessingTest.java | 19 +- ...extStreamIteratorSystemProcessingTest.java | 20 +- .../amazon/ion/TextTestCase.java | 20 +- .../amazon/ion/TimestampBadTest.java | 30 +- .../amazon/ion/TimestampGoodTest.java | 32 +- .../amazon/ion/TimestampTest.java | 274 +- .../amazon/ion/TrBwBrProcessingTest.java | 20 +- .../amazon/ion/TrueSequenceTestCase.java | 42 +- .../amazon/ion/ValueFactorySequenceTest.java | 41 +- .../amazon/ion/facet/FacetsTest.java | 24 +- .../amazon/ion/impl/BinaryWriterTest.java | 39 +- .../BinaryWriterWithLocalSymtabsTest.java | 36 +- test/com/amazon/ion/impl/ByteBufferTest.java | 399 +++ .../amazon/ion/impl/CharacterReaderTest.java | 248 ++ .../com/amazon/ion/impl/EscapingCallback.java | 128 + .../com/amazon/ion/impl/IonImplUtilsTest.java | 40 + .../ion/impl/IonMarkupWriterFilesTest.java | 81 + .../amazon/ion/impl/IonMarkupWriterTest.java | 143 + .../amazon/ion/impl/IonUTF8Test.java | 17 +- .../amazon/ion/impl/IonWriterTestCase.java | 125 +- .../amazon/ion/impl/IonWriterTests.java | 17 +- .../amazon/ion/impl/IterationTest.java | 23 +- .../amazon/ion/impl/LocalSymbolTableTest.java | 38 +- .../amazon/ion/impl/OldBinaryWriterTest.java | 57 + ...timizedBinaryWriterLengthPatchingTest.java | 37 +- .../OptimizedBinaryWriterSymbolTableTest.java | 38 +- .../impl/OptimizedBinaryWriterTestCase.java | 60 +- .../ion/impl/OutputStreamWriterTestCase.java | 42 +- .../ion/impl/RawValueSpanReaderTest.java | 325 ++ .../ion/impl/SharedSymbolTableTest.java | 49 +- .../amazon/ion/impl/SymbolTableTest.java | 107 +- .../amazon/ion/impl/Symtabs.java | 56 +- .../amazon/ion/impl/TestMarkupCallback.java | 148 + .../amazon/ion/impl/TextWriterTest.java | 56 +- .../amazon/ion/impl/TreeReaderTest.java | 34 +- .../ion/impl/UnifiedInputStreamXTest.java | 15 +- .../amazon/ion/impl/ValueWriterTest.java | 36 +- .../amazon/ion/impl/VarIntTest.java | 27 +- .../impl/_Private_ScalarConversionsTest.java | 44 + .../impl/bin/IonManagedBinaryWriterTest.java | 45 +- .../ion/impl/bin/IonRawBinaryWriterTest.java | 160 +- .../bin/PooledBlockAllocatorProviderTest.java | 18 +- .../amazon/ion/impl/bin/WriteBufferTest.java | 20 +- .../BaseIonSequenceLiteSublistTestCase.java | 29 +- .../lite/BaseIonSequenceLiteTestCase.java | 35 +- .../amazon/ion/impl/lite/IonContextTest.java | 49 +- .../impl/lite/IonDatagramLiteSublistTest.java | 31 + .../ion/impl/lite/IonDatagramLiteTest.java | 25 + .../amazon/ion/impl/lite/IonListLiteTest.java | 25 + .../impl/lite/IonListSexpLiteSublistTest.java | 29 +- .../amazon/ion/impl/lite/IonSexpLiteTest.java | 25 + .../impl/lite/SIDPresentLifecycleTest.java | 71 +- .../amazon/ion/junit/Injected.java | 32 +- .../amazon/ion/junit/IonAssert.java | 121 +- .../amazon/ion/profile/IteratorTiming.java | 22 +- .../ion/streaming/BadIonStreamingTest.java | 71 + .../ion/streaming/BinaryStreamingTest.java | 200 +- .../ion/streaming/GoodIonStreamingTest.java | 33 +- .../ion/streaming/InputStreamReaderTest.java | 28 +- .../ion/streaming/MiscStreamingTest.java | 63 +- .../streaming/NonOffsetSpanReaderTest.java | 23 +- .../ion/streaming/NonSeekableReaderTest.java | 21 +- .../ion/streaming/NonSpanReaderTest.java | 21 +- .../amazon/ion/streaming/NonTextSpanTest.java | 23 +- .../streaming/OffsetSpanBinaryReaderTest.java | 30 +- .../ion/streaming/OffsetSpanReaderTest.java | 29 +- .../amazon/ion/streaming/ReaderCompare.java | 50 +- .../ion/streaming/ReaderDomCopyTest.java | 33 +- .../ion/streaming/ReaderFacetTestCase.java | 36 +- .../ion/streaming/ReaderIntegerSizeTest.java | 27 +- .../ion/streaming/ReaderSkippingTest.java | 37 +- .../amazon/ion/streaming/ReaderTest.java | 38 +- .../amazon/ion/streaming/ReaderTestCase.java | 30 +- .../ion/streaming/RoundTripStreamingTest.java | 78 +- .../ion/streaming/SeekableReaderTest.java | 33 +- .../amazon/ion/streaming/SpanReaderTest.java | 27 +- .../amazon/ion/streaming/SpanTests.java | 18 +- .../amazon/ion/streaming/TextSpanTest.java | 29 +- .../system/IonBinaryWriterBuilderTest.java | 66 +- .../ion/system/IonReaderBuilderTest.java | 27 +- .../ion/system/IonSystemBuilderTest.java | 39 +- .../ion/system/IonTextWriterBuilderTest.java | 46 +- .../amazon/ion/system/SimpleCatalogTest.java | 31 +- .../amazon/ion/util/EquivalenceTest.java | 28 +- .../amazon/ion/util/IonStreamUtilsTest.java | 21 +- .../amazon/ion/util/JarInfoTest.java | 35 +- .../amazon/ion/util/NullOutputStream.java | 15 +- test/com/amazon/ion/util/PrinterTest.java | 678 ++++ .../amazon/ion/util/RepeatInputStream.java | 15 +- .../amazon/ion/util/SpansTest.java | 21 +- .../amazon/ion/util/TextTest.java | 39 +- .../amazon/ion/BinaryTestGenerator.py | 271 -- test/software/amazon/ion/EquivsTest.java | 34 - test/software/amazon/ion/NonEquivsTest.java | 34 - .../amazon/ion/impl/IonImplUtilsTest.java | 39 - .../impl/PrivateScalarConversionsTest.java | 33 - .../impl/lite/IonDatagramLiteSublistTest.java | 16 - .../ion/impl/lite/IonDatagramLiteTest.java | 10 - .../amazon/ion/impl/lite/IonListLiteTest.java | 10 - .../amazon/ion/impl/lite/IonSexpLiteTest.java | 10 - .../ion/streaming/BadIonStreamingTest.java | 88 - 411 files changed, 24143 insertions(+), 7425 deletions(-) delete mode 100644 .classpath delete mode 100644 .project delete mode 100644 .settings/AllTests.launch delete mode 100644 .settings/de.loskutov.anyedit.AnyEditTools.prefs delete mode 100644 .settings/org.eclipse.jdt.core.prefs delete mode 100644 .settings/org.eclipse.jdt.ui.prefs delete mode 100644 .settings/org.eclipse.wst.validation.prefs create mode 100644 src/com/amazon/ion/ContainedValueException.java rename src/{software => com}/amazon/ion/Decimal.java (90%) create mode 100644 src/com/amazon/ion/EmptySymbolException.java rename src/{software => com}/amazon/ion/IntegerSize.java (55%) create mode 100644 src/com/amazon/ion/InvalidSystemSymbolException.java create mode 100644 src/com/amazon/ion/IonBinaryWriter.java rename src/{software => com}/amazon/ion/IonBlob.java (72%) rename src/{software => com}/amazon/ion/IonBool.java (75%) rename src/{software => com}/amazon/ion/IonCatalog.java (88%) rename src/{software => com}/amazon/ion/IonClob.java (79%) rename src/{software => com}/amazon/ion/IonContainer.java (88%) rename src/{software => com}/amazon/ion/IonDatagram.java (71%) rename src/{software => com}/amazon/ion/IonDecimal.java (87%) rename src/{software => com}/amazon/ion/IonException.java (90%) rename src/{software => com}/amazon/ion/IonFloat.java (88%) rename src/{software => com}/amazon/ion/IonInt.java (84%) rename src/{software => com}/amazon/ion/IonList.java (59%) rename src/{software => com}/amazon/ion/IonLoader.java (93%) rename src/{software => com}/amazon/ion/IonLob.java (85%) rename src/{software => com}/amazon/ion/IonMutableCatalog.java (64%) rename src/{software => com}/amazon/ion/IonNull.java (63%) rename src/{software => com}/amazon/ion/IonNumber.java (55%) rename src/{software => com}/amazon/ion/IonReader.java (88%) rename src/{software => com}/amazon/ion/IonSequence.java (97%) rename src/{software => com}/amazon/ion/IonSexp.java (60%) rename src/{software => com}/amazon/ion/IonString.java (70%) rename src/{software => com}/amazon/ion/IonStruct.java (96%) rename src/{software => com}/amazon/ion/IonSymbol.java (58%) rename src/{software => com}/amazon/ion/IonSystem.java (92%) rename src/{software => com}/amazon/ion/IonText.java (73%) create mode 100644 src/com/amazon/ion/IonTextReader.java rename src/{software => com}/amazon/ion/IonTimestamp.java (95%) rename src/{software => com}/amazon/ion/IonType.java (81%) rename src/{software => com}/amazon/ion/IonValue.java (94%) rename src/{software => com}/amazon/ion/IonWriter.java (88%) rename src/{software => com}/amazon/ion/NullValueException.java (52%) rename src/{software => com}/amazon/ion/OffsetSpan.java (77%) create mode 100644 src/com/amazon/ion/RawValueSpanProvider.java rename src/{software => com}/amazon/ion/ReadOnlyValueException.java (63%) rename src/{software => com}/amazon/ion/SeekableReader.java (89%) rename src/{software => com}/amazon/ion/Span.java (82%) rename src/{software => com}/amazon/ion/SpanProvider.java (76%) rename src/{software => com}/amazon/ion/SubstituteSymbolTableException.java (61%) rename src/{software => com}/amazon/ion/SymbolTable.java (95%) rename src/{software => com}/amazon/ion/SymbolToken.java (82%) rename src/{software => com}/amazon/ion/SystemSymbols.java (88%) rename src/{software => com}/amazon/ion/TextSpan.java (80%) rename src/{software => com}/amazon/ion/Timestamp.java (95%) rename src/{software => com}/amazon/ion/UnexpectedEofException.java (70%) rename src/{software => com}/amazon/ion/UnknownSymbolException.java (75%) rename src/{software => com}/amazon/ion/UnsupportedIonVersionException.java (69%) rename src/{software => com}/amazon/ion/ValueFactory.java (86%) rename src/{software => com}/amazon/ion/ValueVisitor.java (58%) rename src/{software => com}/amazon/ion/apps/BaseApp.java (91%) create mode 100644 src/com/amazon/ion/apps/EncodeApp.java rename src/{software => com}/amazon/ion/apps/PrintApp.java (87%) rename src/{software => com}/amazon/ion/apps/SymtabApp.java (86%) rename src/{software => com}/amazon/ion/facet/Faceted.java (81%) rename src/{software => com}/amazon/ion/facet/Facets.java (90%) rename src/{software => com}/amazon/ion/facet/UnsupportedFacetException.java (76%) rename src/{software => com}/amazon/ion/facet/package-info.java (83%) rename src/{software => com}/amazon/ion/impl/AppendableFastAppendable.java (77%) rename src/{software => com}/amazon/ion/impl/Base64Encoder.java (97%) create mode 100644 src/com/amazon/ion/impl/BlockedBuffer.java create mode 100644 src/com/amazon/ion/impl/ByteBuffer.java create mode 100644 src/com/amazon/ion/impl/ByteReader.java create mode 100644 src/com/amazon/ion/impl/ByteWriter.java rename src/{software => com}/amazon/ion/impl/DowncastingFaceted.java (57%) create mode 100644 src/com/amazon/ion/impl/IonBinary.java create mode 100644 src/com/amazon/ion/impl/IonCharacterReader.java rename src/{software => com}/amazon/ion/impl/IonIteratorImpl.java (86%) create mode 100644 src/com/amazon/ion/impl/IonMessages.java rename src/{software => com}/amazon/ion/impl/IonReaderBinaryRawX.java (88%) rename src/{software => com}/amazon/ion/impl/IonReaderBinarySystemX.java (83%) rename src/{software => com}/amazon/ion/impl/IonReaderBinaryUserX.java (69%) rename src/{software => com}/amazon/ion/impl/IonReaderTextRawTokensX.java (98%) rename src/{software => com}/amazon/ion/impl/IonReaderTextRawX.java (97%) rename src/{software => com}/amazon/ion/impl/IonReaderTextSystemX.java (94%) rename src/{software => com}/amazon/ion/impl/IonReaderTextUserX.java (91%) rename src/{software => com}/amazon/ion/impl/IonReaderTreeSystem.java (86%) rename src/{software => com}/amazon/ion/impl/IonReaderTreeUserX.java (83%) create mode 100644 src/com/amazon/ion/impl/IonTextBufferedStream.java rename src/{software => com}/amazon/ion/impl/IonTokenConstsX.java (98%) create mode 100644 src/com/amazon/ion/impl/IonTokenReader.java rename src/{software => com}/amazon/ion/impl/IonUTF8.java (98%) rename src/{software => com}/amazon/ion/impl/IonWriterSystem.java (79%) create mode 100644 src/com/amazon/ion/impl/IonWriterSystemBinary.java rename src/{software => com}/amazon/ion/impl/IonWriterSystemText.java (90%) create mode 100644 src/com/amazon/ion/impl/IonWriterSystemTextMarkup.java rename src/{software => com}/amazon/ion/impl/IonWriterSystemTree.java (81%) rename src/{software => com}/amazon/ion/impl/IonWriterUser.java (91%) create mode 100644 src/com/amazon/ion/impl/IonWriterUserBinary.java rename src/{software => com}/amazon/ion/impl/LocalSymbolTable.java (92%) rename src/{software => com}/amazon/ion/impl/LocalSymbolTableAsStruct.java (86%) rename src/{software => com}/amazon/ion/impl/LocalSymbolTableImports.java (93%) rename src/{software => com}/amazon/ion/impl/OutputStreamFastAppendable.java (89%) rename src/{software => com}/amazon/ion/impl/SharedSymbolTable.java (91%) create mode 100644 src/com/amazon/ion/impl/SimpleByteBuffer.java rename src/{software => com}/amazon/ion/impl/SubstituteSymbolTable.java (90%) rename src/{software => com}/amazon/ion/impl/SymbolTableReader.java (93%) create mode 100644 src/com/amazon/ion/impl/SymbolTokenImpl.java create mode 100644 src/com/amazon/ion/impl/SystemValueIterator.java rename src/{software => com}/amazon/ion/impl/UnifiedDataPageX.java (95%) rename src/{software => com}/amazon/ion/impl/UnifiedInputBufferX.java (96%) rename src/{software => com}/amazon/ion/impl/UnifiedInputStreamX.java (97%) rename src/{software => com}/amazon/ion/impl/UnifiedSavePointManagerX.java (97%) create mode 100644 src/com/amazon/ion/impl/_Private_ByteTransferReader.java rename src/{software/amazon/ion/impl/PrivateByteTransferSink.java => com/amazon/ion/impl/_Private_ByteTransferSink.java} (53%) create mode 100644 src/com/amazon/ion/impl/_Private_CallbackBuilder.java rename src/{software/amazon/ion/impl/PrivateCommandLine.java => com/amazon/ion/impl/_Private_CommandLine.java} (72%) rename src/{software/amazon/ion/impl/PrivateCurriedValueFactory.java => com/amazon/ion/impl/_Private_CurriedValueFactory.java} (82%) rename src/{software/amazon/ion/impl/PrivateFastAppendableDecorator.java => com/amazon/ion/impl/_Private_FastAppendableDecorator.java} (69%) create mode 100644 src/com/amazon/ion/impl/_Private_FastAppendableTrampoline.java rename src/{software/amazon/ion/impl/PrivateIonBinaryWriterBuilder.java => com/amazon/ion/impl/_Private_IonBinaryWriterBuilder.java} (56%) create mode 100644 src/com/amazon/ion/impl/_Private_IonBinaryWriterImpl.java rename src/{software/amazon/ion/impl/PrivateIonConstants.java => com/amazon/ion/impl/_Private_IonConstants.java} (92%) create mode 100644 src/com/amazon/ion/impl/_Private_IonContainer.java create mode 100644 src/com/amazon/ion/impl/_Private_IonDatagram.java rename src/{software/amazon/ion/impl/PrivateIonReaderFactory.java => com/amazon/ion/impl/_Private_IonReaderFactory.java} (75%) rename src/{software/amazon/ion/impl/PrivateIonSymbol.java => com/amazon/ion/impl/_Private_IonSymbol.java} (50%) rename src/{software/amazon/ion/impl/PrivateIonSystem.java => com/amazon/ion/impl/_Private_IonSystem.java} (58%) rename src/{software/amazon/ion/impl/PrivateIonTextAppender.java => com/amazon/ion/impl/_Private_IonTextAppender.java} (92%) create mode 100644 src/com/amazon/ion/impl/_Private_IonTextWriterBuilder.java rename src/{software/amazon/ion/impl/PrivateIonValue.java => com/amazon/ion/impl/_Private_IonValue.java} (80%) rename src/{software/amazon/ion/impl/PrivateIonWriter.java => com/amazon/ion/impl/_Private_IonWriter.java} (65%) rename src/{software/amazon/ion/impl/PrivateIonWriterBase.java => com/amazon/ion/impl/_Private_IonWriterBase.java} (89%) rename src/{software/amazon/ion/impl/PrivateIonWriterFactory.java => com/amazon/ion/impl/_Private_IonWriterFactory.java} (71%) rename src/{software/amazon/ion/impl/PrivateListWriter.java => com/amazon/ion/impl/_Private_ListWriter.java} (60%) rename src/{software/amazon/ion/impl/PrivateLocalSymbolTableFactory.java => com/amazon/ion/impl/_Private_LocalSymbolTableFactory.java} (81%) create mode 100644 src/com/amazon/ion/impl/_Private_MarkupCallback.java create mode 100644 src/com/amazon/ion/impl/_Private_ReaderWriter.java rename src/{software/amazon/ion/impl/PrivateScalarConversions.java => com/amazon/ion/impl/_Private_ScalarConversions.java} (97%) create mode 100644 src/com/amazon/ion/impl/_Private_SymbolToken.java rename src/{software/amazon/ion/impl/PrivateSymtabExtendsCache.java => com/amazon/ion/impl/_Private_SymtabExtendsCache.java} (69%) rename src/{software/amazon/ion/impl/PrivateUtils.java => com/amazon/ion/impl/_Private_Utils.java} (90%) rename src/{software/amazon/ion/impl/PrivateValueFactory.java => com/amazon/ion/impl/_Private_ValueFactory.java} (50%) rename src/{software => com}/amazon/ion/impl/bin/AbstractIonWriter.java (66%) rename src/{software => com}/amazon/ion/impl/bin/AbstractSymbolTable.java (73%) rename src/{software => com}/amazon/ion/impl/bin/Block.java (79%) rename src/{software => com}/amazon/ion/impl/bin/BlockAllocator.java (77%) rename src/{software => com}/amazon/ion/impl/bin/BlockAllocatorProvider.java (72%) rename src/{software => com}/amazon/ion/impl/bin/BlockAllocatorProviders.java (78%) create mode 100644 src/com/amazon/ion/impl/bin/IonBinaryWriterAdapter.java rename src/{software => com}/amazon/ion/impl/bin/IonManagedBinaryWriter.java (92%) rename src/{software => com}/amazon/ion/impl/bin/IonRawBinaryWriter.java (90%) rename src/{software => com}/amazon/ion/impl/bin/PooledBlockAllocatorProvider.java (86%) rename src/{software => com}/amazon/ion/impl/bin/Symbols.java (83%) rename src/{software => com}/amazon/ion/impl/bin/WriteBuffer.java (98%) create mode 100644 src/com/amazon/ion/impl/bin/_PrivateIon_HashTrampoline.java rename src/{software/amazon/ion/impl/bin/PrivateIonManagedBinaryWriterBuilder.java => com/amazon/ion/impl/bin/_Private_IonManagedBinaryWriterBuilder.java} (62%) create mode 100644 src/com/amazon/ion/impl/bin/_Private_IonManagedWriter.java create mode 100644 src/com/amazon/ion/impl/bin/_Private_IonRawWriter.java create mode 100644 src/com/amazon/ion/impl/bin/package-info.java rename src/{software => com}/amazon/ion/impl/lite/ContainerlessContext.java (72%) rename src/{software => com}/amazon/ion/impl/lite/IonBlobLite.java (73%) rename src/{software => com}/amazon/ion/impl/lite/IonBoolLite.java (81%) rename src/{software => com}/amazon/ion/impl/lite/IonClobLite.java (75%) rename src/{software => com}/amazon/ion/impl/lite/IonContainerLite.java (91%) rename src/{software => com}/amazon/ion/impl/lite/IonContext.java (80%) rename src/{software => com}/amazon/ion/impl/lite/IonDatagramLite.java (94%) rename src/{software => com}/amazon/ion/impl/lite/IonDecimalLite.java (86%) rename src/{software => com}/amazon/ion/impl/lite/IonFloatLite.java (83%) rename src/{software => com}/amazon/ion/impl/lite/IonIntLite.java (90%) rename src/{software => com}/amazon/ion/impl/lite/IonListLite.java (75%) rename src/{software => com}/amazon/ion/impl/lite/IonLoaderLite.java (77%) rename src/{software => com}/amazon/ion/impl/lite/IonLobLite.java (87%) rename src/{software => com}/amazon/ion/impl/lite/IonNullLite.java (72%) rename src/{software => com}/amazon/ion/impl/lite/IonSequenceLite.java (95%) rename src/{software => com}/amazon/ion/impl/lite/IonSexpLite.java (73%) rename src/{software => com}/amazon/ion/impl/lite/IonStringLite.java (75%) rename src/{software => com}/amazon/ion/impl/lite/IonStructLite.java (95%) rename src/{software => com}/amazon/ion/impl/lite/IonSymbolLite.java (77%) rename src/{software => com}/amazon/ion/impl/lite/IonSystemLite.java (85%) rename src/{software => com}/amazon/ion/impl/lite/IonTextLite.java (74%) rename src/{software => com}/amazon/ion/impl/lite/IonTimestampLite.java (87%) rename src/{software => com}/amazon/ion/impl/lite/IonValueLite.java (92%) rename src/{software => com}/amazon/ion/impl/lite/ReverseBinaryEncoder.java (90%) rename src/{software => com}/amazon/ion/impl/lite/TopLevelContext.java (76%) rename src/{software => com}/amazon/ion/impl/lite/ValueFactoryLite.java (90%) rename src/{software/amazon/ion/impl/lite/PrivateLiteDomTrampoline.java => com/amazon/ion/impl/lite/_Private_LiteDomTrampoline.java} (50%) rename src/{software => com}/amazon/ion/overview.html (65%) create mode 100644 src/com/amazon/ion/package-info.java rename src/{software => com}/amazon/ion/system/IonBinaryWriterBuilder.java (92%) rename src/{software => com}/amazon/ion/system/IonReaderBuilder.java (91%) rename src/{software => com}/amazon/ion/system/IonSystemBuilder.java (89%) rename src/{software => com}/amazon/ion/system/IonTextWriterBuilder.java (94%) rename src/{software => com}/amazon/ion/system/IonWriterBuilder.java (86%) rename src/{software => com}/amazon/ion/system/IonWriterBuilderBase.java (91%) rename src/{software => com}/amazon/ion/system/SimpleCatalog.java (92%) create mode 100644 src/com/amazon/ion/system/SystemFactory.java create mode 100644 src/com/amazon/ion/system/package-info.java rename src/{software => com}/amazon/ion/util/AbstractValueVisitor.java (67%) rename src/{software => com}/amazon/ion/util/Equivalence.java (92%) rename src/{software => com}/amazon/ion/util/GzipOrRawInputStream.java (87%) rename src/{software => com}/amazon/ion/util/IonStreamUtils.java (84%) rename src/{software => com}/amazon/ion/util/IonTextUtils.java (93%) rename src/{software => com}/amazon/ion/util/IonValueUtils.java (59%) rename src/{software => com}/amazon/ion/util/JarInfo.java (89%) create mode 100644 src/com/amazon/ion/util/Printer.java rename src/{software => com}/amazon/ion/util/Spans.java (71%) rename src/{software/amazon/ion/util/PrivateFastAppendable.java => com/amazon/ion/util/_Private_FastAppendable.java} (76%) create mode 100644 src/com/amazon/ion/util/package-info.java delete mode 100644 src/software/amazon/ion/ContainedValueException.java delete mode 100644 src/software/amazon/ion/EmptySymbolException.java delete mode 100644 src/software/amazon/ion/InvalidSystemSymbolException.java delete mode 100644 src/software/amazon/ion/impl/IonMessages.java delete mode 100644 src/software/amazon/ion/impl/PrivateByteTransferReader.java delete mode 100644 src/software/amazon/ion/impl/PrivateFastAppendableTrampoline.java delete mode 100644 src/software/amazon/ion/impl/PrivateIonContainer.java delete mode 100644 src/software/amazon/ion/impl/PrivateIonDatagram.java delete mode 100644 src/software/amazon/ion/impl/PrivateIonTextWriterBuilder.java delete mode 100644 src/software/amazon/ion/impl/PrivateReaderWriter.java delete mode 100644 src/software/amazon/ion/impl/SymbolTokenImpl.java delete mode 100644 src/software/amazon/ion/impl/bin/PrivateIonHashTrampoline.java delete mode 100644 src/software/amazon/ion/impl/bin/package-info.java delete mode 100644 src/software/amazon/ion/package-info.java delete mode 100644 src/software/amazon/ion/system/package-info.java delete mode 100644 src/software/amazon/ion/util/package-info.java rename test/{software => com}/amazon/ion/AnnotationEscapesTest.java (55%) rename test/{software => com}/amazon/ion/AssertionsEnabledTest.java (66%) rename test/{software => com}/amazon/ion/BadIonTest.java (75%) rename test/{software => com}/amazon/ion/BinaryByteArrayIteratorSystemProcessingTest.java (66%) rename test/{software => com}/amazon/ion/BinaryReaderSystemProcessingTest.java (63%) rename test/{software => com}/amazon/ion/BinaryReaderWrappedValueLengthTest.java (76%) rename test/{software => com}/amazon/ion/BinaryStreamIteratorSystemProcessingTest.java (69%) rename test/{software => com}/amazon/ion/BinaryTest.java (91%) rename test/{software => com}/amazon/ion/BitUtils.java (74%) rename test/{software => com}/amazon/ion/BlobTest.java (92%) rename test/{software => com}/amazon/ion/BoolTest.java (86%) rename test/{software => com}/amazon/ion/Checker.java (65%) rename test/{software => com}/amazon/ion/ClobTest.java (88%) rename test/{software => com}/amazon/ion/CloneTest.java (80%) rename test/{software => com}/amazon/ion/ContainerTestCase.java (90%) rename test/{software => com}/amazon/ion/DatagramIteratorSystemProcessingTest.java (56%) rename test/{software => com}/amazon/ion/DatagramMaker.java (80%) rename test/{software => com}/amazon/ion/DatagramTest.java (88%) rename test/{software => com}/amazon/ion/DatagramTreeReaderSystemProcessingTest.java (77%) rename test/{software => com}/amazon/ion/DecimalTest.java (91%) rename test/{software => com}/amazon/ion/EquivTimelineTest.java (75%) create mode 100644 test/com/amazon/ion/EquivsTest.java rename test/{software => com}/amazon/ion/EquivsTestCase.java (90%) rename test/{software => com}/amazon/ion/ExtendedDecimalTest.java (93%) rename test/{software => com}/amazon/ion/FakeSymbolToken.java (62%) rename test/{software => com}/amazon/ion/FieldNameEscapesTest.java (54%) rename test/{software => com}/amazon/ion/FloatTest.java (92%) rename test/{software => com}/amazon/ion/GoodIonTest.java (70%) rename test/{software => com}/amazon/ion/HashCodeCorrectnessTest.java (97%) rename test/{software => com}/amazon/ion/HashCodeDeltaCollisionTest.java (89%) rename test/{software => com}/amazon/ion/HashCodeDistributionTest.java (94%) rename test/{software => com}/amazon/ion/InputStreamWrapper.java (53%) rename test/{software => com}/amazon/ion/IntTest.java (97%) rename test/{software => com}/amazon/ion/IonExceptionTest.java (86%) create mode 100644 test/com/amazon/ion/IonRawWriterBasicTest.java create mode 100644 test/com/amazon/ion/IonRawWriterSymbolsTest.java rename test/{software => com}/amazon/ion/IonReaderToIonValueTest.java (62%) rename test/{software => com}/amazon/ion/IonSystemTest.java (89%) rename test/{software => com}/amazon/ion/IonTestCase.java (86%) rename test/{software => com}/amazon/ion/IonValueChecker.java (81%) rename test/{software => com}/amazon/ion/IonValueDeltaGenerator.java (90%) rename test/{software => com}/amazon/ion/IonValueTest.java (88%) rename test/{software => com}/amazon/ion/IteratorSystemProcessingTestCase.java (84%) rename test/{software => com}/amazon/ion/JavaNumericsTest.java (91%) rename test/{software => com}/amazon/ion/JavaReaderIteratorSystemProcessingTest.java (65%) rename test/{software => com}/amazon/ion/ListTest.java (89%) rename test/{software => com}/amazon/ion/LoadBinaryBytesSystemProcessingTest.java (61%) rename test/{software => com}/amazon/ion/LoadBinaryStreamSystemProcessingTest.java (64%) rename test/{software => com}/amazon/ion/LoadTextBytesSystemProcessingTest.java (53%) rename test/{software => com}/amazon/ion/LoadTextStreamSystemProcessingTest.java (57%) rename test/{software => com}/amazon/ion/LoaderTest.java (93%) rename test/{software => com}/amazon/ion/LongStringTest.java (56%) rename test/{software => com}/amazon/ion/NewDatagramIteratorSystemProcessingTest.java (79%) create mode 100644 test/com/amazon/ion/NonEquivsTest.java rename test/{software => com}/amazon/ion/NopPaddingTest.java (91%) rename test/{software => com}/amazon/ion/NullTest.java (78%) create mode 100644 test/com/amazon/ion/RawValueSpanReaderBasicTest.java rename test/{software => com}/amazon/ion/ReaderChecker.java (85%) rename test/{software => com}/amazon/ion/ReaderMaker.java (90%) rename test/{software => com}/amazon/ion/ReaderSystemProcessingTestCase.java (91%) rename test/{software => com}/amazon/ion/RoundTripTest.java (79%) rename test/{software => com}/amazon/ion/SequenceTestCase.java (95%) rename test/{software => com}/amazon/ion/SexpTest.java (89%) rename test/{software => com}/amazon/ion/SharedSymtabMaker.java (68%) rename test/{software => com}/amazon/ion/StringFieldNameEscapesTest.java (59%) rename test/{software => com}/amazon/ion/StringTest.java (94%) rename test/{software => com}/amazon/ion/StructTest.java (97%) rename test/{software => com}/amazon/ion/SurrogateEscapeTest.java (74%) rename test/{software => com}/amazon/ion/SymbolTest.java (87%) rename test/{software => com}/amazon/ion/SystemProcessingTestCase.java (96%) rename test/{software => com}/amazon/ion/SystemProcessingTests.java (74%) rename test/{software => com}/amazon/ion/TestUtils.java (90%) rename test/{software => com}/amazon/ion/TextByteArrayIteratorSystemProcessingTest.java (58%) rename test/{software => com}/amazon/ion/TextIteratorSystemProcessingTest.java (61%) rename test/{software => com}/amazon/ion/TextReaderSystemProcessingTest.java (61%) rename test/{software => com}/amazon/ion/TextStreamIteratorSystemProcessingTest.java (62%) rename test/{software => com}/amazon/ion/TextTestCase.java (79%) rename test/{software => com}/amazon/ion/TimestampBadTest.java (60%) rename test/{software => com}/amazon/ion/TimestampGoodTest.java (68%) rename test/{software => com}/amazon/ion/TimestampTest.java (93%) rename test/{software => com}/amazon/ion/TrBwBrProcessingTest.java (66%) rename test/{software => com}/amazon/ion/TrueSequenceTestCase.java (91%) rename test/{software => com}/amazon/ion/ValueFactorySequenceTest.java (77%) rename test/{software => com}/amazon/ion/facet/FacetsTest.java (87%) rename test/{software => com}/amazon/ion/impl/BinaryWriterTest.java (90%) rename test/{software => com}/amazon/ion/impl/BinaryWriterWithLocalSymtabsTest.java (89%) create mode 100644 test/com/amazon/ion/impl/ByteBufferTest.java create mode 100644 test/com/amazon/ion/impl/CharacterReaderTest.java create mode 100644 test/com/amazon/ion/impl/EscapingCallback.java create mode 100644 test/com/amazon/ion/impl/IonImplUtilsTest.java create mode 100644 test/com/amazon/ion/impl/IonMarkupWriterFilesTest.java create mode 100644 test/com/amazon/ion/impl/IonMarkupWriterTest.java rename test/{software => com}/amazon/ion/impl/IonUTF8Test.java (92%) rename test/{software => com}/amazon/ion/impl/IonWriterTestCase.java (92%) rename test/{software => com}/amazon/ion/impl/IonWriterTests.java (58%) rename test/{software => com}/amazon/ion/impl/IterationTest.java (77%) rename test/{software => com}/amazon/ion/impl/LocalSymbolTableTest.java (89%) create mode 100644 test/com/amazon/ion/impl/OldBinaryWriterTest.java rename test/{software => com}/amazon/ion/impl/OptimizedBinaryWriterLengthPatchingTest.java (90%) rename test/{software => com}/amazon/ion/impl/OptimizedBinaryWriterSymbolTableTest.java (89%) rename test/{software => com}/amazon/ion/impl/OptimizedBinaryWriterTestCase.java (72%) rename test/{software => com}/amazon/ion/impl/OutputStreamWriterTestCase.java (90%) create mode 100644 test/com/amazon/ion/impl/RawValueSpanReaderTest.java rename test/{software => com}/amazon/ion/impl/SharedSymbolTableTest.java (90%) rename test/{software => com}/amazon/ion/impl/SymbolTableTest.java (94%) rename test/{software => com}/amazon/ion/impl/Symtabs.java (81%) create mode 100644 test/com/amazon/ion/impl/TestMarkupCallback.java rename test/{software => com}/amazon/ion/impl/TextWriterTest.java (89%) rename test/{software => com}/amazon/ion/impl/TreeReaderTest.java (69%) rename test/{software => com}/amazon/ion/impl/UnifiedInputStreamXTest.java (69%) rename test/{software => com}/amazon/ion/impl/ValueWriterTest.java (81%) rename test/{software => com}/amazon/ion/impl/VarIntTest.java (83%) create mode 100644 test/com/amazon/ion/impl/_Private_ScalarConversionsTest.java rename test/{software => com}/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java (83%) rename test/{software => com}/amazon/ion/impl/bin/IonRawBinaryWriterTest.java (75%) rename test/{software => com}/amazon/ion/impl/bin/PooledBlockAllocatorProviderTest.java (81%) rename test/{software => com}/amazon/ion/impl/bin/WriteBufferTest.java (97%) rename test/{software => com}/amazon/ion/impl/lite/BaseIonSequenceLiteSublistTestCase.java (95%) rename test/{software => com}/amazon/ion/impl/lite/BaseIonSequenceLiteTestCase.java (68%) rename test/{software => com}/amazon/ion/impl/lite/IonContextTest.java (91%) create mode 100644 test/com/amazon/ion/impl/lite/IonDatagramLiteSublistTest.java create mode 100644 test/com/amazon/ion/impl/lite/IonDatagramLiteTest.java create mode 100644 test/com/amazon/ion/impl/lite/IonListLiteTest.java rename test/{software => com}/amazon/ion/impl/lite/IonListSexpLiteSublistTest.java (77%) create mode 100644 test/com/amazon/ion/impl/lite/IonSexpLiteTest.java rename test/{software => com}/amazon/ion/impl/lite/SIDPresentLifecycleTest.java (81%) rename test/{software => com}/amazon/ion/junit/Injected.java (88%) rename test/{software => com}/amazon/ion/junit/IonAssert.java (82%) rename test/{software => com}/amazon/ion/profile/IteratorTiming.java (77%) create mode 100644 test/com/amazon/ion/streaming/BadIonStreamingTest.java rename test/{software => com}/amazon/ion/streaming/BinaryStreamingTest.java (88%) rename test/{software => com}/amazon/ion/streaming/GoodIonStreamingTest.java (53%) rename test/{software => com}/amazon/ion/streaming/InputStreamReaderTest.java (72%) rename test/{software => com}/amazon/ion/streaming/MiscStreamingTest.java (87%) rename test/{software => com}/amazon/ion/streaming/NonOffsetSpanReaderTest.java (60%) rename test/{software => com}/amazon/ion/streaming/NonSeekableReaderTest.java (58%) rename test/{software => com}/amazon/ion/streaming/NonSpanReaderTest.java (60%) rename test/{software => com}/amazon/ion/streaming/NonTextSpanTest.java (59%) rename test/{software => com}/amazon/ion/streaming/OffsetSpanBinaryReaderTest.java (76%) rename test/{software => com}/amazon/ion/streaming/OffsetSpanReaderTest.java (75%) rename test/{software => com}/amazon/ion/streaming/ReaderCompare.java (82%) rename test/{software => com}/amazon/ion/streaming/ReaderDomCopyTest.java (55%) rename test/{software => com}/amazon/ion/streaming/ReaderFacetTestCase.java (82%) rename test/{software => com}/amazon/ion/streaming/ReaderIntegerSizeTest.java (85%) rename test/{software => com}/amazon/ion/streaming/ReaderSkippingTest.java (84%) rename test/{software => com}/amazon/ion/streaming/ReaderTest.java (91%) rename test/{software => com}/amazon/ion/streaming/ReaderTestCase.java (74%) rename test/{software => com}/amazon/ion/streaming/RoundTripStreamingTest.java (88%) rename test/{software => com}/amazon/ion/streaming/SeekableReaderTest.java (88%) rename test/{software => com}/amazon/ion/streaming/SpanReaderTest.java (86%) rename test/{software => com}/amazon/ion/streaming/SpanTests.java (60%) rename test/{software => com}/amazon/ion/streaming/TextSpanTest.java (60%) rename test/{software => com}/amazon/ion/system/IonBinaryWriterBuilderTest.java (86%) rename test/{software => com}/amazon/ion/system/IonReaderBuilderTest.java (83%) rename test/{software => com}/amazon/ion/system/IonSystemBuilderTest.java (80%) rename test/{software => com}/amazon/ion/system/IonTextWriterBuilderTest.java (90%) rename test/{software => com}/amazon/ion/system/SimpleCatalogTest.java (87%) rename test/{software => com}/amazon/ion/util/EquivalenceTest.java (93%) rename test/{software => com}/amazon/ion/util/IonStreamUtilsTest.java (78%) rename test/{software => com}/amazon/ion/util/JarInfoTest.java (82%) rename test/{software => com}/amazon/ion/util/NullOutputStream.java (63%) create mode 100644 test/com/amazon/ion/util/PrinterTest.java rename test/{software => com}/amazon/ion/util/RepeatInputStream.java (82%) rename test/{software => com}/amazon/ion/util/SpansTest.java (61%) rename test/{software => com}/amazon/ion/util/TextTest.java (84%) delete mode 100755 test/software/amazon/ion/BinaryTestGenerator.py delete mode 100644 test/software/amazon/ion/EquivsTest.java delete mode 100644 test/software/amazon/ion/NonEquivsTest.java delete mode 100644 test/software/amazon/ion/impl/IonImplUtilsTest.java delete mode 100644 test/software/amazon/ion/impl/PrivateScalarConversionsTest.java delete mode 100644 test/software/amazon/ion/impl/lite/IonDatagramLiteSublistTest.java delete mode 100644 test/software/amazon/ion/impl/lite/IonDatagramLiteTest.java delete mode 100644 test/software/amazon/ion/impl/lite/IonListLiteTest.java delete mode 100644 test/software/amazon/ion/impl/lite/IonSexpLiteTest.java delete mode 100644 test/software/amazon/ion/streaming/BadIonStreamingTest.java diff --git a/.classpath b/.classpath deleted file mode 100644 index aee63273dc..0000000000 --- a/.classpath +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - diff --git a/.gitignore b/.gitignore index 708300db93..c6d00a25a0 100644 --- a/.gitignore +++ b/.gitignore @@ -136,7 +136,7 @@ flycheck_*.el # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 -.idea +.idea # User-specific stuff .idea/**/workspace.xml @@ -263,4 +263,3 @@ gradle-app.setting **/build/ # End of https://www.gitignore.io/api/vim,emacs,maven,gradle,eclipse,intellij - diff --git a/.gitmodules b/.gitmodules index 776319e243..38171e6698 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,3 @@ [submodule "ion-tests"] path = ion-tests url = https://github.com/amzn/ion-tests.git - branch = master diff --git a/.project b/.project deleted file mode 100644 index 333d659778..0000000000 --- a/.project +++ /dev/null @@ -1,15 +0,0 @@ - - - IonJava - - - - org.eclipse.jdt.core.javabuilder - - - - - - org.eclipse.jdt.core.javanature - - diff --git a/.settings/AllTests.launch b/.settings/AllTests.launch deleted file mode 100644 index 1cab7f7578..0000000000 --- a/.settings/AllTests.launch +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - - - - - - - diff --git a/.settings/de.loskutov.anyedit.AnyEditTools.prefs b/.settings/de.loskutov.anyedit.AnyEditTools.prefs deleted file mode 100644 index 13aed0422b..0000000000 --- a/.settings/de.loskutov.anyedit.AnyEditTools.prefs +++ /dev/null @@ -1,13 +0,0 @@ -#Sun Apr 29 18:50:10 PDT 2007 -activeContentFilterList=*.makefile,makefile,*.Makefile,Makefile,Makefile.*,*.mk -convertActionOnSaave=AnyEdit.CnvrtTabToSpaces -eclipse.preferences.version=1 -inActiveContentFilterList= -javaTabWidthForJava=true -org.eclipse.jdt.ui.editor.tab.width=4 -projectPropsEnabled=false -removeTrailingSpaces=true -replaceAllTabs=false -saveAndConvert=true -saveAndTrim=true -useModulo4Tabs=false diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs deleted file mode 100644 index 2298a99ff3..0000000000 --- a/.settings/org.eclipse.jdt.core.prefs +++ /dev/null @@ -1,354 +0,0 @@ -eclipse.preferences.version=1 -org.eclipse.jdt.core.builder.cleanOutputFolder=clean -org.eclipse.jdt.core.builder.duplicateResourceTask=warning -org.eclipse.jdt.core.builder.invalidClasspath=ignore -org.eclipse.jdt.core.builder.recreateModifiedClassFileInOutputFolder=ignore -org.eclipse.jdt.core.builder.resourceCopyExclusionFilter=*.launch -org.eclipse.jdt.core.circularClasspath=error -org.eclipse.jdt.core.classpath.exclusionPatterns=enabled -org.eclipse.jdt.core.classpath.multipleOutputLocations=enabled -org.eclipse.jdt.core.codeComplete.argumentPrefixes= -org.eclipse.jdt.core.codeComplete.argumentSuffixes= -org.eclipse.jdt.core.codeComplete.fieldPrefixes= -org.eclipse.jdt.core.codeComplete.fieldSuffixes= -org.eclipse.jdt.core.codeComplete.localPrefixes= -org.eclipse.jdt.core.codeComplete.localSuffixes= -org.eclipse.jdt.core.codeComplete.staticFieldPrefixes= -org.eclipse.jdt.core.codeComplete.staticFieldSuffixes= -org.eclipse.jdt.core.codeComplete.staticFinalFieldPrefixes= -org.eclipse.jdt.core.codeComplete.staticFinalFieldSuffixes= -org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled -org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 -org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=1.8 -org.eclipse.jdt.core.compiler.debug.lineNumber=generate -org.eclipse.jdt.core.compiler.debug.localVariable=generate -org.eclipse.jdt.core.compiler.debug.sourceFile=generate -org.eclipse.jdt.core.compiler.doc.comment.support=enabled -org.eclipse.jdt.core.compiler.maxProblemPerUnit=100 -org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning -org.eclipse.jdt.core.compiler.problem.assertIdentifier=error -org.eclipse.jdt.core.compiler.problem.autoboxing=ignore -org.eclipse.jdt.core.compiler.problem.deprecation=warning -org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled -org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled -org.eclipse.jdt.core.compiler.problem.discouragedReference=warning -org.eclipse.jdt.core.compiler.problem.emptyStatement=warning -org.eclipse.jdt.core.compiler.problem.enumIdentifier=error -org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore -org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore -org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning -org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning -org.eclipse.jdt.core.compiler.problem.forbiddenReference=error -org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning -org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning -org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning -org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore -org.eclipse.jdt.core.compiler.problem.invalidJavadoc=warning -org.eclipse.jdt.core.compiler.problem.invalidJavadocTags=enabled -org.eclipse.jdt.core.compiler.problem.invalidJavadocTagsDeprecatedRef=enabled -org.eclipse.jdt.core.compiler.problem.invalidJavadocTagsNotVisibleRef=disabled -org.eclipse.jdt.core.compiler.problem.invalidJavadocTagsVisibility=protected -org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore -org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning -org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore -org.eclipse.jdt.core.compiler.problem.missingJavadocComments=ignore -org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsOverriding=enabled -org.eclipse.jdt.core.compiler.problem.missingJavadocCommentsVisibility=public -org.eclipse.jdt.core.compiler.problem.missingJavadocTags=ignore -org.eclipse.jdt.core.compiler.problem.missingJavadocTagsOverriding=enabled -org.eclipse.jdt.core.compiler.problem.missingJavadocTagsVisibility=public -org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=warning -org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning -org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning -org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning -org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore -org.eclipse.jdt.core.compiler.problem.nullReference=warning -org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning -org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore -org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore -org.eclipse.jdt.core.compiler.problem.rawTypeReference=ignore -org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled -org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning -org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled -org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore -org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning -org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=warning -org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore -org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=warning -org.eclipse.jdt.core.compiler.problem.unnecessaryElse=warning -org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=warning -org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore -org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=ignore -org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled -org.eclipse.jdt.core.compiler.problem.unusedImport=warning -org.eclipse.jdt.core.compiler.problem.unusedLabel=warning -org.eclipse.jdt.core.compiler.problem.unusedLocal=warning -org.eclipse.jdt.core.compiler.problem.unusedParameter=ignore -org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled -org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled -org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning -org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning -org.eclipse.jdt.core.compiler.source=1.8 -org.eclipse.jdt.core.compiler.taskCaseSensitive=enabled -org.eclipse.jdt.core.compiler.taskPriorities=NORMAL,HIGH,NORMAL -org.eclipse.jdt.core.compiler.taskTags=TODO,FIXME,XXX -org.eclipse.jdt.core.formatter.align_type_members_on_columns=false -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=18 -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=16 -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=16 -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=18 -org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=16 -org.eclipse.jdt.core.formatter.alignment_for_assignment=0 -org.eclipse.jdt.core.formatter.alignment_for_binary_expression=18 -org.eclipse.jdt.core.formatter.alignment_for_compact_if=16 -org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=82 -org.eclipse.jdt.core.formatter.alignment_for_enum_constants=0 -org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=18 -org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16 -org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=18 -org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=18 -org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=16 -org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration=17 -org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration=16 -org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=17 -org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=32 -org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=32 -org.eclipse.jdt.core.formatter.blank_lines_after_imports=2 -org.eclipse.jdt.core.formatter.blank_lines_after_package=1 -org.eclipse.jdt.core.formatter.blank_lines_before_field=1 -org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0 -org.eclipse.jdt.core.formatter.blank_lines_before_imports=1 -org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1 -org.eclipse.jdt.core.formatter.blank_lines_before_method=1 -org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=1 -org.eclipse.jdt.core.formatter.blank_lines_before_package=0 -org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1 -org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations=2 -org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=next_line -org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_block=next_line -org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=next_line -org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration=next_line -org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line -org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=next_line -org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=next_line -org.eclipse.jdt.core.formatter.brace_position_for_switch=next_line -org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=next_line -org.eclipse.jdt.core.formatter.comment.clear_blank_lines=false -org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false -org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false -org.eclipse.jdt.core.formatter.comment.format_block_comments=true -org.eclipse.jdt.core.formatter.comment.format_comments=true -org.eclipse.jdt.core.formatter.comment.format_header=false -org.eclipse.jdt.core.formatter.comment.format_html=true -org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=true -org.eclipse.jdt.core.formatter.comment.format_line_comments=true -org.eclipse.jdt.core.formatter.comment.format_source_code=true -org.eclipse.jdt.core.formatter.comment.indent_parameter_description=false -org.eclipse.jdt.core.formatter.comment.indent_root_tags=true -org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert -org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=insert -org.eclipse.jdt.core.formatter.comment.line_length=79 -org.eclipse.jdt.core.formatter.compact_else_if=true -org.eclipse.jdt.core.formatter.continuation_indentation=1 -org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2 -org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=true -org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true -org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header=true -org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true -org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true -org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true -org.eclipse.jdt.core.formatter.indent_empty_lines=false -org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true -org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true -org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true -org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=true -org.eclipse.jdt.core.formatter.indentation.size=8 -org.eclipse.jdt.core.formatter.insert_new_line_after_annotation=insert -org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=insert -org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=insert -org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert -org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=insert -org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement=insert -org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body=insert -org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert -org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert -org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_binary_operator=insert -org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=insert -org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert -org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert -org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert -org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert -org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert -org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert -org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert -org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert -org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert -org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert -org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard=do not insert -org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert -org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert -org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert -org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_binary_operator=insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert -org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional=insert -org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for=insert -org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_ellipsis=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert -org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert -org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert -org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert -org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert -org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert -org.eclipse.jdt.core.formatter.insert_space_before_unary_operator=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert -org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert -org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false -org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false -org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false -org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false -org.eclipse.jdt.core.formatter.lineSplit=79 -org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false -org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false -org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0 -org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1 -org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true -org.eclipse.jdt.core.formatter.tabulation.char=space -org.eclipse.jdt.core.formatter.tabulation.size=4 -org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=false -org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true -org.eclipse.jdt.core.incompatibleJDKLevel=ignore -org.eclipse.jdt.core.incompleteClasspath=warning diff --git a/.settings/org.eclipse.jdt.ui.prefs b/.settings/org.eclipse.jdt.ui.prefs deleted file mode 100644 index 266ef3e388..0000000000 --- a/.settings/org.eclipse.jdt.ui.prefs +++ /dev/null @@ -1,116 +0,0 @@ -#Fri Feb 13 19:25:41 PST 2009 -cleanup.add_default_serial_version_id=true -cleanup.add_generated_serial_version_id=false -cleanup.add_missing_annotations=true -cleanup.add_missing_deprecated_annotations=true -cleanup.add_missing_nls_tags=false -cleanup.add_missing_override_annotations=true -cleanup.add_serial_version_id=false -cleanup.always_use_blocks=true -cleanup.always_use_parentheses_in_expressions=false -cleanup.always_use_this_for_non_static_field_access=false -cleanup.always_use_this_for_non_static_method_access=false -cleanup.convert_to_enhanced_for_loop=false -cleanup.format_source_code=false -cleanup.make_local_variable_final=true -cleanup.make_parameters_final=false -cleanup.make_private_fields_final=true -cleanup.make_variable_declarations_final=false -cleanup.never_use_blocks=false -cleanup.never_use_parentheses_in_expressions=true -cleanup.organize_imports=true -cleanup.qualify_static_field_accesses_with_declaring_class=false -cleanup.qualify_static_member_accesses_through_instances_with_declaring_class=true -cleanup.qualify_static_member_accesses_through_subtypes_with_declaring_class=true -cleanup.qualify_static_member_accesses_with_declaring_class=true -cleanup.qualify_static_method_accesses_with_declaring_class=false -cleanup.remove_private_constructors=true -cleanup.remove_trailing_whitespaces=true -cleanup.remove_trailing_whitespaces_all=true -cleanup.remove_trailing_whitespaces_ignore_empty=false -cleanup.remove_unnecessary_casts=false -cleanup.remove_unnecessary_nls_tags=true -cleanup.remove_unused_imports=true -cleanup.remove_unused_local_variables=false -cleanup.remove_unused_private_fields=true -cleanup.remove_unused_private_members=false -cleanup.remove_unused_private_methods=true -cleanup.remove_unused_private_types=true -cleanup.sort_members=false -cleanup.sort_members_all=false -cleanup.use_blocks=false -cleanup.use_blocks_only_for_return_and_throw=false -cleanup.use_parentheses_in_expressions=false -cleanup.use_this_for_non_static_field_access=false -cleanup.use_this_for_non_static_field_access_only_if_necessary=true -cleanup.use_this_for_non_static_method_access=false -cleanup.use_this_for_non_static_method_access_only_if_necessary=true -cleanup_profile=_Custom -cleanup_settings_version=2 -eclipse.preferences.version=1 -editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true -formatter_profile=_CPT -formatter_settings_version=11 -instance/org.eclipse.core.net/org.eclipse.core.net.hasMigrated=true -internal.default.compliance=default -org.eclipse.jdt.ui.exception.name=e -org.eclipse.jdt.ui.gettersetter.use.is=true -org.eclipse.jdt.ui.ignorelowercasenames=true -org.eclipse.jdt.ui.importorder=; -org.eclipse.jdt.ui.javadoc=true -org.eclipse.jdt.ui.keywordthis=false -org.eclipse.jdt.ui.ondemandthreshold=99 -org.eclipse.jdt.ui.overrideannotation=true -org.eclipse.jdt.ui.staticondemandthreshold=99 -org.eclipse.jdt.ui.text.custom_code_templates= -sp_cleanup.add_default_serial_version_id=true -sp_cleanup.add_generated_serial_version_id=false -sp_cleanup.add_missing_annotations=true -sp_cleanup.add_missing_deprecated_annotations=true -sp_cleanup.add_missing_methods=false -sp_cleanup.add_missing_nls_tags=false -sp_cleanup.add_missing_override_annotations=false -sp_cleanup.add_serial_version_id=false -sp_cleanup.always_use_blocks=true -sp_cleanup.always_use_parentheses_in_expressions=false -sp_cleanup.always_use_this_for_non_static_field_access=false -sp_cleanup.always_use_this_for_non_static_method_access=false -sp_cleanup.convert_to_enhanced_for_loop=false -sp_cleanup.correct_indentation=false -sp_cleanup.format_source_code=false -sp_cleanup.format_source_code_changes_only=false -sp_cleanup.make_local_variable_final=false -sp_cleanup.make_parameters_final=false -sp_cleanup.make_private_fields_final=true -sp_cleanup.make_type_abstract_if_missing_method=false -sp_cleanup.make_variable_declarations_final=false -sp_cleanup.never_use_blocks=false -sp_cleanup.never_use_parentheses_in_expressions=true -sp_cleanup.on_save_use_additional_actions=true -sp_cleanup.organize_imports=true -sp_cleanup.qualify_static_field_accesses_with_declaring_class=false -sp_cleanup.qualify_static_member_accesses_through_instances_with_declaring_class=true -sp_cleanup.qualify_static_member_accesses_through_subtypes_with_declaring_class=true -sp_cleanup.qualify_static_member_accesses_with_declaring_class=false -sp_cleanup.qualify_static_method_accesses_with_declaring_class=false -sp_cleanup.remove_private_constructors=true -sp_cleanup.remove_trailing_whitespaces=true -sp_cleanup.remove_trailing_whitespaces_all=true -sp_cleanup.remove_trailing_whitespaces_ignore_empty=false -sp_cleanup.remove_unnecessary_casts=false -sp_cleanup.remove_unnecessary_nls_tags=false -sp_cleanup.remove_unused_imports=true -sp_cleanup.remove_unused_local_variables=false -sp_cleanup.remove_unused_private_fields=true -sp_cleanup.remove_unused_private_members=false -sp_cleanup.remove_unused_private_methods=true -sp_cleanup.remove_unused_private_types=true -sp_cleanup.sort_members=false -sp_cleanup.sort_members_all=false -sp_cleanup.use_blocks=false -sp_cleanup.use_blocks_only_for_return_and_throw=false -sp_cleanup.use_parentheses_in_expressions=false -sp_cleanup.use_this_for_non_static_field_access=false -sp_cleanup.use_this_for_non_static_field_access_only_if_necessary=true -sp_cleanup.use_this_for_non_static_method_access=false -sp_cleanup.use_this_for_non_static_method_access_only_if_necessary=true diff --git a/.settings/org.eclipse.wst.validation.prefs b/.settings/org.eclipse.wst.validation.prefs deleted file mode 100644 index 5522cbf3b0..0000000000 --- a/.settings/org.eclipse.wst.validation.prefs +++ /dev/null @@ -1,6 +0,0 @@ -#Tue Feb 20 16:42:38 PST 2007 -DELEGATES_PREFERENCE=delegateValidatorListorg.eclipse.wst.xsd.core.internal.validation.eclipse.XSDDelegatingValidator\=org.eclipse.wst.xsd.core.internal.validation.eclipse.Validator;org.eclipse.wst.wsdl.validation.internal.eclipse.WSDLDelegatingValidator\=org.eclipse.wst.wsdl.validation.internal.eclipse.Validator; -USER_BUILD_PREFERENCE=enabledBuildValidatorListorg.eclipse.wst.wsi.ui.internal.WSIMessageValidator;org.eclipse.wst.xml.core.internal.validation.eclipse.Validator;org.eclipse.wst.dtd.core.internal.validation.eclipse.Validator;org.eclipse.wst.xsd.core.internal.validation.eclipse.XSDDelegatingValidator;org.eclipse.wst.wsdl.validation.internal.eclipse.WSDLDelegatingValidator;org.eclipse.wst.html.internal.validation.HTMLValidator; -USER_MANUAL_PREFERENCE=enabledManualValidatorListorg.eclipse.wst.wsi.ui.internal.WSIMessageValidator;org.eclipse.wst.xml.core.internal.validation.eclipse.Validator;org.eclipse.wst.dtd.core.internal.validation.eclipse.Validator;org.eclipse.wst.xsd.core.internal.validation.eclipse.XSDDelegatingValidator;org.eclipse.wst.wsdl.validation.internal.eclipse.WSDLDelegatingValidator;org.eclipse.wst.html.internal.validation.HTMLValidator; -USER_PREFERENCE=overrideGlobalPreferencesfalse -eclipse.preferences.version=1 diff --git a/.travis.yml b/.travis.yml index a8b579c681..f1dd7898a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,28 +1,28 @@ sudo: false language: java jdk: -- openjdk8 -- openjdk9 -- openjdk10 -- openjdk11 -- oraclejdk8 -- oraclejdk9 -- oraclejdk11 + - openjdk8 + - openjdk9 + - openjdk10 + - openjdk11 + - oraclejdk8 + - oraclejdk9 + - oraclejdk11 -script: mvn test +script: mvn test -jobs: - include: - stage: report generation - jdk: openjdk11 - script: mvn test site +jobs: + include: + stage: report generation + jdk: openjdk11 + script: mvn test site after_success: - - bash <(curl -s https://codecov.io/bash) - deploy: + - bash <(curl -s https://codecov.io/bash) + deploy: provider: pages local-dir: "./target/site/" skip-cleanup: true github-token: "$GITHUB_TOKEN" keep-history: true # keeps commit history of gh-pages branch on: - branch: master + branch: master \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 3b64466870..a0ea08c6ac 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,4 +1,4 @@ ## Code of Conduct -This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). -For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact -opensource-codeofconduct@amazon.com with any additional questions or comments. +This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). +For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact +opensource-codeofconduct@amazon.com with any additional questions or comments. \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e5289c5300..f226f24d2d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,9 +1,9 @@ # Contributing Guidelines -Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional +Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional documentation, we greatly value feedback and contributions from our community. -Please read through this document before submitting any issues or pull requests to ensure we have all the necessary +Please read through this document before submitting any issues or pull requests to ensure we have all the necessary information to effectively respond to your bug report or contribution. @@ -11,7 +11,7 @@ information to effectively respond to your bug report or contribution. We welcome you to use the GitHub issue tracker to report bugs or suggest features. -When filing an issue, please check [existing open](https://github.com/amzn/ion-java/issues), or [recently closed](https://github.com/amzn/ion-java/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already +When filing an issue, please check [existing open](https://github.com/amzn/ion-java/issues), or [recently closed](https://github.com/amzn/ion-java/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: * A reproducible test case or series of steps @@ -36,17 +36,17 @@ To send us a pull request, please: 5. Send us a pull request, answering any default questions in the pull request interface. 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. -GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and +GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). ## Finding contributions to work on -Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/amzn/ion-java/labels/help%20wanted) issues is a great place to start. +Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/amzn/ion-java/labels/help%20wanted) issues is a great place to start. ## Code of Conduct -This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). -For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact +This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). +For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact opensource-codeofconduct@amazon.com with any additional questions or comments. @@ -58,4 +58,4 @@ If you discover a potential security issue in this project we ask that you notif See the [LICENSE](https://github.com/amzn/ion-java/blob/master/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. -We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. +We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. \ No newline at end of file diff --git a/LICENSE b/LICENSE index d9a10c0d8e..2bb9ad240f 100644 --- a/LICENSE +++ b/LICENSE @@ -173,4 +173,4 @@ incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. - END OF TERMS AND CONDITIONS + END OF TERMS AND CONDITIONS \ No newline at end of file diff --git a/NOTICE b/NOTICE index 2225c71e25..884a507a45 100644 --- a/NOTICE +++ b/NOTICE @@ -1,2 +1,2 @@ Amazon Ion Java -Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. +Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. \ No newline at end of file diff --git a/README.md b/README.md index 8a9da762ad..5c27014596 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ A Java implementation of the [Ion data notation](http://amzn.github.io/ion-docs). [![Build Status](https://travis-ci.org/amzn/ion-java.svg?branch=master)](https://travis-ci.org/amzn/ion-java) -[![Maven Central](https://maven-badges.herokuapp.com/maven-central/software.amazon.ion/ion-java/badge.svg)](https://maven-badges.herokuapp.com/maven-central/software.amazon.ion/ion-java) -[![Javadoc](https://javadoc-badge.appspot.com/software.amazon.ion/ion-java.svg?label=javadoc)](http://www.javadoc.io/doc/software.amazon.ion/ion-java) +[![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.amazon.ion/ion-java/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.amazon.ion/ion-java) +[![Javadoc](https://javadoc-badge.appspot.com/com.amazon.ion/ion-java.svg?label=javadoc)](http://www.javadoc.io/doc/com.amazon.ion/ion-java) ## Setup This repository contains a [git submodule](https://git-scm.com/docs/git-submodule) @@ -58,7 +58,7 @@ dependency into your project's `pom.xml`: ``` - software.amazon.ion + com.amazon.ion ion-java 1.3.1 @@ -66,5 +66,5 @@ dependency into your project's `pom.xml`: ## Using the Library A great way to get started is to use the [Ion cookbook](http://amzn.github.io/ion-docs/cookbook.html). -The [API documentation](http://www.javadoc.io/doc/software.amazon.ion/ion-java) will give a lot -of detailed information about how to use the library. +The [API documentation](http://www.javadoc.io/doc/com.amazon.ion/ion-java) will give a lot +of detailed information about how to use the library. \ No newline at end of file diff --git a/ion-tests b/ion-tests index 0281240a90..48a63157b2 160000 --- a/ion-tests +++ b/ion-tests @@ -1 +1 @@ -Subproject commit 0281240a9039a1f609fc8e289721efff489956be +Subproject commit 48a63157b28b81ed72c71789dac7d7f0ffe87038 diff --git a/pom.xml b/pom.xml index 9550562224..33792c29df 100644 --- a/pom.xml +++ b/pom.xml @@ -1,7 +1,22 @@ + + 4.0.0 - software.amazon.ion + com.amazon.ion ion-java 1.3.1 bundle @@ -20,13 +35,13 @@ - - Amazon Ion Team - ion-team@amazon.com - Amazon Labs - https://github.com/amzn - - + + Amazon Ion Team + ion-team@amazon.com + Amazon + https://github.com/amzn + + scm:git:git@github.com:amzn/ion-java.git @@ -111,16 +126,16 @@ - !software.amazon.ion.apps.*, - !software.amazon.ion.impl.*, - software.amazon.ion.*, + !com.amazon.ion.apps.*, + !com.amazon.ion.impl.*, + com.amazon.ion.*, - software.amazon.ion.impl.PrivateCommandLine + com.amazon.ion.impl.PrivateCommandLine ${build.time} ${project.version} - software.amazon.ion + com.amazon.ion @@ -192,11 +207,11 @@ - software.amazon.ion.apps:software.amazon.ion.impl + com.amazon.ion.apps:com.amazon.ion.impl **/*Private* - ${project.basedir}/src/software/amazon/ion/overview.html + ${project.basedir}/src/com/amazon/ion/overview.html Amazon Ion Java ${project.version} API Reference Ion Java ${project.version}
    Amazon Ion Java ${project.version} API Reference
    @@ -262,11 +277,11 @@ maven-javadoc-plugin 2.10.3 - software.amazon.ion.apps:software.amazon.ion.impl + com.amazon.ion.apps:com.amazon.ion.impl **/*Private* - ${project.basedir}/src/software/amazon/ion/overview.html + ${project.basedir}/src/com/amazon/ion/overview.html Amazon Ion Java ${project.version} API Reference Ion Java ${project.version}
    Amazon Ion Java ${project.version} API Reference
    @@ -316,4 +331,4 @@ -
    + \ No newline at end of file diff --git a/src/com/amazon/ion/ContainedValueException.java b/src/com/amazon/ion/ContainedValueException.java new file mode 100644 index 0000000000..a7c5deea20 --- /dev/null +++ b/src/com/amazon/ion/ContainedValueException.java @@ -0,0 +1,40 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion; + +/** + * An error caused by adding an {@link IonValue} into a container when it's + * already contained elsewhere. + */ +public class ContainedValueException + extends IonException +{ + private static final long serialVersionUID = 1L; + + + public ContainedValueException() + { + super(); + } + + /** + * @param message + */ + public ContainedValueException(String message) + { + super(message); + } +} diff --git a/src/software/amazon/ion/Decimal.java b/src/com/amazon/ion/Decimal.java similarity index 90% rename from src/software/amazon/ion/Decimal.java rename to src/com/amazon/ion/Decimal.java index a92dc06644..bde45ee1f1 100644 --- a/src/software/amazon/ion/Decimal.java +++ b/src/com/amazon/ion/Decimal.java @@ -1,18 +1,19 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.math.BigDecimal; import java.math.BigInteger; @@ -68,14 +69,16 @@ private NegativeZero(int scale, MathContext mc) public float floatValue() { float v = super.floatValue(); - return Float.compare(0f, v) <= 0 ? -1 * v : v; + if (Float.compare(0f, v) <= 0) v = -1 * v; + return v; } @Override public double doubleValue() { double v = super.doubleValue(); - return Double.compare(0d, v) <= 0 ? -1 * v : v; + if (Double.compare(0d, v) <= 0) v = -1 * v; + return v; } @@ -157,8 +160,12 @@ public static boolean isNegativeZero(BigDecimal val) */ public static BigDecimal bigDecimalValue(BigDecimal val) { - return val == null || val.getClass() == BigDecimal.class ? - val : new BigDecimal(val.unscaledValue(), val.scale()); + if (val == null + || val.getClass() == BigDecimal.class) + { + return val; + } + return new BigDecimal(val.unscaledValue(), val.scale()); } /** @@ -263,12 +270,17 @@ public static Decimal valueOf(double val) public static Decimal valueOf(double val, MathContext mc) { - return Double.compare(val, -0d) == 0 ? new NegativeZero(1, mc) : new Decimal(Double.toString(val), mc); + if (Double.compare(val, -0d) == 0) + { + return new NegativeZero(1, mc); + } + return new Decimal(Double.toString(val), mc); } public static Decimal valueOf(BigDecimal val) { - return val == null || val instanceof Decimal ? (Decimal) val : new Decimal(val.unscaledValue(), val.scale()); + if (val == null || val instanceof Decimal) return (Decimal) val; + return new Decimal(val.unscaledValue(), val.scale()); } public static Decimal valueOf(BigDecimal val, MathContext mc) @@ -410,6 +422,7 @@ public final boolean isNegativeZero() */ public final BigDecimal bigDecimalValue() { + return new BigDecimal(unscaledValue(), scale()); } } diff --git a/src/com/amazon/ion/EmptySymbolException.java b/src/com/amazon/ion/EmptySymbolException.java new file mode 100644 index 0000000000..eb1351906f --- /dev/null +++ b/src/com/amazon/ion/EmptySymbolException.java @@ -0,0 +1,35 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion; + +/** + * An error caused by a symbol not containing at least one character for + * its text. + * + * @deprecated this exception is not used as empty symbols are valid. In cases where null is used as the symbol value it was + * replaced by {@link NullPointerException} + */ +@Deprecated +public class EmptySymbolException + extends IonException +{ + private static final long serialVersionUID = -7801632953459636349L; + + public EmptySymbolException() + { + super("Symbols must contain at least one character."); + } +} diff --git a/src/software/amazon/ion/IntegerSize.java b/src/com/amazon/ion/IntegerSize.java similarity index 55% rename from src/software/amazon/ion/IntegerSize.java rename to src/com/amazon/ion/IntegerSize.java index 732dab211a..7ced059a0f 100644 --- a/src/software/amazon/ion/IntegerSize.java +++ b/src/com/amazon/ion/IntegerSize.java @@ -1,6 +1,19 @@ -// Copyright (c) 2016 Amazon.com, Inc. All rights reserved. +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ -package software.amazon.ion; +package com.amazon.ion; /** * Indicates the smallest-possible Java type of an Ion {@code int} value. diff --git a/src/com/amazon/ion/InvalidSystemSymbolException.java b/src/com/amazon/ion/InvalidSystemSymbolException.java new file mode 100644 index 0000000000..c718ba46e4 --- /dev/null +++ b/src/com/amazon/ion/InvalidSystemSymbolException.java @@ -0,0 +1,42 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion; + + +/** + * An error caused by use of an invalid symbol starting with + * "$ion_". + */ +public class InvalidSystemSymbolException + extends IonException +{ + private static final long serialVersionUID = 2206499395645594047L; + + private String myBadSymbol; + + + public InvalidSystemSymbolException(String badSymbol) + { + super("Invalid system symbol '" + badSymbol + "'"); + myBadSymbol = badSymbol; + } + + + public String getBadSymbol() + { + return myBadSymbol; + } +} diff --git a/src/com/amazon/ion/IonBinaryWriter.java b/src/com/amazon/ion/IonBinaryWriter.java new file mode 100644 index 0000000000..fcbf94edaa --- /dev/null +++ b/src/com/amazon/ion/IonBinaryWriter.java @@ -0,0 +1,85 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * An {@link IonWriter} that encodes Ion binary data. + * + * @deprecated Use {@link IonSystem#newBinaryWriter(OutputStream, SymbolTable...)} instead. + */ +@Deprecated +public interface IonBinaryWriter + extends IonWriter +{ + + /** + * Gets the size in bytes of this binary data. + * This is generally needed before calling {@link #getBytes()} or + * {@link #getBytes(byte[], int, int)}. + * + * @return the size in bytes. + */ + @Deprecated + public int byteSize(); + + /** + * Copies the current contents of this writer as a new byte array holding + * Ion binary-encoded data. + * This allocates an array of the size needed to exactly + * hold the output and copies the entire value to it. + * + * @return the byte array with the writers output + * @throws IOException + */ + @Deprecated + public byte[] getBytes() + throws IOException; + + + /** + * Copies the current contents of the writer to a given byte array + * array. This starts writing to the array at offset and writes + * up to maxlen bytes. + * If this writer is not able to stop in the middle of its + * work this may overwrite the array and later throw and exception. + * + * @param bytes users byte array to write into + * @param offset initial offset in the array to write into + * @param maxlen maximum number of bytes to write + * @return number of bytes written + * @throws IOException + */ + @Deprecated + public int getBytes(byte[] bytes, int offset, int maxlen) + throws IOException; + + /** + * Writes the current contents of the writer to the output + * stream. This is only valid if the writer is not in the + * middle of writing a container. + * + * @param userstream OutputStream to write the bytes to + * @return int length of bytes written + * @throws IOException + */ + @Deprecated + public int writeBytes(OutputStream userstream) + throws IOException; + +} diff --git a/src/software/amazon/ion/IonBlob.java b/src/com/amazon/ion/IonBlob.java similarity index 72% rename from src/software/amazon/ion/IonBlob.java rename to src/com/amazon/ion/IonBlob.java index f90468e703..503a865579 100644 --- a/src/software/amazon/ion/IonBlob.java +++ b/src/com/amazon/ion/IonBlob.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.io.IOException; diff --git a/src/software/amazon/ion/IonBool.java b/src/com/amazon/ion/IonBool.java similarity index 75% rename from src/software/amazon/ion/IonBool.java rename to src/com/amazon/ion/IonBool.java index 9bbd928fac..4ed612194e 100644 --- a/src/software/amazon/ion/IonBool.java +++ b/src/com/amazon/ion/IonBool.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; diff --git a/src/software/amazon/ion/IonCatalog.java b/src/com/amazon/ion/IonCatalog.java similarity index 88% rename from src/software/amazon/ion/IonCatalog.java rename to src/com/amazon/ion/IonCatalog.java index a852ae080b..cf79390d92 100644 --- a/src/software/amazon/ion/IonCatalog.java +++ b/src/com/amazon/ion/IonCatalog.java @@ -1,21 +1,22 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import software.amazon.ion.system.IonSystemBuilder; -import software.amazon.ion.system.SimpleCatalog; +import com.amazon.ion.system.IonSystemBuilder; +import com.amazon.ion.system.SimpleCatalog; /** @@ -71,7 +72,7 @@ * Binary decoding prefers an exact match, and in a couple edge cases, * requires it. Therefore a single "get latest version" method is insufficient. * See the - * Ion Symbols page + * Ion Symbols page * for more details on this topic. *

    * It's expected that many if not most applications will implement a dynamic diff --git a/src/software/amazon/ion/IonClob.java b/src/com/amazon/ion/IonClob.java similarity index 79% rename from src/software/amazon/ion/IonClob.java rename to src/com/amazon/ion/IonClob.java index 9ca6b5fe30..9004337913 100644 --- a/src/software/amazon/ion/IonClob.java +++ b/src/com/amazon/ion/IonClob.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.io.Reader; import java.nio.charset.Charset; diff --git a/src/software/amazon/ion/IonContainer.java b/src/com/amazon/ion/IonContainer.java similarity index 88% rename from src/software/amazon/ion/IonContainer.java rename to src/com/amazon/ion/IonContainer.java index 8c1f621975..12805d8ea4 100644 --- a/src/software/amazon/ion/IonContainer.java +++ b/src/com/amazon/ion/IonContainer.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.util.Collection; import java.util.Iterator; diff --git a/src/software/amazon/ion/IonDatagram.java b/src/com/amazon/ion/IonDatagram.java similarity index 71% rename from src/software/amazon/ion/IonDatagram.java rename to src/com/amazon/ion/IonDatagram.java index fad4c6ce62..72394b35d3 100644 --- a/src/software/amazon/ion/IonDatagram.java +++ b/src/com/amazon/ion/IonDatagram.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.io.IOException; import java.io.OutputStream; @@ -41,11 +42,11 @@ public interface IonDatagram /** * This inherited method is not yet supported by datagrams. *

    - * Vote for issue amzn/ion-java#48 if you need this. + * Vote for issue amzn/ion-java/issues/48 if you need this. * * @throws UnsupportedOperationException at every call. * - * @see amzn/ion-java#48 + * @see amzn/ion-java/issues/48 */ public void add(int index, IonValue element) throws ContainedValueException, NullPointerException; @@ -53,11 +54,11 @@ public void add(int index, IonValue element) /** * This inherited method is not yet supported by datagrams. *

    - * Vote for issue amzn/ion-java#48 if you need this. + * Vote for issue amzn/ion-java/issues/48 if you need this. * * @throws UnsupportedOperationException at every call. * - * @see amzn/ion-java#48 + * @see amzn/ion-java/issues/48 */ public ValueFactory add(int index) throws ContainedValueException, NullPointerException; @@ -65,22 +66,22 @@ public ValueFactory add(int index) /** * This inherited method is not yet supported by datagrams. *

    - * Vote for issue amzn/ion-java#47 if you need this. + * Vote for issue amzn/ion-java/issues/47 if you need this. * * @throws UnsupportedOperationException at every call. * - * @see amzn/ion-java#47 + * @see amzn/ion-java/issues/47 */ public boolean addAll(int index, Collection c); /** * This inherited method is not yet supported by datagrams. *

    - * Vote for issue amzn/ion-java#50 if you need this. + * Vote for issue amzn/ion-java/issues/50 if you need this. * * @throws UnsupportedOperationException at every call. * - * @see amzn/ion-java#50 + * @see amzn/ion-java/issues/50 */ public IonValue set(int index, IonValue element); @@ -188,6 +189,19 @@ public int byteSize() throws IonException; + /** + * Copies the binary-encoded form of this datagram into a new byte array. + * + * @return a new, non-empty byte array containing the encoded datagram. + * + * @throws IonException if there's an error encoding the data. + * + * @deprecated Use {@link #getBytes()} instead, which is renamed for consistency with other interfaces. + */ + @Deprecated + public byte[] toBytes() + throws IonException; + /** * Copies the binary-encoded form of this datagram into a new byte array. * @@ -199,6 +213,54 @@ public byte[] getBytes() throws IonException; + /** + * Copies the binary-encoded form of this datagram into a given array. + *

    + * The given array must be large enough to contain all the bytes of this + * datagram. + *

    + * An invocation of this method of the form {@code dg.get(a)} behaves in + * exactly the same way as the invocation: + *

    +     *    dg.get(a, 0)
    +     *
    + * + * @param dst the array into which bytes are to be written. + * + * @return the number of bytes copied into {@code dst}. + * + * @throws IonException if there's an error encoding the data. + * @throws IndexOutOfBoundsException if {@code dst.length} is + * smaller than the result of {@link #byteSize()}. + * + * @see #getBytes(byte[],int) + */ + @Deprecated + public int getBytes(byte[] dst) + throws IonException; + + + /** + * Copies the binary-encoded form of this datagram into a given sub-array. + *

    + * The given subarray must be large enough to contain all the bytes of this + * datagram. + * + * @param dst the array into which bytes are to be written. + * @param offset the offset within the array of the first byte to be + * written; must be non-negative and no larger than {@code dst.length} + * + * @return the number of bytes copied into {@code dst}. + * + * @throws IonException if there's an error encoding the data. + * @throws IndexOutOfBoundsException if {@code (dst.length - offset)} is + * smaller than the result of {@link #byteSize()}. + */ + @Deprecated + public int getBytes(byte[] dst, int offset) + throws IonException; + + /** * Copies the binary-encoded form of this datagram to a specified stream. * diff --git a/src/software/amazon/ion/IonDecimal.java b/src/com/amazon/ion/IonDecimal.java similarity index 87% rename from src/software/amazon/ion/IonDecimal.java rename to src/com/amazon/ion/IonDecimal.java index 827b4749f3..40871bb2a3 100644 --- a/src/software/amazon/ion/IonDecimal.java +++ b/src/com/amazon/ion/IonDecimal.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.math.BigDecimal; diff --git a/src/software/amazon/ion/IonException.java b/src/com/amazon/ion/IonException.java similarity index 90% rename from src/software/amazon/ion/IonException.java rename to src/com/amazon/ion/IonException.java index 6410f0dfb1..f4d577f8ea 100644 --- a/src/software/amazon/ion/IonException.java +++ b/src/com/amazon/ion/IonException.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.io.IOException; import java.util.IdentityHashMap; diff --git a/src/software/amazon/ion/IonFloat.java b/src/com/amazon/ion/IonFloat.java similarity index 88% rename from src/software/amazon/ion/IonFloat.java rename to src/com/amazon/ion/IonFloat.java index c8db2cef40..e4fa2be69d 100644 --- a/src/software/amazon/ion/IonFloat.java +++ b/src/com/amazon/ion/IonFloat.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.math.BigDecimal; diff --git a/src/software/amazon/ion/IonInt.java b/src/com/amazon/ion/IonInt.java similarity index 84% rename from src/software/amazon/ion/IonInt.java rename to src/com/amazon/ion/IonInt.java index d2eabbca77..2270b0a918 100644 --- a/src/software/amazon/ion/IonInt.java +++ b/src/com/amazon/ion/IonInt.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.math.BigInteger; diff --git a/src/software/amazon/ion/IonList.java b/src/com/amazon/ion/IonList.java similarity index 59% rename from src/software/amazon/ion/IonList.java rename to src/com/amazon/ion/IonList.java index 4834f0926a..0b697fb368 100644 --- a/src/software/amazon/ion/IonList.java +++ b/src/com/amazon/ion/IonList.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.util.Collection; diff --git a/src/software/amazon/ion/IonLoader.java b/src/com/amazon/ion/IonLoader.java similarity index 93% rename from src/software/amazon/ion/IonLoader.java rename to src/com/amazon/ion/IonLoader.java index fe91b340ef..aaab75938a 100644 --- a/src/software/amazon/ion/IonLoader.java +++ b/src/com/amazon/ion/IonLoader.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.io.File; import java.io.IOException; diff --git a/src/software/amazon/ion/IonLob.java b/src/com/amazon/ion/IonLob.java similarity index 85% rename from src/software/amazon/ion/IonLob.java rename to src/com/amazon/ion/IonLob.java index 3245ca327c..780b3a61a2 100644 --- a/src/software/amazon/ion/IonLob.java +++ b/src/com/amazon/ion/IonLob.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.io.InputStream; diff --git a/src/software/amazon/ion/IonMutableCatalog.java b/src/com/amazon/ion/IonMutableCatalog.java similarity index 64% rename from src/software/amazon/ion/IonMutableCatalog.java rename to src/com/amazon/ion/IonMutableCatalog.java index 6d14f05384..18c528812c 100644 --- a/src/software/amazon/ion/IonMutableCatalog.java +++ b/src/com/amazon/ion/IonMutableCatalog.java @@ -1,18 +1,19 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; /** * An {@link IonCatalog} that can be updated. diff --git a/src/software/amazon/ion/IonNull.java b/src/com/amazon/ion/IonNull.java similarity index 63% rename from src/software/amazon/ion/IonNull.java rename to src/com/amazon/ion/IonNull.java index 477fcd3909..a841a2901d 100644 --- a/src/software/amazon/ion/IonNull.java +++ b/src/com/amazon/ion/IonNull.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; /** * The Ion null value, also known as null.null. diff --git a/src/software/amazon/ion/IonNumber.java b/src/com/amazon/ion/IonNumber.java similarity index 55% rename from src/software/amazon/ion/IonNumber.java rename to src/com/amazon/ion/IonNumber.java index 593cb1d336..dd17c55975 100644 --- a/src/software/amazon/ion/IonNumber.java +++ b/src/com/amazon/ion/IonNumber.java @@ -1,18 +1,19 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; /** * The IonNumber interface is a fore runner of a common base for the @@ -23,7 +24,7 @@ * WARNING: This interface should not be implemented or extended by * code outside of this library. */ -public interface IonNumber // TODO amzn/ion-java#53 Complete this interface +public interface IonNumber // TODO amzn/ion-java/issues/53 Complete this interface extends IonValue { } diff --git a/src/software/amazon/ion/IonReader.java b/src/com/amazon/ion/IonReader.java similarity index 88% rename from src/software/amazon/ion/IonReader.java rename to src/com/amazon/ion/IonReader.java index b849f36f36..7675b4b436 100644 --- a/src/software/amazon/ion/IonReader.java +++ b/src/com/amazon/ion/IonReader.java @@ -1,26 +1,27 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; +import com.amazon.ion.facet.Faceted; import java.io.Closeable; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.util.Date; import java.util.Iterator; -import software.amazon.ion.facet.Faceted; /* One design goal is for the readers and writers to be independent of an * IonSystem or ValueFactory and thus independent of particular implementations @@ -40,7 +41,7 @@ * WARNING: This interface should not be implemented or extended by * code outside of this library. * We still have some work to do before this interface is stable. - * See issue amzn/ion-java#11 + * See issue amzn/ion-java/issues/11 *

    * An {@code IonReader} has a "cursor" tracking the current value on * which the reader is positioned. Generally, newly created readers are not @@ -87,7 +88,7 @@ *

    The {@link SeekableReader} Facet

    * This facet is available on all readers except those created from * an {@link java.io.InputStream InputStream}. - * (See issue amzn/ion-java#17.) + * (See issue amzn/ion-java/issues/17.) * It allows the user to reposition the reader to a {@link Span} over the * same reader instance or another reader with the same source. * @@ -104,6 +105,24 @@ public interface IonReader extends Closeable, Faceted { + + /** + * Determines whether there is another value at the current depth; + * in other words whether there is a sibling value that may be reached + * using {@link #next()}. + * This method may be + * called multiple times, which does not move the current position. + *

    + * WARNING: this method alters the internal state of the reader such + * that you cannot reliably get values from the "current" element. The only + * thing you should call after {@code hasNext()} is {@link #next()}! + * + * @deprecated Applications should detect the end of the current level by + * checking for a {@code null} response from {@link #next()}. + */ + @Deprecated + public boolean hasNext(); + /** * Positions this reader on the next sibling after the current value, * returning the type of that value. Once so positioned the contents of @@ -213,6 +232,24 @@ public interface IonReader */ public Iterator iterateTypeAnnotations(); + /** + + /** + * Gets the symbol ID of the field name attached to the current value. + *

    + * This is an "expert method": correct use requires deep understanding + * of the Ion binary format. You almost certainly don't want to use it. + * + * @return the symbol ID of the field name, if the current value is a + * field within a struct. + * If the current value is not a field, or if the symbol ID cannot be + * determined, this method returns a value less than one. + * + * @deprecated Use {@link #getFieldNameSymbol()} instead. + */ + @Deprecated + public int getFieldId(); + /** * Return the field name of the current value. Or null if there is no valid * current value or if the current value is not a field of a struct. @@ -233,6 +270,7 @@ public interface IonReader * @return null if there is no current value or if the current value is * not a field of a struct. * + */ public SymbolToken getFieldNameSymbol(); @@ -355,6 +393,7 @@ public interface IonReader * * @return null if {@link #isNullValue()} * + */ public SymbolToken symbolValue(); diff --git a/src/software/amazon/ion/IonSequence.java b/src/com/amazon/ion/IonSequence.java similarity index 97% rename from src/software/amazon/ion/IonSequence.java rename to src/com/amazon/ion/IonSequence.java index f42a9c6dfe..9621859e59 100644 --- a/src/software/amazon/ion/IonSequence.java +++ b/src/com/amazon/ion/IonSequence.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.util.Collection; import java.util.Iterator; diff --git a/src/software/amazon/ion/IonSexp.java b/src/com/amazon/ion/IonSexp.java similarity index 60% rename from src/software/amazon/ion/IonSexp.java rename to src/com/amazon/ion/IonSexp.java index 4003d1b9e7..961d5a5354 100644 --- a/src/software/amazon/ion/IonSexp.java +++ b/src/com/amazon/ion/IonSexp.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.util.Collection; diff --git a/src/software/amazon/ion/IonString.java b/src/com/amazon/ion/IonString.java similarity index 70% rename from src/software/amazon/ion/IonString.java rename to src/com/amazon/ion/IonString.java index 6c6c311844..c7d471610e 100644 --- a/src/software/amazon/ion/IonString.java +++ b/src/com/amazon/ion/IonString.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; /** * An Ion string value. diff --git a/src/software/amazon/ion/IonStruct.java b/src/com/amazon/ion/IonStruct.java similarity index 96% rename from src/software/amazon/ion/IonStruct.java rename to src/com/amazon/ion/IonStruct.java index f554fdc05f..6ebb0be081 100644 --- a/src/software/amazon/ion/IonStruct.java +++ b/src/com/amazon/ion/IonStruct.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.util.Map; diff --git a/src/software/amazon/ion/IonSymbol.java b/src/com/amazon/ion/IonSymbol.java similarity index 58% rename from src/software/amazon/ion/IonSymbol.java rename to src/com/amazon/ion/IonSymbol.java index e0f4170869..78fc2b711d 100644 --- a/src/software/amazon/ion/IonSymbol.java +++ b/src/com/amazon/ion/IonSymbol.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; /** * An Ion symbol value. @@ -35,6 +36,21 @@ public String stringValue() throws UnknownSymbolException; + /** + * Gets the integer symbol id used in the binary encoding of this symbol. + * + * @return an integer greater than zero, if this value has an associated + * symbol table. Otherwise, return {@link SymbolTable#UNKNOWN_SYMBOL_ID}. + * + * @throws NullValueException if this is null.symbol. + * + * @deprecated Use {@link #symbolValue()} instead. + */ + @Deprecated + public int getSymbolId() + throws NullValueException; + + /** * Returns this value as a symbol token (text + ID). * diff --git a/src/software/amazon/ion/IonSystem.java b/src/com/amazon/ion/IonSystem.java similarity index 92% rename from src/software/amazon/ion/IonSystem.java rename to src/com/amazon/ion/IonSystem.java index 57c2e92987..56871c9ec7 100644 --- a/src/software/amazon/ion/IonSystem.java +++ b/src/com/amazon/ion/IonSystem.java @@ -1,27 +1,28 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; +import com.amazon.ion.system.IonSystemBuilder; +import com.amazon.ion.system.IonTextWriterBuilder; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.Reader; import java.util.Date; import java.util.Iterator; -import software.amazon.ion.system.IonSystemBuilder; -import software.amazon.ion.system.IonTextWriterBuilder; /** * Entry point to all things Ion. @@ -38,7 +39,7 @@ * {@link #clone(IonValue)}. *

    * To create an {@code IonSystem}, - * see {@link software.amazon.ion.system.IonSystemBuilder}. + * see {@link com.amazon.ion.system.IonSystemBuilder}. *

    * Implementations of this interface are safe for use by multiple * threads. @@ -93,7 +94,7 @@ public SymbolTable getSystemSymbolTable(String ionVersionId) * or if any but the first is a system table. * @throws NullPointerException if any import is null. */ - // TODO amzn/ion-java#38 Should we allow substituted imports as valid args? + // TODO amzn/ion-java/issues/38 Should we allow substituted imports as valid args? public SymbolTable newLocalSymbolTable(SymbolTable... imports); @@ -218,7 +219,7 @@ public SymbolTable newSharedSymbolTable(String name, * schema. *

    * Applications should generally use {@link #iterate(InputStream)} - * whenever possible, since this library has much faster UTF-8 decoding + * whenever possible, since this library has much faster Unicode decoding * than the Java IO framework. *

    * Because this library performs its own buffering, it's recommended that @@ -394,7 +395,8 @@ public SymbolTable newSharedSymbolTable(String name, * * @param ionText must not be null. */ - public IonReader newReader(String ionText); + @SuppressWarnings("deprecation") + public IonTextReader newReader(String ionText); /** * Creates an new {@link IonReader} instance over a block of Ion data, @@ -447,7 +449,7 @@ public SymbolTable newSharedSymbolTable(String name, * Creates an new {@link IonReader} instance over Ion text data. *

    * Applications should generally us {@link #newReader(InputStream)} - * whenever possible, since this library has much faster UTF-8 decoding + * whenever possible, since this library has much faster Unicode decoding * than the Java IO framework. *

    * Because this library performs its own buffering, it's recommended that @@ -569,6 +571,32 @@ public IonWriter newTextWriter(Appendable out, SymbolTable... imports) */ public IonWriter newBinaryWriter(OutputStream out, SymbolTable... imports); + /** + * Creates a new writer that will encode binary Ion data. + * + * @return a new {@link IonBinaryWriter} instance; not {@code null}. + * + * @deprecated Use {@link #newBinaryWriter(OutputStream, SymbolTable...)} instead. + */ + @Deprecated + public IonBinaryWriter newBinaryWriter(); + + /** + * Creates a new writer that will encode binary Ion data, + * using the given shared symbol tables as imports. + *

    + * The output stream will be start with an Ion Version Marker and a + * local symbol table that uses the given {@code imports}. + * + * @param imports a sequence of shared symbol tables + * + * @return a new {@link IonBinaryWriter} instance; not {@code null}. + * + * @deprecated Use {@link #newBinaryWriter(OutputStream, SymbolTable...)} instead. + */ + @Deprecated + public IonBinaryWriter newBinaryWriter(SymbolTable... imports); + //------------------------------------------------------------------------- // DOM creation @@ -602,8 +630,8 @@ public IonWriter newTextWriter(Appendable out, SymbolTable... imports) /** * Creates a new datagram, bootstrapped with imported symbol tables. - * Generally an application will use this to acquire a datagram, then adds - * values to it, then calls {@link IonDatagram#getBytes()} + * Generally an application will use this to aquire a datagram, then adds + * values to it, then calls {@link IonDatagram#getBytes(byte[])} * (or similar) to extract binary data. * * @param imports the set of shared symbol tables to import. diff --git a/src/software/amazon/ion/IonText.java b/src/com/amazon/ion/IonText.java similarity index 73% rename from src/software/amazon/ion/IonText.java rename to src/com/amazon/ion/IonText.java index d8057f8179..efb510c521 100644 --- a/src/software/amazon/ion/IonText.java +++ b/src/com/amazon/ion/IonText.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; /** * Common functionality of Ion string and symbol diff --git a/src/com/amazon/ion/IonTextReader.java b/src/com/amazon/ion/IonTextReader.java new file mode 100644 index 0000000000..3a41c6cdda --- /dev/null +++ b/src/com/amazon/ion/IonTextReader.java @@ -0,0 +1,35 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion; + +import com.amazon.ion.util.Spans; + +/** + * Extends {@link IonReader} with capabilites specialized to Ion text data + * streams. + * + * @deprecated Use {@link TextSpan} instead. + * + * @see SpanProvider + * @see TextSpan + * @see Spans#currentSpan(Class, Object) + */ +@Deprecated +public interface IonTextReader + extends IonReader +{ + +} diff --git a/src/software/amazon/ion/IonTimestamp.java b/src/com/amazon/ion/IonTimestamp.java similarity index 95% rename from src/software/amazon/ion/IonTimestamp.java rename to src/com/amazon/ion/IonTimestamp.java index af3fdd3ba7..63bb78c5bc 100644 --- a/src/software/amazon/ion/IonTimestamp.java +++ b/src/com/amazon/ion/IonTimestamp.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.math.BigDecimal; import java.util.Date; @@ -28,7 +29,7 @@ public interface IonTimestamp extends IonValue { - // TODO amzn/ion-java#33 Deprecate setters and getters + // TODO amzn/ion-java/issues/223 Deprecate setters and getters /** * Gets the value of this timestamp in a form suitable for diff --git a/src/software/amazon/ion/IonType.java b/src/com/amazon/ion/IonType.java similarity index 81% rename from src/software/amazon/ion/IonType.java rename to src/com/amazon/ion/IonType.java index dd7edd29d1..1a2e112168 100644 --- a/src/software/amazon/ion/IonType.java +++ b/src/com/amazon/ion/IonType.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; /** @@ -63,6 +64,7 @@ public static boolean isContainer(IonType t) * * @return true when {@code t} is {@link #STRING} or {@link #SYMBOL}. * + */ public static boolean isText(IonType t) { @@ -77,6 +79,7 @@ public static boolean isText(IonType t) * * @return true when {@code t} is {@link #BLOB} or {@link #CLOB}. * + */ public static boolean isLob(IonType t) { diff --git a/src/software/amazon/ion/IonValue.java b/src/com/amazon/ion/IonValue.java similarity index 94% rename from src/software/amazon/ion/IonValue.java rename to src/com/amazon/ion/IonValue.java index d22a58ed43..74e8285886 100644 --- a/src/software/amazon/ion/IonValue.java +++ b/src/com/amazon/ion/IonValue.java @@ -1,22 +1,23 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; +import com.amazon.ion.system.IonTextWriterBuilder; import java.util.Collections; import java.util.concurrent.CountDownLatch; -import software.amazon.ion.system.IonTextWriterBuilder; /** * Base type for all Ion data nodes. @@ -214,7 +215,7 @@ public interface IonValue * local or system symbol table (or null). * * @return the symbol table, or null if this value is not - * currently backed by a symbol table. + * currently backed by binary-encoded data. */ public SymbolTable getSymbolTable(); @@ -238,6 +239,19 @@ public interface IonValue public SymbolToken getFieldNameSymbol(); + /** + * Gets the symbol ID of the field name attached to this value. + * + * @return the symbol ID of the field name, if this is part of an + * {@link IonStruct}. If this is not a field, or if the symbol ID cannot be + * determined, this method returns a value less than one. + * + * @deprecated Use {@link #getFieldNameSymbol()} instead. + */ + @Deprecated + public int getFieldId(); + + /** * Gets the container of this value, * or null if this is not part of one. @@ -264,6 +278,7 @@ public interface IonValue * * @throws UnsupportedOperationException if this is an {@link IonDatagram}. * + */ public IonValue topLevelValue(); @@ -285,6 +300,7 @@ public interface IonValue * @return the (ordered) annotations on the current value, or an empty * array (not {@code null}) if there are none. * + */ public SymbolToken[] getTypeAnnotationSymbols(); @@ -306,6 +322,7 @@ public interface IonValue * * @throws NullPointerException if any of the annotations are null * + */ public void setTypeAnnotations(String... annotations); @@ -321,6 +338,7 @@ public interface IonValue * If null or empty array, then all annotations are removed. * Any duplicates are preserved. * + */ public void setTypeAnnotationSymbols(SymbolToken... annotations); @@ -358,6 +376,7 @@ public interface IonValue * and performs a deep write, including the contents of * any containers encountered. * + */ public void writeTo(IonWriter writer); @@ -428,7 +447,7 @@ public IonValue clone() * two strings returned by this method. *

    * For more configurable rendering, see - * {@link software.amazon.ion.system.IonTextWriterBuilder}. + * {@link com.amazon.ion.system.IonTextWriterBuilder}. *

    * This is not the correct way to retrieve the content of an * {@link IonString} or {@link IonSymbol}! @@ -460,6 +479,7 @@ public IonValue clone() * * @return Ion text data equivalent to this value. * + */ public String toPrettyString(); @@ -473,6 +493,7 @@ public IonValue clone() * * @return Ion text data equivalent to this value. * + */ public String toString(IonTextWriterBuilder writerBuilder); @@ -484,7 +505,7 @@ public IonValue clone() * traverses the hierarchy, and as such it should be considered an * expensive operation. * - * @see software.amazon.ion.util.Equivalence + * @see com.amazon.ion.util.Equivalence * * @param other The value to compare with. * diff --git a/src/software/amazon/ion/IonWriter.java b/src/com/amazon/ion/IonWriter.java similarity index 88% rename from src/software/amazon/ion/IonWriter.java rename to src/com/amazon/ion/IonWriter.java index 3dc1588692..fe1e0e960d 100644 --- a/src/software/amazon/ion/IonWriter.java +++ b/src/com/amazon/ion/IonWriter.java @@ -1,27 +1,30 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; +import com.amazon.ion.facet.Faceted; +import com.amazon.ion.system.IonTextWriterBuilder; +import com.amazon.ion.util.IonStreamUtils; import java.io.ByteArrayOutputStream; import java.io.Closeable; import java.io.Flushable; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; -import software.amazon.ion.system.IonTextWriterBuilder; -import software.amazon.ion.util.IonStreamUtils; +import java.util.Date; /** * Writes Ion data to an output source. @@ -32,7 +35,7 @@ * WARNING: This interface should not be implemented or extended by * code outside of this library. * We still have some work to do before this interface is stable. - * See issue amzn/ion-java#10 + * See issue amzn/ion-java/issues/10 *

    * A value is written via the set of typed {@code write*()} methods such as * {@link #writeBool(boolean)} and {@link #writeInt(long)}. @@ -75,7 +78,7 @@ * @see IonTextWriterBuilder */ public interface IonWriter - extends Closeable, Flushable + extends Closeable, Flushable, Faceted { /** * Gets the symbol table that is currently in use by the writer. @@ -138,7 +141,7 @@ public interface IonWriter * as if it were preceded by an Ion version marker, resetting the stream * context as if this were a new stream. (Whether or not an IVM is written * may depend upon the writer's configuration; see - * {@link software.amazon.ion.system.IonWriterBuilder.IvmMinimizing + * {@link com.amazon.ion.system.IonWriterBuilder.IvmMinimizing * IvmMinimizing}.) *

    * This feature can be used to flush reliably before writing more values. @@ -242,6 +245,7 @@ public interface IonWriter * @param annotations * If null or empty, any pending annotations are cleared. * + */ public void setTypeAnnotationSymbols(SymbolToken... annotations); @@ -297,6 +301,19 @@ public interface IonWriter //========================================================================= // Value writing + /** + * writes the contents of the passed in Ion value to the output. + *

    + * This method also writes annotations and field names (if in a struct), + * and performs a deep write, including the contents of + * any containers encountered. + * + * @param value may be null, in which case this method does nothing. + * + * @deprecated Use {@link IonValue#writeTo(IonWriter)} instead. + */ + @Deprecated + public void writeValue(IonValue value) throws IOException; /** * Writes the current value from a reader. @@ -391,6 +408,19 @@ public interface IonWriter */ public void writeTimestamp(Timestamp value) throws IOException; + /** + * writes the passed in Date (in milliseconds since the epoch) as an + * IonTimestamp. The Date value is treated as a UTC value with an + * unknown timezone offset (a z value). + * @param value java.util Date holding the UTC timestamp; + * may be null to represent {@code null.timestamp}. + * + * @deprecated Use {@link #writeTimestamp(Timestamp) + * IonWriter.writeTimestamp(}{@link Timestamp#forDateZ(Date) Timestamp.forDateZ(Date))} instead. + */ + @Deprecated + public void writeTimestampUTC(Date value) throws IOException; + /** * Writes the text of an Ion symbol value. * @@ -403,17 +433,32 @@ public interface IonWriter /** * Writes the content of an Ion symbol value. - * If the token has text, it is considered canonical and any SID in the - * token may be ignored or reassigned. * * @param content may be null to represent {@code null.symbol}. * * @throws IllegalArgumentException if the value contains an invalid UTF-16 * surrogate pair. * + */ public void writeSymbolToken(SymbolToken content) throws IOException; + /** + * Write an Ion version marker symbol to the output. This + * is the $ion_1_0 value currently (in later versions the + * number may change). In text output this appears as the + * text symbol. In binary this will be the symbol id if + * the writer is in a list, sexp or struct. If the writer + * is currently at the top level this will write the + * "magic cookie" value. + * + * Writing a version marker will reset the symbol table + * to be the system symbol table. + * + * @throws IOException + */ +// public void writeIonVersionMarker() throws IOException; + /** * Writes a {@link java.lang.String} as an Ion string. Since Ion strings are * UTF-8 and Java Strings are Unicode 16. As such the resulting diff --git a/src/software/amazon/ion/NullValueException.java b/src/com/amazon/ion/NullValueException.java similarity index 52% rename from src/software/amazon/ion/NullValueException.java rename to src/com/amazon/ion/NullValueException.java index 3b2225b626..ac43f07e0b 100644 --- a/src/software/amazon/ion/NullValueException.java +++ b/src/com/amazon/ion/NullValueException.java @@ -1,8 +1,19 @@ /* - * Copyright (c) 2007 Amazon.com, Inc. All rights reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; /** @@ -14,6 +25,7 @@ public class NullValueException { private static final long serialVersionUID = 1L; + public NullValueException() { super(); diff --git a/src/software/amazon/ion/OffsetSpan.java b/src/com/amazon/ion/OffsetSpan.java similarity index 77% rename from src/software/amazon/ion/OffsetSpan.java rename to src/com/amazon/ion/OffsetSpan.java index a4341ee890..6ed135351b 100644 --- a/src/software/amazon/ion/OffsetSpan.java +++ b/src/com/amazon/ion/OffsetSpan.java @@ -1,22 +1,23 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; +import com.amazon.ion.facet.Faceted; +import com.amazon.ion.facet.Facets; import java.io.InputStream; -import software.amazon.ion.facet.Faceted; -import software.amazon.ion.facet.Facets; /** * Exposes the positions of a {@link Span} in the form of zero-based offsets diff --git a/src/com/amazon/ion/RawValueSpanProvider.java b/src/com/amazon/ion/RawValueSpanProvider.java new file mode 100644 index 0000000000..5179d7eda7 --- /dev/null +++ b/src/com/amazon/ion/RawValueSpanProvider.java @@ -0,0 +1,59 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion; + +/** + * Provide the ability to retrieve {@link Span}s (abstract value positions) + * of raw Ion values, excluding type and length octets. + *

    + * WARNING: This interface should not be implemented or extended by + * code outside of this library. + *

    + * This functionality may be accessed as a facet of binary {@link IonReader}s. + * + * @deprecated This is a private API subject to change without notice. + */ +@Deprecated +public interface RawValueSpanProvider +{ + + /** + * Constructs a Span, which may be faceted as an {@link OffsetSpan}, that + * provides the start and end byte positions of the current value. + *

    + * NOTE: for Ion {@code int} values, users should not use these byte positions to + * determine which primitive (if any) the value can fit into, because the + * sign of Ion {@code int} values is encoded into the type ID byte, which is + * not included in this span. + *

    + * WARNING: Spans provided by this method are not compatible with the + * {@link SeekableReader} facet, because they lack the type ID and length + * bytes that are important when reconstructing the context on seek. For + * {@link SpanProvider#currentSpan()} should be used to retrieve seekable + * Spans. + * @return the constructed Span + */ + public Span valueSpan(); + + /** + * @return the byte[] that backs this span. This span's start and end positions + * may be used as indices into the returned buffer. NOTE: does NOT perform + * a copy of the buffer; care must be taken not to mutate and corrupt the + * data. + */ + public byte[] buffer(); + +} diff --git a/src/software/amazon/ion/ReadOnlyValueException.java b/src/com/amazon/ion/ReadOnlyValueException.java similarity index 63% rename from src/software/amazon/ion/ReadOnlyValueException.java rename to src/com/amazon/ion/ReadOnlyValueException.java index e9f98daa70..daa3ed3763 100644 --- a/src/software/amazon/ion/ReadOnlyValueException.java +++ b/src/com/amazon/ion/ReadOnlyValueException.java @@ -1,18 +1,19 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; /** diff --git a/src/software/amazon/ion/SeekableReader.java b/src/com/amazon/ion/SeekableReader.java similarity index 89% rename from src/software/amazon/ion/SeekableReader.java rename to src/com/amazon/ion/SeekableReader.java index 59a6f9edd2..0603e6c4d8 100644 --- a/src/software/amazon/ion/SeekableReader.java +++ b/src/com/amazon/ion/SeekableReader.java @@ -1,18 +1,19 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; /** * An {@link IonReader} facet providing the ability to retrieve diff --git a/src/software/amazon/ion/Span.java b/src/com/amazon/ion/Span.java similarity index 82% rename from src/software/amazon/ion/Span.java rename to src/com/amazon/ion/Span.java index d3c7ec9e7f..c40bb6173e 100644 --- a/src/software/amazon/ion/Span.java +++ b/src/com/amazon/ion/Span.java @@ -1,21 +1,22 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import software.amazon.ion.facet.Faceted; -import software.amazon.ion.util.Spans; +import com.amazon.ion.facet.Faceted; +import com.amazon.ion.util.Spans; /** * An immutable reference to a consecutive sequence of values (perhaps diff --git a/src/software/amazon/ion/SpanProvider.java b/src/com/amazon/ion/SpanProvider.java similarity index 76% rename from src/software/amazon/ion/SpanProvider.java rename to src/com/amazon/ion/SpanProvider.java index 3b22b1c1df..a506a6e719 100644 --- a/src/software/amazon/ion/SpanProvider.java +++ b/src/com/amazon/ion/SpanProvider.java @@ -1,18 +1,19 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; /** * Provide the ability to retrieve {@link Span}s (abstract value positions) diff --git a/src/software/amazon/ion/SubstituteSymbolTableException.java b/src/com/amazon/ion/SubstituteSymbolTableException.java similarity index 61% rename from src/software/amazon/ion/SubstituteSymbolTableException.java rename to src/com/amazon/ion/SubstituteSymbolTableException.java index 70e56e0cf9..7472ec8657 100644 --- a/src/software/amazon/ion/SubstituteSymbolTableException.java +++ b/src/com/amazon/ion/SubstituteSymbolTableException.java @@ -1,18 +1,19 @@ /* - * Copyright 2013-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; /** * An error caused by an operation that requires an exact match on an import @@ -20,7 +21,7 @@ * * @see SymbolTable#isSubstitute() */ -// TODO amzn/ion-java#40 Provide some useful info to assist callers with handling this +// TODO amzn/ion-java/issues/40 Provide some useful info to assist callers with handling this // exception. E.g. reference to the substitute import in violation. public class SubstituteSymbolTableException extends IonException diff --git a/src/software/amazon/ion/SymbolTable.java b/src/com/amazon/ion/SymbolTable.java similarity index 95% rename from src/software/amazon/ion/SymbolTable.java rename to src/com/amazon/ion/SymbolTable.java index e9651e22e4..e73e1da738 100644 --- a/src/software/amazon/ion/SymbolTable.java +++ b/src/com/amazon/ion/SymbolTable.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.io.IOException; import java.util.Iterator; @@ -42,7 +43,7 @@ * Implementations of this interface are safe for use by multiple * threads. * - * @see Ion Symbols page + * @see Ion Symbols page */ public interface SymbolTable { @@ -123,6 +124,7 @@ public interface SymbolTable * * @see #makeReadOnly() * + */ public boolean isReadOnly(); @@ -135,6 +137,7 @@ public interface SymbolTable * * @see #isReadOnly() * + */ public void makeReadOnly(); @@ -224,6 +227,7 @@ public interface SymbolTable * * @see #find(String) * + */ public SymbolToken intern(String text); @@ -241,6 +245,7 @@ public interface SymbolTable * * @see #intern(String) * + */ public SymbolToken find(String text); diff --git a/src/software/amazon/ion/SymbolToken.java b/src/com/amazon/ion/SymbolToken.java similarity index 82% rename from src/software/amazon/ion/SymbolToken.java rename to src/com/amazon/ion/SymbolToken.java index 3a88a02c58..a531823081 100644 --- a/src/software/amazon/ion/SymbolToken.java +++ b/src/com/amazon/ion/SymbolToken.java @@ -1,18 +1,19 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; /** diff --git a/src/software/amazon/ion/SystemSymbols.java b/src/com/amazon/ion/SystemSymbols.java similarity index 88% rename from src/software/amazon/ion/SystemSymbols.java rename to src/com/amazon/ion/SystemSymbols.java index 62e50c69e6..57d5745947 100644 --- a/src/software/amazon/ion/SystemSymbols.java +++ b/src/com/amazon/ion/SystemSymbols.java @@ -1,18 +1,19 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; /** * Constants for symbols defined by the Ion specification. diff --git a/src/software/amazon/ion/TextSpan.java b/src/com/amazon/ion/TextSpan.java similarity index 80% rename from src/software/amazon/ion/TextSpan.java rename to src/com/amazon/ion/TextSpan.java index db76c60e39..c81772913f 100644 --- a/src/software/amazon/ion/TextSpan.java +++ b/src/com/amazon/ion/TextSpan.java @@ -1,21 +1,22 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import software.amazon.ion.facet.Faceted; -import software.amazon.ion.util.Spans; +import com.amazon.ion.facet.Faceted; +import com.amazon.ion.util.Spans; /** * Exposes the positions of a {@link Span} in the form of one-based diff --git a/src/software/amazon/ion/Timestamp.java b/src/com/amazon/ion/Timestamp.java similarity index 95% rename from src/software/amazon/ion/Timestamp.java rename to src/com/amazon/ion/Timestamp.java index 1dcd2587f2..39df6e0827 100644 --- a/src/software/amazon/ion/Timestamp.java +++ b/src/com/amazon/ion/Timestamp.java @@ -1,22 +1,25 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static software.amazon.ion.impl.PrivateUtils.safeEquals; -import static software.amazon.ion.util.IonTextUtils.printCodePointAsString; +import static com.amazon.ion.impl._Private_Utils.safeEquals; +import static com.amazon.ion.util.IonTextUtils.printCodePointAsString; +import com.amazon.ion.impl._Private_Utils; +import com.amazon.ion.util.IonTextUtils; import java.io.IOException; import java.math.BigDecimal; import java.math.RoundingMode; @@ -24,9 +27,6 @@ import java.util.Date; import java.util.GregorianCalendar; -import software.amazon.ion.impl.PrivateUtils; -import software.amazon.ion.util.IonTextUtils; - /** * An immutable representation of a point in time. Ion defines a simple * representation of time based on Coordinated Universal Time (UTC). @@ -121,7 +121,16 @@ public static enum Precision { DAY (FLAG_YEAR | FLAG_MONTH | FLAG_DAY), // HOUR is not a supported precision per https://www.w3.org/TR/NOTE-datetime MINUTE (FLAG_YEAR | FLAG_MONTH | FLAG_DAY | FLAG_MINUTE), - SECOND (FLAG_YEAR | FLAG_MONTH | FLAG_DAY | FLAG_MINUTE | FLAG_SECOND); + SECOND (FLAG_YEAR | FLAG_MONTH | FLAG_DAY | FLAG_MINUTE | FLAG_SECOND), + + /** + * DEPRECATED! Treating the fractional part of seconds separate from + * the integer part has led to countless bugs. We intend to combine + * the two under the SECOND precision. + * + */ + @Deprecated + FRACTION(FLAG_YEAR | FLAG_MONTH | FLAG_DAY | FLAG_MINUTE | FLAG_SECOND); /** Bit flags for the precision. */ private final int flags; @@ -131,15 +140,11 @@ private Precision(int flags) this.flags = flags; } - private boolean alwaysUnknownOffset() - { - return this.ordinal() <= DAY.ordinal(); - } - public boolean includes(Precision isIncluded) { switch (isIncluded) { + case FRACTION: case SECOND: return (flags & FLAG_SECOND) != 0; case MINUTE: return (flags & FLAG_MINUTE) != 0; case DAY: return (flags & FLAG_DAY) != 0; @@ -149,6 +154,11 @@ public boolean includes(Precision isIncluded) } throw new IllegalStateException("unrecognized precision" + isIncluded); } + + private boolean alwaysUnknownOffset() + { + return this.ordinal() <= DAY.ordinal(); + } } private static final int HASH_SIGNATURE = @@ -354,6 +364,7 @@ private void set_fields_from_calendar(Calendar cal, boolean calendarHasMilliseconds = cal.isSet(Calendar.MILLISECOND); switch (this._precision) { + case FRACTION: case SECOND: this._second = checkAndCastSecond(cal.get(Calendar.SECOND)); if (calendarHasMilliseconds) { @@ -431,9 +442,11 @@ private Timestamp(int zyear, int zmonth) * Creates a new Timestamp, precise to the day, with unknown local offset. *

    * This is equivalent to the corresponding Ion value {@code YYYY-MM-DD}. + * + * @deprecated Use {@link #forDay(int, int, int)} instead. */ @Deprecated - private Timestamp(int zyear, int zmonth, int zday) + public Timestamp(int zyear, int zmonth, int zday) { this(Precision.DAY, zyear, zmonth, zday, NO_HOURS, NO_MINUTES, NO_SECONDS, NO_FRACTIONAL_SECONDS, UNKNOWN_OFFSET, APPLY_OFFSET_NO); } @@ -450,9 +463,11 @@ private Timestamp(int zyear, int zmonth, int zday) * @param offset * the local offset from UTC, measured in minutes; * may be {@code null} to represent an unknown local offset + * + * @deprecated Use {@link #forMinute(int, int, int, int, int, Integer)} instead. */ @Deprecated - private Timestamp(int year, int month, int day, + public Timestamp(int year, int month, int day, int hour, int minute, Integer offset) { @@ -470,15 +485,46 @@ private Timestamp(int year, int month, int day, * @param offset * the local offset from UTC, measured in minutes; * may be {@code null} to represent an unknown local offset. + * + * @deprecated Use {@link #forSecond(int, int, int, int, int, int, Integer)} instead. */ @Deprecated - private Timestamp(int year, int month, int day, + public Timestamp(int year, int month, int day, int hour, int minute, int second, Integer offset) { this(Precision.SECOND, year, month, day, hour, minute, second, NO_FRACTIONAL_SECONDS, offset, APPLY_OFFSET_YES); } + /** + * Creates a new Timestamp, precise to the second or fractional second, + * with a given local offset. + *

    + * This is equivalent to the corresponding Ion value + * {@code YYYY-MM-DDThh:mm:ss.fff+-oo:oo}, where {@code oo:oo} represents + * the hour and minutes of the local offset from UTC, and {@code fff} + * represents the fractional seconds. + * + * @param frac + * the fractional seconds; must not be {@code null}; if negative, + * its absolute value is used + * @param offset + * the local offset from UTC, measured in minutes; + * may be {@code null} to represent an unknown local offset + * + * @throws NullPointerException if {@code frac} is {@code null} + * + * @deprecated Use {@link #forSecond(int, int, int, int, int, BigDecimal, Integer)} + * instead. + */ + @Deprecated + public Timestamp(int year, int month, int day, + int hour, int minute, int second, BigDecimal frac, + Integer offset) + { + this(Precision.SECOND, year, month, day, hour, minute, second, frac, offset, APPLY_OFFSET_YES); + } + /** * Creates a new Timestamp from the individual time components. The * individual time components are expected to be in UTC, @@ -506,6 +552,7 @@ private Timestamp(Precision p, int zyear, int zmonth, int zday, switch (p) { default: throw new IllegalArgumentException("invalid Precision passed to constructor"); + case FRACTION: case SECOND: if (frac == null || frac.equals(BigDecimal.ZERO)) { @@ -587,8 +634,6 @@ private Timestamp(Precision p, int zyear, int zmonth, int zday, * @param offset * the local offset from UTC, measured in minutes; * may be {@code null} to represent an unknown local offset. - * - * @deprecated This is an internal API that is subject to change without notice. */ @Deprecated public static Timestamp @@ -620,9 +665,11 @@ private Timestamp(Precision p, int zyear, int zmonth, int zday, * * @throws IllegalArgumentException * if {@code cal} has no appropriate calendar fields set. + * + * @deprecated Use {@link #forCalendar(Calendar)} instead. */ @Deprecated - private Timestamp(Calendar cal) + public Timestamp(Calendar cal) { Precision precision; @@ -686,6 +733,7 @@ private Timestamp(BigDecimal millis, Precision precision, Integer localOffset) case MINUTE: _second = 0; case SECOND: + case FRACTION: } _offset = localOffset; @@ -693,7 +741,7 @@ private Timestamp(BigDecimal millis, Precision precision, Integer localOffset) // a Calendar can handle. Set the _fraction here so that extra precision (if any) is not lost. // However, don't set the fraction if the given BigDecimal does not have precision at least to the tenth of // a second. - if (precision.includes(Precision.SECOND) && millis.scale() > -3) { + if ((precision.includes(Precision.SECOND)) && millis.scale() > -3) { BigDecimal secs = millis.movePointLeft(3); BigDecimal secsDown = fastRoundZeroFloor(secs); _fraction = secs.subtract(secsDown); @@ -735,9 +783,11 @@ private Timestamp(BigDecimal millis, Precision precision, Integer localOffset) * may be {@code null} to represent an unknown local offset * * @throws NullPointerException if {@code millis} is {@code null} + * + * @deprecated Use {@link #forMillis(BigDecimal, Integer)} instead. */ @Deprecated - private Timestamp(BigDecimal millis, Integer localOffset) + public Timestamp(BigDecimal millis, Integer localOffset) { if (millis == null) throw new NullPointerException("millis is null"); @@ -801,9 +851,11 @@ private static void throwTimestampOutOfRangeError(Number millis) { * @param localOffset * the local offset from UTC, measured in minutes; * may be {@code null} to represent an unknown local offset. + * + * @deprecated Use {@link #forMillis(long, Integer)} instead. */ @Deprecated - private Timestamp(long millis, Integer localOffset) + public Timestamp(long millis, Integer localOffset) { if(millis < MINIMUM_TIMESTAMP_IN_MILLIS || millis >= MAXIMUM_TIMESTAMP_IN_MILLIS) { throwTimestampOutOfRangeError(millis); @@ -861,7 +913,7 @@ private static IllegalArgumentException fail(CharSequence input) * @return * {@code null} if the {@code CharSequence} is "null.timestamp" * - * @see Ion Timestamp Page + * @see Ion Timestamp Page * @see W3C Note on Date and Time Formats */ public static Timestamp valueOf(CharSequence ionFormattedTimestamp) @@ -970,7 +1022,6 @@ public static Timestamp valueOf(CharSequence ionFormattedTimestamp) { break; } - precision = Precision.SECOND; pos = END_OF_SECONDS + 1; while (length > pos && Character.isDigit(in.charAt(pos))) { pos++; @@ -1212,6 +1263,7 @@ public static Timestamp forDay(int yearZ, int monthZ, int dayZ) * the local offset from UTC, measured in minutes; * may be {@code null} to represent an unknown local offset * + */ public static Timestamp forMinute(int year, int month, int day, int hour, int minute, @@ -1232,6 +1284,7 @@ public static Timestamp forMinute(int year, int month, int day, * the local offset from UTC, measured in minutes; * may be {@code null} to represent an unknown local offset * + */ public static Timestamp forSecond(int year, int month, int day, int hour, int minute, int second, @@ -1255,6 +1308,7 @@ public static Timestamp forSecond(int year, int month, int day, * the local offset from UTC, measured in minutes; * may be {@code null} to represent an unknown local offset * + */ public static Timestamp forSecond(int year, int month, int day, int hour, int minute, BigDecimal second, @@ -1280,6 +1334,7 @@ public static Timestamp forSecond(int year, int month, int day, * the local offset from UTC, measured in minutes; * may be {@code null} to represent an unknown local offset. * + */ public static Timestamp forMillis(long millis, Integer localOffset) { @@ -1319,6 +1374,7 @@ public static Timestamp forMillis(long millis, Integer localOffset) * * @throws NullPointerException if {@code millis} is {@code null} * + */ public static Timestamp forMillis(BigDecimal millis, Integer localOffset) { @@ -1335,6 +1391,7 @@ public static Timestamp forMillis(BigDecimal millis, Integer localOffset) * field set in the {@code Calendar}; * or {@code null} if {@code calendar} is {@code null} * + */ public static Timestamp forCalendar(Calendar calendar) { @@ -1353,6 +1410,7 @@ public static Timestamp forCalendar(Calendar calendar) * a new Timestamp instance, in UTC, precise to the millisecond; * {@code null} if {@code date} is {@code null} * + */ public static Timestamp forDateZ(Date date) { @@ -1375,6 +1433,7 @@ public static Timestamp forDateZ(Date date) * nanosecond * {@code null} if {@code sqlTimestamp} is {@code null} * + */ public static Timestamp forSqlTimestampZ(java.sql.Timestamp sqlTimestamp) { @@ -1414,6 +1473,7 @@ public static Timestamp now() * a new Timestamp instance, in UTC, representing the current * time. * + */ public static Timestamp nowZ() { @@ -1450,10 +1510,11 @@ public Date dateValue() * * @return a new {@code Calendar} instance, in its local time. * + */ public Calendar calendarValue() { - Calendar cal = new GregorianCalendar(PrivateUtils.UTC); + Calendar cal = new GregorianCalendar(_Private_Utils.UTC); long millis = getMillis(); Integer offset = _offset; @@ -1537,6 +1598,7 @@ public BigDecimal getDecimalMillis() case DAY: case MINUTE: case SECOND: + case FRACTION: long millis = Date.UTC(this._year - 1900, this._month - 1, this._day, this._hour, this._minute, this._second); BigDecimal dec = BigDecimal.valueOf(millis); if (_fraction != null) { @@ -1721,6 +1783,29 @@ public BigDecimal getDecimalSecond() } + /** + * Returns the fractional second of this Timestamp. + *

    + * Fractional seconds are not affected by local offsets. + * As such, this method produces the same output as + * {@link #getZFractionalSecond()}. + * + * @return + * a BigDecimal within the range [0, 1); + * {@code null} is returned if the Timestamp isn't + * precise to the fractional second + * + * @see #getZFractionalSecond() + * + * Use {@link #getDecimalSecond()} instead. + */ + @Deprecated + public BigDecimal getFractionalSecond() + { + return this._fraction; + } + + /** * Returns the year of this Timestamp, in UTC. * @@ -1831,13 +1916,17 @@ public BigDecimal getZDecimalSecond() * Returns the fractional second of this Timestamp. *

    * Fractional seconds are not affected by local offsets. + * As such, this method produces the same output as + * {@link #getFractionalSecond()}. * * @return * a BigDecimal within the range [0, 1); * {@code null} is returned if the Timestamp isn't * precise to the fractional second * - * @deprecated This is an internal API that is subject to change without notice. + * @see #getFractionalSecond() + * + * @deprecated Use {@link #getZDecimalSecond()} instead. */ @Deprecated public BigDecimal getZFractionalSecond() @@ -1904,6 +1993,7 @@ public String toString() return buffer.toString(); } + /** * Returns the string representation (in Ion format) of this Timestamp * in UTC. @@ -1921,11 +2011,12 @@ public String toZString() catch (IOException e) { throw new RuntimeException("Exception printing to StringBuilder", - e); + e); } return buffer.toString(); } + /** * Prints to an {@code Appendable} the string representation (in Ion format) * of this Timestamp in its local time. @@ -1954,6 +2045,7 @@ public void print(Appendable out) print(out, adjusted); } + /** * Prints to an {@code Appendable} the string representation (in Ion format) * of this Timestamp in UTC. @@ -1982,6 +2074,7 @@ public void printZ(Appendable out) } case MINUTE: case SECOND: + case FRACTION: { Timestamp ztime = this.clone(); ztime._offset = UTC_OFFSET; @@ -1991,6 +2084,7 @@ public void printZ(Appendable out) } } + /** * helper for print(out) and printZ(out) so that printZ can create * a zulu time and pass it directly and print can apply the local @@ -2124,6 +2218,7 @@ private static void print_fractional_digits(Appendable out, BigDecimal value) public final Timestamp adjustMillis(long amount) { if (amount == 0) return this; Timestamp ts = addMillisForPrecision(amount, _precision, false); + //ts._precision = _precision; ts.clearUnusedPrecision(); if (ts._precision.includes(Precision.SECOND)) { // Maintain the same amount of fractional precision. @@ -2152,7 +2247,8 @@ public final Timestamp adjustMillis(long amount) { * * @param amount a number of milliseconds. */ - public final Timestamp addMillis(long amount) { + public final Timestamp addMillis(long amount) + { if (amount == 0 && _precision.includes(Precision.SECOND) && _fraction != null && _fraction.scale() >= 3) { // Zero milliseconds are to be added, and the precision does not need to be increased. return this; @@ -2465,7 +2561,8 @@ public int hashCode() result ^= (result << 19) ^ (result >> 13); - result = prime * result + this._precision.toString().hashCode(); + Precision precision = this._precision == Precision.FRACTION ? Precision.SECOND : this._precision; + result = prime * result + precision.toString().hashCode(); result ^= (result << 19) ^ (result >> 13); @@ -2608,9 +2705,13 @@ public boolean equals(Timestamp t) if (this == t) return true; if (t == null) return false; + // FRACTION precision is equivalent to SECOND precision. + Precision thisPrecision = this._precision.includes(Precision.SECOND) ? Precision.SECOND : this._precision; + Precision thatPrecision = t._precision.includes(Precision.SECOND) ? Precision.SECOND : t._precision; + // if the precisions are not the same the values are not // precision doesn't matter WRT equality - if (this._precision != t._precision) return false; + if (thisPrecision != thatPrecision) return false; // if the local offset are not the same the values are not if (this._offset == null) { @@ -2635,7 +2736,7 @@ public boolean equals(Timestamp t) // we only look at the fraction if we know that it's actually there if ((this._fraction != null && t._fraction == null) - || (this._fraction == null && t._fraction != null)) { + || (this._fraction == null && t._fraction != null)) { // one of the fractions are null return false; } diff --git a/src/software/amazon/ion/UnexpectedEofException.java b/src/com/amazon/ion/UnexpectedEofException.java similarity index 70% rename from src/software/amazon/ion/UnexpectedEofException.java rename to src/com/amazon/ion/UnexpectedEofException.java index a509009899..95280cb7a3 100644 --- a/src/software/amazon/ion/UnexpectedEofException.java +++ b/src/com/amazon/ion/UnexpectedEofException.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; /** @@ -24,6 +25,7 @@ public class UnexpectedEofException { private static final long serialVersionUID = 1L; + public UnexpectedEofException() { super(); diff --git a/src/software/amazon/ion/UnknownSymbolException.java b/src/com/amazon/ion/UnknownSymbolException.java similarity index 75% rename from src/software/amazon/ion/UnknownSymbolException.java rename to src/com/amazon/ion/UnknownSymbolException.java index e3cba54630..9cc7a50ec9 100644 --- a/src/software/amazon/ion/UnknownSymbolException.java +++ b/src/com/amazon/ion/UnknownSymbolException.java @@ -1,18 +1,19 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; /** * An error caused by a symbol ID that could not be translated into text diff --git a/src/software/amazon/ion/UnsupportedIonVersionException.java b/src/com/amazon/ion/UnsupportedIonVersionException.java similarity index 69% rename from src/software/amazon/ion/UnsupportedIonVersionException.java rename to src/com/amazon/ion/UnsupportedIonVersionException.java index 96102070f7..275381eee2 100644 --- a/src/software/amazon/ion/UnsupportedIonVersionException.java +++ b/src/com/amazon/ion/UnsupportedIonVersionException.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; /** * An error caused by a request for an Ion version that is not supported by diff --git a/src/software/amazon/ion/ValueFactory.java b/src/com/amazon/ion/ValueFactory.java similarity index 86% rename from src/software/amazon/ion/ValueFactory.java rename to src/com/amazon/ion/ValueFactory.java index 7b9f5d9b07..a74f7c0f19 100644 --- a/src/software/amazon/ion/ValueFactory.java +++ b/src/com/amazon/ion/ValueFactory.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.math.BigDecimal; import java.math.BigInteger; @@ -249,17 +250,42 @@ public interface ValueFactory */ public IonList newEmptyList(); + /** + * Constructs a new list with given children. + * + * @param values + * the initial set of children. If null, then the new + * instance will have {@link IonValue#isNullValue()} == true. + * + * @throws ContainedValueException + * if any value in {@code values} + * has {@link IonValue#getContainer()} != null. + * @throws NullPointerException + * if any value in {@code values} is null. + * @throws IllegalArgumentException + * if any value in {@code values} is an {@link IonDatagram}. + * + * @deprecated This method can be invoked + * (accidentally and incorrectly) with an {@link IonSequence}! + * Use either {@link #newList(IonValue...)} + * or {@link #newList(IonValue...) newList()}{@link + * IonSequence#addAll(Collection) .addAll(Collection)}. + */ + @Deprecated + public IonList newList(Collection values) + throws ContainedValueException, NullPointerException; + /** * Constructs a new {@code list} with the given child. - * - * @param child the initial child of the new list. *

    - * This method is temporary until "newList(Collection)" is + * This method is temporary until {@link #newList(Collection)} is * removed. It's sole purpose is to avoid the doomed attempt to add all * of the parameter's children to the new list; that will always throw * {@link ContainedValueException}. * + * @param child the initial child of the new list. + * * @throws NullPointerException if {@code child} is null. * @throws IllegalArgumentException if {@code child} is an {@link IonDatagram}. * @throws ContainedValueException @@ -357,10 +383,36 @@ public IonList newList(IonValue... children) */ public IonSexp newEmptySexp(); + + /** + * Constructs a new sexp with given child elements. + * + * @param values + * the initial set of children. If null, then the new + * instance will have {@link IonValue#isNullValue()} == true. + * + * @throws ContainedValueException + * if any value in {@code values} + * has {@link IonValue#getContainer()} != null. + * @throws NullPointerException + * if any value in {@code values} is null. + * @throws IllegalArgumentException + * if any value in {@code values} is an {@link IonDatagram}. + * + * @deprecated This method can be invoked + * (accidentally and incorrectly) with an {@link IonSequence}! + * Use either {@link #newSexp(IonValue...)} + * or {@link #newSexp(IonValue...) newSexp()}{@link + * IonSequence#addAll(Collection) .addAll(Collection)}. + */ + @Deprecated + public IonSexp newSexp(Collection values) + throws ContainedValueException, NullPointerException; + /** * Constructs a new {@code sexp} with the given child. *

    - * This method is temporary until "newSexp(Collection)" is + * This method is temporary until {@link #newSexp(Collection)} is * removed. It's sole purpose is to avoid the doomed attempt to add all * of the parameter's children to the new sequence; that will always throw * {@link ContainedValueException}. diff --git a/src/software/amazon/ion/ValueVisitor.java b/src/com/amazon/ion/ValueVisitor.java similarity index 58% rename from src/software/amazon/ion/ValueVisitor.java rename to src/com/amazon/ion/ValueVisitor.java index b9b761e01e..2aa3043dff 100644 --- a/src/software/amazon/ion/ValueVisitor.java +++ b/src/com/amazon/ion/ValueVisitor.java @@ -1,10 +1,21 @@ /* - * Copyright (c) 2007-2013 Amazon.com, Inc. All rights reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import software.amazon.ion.util.AbstractValueVisitor; +import com.amazon.ion.util.AbstractValueVisitor; /** * A Visitor for the Ion value hierarchy. diff --git a/src/software/amazon/ion/apps/BaseApp.java b/src/com/amazon/ion/apps/BaseApp.java similarity index 91% rename from src/software/amazon/ion/apps/BaseApp.java rename to src/com/amazon/ion/apps/BaseApp.java index 2f635e30d2..d5890bb5ae 100644 --- a/src/software/amazon/ion/apps/BaseApp.java +++ b/src/com/amazon/ion/apps/BaseApp.java @@ -1,19 +1,27 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.apps; +package com.amazon.ion.apps; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonException; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonSystem; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.system.IonSystemBuilder; +import com.amazon.ion.system.SimpleCatalog; import java.io.BufferedInputStream; import java.io.ByteArrayOutputStream; import java.io.File; @@ -21,13 +29,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonException; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonSystem; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.system.IonSystemBuilder; -import software.amazon.ion.system.SimpleCatalog; /** * ion_encode ion_print diff --git a/src/com/amazon/ion/apps/EncodeApp.java b/src/com/amazon/ion/apps/EncodeApp.java new file mode 100644 index 0000000000..40045b6d43 --- /dev/null +++ b/src/com/amazon/ion/apps/EncodeApp.java @@ -0,0 +1,175 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.apps; + +import com.amazon.ion.IonBinaryWriter; +import com.amazon.ion.IonException; +import com.amazon.ion.IonReader; +import com.amazon.ion.SymbolTable; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; + + +public class EncodeApp + extends BaseApp +{ + private SymbolTable[] myImports; + private File myOutputDir; + private String myOutputFile; + + + //========================================================================= + // Static methods + + public static void main(String[] args) + { + EncodeApp app = new EncodeApp(); + app.doMain(args); + } + + + //========================================================================= + // Construction and Configuration + + public EncodeApp() + { + } + + + //========================================================================= + + + /** + * + * @param args + * @return the next index to process + */ + @Override + protected int processOptions(String[] args) + { + ArrayList imports = new ArrayList(); + + int i; + for (i = 0; i < args.length; i++) + { + String arg = args[i]; + if ("--catalog".equals(arg)) + { + String symtabPath = args[++i]; + loadCatalog(symtabPath); + } + else if ("--import".equals(arg)) + { + // We'll use the latest version available. + String name = args[++i]; + SymbolTable symtab = getLatestSharedSymtab(name); + imports.add(symtab); + } + else if ("--output-dir".equals(arg)) + { + String path = args[++i]; + myOutputDir = new File(path); + if (! myOutputDir.isDirectory() || ! myOutputDir.canWrite()) + { + throw new RuntimeException("Not a writeable directory: " + + path); + } + } + else if ("--output".equals(arg)) + { + String path = args[++i]; + myOutputFile = path; + myOutputDir = new File(path).getParentFile(); + if (! myOutputDir.isDirectory() || ! myOutputDir.canWrite()) + { + throw new RuntimeException("Not a writeable directory: " + + path); + } + } + else + { + // this arg is not an option, we're done here + break; + } + } + + myImports = imports.toArray(new SymbolTable[0]); + + return i; + } + + + @Override + protected void process(File inputFile, IonReader reader) + throws IOException, IonException + { + IonBinaryWriter writer = mySystem.newBinaryWriter(myImports); + + writer.writeValues(reader); + + byte[] binaryBytes = writer.getBytes(); + + if (myOutputDir != null) + { + String fileName = inputFile.getName(); + File outputFile = new File(myOutputDir, fileName); + FileOutputStream out = new FileOutputStream(outputFile); + try + { + out.write(binaryBytes); + } + finally + { + out.close(); + } + } + else + { + System.out.write(binaryBytes); + } + } + + @Override + protected void process(IonReader reader) + throws IOException, IonException + { + IonBinaryWriter writer = mySystem.newBinaryWriter(myImports); + + writer.writeValues(reader); + + byte[] binaryBytes = writer.getBytes(); + + if (myOutputDir != null) + { + File outputFile = new File(myOutputFile); + FileOutputStream out = new FileOutputStream(outputFile); + try + { + out.write(binaryBytes); + } + finally + { + out.close(); + } + } + else + { + System.out.write(binaryBytes); + } + } +} diff --git a/src/software/amazon/ion/apps/PrintApp.java b/src/com/amazon/ion/apps/PrintApp.java similarity index 87% rename from src/software/amazon/ion/apps/PrintApp.java rename to src/com/amazon/ion/apps/PrintApp.java index 260f29a855..ae9980e1a9 100644 --- a/src/software/amazon/ion/apps/PrintApp.java +++ b/src/com/amazon/ion/apps/PrintApp.java @@ -1,27 +1,28 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.apps; +package com.amazon.ion.apps; +import com.amazon.ion.IonException; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonWriter; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; -import software.amazon.ion.IonException; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonWriter; /** * TODO pretty-print on/off and configure diff --git a/src/software/amazon/ion/apps/SymtabApp.java b/src/com/amazon/ion/apps/SymtabApp.java similarity index 86% rename from src/software/amazon/ion/apps/SymtabApp.java rename to src/com/amazon/ion/apps/SymtabApp.java index 1d57441790..63216a53b7 100644 --- a/src/software/amazon/ion/apps/SymtabApp.java +++ b/src/com/amazon/ion/apps/SymtabApp.java @@ -1,29 +1,31 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.apps; +package com.amazon.ion.apps; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonException; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; import java.util.Iterator; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonException; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonType; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; + public class SymtabApp extends BaseApp @@ -187,9 +189,13 @@ public void processFiles(String[] filePaths) protected void process(IonReader reader) throws IonException { - IonType type; - while ((type = reader.next()) != null) + while (reader.hasNext()) { + IonType type = reader.next(); + +// System.err.println("Next: " + type); +// System.err.println("isInStruct=" + reader.isInStruct()); + String fieldName = reader.getFieldName(); intern(fieldName); @@ -206,6 +212,7 @@ protected void process(IonReader reader) case SEXP: case STRUCT: { +// System.err.println("stepping in"); reader.stepIn(); break; } @@ -216,8 +223,9 @@ protected void process(IonReader reader) } } - while (reader.next() != null && reader.getDepth() > 0) + while (! reader.hasNext() && reader.getDepth() > 0) { +// System.err.println("stepping out"); reader.stepOut(); } } diff --git a/src/software/amazon/ion/facet/Faceted.java b/src/com/amazon/ion/facet/Faceted.java similarity index 81% rename from src/software/amazon/ion/facet/Faceted.java rename to src/com/amazon/ion/facet/Faceted.java index 2e3d172548..087d821909 100644 --- a/src/software/amazon/ion/facet/Faceted.java +++ b/src/com/amazon/ion/facet/Faceted.java @@ -1,18 +1,19 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.facet; +package com.amazon.ion.facet; /** * Provides access to optional extension interfaces of a subject instance. diff --git a/src/software/amazon/ion/facet/Facets.java b/src/com/amazon/ion/facet/Facets.java similarity index 90% rename from src/software/amazon/ion/facet/Facets.java rename to src/com/amazon/ion/facet/Facets.java index bfec7aabbc..ba5036dc70 100644 --- a/src/software/amazon/ion/facet/Facets.java +++ b/src/com/amazon/ion/facet/Facets.java @@ -1,18 +1,19 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.facet; +package com.amazon.ion.facet; /** * Utility methods for working with facets. diff --git a/src/software/amazon/ion/facet/UnsupportedFacetException.java b/src/com/amazon/ion/facet/UnsupportedFacetException.java similarity index 76% rename from src/software/amazon/ion/facet/UnsupportedFacetException.java rename to src/com/amazon/ion/facet/UnsupportedFacetException.java index 052d07adca..f8f807d618 100644 --- a/src/software/amazon/ion/facet/UnsupportedFacetException.java +++ b/src/com/amazon/ion/facet/UnsupportedFacetException.java @@ -1,18 +1,19 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.facet; +package com.amazon.ion.facet; /** diff --git a/src/software/amazon/ion/facet/package-info.java b/src/com/amazon/ion/facet/package-info.java similarity index 83% rename from src/software/amazon/ion/facet/package-info.java rename to src/com/amazon/ion/facet/package-info.java index c7cf02c639..1f05ee0161 100644 --- a/src/software/amazon/ion/facet/package-info.java +++ b/src/com/amazon/ion/facet/package-info.java @@ -1,15 +1,16 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ /** @@ -36,7 +37,7 @@ * to support different sets of facets, based on its particular state. *

    * The central focus of the pattern is the - * {@link software.amazon.ion.facet.Faceted#asFacet(Class)} method. + * {@link com.amazon.ion.facet.Faceted#asFacet(Class)} method. * Subjects that wish to support facets implement it to allow users to request * the desired facet. * This method returns null when the subject doesn't support the facet. @@ -56,10 +57,10 @@ * especially when the subject is a decorator, adaptor, or similar wrapper * around the actual provider of the facet. *

    - * Given a concrete {@link software.amazon.ion.facet.Faceted} class, it may be that + * Given a concrete {@link com.amazon.ion.facet.Faceted} class, it may be that * some instances support a particular facet while others do not, depending on * the state of the subject or the way it was constructed. In such cases - * {@link software.amazon.ion.facet.Faceted#asFacet asFacet} should choose whether + * {@link com.amazon.ion.facet.Faceted#asFacet asFacet} should choose whether * to return the facet based on the subject's state. * Such classes should not extend the facet interface (directly or * indirectly), since that allows clients to bypass {@code asfacet} and @@ -73,4 +74,4 @@ * Erich Gamma. It was primarily inspired by ISO C++ {@code locale} facets. * */ -package software.amazon.ion.facet; +package com.amazon.ion.facet; diff --git a/src/software/amazon/ion/impl/AppendableFastAppendable.java b/src/com/amazon/ion/impl/AppendableFastAppendable.java similarity index 77% rename from src/software/amazon/ion/impl/AppendableFastAppendable.java rename to src/com/amazon/ion/impl/AppendableFastAppendable.java index ac06cbb4dd..a7f2c1e8fc 100644 --- a/src/software/amazon/ion/impl/AppendableFastAppendable.java +++ b/src/com/amazon/ion/impl/AppendableFastAppendable.java @@ -1,29 +1,30 @@ /* - * Copyright 2013-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; +import com.amazon.ion.util._Private_FastAppendable; import java.io.Closeable; import java.io.Flushable; import java.io.IOException; -import software.amazon.ion.util.PrivateFastAppendable; /** - * Adapts an {@link Appendable} to implement {@link PrivateFastAppendable}. + * Adapts an {@link Appendable} to implement {@link _Private_FastAppendable}. */ final class AppendableFastAppendable - implements PrivateFastAppendable, Closeable, Flushable + implements _Private_FastAppendable, Closeable, Flushable { private final Appendable _out; diff --git a/src/software/amazon/ion/impl/Base64Encoder.java b/src/com/amazon/ion/impl/Base64Encoder.java similarity index 97% rename from src/software/amazon/ion/impl/Base64Encoder.java rename to src/com/amazon/ion/impl/Base64Encoder.java index 0e4e9160b1..ac01dcb575 100644 --- a/src/software/amazon/ion/impl/Base64Encoder.java +++ b/src/com/amazon/ion/impl/Base64Encoder.java @@ -1,33 +1,32 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; -import java.io.IOException; -import java.io.InputStream; -import java.io.Reader; -import software.amazon.ion.IonException; -import software.amazon.ion.util.IonTextUtils; - -/** +/* * This is a class that supports encoding and decoding binary * data in base 64 encodings. + * *

    * The default encoding is the URL Safe encoding variant specified * in http://tools.ietf.org/html/rfc4648 + *

    + * *

    * It's character translation table is: + * *

      *    Table 2: The "URL and Filename safe" Base 64 Alphabet
      *
    @@ -52,7 +51,16 @@
      *       15 P            32 g            49 x
      *       16 Q            33 h            50 y            (pad) =
      * 
    + *

    */ + +import com.amazon.ion.IonException; +import com.amazon.ion.util.IonTextUtils; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; + + final class Base64Encoder { static class EL { // EncoderLetter diff --git a/src/com/amazon/ion/impl/BlockedBuffer.java b/src/com/amazon/ion/impl/BlockedBuffer.java new file mode 100644 index 0000000000..06bd00f4c3 --- /dev/null +++ b/src/com/amazon/ion/impl/BlockedBuffer.java @@ -0,0 +1,1778 @@ + +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; +import com.amazon.ion.IonException; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.SortedSet; +import java.util.TreeSet; +/** + * This implements a blocked byte buffer and both an input and output stream + * that operates over it. It is designed to be able to be randomly accessed. + * The output steam supports both inserting data (with "stretching") in the + * middle of the stream and over-write. The output steam also supports remove + * which shrinks the overall data buffer. The underlying buffer is backed by + * one or more byte arrays to minimize data movement. + *

    + * It is also meant to be reused, so that it does not have to pressure the + * GC, if that is desirable. + */ +final class BlockedBuffer +{ + /////////////////////////////////////////////////////////////////////////////// + // + // updatable, insertable, and possibly fragmented byte buffer + // + // these manage the set of memory (byte) buffers + ArrayList _blocks; + int _next_block_position; // next position in _blocks for active block, may be less than _blocks.size() + int _lastCapacity; // used to allocate new blocks + int _buf_limit; // high water mark of _position + int _version; + int _mutation_version; + Object _mutator; +// BUGBUG - this is just a test, it shouldn't be in checked in code +static final boolean test_with_no_version_checking = false; + void start_mutate(Object caller, int version) { + if (test_with_no_version_checking) return; + if (_mutation_version != 0 || _mutator != null) + throw new BlockedBufferException("lock conflict"); + if (version != _version) + throw new BlockedBufferException("version conflict on update"); + _mutator = caller; + _mutation_version = version; + } + int end_mutate(Object caller) { + if (test_with_no_version_checking) return _version; + if (_version != _mutation_version) + throw new BlockedBufferException("version mismatch failure"); + if (caller != _mutator) + throw new BlockedBufferException("caller mismatch failure"); + _version = _mutation_version + 1; + _mutation_version = 0; + _mutator = null; + return _version; + } + boolean mutation_in_progress(Object caller, int version) { + if (test_with_no_version_checking) return false; + if (_mutation_version != version) + throw new BlockedBufferException("unexpected update lock conflict"); + if (caller != _mutator) + throw new BlockedBufferException("caller mismatch failure"); + return true; + } + int getVersion() { + return _version; + } + static boolean debugValidation = false; + static int _defaultBlockSizeMin; + static int _defaultBlockSizeUpperLimit; + static { + resetParameters(); + } + public static void resetParameters() { + debugValidation = false; + _defaultBlockSizeMin = 4096 * 8; + _defaultBlockSizeUpperLimit = 4096 * 8; + } + public int _blockSizeMin = _defaultBlockSizeMin; + public int _blockSizeUpperLimit = _defaultBlockSizeUpperLimit; + static void setBlockSizeParameters(int min, int max, + boolean intenseValidation) { + debugValidation = intenseValidation; + setBlockSizeParameters(min, max); + } + public static void setBlockSizeParameters(int min, int max) { + if (min < 0 || max < min) { + throw new IllegalArgumentException(); + } + _defaultBlockSizeMin = min; + _defaultBlockSizeUpperLimit = max; + return; + } + /////////////////////////////////////////////////////////////////////////// + /** + * Creates a new buffer without preallocating any space. + */ + public BlockedBuffer() { + start_mutate(this, 0); + init(0, null); + end_mutate(this); + } + /** + * Creates a new buffer, preallocating some initial capacity. + * + * @param initialSize the number of bytes to allocate. + */ + public BlockedBuffer(int initialSize) { + start_mutate(this, 0); + init(initialSize, null); + end_mutate(this); + } + /** + * Creates a new buffer, assuming ownership of given data. + * This method assumes ownership of the data array + * and will modify it at will. + * + * @param data the initial data to be buffered. + * + * @throws NullPointerException if buffer is null. + */ + public BlockedBuffer(byte[] data) { + start_mutate(this, 0); + init(0, new bbBlock(data)); + _buf_limit = data.length; + end_mutate(this); + } + /** + * Creates a new buffer containing all data remaining on an + * {@link InputStream}. The stream is closed before returning. + * + * @param data must not be null. + * + * @throws IOException + */ + public BlockedBuffer(InputStream data) + throws IOException + { + IonBinary.Writer writer = new IonBinary.Writer(this); + try { + writer.write(data); + } + finally { + data.close(); + } + } + /** + * creates a logical copy of the buffer. This does not preserve + * the position state and is equivalent to constructing a new + * buffer from the old by getting the bytes from the original + * and writing them to a new buffer. + */ + @Override + public BlockedBuffer clone() + { + BlockedBuffer clone = new BlockedBuffer(this._buf_limit); + int end = this._buf_limit; + bbBlock dst_block = clone._blocks.get(0); + int dst_offset = 0; + int dst_limit = dst_block.blockCapacity(); + for (int ii=0; ii dst_limit - dst_offset) { + to_copy = dst_limit - dst_offset; + } + System.arraycopy(src_block._buffer, 0, dst_block._buffer, dst_offset, to_copy); + dst_offset += to_copy; + // the cloned BlockedBuffer should be able to hold all the data + // in it's single block + assert dst_offset <= dst_limit; + // see if we're done (and break out in that case) + if (src_end >= end) break; + } + dst_block._limit = dst_offset; + clone._buf_limit = dst_offset; + return clone; + } + /** + * Initializes the various members such as the block arraylist + * the initial block and the various values like the block size upper limit. + * @param initialSize or 0 + * @param initialBlock or null + * @return bbBlock the initial current block + */ + private bbBlock init(int initialSize, bbBlock initialBlock) + { + this._lastCapacity = BlockedBuffer._defaultBlockSizeMin; + this._blockSizeUpperLimit = BlockedBuffer._defaultBlockSizeUpperLimit; + while (this._lastCapacity < initialSize && + this._lastCapacity < this._blockSizeUpperLimit) + { + this.nextBlockSize(this, 0); + } + int count = initialSize / this._lastCapacity; + if (initialBlock != null) count = 1; + this._blocks = new ArrayList(count); + if (initialBlock == null) { + initialBlock = new bbBlock(this.nextBlockSize(this, 0)); + } + this._blocks.add(initialBlock); + this._next_block_position = 1; + // create any preallocated blocks (following _next_block_position) + bbBlock b; + for (int need = initialSize - initialBlock.blockCapacity() + ; need > 0 + ; need -= b.blockCapacity() + ) { + b = new bbBlock(this.nextBlockSize(this, 0)); + b._idx = -1; + this._blocks.add(b); + } + return initialBlock; + } + /** + * Gets the number of bytes of content in this buffer. + * This isn't the same as its capacity. + */ + public final int size() { + return _buf_limit; + } + /** + * empties the entire contents of the buffer + */ + private void clear(Object caller, int version) { + assert mutation_in_progress(caller, version); + _buf_limit = 0; + for (int ii=0; ii<_blocks.size(); ii++) { + _blocks.get(ii).clearBlock(); + // _blocks.get(ii)._idx = -1; this is done in clearBlock() + } + bbBlock first = _blocks.get(0); + first._idx = 0; // cas: 26 dec 2008 + first._offset = 0; + first._limit = 0; + _next_block_position = 1; + return; + } + /** + * treat the limit as the end of file + */ + bbBlock truncate(Object caller, int version, int pos) { + assert mutation_in_progress(caller, version); + if (0 > pos || pos > this._buf_limit ) + throw new IllegalArgumentException(); + // clear out all the blocks in use from the last in use + // to the block where the eof will be located + bbBlock b = null; + for (int idx = this._next_block_position - 1; idx >= 0; idx--) { + b = this._blocks.get(idx); + if (b._offset <= pos) break; + b.clearBlock(); + } + if (b == null) { + throw new IllegalStateException("block missing at position "+pos); + } + // reset the next block position to account for this. + this._next_block_position = b._idx + 1; + // on the block where eof is, set it's limit appropriately + b._limit = pos - b._offset; + // set the overall buffer limits + this._buf_limit = pos; + b = this.findBlockForRead(pos, version, b, pos); + return b; + } + private bbBlock addBlock(Object caller, int version, int idx, int offset, + int needed) + { + assert mutation_in_progress(caller, version); + bbBlock newblock = null; + for (int ii=this._next_block_position; ii < this._blocks.size(); ii++) + { + bbBlock tmpblock = this._blocks.get(this._next_block_position); + if (tmpblock._buffer.length >= needed) { + this._blocks.remove(this._next_block_position); + newblock = tmpblock; + break; + } + } + if (newblock == null) { + // if there's nothing big enough to recycle + // so we have to really make more space + int bufcapacity = 0; + if (needed > _blockSizeUpperLimit) { + bufcapacity = needed; + } + else { + while (bufcapacity < needed) { + bufcapacity = this.nextBlockSize(caller, version); + } + } + newblock = new bbBlock(bufcapacity); + } + // if the caller didn't specify an index + // we'll have to find out where this goes + if (idx == -1) { + for (idx = 0; idx < this._next_block_position; idx++) { + if (this._blocks.get(idx)._offset < 0) { + break; + } + if (offset >= this._blocks.get(idx)._offset) { + break; + } + } + } + // initialize the buffer and add it to the list in the right spot + newblock._idx = idx; + newblock._offset = offset; + _blocks.add(idx, newblock); + _next_block_position++; + // if this isn't the last buffer, bump the idx of the trailing buffers + for (int ii = idx + 1; ii < _next_block_position; ii++) { + this._blocks.get(ii)._idx = ii; + } + return newblock; + } + private int nextBlockSize(Object caller, int version) + { + assert mutation_in_progress(caller, version); + if (_lastCapacity == 0) { + _lastCapacity = _blockSizeMin; + } + else if (_lastCapacity < _blockSizeUpperLimit) { + _lastCapacity *= 2; + } + return _lastCapacity; + } + // starts with (pos, 0, _next_block_position) so we're really + // looking in blocks with indices from lo to (hi-1) inclusive + final bbBlock findBlockHelper(int pos, int lo, int hi) + { + bbBlock block; + int ii; + if ((hi - lo) <= 3) { + for (ii=lo; ii block._offset + block._limit) continue; + if (block.containsForRead(pos)) { + return block; + } + if (block._offset >= pos) break; + } + return this._blocks.get(ii - 1); // this will always be > 0 + } + int mid = (hi + lo) / 2; + block = this._blocks.get(mid); + assert block != null; + if (block._offset > pos) { + return findBlockHelper(pos, lo, mid); + } + return findBlockHelper(pos, mid, hi); + } + /** + * find the block where this offset (newPosition) has already + * been written. Typically the caller will set _curr to be the + * returned block. + * @param pos global position to be read from + * @return curr block ready to be read from + */ + bbBlock findBlockForRead(Object caller, int version, bbBlock curr, int pos) + { + assert pos >= 0 && "buffer positions are never negative".length() > 0; + if (pos > this._buf_limit) { + throw new BlockedBufferException("invalid position"); + } + assert _validate(); + if (curr != null) { + if (curr.containsForRead(pos)) { + return curr; + } + if (pos == this._buf_limit && (pos - curr._offset) == curr._limit) { + return curr; + } + } + boolean at_eof = (pos == this._buf_limit); + if (at_eof) { + // if this is the last block actually in use + // and we're looking for the eof position then + // we can check for the "last byte not quite + // written yet" case, which is fine + bbBlock block = this._blocks.get(this._next_block_position - 1); + if (block.containsForWrite(pos)) return block; + } + else { + bbBlock block = this.findBlockHelper(pos, 0, this._next_block_position); + return block; + } + throw new BlockedBufferException("valid position can't be found!"); + } + /** + * find the block where this offset (newPosition) should be written. + * typicall the caller will set _curr to be the returned block. + * @param pos global position to be written to + * @return curr block ready to be written to + */ + bbBlock findBlockForWrite(Object caller, int version, bbBlock curr, int pos) + { + assert mutation_in_progress(caller, version); + assert (pos >= 0 && "invalid position, positions must be >= 0".length() > 0); + if (pos > this._buf_limit + 1) { + throw new BlockedBufferException("writes must be contiguous"); + } + assert _validate(); + if (curr != null && curr.hasRoomToWrite(pos, 1) == true) { + if (curr._offset + curr._limit == pos && curr._idx < this._next_block_position) { + bbBlock b = this._blocks.get(curr._idx + 1); + if (b.containsForWrite(pos)) { + curr = b; + } + } + return curr; + } + // we're not going to write into curr, so find out the right block + bbBlock block; + if (pos == this._buf_limit) { + // if we're at the limit the only possible (existing) block + // will be the very last block - shortcut to optimize append + assert this._next_block_position > 0; + block = this._blocks.get(this._next_block_position - 1); + } + else if (curr != null && pos == curr._offset + curr._limit) { + // if our current position is exactly at the end (and we already know + // we can't write into this block if we can write at all we'll have + // to write into the next block (inner blocks can't be 0 bytes long) + block = this._blocks.get(curr._idx + 1); + } + else { + // since we're not at the limit and we don't have a current block + // we'll go find the block in the list (this is an abnormal case) + // since append if usual for writing + block = findBlockHelper(pos, 0, this._next_block_position); + } + assert block != null; + assert block.containsForWrite(pos); + // chech our candidate block to see if it's the one we'd write into + if (block.hasRoomToWrite(pos, 1)) { + return block; + } + // at this point, we can't use _curr in any event so we can just + // move on to the next block since findHelper will have returned + // either the right block (which it didn't) or the one just in + // front of the right block - so let's see if there is an allocated + // block just following this + if (block._idx < this._next_block_position - 1) { + block = this._blocks.get(block._idx + 1); + return block; + } + // there wasn't a following block (actually a common case when + // you're appending) so we have to go ahead an actually add a new block + int newIdx = block._idx + 1; + assert newIdx == this._next_block_position; + bbBlock ret = this.addBlock(caller + ,version + ,newIdx + ,pos + ,this.nextBlockSize(caller, version) + ); + return ret; + } + /** + * dispatcher for the various forms of insert we encounter + * calls one of the four helpers depending on the case + * that is needed to inser here + * @param len number of bytes to make space for + * @return int number of bytes inserted + */ + int insert(Object caller, int version, bbBlock curr, int pos, int len) + { + assert mutation_in_progress(caller, version); + // DEBUG: int amountMoved = 0; + // DEBUG: int before = this._buf_limit; + // DEBUG: assert _validate(); + // if there's room in the current block - just + // move the "trailing" bytes down and we're done + int neededSpace = len - curr.unusedBlockCapacity(); + if (neededSpace <= 0) { + // we have all the space we need in the current block + // DEBUG: amountMoved = + insertInCurrOnly(caller, version, curr, pos, len); + } + else { + // we'll need at least some additional space beyond the curr + // block, see if there's room in the + // next one, otherwise we'll make more (blocks) + bbBlock next = null; + if (curr._idx < this._next_block_position - 1) { + // if there is another block + next = this._blocks.get(curr._idx + 1); + } + if (next != null && + (neededSpace <= next.unusedBlockCapacity()) + ) { + // with the addition of the free space in the following block we have enough + // DEBUG: amountMoved = + insertInCurrAndNext(caller, version, curr, pos, len, next); + } + else { + // we'll have to make one or more new blocks + // first figure out much will be in the first + // and last blocks (i.e. ignoring the whole + // blocks + int lenNeededInLastAddedBlock = neededSpace % _blockSizeUpperLimit; + int tailLen = curr.bytesAvailableToRead(pos); + if (lenNeededInLastAddedBlock < tailLen) lenNeededInLastAddedBlock = tailLen; + if (lenNeededInLastAddedBlock < neededSpace + && neededSpace < this._blockSizeUpperLimit) { + // if we need less than the largest block then we should + // make *one* block with all of the needed space + lenNeededInLastAddedBlock = neededSpace; + } + bbBlock newblock = insertMakeNewTailBlock(caller, version, curr, lenNeededInLastAddedBlock); + // now see if the curr block and this newblock have enough + // available space to do the job, and if there's some trailing + // data from curr that will end up staying in curr + if (len <= (curr.unusedBlockCapacity() + + newblock.unusedBlockCapacity()) + ) { + // insert this as a zero length block immediately after _curr + // insertBlock also adjusts the trailing blocks idx values + insertBlock(newblock); + // now pretend we just have the "push into the next block" case + // DEBUG: amountMoved = + insertInCurrAndNext(caller, version, curr, pos, len, newblock); + } + else { + // and last we have the case of having to insert more than 1 block + // which means all of the trailing bytes in _curr move into the last + // block + // DEBUG: amountMoved = + insertAsManyBlocksAsNeeded(caller, version, curr, pos, len, newblock); + } + } + } + // DEBUG: if (this._buf_limit - before != len + // DEBUG: || amountMoved != len) { + // DEBUG: throw new BlockedBufferException("insert went wrong #1 !!!"); + // DEBUG: } + assert _validate(); + return len; + } + /** + * this handles insert when there's enough room in the + * current block + */ + private int insertInCurrOnly(Object caller, int version, bbBlock curr, int pos, int len) + { + assert mutation_in_progress(caller, version); + // the space we need is available right in the block + assert curr.unusedBlockCapacity() >= len; + System.arraycopy(curr._buffer, curr.blockOffsetFromAbsolute(pos) + ,curr._buffer, curr.blockOffsetFromAbsolute(pos) + len, curr.bytesAvailableToRead(pos)); + curr._limit += len; + this.adjustOffsets(curr._idx, len, 0); + notifyInsert(pos, len); + return len; + } + private int insertInCurrAndNext(Object caller, int version, bbBlock curr, int pos, int len, bbBlock next) + { + assert mutation_in_progress(caller, version); + // DEBUG: int amountMoved = 0; + // all the space we need (len) fits in these two blocks + assert curr.unusedBlockCapacity() + next.unusedBlockCapacity() >= len; + // and we need to use space in both of these blocks + assert curr.unusedBlockCapacity() < len; + int availableToRead = curr.bytesAvailableToRead(pos); + int tailInCurr = availableToRead; + int deltaOfNextData = len - curr.unusedBlockCapacity(); + int tailCopiedToNext = deltaOfNextData; + if (tailCopiedToNext > availableToRead) { + tailCopiedToNext = availableToRead; + } + // first we copy the data in the next block down to make room + // for data we're pushing off the end of the _curr block + // if we need to, there may not be any data in the next block + if (next._limit > 0) { + System.arraycopy(next._buffer, 0, next._buffer, deltaOfNextData, next._limit); + } + next._limit += deltaOfNextData; + // DEBUG: amountMoved += deltaOfNextData; + // next we copy the data from the tail of _curr into the front of next + // since we don't have room for it any longer in the _curr block + // but it is possible that there is not tail at all (pos == limit) + if (tailCopiedToNext > 0) { + System.arraycopy(curr._buffer, curr._limit - tailCopiedToNext + , next._buffer, deltaOfNextData - tailCopiedToNext, tailCopiedToNext); + } + // finally if there's any tail left in the _curr block we copy that + // down to the end of the _curr block (if all of the tail moved into + // the next block nothing happens here + int leftInCurr = tailInCurr - tailCopiedToNext; + if (leftInCurr > 0) { + int blockPosition = curr.blockOffsetFromAbsolute(pos); + System.arraycopy(curr._buffer, blockPosition + ,curr._buffer, blockPosition + len, leftInCurr); + } + // finally if we reused from space in _curr (between _limit and the unreserved capacity) + // we adjust for that as well as the space adjusted in the newblock + int addedInCurr = curr.unusedBlockCapacity(); + if (addedInCurr > 0) { + curr._limit += addedInCurr; + // DEBUG: amountMoved += addedInCurr; + next._offset += addedInCurr; + } + assert (curr.blockOffsetFromAbsolute(pos) + tailCopiedToNext + addedInCurr + leftInCurr) == curr._limit; + this.adjustOffsets(next._idx, len, 0); + notifyInsert(pos, len); + // DEBUG: if (amountMoved != len) { + // DEBUG: throw new BlockedBufferException("insert went wrong #4 !!!"); + // DEBUG: } + return len; + } + private bbBlock insertMakeNewTailBlock(Object caller, int version, bbBlock curr, int minimumBlockSize) + { + assert mutation_in_progress(caller, version); + // needed is the amount of data we'll put into the + // final added block (which is actually added first) + int newblocksize = minimumBlockSize; + if (newblocksize < _blockSizeUpperLimit) { + // if we don't need an oversize block then find a block + // size that will be big enough + while ((newblocksize = this.nextBlockSize(caller, version)) < minimumBlockSize) { + // bump up requested block capacity until we get + // at least enough to hold the request, or we + // hit the max blocksize whichever comes first + } + } + // allocate and initialize a new block that will be the + // tail of our interesting blocks + bbBlock newblock = new bbBlock(newblocksize); + newblock._idx = curr._idx + 1; + newblock._offset = curr._offset + curr._limit; // we'll adjust this later like any existing block + return newblock; + } + private int insertAsManyBlocksAsNeeded(Object caller, int version, bbBlock curr, int pos, int len, bbBlock newLastBlock) + { + assert mutation_in_progress(caller, version); + // DEBUG: int amountAllocated = 0; + // DEBUG: int origPos = this._buf_position; + // this is the case where the old tail is pushed entirely out of the + // old block into a new trailing block and then as many whole new + // blocks as needed (which maybe none) are inserted between these two + bbBlock oldCurr = curr; + int oldPosition = curr.blockOffsetFromAbsolute(pos); + int oldBlockTail = curr._limit - oldPosition; + int newSpaceInCurr = curr.unusedBlockCapacity(); + // adjust the curr blocks limit + curr._limit += newSpaceInCurr; + // DEBUG: amountAllocated += newSpaceInCurr; + int newoffset = curr._offset + curr._limit; + int spaceNeededInMiddle = len - newSpaceInCurr - newLastBlock._buffer.length; + int addedblocks = 0; + bbBlock newblock = null; + assert (spaceNeededInMiddle > 0); // this is the "as many as needed" case not "this and next" + // add blocks until we're ready for the last block + while (spaceNeededInMiddle > 0) { + addedblocks++; + newblock = new bbBlock(this.nextBlockSize(caller, version)); + newblock._limit = newblock._buffer.length; + if (newblock._limit > spaceNeededInMiddle) newblock._limit = spaceNeededInMiddle; + // DEBUG: amountAllocated += newblock._limit; + newblock._idx = curr._idx + addedblocks; + newblock._offset = newoffset; + this._blocks.add(newblock._idx, newblock); + spaceNeededInMiddle -= newblock._limit; + newoffset += newblock._limit; + } + // add the last block + addedblocks++; + newblock = newLastBlock; + newblock._limit = newblock._buffer.length; + // DEBUG: amountAllocated += newblock._limit; + newblock._idx = curr._idx + addedblocks; + newblock._offset = newoffset; + this._blocks.add(newblock._idx, newblock); + // DEBUG: assert (amountAllocated == len); + // now adjust the trailing blocks + adjustOffsets(newblock._idx, len, addedblocks); + notifyInsert(pos, len); + // now we copy the tail of the _curr block to the end of the space + // note that this only works because the tail is being copied to + // an altogether different block in the buffer, so it can't overlap + if (oldBlockTail > 0) { + System.arraycopy(oldCurr._buffer, oldPosition, newLastBlock._buffer, newLastBlock._limit - oldBlockTail, oldBlockTail); + } + // DEBUG: assert this.position() == origPos; + // DEBUG: assert (amountAllocated == len); + return len; + } + private void insertBlock(bbBlock newblock) { + // in both cases we need to insert the new block after _curr + // and adjust the idx values to go with that + this._blocks.add(newblock._idx, newblock); + _next_block_position++; + for (int ii=newblock._idx + 1; ii < this._next_block_position; ii++) { + this._blocks.get(ii)._idx++; + } + } + private void adjustOffsets(int lastidx, int addedBytes, int addedBlocks) { + bbBlock b; + // now we adjust the trailing offsets + if (addedBytes != 0 || addedBlocks != 0) { + this._next_block_position += addedBlocks; + for (int ii=lastidx + 1; ii < this._next_block_position; ii++) { + b = this._blocks.get(ii); + b._offset += addedBytes; + b._idx += addedBlocks; + } + this._buf_limit += addedBytes; + } + } + bbBlock remove(Object caller, int version, bbBlock curr, int pos, int len) + { + assert mutation_in_progress(caller, version); + if (len == 0) return curr; + if (len < 0 || (pos + len) > this._buf_limit) { + throw new IllegalArgumentException(); + } + int amountToRemove = len; + int removedBlocks = 0; + int startingIdx = curr._idx; + int currIdx = curr._idx; + bbBlock currBlock = curr; + assert (curr._offset <= pos); + assert (pos - curr._offset <= curr._limit); + assert _validate(); + // this is to simply eliminate a big edge case + if (pos == 0 && len == this._buf_limit) { + this.clear(caller, version); + notifyRemove(0, len); + return null; + } + // remove from the initial block + int currBlockPosition = currBlock.blockOffsetFromAbsolute(pos); + int removedFromThisBlock = currBlock._limit - currBlockPosition; + if (removedFromThisBlock > amountToRemove) removedFromThisBlock = amountToRemove; + if (removedFromThisBlock == currBlock._limit) { + // we'll be removing the whole block in the whole block loop below + startingIdx--; // so we have to back up on to fix the next block that will + // "fall" down into the soon to be emptied slot here + } + else { + // we always copy into position, and we copy whatever is still + // left in the end of the block + int moveAmount = currBlock._limit - currBlockPosition - removedFromThisBlock; + if (moveAmount > 0) { + System.arraycopy(currBlock._buffer, currBlock._limit - moveAmount + ,currBlock._buffer, currBlockPosition, moveAmount); + } + amountToRemove -= removedFromThisBlock; + currBlock._limit -= removedFromThisBlock; + if (amountToRemove > 0) { + // when we're on the last block, there'll be nothing to remove, + // and no block to get either + currIdx = currBlock._idx + 1; + currBlock = this._blocks.get(currIdx); + } + } + while (amountToRemove > 0 && amountToRemove >= currBlock._limit) { + amountToRemove -= currBlock._limit; + // remove the whole block - so first hang onto a reference + bbBlock temp = currBlock; + this._blocks.remove(currIdx); + removedBlocks++; + temp.clearBlock(); + this._blocks.add(temp); // dump it at the end (marked as not in use) + // and we don't move currIdx because we bumped it out of the whole array + if (currIdx < this._next_block_position - removedBlocks) { + currBlock = this._blocks.get(currIdx); + } + else if (currIdx > 0) { + currIdx--; + currBlock = this._blocks.get(currIdx); + } + else { + throw new BlockedBufferException("fatal - no current block!"); + } + } + if (amountToRemove > 0) { + assert amountToRemove < currBlock._limit; + System.arraycopy(currBlock._buffer, amountToRemove + ,currBlock._buffer, 0, currBlock._limit - amountToRemove); + assert amountToRemove < currBlock._limit; + currBlock._limit -= amountToRemove; + currBlock._offset += amountToRemove; + } + // we'll even adjust the offset of the first block (if it's the last as well) + adjustOffsets(startingIdx, -len, -removedBlocks); + notifyRemove(pos, len); + // DEBUG: int shouldBe = 0; + // DEBUG: int is = currBlock._offset; + // DEBUG: if (currBlock._idx > 0) { + // DEBUG: shouldBe = this._blocks.get(currBlock._idx - 1)._offset + this._blocks.get(currBlock._idx - 1)._limit; + // DEBUG: if (currIdx != startingIdx) assert (shouldBe == this._buf_position); + // DEBUG: } + // DEBUG: int delta = shouldBe - is; + // DEBUG: assert(delta == 0); + assert _validate(); + return currBlock; + } + static int _validate_count; + public boolean _validate() { + int pos = 0; + int idx; + boolean err = false; + _validate_count++; + if ((_validate_count % 128) != 0) return true; + // you can change the 0 below (in from of the -2) to be the validation counter + // which reported the failure and the test will be true when _validate() is + // called on the last GOOD check. + if (_validate_count == 30 -2) { + // used to set breakpoints on particular calls for validation + err = (_validate_count < 0); + } + for (idx=0; idx b._buffer.length /* - b._reserved */ ) { + System.out.println("block "+idx+": "+ + "limit is out of range"+ + ", it is "+b._limit+ + " should be between 0 and "+ (b._buffer.length /* - b._reserved */)); + err = true; + } + else if (b._limit == 0) { + if ( ! (b._idx == (this._next_block_position - 1) + && b._offset == this._buf_limit) + ) { + System.out.println("block "+idx+": "+ + "has a ZERO limit"); + err = true; + } + } + pos += b._limit; + } + if (idx != this._next_block_position) { + System.out.println("next block position is wrong, is "+this._next_block_position+" should be "+idx); + err = true; + } + for (idx++; idx 0) { + bbBlock last = this._blocks.get(this._next_block_position - 1); + if (last._offset + last._limit != this._buf_limit){ + System.out.println("last block "+last._idx+" limit isn't "+ + "_buf_limit ("+this._buf_limit+"): "+ + " calc'd last block limit is " + + last._offset +" + "+ last._limit + +" = "+(last._offset + last._limit) + ); + err = true; + } + } + if (this._buf_limit < 0 || (this._buf_limit > 0 && this._next_block_position < 1)){ + System.out.println("this._buf_limit "+ this._buf_limit+ " is invalid"); + err = true; + } + if (err == true) { + System.out.println("failed with validation count = " + _validate_count); + } + return err == false; // validate is true if all is ok so that assert _validate(); works as expected + } + final static class bbBlock { + public int _idx; + public int _offset; + public int _limit; + public byte[] _buffer; + public bbBlock(int capacity) { + _buffer = new byte[capacity]; + } + /** + * Assumes ownership of an array to create a new block. The data + * within the buffer is maintained. + * + * @param buffer contains the data for the block. + * + * @throws NullPointerException if buffer is null. + */ + bbBlock(byte[] buffer) { + _buffer = buffer; + _limit = buffer.length; + } + public bbBlock clearBlock() { + _idx = -1; + _offset = -1; + _limit = 0; + return this; + } + /** + * maximimum number of bytes that can be held in this block. + */ + final int blockCapacity() { + assert this._offset >= 0; + return this._buffer.length ; + } + /** + * maximimum number of bytes that can be appended in this block currently. + */ + final int unusedBlockCapacity() { + assert this._offset >= 0; + return this._buffer.length - this._limit; + } + /** + * Gets the number of bytes between the current position and the + * writable capacity of this block. + * @param pos absolute position + */ + final int bytesAvailableToWrite(int pos) { + assert this._offset >= 0; + return this._buffer.length - (pos - _offset); + } + /** + * Gets the number of, as yet, unused bytes in this block. That's the number + * of bytes that can be inserted into this block without overflowing, or the + * number of bytes between the current position and the end of the written bytes + * in this block + * @param pos absolute position + */ + public final int bytesAvailableToRead(int pos) { + assert this._offset >= 0; + return this._limit - (pos - _offset); + } + /** + * is there space between position and capacity? + * @param pos absolute position + * @param needed + * @return boolean + */ + final boolean hasRoomToWrite(int pos, int needed) { + assert this._offset >= 0; + return (needed <= (this._buffer.length - (pos - _offset))); + } + final boolean containsForRead(int pos) { + assert this._offset >= 0; + return (pos >= _offset && pos < _offset + _limit); + } + final boolean containsForWrite(int pos) { + assert this._offset >= 0; + return (pos >= _offset && pos <= _offset + _limit); + } + final int blockOffsetFromAbsolute(int pos) { + assert this._offset >= 0; + return pos - _offset; + } + } + public interface Monitor + { + public boolean notifyInsert(int pos, int len); + public boolean notifyRemove(int pos, int len); + public int getMemberIdOffset(); + } + private final static class PositionMonitor implements Monitor + { + int _pos; + PositionMonitor(int pos) { _pos = pos; } + public int getMemberIdOffset() { return _pos; } + public boolean notifyInsert(int pos, int len) { return false; } + public boolean notifyRemove(int pos, int len) { return false; } + } + private final static class CompareMonitor implements Comparator { + static CompareMonitor instance = new CompareMonitor(); + private CompareMonitor() {} + static CompareMonitor getComparator() + { + return instance; + } + public int compare(Monitor arg0, Monitor arg1) + { + return arg0.getMemberIdOffset() - arg1.getMemberIdOffset(); + } + } + TreeSet _updatelist = new TreeSet(CompareMonitor.getComparator()); + public void notifyRegister(Monitor item) { + _updatelist.add(item); + } + public void notifyUnregister(Monitor item) { + _updatelist.remove(item); + } + public void notifyInsert(int pos, int len) { + if (len == 0) return; + PositionMonitor pm = new PositionMonitor(pos); + SortedSet follows = _updatelist.tailSet(pm); + for (Monitor m : follows) { + if (m.notifyInsert(pos, len)) { + follows.remove(m); + } + } + } + public void notifyRemove(int pos, int len) { + if (len == 0) return; + PositionMonitor pm = new PositionMonitor(pos); + SortedSet follows = _updatelist.tailSet(pm); + for (Monitor m : follows) { + if (m.notifyRemove(pos, len)) { + follows.remove(m); + } + } + } + /** + * Reads data from a byte buffer, keeps a local position and + * a current block. Snaps a buffer length on creation; + */ + public static class BlockedByteInputStream extends java.io.InputStream + { + BlockedBuffer _buf; + int _pos; + int _mark; + bbBlock _curr; + int _blockPosition; + int _version; + /** + * @param bb blocked buffer to read from + */ + public BlockedByteInputStream(BlockedBuffer bb) + { + this(0, bb); + } + /** + * @param bb blocked buffer to read from + * @param pos initial offset to read + */ + public BlockedByteInputStream(BlockedBuffer bb, int pos) + { + this(pos, bb); + } + /** + * @param pos initial offset to read + * @param end is the local limit, or -1 (_end_unspecified) + * @param bb blocked buffer to read from + */ + private BlockedByteInputStream(int pos, BlockedBuffer bb) + { + if (bb == null) throw new IllegalArgumentException(); + _version = bb.getVersion(); + _buf = bb; + _set_position(pos); + _mark = -1; + } + @Override + public final void mark(int readlimit) { + this._mark = this._pos; + } + @Override + public final void reset() throws IOException { + if (this._mark == -1) throw new IOException("mark not set"); + _set_position(this._mark); + } + /** + * the current offset in the buffer + */ + public final int position() { + return this._pos; + } + /** + * this forces a version sync with the underlying blocked buffer. + * The current position is lost during this call. + * + */ + public final void sync() throws IOException + { + if (_buf == null) throw new IOException("stream is closed"); + _version = _buf.getVersion(); + _curr = null; + _pos = 0; + } + /** + * debug api to force check for internal validity of the + * underlying buffer + */ + public final boolean _validate() { + return this._buf._validate(); + } + /** + * sets the position of the stream to be pos. The next operation + * (such as read) will return the byte at that offset. + * @param pos new offset to read from + * @return this stream + */ + public final BlockedByteInputStream setPosition(int pos) throws IOException + { + if (_buf == null) throw new IOException("stream is closed"); + fail_on_version_change(); + if (pos < 0 || pos > _buf.size()) { + throw new IllegalArgumentException(); + } + // call our unfailing private method to do the real work + _set_position(pos); + fail_on_version_change(); + return this; + } + private final void _set_position(int pos) + { + _pos = pos; + _curr = _buf.findBlockForRead(this, _version, _curr, pos); + _blockPosition = _pos - _curr._offset; + } + /** + * closes the steam and clears its reference to the + * byte buffer. Once closed it cannot be used. + */ + @Override + public final void close() throws IOException + { + this._buf = null; + this._pos = -1; + } + public final int writeTo(OutputStream out, int len) throws IOException + { + if (_buf == null) throw new IOException("stream is closed"); + fail_on_version_change(); + if (_pos > _buf.size()) throw new IllegalArgumentException(); + + int startingPos = _pos; + int localEnd = _pos + len; + if (localEnd > _buf.size()) localEnd = _buf.size(); + assert(_curr.blockOffsetFromAbsolute(_pos) == _blockPosition); + + while (_pos < localEnd) { + int available = _curr._limit - _blockPosition; + boolean partial_read = available > localEnd - _pos; + if (partial_read) { + available = localEnd - _pos; + } + + out.write(_curr._buffer, _blockPosition, available); + _pos += available; + if (partial_read) { + _blockPosition += available; + break; + } + _curr = _buf.findBlockForRead(this, _version, _curr, _pos); + _blockPosition =_curr.blockOffsetFromAbsolute(_pos); + } + + fail_on_version_change(); + return _pos - startingPos; + } + + public final int writeTo(ByteWriter out, int len) throws IOException + { + if (_buf == null) throw new IOException("stream is closed"); + fail_on_version_change(); + if (_pos > _buf.size()) throw new IllegalArgumentException(); + + int startingPos = _pos; + int localEnd = _pos + len; + if (localEnd > _buf.size()) localEnd = _buf.size(); + assert(_curr.blockOffsetFromAbsolute(_pos) == _blockPosition); + + while (_pos < localEnd) { + int available = _curr._limit - _blockPosition; + boolean partial_read = available > localEnd - _pos; + if (partial_read) { + available = localEnd - _pos; + } + out.write(_curr._buffer, _blockPosition, available); + _pos += available; + if (partial_read) { + _blockPosition += available; + break; + } + _curr = _buf.findBlockForRead(this, _version, _curr, _pos); + _blockPosition =_curr.blockOffsetFromAbsolute(_pos); + } + + fail_on_version_change(); + return _pos - startingPos; + } + + /** + * reads (up to) {@code len} bytes from the buffer and copies them into + * the user supplied byte array (bytes) starting at offset + * off in the users array. This returns the number of bytes + * read, which may be less than the number requested if + * there is not enough data available in the buffer. + * + * @throws IndexOutOfBoundsException + * if {@code (dst.length - offset) < len} + */ + @Override + public final int read(byte[] bytes, int offset, int len) throws IOException + { + if (_buf == null) throw new IOException("stream is closed"); + fail_on_version_change(); + if (_pos > _buf.size()) throw new IllegalArgumentException(); + int startingPos = _pos; + int localEnd = _pos + len; + if (localEnd > _buf.size()) localEnd = _buf.size(); + while (_pos < localEnd) { + bbBlock block = _curr; + int block_offset = _blockPosition; + int available = block._limit - _blockPosition; + if (available > localEnd - _pos) { + // we aren't emptying this block so adjust our location + available = localEnd - _pos; + _blockPosition += available; + } + else { + // TODO can't we just move to the next block? + _curr = _buf.findBlockForRead(this, _version, _curr, _pos + available); + _blockPosition = 0; + } + System.arraycopy(block._buffer, block_offset, bytes, offset, available); + _pos += available; + offset += available; + } + fail_on_version_change(); + return _pos - startingPos; + } + /** + * reads the next byte in the buffer. This returns -1 + * if there is no data available to be read. + */ + @Override + public final int read() throws IOException + { + if (_buf == null) { + throw new IOException("input stream is closed"); + } + fail_on_version_change(); + if (_pos >= _buf.size()) return -1; + if (_blockPosition >= _curr._limit) { + _curr = this._buf.findBlockForRead(this, _version, _curr, _pos); + _blockPosition = 0; + } + int nextByte = (0xff & _curr._buffer[_blockPosition]); + _blockPosition++; + _pos++; + fail_on_version_change(); + return nextByte; + } + private final void fail_on_version_change() throws IOException + { + if (_buf.getVersion() != _version) { + this.close(); + throw new BlockedBufferException("buffer has been changed!"); + } + } + @Override + public final long skip(long n) throws IOException + { + if (n < 0 || n > Integer.MAX_VALUE) throw new IllegalArgumentException("we only handle buffer less than "+Integer.MAX_VALUE+" bytes in length"); + if (_buf == null) throw new IOException("stream is closed"); + fail_on_version_change(); + if (_pos >= _buf.size()) return -1; + int len = (int)n; + if (len == 0) return 0; + int startingPos = _pos; + int localEnd = _pos + len; + if (localEnd > _buf.size()) localEnd = _buf.size(); + // if we run off the end of this block, we need to update + // our current block ( _curr ) we'll update the block position + // in any event (once we know the right block, of course) + if (localEnd > _blockPosition + _curr._offset) { + _curr = _buf.findBlockForRead(this, _version, _curr, localEnd); + } + _blockPosition = localEnd - _curr._offset; + _pos = localEnd; + fail_on_version_change(); + return _pos - startingPos; + } + } + /** + * Reads data from a byte buffer, keeps a local position and + * a current block. Snaps a buffer length on creation; + */ + public static class BlockedByteOutputStream extends java.io.OutputStream + { + BlockedBuffer _buf; + int _pos; + bbBlock _curr; + int _blockPosition; + int _version; + /** + * creates writable stream (OutputStream) that writes + * to a fresh blocked buffer. The stream is initially + * position at offset 0. + */ + public BlockedByteOutputStream() { + _buf = new BlockedBuffer(); + _version = _buf.getVersion(); + _set_position(0); + } + /** + * creates writable stream (OutputStream) that writes + * to the supplied byte buffer. The stream is initially + * position at offset 0. + * @param bb blocked buffer to write to + */ + public BlockedByteOutputStream(BlockedBuffer bb) { + _buf = bb; + _version = _buf.getVersion(); + _set_position(0); + } + /** + * creates writable stream (OutputStream) that can write + * to the supplied byte buffer. The stream is initially + * position at offset off. + * @param bb blocked buffer to write to + * @param off initial offset to write to + */ + public BlockedByteOutputStream(BlockedBuffer bb, int off) { + if (bb == null || off < 0 || off > bb.size() ) { + throw new IllegalArgumentException(); + } + _buf = bb; + _version = _buf.getVersion(); + _set_position(0); + } + /** + * the current offset in the buffer + */ + public final int position() { + return this._pos; + } + /** + * this forces a version sync with the underlying blocked buffer. + * The current position is lost during this call. + * + */ + public final void sync() throws IOException + { + if (_buf == null) throw new IOException("stream is closed"); + _version = _buf.getVersion(); + _pos = 0; + _curr = null; + } + /** + * debug api to force check for internal validity of the + * underlying buffer + */ + public final boolean _validate() { + return this._buf._validate(); + } + /** + * repositions this stream in the buffer. The next + * read, write, or insert operation will take place + * at the specified position. The position must + * be within the contiguous range of written bytes, + * including the pseudo end of file character just + * past the end, which can be written on and returns + * -1 if read. + */ + public final BlockedByteOutputStream setPosition(int pos) throws IOException + { + if (_buf == null) throw new IOException("stream is closed"); + fail_on_version_change(); + if (pos < 0 || pos > _buf.size()) { + throw new IllegalArgumentException(); + } + this._set_position(pos); + fail_on_version_change(); + return this; + } + private final void _set_position(int pos) + { + _pos = pos; + _curr = _buf.findBlockForRead(this, _version, _curr, pos); + _blockPosition = _pos - _curr._offset; + return; + } + /** + * closes the steam and clears its reference to the + * byte buffer. Once closed it cannot be used. + */ + @Override + public final void close() throws IOException + { + this._buf = null; + this._pos = -1; + return; + } + /** + * Inserts space and writes 1 byte to the current + * position in this output stream. Only the low + * order byte of the passed in int is written the + * high order bits are ignored. + */ + @Override + public final void write(int b) throws IOException + { + if (_buf == null) throw new IOException("stream is closed"); + _buf.start_mutate(this, _version); + _write(b); + _version = _buf.end_mutate(this); + return; + } + final void start_write() { + _buf.start_mutate(this, _version); + } + final void end_write() { + _version = _buf.end_mutate(this); + } + final void _write(int b) throws IOException + { + if (bytesAvailableToWriteInCurr(_pos) < 1) { + _curr = _buf.findBlockForWrite(this, _version, _curr, _pos); + assert _curr._offset == _pos; + _blockPosition = 0; + } + _curr._buffer[_blockPosition++] = (byte)(b & 0xff); + _pos++; + if (_blockPosition > _curr._limit) { + _curr._limit = _blockPosition; + if (_pos > _buf._buf_limit ) _buf._buf_limit = _pos; + } + } + private final int bytesAvailableToWriteInCurr(int pos) { + assert _curr != null; + assert _curr._offset <= pos; + assert _curr._offset + _curr._limit >= pos; + if (_curr._idx < this._buf._next_block_position - 1) { + return _curr.bytesAvailableToRead(pos); + } + int ret = _curr._buffer.length - (pos - _curr._offset); + return ret; // _curr.bytesAvailableToWrite(pos); + } + /** + * Writes len bytes from the specified byte array starting + * at in the user array at offset off to the current position + * in this output stream. + */ + @Override + public final void write(byte[] b, int off, int len) throws IOException + { + if (_buf == null) throw new IOException("stream is closed"); + _buf.start_mutate(this, _version); + _write(b, off, len); + _version = _buf.end_mutate(this); + } + private final void _write(byte[] b, int off, int len) + { + int end_b = off + len; + while (off < end_b) + { + int writeInThisBlock = bytesAvailableToWriteInCurr(_pos); + if (writeInThisBlock > end_b - off) { + writeInThisBlock = end_b - off; + } + assert writeInThisBlock >= 0; + if (writeInThisBlock > 0) { + System.arraycopy(b, off, _curr._buffer, _blockPosition, writeInThisBlock); + off += writeInThisBlock; + _pos += writeInThisBlock; + _blockPosition += writeInThisBlock; + if (_blockPosition > _curr._limit) { + _curr._limit = _blockPosition; + if (_pos > _buf._buf_limit) _buf._buf_limit = _pos; + } + else { + assert _pos <= _buf._buf_limit; + } + } + if (off >= end_b) break; + + _curr = _buf.findBlockForWrite(this, _version, _curr, _pos); + _blockPosition = _curr.blockOffsetFromAbsolute(_pos); + assert _curr._offset == _pos || off >= end_b; + } + } + /** + * Writes bytes from the specified byte stream from its current + * stream position to the end of the stream. Writing the bytes + * to the current position in this output stream. + * @throws IOException + */ + public final void write(InputStream bytestream) throws IOException + { + if (_buf == null) throw new IOException("stream is closed"); + _buf.start_mutate(this, _version); + _write(bytestream, -1); + _version = _buf.end_mutate(this); + } + /** + * Writes bytes from the specified byte stream from its current + * stream position up to length bytes from the stream. Writing the + * bytes to the current position in this output stream. + * @throws IOException + */ + public final void write(InputStream bytestream, int len) throws IOException + { + if (_buf == null) throw new IOException("stream is closed"); + _buf.start_mutate(this, _version); + _write(bytestream, len); + _version = _buf.end_mutate(this); + } + /** + * helper to write data. This does not check input arguments. + * @param bytestream source of the data + * @param len number of bytes to read from the input stream, -1 for all + * @throws IOException + */ + private final void _write(InputStream bytestream, int len) throws IOException + { + if (len == 0) return; + + int written = 0; + boolean read_all = (len == -1); + + for (;;) + { + int writeInThisBlock = bytesAvailableToWriteInCurr(_pos); + assert writeInThisBlock >= 0; + + int to_read = read_all ? writeInThisBlock : len; + if (to_read > writeInThisBlock) { + to_read = writeInThisBlock; + } + int len_read = bytestream.read(_curr._buffer, _blockPosition, to_read); + if (len_read == -1) break; + if (len_read > 0) { + _pos += len_read; + _blockPosition += len_read; + if (_blockPosition > _curr._limit) { + _curr._limit = _blockPosition; + if (_pos > _buf._buf_limit) _buf._buf_limit = _pos; + } + else { + assert _pos <= _buf._buf_limit; + } + } + + if (len_read == writeInThisBlock) { + _curr = _buf.findBlockForWrite(this, _version, _curr, _pos); + _blockPosition = _curr.blockOffsetFromAbsolute(_pos); + assert _curr._offset == _pos || written < len_read; + } + else { + assert len_read < writeInThisBlock; + } + if (!read_all) { + len -= len_read; + if (len < 1) break; + } + } + } + /** + * Inserts the amount space requested at the current + * position in this output stream. No data is written + * into the output stream. + */ + public final void insert(int len) throws IOException + { + if (_buf == null) throw new IOException("stream is closed"); + if (len < 0) { + throw new IllegalArgumentException(); + } + if (len > 0) { + _buf.start_mutate(this, _version); + _buf.insert(this, _version, _curr, _pos, len); + _version = _buf.end_mutate(this); + } + return; + } + /** + * Inserts space and writes 1 byte to the current + * position in this output stream. Only the low + * order byte of the passed in int is written the + * high order bits are ignored. + */ + public final void insert(byte b) throws IOException + { + if (_buf == null) throw new IOException("stream is closed"); + _buf.start_mutate(this, _version); + _buf.insert(this, _version, _curr, _pos, 1); + _write(b); + _version = _buf.end_mutate(this); + } + /** + * Inserts space and writes len bytes from the specified + * byte array starting at in the user array at offset off + * to the current position in this output stream. + */ + public final void insert(byte[] b, int off, int len) throws IOException + { + if (_buf == null) throw new IOException("stream is closed"); + _buf.start_mutate(this, _version); + _buf.insert(this, _version, _curr, _pos, len); + _write(b, off, len); + _version = _buf.end_mutate(this); + } + /** + * Inserts space and writes len bytes from the specified + * byte array starting at in the user array at offset off + * to the current position in this output stream. + */ + public final void remove(int len) throws IOException + { + if (_buf == null) throw new IOException("stream is closed"); + _buf.start_mutate(this, _version); + _curr = _buf.remove(this, _version, _curr, _pos, len); + _version = _buf.end_mutate(this); + } + /** + * trucates the buffer at the current location after this + * call the last previously written or read byte will be + * the end of the buffer. + */ + public final void truncate() throws IOException + { + if (_buf == null) throw new IOException("stream is closed"); + if (this._buf._buf_limit == _pos) return; + _buf.start_mutate(this, _version); + _curr = _buf.truncate(this, _version, _pos); + _version = _buf.end_mutate(this); + } + private final void fail_on_version_change() throws IOException + { + if (_buf.getVersion() != _version) { + this.close(); + throw new BlockedBufferException("buffer has been changed!"); + } + } + } + public static class BlockedBufferException extends IonException + { + private static final long serialVersionUID = 1582507845614969389L; + public BlockedBufferException() { super(); } + public BlockedBufferException(String message) { super(message); } + public BlockedBufferException(String message, Throwable cause) { + super(message, cause); + } + public BlockedBufferException(Throwable cause) { super(cause); } + } + public static class BufferedOutputStream + extends OutputStream + { + BlockedBuffer _buffer; + BlockedByteOutputStream _writer; + public BufferedOutputStream() { + this(new BlockedBuffer()); + } + public BufferedOutputStream(BlockedBuffer buffer) { + _buffer = buffer; + _writer = new BlockedByteOutputStream(_buffer); + } + /** + * Gets the size in bytes of this binary data. + * This is generally needed before calling {@link #getBytes()} or + * {@link #getBytes(byte[], int, int)}. + * + * @return the size in bytes. + */ + public int byteSize() + { + return _buffer.size(); + } + /** + * Copies the current contents of this writer as a new byte array holding + * Ion binary-encoded data. + * This allocates an array of the size needed to exactly + * hold the output and copies the entire value to it. + * + * @return the byte array with the writers output + * @throws IOException + */ + public byte[] getBytes() + throws IOException + { + int size = byteSize(); + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(size); + writeBytes(byteStream); + byte[] bytes = byteStream.toByteArray(); + return bytes; + } + /** + * Copies the current contents of the writer to a given byte array + * array. This starts writing to the array at offset and writes + * up to len bytes. + * If this writer is not able to stop in the middle of its + * work this may overwrite the array and later throw and exception. + * + * @param bytes users byte array to write into + * @param offset initial offset in the array to write into + * @param len maximum number of bytes to write from offset on + * @return number of bytes written + * @throws IOException + */ + public int getBytes(byte[] bytes, int offset, int len) + throws IOException + { + SimpleByteBuffer outbuf = new SimpleByteBuffer(bytes, offset, len); + OutputStream writer = (OutputStream)outbuf.getWriter(); + int written = writeBytes(writer); + return written; + } + /** + * Writes the current contents of the writer to the output + * stream. This is only valid if the writer is not in the + * middle of writing a container. + * + * @param userstream OutputStream to write the bytes to + * @return int length of bytes written + * @throws IOException + */ + public int writeBytes(OutputStream userstream) + throws IOException + { + int limit = _buffer.size(); + int pos = 0; + int version = _buffer.getVersion(); + bbBlock curr = null; + _buffer.start_mutate(this, version); + while (pos < limit) { + curr = _buffer.findBlockForRead(this, version, curr, pos); + if (curr == null) { + throw new IOException("buffer missing expected bytes"); + } + int len = curr.bytesAvailableToRead(pos); + if (len <= 0) { + throw new IOException("buffer missing expected bytes"); + } + userstream.write(curr._buffer, 0, len); + pos += len; + } + _buffer.end_mutate(this); + return pos; + } + @Override + public void write(int b) throws IOException + { + _writer.write(b); + } + @Override + public void write(byte[] bytes) throws IOException + { + write(bytes, 0, bytes.length); + } + @Override + public void write(byte[] bytes, int off, int len) throws IOException + { + _writer.write(bytes, off, len); + } + } +} diff --git a/src/com/amazon/ion/impl/ByteBuffer.java b/src/com/amazon/ion/impl/ByteBuffer.java new file mode 100644 index 0000000000..941ea7bf27 --- /dev/null +++ b/src/com/amazon/ion/impl/ByteBuffer.java @@ -0,0 +1,72 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * This is a general purpose interface to support operations + * over byte sources for Ion operations. The implementations + * are resposible for managing container state during operations + * as well as the underlying data in the buffer itself. + */ +interface ByteBuffer +{ + /** + * creates a reader and sets it to the beginning of the buffer + * under managment here for use. + * @return a new reader + */ + public abstract ByteReader getReader(); + + /** + * creates a writer and sets it to the beginning of the buffer + * under managment here for use. + * @return a new writer + */ + public abstract ByteWriter getWriter(); + + /** + * wraps up the current buffer as a new byte array and passed + * ownership of the copy to the caller. + * @return a copy of the buffer in a new byte array + */ + public abstract byte[] getBytes(); + + /** + * copies the contents of the buffer into a user supplied + * byte array. It copies into the user buffer starting at + * the supplied offset and attempts to copy no more than + * length bytes. If the implmentation cannot determine the + * length to be copied before copying it may overwrite some + * of the user buffer and will throw at the end of the copy. + * (note that is may throw an out of bounds exception if + * the write extends beyond the length of the callers array) + * @param buffer users byte array to copy into + * @param offset first byte of the array to use + * @param length length of the byte array available to copy into + * @return number of bytes actually copied + */ + public abstract int getBytes(byte[] buffer, int offset, int length); + + /** + * copies the contents of the buffer into and output stream. + * @param out stream to write into + * @throws IOException + */ + public abstract void writeBytes(OutputStream out) throws IOException; +} diff --git a/src/com/amazon/ion/impl/ByteReader.java b/src/com/amazon/ion/impl/ByteReader.java new file mode 100644 index 0000000000..3bf0e2a771 --- /dev/null +++ b/src/com/amazon/ion/impl/ByteReader.java @@ -0,0 +1,185 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import com.amazon.ion.Decimal; +import com.amazon.ion.Timestamp; +import java.io.IOException; +/** + * Interface to read bytes over a variety of sources. + * Avoiding the Java standard overhead in so far as possible. + * This requires the ability to access any part of the underlying + * data source randomly. However forward access is expected to be + * the standard direction. + */ +interface ByteReader +{ + /** + * yet another definition of eof of file as -1 + */ + public static final int EOF = -1; + + /** + * returns the current position of the reader in the underlying + * byte buffer + * @return offset of the next byte to read + */ + public int position(); + /** + * sets the position of the reader so that it will read the + * byte at the passed in offset (newPosition) on the next call + * to read() + * @param newPosition offset to position the reader to + */ + public void position(int newPosition); + /** + * moves the reader forward length bytes. This is typically + * faster than read as it has no need to actually move any + * bytes of data. It may error if the skip would take the + * current position off the end of the buffer. + * @param length number of bytes to skip + */ + public void skip(int length); + + /** + * reads a single byte and returns it as an unsigned value, + * that is from 0 to 255. This returns a ByteReader.EOF (-1) + * if there is not data to read. + * @return next byte of data + * @throws IOException + */ + public int read() throws IOException; + /** + * reads a number of bytes into the callers supplied buffer (dst). + * This will return the number of bytes read, if any. + * @param dst callers destination byte array + * @param start offset of the first destination array element to write into + * @param len maximum number of bytes to copy + * @return number of bytes actually copied + * @throws IOException + */ + public int read(byte[] dst, int start, int len) throws IOException; + + /** + * reads the next byte as a typedesc bytes + * @return typedesc byte + * @throws IOException + */ + public int readTypeDesc() throws IOException; + + /** + * reads the upcoming bytes as a signed long. This uses + * 7 bits per byte, expects the bytes in bigendian order, and + * looks for the high order bit (0x80) to be set to mark the + * end of the value. It expects the 2nd bit (0x40 of the first + * byte) to be the sign of the value. The remaining bits are + * the absolute value of the orginal value. It + * returns the aggregated bytes as a Java long (signed 64 bit + * value) correctly signed. + * @return long value read + * @throws IOException + */ + public long readVarLong() throws IOException; + /** + * reads the upcoming bytes as a unsigned long. This uses + * 7 bits per byte, expects the bytes in bigendian order, and + * looks for the high order bit (0x80) to be set to mark the + * end of the value. It does not include a sign bit so all 7 bits + * in the first byte are treated as positive. It + * returns the aggregated bytes as a Java long (signed 64 bit + * value). + * @return long value read + * @throws IOException + */ + public long readVarULong() throws IOException; + /** + * reads the next len bytes as a signed long. This uses + * 8 bits per byte and expects the bytes in bigendian order. It + * returns the aggregated bytes as a Java long (signed 64 bit + * value). + * @param len number of bytes to read into the long + * @return long value read + * @throws IOException + */ + public long readULong(int len) throws IOException; + + /** + * reads the upcoming bytes as a signed int. This uses + * 7 bits per byte, expects the bytes in bigendian order, and + * looks for the high order bit (0x80) to be set to mark the + * end of the value. It expects the 2nd bit (0x40 of the first + * byte) to be the sign of the value. The remaining bits are + * the absolute value of the orginal value. It + * returns the aggregated bytes as a Java long (signed 32 bit + * value) correctly signed. + * @return int value read + * @throws IOException + */ + public int readVarInt() throws IOException; + /** + * reads the upcoming bytes as a unsigned int. This uses + * 7 bits per byte, expects the bytes in bigendian order, and + * looks for the high order bit (0x80) to be set to mark the + * end of the value. It does not include a sign bit so all 7 bits + * in the first byte are treated as positive. It + * returns the aggregated bytes as a Java long (signed 32 bit + * value). + * @return int value read + * @throws IOException + */ + public int readVarUInt() throws IOException; + + /** + * reads len bytes and treats them as an IEEE 64 binary + * floating point value. Currently the only valid values for + * len are 0 (which returns 0.0e1) or 8 which is treated + * as a 64 bit binary float and returned. + * @param len 0 or 8 + * @return 0.0 or the double represented by the next 8 bytes + * @throws IOException + */ + public double readFloat(int len) throws IOException; + /** + * reads the next len bytes and converts them to a {@link Decimal} value. + * @param len number of bytes over which the value extends + * @return Decimal object initialized to the represented value; not null. + * @throws IOException + */ + public Decimal readDecimal(int len) throws IOException; + + /** + * Reads the next len bytes and returns them as a {@link Timestamp} + * object. This contains an UTC date and time value and a + * timezone offset of the original value. + * + * @param len + * @return a new {@link Timestamp} instance, + * or {@code null} if the encoded value is {@code null.timestamp}. + * @throws IOException + */ + public Timestamp readTimestamp(int len) throws IOException; + + /** reads the next len bytes and converts them from UTF-8 into Java + * characters and returns those characters as a String. This may throw + * an exception in the event it encounters byte sequences that are not + * value UTF-8. This converts some Unicode values into character pairs + * when the value requires the use of surrogate pairs in Unicode 16. + * @param len number of bytes to read + * @return resultant Java string + * @throws IOException + */ + public String readString(int len) throws IOException; +} diff --git a/src/com/amazon/ion/impl/ByteWriter.java b/src/com/amazon/ion/impl/ByteWriter.java new file mode 100644 index 0000000000..b1f9251aab --- /dev/null +++ b/src/com/amazon/ion/impl/ByteWriter.java @@ -0,0 +1,53 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import java.io.IOException; +import java.math.BigDecimal; + +/** + * Interface to write Ion value to bytes over a variety of destinations. + * Avoiding the Java standard overhead in so far as possible. + * This requires the ability to access any part of the underlying + * data source randomly and insert space at any position. However forward + * access is expected to be the standard direction. + */ +interface ByteWriter +{ + public void write(byte b) throws IOException; + public void write(byte[] dst, int start, int len) throws IOException; + public int position(); + public void position(int newPosition); + + public void insert(int length); + public void remove(int length); + + public void writeTypeDesc(int typeDescByte) throws IOException; + public int writeTypeDescWithLength(int typeid, int lenOfLength, int valueLength) throws IOException; + public int writeTypeDescWithLength(int typeid, int valueLength) throws IOException; + + public int writeIonInt(long value, int len) throws IOException; + public int writeVarInt(long value, int len, boolean force_zero_write) throws IOException; + public int writeVarUInt(long value, int len, boolean force_zero_write) throws IOException; + + public int writeIonInt(int value, int len) throws IOException; + public int writeVarInt(int value, int len, boolean force_zero_write) throws IOException; + public int writeVarUInt(int value, int len, boolean force_zero_write) throws IOException; + + public int writeFloat(double value) throws IOException; + public int writeDecimal(BigDecimal value) throws IOException; + public int writeString(String value) throws IOException; +} diff --git a/src/software/amazon/ion/impl/DowncastingFaceted.java b/src/com/amazon/ion/impl/DowncastingFaceted.java similarity index 57% rename from src/software/amazon/ion/impl/DowncastingFaceted.java rename to src/com/amazon/ion/impl/DowncastingFaceted.java index 13a37e1452..171f7b1f77 100644 --- a/src/software/amazon/ion/impl/DowncastingFaceted.java +++ b/src/com/amazon/ion/impl/DowncastingFaceted.java @@ -1,19 +1,22 @@ + /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; -import software.amazon.ion.facet.Faceted; +package com.amazon.ion.impl; + +import com.amazon.ion.facet.Faceted; /** * Provides a simple implementation of {@link Faceted} diff --git a/src/com/amazon/ion/impl/IonBinary.java b/src/com/amazon/ion/impl/IonBinary.java new file mode 100644 index 0000000000..956d49212c --- /dev/null +++ b/src/com/amazon/ion/impl/IonBinary.java @@ -0,0 +1,2942 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import static com.amazon.ion.impl._Private_IonConstants.BINARY_VERSION_MARKER_SIZE; +import static com.amazon.ion.impl._Private_Utils.EMPTY_BYTE_ARRAY; +import static com.amazon.ion.impl._Private_Utils.readFully; +import static com.amazon.ion.util.IonStreamUtils.isIonBinary; + +import com.amazon.ion.Decimal; +import com.amazon.ion.IonException; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.Timestamp; +import com.amazon.ion.Timestamp.Precision; +import com.amazon.ion.UnexpectedEofException; +import com.amazon.ion.impl._Private_IonConstants.HighNibble; +import com.amazon.ion.util.IonTextUtils; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PushbackReader; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.MathContext; +import java.util.ArrayList; +import java.util.Stack; + +final class IonBinary +{ + static boolean debugValidation = false; + + private static final BigInteger MAX_LONG_VALUE = new BigInteger(Long.toString(Long.MAX_VALUE)); + private static final int SIZE_OF_LONG = 8; + + final static int _ib_TOKEN_LEN = 1; + final static int _ib_VAR_INT32_LEN_MAX = 5; // 31 bits (java limit) / 7 bits per byte = 5 bytes + final static int _ib_VAR_INT64_LEN_MAX = 10; // 63 bits (java limit) / 7 bits per byte = 10 bytes + final static int _ib_INT64_LEN_MAX = 8; + final static int _ib_FLOAT64_LEN = 8; + + private static final Double DOUBLE_POS_ZERO = Double.valueOf(0.0); + + static final int ZERO_DECIMAL_TYPEDESC = + _Private_IonConstants.makeTypeDescriptor(_Private_IonConstants.tidDecimal, + _Private_IonConstants.lnNumericZero); + + static final int NULL_DECIMAL_TYPEDESC = + _Private_IonConstants.makeTypeDescriptor(_Private_IonConstants.tidDecimal, + _Private_IonConstants.lnIsNullAtom); + + private IonBinary() { } + + + /** + * Verifies that a reader starts with a valid Ion cookie, throwing an + * exception if it does not. + * + * @param reader must not be null. + * @throws IonException if there's a problem reading the cookie, or if the + * data does not start with {@link _Private_IonConstants#BINARY_VERSION_MARKER_1_0}. + */ + public static void verifyBinaryVersionMarker(Reader reader) + throws IonException + { + try + { + int pos = reader.position(); + //reader.sync(); + //reader.setPosition(0); + byte[] bvm = new byte[BINARY_VERSION_MARKER_SIZE]; + int len = readFully(reader, bvm); + if (len < BINARY_VERSION_MARKER_SIZE) + { + String message = + "Binary data is too short: at least " + + BINARY_VERSION_MARKER_SIZE + + " bytes are required, but only " + len + " were found."; + throw new IonException(message); + + } + + if (! isIonBinary(bvm)) + { + StringBuilder buf = new StringBuilder(); + buf.append("Binary data has unrecognized header"); + for (int i = 0; i < bvm.length; i++) + { + int b = bvm[i] & 0xFF; + buf.append(" 0x"); + buf.append(Integer.toHexString(b).toUpperCase()); + } + throw new IonException(buf.toString()); + } + + //reader.setPosition(0); // cas 19 apr 2008 + reader.setPosition(pos); // cas 5 may 2009 :) + } + catch (IOException e) + { + throw new IonException(e); + } + } + + /* imported from SimpleByteBuffer.SimpleByteWriter */ + + static public int writeTypeDescWithLength(OutputStream userstream, int typeid, int lenOfLength, int valueLength) throws IOException + { + int written_len = 1; + + int td = ((typeid & 0xf) << 4); + if (valueLength >= _Private_IonConstants.lnIsVarLen) { + td |= _Private_IonConstants.lnIsVarLen; + userstream.write((byte)(td & 0xff)); + written_len += writeVarUInt(userstream, (long)valueLength, lenOfLength, true); + } + else { + td |= (valueLength & 0xf); + userstream.write((byte)(td & 0xff)); + } + return written_len; + } + + static public int writeTypeDescWithLength(OutputStream userstream, int typeid, int valueLength) throws IOException + { + int written_len = 1; + int td = ((typeid & 0xf) << 4); + + if (valueLength >= _Private_IonConstants.lnIsVarLen) { + td |= _Private_IonConstants.lnIsVarLen; + userstream.write((byte)(td & 0xff)); + int lenOfLength = IonBinary.lenVarUInt(valueLength); + written_len += writeVarUInt(userstream, (long)valueLength, lenOfLength, true); + } + else { + td |= (valueLength & 0xf); + userstream.write((byte)(td & 0xff)); + } + return written_len; + } + + static public int writeIonInt(OutputStream userstream, long value, int len) throws IOException + { + // we shouldn't be writing out 0's as an Ion int value + if (value == 0) { + assert len == 0; + return len; // aka 0 + } + + // figure out how many we have bytes we have to write out + long mask = 0xffL; + boolean is_negative = (value < 0); + + assert len == IonBinary.lenIonInt(value); + + if (is_negative) { + value = -value; + // note for Long.MIN_VALUE the negation returns + // itself as a value, but that's also the correct + // "positive" value to write out anyway, so it + // all works out + } + + // write the rest + switch (len) { // we already wrote 1 byte + case 8: userstream.write((byte)((value >> (8*7)) & mask)); + case 7: userstream.write((byte)((value >> (8*6)) & mask)); + case 6: userstream.write((byte)((value >> (8*5)) & mask)); + case 5: userstream.write((byte)((value >> (8*4)) & mask)); + case 4: userstream.write((byte)((value >> (8*3)) & mask)); + case 3: userstream.write((byte)((value >> (8*2)) & mask)); + case 2: userstream.write((byte)((value >> (8*1)) & mask)); + case 1: userstream.write((byte)(value & mask)); + } + + return len; + } + + static public int writeVarUInt(OutputStream userstream, long value) + throws IOException + { + int len = IonBinary.lenVarUInt(value); + writeVarUInt(userstream, value, len, false); + return len; + } + + static public int writeVarUInt(OutputStream userstream, long value, int len, + boolean force_zero_write) + throws IOException + { + int mask = 0x7F; + assert len == IonBinary.lenVarUInt(value); + assert value >= 0; + + switch (len - 1) { + case 9: userstream.write((byte)((value >> (7*9)) & mask)); + case 8: userstream.write((byte)((value >> (7*8)) & mask)); + case 7: userstream.write((byte)((value >> (7*7)) & mask)); + case 6: userstream.write((byte)((value >> (7*6)) & mask)); + case 5: userstream.write((byte)((value >> (7*5)) & mask)); + case 4: userstream.write((byte)((value >> (7*4)) & mask)); + case 3: userstream.write((byte)((value >> (7*3)) & mask)); + case 2: userstream.write((byte)((value >> (7*2)) & mask)); + case 1: userstream.write((byte)((value >> (7*1)) & mask)); + case 0: userstream.write((byte)((value & mask) | 0x80L)); + break; + case -1: // or len == 0 + if (force_zero_write) { + userstream.write((byte)0x80); + assert len == 1; + } + else { + assert len == 0; + } + break; + } + return len; + } + + static public int writeString(OutputStream userstream, String value) + throws IOException + { + int len = 0; + for (int ii=0; ii= 0xD800 && c <= 0xDFFF) { + if (_Private_IonConstants.isHighSurrogate(c)) { + // houston we have a high surrogate (let's hope it has a partner + if (++ii >= value.length()) { + throw new IllegalArgumentException("invalid string, unpaired high surrogate character"); + } + int c2 = value.charAt(ii); + if (!_Private_IonConstants.isLowSurrogate(c2)) { + throw new IllegalArgumentException("invalid string, unpaired high surrogate character"); + } + c = _Private_IonConstants.makeUnicodeScalar(c, c2); + } + else if (_Private_IonConstants.isLowSurrogate(c)) { + // it's a loner low surrogate - that's an error + throw new IllegalArgumentException("invalid string, unpaired low surrogate character"); + } + } + + for (c = makeUTF8IntFromScalar(c); (c & 0xffffff00) != 0; ++len) { + userstream.write((byte)(c & 0xff)); + c = c >>> 8; + } + } + return len; + } + + // TODO: move this to IonConstants or IonUTF8 + static final public int makeUTF8IntFromScalar(int c) throws IOException + { + // TO DO: check this encoding, it is from: + // http://en.wikipedia.org/wiki/UTF-8 + // we probably should use some sort of Java supported + // library for this. this class might be of interest: + // CharsetDecoder(Charset cs, float averageCharsPerByte, float maxCharsPerByte) + // in: java.nio.charset.CharsetDecoder + + int value = 0; + + // first the quick, easy and common case - ascii + if (c < 0x80) { + value = (0xff & c ); + } + else if (c < 0x800) { + // 2 bytes characters from 0x000080 to 0x0007FF + value = ( 0xff & (0xC0 | (c >> 6) ) ); + value |= ( 0xff & (0x80 | (c & 0x3F)) ) << 8; + } + else if (c < 0x10000) { + // 3 byte characters from 0x800 to 0xFFFF + // but only 0x800...0xD7FF and 0xE000...0xFFFF are valid + if (c > 0xD7FF && c < 0xE000) { + throwUTF8Exception(); + } + value = ( 0xff & (0xE0 | (c >> 12) ) ); + value |= ( 0xff & (0x80 | ((c >> 6) & 0x3F)) ) << 8; + value |= ( 0xff & (0x80 | (c & 0x3F)) ) << 16; + + } + else if (c <= 0x10FFFF) { + // 4 byte characters 0x010000 to 0x10FFFF + // these are are valid + value = ( 0xff & (0xF0 | (c >> 18)) ); + value |= ( 0xff & (0x80 | ((c >> 12) & 0x3F)) ) << 8; + value |= ( 0xff & (0x80 | ((c >> 6) & 0x3F)) ) << 16; + value |= ( 0xff & (0x80 | (c & 0x3F)) ) << 24; + } + else { + throwUTF8Exception(); + } + return value; + } + static void throwUTF8Exception() throws IOException + { + throw new IOException("Invalid UTF-8 character encountered"); + } + + + public static class BufferManager + { + BlockedBuffer _buf; + IonBinary.Reader _reader; + IonBinary.Writer _writer; + + public BufferManager() { + _buf = new BlockedBuffer(); + this.openReader(); + this.openWriter(); + } + public BufferManager(BlockedBuffer buf) { + _buf = buf; + this.openReader(); + this.openWriter(); + } + + /** + * Creates a new buffer containing the entire content of an + * {@link InputStream}. + * + * @param bytestream a stream interface the byte image to buffer + * + * @throws IonException wrapping any {@link IOException}s thrown by the + * stream. + */ + public BufferManager(InputStream bytestream) + { + this(); // this will create a fresh buffer + // as well as a reader and writer + + // now we move the data from the input stream to the buffer + // more or less as fast as we can. I am (perhaps foolishly) + // assuming the "available()" is a useful size. + try + { + _writer.write(bytestream); + } + catch (IOException e) + { + throw new IonException(e); + } + } + + /** + * Creates a new buffer containing the entire content of an + * {@link InputStream}. + * + * @param bytestream a stream interface the byte image to buffer + * + * @throws IonException wrapping any {@link IOException}s thrown by the + * stream. + */ + public BufferManager(InputStream bytestream, int len) + { + this(); // this will create a fresh buffer + // as well as a reader and writer + + // now we move the data from the input stream to the buffer + // more or less as fast as we can. I am (perhaps foolishly) + // assuming the "available()" is a useful size. + try + { + _writer.write(bytestream, len); + } + catch (IOException e) + { + throw new IonException(e); + } + } + + @Override + public BufferManager clone() throws CloneNotSupportedException + { + BlockedBuffer buffer_clone = this._buf.clone(); + BufferManager clone = new BufferManager(buffer_clone); + return clone; + } + + public IonBinary.Reader openReader() { + if (_reader == null) { + _reader = new IonBinary.Reader(_buf); + } + return _reader; + } + public IonBinary.Writer openWriter() { + if (_writer == null) { + _writer = new IonBinary.Writer(_buf); + } + return _writer; + } + + public BlockedBuffer buffer() { return _buf; } + public IonBinary.Reader reader() { return _reader; } + public IonBinary.Writer writer() { return _writer; } + + public IonBinary.Reader reader(int pos) throws IOException + { + _reader.setPosition(pos); + return _reader; + } + public IonBinary.Writer writer(int pos) throws IOException + { + _writer.setPosition(pos); + return _writer; + } + public static BufferManager makeReadManager(BlockedBuffer buf) { + BufferManager bufmngr = new BufferManager(buf); + bufmngr.openReader(); + return bufmngr; + } + public static BufferManager makeReadWriteManager(BlockedBuffer buf) { + BufferManager bufmngr = new BufferManager(buf); + bufmngr.openReader(); + bufmngr.openWriter(); + return bufmngr; + } + } + + /** + * Variable-length, high-bit-terminating integer, 7 data bits per byte. + */ + public static int lenVarUInt(long longVal) { + assert longVal >= 0; + if (longVal < (1L << (7 * 1))) return 1; // 7 bits + if (longVal < (1L << (7 * 2))) return 2; // 14 bits + if (longVal < (1L << (7 * 3))) return 3; // 21 bits + if (longVal < (1L << (7 * 4))) return 4; // 28 bits + if (longVal < (1L << (7 * 5))) return 5; // 35 bits + if (longVal < (1L << (7 * 6))) return 6; // 42 bits + if (longVal < (1L << (7 * 7))) return 7; // 49 bits + if (longVal < (1L << (7 * 8))) return 8; // 56 bits + if (longVal < (1L << (7 * 9))) return 9; // 63 bits + return 10; + } + + // TODO maybe add lenVarInt(int) to micro-optimize, or? + + public static int lenVarInt(long longVal) { + if (longVal == 0) { + return 0; + } + if (longVal < 0) longVal = -longVal; + if (longVal < (1L << (7 * 1 - 1))) return 1; // 6 bits + if (longVal < (1L << (7 * 2 - 1))) return 2; // 13 bits + if (longVal < (1L << (7 * 3 - 1))) return 3; // 20 bits + if (longVal < (1L << (7 * 4 - 1))) return 4; // 27 bits + if (longVal < (1L << (7 * 5 - 1))) return 5; // 34 bits + if (longVal < (1L << (7 * 6 - 1))) return 6; // 41 bits + if (longVal < (1L << (7 * 7 - 1))) return 7; // 48 bits + if (longVal < (1L << (7 * 8 - 1))) return 8; // 55 bits + if (longVal < (1L << (7 * 9 - 1))) return 9; // 62 bits + return 10; + } + + /** + * @return zero if input is zero + */ + public static int lenUInt(long longVal) { + if (longVal == 0) { + return 0; + } + if (longVal < 0) { + throw new BlockedBuffer.BlockedBufferException("fatal signed long where unsigned was promised"); + } + if (longVal < (1L << (8 * 1))) return 1; // 8 bits + if (longVal < (1L << (8 * 2))) return 2; // 16 bits + if (longVal < (1L << (8 * 3))) return 3; // 24 bits + if (longVal < (1L << (8 * 4))) return 4; // 32 bits + if (longVal < (1L << (8 * 5))) return 5; // 40 bits + if (longVal < (1L << (8 * 6))) return 6; // 48 bits + if (longVal < (1L << (8 * 7))) return 7; // 56 bits + return 8; + } + + public static int lenUInt(BigInteger bigVal) + { + if (bigVal.signum() < 0) { + throw new IllegalArgumentException("lenUInt expects a non-negative a value"); + } + final int bits = bigVal.bitLength(); + // determine the number of octets needed to represent this bit pattern + // (div 8) + int bytes = bits >> 3; + // if we have a bit more than what can fit in an octet, we need to add + // an extra octet (mod 8) + if ((bits & 0x7) != 0) + { + bytes++; + } + return bytes; + } + + // TODO maybe add lenInt(int) to micro-optimize, or? + + public static int lenInt(long longVal) { + if (longVal != 0) { + return 0; + } + if (longVal < 0) longVal = -longVal; + if (longVal < (1L << (8 * 1 - 1))) return 1; // 7 bits + if (longVal < (1L << (8 * 2 - 1))) return 2; // 15 bits + if (longVal < (1L << (8 * 3 - 1))) return 3; // 23 bits + if (longVal < (1L << (8 * 4 - 1))) return 4; // 31 bits + if (longVal < (1L << (8 * 5 - 1))) return 5; // 39 bits + if (longVal < (1L << (8 * 6 - 1))) return 6; // 47 bits + if (longVal < (1L << (8 * 7 - 1))) return 7; // 55 bits + if (longVal == Long.MIN_VALUE) return 9; + return 8; + } + + public static int lenInt(BigInteger bi, boolean force_zero_writes) + { + int len = bi.abs().bitLength() + 1; // add 1 for the sign bit (which must always be present) + int bytelen = 0; + + switch (bi.signum()) { + case 0: + bytelen = force_zero_writes ? 1 : 0; + break; + case 1: + case -1: + bytelen = ((len-1) / 8) + 1; + break; + } + + return bytelen; + } + + public static int lenIonInt(long v) { + if (v < 0) { + // note that for Long.MIN_VALUE (0x8000000000000000) the negative + // is the same, but that's also the bit pattern we need to + // write out, but the UInt method won't like it, so we just + // return then value that we actually know. + if (v == Long.MIN_VALUE) return SIZE_OF_LONG; + return IonBinary.lenUInt(-v); + } + else if (v > 0) { + return IonBinary.lenUInt(v); + } + return 0; // CAS UPDATE, was 1 + } + + public static int lenIonInt(BigInteger v) + { + if (v.signum() < 0) + { + v = v.negate(); + } + int len = lenUInt(v); + return len; + } + + /** + * The size of length value when short lengths are recorded in the + * typedesc low-nibble. + * @param valuelen + * @return zero if valuelen < 14 + */ + public static int lenLenFieldWithOptionalNibble(int valuelen) { + if (valuelen < _Private_IonConstants.lnIsVarLen) { + return 0; + } + return lenVarUInt(valuelen); + } + + public static int lenTypeDescWithAppropriateLenField(int type, int valuelen) + { + switch (type) { + case _Private_IonConstants.tidNull: // null(0) + case _Private_IonConstants.tidBoolean: // boolean(1) + return _Private_IonConstants.BB_TOKEN_LEN; + + case _Private_IonConstants.tidPosInt: // 2 + case _Private_IonConstants.tidNegInt: // 3 + case _Private_IonConstants.tidFloat: // float(4) + case _Private_IonConstants.tidDecimal: // decimal(5) + case _Private_IonConstants.tidTimestamp: // timestamp(6) + case _Private_IonConstants.tidSymbol: // symbol(7) + case _Private_IonConstants.tidString: // string (8) + case _Private_IonConstants.tidClob: // clob(9) + case _Private_IonConstants.tidBlob: // blob(10) + case _Private_IonConstants.tidList: // 11 -- cas mar 6 2008, moved containers + case _Private_IonConstants.tidSexp: // 12 -- up heresince they now use the same + case _Private_IonConstants.tidStruct: // 13 -- length encodings as scalars + case _Private_IonConstants.tidTypedecl: // 14 + if (valuelen < _Private_IonConstants.lnIsVarLen) { + return _Private_IonConstants.BB_TOKEN_LEN; + } + return _Private_IonConstants.BB_TOKEN_LEN + lenVarUInt(valuelen); + + case _Private_IonConstants.tidUnused: // unused(15) + default: + throw new IonException("invalid type"); + } + } + + public static int lenIonFloat(double value) { + if (Double.valueOf(value).equals(DOUBLE_POS_ZERO)) + { + // pos zero special case + return 0; + } + + // always 8-bytes for IEEE-754 64-bit + return _ib_FLOAT64_LEN; + } + + /** + * Would this value have a zero length-nibble? That is: is it 0d0 ? + */ + public static boolean isNibbleZero(BigDecimal bd) + { + if (Decimal.isNegativeZero(bd)) return false; + if (bd.signum() != 0) return false; + int scale = bd.scale(); + return (scale == 0); + } + + /** + * @param bd must not be null. + */ + public static int lenIonDecimal(BigDecimal bd) + { + // first check for the special cases of null + // and 0d0 which are encoded in the nibble + if (bd == null) return 0; + if (isNibbleZero(bd)) return 0; + + // otherwise do this the hard way + BigInteger mantissa = bd.unscaledValue(); + + // negative zero mantissa must be written, positive zero does not + boolean forceMantissa = Decimal.isNegativeZero(bd); + int mantissaByteCount = lenInt(mantissa, forceMantissa); + + // We really need the length of the exponent (-scale) but in our + // representation the length is the same regardless of sign. + int scale = bd.scale(); + int exponentByteCount = lenVarInt(scale); + + // Exponent is always at least one byte. + if (exponentByteCount == 0) exponentByteCount = 1; + + return exponentByteCount + mantissaByteCount; + } + + + /** + * this method computes the output length of this timestamp value + * in the Ion binary format. It does not include the length of + * the typedesc byte that preceeds the actual value. The output + * length of a null value is 0, as a result this this. + * @param di may be null + */ + public static int lenIonTimestamp(Timestamp di) + { + if (di == null) return 0; + + int len = 0; + switch (di.getPrecision()) { + case FRACTION: + case SECOND: + { + BigDecimal fraction = di.getFractionalSecond(); + if (fraction != null) + { + assert fraction.signum() >=0 && ! fraction.equals(BigDecimal.ZERO) + : "Bad timestamp fraction: " + fraction; + + // Since the fraction is not 0d0, at least one subfield of the + // exponent and mantissa is non-zero, so this will always write at + // least one byte. + int fracLen = IonBinary.lenIonDecimal(fraction); + assert fracLen > 0; + len += fracLen; + } + + len++; // len of seconds < 60 + } + case MINUTE: + len += 2; // len of hour and minutes (both < 127) + case DAY: + len += 1; // len of month and day (both < 127) + case MONTH: + len += 1; // len of month and day (both < 127) + case YEAR: + len += IonBinary.lenVarUInt(di.getZYear()); + } + Integer offset = di.getLocalOffset(); + if (offset == null) { + len++; // room for the -0 (i.e. offset is "no specified offset") + } + else if (offset == 0) { + len++; + } + else { + len += IonBinary.lenVarInt(offset.longValue()); + } + return len; + } + + /** + * @param v may be null. + * @throws IllegalArgumentException if the text contains bad UTF-16 data. + */ + public static int lenIonString(String v) + { + if (v == null) return 0; + int len = 0; + + for (int ii=0; ii= 0xD800 && c <= 0xDFFF) { + // look for surrogate pairs and merge them (and throw on bad data) + if (_Private_IonConstants.isHighSurrogate(c)) { + ii++; + if (ii >= v.length()) { + String message = + "Text ends with unmatched UTF-16 surrogate " + + IonTextUtils.printCodePointAsString(c); + throw new IllegalArgumentException(message); + } + int c2 = v.charAt(ii); + if (!_Private_IonConstants.isLowSurrogate(c2)) { + String message = + "Text contains unmatched UTF-16 high surrogate " + + IonTextUtils.printCodePointAsString(c) + + " at index " + (ii-1); + throw new IllegalArgumentException(message); + } + c = _Private_IonConstants.makeUnicodeScalar(c, c2); + } + else if (_Private_IonConstants.isLowSurrogate(c)) { + String message = + "Text contains unmatched UTF-16 low surrogate " + + IonTextUtils.printCodePointAsString(c) + + " at index " + ii; + throw new IllegalArgumentException(message); + } + } + // no need to check the 0x10FFFF overflow as it is checked in lenUnicodeScalarAsUTF8 + + // and now figure out how long this "complicated" (non-ascii) character is + if (c < 0x80) { + ++len; + } + else if (c < 0x800) { + len += 2; + } + else if (c < 0x10000) { + len += 3; + } + else if (c <= 0x10FFFF) { // just about as cheap as & == 0 and checks for some out of range values + len += 4; + } else { + // TODO how is this possible? + throw new IllegalArgumentException("invalid string, illegal Unicode scalar (character) encountered"); + } + } + + return len; + } + + public static int lenAnnotationListWithLen(String[] annotations, + SymbolTable symbolTable) + { + int annotationLen = 0; + + if (annotations != null) { + // add up the length of the encoded symbols + for (int ii=0; ii 0; // TODO amzn/ion-java/issues/12 + annotationLen += IonBinary.lenVarUInt(symid); + } + + // now the len of the list + annotationLen += IonBinary.lenVarUInt(annotationLen); + } + return annotationLen; + } + + public static int lenAnnotationListWithLen(ArrayList annotations) + { + int annotationLen = 0; + + // add up the length of the encoded symbols + for (Integer ii : annotations) { + int symid = ii.intValue(); + annotationLen += IonBinary.lenVarUInt(symid); + } + + // now the len of the list + annotationLen += IonBinary.lenVarUInt(annotationLen); + + return annotationLen; + } + + public static int lenIonNullWithTypeDesc() { + return _ib_TOKEN_LEN; + } + public static int lenIonBooleanWithTypeDesc(Boolean v) { + return _ib_TOKEN_LEN; + } + public static int lenIonIntWithTypeDesc(Long v) { + int len = 0; + if (v != null) { + long vl = v.longValue(); + int vlen = lenIonInt(vl); + len += vlen; + len += lenLenFieldWithOptionalNibble(vlen); + } + return len + _ib_TOKEN_LEN; + } + public static int lenIonFloatWithTypeDesc(Double v) { + int len = 0; + if (v != null) { + int vlen = lenIonFloat(v); + len += vlen; + len += lenLenFieldWithOptionalNibble(vlen); + } + return len + _ib_TOKEN_LEN; + } + public static int lenIonDecimalWithTypeDesc(BigDecimal v) { + int len = 0; + if (v != null) { + int vlen = lenIonDecimal(v); + len += vlen; + len += lenLenFieldWithOptionalNibble(vlen); + } + return len + _ib_TOKEN_LEN; + } + public static int lenIonTimestampWithTypeDesc(Timestamp di) { + int len = 0; + if (di != null) { + int vlen = IonBinary.lenIonTimestamp(di); + len += vlen; + len += lenLenFieldWithOptionalNibble(vlen); + } + return len + _ib_TOKEN_LEN; + } + public static int lenIonStringWithTypeDesc(String v) { + int len = 0; + if (v != null) { + int vlen = lenIonString(v); + if (vlen < 0) return -1; + len += vlen; + len += lenLenFieldWithOptionalNibble(vlen); + } + return len + _ib_TOKEN_LEN; + } + public static int lenIonClobWithTypeDesc(String v) { + return lenIonStringWithTypeDesc(v); + } + public static int lenIonBlobWithTypeDesc(byte[] v) { + int len = 0; + if (v != null) { + int vlen = v.length; + len += vlen; + len += lenLenFieldWithOptionalNibble(vlen); + } + return len + _ib_TOKEN_LEN; + } + + + /** Utility method to convert an unsigned magnitude stored as a long to a {@link BigInteger}. */ + public static BigInteger unsignedLongToBigInteger(int signum, long val) + { + byte[] magnitude = { + (byte) ((val >> 56) & 0xFF), + (byte) ((val >> 48) & 0xFF), + (byte) ((val >> 40) & 0xFF), + (byte) ((val >> 32) & 0xFF), + (byte) ((val >> 24) & 0xFF), + (byte) ((val >> 16) & 0xFF), + (byte) ((val >> 8) & 0xFF), + (byte) (val & 0xFF), + }; + return new BigInteger(signum, magnitude); + } + + /** + * Uses {@link InputStream#read(byte[], int, int)} until the entire length is read. + * This method will block until the request is satisfied. + * + * @param in The stream to read from. + * @param buf The buffer to read to. + * @param offset The offset of the buffer to read from. + * @param len The length of the data to read. + */ + public static void readAll(InputStream in, byte[] buf, int offset, int len) throws IOException + { + int rem = len; + while (rem > 0) + { + int amount = in.read(buf, offset, rem); + if (amount <= 0) + { + // try to throw a useful exception + if (in instanceof Reader) + { + ((Reader) in).throwUnexpectedEOFException(); + } + // defer to a plain exception + throw new IonException("Unexpected EOF"); + } + rem -= amount; + offset += amount; + } + } + + public static final class Reader + extends BlockedBuffer.BlockedByteInputStream + { + /** + * @param bb blocked buffer to read from + */ + public Reader(BlockedBuffer bb) + { + super(bb); + } + /** + * @param bb blocked buffer to read from + * @param pos initial offset to read + */ + public Reader(BlockedBuffer bb, int pos) + { + super(bb, pos); + } + + /** + * return the underlying bytes as a single buffer + * + * @return bytes[] + * @throws IOException + * @throws UnexpectedEofException if end of file is hit. + * @throws IOException if there's other problems reading input. + */ + public byte[] getBytes() throws IOException { + if (this._buf == null) return null; + this.sync(); + this.setPosition(0); + int len = _buf.size(); + byte[] buf = new byte[len]; + if (readFully(this, buf) != len) { + throw new UnexpectedEofException(); + } + return buf; + } + + + /** + * Read exactly one byte of input. + * + * @return 0x00 through 0xFF as a positive int. + * @throws UnexpectedEofException if end of file is hit. + * @throws IOException if there's other problems reading input. + */ + public int readToken() throws UnexpectedEofException, IOException + { + int c = read(); + if (c < 0) throwUnexpectedEOFException(); + return c; + } + + public int readActualTypeDesc() throws IOException + { + int c = read(); + if (c < 0) throwUnexpectedEOFException(); + int typeid = _Private_IonConstants.getTypeCode(c); + if (typeid == _Private_IonConstants.tidTypedecl) { + int lownibble = _Private_IonConstants.getLowNibble(c); + if (lownibble == 0) { + // 0xE0 is the first byte of the IonVersionMarker + // so we'll return it as is, the caller has to + // verify the remaining bytes and handle them + // appropriately - so here we do nothing + } + else { + this.readLength(typeid, lownibble); + int alen = this.readVarIntAsInt(); + // TODO add skip(int) method instead of this loop. + while (alen > 0) { + if (this.read() < 0) throwUnexpectedEOFException(); + alen--; + } + c = read(); + if (c < 0) throwUnexpectedEOFException(); + } + } + return c; + } + + public int[] readAnnotations() throws IOException + { + int[] annotations = null; + + int annotationLen = this.readVarUIntAsInt(); + int annotationPos = this.position(); // pos at the first ann sid + int annotationEnd = annotationPos + annotationLen; + int annotationCount = 0; + + // first we read through and count + while(this.position() < annotationEnd) { + // read the annotation symbol id itself + // and for this first pass we just throw that + // value away, since we're just counting + this.readVarUIntAsInt(); + annotationCount++; + } + if (annotationCount > 0) { + // then, if there are any there, we + // allocate the array, and re-read the sids + // look them up and fill in the array + annotations = new int[annotationCount]; + int annotationIdx = 0; + this.setPosition(annotationPos); + while(this.position() < annotationEnd) { + // read the annotation symbol id itself + int sid = this.readVarUIntAsInt(); + annotations[annotationIdx++] = sid; + } + } + + return annotations; + } + + + public int readLength(int td, int ln) throws IOException + { + // TODO check for invalid lownibbles + switch (td) { + case _Private_IonConstants.tidNull: // null(0) + case _Private_IonConstants.tidBoolean: // boolean(1) + return 0; + case _Private_IonConstants.tidPosInt: // 2 + case _Private_IonConstants.tidNegInt: // 3 + case _Private_IonConstants.tidFloat: // float(4) + case _Private_IonConstants.tidDecimal: // decimal(5) + case _Private_IonConstants.tidTimestamp: // timestamp(6) + case _Private_IonConstants.tidSymbol: // symbol(7) + case _Private_IonConstants.tidString: // string (8) + case _Private_IonConstants.tidClob: // clob(9) + case _Private_IonConstants.tidBlob: // blob(10) + case _Private_IonConstants.tidList: // 11 + case _Private_IonConstants.tidSexp: // 12 + case _Private_IonConstants.tidTypedecl: // 14 + switch (ln) { + case 0: + case _Private_IonConstants.lnIsNullAtom: + return 0; + case _Private_IonConstants.lnIsVarLen: + return readVarUIntAsInt(); + default: + return ln; + } + case _Private_IonConstants.tidNopPad: // 99 + switch (ln) { + case _Private_IonConstants.lnIsVarLen: + return readVarUIntAsInt(); + default: + return ln; + } + case _Private_IonConstants.tidStruct: // 13 + switch (ln) { + case _Private_IonConstants.lnIsEmptyContainer: + case _Private_IonConstants.lnIsNullStruct: + return 0; + case _Private_IonConstants.lnIsOrderedStruct: + case _Private_IonConstants.lnIsVarLen: + return readVarUIntAsInt(); + default: + return ln; + } + case _Private_IonConstants.tidUnused: // unused(15) + default: + // TODO use InvalidBinaryDataException + throw new BlockedBuffer.BlockedBufferException("invalid type id encountered: " + td); + } + } + + /* + public long readFixedIntLongValue(int len) throws IOException { + long retvalue = 0; + int b; + + switch (len) { + case 8: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue |= b << (7*8); + case 7: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue |= b << (6*8); + case 6: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue |= b << (5*8); + case 5: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue |= b << (4*8); + case 4: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue |= b << (3*8); + case 3: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue |= b << (2*8); + case 2: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue |= b << (1*8); + case 1: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue |= b << (0*8); + } + return retvalue; + } + public int readFixedIntIntValue(int len) throws IOException { + int retvalue = 0; + int b; + + switch (len) { + case 4: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue |= b << (3*8); + case 3: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue |= b << (2*8); + case 2: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue |= b << (1*8); + case 1: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue |= b << (0*8); + } + return retvalue; + } + */ + public long readIntAsLong(int len) throws IOException { + long retvalue = 0; + boolean is_negative = false; + int b; + + if (len > 0) { + // read the first byte + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (b & 0x7F); + is_negative = ((b & 0x80) != 0); + + switch (len - 1) { // we read 1 already + case 8: // even after reading the 1st byte we may have 8 remaining + // bytes of value when the value is Long.MIN_VALUE since it + // has all 8 bytes for data and the ninth for the sign bit + // all by itself (which we read above) + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 7: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 6: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 5: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 4: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 3: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 2: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 1: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + default: + } + if (is_negative) { + // this is correct even when retvalue == Long.MIN_VALUE + retvalue = -retvalue; + } + } + return retvalue; + } + /** @throws IOException + * @deprecated */ + @Deprecated + public int readIntAsInt(int len) throws IOException { + int retvalue = 0; + boolean is_negative = false; + int b; + + if (len > 0) { + // read the first byte + b = read(); + if (b < 0) throwUnexpectedEOFException(); + retvalue = (b & 0x7F); + is_negative = ((b & 0x80) != 0); + + switch (len - 1) { // we read 1 already + case 7: + case 6: + case 5: + case 4: + throw new IonException("overflow attempt to read long value into an int"); + case 3: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 2: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + } + if (is_negative) { + // this is correct even when retvalue == Integer.MIN_VALUE + retvalue = -retvalue; + } + } + return retvalue; + } + + /** + * Reads the specified magnitude as a {@link BigInteger}. + * + * @param len The length of the UInt octets to read. + * @param signum The sign as per {@link BigInteger#BigInteger(int, byte[])}. + * @return The signed {@link BigInteger}. + * + * @throws IOException Thrown if there are an I/O errors on the underlying stream. + */ + public BigInteger readUIntAsBigInteger(int len, int signum) throws IOException { + byte[] magnitude = new byte[len]; + for (int i = 0; i < len; i++) { + int octet = read(); + if (octet < 0) { + throwUnexpectedEOFException(); + } + magnitude[i] = (byte) octet; + } + + // both BigInteger and ion store this magnitude as big-endian + return new BigInteger(signum, magnitude); + } + + public long readUIntAsLong(int len) throws IOException { + long retvalue = 0; + int b; + + switch (len) { + default: + throw new IonException("overflow attempt to read long value into an int"); + case 8: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 7: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 6: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 5: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 4: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 3: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 2: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 1: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 0: + } + return retvalue; + } + public int readUIntAsInt(int len) throws IOException { + int retvalue = 0; + int b; + + switch (len) { + default: + throw new IonException("overflow attempt to read long value into an int"); + case 4: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = b; + case 3: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 2: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 1: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 0: + } + return retvalue; + } + public long readVarUIntAsLong() throws IOException { + long retvalue = 0; + int b; + + for (;;) { + if ((b = read()) < 0) throwUnexpectedEOFException(); + if ((retvalue & 0xFE00000000000000L) != 0) { + // if any of the top 7 bits are set at this point there's + // a problem, because we'll be shifting the into oblivian + // just below, so ... + throw new IonException("overflow attempt to read long value into a long"); + } + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break; + } + return retvalue; + } + public int readVarUIntAsInt() throws IOException { + int retvalue = 0; + int b; + + for (;;) { // fake loop to create a "goto done" + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break; + + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break; + + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break; + + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break; + + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break; + + // if we get here we have more bits than we have room for :( + throw new IonException("var int overflow at: "+this.position()); + } + return retvalue; + } + public long readVarIntAsLong() throws IOException + { + long retvalue = 0; + boolean is_negative = false; + int b; + + // synthetic label "done" (yuck) +done: for (;;) { + // read the first byte - it has the sign bit + if ((b = read()) < 0) throwUnexpectedEOFException(); + if ((b & 0x40) != 0) { + is_negative = true; + } + retvalue = (b & 0x3F); + if ((b & 0x80) != 0) break done; + + // for the second byte we shift our eariler bits just as much, + // but there are fewer of them there to shift + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break done; + + // for the rest, they're all the same + for (;;) { + if ((b = read()) < 0) throwUnexpectedEOFException(); + if ((retvalue & 0xFE00000000000000L) != 0) { + // if any of the top 7 bits are set at this point there's + // a problem, because we'll be shifting the into oblivian + // just below, so ... + throw new IonException("overflow attempt to read long value into a long"); + } + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break done; + } + } + if (is_negative) { + // this is correct even when retvalue == Long.MIN_VALUE + retvalue = -retvalue; + } + return retvalue; + } + public int readVarIntAsInt() throws IOException + { + int retvalue = 0; + boolean is_negative = false; + int b; + + // synthetic label "done" (yuck) +done: for (;;) { + // read the first byte - it has the sign bit + if ((b = read()) < 0) throwUnexpectedEOFException(); + if ((b & 0x40) != 0) { + is_negative = true; + } + retvalue = (b & 0x3F); + if ((b & 0x80) != 0) break done; + + // for the second byte we shift our eariler bits just as much, + // but there are fewer of them there to shift + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break done; + + // for the rest, they're all the same + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break done; + + // for the rest, they're all the same + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break done; + + // for the rest, they're all the same + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break done; + + // if we get here we have more bits than we have room for :( + throw new IonException("var int overflow at: "+this.position()); + } + if (is_negative) { + // this is correct even when retvalue == Integer.MIN_VALUE + retvalue = -retvalue; + } + return retvalue; + } + + /** + * Reads an integer value, returning null to mean -0. + * @throws IOException + */ + public Integer readVarIntWithNegativeZero() throws IOException + { + int retvalue = 0; + boolean is_negative = false; + int b; + + // sythetic label "done" (yuck) +done: for (;;) { + // read the first byte - it has the sign bit + if ((b = read()) < 0) throwUnexpectedEOFException(); + if ((b & 0x40) != 0) { + is_negative = true; + } + retvalue = (b & 0x3F); + if ((b & 0x80) != 0) break done; + + // for the second byte we shift our eariler bits just as much, + // but there are fewer of them there to shift + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break done; + + // for the rest, they're all the same + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break done; + + // for the rest, they're all the same + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break done; + + // for the rest, they're all the same + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break done; + + // if we get here we have more bits than we have room for :( + throw new IonException("var int overflow at: "+this.position()); + } + + Integer retInteger = null; + if (is_negative) { + if (retvalue != 0) { + // this is correct even when retvalue == Long.MIN_VALUE + retvalue = -retvalue; + retInteger = new Integer(retvalue); + } + } + else { + retInteger = new Integer(retvalue); + } + return retInteger; + } + + public double readFloatValue(int len) throws IOException + { + if (len == 0) + { + // special case, return pos zero + return 0.0d; + } + + if (len != 8) + { + throw new IonException("Length of float read must be 0 or 8"); + } + + long dBits = readUIntAsLong(len); + return Double.longBitsToDouble(dBits); + } + + + /** + * Near clone of {@link SimpleByteBuffer.SimpleByteReader#readDecimal(int)} + * and {@link IonReaderBinaryRawX#readDecimal(int)} + * so keep them in sync! + */ + public Decimal readDecimalValue(int len) throws IOException + { + MathContext mathContext = MathContext.UNLIMITED; + + Decimal bd; + + // we only write out the '0' value as the nibble 0 + if (len == 0) { + bd = Decimal.valueOf(0, mathContext); + } + else { + // otherwise we to it the hard way .... + int startpos = this.position(); + int exponent = this.readVarIntAsInt(); + int bitlen = len - (this.position() - startpos); + + BigInteger value; + int signum; + if (bitlen > 0) + { + byte[] bits = new byte[bitlen]; + readAll(this, bits, 0, bitlen); + + signum = 1; + if (bits[0] < 0) + { + // value is negative, clear the sign + bits[0] &= 0x7F; + signum = -1; + } + value = new BigInteger(signum, bits); + } + else { + signum = 0; + value = BigInteger.ZERO; + } + + // Ion stores exponent, BigDecimal uses the negation "scale" + int scale = -exponent; + if (value.signum() == 0 && signum == -1) + { + assert value.equals(BigInteger.ZERO); + bd = Decimal.negativeZero(scale, mathContext); + } + else + { + bd = Decimal.valueOf(value, scale, mathContext); + } + } + + return bd; + } + + /** + * @see IonReaderBinaryRawX#readTimestamp + */ + public Timestamp readTimestampValue(int len) throws IOException + { + if (len < 1) { + // nothing to do here - and the timestamp will be NULL + return null; + } + + int year = 0, month = 0, day = 0, hour = 0, minute = 0, second = 0; + Decimal frac = null; + int end = this.position() + len; + + // first up is the offset, which requires a special int reader + // to return the -0 as a null Integer + Integer offset = this.readVarIntWithNegativeZero(); + + // now we'll read the struct values from the input stream + + // year is from 0001 to 9999 + // or 0x1 to 0x270F or 14 bits - 1 or 2 bytes + year = readVarUIntAsInt(); + Precision p = Precision.YEAR; // our lowest significant option + + if (position() < end) { + month = readVarUIntAsInt(); + p = Precision.MONTH; // our lowest significant option + + if (position() < end) { + day = readVarUIntAsInt(); + p = Precision.DAY; // our lowest significant option + + // now we look for hours and minutes + if (position() < end) { + hour = readVarUIntAsInt(); + minute = readVarUIntAsInt(); + p = Precision.MINUTE; + + if (position() < end) { + second = readVarUIntAsInt(); + p = Precision.SECOND; + + int remaining = end - position(); + if (remaining > 0) { + // now we read in our actual "milliseconds since the epoch" + frac = this.readDecimalValue(remaining); + } + } + } + } + } + + // now we let timestamp put it all together + try { + Timestamp val = + Timestamp.createFromUtcFields(p, year, month, day, + hour, minute, second, + frac, offset); + return val; + } + catch (IllegalArgumentException e) { + throw new IonException(e.getMessage() + " at: " + position()); + } + } + + public String readString(int len) throws IOException + { + char[] cb = new char[len]; // since we know the length in bytes (which must be + // greater than or equal to chars) there's no need + // for a stringbuffer (even surrgated char's are ok) + int ii=0; + int c; + int endPosition = this.position() + len; + + while (this.position() < endPosition) { + c = readUnicodeScalar(); + if (c < 0) throwUnexpectedEOFException(); + //sb.append((char)c); + if (c < 0x10000) { + cb[ii++] = (char)c; + } + else { + cb[ii++] = (char)_Private_IonConstants.makeHighSurrogate(c); + cb[ii++] = (char)_Private_IonConstants.makeLowSurrogate(c); + } + } + + if (this.position() < endPosition) throwUnexpectedEOFException(); + + return new String(cb, 0, ii); // sb.toString(); + } + public int readUnicodeScalar() throws IOException { + int c = -1, b; + + b = read(); + if (b < 0) return -1; + + // ascii is all good + if ((b & 0x80) == 0) { + return b; + } + + // now we start gluing the multi-byte value together + if ((b & 0xe0) == 0xc0) { + // for values from 0x80 to 0x7FF (all legal) + c = (b & ~0xe0); + b = read(); + if ((b & 0xc0) != 0x80) throwUTF8Exception(); + c <<= 6; + c |= (b & ~0x80); + } + else if ((b & 0xf0) == 0xe0) { + // for values from 0x800 to 0xFFFFF (NOT all legal) + c = (b & ~0xf0); + b = read(); + if ((b & 0xc0) != 0x80) throwUTF8Exception(); + c <<= 6; + c |= (b & ~0x80); + b = read(); + if ((b & 0xc0) != 0x80) throwUTF8Exception(); + c <<= 6; + c |= (b & ~0x80); + if (c > 0x00D7FF && c < 0x00E000) { + throw new IonException("illegal surrgate value encountered in input utf-8 stream"); + } + } + else if ((b & 0xf8) == 0xf0) { + // for values from 0x010000 to 0x1FFFFF (NOT all legal) + c = (b & ~0xf8); + b = read(); + if ((b & 0xc0) != 0x80) throwUTF8Exception(); + c <<= 6; + c |= (b & ~0x80); + b = read(); + if ((b & 0xc0) != 0x80) throwUTF8Exception(); + c <<= 6; + c |= (b & ~0x80); + b = read(); + if ((b & 0xc0) != 0x80) throwUTF8Exception(); + c <<= 6; + c |= (b & ~0x80); + if (c > 0x10FFFF) { + throw new IonException("illegal surrgate value encountered in input utf-8 stream"); + } + } + else { + throwUTF8Exception(); + } + return c; + } + void throwUTF8Exception() + { + throw new IonException("Invalid UTF-8 character encounter in a string at pos " + this.position()); + } + void throwUnexpectedEOFException() { + throw new BlockedBuffer.BlockedBufferException("unexpected EOF in value at offset " + this.position()); + } + + public String readString() throws IOException { + int td = read(); + if (_Private_IonConstants.getTypeCode(td) != _Private_IonConstants.tidString) { + throw new IonException("readString helper only works for string(7) not "+((td >> 4 & 0xf))); + } + int len = (td & 0xf); + if (len == _Private_IonConstants.lnIsNullAtom) { + return null; + } + else if (len == _Private_IonConstants.lnIsVarLen) { + len = this.readVarUIntAsInt(); + } + return readString(len); + } + + /** + * Skips through a NOP pad if there is one. Must be called at the beginning of type description + * + * It's no-op if there is no Nop pad to skip through at the current position + * + * @return true if it skipped through a nop pad + */ + boolean skipThroughNopPad() throws IOException + { + int originalPosition = this._pos; + + // check to see if there is a type declaration as it's not valid to annotate nop pads + int c = this.read(); + boolean hasTypedecl = _Private_IonConstants.getTypeCode(c) == _Private_IonConstants.tidTypedecl; + this.setPosition(originalPosition); + + int typeDesc = this.readActualTypeDesc(); + int tid = _Private_IonConstants.getTypeCode(typeDesc); + int len = _Private_IonConstants.getLowNibble(typeDesc); + + if(tid == _Private_IonConstants.tidNull && len != _Private_IonConstants.lnIsNull){ + if(hasTypedecl) { + throw new IonException("NOP padding is not allowed within annotation wrappers."); + } + + int toSkip = this.readLength(_Private_IonConstants.tidNopPad, len); + long skipped = this.skip(toSkip); + if(toSkip > 0 && toSkip != skipped) { + throw new IonException("Nop pad too short declared length: " + toSkip + " pad actual size: " + skipped); + } + + return true; + } + + // resets the reader if it wasn't a NOP pad + this.setPosition(originalPosition); + + return false; + } + } + + public static final class Writer + extends BlockedBuffer.BlockedByteOutputStream + { + /** + * creates writable stream (OutputStream) that writes + * to a fresh blocked buffer. The stream is initially + * position at offset 0. + */ + public Writer() { super(); } + /** + * creates writable stream (OutputStream) that writes + * to the supplied byte buffer. The stream is initially + * position at offset 0. + * @param bb blocked buffer to write to + */ + public Writer(BlockedBuffer bb) { super(bb); } + /** + * creates writable stream (OutputStream) that can write + * to the supplied byte buffer. The stream is initially + * position at offset off. + * @param bb blocked buffer to write to + * @param off initial offset to write to + */ + public Writer(BlockedBuffer bb, int off) { super(bb, off); } + + Stack _pos_stack; + Stack _pending_high_surrogate_stack; + int _pending_high_surrogate; + public void pushPosition(Object o) + { + PositionMarker pm = new PositionMarker(this.position(), o); + if (_pos_stack == null) { + _pos_stack = new Stack(); + _pending_high_surrogate_stack = new Stack(); + } + _pos_stack.push(pm); + _pending_high_surrogate_stack.push(_pending_high_surrogate); + _pending_high_surrogate = 0; + } + public PositionMarker popPosition() + { + if (_pending_high_surrogate != 0) { + throw new IonException("unmatched high surrogate encountered in input, illegal utf-16 character sequence"); + } + PositionMarker pm = _pos_stack.pop(); + _pending_high_surrogate = _pending_high_surrogate_stack.pop(); + return pm; + } + + /***************************************************************************** + * + * These routines work together to write very long values from an input + * reader where we don't know how long the value is going to be in advance. + * + * Basically it tries to write a short value (len <= BB_SHORT_LEN_MAX) and if + * that fails it moves the data around in the buffers and moves buffers worth + * of data at a time. + * + */ + static class lhNode + { + int _hn; + int _lownibble; + boolean _length_follows; + + lhNode(int hn + ,int lownibble + ,boolean length_follows + ) { + _hn = hn; + _lownibble = lownibble; + _length_follows = length_follows; + } + } + + public void startLongWrite(int hn) throws IOException + { + if (debugValidation) _validate(); + + pushLongHeader(hn, 0, false); + + this.writeCommonHeader(hn, 0); + + if (debugValidation) _validate(); + } + + public void pushLongHeader(int hn + ,int lownibble + ,boolean length_follows + ) { + lhNode n = new lhNode(hn, lownibble, length_follows); + pushPosition(n); + } + + + /** + * Update the TD/LEN header of a value. + * + * @param hn high nibble value (or type id) + * @param lownibble is the low nibble value. + * If -1, then the low nibble is pulled from the header node pushed + * previously. If the header node had lengthFollows==false then this + * is ignored and the LN is computed from what was actually written. + */ + public void patchLongHeader(int hn, int lownibble) throws IOException + { + int currpos = this.position(); + + if (debugValidation) _validate(); + + // get the header description we pushed on the stack a while ago + // pop also checks to make sure we don't have a dangling high + // surrogate pending + PositionMarker pm = this.popPosition(); + lhNode n = (lhNode)pm.getUserData(); + int totallen = currpos - pm.getPosition(); + + // fix up the low nibble if we need to + if (lownibble == -1) { + lownibble = n._lownibble; + } + + // calculate the length just the value itself + // we don't count the type descriptor in the value len + int writtenValueLen = totallen - _Private_IonConstants.BB_TOKEN_LEN; + + // now we can figure out how long the value is going to be + + // This is the length of the length (it does NOT + // count the typedesc byte however) + int len_o_len = IonBinary.lenVarUInt(writtenValueLen); + + // TODO cleanup this logic. lengthFollows == is struct + if (n._length_follows) { + assert hn == _Private_IonConstants.tidStruct; + + if (lownibble == _Private_IonConstants.lnIsOrderedStruct) + { + // leave len_o_len alone + } + else + { + if (writtenValueLen < _Private_IonConstants.lnIsVarLen) { + lownibble = writtenValueLen; + len_o_len = 0; + } + else { + lownibble = _Private_IonConstants.lnIsVarLen; + } + assert lownibble != _Private_IonConstants.lnIsOrderedStruct; + } + } + else { + if (writtenValueLen < _Private_IonConstants.lnIsVarLen) { + lownibble = writtenValueLen; + len_o_len = 0; + } + else { + lownibble = _Private_IonConstants.lnIsVarLen; + } + } + + // first we go back to the beginning + this.setPosition(pm.getPosition()); + + // figure out if we need to move the trailing data to make + // room for the variable length length + int needed = len_o_len; + if (needed > 0) { + // insert does the heavy lifting of making room for us + // at the current position for "needed" additional bytes + insert(needed); + } + + // so, we have room (or already had enough) now we can write + // replacement type descriptor and the length and the reset the pos + this.writeByte(_Private_IonConstants.makeTypeDescriptor(hn, lownibble)); + if (len_o_len > 0) { + this.writeVarUIntValue(writtenValueLen, true); + } + + if (needed < 0) { + // in the unlikely event we wrote more than we needed, now + // is the time to remove it. (which does happen from time to time) + this.remove(-needed); + } + + // return the cursor to it's correct location, + // taking into account the added length data + this.setPosition(currpos + needed); + + if (debugValidation) _validate(); + + } + + // TODO - this may have problems with unicode utf16/utf8 conversions + // note that this reads characters that have already had the escape + // sequence processed (so we don't want to do that a second time) + // the chars here were filled by the IonTokenReader which already + // de-escaped the escaped sequences from the input, so all chars are "real" + public void appendToLongValue(CharSequence chars, boolean onlyByteSizedCharacters) throws IOException + { + if (debugValidation) _validate(); + + int len = chars.length(); // TODO is this ever >0 for clob? + + for (int ii = 0; ii < len; ii++) + { + int c = chars.charAt(ii); + if (onlyByteSizedCharacters) { + if (c > 255) { + throw new IonException("escaped character value too large in clob (0 to 255 only)"); + } + write((byte)(0xff & c)); + } + else { + if (_pending_high_surrogate != 0) { + if ((c & _Private_IonConstants.surrogate_mask) != _Private_IonConstants.low_surrogate_value) { + throw new IonException("unmatched high surrogate character encountered, invalid utf-16"); + } + c = _Private_IonConstants.makeUnicodeScalar(_pending_high_surrogate, c); + _pending_high_surrogate = 0; + } + else if ((c & _Private_IonConstants.surrogate_mask) == _Private_IonConstants.high_surrogate_value) { + ii++; + if (ii >= len) { + // a trailing high surrogate, we just remember it for later + // and (hopefully, and usually, the low surrogate will be + // appended shortly + _pending_high_surrogate = c; + break; + } + int c2 = chars.charAt(ii); + if ((c2 & _Private_IonConstants.surrogate_mask) != _Private_IonConstants.low_surrogate_value) { + throw new IonException("unmatched high surrogate character encountered, invalid utf-16"); + } + c = _Private_IonConstants.makeUnicodeScalar(c, c2); + } + else if ((c & _Private_IonConstants.surrogate_mask) == _Private_IonConstants.low_surrogate_value) { + throw new IonException("unmatched low surrogate character encountered, invalid utf-16"); + } + writeUnicodeScalarAsUTF8(c); + } + } + + if (debugValidation) _validate(); + } + + // helper for appendToLongValue below - this never cares about surrogates + // as it only consumes terminators which are ascii + final boolean isLongTerminator(int terminator, PushbackReader r) throws IOException { + int c; + + // look for terminator 2 - if it's not there put back what we saw + c = r.read(); + if (c != terminator) { + r.unread(c); + return false; + } + + // look for terminator 3 - otherwise put back what we saw and the + // preceeding terminator we found above + c = r.read(); + if (c != terminator) { + r.unread(c); + r.unread(terminator); + return false; + } + // we found 3 in a row - now that's a long terminator + return true; + } + + /** + * Reads the remainder of a quoted string/symbol into this buffer. + * The closing quotes are consumed from the reader. + *

    + * XXX WARNING XXX + * Almost identical logic is found in + * {@link IonTokenReader#finishScanString(boolean)} + * + * @param terminator the closing quote character. + * @param longstring + * @param r + * @throws IOException + */ + public void appendToLongValue(int terminator + ,boolean longstring + ,boolean onlyByteSizedCharacters + ,boolean decodeEscapeSequences + ,PushbackReader r + ) + throws IOException, UnexpectedEofException + { + int c; + + if (debugValidation) { + if (terminator == -1 && longstring) { + throw new IllegalStateException("longstrings have to have a terminator, no eof termination"); + } + _validate(); + } + + assert(terminator != '\\'); + for (;;) { + c = r.read(); // we put off the surrogate logic as long as possible (so not here) + + if (c == terminator) { + if (!longstring || isLongTerminator(terminator, r)) { + // if it's not a long string one quote is enough otherwise look ahead + break; + } + } + else if (c == -1) { + throw new UnexpectedEofException(); + } + else if (c == '\n' || c == '\r') { + // here we'll handle embedded new line detection and escaped characters + if ((terminator != -1) && !longstring) { + throw new IonException("unexpected line terminator encountered in quoted string"); + } + } + else if (decodeEscapeSequences && c == '\\') { + // if this is an escape sequence we need to process it now + // since we allow a surrogate to be encoded using \ u (or \ U) + // encoding + c = IonTokenReader.readEscapedCharacter(r, onlyByteSizedCharacters); + if (c == IonTokenReader.EMPTY_ESCAPE_SEQUENCE) { + continue; + } + } + + if (onlyByteSizedCharacters) { + assert(_pending_high_surrogate == 0); // if it's byte sized only, then we shouldn't have a dangling surrogate + if ((c & (~0xff)) != 0) { + throw new IonException("escaped character value too large in clob (0 to 255 only)"); + } + write((byte)(0xff & c)); + } + else { + // for larger characters we have to glue together surrogates, regardless + // of how they were encoded. If we have a high surrogate and go to peek + // for the low surrogate and hit the end of a segment of a long string + // (triple quoted multi-line string) we leave a dangling high surrogate + // that will get picked up on the next call into this routine when the + // next segment of the long string is processed + if (_pending_high_surrogate != 0) { + if ((c & _Private_IonConstants.surrogate_mask) != _Private_IonConstants.low_surrogate_value) { + String message = + "Text contains unmatched UTF-16 high surrogate " + + IonTextUtils.printCodePointAsString(_pending_high_surrogate); + throw new IonException(message); + } + c = _Private_IonConstants.makeUnicodeScalar(_pending_high_surrogate, c); + _pending_high_surrogate = 0; + } + else if ((c & _Private_IonConstants.surrogate_mask) == _Private_IonConstants.high_surrogate_value) { + int c2 = r.read(); + if (c2 == terminator) { + if (longstring && isLongTerminator(terminator, r)) { + // if it's a long string termination we'll hang onto the current c as the pending surrogate + _pending_high_surrogate = c; + c = terminator; + break; + } + // otherwise this is an error + String message = + "Text contains unmatched UTF-16 high surrogate " + + IonTextUtils.printCodePointAsString(c); + throw new IonException(message); + } + else if (c2 == -1) { + // eof is also an error - really two errors + throw new UnexpectedEofException(); + } + //here we convert escape sequences into characters and continue until + //we encounter a non-newline escape (typically immediately) + while (decodeEscapeSequences && c2 == '\\') { + c2 = IonTokenReader.readEscapedCharacter(r, onlyByteSizedCharacters); + if (c2 != IonTokenReader.EMPTY_ESCAPE_SEQUENCE) break; + c2 = r.read(); + if (c2 == terminator) { + if (longstring && isLongTerminator(terminator, r)) { + // if it's a long string termination we'll hang onto the current c as the pending surrogate + _pending_high_surrogate = c; + c = c2; // we'll be checking this below + break; + } + // otherwise this is an error + String message = + "Text contains unmatched UTF-16 high surrogate " + + IonTextUtils.printCodePointAsString(c); + throw new IonException(message); + } + else if (c2 == -1) { + // eof is also an error - really two errors + throw new UnexpectedEofException(); + } + } + // check to see how we broke our of the while loop above, we may be "done" + if (_pending_high_surrogate != 0) { + break; + } + + if ((c2 & _Private_IonConstants.surrogate_mask) != _Private_IonConstants.low_surrogate_value) { + String message = + "Text contains unmatched UTF-16 high surrogate " + + IonTextUtils.printCodePointAsString(c); + throw new IonException(message); + } + c = _Private_IonConstants.makeUnicodeScalar(c, c2); + } + else if ((c & _Private_IonConstants.surrogate_mask) == _Private_IonConstants.low_surrogate_value) { + String message = + "Text contains unmatched UTF-16 low surrogate " + + IonTextUtils.printCodePointAsString(c); + throw new IonException(message); + } + writeUnicodeScalarAsUTF8(c); + } + } + + if (c != terminator) { + // TODO determine if this can really happen. + throw new UnexpectedEofException(); + } + if (debugValidation) _validate(); + + return; + } + + + /* + * this is a set of routines that put data into an Ion buffer + * using the various type encodeing techniques + * + * these are the "high" level write routines (and read as well, + * in the fullness of time) + * + * + * methods with names of the form: + * void writeValue( value ... ) + * only write the value portion of the data + * + * methods with name like: + * int(len) writeWithLength( ... value ... ) + * write the trailing length and the value they return the + * length written in case it is less than BB_SHORT_LEN_MAX (13) + * so that the caller can backpatch the type descriptor if they + * need to + * + * methods with names of the form: + * void write( nibble, ..., value ) + * write the type descriptor the length if needed and the value + * + * methods with a name like: + * int ValueLength( ... ) + * return the number of bytes that will be needed to write + * the value out + * + */ + + + /** + * Buffer for decoding (un)signed VarInt's and Int's + */ + private byte[] numberBuffer = new byte[10]; + + public int writeVarUIntValue(long value, boolean force_zero_write) throws IOException + { + assert value >= 0; + int len = 0; + if (value == 0) { + if (force_zero_write) { + write((byte)0x80); + len = 1; + } + } else { + int i = numberBuffer.length; + // write every 7 bits of the value + while (value > 0) { + numberBuffer[--i] = (byte)(value & 0x7f); + value = value >>> 7; + } + // set the end bit + numberBuffer[numberBuffer.length - 1] |= 0x80; + len = numberBuffer.length - i; + write(numberBuffer, i, len); + } + return len; + } + + /** + * Writes a uint field of maximum length 8. + * Note that this will write from the lowest to highest + * order bits in the long value given. + */ + public int writeUIntValue(long value) throws IOException + { + int i = numberBuffer.length; + // even if value is Long.MIN_VALUE we will still serialize it correctly :) + while (value != 0) { + numberBuffer[--i] = (byte)(value & 0xff); + value = value >>> 8; + } + int len = numberBuffer.length - i; + write(numberBuffer, i, len); + return len; + } + + /** + * Writes a uint field at given length + * Note that this will write from the lowest to highest + * order bits in the long value given. + */ + public int writeUIntValue(long value, int len) throws IOException + { + int i = numberBuffer.length; + for (int j = 0; j < len; ++j) { + numberBuffer[--i] = (byte)(value & 0xff); + value = value >>> 8; + } + write(numberBuffer, i, len); + return len; + } + + /** + * Writes a uint field of arbitary length. It does + * check the value to see if a simpler routine can + * handle the work; + * Note that this will write from the lowest to highest + * order bits in the long value given. + */ + public int writeUIntValue(BigInteger value, int len) throws IOException + { + int returnlen = 0; + int signum = value.signum(); + + if (signum == 0) { + // Zero has no bytes of data at all! Nothing to write. + } + else if (signum < 0) { + throw new IllegalArgumentException("value must be greater than or equal to 0"); + } + else if (value.compareTo(MAX_LONG_VALUE) == -1) { + long lvalue = value.longValue(); + returnlen = writeUIntValue(lvalue, len); + } + else { + assert(signum > 0); + byte[] bits = value.toByteArray(); + // BigInteger will pad this with a null byte sometimes + // for negative numbers...let's skip past any leading null bytes + int offset = 0; + while (offset < bits.length && bits[offset] == 0) { + offset++; + } + int bitlen = bits.length - offset; + this.write(bits, offset, bitlen); + returnlen += bitlen; + } + assert(returnlen == len); + return len; + } + + + public int writeVarIntValue(long value, boolean force_zero_write) throws IOException + { + int len = 0; + if (value == 0) { + if (force_zero_write) { + write((byte)0x80); + len = 1; + } + } else { + int i = numberBuffer.length; + boolean negative = value < 0; + if (negative) { + value = -value; + } + // write every 7 bits of the value + while (value > 0) { + numberBuffer[--i] = (byte)(value & 0x7f); + value = value >>> 7; + } + // set the end bit + numberBuffer[numberBuffer.length - 1] |= 0x80; + // increase the length of VarInt if the sign bit is 'occupied' + // by the value to properly flag it + if ((numberBuffer[i] & 0x40) == 0x40) { + numberBuffer[--i] = 0x00; + } + // set the sign bit + if (negative) { + // add the sign bit to MSB + numberBuffer[i] |= 0x40; + } + len = numberBuffer.length - i; + write(numberBuffer, i, len); + } + return len; + } + + public int writeIntValue(long value) throws IOException { + int i = numberBuffer.length; + boolean negative = value < 0; + if (negative) { + value = -value; + } + while (value > 0) { + numberBuffer[--i] = (byte)(value & 0xff); + value = value >>> 8; + } + // increase the length of Int if the sign bit is 'occupied' + // by the value to properly flag it + if ((numberBuffer[i] & 0x80) == 0x40) { + numberBuffer[--i] = 0x00; + } + // set the sign bit + if (negative) { + // add the sign bit to MSB + numberBuffer[i] |= 0x80; + } + int len = numberBuffer.length - i; + write(numberBuffer, i, len); + return len; + } + public int writeFloatValue(double d) throws IOException + { + if (Double.valueOf(d).equals(DOUBLE_POS_ZERO)) + { + // pos zero special case + return 0; + } + + // TODO write "custom" serialization or verify that + // the java routine is doing the right thing + long dBits = Double.doubleToRawLongBits(d); + return writeUIntValue(dBits, _ib_FLOAT64_LEN); + } + + byte[] singleCodepointUtf8Buffer = new byte[4]; + public int writeUnicodeScalarAsUTF8(int c) throws IOException + { + int len; + if (c < 0x80) { + len = 1; + this.start_write(); + _write((byte)(c & 0xff)); + this.end_write(); + } else { + len = _writeUnicodeScalarToByteBuffer(c, singleCodepointUtf8Buffer, 0); + this.write(singleCodepointUtf8Buffer, 0, len); + } + return len; + } + + // this will write at least 2 byte unicode scalar to buffer + private final int _writeUnicodeScalarToByteBuffer(int c, byte[] buffer, int offset) throws IOException + { + int len = -1; + + assert offset + 4 <= buffer.length; + + // first the quick, easy and common case - ascii + if (c < 0x800) { + // 2 bytes characters from 0x000080 to 0x0007FF + buffer[offset] = (byte)( 0xff & (0xC0 | (c >> 6)) ); + buffer[++offset] = (byte)( 0xff & (0x80 | (c & 0x3F)) ); + len = 2; + } + else if (c < 0x10000) { + // 3 byte characters from 0x800 to 0xFFFF + // but only 0x800...0xD7FF and 0xE000...0xFFFF are valid + if (c > 0xD7FF && c < 0xE000) { + this.throwUTF8Exception(); + } + buffer[offset] = (byte)( 0xff & (0xE0 | (c >> 12)) ); + buffer[++offset] = (byte)( 0xff & (0x80 | ((c >> 6) & 0x3F)) ); + buffer[++offset] = (byte)( 0xff & (0x80 | (c & 0x3F)) ); + len = 3; + } + else if (c <= 0x10FFFF) { + // 4 byte characters 0x010000 to 0x10FFFF + // these are are valid + buffer[offset] = (byte)( 0xff & (0xF0 | (c >> 18)) ); + buffer[++offset] = (byte)( 0xff & (0x80 | ((c >> 12) & 0x3F)) ); + buffer[++offset] = (byte)( 0xff & (0x80 | ((c >> 6) & 0x3F)) ); + buffer[++offset] = (byte)( 0xff & (0x80 | (c & 0x3F)) ); + len = 4; + } + else { + this.throwUTF8Exception(); + } + return len; + } + + /****************************** + * + * These are the "write typed value with header" routines + * they all are of the form: + * + * void write(nibble, ...) + */ + public int writeByte(HighNibble hn, int len) throws IOException + { + if (len < 0) { + throw new IonException("negative token length encountered"); + } + if (len > 13) len = 14; // TODO remove magic numbers + int t = _Private_IonConstants.makeTypeDescriptor( hn.value(), len ); + write(t); + return 1; + } + + /** + * Write one byte. + * + * @param b the byte to write. + * @return the number of bytes written (always 1). + * @throws IOException + */ + public int writeByte(byte b) throws IOException + { + write(b); + return 1; + } + + /** + * Write one byte. + * + * @param b integer containing the byte to write; only the 8 low-order + * bits are written. + * + * @return the number of bytes written (always 1). + * @throws IOException + */ + public int writeByte(int b) throws IOException + { + write(b); + return 1; + } + + public int writeAnnotations(SymbolToken[] annotations, + SymbolTable symbolTable) throws IOException + { + int startPosition = this.position(); + + int annotationLen = 0; + for (int ii=0; ii annotations) + throws IOException + { + int startPosition = this.position(); + + int annotationLen = 0; + for (Integer ii : annotations) { + annotationLen += IonBinary.lenVarUInt(ii.intValue()); + } + + // write the len of the list + this.writeVarUIntValue(annotationLen, true); + + // write the symbol id's + for (Integer ii : annotations) { + this.writeVarUIntValue(ii.intValue(), true); + } + + return this.position() - startPosition; + } + + + public void writeStubStructHeader(int hn, int ln) + throws IOException + { + // write the hn and ln as the typedesc, we'll patch it later. + writeByte(_Private_IonConstants.makeTypeDescriptor(hn, ln)); + } + + /** + * Write typedesc and length for the common case. + */ + public int writeCommonHeader(int hn, int len) + throws IOException + { + int returnlen = 0; + + // write then len in low nibble + if (len < _Private_IonConstants.lnIsVarLen) { + returnlen += writeByte(_Private_IonConstants.makeTypeDescriptor(hn, len)); + } + else { + returnlen += writeByte(_Private_IonConstants.makeTypeDescriptor(hn, _Private_IonConstants.lnIsVarLen)); + returnlen += writeVarUIntValue(len, false); + } + return returnlen; + } + + + /*************************************** + * + * Here are the write type value with type descriptor and everything + * family of methods, these depend on the others to do much of the work + * + */ + + /** + * @param sid must be valid id (>=1) + */ + public int writeSymbolWithTD(int sid) // was: (String s, SymbolTable symtab) + throws IOException + { + // was: int sid = symtab.addSymbol(s); + assert sid > 0; + + int vlen = lenUInt(sid); + int len = this.writeCommonHeader( + _Private_IonConstants.tidSymbol + ,vlen + ); + len += this.writeUIntValue(sid, vlen); + + return len; + } + + + /** + * Writes the full string header + data. + */ + public int writeStringWithTD(String s) throws IOException + { + // first we have to see how long this will be in the output + /// buffer - part of the cost of length prefixed values + int len = IonBinary.lenIonString(s); + if (len < 0) this.throwUTF8Exception(); + + // first we write the type desc and length + len += this.writeCommonHeader(_Private_IonConstants.tidString, len); + + // now we write just the value out + writeStringData(s); + //for (int ii=0; ii stringBufferLen - 4) { // 4 is the max UTF-8 encoding size + this.write(stringBuffer, 0, bufPos); + bufPos = 0; + } + // at this point stringBuffer contains enough space for UTF-8 encoded code point + if (c < 128) { + // don't even both to call the "utf8" converter for ascii + stringBuffer[bufPos++] = (byte)c; + len++; + continue; + } + // multi-byte utf8 + + if (c >= 0xD800 && c <= 0xDFFF) { + if (_Private_IonConstants.isHighSurrogate(c)) { + // houston we have a high surrogate (let's hope it has a partner + if (++ii >= s.length()) { + throw new IllegalArgumentException("invalid string, unpaired high surrogate character"); + } + int c2 = s.charAt(ii); + if (!_Private_IonConstants.isLowSurrogate(c2)) { + throw new IllegalArgumentException("invalid string, unpaired high surrogate character"); + } + c = _Private_IonConstants.makeUnicodeScalar(c, c2); + } + else if (_Private_IonConstants.isLowSurrogate(c)) { + // it's a loner low surrogate - that's an error + throw new IllegalArgumentException("invalid string, unpaired low surrogate character"); + } + // from 0xE000 up the _writeUnicodeScalar will check for us + } + int utf8len = this._writeUnicodeScalarToByteBuffer(c, stringBuffer, bufPos); + bufPos += utf8len; + len += utf8len; + } + if (bufPos > 0) { + this.write(stringBuffer, 0, bufPos); + } + + return len; + } + + public int writeNullWithTD(HighNibble hn) throws IOException + { + writeByte(hn, _Private_IonConstants.lnIsNullAtom); + return 1; + } + + public int writeTimestampWithTD(Timestamp di) + throws IOException + { + int returnlen; + + if (di == null) { + returnlen = this.writeCommonHeader( + _Private_IonConstants.tidTimestamp + ,_Private_IonConstants.lnIsNullAtom); + } + else { + int vlen = IonBinary.lenIonTimestamp(di); + + returnlen = this.writeCommonHeader( + _Private_IonConstants.tidTimestamp + ,vlen); + + int wroteLen = writeTimestamp(di); + assert wroteLen == vlen; + returnlen += wroteLen; + } + return returnlen; + } + + public int writeTimestamp(Timestamp di) + throws IOException + { + if (di == null) return 0; + int returnlen = 0; + Precision precision = di.getPrecision(); + + Integer offset = di.getLocalOffset(); + if (offset == null) { + // TODO don't use magic numbers! + this.write((byte)(0xff & (0x80 | 0x40))); // negative 0 (no timezone) + returnlen ++; + } + else { + returnlen += this.writeVarIntValue(offset.intValue(), true); + } + + // now the date - year, month, day as VarUInts + // if we have a non-null value we have at least the date + if (precision.includes(Precision.YEAR)) { + returnlen += this.writeVarUIntValue(di.getZYear(), true); + } + if (precision.includes(Precision.MONTH)) { + returnlen += this.writeVarUIntValue(di.getZMonth(), true); + } + if (precision.includes(Precision.DAY)) { + returnlen += this.writeVarUIntValue(di.getZDay(), true); + } + + // now the time portion + if (precision.includes(Precision.MINUTE)) { + returnlen += this.writeVarUIntValue(di.getZHour(), true); + returnlen += this.writeVarUIntValue(di.getZMinute(), true); + } + if (precision.includes(Precision.SECOND)) { + returnlen += this.writeVarUIntValue(di.getZSecond(), true); + // and, finally, any fractional component that is known + BigDecimal fraction = di.getZFractionalSecond(); + if (fraction != null) { + assert !fraction.equals(BigDecimal.ZERO); + returnlen += this.writeDecimalContent(fraction); + } + } + return returnlen; + } + + public int writeDecimalWithTD(BigDecimal bd) throws IOException + { + int returnlen; + + // we only write out the '0' value as the nibble 0 + if (bd == null) { + returnlen = + this.writeByte(NULL_DECIMAL_TYPEDESC); + } + else if (isNibbleZero(bd)) { + returnlen = + this.writeByte(ZERO_DECIMAL_TYPEDESC); + } + else { + // otherwise we to it the hard way .... + int len = IonBinary.lenIonDecimal(bd); + + if (len < _Private_IonConstants.lnIsVarLen) { + returnlen = this.writeByte( + _Private_IonConstants.makeTypeDescriptor( + _Private_IonConstants.tidDecimal + , len + ) + ); + } + else { + returnlen = this.writeByte( + _Private_IonConstants.makeTypeDescriptor( + _Private_IonConstants.tidDecimal + , _Private_IonConstants.lnIsVarLen + ) + ); + this.writeVarIntValue(len, false); + } + int wroteDecimalLen = writeDecimalContent(bd); + assert wroteDecimalLen == len; + returnlen += wroteDecimalLen; + } + return returnlen; + } + + private static final byte[] negativeZeroBitArray = new byte[] { (byte)0x80 }; + + /** Zero-length byte array. */ + private static final byte[] positiveZeroBitArray = EMPTY_BYTE_ARRAY; + + + /** + * @see com.amazon.ion.impl.lite.ReverseBinaryEncoder#writeIonDecimalContent + */ + public int writeDecimalContent(BigDecimal bd) + throws IOException + { + // check for null and 0. which are encoded in the nibble itself. + if (bd == null) return 0; + + if (isNibbleZero(bd)) return 0; + + // Ion stores exponent, BigDecimal uses the negation "scale" + int exponent = -bd.scale(); + + // The exponent isn't optional (except for the 0d0 case above). + int returnlen = writeVarIntValue(exponent, + /* force_zero_write*/ true); + + BigInteger mantissa = bd.unscaledValue(); + + byte[] mantissaBits; + switch (mantissa.signum()) { + case 0: + if (Decimal.isNegativeZero(bd)) { + mantissaBits = negativeZeroBitArray; + } + else { + mantissaBits = positiveZeroBitArray; + } + break; + case -1: + // Obtain the unsigned value of the BigInteger + // We cannot use the twos complement representation of a + // negative BigInteger as this is different from the encoding + // of basic field Int. + mantissaBits = mantissa.negate().toByteArray(); + // Set the sign on the highest order bit of the first octet + mantissaBits[0] |= 0x80; + break; + case 1: + mantissaBits = mantissa.toByteArray(); + break; + default: + throw new IllegalStateException("mantissa signum out of range"); + } + + this.write(mantissaBits, 0, mantissaBits.length); + returnlen += mantissaBits.length; + + return returnlen; + } + + + void throwUTF8Exception() + { + throwException("Invalid UTF-8 character encounter in a string at pos " + this.position()); + } + + void throwException(String s) + { + throw new BlockedBuffer.BlockedBufferException(s); + } + } + + public static class PositionMarker + { + int _pos; + Object _userData; + + public PositionMarker() {} + public PositionMarker(int pos, Object o) { + _pos = pos; + _userData = o; + } + + public int getPosition() { return _pos; } + public Object getUserData() { return _userData; } + + public void setPosition(int pos) { + _pos = pos; + } + public void setUserData(Object o) { + _userData = o; + } + } +} diff --git a/src/com/amazon/ion/impl/IonCharacterReader.java b/src/com/amazon/ion/impl/IonCharacterReader.java new file mode 100644 index 0000000000..d4590c723f --- /dev/null +++ b/src/com/amazon/ion/impl/IonCharacterReader.java @@ -0,0 +1,284 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import java.io.IOException; +import java.io.PushbackReader; +import java.io.Reader; +import java.util.LinkedList; + +/** + * Extension of the {@link java.io.PushbackReader} to abstract line/offset counting + * and push back support. + * + * Note that this class does not leverage {@link java.io.LineNumberReader} as we + * need to un-roll the line number on push back and we need to have the logic to + * deal with newline combinations anyhow. + */ +final class IonCharacterReader extends PushbackReader { + + /** + * The default buffer size + * + * @see _Private_Utils#MAX_LOOKAHEAD_UTF16 + */ + public static final int DEFAULT_BUFFER_SIZE = 12; + + /** + * The additional buffer padding--this is to add to the fact that the + * pushback buffer may have to be larger than user specified characters + * a CR/LF combo is 2 characters, but logically one newline. + * + * FIXME does this properly account for surrogates? + * + * @see _Private_Utils#MAX_LOOKAHEAD_UTF16 + */ + public static final int BUFFER_PADDING = 1; + + private long m_consumed; + private int m_line; + private int m_column; + private int m_size; + + // our offset stack--really, for efficiency this should + // be a linked list of primitive ints to avoid boxing costs + // however, pursue this only if it is really a performance + // issue. Also, we are not using the List interface for code + // clarity as this is used as a deque. (Java 6 has java.util.Deque) + + /** + * This offset stack is for pushing back characters that unroll lines + * and keeps our offset/line number counts correct. This is essentially + * a pushback stack for character offsets. + */ + private LinkedList m_columns; + + /** + * Constructs a character reader with an explicit buffer size. Note that this + * is a minimum and the implementation is allowed to make it larger for + * internal operations. + * + * @param in The underlying reader to wrap. + * @param size The size of the push back buffer. + */ + public IonCharacterReader( final Reader in, final int size ) { + super( in, size + BUFFER_PADDING ); + + assert size > 0; + + m_consumed = 0; + m_line = 1; + m_column = 0; + m_columns = new LinkedList(); + m_size = size + BUFFER_PADDING; + } + + /** + * Constructs a character reader with the default buffer size. + * + * @param in The underlying reader to wrap. + */ + public IonCharacterReader( final Reader in ) { + this( in, DEFAULT_BUFFER_SIZE ); + } + + /** + * Returns the logical number of consumed characters. + * This does not necessarily equal to the number of + * actual characters read as newline combinations are + * treated as one. + * + * @return The logical number of consumed characters. + */ + public final long getConsumedAmount() { + return m_consumed; + } + + /** + * Returns the current line number in the stream based on the last call + * to read(). + * + * @return The line number, 1-based. + */ + public final int getLineNumber() { + return m_line; + } + + /** + * Returns the offset within the line based on the last call to read(). + * + * @return The offset, 1-based. + */ + public final int getColumn() { + return m_column; + } + + /** + * Uses the push back implementation but normalizes newlines to "\n". + */ + @Override + public int read() throws IOException { + int nextChar = super.read(); + + // process the character + if ( nextChar != -1 ) { + if ( nextChar == '\n' ) { + m_line++; + pushColumn( m_column ); + m_column = 0; + } + else if ( nextChar == '\r') { + int aheadChar = super.read(); + + // if the lookahead is not a newline combo, unread it + if ( aheadChar != '\n' ) { + // no need to unread it with line/offset update. + unreadImpl( aheadChar, false ); + } + + m_line++; + pushColumn( m_column ); + m_column = 0; + + // normalize + nextChar = '\n'; + } else { + m_column++; + } + m_consumed++; + } + + return nextChar; + } + + private final void pushColumn( final int offset ) { + // constrain the offset stack + // to the buffer size + if ( m_columns.size() == m_size ) { + m_columns.removeFirst(); + } + + // box into collection + m_columns.addLast( offset ); + } + + private final int popColumn() throws IOException { + if ( m_columns.isEmpty() ) { + throw new IOException( "Cannot unread past buffer" ); + } + + // unbox out of collection + return m_columns.removeLast(); + } + + /** + * Readers a buffer's worth of data. This implementation + * simply leverages {@link #read()} over the buffer. + */ + @Override + public int read( char[] cbuf, int off, int len ) throws IOException { + assert len >= 0; + assert off >= 0; + + int amountRead = 0; + final int endIndex = off + len; + for ( int index = off; index < endIndex; index++ ) { + int readChar = read(); + if ( readChar == -1 ) { + break; + } + + cbuf[ index ] = ( char ) readChar; + amountRead++; + } + + return amountRead == 0 ? -1 : amountRead; + } + + /** + * Skips over some number of characters. + * This is implemented as a series of {@link #read()} calls. + */ + @Override + public long skip( final long n ) throws IOException { + assert n > 0; + + long charsLeft = n; + // note the read side effect + while ( charsLeft > 0 && read() != -1 ) { + charsLeft--; + } + return n - charsLeft; + } + + /** + * Delegates to {@link #unread(int)}. + */ + @Override + public void unread( char[] cbuf, int off, int len ) throws IOException { + assert len >= 0; + assert off >= 0; + + final int endIndex = off + len; + for ( int index = endIndex - 1; index >= off; index-- ) { + unread( cbuf[ index ] ); + } + } + + /** + * Delegates to {@link #unread(char[],int,int)}. + */ + @Override + public void unread( char[] cbuf ) throws IOException { + unread( cbuf, 0, cbuf.length ); + } + + /** + * Will unread a character and update the line number if necessary. This will throw an + * exception if a carriage return is given as this character is never yielded from + * this stream. + */ + @Override + public void unread( int c ) throws IOException { + if ( c == '\r' ) { + throw new IOException( "Cannot unread a carriage return" ); + } + + unreadImpl( c, true ); + } + + /** + * Performs ths actual unread operation. + * + * @param c the character to unread. + * @param updateCounts Whether or not we actually update the line number. + */ + private void unreadImpl(int c, boolean updateCounts ) throws IOException { + if ( c != -1 ) { + if ( updateCounts ) { + if ( c == '\n' ) { + m_line--; + m_column = popColumn(); + } else { + m_column--; + } + m_consumed--; + } + super.unread( c ); + } + } + +} diff --git a/src/software/amazon/ion/impl/IonIteratorImpl.java b/src/com/amazon/ion/impl/IonIteratorImpl.java similarity index 86% rename from src/software/amazon/ion/impl/IonIteratorImpl.java rename to src/com/amazon/ion/impl/IonIteratorImpl.java index 184d5a8b05..2083575f07 100644 --- a/src/software/amazon/ion/impl/IonIteratorImpl.java +++ b/src/com/amazon/ion/impl/IonIteratorImpl.java @@ -1,30 +1,31 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; - +package com.amazon.ion.impl; + +import com.amazon.ion.IonLob; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonSequence; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.ValueFactory; import java.util.Iterator; import java.util.NoSuchElementException; -import software.amazon.ion.IonLob; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonSequence; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.ValueFactory; final class IonIteratorImpl @@ -186,10 +187,10 @@ private IonValue readValue() // TODO this is too late in the case of system reading // when v is a local symtab (it will get itself, not the prior symtab) SymbolTable symtab = _reader.getSymbolTable(); - ((PrivateIonValue)v).setSymbolTable(symtab); + ((_Private_IonValue)v).setSymbolTable(symtab); if (annotations.length != 0) { - ((PrivateIonValue)v).setTypeAnnotationSymbols(annotations); + ((_Private_IonValue)v).setTypeAnnotationSymbols(annotations); } return v; diff --git a/src/com/amazon/ion/impl/IonMessages.java b/src/com/amazon/ion/impl/IonMessages.java new file mode 100644 index 0000000000..c34b11c3fb --- /dev/null +++ b/src/com/amazon/ion/impl/IonMessages.java @@ -0,0 +1,24 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + + +final class IonMessages +{ + static final String CANNOT_STEP_OUT = + "Cannot stepOut any further, already at top level."; + +} diff --git a/src/software/amazon/ion/impl/IonReaderBinaryRawX.java b/src/com/amazon/ion/impl/IonReaderBinaryRawX.java similarity index 88% rename from src/software/amazon/ion/impl/IonReaderBinaryRawX.java rename to src/com/amazon/ion/impl/IonReaderBinaryRawX.java index aca36e5de9..1ef4ffde21 100644 --- a/src/software/amazon/ion/impl/IonReaderBinaryRawX.java +++ b/src/com/amazon/ion/impl/IonReaderBinaryRawX.java @@ -1,35 +1,36 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; -import static software.amazon.ion.SystemSymbols.ION_1_0_SID; +import static com.amazon.ion.SystemSymbols.ION_1_0_SID; +import com.amazon.ion.Decimal; +import com.amazon.ion.IonException; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonType; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.Timestamp; +import com.amazon.ion.Timestamp.Precision; +import com.amazon.ion.impl.UnifiedSavePointManagerX.SavePoint; +import com.amazon.ion.impl._Private_ScalarConversions.AS_TYPE; +import com.amazon.ion.impl._Private_ScalarConversions.ValueVariant; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.math.MathContext; -import software.amazon.ion.Decimal; -import software.amazon.ion.IonException; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonType; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.Timestamp; -import software.amazon.ion.Timestamp.Precision; -import software.amazon.ion.impl.PrivateScalarConversions.AS_TYPE; -import software.amazon.ion.impl.PrivateScalarConversions.ValueVariant; -import software.amazon.ion.impl.UnifiedSavePointManagerX.SavePoint; /** @@ -51,7 +52,6 @@ abstract class IonReaderBinaryRawX static final int DEFAULT_CONTAINER_STACK_SIZE = 12; // a multiple of 3 static final int DEFAULT_ANNOTATION_SIZE = 10; static final int NO_LIMIT = Integer.MIN_VALUE; - protected enum State { S_INVALID, S_BEFORE_FIELD, // only true in structs @@ -77,6 +77,7 @@ protected enum State { int _value_field_id; int _value_tid; int _value_len; + long _value_start; int _value_lob_remaining; boolean _value_lob_is_ready; @@ -120,7 +121,7 @@ protected final void init_raw(UnifiedInputStreamX uis) { final void re_init_raw() { _local_remaining = NO_LIMIT; - _parent_tid = PrivateIonConstants.tidDATAGRAM; + _parent_tid = _Private_IonConstants.tidDATAGRAM; _value_field_id = SymbolTable.UNKNOWN_SYMBOL_ID; _state = State.S_BEFORE_TID; // this is where we always start _has_next_needed = true; @@ -131,6 +132,7 @@ final void re_init_raw() { _value_is_true = false; _value_len = 0; + _value_start = 0; _value_lob_remaining = 0; _value_lob_is_ready = false; @@ -179,7 +181,7 @@ private final int get_top_type() { assert(_container_top > 0); long type_limit = _container_stack[(_container_top - POS_STACK_STEP) + TYPE_LIMIT_OFFSET]; int type = (int)(type_limit & TYPE_MASK); - if (type < 0 || type > PrivateIonConstants.tidDATAGRAM) { + if (type < 0 || type > _Private_IonConstants.tidDATAGRAM) { throwErrorAt("invalid type id in parent stack"); } return type; @@ -194,8 +196,7 @@ private final void pop() { assert(_container_top > 0); _container_top -= POS_STACK_STEP; } - - boolean hasNext() + public boolean hasNext() { if (!_eof && _has_next_needed) { try { @@ -225,14 +226,14 @@ public IonType next() assert( _value_type != null || _eof == true); return _value_type; } - // from IonConstants - // public static final byte[] BINARY_VERSION_MARKER_1_0 = + //from IonConstants + //public static final byte[] BINARY_VERSION_MARKER_1_0 = // { (byte) 0xE0, // (byte) 0x01, // (byte) 0x00, // (byte) 0xEA }; - private static final int BINARY_VERSION_MARKER_TID = PrivateIonConstants.getTypeCode(PrivateIonConstants.BINARY_VERSION_MARKER_1_0[0] & 0xff); - private static final int BINARY_VERSION_MARKER_LEN = PrivateIonConstants.getLowNibble(PrivateIonConstants.BINARY_VERSION_MARKER_1_0[0] & 0xff); + private static final int BINARY_VERSION_MARKER_TID = _Private_IonConstants.getTypeCode(_Private_IonConstants.BINARY_VERSION_MARKER_1_0[0] & 0xff); + private static final int BINARY_VERSION_MARKER_LEN = _Private_IonConstants.getLowNibble(_Private_IonConstants.BINARY_VERSION_MARKER_1_0[0] & 0xff); private final void has_next_helper_raw() throws IOException { clear_value(); @@ -256,13 +257,13 @@ private final void has_next_helper_raw() throws IOException _eof = true; break; } - if (_value_tid == PrivateIonConstants.tidNopPad) { + if (_value_tid == _Private_IonConstants.tidNopPad) { // skips size of pad and resets State machine skip(_value_len); clear_value(); break; } - else if (_value_tid == PrivateIonConstants.tidTypedecl) { + else if (_value_tid == _Private_IonConstants.tidTypedecl) { assert (_value_tid == (BINARY_VERSION_MARKER_TID & 0xff)); // the bvm tid happens to be type decl if (_value_len == BINARY_VERSION_MARKER_LEN ) { // this isn't valid for any type descriptor except the first byte @@ -320,16 +321,17 @@ else if (_value_tid == PrivateIonConstants.tidTypedecl) { } private final void load_version_marker() throws IOException { - for (int ii=1; ii= 0) { throwErrorAt( - "The fractional seconds value in a timestamp must be greater than or " - + "equal to zero and less than one." + "The fractional seconds value in a timestamp must be greater than or " + + "equal to zero and less than one." ); } } @@ -1151,7 +1163,6 @@ protected final Timestamp readTimestamp(int len) throws IOException _local_remaining = save_limit; // now we let timestamp put it all together try { - @SuppressWarnings("deprecation") Timestamp val = Timestamp.createFromUtcFields(p, year, month, day, hour, minute, second, frac, offset); @@ -1180,8 +1191,8 @@ protected final String readString(int len) throws IOException chars[ii++] = (char)c; } else { // when c is >= 0x10000 we need surrogate encoding - chars[ii++] = (char)PrivateIonConstants.makeHighSurrogate(c); - chars[ii++] = (char)PrivateIonConstants.makeLowSurrogate(c); + chars[ii++] = (char)_Private_IonConstants.makeHighSurrogate(c); + chars[ii++] = (char)_Private_IonConstants.makeLowSurrogate(c); } } _local_remaining = save_limit; diff --git a/src/software/amazon/ion/impl/IonReaderBinarySystemX.java b/src/com/amazon/ion/impl/IonReaderBinarySystemX.java similarity index 83% rename from src/software/amazon/ion/impl/IonReaderBinarySystemX.java rename to src/com/amazon/ion/impl/IonReaderBinarySystemX.java index f28361e7e8..614ef7f5a1 100644 --- a/src/software/amazon/ion/impl/IonReaderBinarySystemX.java +++ b/src/com/amazon/ion/impl/IonReaderBinarySystemX.java @@ -1,49 +1,57 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; - -import static software.amazon.ion.IonType.SYMBOL; -import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; - +package com.amazon.ion.impl; + +import com.amazon.ion.IonSystem; +import static com.amazon.ion.IonType.SYMBOL; +import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; + +import com.amazon.ion.Decimal; +import com.amazon.ion.IntegerSize; +import com.amazon.ion.IonType; +import com.amazon.ion.NullValueException; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.Timestamp; +import com.amazon.ion.UnknownSymbolException; +import com.amazon.ion.impl._Private_ScalarConversions.AS_TYPE; +import com.amazon.ion.impl._Private_ScalarConversions.ValueVariant; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.util.Date; import java.util.Iterator; -import software.amazon.ion.Decimal; -import software.amazon.ion.IntegerSize; -import software.amazon.ion.IonType; -import software.amazon.ion.NullValueException; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.Timestamp; -import software.amazon.ion.UnknownSymbolException; -import software.amazon.ion.impl.PrivateScalarConversions.AS_TYPE; -import software.amazon.ion.impl.PrivateScalarConversions.ValueVariant; + class IonReaderBinarySystemX extends IonReaderBinaryRawX - implements PrivateReaderWriter + implements _Private_ReaderWriter { SymbolTable _symbols; + @Deprecated + IonReaderBinarySystemX(byte[] bytes, int offset, int length) { + this(UnifiedInputStreamX.makeStream(bytes, offset, length)); + } + IonReaderBinarySystemX(UnifiedInputStreamX in) { super(); init_raw(in); - // TODO check IVM to determine version: amznlabs/ion-java#19, amznlabs/ion-java#24 + // TODO check IVM to determine version: amzn/ion-java#19, amzn/ion-java#24 _symbols = SharedSymbolTable.getSystemSymbolTable(1); } @@ -54,6 +62,11 @@ class IonReaderBinarySystemX // or the user reader. Here they just fail. // + public final int getFieldId() + { + return _value_field_id; + } + public SymbolToken[] getTypeAnnotationSymbols() { load_annotations(); @@ -100,9 +113,9 @@ protected final void prepare_value(int as_type) { if (!_v.can_convert(as_type)) { String message = "can't cast from " - +PrivateScalarConversions.getValueTypeName(_v.getAuthoritativeType()) + +_Private_ScalarConversions.getValueTypeName(_v.getAuthoritativeType()) +" to " - +PrivateScalarConversions.getValueTypeName(as_type); + +_Private_ScalarConversions.getValueTypeName(as_type); throw new IllegalStateException(message); } int fnid = _v.get_conversion_fnid(as_type); @@ -128,22 +141,8 @@ protected final void load_cached_value(int value_type) throws IOException } } - /** Utility method to convert an unsigned magnitude stored as a long to a {@link BigInteger}. */ - private static BigInteger unsignedLongToBigInteger(int signum, long val) - { - byte[] magnitude = { - (byte) ((val >> 56) & 0xFF), - (byte) ((val >> 48) & 0xFF), - (byte) ((val >> 40) & 0xFF), - (byte) ((val >> 32) & 0xFF), - (byte) ((val >> 24) & 0xFF), - (byte) ((val >> 16) & 0xFF), - (byte) ((val >> 8) & 0xFF), - (byte) (val & 0xFF), - }; - return new BigInteger(signum, magnitude); - } - + static final int MAX_BINARY_LENGTH_INT = 4; + static final int MAX_BINARY_LENGTH_LONG = 8; static final BigInteger MIN_LONG_VALUE = BigInteger.valueOf(Long.MIN_VALUE); static final BigInteger MAX_LONG_VALUE = BigInteger.valueOf(Long.MAX_VALUE); @@ -180,24 +179,23 @@ private final void load_scalar_value() throws IOException _v.setAuthoritativeType(AS_TYPE.boolean_value); break; case INT: - boolean is_negative = _value_tid == PrivateIonConstants.tidNegInt; + boolean is_negative = _value_tid == _Private_IonConstants.tidNegInt; if (_value_len == 0) { if (is_negative) { throwIllegalNegativeZeroException(); } - int v = 0; _v.setValue(v); _v.setAuthoritativeType(AS_TYPE.int_value); } - else if (_value_len <= Long.BYTES) { + else if (_value_len <= MAX_BINARY_LENGTH_LONG) { long v = readULong(_value_len); if (v < 0) { // we probably can't fit this magnitude properly into a Java long int signum = !is_negative ? 1 : -1; - BigInteger big = unsignedLongToBigInteger(signum, v); + BigInteger big = IonBinary.unsignedLongToBigInteger(signum, v); _v.setValue(big); // boundary condition if (big.compareTo(MIN_LONG_VALUE) < 0 || big.compareTo(MAX_LONG_VALUE) > 0) { @@ -376,7 +374,6 @@ public Timestamp timestampValue() return _v.getTimestamp(); } - @Override public IntegerSize getIntegerSize() { load_once(); @@ -384,12 +381,10 @@ public IntegerSize getIntegerSize() { return null; } - return PrivateScalarConversions.getIntegerSize(_v.getAuthoritativeType()); + return _Private_ScalarConversions.getIntegerSize(_v.getAuthoritativeType()); } - // TODO amzn/ion-java#63 this needs to use the appropriate symbol table for user values - - public final String stringValue() + public String stringValue() { if (! IonType.isText(_value_type)) throw new IllegalStateException(); if (_value_is_null) return null; @@ -431,7 +426,11 @@ int getSymbolId() return _v.getInt(); } - public final String getFieldName() + // + // unsupported public methods that require a symbol table + // to operate - which is only supported on a user reader + // + public String getFieldName() { String name; if (_value_field_id == SymbolTable.UNKNOWN_SYMBOL_ID) { @@ -457,7 +456,7 @@ public SymbolToken getFieldNameSymbol() public final Iterator iterateTypeAnnotations() { String[] annotations = getTypeAnnotations(); - return PrivateUtils.stringIterator(annotations); + return _Private_Utils.stringIterator(annotations); } public final String[] getTypeAnnotations() @@ -465,7 +464,7 @@ public final String[] getTypeAnnotations() load_annotations(); String[] anns; if (_annotation_count < 1) { - anns = PrivateUtils.EMPTY_STRING_ARRAY; + anns = _Private_Utils.EMPTY_STRING_ARRAY; } else { anns = new String[_annotation_count]; @@ -479,7 +478,7 @@ public final String[] getTypeAnnotations() return anns; } - public final SymbolTable getSymbolTable() + public SymbolTable getSymbolTable() { return _symbols; } diff --git a/src/software/amazon/ion/impl/IonReaderBinaryUserX.java b/src/com/amazon/ion/impl/IonReaderBinaryUserX.java similarity index 69% rename from src/software/amazon/ion/impl/IonReaderBinaryUserX.java rename to src/com/amazon/ion/impl/IonReaderBinaryUserX.java index 16c04d6192..31d3d775be 100644 --- a/src/software/amazon/ion/impl/IonReaderBinaryUserX.java +++ b/src/com/amazon/ion/impl/IonReaderBinaryUserX.java @@ -1,39 +1,44 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; - -import static software.amazon.ion.SystemSymbols.ION_1_0_SID; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; - +package com.amazon.ion.impl; + +import com.amazon.ion.IonSystem; +import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; +import static com.amazon.ion.SystemSymbols.ION_1_0_SID; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; + +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonType; +import com.amazon.ion.OffsetSpan; +import com.amazon.ion.RawValueSpanProvider; +import com.amazon.ion.SeekableReader; +import com.amazon.ion.Span; +import com.amazon.ion.SpanProvider; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.UnknownSymbolException; +import com.amazon.ion.impl.UnifiedInputStreamX.FromByteArray; +import com.amazon.ion.impl.UnifiedSavePointManagerX.SavePoint; +import com.amazon.ion.impl._Private_ScalarConversions.AS_TYPE; import java.io.IOException; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonType; -import software.amazon.ion.OffsetSpan; -import software.amazon.ion.SeekableReader; -import software.amazon.ion.Span; -import software.amazon.ion.SpanProvider; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.UnknownSymbolException; -import software.amazon.ion.impl.PrivateScalarConversions.AS_TYPE; -import software.amazon.ion.impl.UnifiedInputStreamX.FromByteArray; -import software.amazon.ion.impl.UnifiedSavePointManagerX.SavePoint; +import java.util.Iterator; final class IonReaderBinaryUserX extends IonReaderBinarySystemX - implements PrivateReaderWriter + implements _Private_ReaderWriter { /** * This is the physical start-of-stream offset when this reader was created. @@ -41,14 +46,22 @@ final class IonReaderBinaryUserX * {@link OffsetSpan}s. */ private final int _physical_start_offset; - private final PrivateLocalSymbolTableFactory _lstFactory; + private final _Private_LocalSymbolTableFactory _lstFactory; IonCatalog _catalog; - private static final class IonReaderBinarySpan + private static class IonReaderBinarySpan extends DowncastingFaceted implements Span, OffsetSpan { + private final boolean _isSeekable; + + public IonReaderBinarySpan(boolean isSeekable) + { + _isSeekable = isSeekable; + } + + State _state; long _offset; long _limit; SymbolTable _symbol_table; @@ -62,10 +75,16 @@ public long getFinishOffset() { return _limit; } + + public boolean isSeekable() + { + return _isSeekable; + } + } public IonReaderBinaryUserX(IonCatalog catalog, - PrivateLocalSymbolTableFactory lstFactory, + _Private_LocalSymbolTableFactory lstFactory, UnifiedInputStreamX userBytes, int physicalStartOffset) { @@ -83,7 +102,6 @@ final void init_user(IonCatalog catalog) _catalog = catalog; } - /** * Determines the abstract position of the reader, such that one can * later {@link #seek} back to it. @@ -92,38 +110,39 @@ final void init_user(IonCatalog catalog) * a value (not before, between, or after values). In other words, one * should only call this method when {@link #getType()} is non-null. * + * @param beforeTid - + * When true, the position returned starts before the + * type/length octet. + * When false, the position returned starts after the + * type/length octet and any optional length octets. + * * @return the current position; not null. * * @throws IllegalStateException if the reader doesn't have a current * value. */ - public Span getCurrentPosition() + public Span getCurrentPosition(boolean beforeTid) { - IonReaderBinarySpan pos = new IonReaderBinarySpan(); - if (getType() == null) { - String message = "IonReader isn't positioned on a value"; - throw new IllegalStateException(message); + throw new IllegalStateException("IonReader isn't positioned on a value"); } - - if (_position_start == -1) // TODO remove? should be unreachable. - { - // special case of the position before the first call to next - pos._offset = _input._pos; - pos._limit = _input._limit; - pos._symbol_table = _symbols; - } - else - { - pos._offset = _position_start - _physical_start_offset; - pos._limit = pos._offset + _position_len; - pos._symbol_table = _symbols; - } - + // Only spans that include the TID octet are seekable. + IonReaderBinarySpan pos = new IonReaderBinarySpan(beforeTid); + long start = beforeTid ? _position_start : _value_start; + long len = beforeTid ? _position_len : _value_len; + pos._offset = start - _physical_start_offset; + pos._limit = pos._offset + len; + pos._symbol_table = _symbols; + pos._state = _state; return pos; } + public byte[] getCurrentBuffer() + { + return _input._bytes; + } + public void seek(IonReaderBinarySpan position) { @@ -166,6 +185,19 @@ public void seek(IonReaderBinarySpan position) // now we need to set our symbol table _symbols = pos._symbol_table; + + // and the other misc state variables we had + // read past before getPosition gets called + // Don't do this, we'll re-read the data from the stream. + // Otherwise, this reader will be in the wrong state. + // For example, getType() will return non-null but that + // shouldn't happen until the user calls next(). +// _state = pos._state; +// _value_type = pos._value_type; +// _value_is_null = pos._value_is_null; +// _value_is_true = pos._value_is_true; + +// _is_in_struct = false; } @@ -181,7 +213,7 @@ public IonType next() } @Override - boolean hasNext() + public boolean hasNext() { if (!_eof && _has_next_needed) { clear_system_value_stack(); @@ -202,7 +234,7 @@ private final void has_next_helper_user() throws IOException { super.hasNext(); if (getDepth() == 0 && !_value_is_null) { - if (_value_tid == PrivateIonConstants.tidSymbol) { + if (_value_tid == _Private_IonConstants.tidSymbol) { if (load_annotations() == 0) { // $ion_1_0 is read as an IVM only if it is not annotated load_cached_value(AS_TYPE.int_value); @@ -214,7 +246,7 @@ private final void has_next_helper_user() throws IOException } } } - else if (_value_tid == PrivateIonConstants.tidStruct) { + else if (_value_tid == _Private_IonConstants.tidStruct) { int count = load_annotations(); if (count > 0 && _annotation_ids[0] == ION_SYMBOL_TABLE_SID) { _symbols = _lstFactory.newLocalSymtab(_catalog, this, false); @@ -223,7 +255,7 @@ else if (_value_tid == PrivateIonConstants.tidStruct) { } } else { - assert (_value_tid != PrivateIonConstants.tidTypedecl); + assert (_value_tid != _Private_IonConstants.tidTypedecl); } } } @@ -259,7 +291,6 @@ public final SymbolToken symbolValue() { return symbol; } - // // This code handles the skipped symbol table // support - it is cloned in IonReaderTextUserX, // IonReaderBinaryUserX and _Private_IonWriterBase @@ -312,22 +343,26 @@ public T asFacet(Class facetType) return facetType.cast(new SpanProviderFacet()); } - // TODO amzn/ion-java#17 support seeking over InputStream + // TODO amzn/ion-java/issues/17 support seeking over InputStream if (_input instanceof FromByteArray) { if (facetType == SeekableReader.class) { return facetType.cast(new SeekableReaderFacet()); } + if (facetType == RawValueSpanProvider.class) + { + return facetType.cast(new RawValueSpanProviderFacet()); + } } - if (facetType == PrivateByteTransferReader.class) + if (facetType == _Private_ByteTransferReader.class) { // This is a rather sketchy use of Facets, since the availability // of the facet depends upon the current state of this subject, // and that can change over time. - // TODO amzn/ion-java#16 Our {@link #transferCurrentValue} doesn't handle + // TODO amzn/ion-java/issues/16 Our {@link #transferCurrentValue} doesn't handle // field names and annotations. // Ensure there's a contiguous buffer we can copy. @@ -347,30 +382,46 @@ private class SpanProviderFacet implements SpanProvider { public Span currentSpan() { - return getCurrentPosition(); + return getCurrentPosition(true); } } + private class RawValueSpanProviderFacet implements RawValueSpanProvider + { + + public Span valueSpan() + { + return getCurrentPosition(false); + } + + public byte[] buffer() + { + return getCurrentBuffer(); + } + + } private class SeekableReaderFacet extends SpanProviderFacet implements SeekableReader { + public void hoist(Span span) { - if (! (span instanceof IonReaderBinarySpan)) + if (! (span instanceof IonReaderBinarySpan) || !((IonReaderBinarySpan)span).isSeekable()) { throw new IllegalArgumentException("Span isn't compatible with this reader."); } seek((IonReaderBinarySpan) span); } + } - private class ByteTransferReaderFacet implements PrivateByteTransferReader + private class ByteTransferReaderFacet implements _Private_ByteTransferReader { - public void transferCurrentValue(PrivateByteTransferSink sink) + public void transferCurrentValue(_Private_ByteTransferSink sink) throws IOException { // Ensure there's a contiguous buffer we can copy. @@ -380,7 +431,7 @@ public void transferCurrentValue(PrivateByteTransferSink sink) throw new UnsupportedOperationException(); } - // TODO amzn/ion-java#16 wrong if current value has a field name or + // TODO amzn/ion-java/issues/16 wrong if current value has a field name or // annotations since the position is in the wrong place. // TODO when implementing that, be careful to handle the case where // the writer already holds a pending field name or annotations! diff --git a/src/software/amazon/ion/impl/IonReaderTextRawTokensX.java b/src/com/amazon/ion/impl/IonReaderTextRawTokensX.java similarity index 98% rename from src/software/amazon/ion/impl/IonReaderTextRawTokensX.java rename to src/com/amazon/ion/impl/IonReaderTextRawTokensX.java index 4e815ebb0d..bdf3462fb4 100644 --- a/src/software/amazon/ion/impl/IonReaderTextRawTokensX.java +++ b/src/com/amazon/ion/impl/IonReaderTextRawTokensX.java @@ -1,34 +1,35 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; - -import static software.amazon.ion.impl.IonTokenConstsX.CharacterSequence.CHAR_SEQ_ESCAPED_NEWLINE_SEQUENCE_1; -import static software.amazon.ion.impl.IonTokenConstsX.CharacterSequence.CHAR_SEQ_ESCAPED_NEWLINE_SEQUENCE_2; -import static software.amazon.ion.impl.IonTokenConstsX.CharacterSequence.CHAR_SEQ_ESCAPED_NEWLINE_SEQUENCE_3; -import static software.amazon.ion.impl.IonTokenConstsX.CharacterSequence.CHAR_SEQ_NEWLINE_SEQUENCE_1; -import static software.amazon.ion.impl.IonTokenConstsX.CharacterSequence.CHAR_SEQ_NEWLINE_SEQUENCE_2; -import static software.amazon.ion.impl.IonTokenConstsX.CharacterSequence.CHAR_SEQ_NEWLINE_SEQUENCE_3; -import static software.amazon.ion.util.IonTextUtils.printCodePointAsString; - +package com.amazon.ion.impl; + +import static com.amazon.ion.impl.IonTokenConstsX.CharacterSequence.CHAR_SEQ_ESCAPED_NEWLINE_SEQUENCE_1; +import static com.amazon.ion.impl.IonTokenConstsX.CharacterSequence.CHAR_SEQ_ESCAPED_NEWLINE_SEQUENCE_2; +import static com.amazon.ion.impl.IonTokenConstsX.CharacterSequence.CHAR_SEQ_ESCAPED_NEWLINE_SEQUENCE_3; +import static com.amazon.ion.impl.IonTokenConstsX.CharacterSequence.CHAR_SEQ_NEWLINE_SEQUENCE_1; +import static com.amazon.ion.impl.IonTokenConstsX.CharacterSequence.CHAR_SEQ_NEWLINE_SEQUENCE_2; +import static com.amazon.ion.impl.IonTokenConstsX.CharacterSequence.CHAR_SEQ_NEWLINE_SEQUENCE_3; +import static com.amazon.ion.util.IonTextUtils.printCodePointAsString; + +import com.amazon.ion.IonException; +import com.amazon.ion.IonType; +import com.amazon.ion.UnexpectedEofException; +import com.amazon.ion.impl.IonTokenConstsX.CharacterSequence; +import com.amazon.ion.impl.UnifiedSavePointManagerX.SavePoint; +import com.amazon.ion.util.IonTextUtils; import java.io.IOException; -import software.amazon.ion.IonException; -import software.amazon.ion.IonType; -import software.amazon.ion.UnexpectedEofException; -import software.amazon.ion.impl.IonTokenConstsX.CharacterSequence; -import software.amazon.ion.impl.UnifiedSavePointManagerX.SavePoint; -import software.amazon.ion.util.IonTextUtils; /** * Tokenizer for the Ion text parser in IonTextIterator. This @@ -2351,10 +2352,10 @@ private final int read_large_char_sequence(int c) throws IOException if (_stream._is_byte_data) { return read_ut8_sequence(c); } - if (PrivateIonConstants.isHighSurrogate(c)) { + if (_Private_IonConstants.isHighSurrogate(c)) { int c2 = read_char(); - if (PrivateIonConstants.isLowSurrogate(c2)) { - c = PrivateIonConstants.makeUnicodeScalar(c, c2); + if (_Private_IonConstants.isLowSurrogate(c2)) { + c = _Private_IonConstants.makeUnicodeScalar(c, c2); } else { // we don't always pair up surrogates here diff --git a/src/software/amazon/ion/impl/IonReaderTextRawX.java b/src/com/amazon/ion/impl/IonReaderTextRawX.java similarity index 97% rename from src/software/amazon/ion/impl/IonReaderTextRawX.java rename to src/com/amazon/ion/impl/IonReaderTextRawX.java index d0b1369713..9a9049aa4f 100644 --- a/src/software/amazon/ion/impl/IonReaderTextRawX.java +++ b/src/com/amazon/ion/impl/IonReaderTextRawX.java @@ -1,36 +1,37 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; - -import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; -import static software.amazon.ion.impl.IonTokenConstsX.TOKEN_CLOSE_BRACE; -import static software.amazon.ion.impl.IonTokenConstsX.TOKEN_CLOSE_PAREN; -import static software.amazon.ion.impl.IonTokenConstsX.TOKEN_CLOSE_SQUARE; - +package com.amazon.ion.impl; + +import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; +import static com.amazon.ion.impl.IonTokenConstsX.TOKEN_CLOSE_BRACE; +import static com.amazon.ion.impl.IonTokenConstsX.TOKEN_CLOSE_PAREN; +import static com.amazon.ion.impl.IonTokenConstsX.TOKEN_CLOSE_SQUARE; + +import com.amazon.ion.IonException; +import com.amazon.ion.IonTextReader; +import com.amazon.ion.IonType; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.UnknownSymbolException; +import com.amazon.ion.impl.UnifiedSavePointManagerX.SavePoint; +import com.amazon.ion.impl._Private_ScalarConversions.AS_TYPE; +import com.amazon.ion.impl._Private_ScalarConversions.ValueVariant; import java.io.IOException; import java.math.BigInteger; import java.util.Iterator; -import software.amazon.ion.IonException; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonType; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.UnknownSymbolException; -import software.amazon.ion.impl.PrivateScalarConversions.AS_TYPE; -import software.amazon.ion.impl.PrivateScalarConversions.ValueVariant; -import software.amazon.ion.impl.UnifiedSavePointManagerX.SavePoint; /** * Reader implementation that reads the token stream and validates @@ -49,7 +50,7 @@ * * This reader scan skip values and in doing so it does not * materialize the contents and it does not validate the contents. - * TODO amzn/ion-java#7 We may want to make validation on skip optional. + * TODO amzn/ion-java/issues/7 We may want to make validation on skip optional. * * This manages the value buffer (_v ValueVariant) and the lob * content (_lob_*) which is cached in some cases. It's main @@ -69,7 +70,7 @@ * */ abstract class IonReaderTextRawX - implements IonReader + implements IonTextReader { public abstract BigInteger bigIntegerValue(); @@ -405,7 +406,7 @@ private final void current_value_is_null(IonType null_type) clear_current_value_buffer(); _value_type = _null_type; _v.setValueToNull(null_type); - _v.setAuthoritativeType(PrivateScalarConversions.AS_TYPE.null_value); + _v.setAuthoritativeType(_Private_ScalarConversions.AS_TYPE.null_value); } private final void current_value_is_bool(boolean value) @@ -413,7 +414,7 @@ private final void current_value_is_bool(boolean value) clear_current_value_buffer(); _value_type = IonType.BOOL; _v.setValue(value); - _v.setAuthoritativeType(PrivateScalarConversions.AS_TYPE.boolean_value); + _v.setAuthoritativeType(_Private_ScalarConversions.AS_TYPE.boolean_value); } private final void set_fieldname(SymbolToken sym) { @@ -450,7 +451,7 @@ private final void clear_annotation_list() { * but the user has chosen not to step into the collection). * @return true if more data remains, false on eof */ - boolean hasNext() + public boolean hasNext() { boolean has_next = has_next_raw_value(); return has_next; @@ -1297,7 +1298,7 @@ final String getRawFieldName() return _field_name; } - int getFieldId() + public int getFieldId() { // For hoisting if (getDepth() == 0 && is_in_struct_internal()) return UNKNOWN_SYMBOL_ID; @@ -1317,12 +1318,12 @@ public SymbolToken getFieldNameSymbol() public Iterator iterateTypeAnnotations() { - return PrivateUtils.stringIterator(getTypeAnnotations()); + return _Private_Utils.stringIterator(getTypeAnnotations()); } public String[] getTypeAnnotations() { - return PrivateUtils.toStrings(_annotations, _annotation_count); + return _Private_Utils.toStrings(_annotations, _annotation_count); } diff --git a/src/software/amazon/ion/impl/IonReaderTextSystemX.java b/src/com/amazon/ion/impl/IonReaderTextSystemX.java similarity index 94% rename from src/software/amazon/ion/impl/IonReaderTextSystemX.java rename to src/com/amazon/ion/impl/IonReaderTextSystemX.java index 07c5668365..c876386b72 100644 --- a/src/software/amazon/ion/impl/IonReaderTextSystemX.java +++ b/src/com/amazon/ion/impl/IonReaderTextSystemX.java @@ -1,46 +1,47 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; - -import static software.amazon.ion.impl.PrivateScalarConversions.getValueTypeName; - +package com.amazon.ion.impl; + +import static com.amazon.ion.impl._Private_ScalarConversions.getValueTypeName; + +import com.amazon.ion.Decimal; +import com.amazon.ion.IntegerSize; +import com.amazon.ion.IonBlob; +import com.amazon.ion.IonClob; +import com.amazon.ion.IonException; +import com.amazon.ion.IonList; +import com.amazon.ion.IonSequence; +import com.amazon.ion.IonSexp; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonTimestamp; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.Timestamp; +import com.amazon.ion.UnknownSymbolException; +import com.amazon.ion.impl.IonReaderTextRawTokensX.IonReaderTextTokenException; +import com.amazon.ion.impl.IonTokenConstsX.CharacterSequence; +import com.amazon.ion.impl._Private_ScalarConversions.AS_TYPE; +import com.amazon.ion.impl._Private_ScalarConversions.CantConvertException; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.util.Date; -import software.amazon.ion.Decimal; -import software.amazon.ion.IntegerSize; -import software.amazon.ion.IonBlob; -import software.amazon.ion.IonClob; -import software.amazon.ion.IonException; -import software.amazon.ion.IonList; -import software.amazon.ion.IonSequence; -import software.amazon.ion.IonSexp; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonTimestamp; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.Timestamp; -import software.amazon.ion.UnknownSymbolException; -import software.amazon.ion.impl.IonReaderTextRawTokensX.IonReaderTextTokenException; -import software.amazon.ion.impl.IonTokenConstsX.CharacterSequence; -import software.amazon.ion.impl.PrivateScalarConversions.AS_TYPE; -import software.amazon.ion.impl.PrivateScalarConversions.CantConvertException; import java.lang.Character; /** @@ -55,7 +56,7 @@ */ class IonReaderTextSystemX extends IonReaderTextRawX - implements PrivateReaderWriter + implements _Private_ReaderWriter { private static int UNSIGNED_BYTE_MAX_VALUE = 255; @@ -63,7 +64,7 @@ class IonReaderTextSystemX protected IonReaderTextSystemX(UnifiedInputStreamX iis) { - _system_symtab = PrivateUtils.systemSymtab(1); // TODO check IVM to determine version: amznlabs/ion-java#19, amznlabs/ion-java#24 + _system_symtab = _Private_Utils.systemSymtab(1); // TODO check IVM to determine version: amzn/ion-java/issues/19 init_once(); init(iis, IonType.DATAGRAM); } @@ -72,7 +73,6 @@ protected IonReaderTextSystemX(UnifiedInputStreamX iis) // be consolidated into a single location, but that would have to be part // of a larger refactor of common logic from both IonReader*SystemX classes // into a base class (the *Value() methods also share a lot of similarity). - @Override public IntegerSize getIntegerSize() { load_once(); @@ -80,7 +80,7 @@ public IntegerSize getIntegerSize() { return null; } - return PrivateScalarConversions.getIntegerSize(_v.getAuthoritativeType()); + return _Private_ScalarConversions.getIntegerSize(_v.getAuthoritativeType()); } private void load_once() @@ -486,7 +486,7 @@ public SymbolToken[] getTypeAnnotationSymbols() for (int i = 0; i < count; i++) { SymbolToken sym = _annotations[i]; - SymbolToken updated = PrivateUtils.localize(symbols, sym); + SymbolToken updated = _Private_Utils.localize(symbols, sym); if (updated != sym) _annotations[i] = updated; result[i] = updated; } @@ -615,7 +615,7 @@ public SymbolTable getSymbolTable() @Override - final int getFieldId() + public final int getFieldId() { // Superclass handles hoisting logic int id = super.getFieldId(); @@ -658,7 +658,7 @@ public SymbolToken getFieldNameSymbol() SymbolToken sym = super.getFieldNameSymbol(); if (sym != null) { - sym = PrivateUtils.localize(getSymbolTable(), sym); + sym = _Private_Utils.localize(getSymbolTable(), sym); } return sym; } @@ -874,7 +874,7 @@ private int readBytes(byte[] buffer, int offset, int len) buffer[offset++] = (byte)c; } break; - // CLOB + // CLOB case IonTokenConstsX.TOKEN_STRING_TRIPLE_QUOTE: while (len-- > 0) { c = _scanner.read_triple_quoted_char(true); diff --git a/src/software/amazon/ion/impl/IonReaderTextUserX.java b/src/com/amazon/ion/impl/IonReaderTextUserX.java similarity index 91% rename from src/software/amazon/ion/impl/IonReaderTextUserX.java rename to src/com/amazon/ion/impl/IonReaderTextUserX.java index b1dfc8d5a4..1b0d0dcd7d 100644 --- a/src/software/amazon/ion/impl/IonReaderTextUserX.java +++ b/src/com/amazon/ion/impl/IonReaderTextUserX.java @@ -1,34 +1,35 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; - -import static software.amazon.ion.SystemSymbols.ION_1_0; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; - +package com.amazon.ion.impl; + +import static com.amazon.ion.SystemSymbols.ION_1_0; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; + +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonType; +import com.amazon.ion.OffsetSpan; +import com.amazon.ion.SeekableReader; +import com.amazon.ion.Span; +import com.amazon.ion.SpanProvider; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.TextSpan; +import com.amazon.ion.UnknownSymbolException; +import com.amazon.ion.UnsupportedIonVersionException; import java.util.regex.Pattern; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonType; -import software.amazon.ion.OffsetSpan; -import software.amazon.ion.SeekableReader; -import software.amazon.ion.Span; -import software.amazon.ion.SpanProvider; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.TextSpan; -import software.amazon.ion.UnknownSymbolException; -import software.amazon.ion.UnsupportedIonVersionException; /** * The text user reader add support for symbols and recognizes, @@ -49,10 +50,11 @@ * symbol is a system symbol or if there is a local symbol * table in the input stream. Otherwise it return the * undefined symbol value. + * */ class IonReaderTextUserX extends IonReaderTextSystemX - implements PrivateReaderWriter + implements _Private_ReaderWriter { private static final Pattern ION_VERSION_MARKER_REGEX = Pattern.compile("^\\$ion_[0-9]+_[0-9]+$"); @@ -62,7 +64,7 @@ class IonReaderTextUserX * {@link OffsetSpan}s. */ private final int _physical_start_offset; - private final PrivateLocalSymbolTableFactory _lstFactory; + private final _Private_LocalSymbolTableFactory _lstFactory; // IonSystem _system; now in IonReaderTextSystemX where it could be null IonCatalog _catalog; @@ -70,7 +72,7 @@ class IonReaderTextUserX protected IonReaderTextUserX(IonCatalog catalog, - PrivateLocalSymbolTableFactory lstFactory, + _Private_LocalSymbolTableFactory lstFactory, UnifiedInputStreamX uis, int physicalStartOffset) { @@ -82,7 +84,7 @@ protected IonReaderTextUserX(IonCatalog catalog, } protected IonReaderTextUserX(IonCatalog catalog, - PrivateLocalSymbolTableFactory lstFactory, + _Private_LocalSymbolTableFactory lstFactory, UnifiedInputStreamX uis) { this(catalog, lstFactory, uis, 0); } @@ -103,7 +105,7 @@ protected IonReaderTextUserX(IonCatalog catalog, * @return true if more data remains, false on eof */ @Override - boolean hasNext() + public boolean hasNext() { boolean has_next = has_next_user_value(); return has_next; diff --git a/src/software/amazon/ion/impl/IonReaderTreeSystem.java b/src/com/amazon/ion/impl/IonReaderTreeSystem.java similarity index 86% rename from src/software/amazon/ion/impl/IonReaderTreeSystem.java rename to src/com/amazon/ion/impl/IonReaderTreeSystem.java index 55cb5b5245..f8a12181b0 100644 --- a/src/software/amazon/ion/impl/IonReaderTreeSystem.java +++ b/src/com/amazon/ion/impl/IonReaderTreeSystem.java @@ -1,59 +1,61 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; - -import static software.amazon.ion.impl.PrivateUtils.readFully; - +package com.amazon.ion.impl; + +import static com.amazon.ion.impl._Private_Utils.readFully; + +import com.amazon.ion.Decimal; +import com.amazon.ion.IntegerSize; +import com.amazon.ion.IonBool; +import com.amazon.ion.IonContainer; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonDecimal; +import com.amazon.ion.IonException; +import com.amazon.ion.IonFloat; +import com.amazon.ion.IonInt; +import com.amazon.ion.IonLob; +import com.amazon.ion.IonNull; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSymbol; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonText; +import com.amazon.ion.IonTimestamp; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.Timestamp; +import com.amazon.ion.impl._Private_IonValue.SymbolTableProvider; import java.io.IOException; import java.io.InputStream; import java.math.BigDecimal; import java.math.BigInteger; import java.util.Date; import java.util.Iterator; -import software.amazon.ion.Decimal; -import software.amazon.ion.IntegerSize; -import software.amazon.ion.IonBool; -import software.amazon.ion.IonContainer; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonDecimal; -import software.amazon.ion.IonException; -import software.amazon.ion.IonFloat; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonLob; -import software.amazon.ion.IonNull; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonText; -import software.amazon.ion.IonTimestamp; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.Timestamp; -import software.amazon.ion.impl.PrivateIonValue.SymbolTableProvider; + class IonReaderTreeSystem - implements IonReader, PrivateReaderWriter + implements IonReader, _Private_ReaderWriter { protected final SymbolTable _system_symtab; protected Iterator _iter; protected IonValue _parent; - protected PrivateIonValue _next; - protected PrivateIonValue _curr; + protected _Private_IonValue _next; + protected _Private_IonValue _curr; protected boolean _eof; /** Holds pairs: IonValue parent, Iterator cursor */ @@ -103,7 +105,7 @@ void re_init(IonValue value, boolean hoisted) if (value instanceof IonDatagram) { // datagrams interacting with these readers must be // IonContainerPrivate containers - assert(value instanceof PrivateIonContainer); + assert(value instanceof _Private_IonContainer); IonDatagram dg = (IonDatagram) value; _parent = dg; _next = null; @@ -111,7 +113,7 @@ void re_init(IonValue value, boolean hoisted) } else { _parent = (hoisted ? null : value.getContainer()); - _next = (PrivateIonValue) value; + _next = (_Private_IonValue) value; } } @@ -148,9 +150,15 @@ private void pop() { _eof = false; } + public boolean hasNext() + { + IonType next_type = next_helper_system(); + return (next_type != null); + } + public IonType next() { - if (this._next == null && next_helper_system() == null) { + if (this._next == null && !this.hasNext()) { this._curr = null; return null; } @@ -166,7 +174,7 @@ IonType next_helper_system() if (this._next != null) return this._next.getType(); if (this._iter != null && this._iter.hasNext()) { - this._next = (PrivateIonValue) this._iter.next(); + this._next = (_Private_IonValue) this._iter.next(); } if ((this._eof =(this._next == null)) == true) { @@ -230,7 +238,7 @@ public final SymbolToken[] getTypeAnnotationSymbols() public final Iterator iterateTypeAnnotations() { String [] annotations = getTypeAnnotations(); - return PrivateUtils.stringIterator(annotations); + return _Private_Utils.stringIterator(annotations); } @@ -249,6 +257,12 @@ public boolean isNullValue() return _curr.isNullValue(); } + public int getFieldId() + { + // FIXME IonValueImpl.getFieldId doesn't return -1 as specced here! + return (_curr == null || _top == 0) ? SymbolTable.UNKNOWN_SYMBOL_ID : _curr.getFieldId(); + } + public String getFieldName() { return (_curr == null || _top == 0) ? null : _curr.getFieldName(); @@ -445,13 +459,13 @@ private static final class Children implements Iterator { boolean _eof; int _next_idx; - PrivateIonContainer _parent; + _Private_IonContainer _parent; IonValue _curr; Children(IonContainer parent) { - if (parent instanceof PrivateIonContainer) { - _parent = (PrivateIonContainer)parent; + if (parent instanceof _Private_IonContainer) { + _parent = (_Private_IonContainer) parent; _next_idx = 0; _curr = null; if (_parent.isNullValue()) { @@ -519,7 +533,6 @@ public SymbolTable pop_passed_symbol_table() } - @Override public IntegerSize getIntegerSize() { if(_curr instanceof IonInt) diff --git a/src/software/amazon/ion/impl/IonReaderTreeUserX.java b/src/com/amazon/ion/impl/IonReaderTreeUserX.java similarity index 83% rename from src/software/amazon/ion/impl/IonReaderTreeUserX.java rename to src/com/amazon/ion/impl/IonReaderTreeUserX.java index 9b24511825..1ed1c31bfe 100644 --- a/src/software/amazon/ion/impl/IonReaderTreeUserX.java +++ b/src/com/amazon/ion/impl/IonReaderTreeUserX.java @@ -1,46 +1,48 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; -import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; -import static software.amazon.ion.SystemSymbols.ION_1_0_SID; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; +import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; +import static com.amazon.ion.SystemSymbols.ION_1_0_SID; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; + +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSymbol; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.SeekableReader; +import com.amazon.ion.Span; +import com.amazon.ion.SpanProvider; +import com.amazon.ion.SymbolTable; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.SeekableReader; -import software.amazon.ion.Span; -import software.amazon.ion.SpanProvider; -import software.amazon.ion.SymbolTable; final class IonReaderTreeUserX extends IonReaderTreeSystem - implements PrivateReaderWriter + implements _Private_ReaderWriter { - private final PrivateLocalSymbolTableFactory _lstFactory; + private final _Private_LocalSymbolTableFactory _lstFactory; IonCatalog _catalog; private SymbolTable _symbols; - public IonReaderTreeUserX(IonValue value, IonCatalog catalog, PrivateLocalSymbolTableFactory lstFactory) + public IonReaderTreeUserX(IonValue value, IonCatalog catalog, _Private_LocalSymbolTableFactory lstFactory) { super(value); // calls re_init _catalog = catalog; @@ -62,6 +64,12 @@ public SymbolTable getSymbolTable() return _symbols; } + @Override + public boolean hasNext() + { + return next_helper_user(); + } + @Override public IonType next() { @@ -74,7 +82,7 @@ public IonType next() return this._curr.getType(); } - private boolean next_helper_user() + boolean next_helper_user() { if (_eof) return false; if (_next != null) return true; @@ -97,7 +105,7 @@ private boolean next_helper_user() // there are no null values we will consume here break; } - int sid = sym.symbolValue().getSid(); + int sid = sym.getSymbolId(); if (sid == UNKNOWN_SYMBOL_ID) { String name = sym.stringValue(); if (name != null) { diff --git a/src/com/amazon/ion/impl/IonTextBufferedStream.java b/src/com/amazon/ion/impl/IonTextBufferedStream.java new file mode 100644 index 0000000000..2be9e8f4ab --- /dev/null +++ b/src/com/amazon/ion/impl/IonTextBufferedStream.java @@ -0,0 +1,311 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import java.io.IOException; +import java.io.InputStream; + +/** + * InputStream implementations over a number of types (byte[] and String) + * that do not handle character decoding. Each byte is treated as just + * a character from 0 to 255. These are used internally to wire up some + * of the Iterator code to some of the base64 encoding (or decoding) + * routins in ion.impl. + */ +abstract class IonTextBufferedStream extends InputStream +{ + public static IonTextBufferedStream makeStream(byte[] bytes) { + return new SimpleBufferStream(bytes); + } + public static IonTextBufferedStream makeStream(byte[] bytes, int offset, int len) { + return new OffsetBufferStream(bytes, offset, len); + } + public static IonTextBufferedStream makeStream(String text) { + return new StringStream(text); + } + /*** FIXME: REMOVE OR FINISH + public static IonTextBufferedStream makeStream(InputStream stream) { + return new StreamStream(stream); + } + */ + public abstract int getByte(int pos); + public abstract int position(); + public abstract IonTextBufferedStream setPosition(int pos); + + /*** FIXME: REMOVE OR FINISH + static final class StreamStream extends IonTextBufferedStream + { + static final int DEFAULT_BUFFER_SIZE = (32*1024); + + InputStream _stream; + byte [] _buffer1; + byte [] _buffer2; + long _file_pos1; + long _file_pos2; + int _len; + int _pos; + + public StreamStream (InputStream stream) + { + _stream = stream; + _buffer1 = new byte[DEFAULT_BUFFER_SIZE]; + _buffer2 = new byte[DEFAULT_BUFFER_SIZE]; + _pos = 0; + _file_pos1 = 0; + _file_pos2 = -1; + _len = load(_buffer1); + } + int load(byte[] buffer) { + + } + + @Override + public final int getByte(int pos) { + if (pos < 0 || pos >= _len) return -1; + return _buffer[pos] & 0xff; + } + + @Override + public final int read() + throws IOException + { + if (_pos >= _len) return -1; + return _buffer[_pos++]; + } + + @Override + public final int read(byte[] bytes, int offset, int len) throws IOException + { + int copied = 0; + if (offset < 0) throw new IllegalArgumentException(); + copied = len; + if (_pos + len >= _len) copied = _len - _pos; + System.arraycopy(_buffer, _pos, bytes, offset, copied); + _pos += copied; + return copied; + } + + @Override + public final int position() { + return _pos; + } + + @Override + public final SimpleBufferStream setPosition(int pos) + { + if (_pos < 0 || _pos > _len) throw new IllegalArgumentException(); + _pos = pos; + return this; + } + + @Override + public void close() throws IOException + { + _stream.close(); + super.close(); + } + } + */ + + static final class SimpleBufferStream extends IonTextBufferedStream + { + byte [] _buffer; + int _len; + int _pos; + + public SimpleBufferStream(byte[] buffer) + { + _buffer = buffer; + _pos = 0; + _len = buffer.length; + } + + @Override + public final int getByte(int pos) { + if (pos < 0 || pos >= _len) return -1; + return _buffer[pos] & 0xff; + } + + @Override + public final int read() + throws IOException + { + if (_pos >= _len) return -1; + return _buffer[_pos++]; + } + + @Override + public final int read(byte[] bytes, int offset, int len) throws IOException + { + int copied = 0; + if (offset < 0) throw new IllegalArgumentException(); + copied = len; + if (_pos + len >= _len) copied = _len - _pos; + System.arraycopy(_buffer, _pos, bytes, offset, copied); + _pos += copied; + return copied; + } + + @Override + public final int position() { + return _pos; + } + + @Override + public final SimpleBufferStream setPosition(int pos) + { + if (_pos < 0 || _pos > _len) throw new IllegalArgumentException(); + _pos = pos; + return this; + } + + @Override + public void close() + throws IOException + { + _pos = _len; + super.close(); + } + } + + static final class StringStream extends IonTextBufferedStream + { + + String _string; + int _end; + int _pos; + + public StringStream(String text) + { + _string = text; + _pos = 0; + _end = text.length(); + } + + @Override + public final int getByte(int pos) { + if (pos < 0) return -1; + if (pos >= _end) return -1; + return _string.charAt(pos); + } + + @Override + public final int read() + throws IOException + { + if (_pos >= _end) return -1; + char c = _string.charAt(_pos++); + return c; + } + + @Override + public final int read(byte[] bytes, int offset, int len) throws IOException + { + throw new UnsupportedOperationException(); + } + + @Override + public final int position() { + return _pos; + } + + @Override + public final StringStream setPosition(int pos) + { + if (pos < 0) throw new IllegalArgumentException(); + if (pos > _end) throw new IllegalArgumentException(); + _pos = pos; + return this; + } + + @Override + public void close() + throws IOException + { + _pos = _end; + super.close(); + } + } + + static final class OffsetBufferStream extends IonTextBufferedStream + { + byte [] _buffer; + int _start; + int _end; + int _pos; + + public OffsetBufferStream(byte[] buffer, int start, int max) + { + _buffer = buffer; + _pos = start; + _start = start; + _end = start + max; + } + + @Override + public final int getByte(int pos) { + if (pos < 0) return -1; + pos += _start; + if (pos >= _end) return -1; + return _buffer[pos] & 0xff; + } + + @Override + public final int read() + throws IOException + { + int c; + if (_pos >= _end) return -1; + c = (((int)_buffer[_pos++]) & 0xFF); // trim sign extension bits + return c; + } + + @Override + public final int read(byte[] bytes, int offset, int len) throws IOException + { + int copied = 0; + if (offset < 0) throw new IllegalArgumentException(); + copied = len; + if (_pos + len >= _end) copied = _end - _pos; + System.arraycopy(_buffer, _pos, bytes, offset, copied); + _pos += copied; + return copied; + } + + @Override + public final int position() { + return _pos - _start; + } + + @Override + public final OffsetBufferStream setPosition(int pos) + { + if (pos < 0) throw new IllegalArgumentException(); + pos += _start; + if (pos > _end) throw new IllegalArgumentException(); + _pos = pos; + return this; + } + + @Override + public void close() + throws IOException + { + _pos = _end; + super.close(); + } + } +} diff --git a/src/software/amazon/ion/impl/IonTokenConstsX.java b/src/com/amazon/ion/impl/IonTokenConstsX.java similarity index 98% rename from src/software/amazon/ion/impl/IonTokenConstsX.java rename to src/com/amazon/ion/impl/IonTokenConstsX.java index 670bdf3183..c361606636 100644 --- a/src/software/amazon/ion/impl/IonTokenConstsX.java +++ b/src/com/amazon/ion/impl/IonTokenConstsX.java @@ -1,22 +1,23 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; -import software.amazon.ion.IonException; -import software.amazon.ion.IonType; -import software.amazon.ion.impl.PrivateScalarConversions.CantConvertException; +import com.amazon.ion.IonException; +import com.amazon.ion.IonType; +import com.amazon.ion.impl._Private_ScalarConversions.CantConvertException; /** diff --git a/src/com/amazon/ion/impl/IonTokenReader.java b/src/com/amazon/ion/impl/IonTokenReader.java new file mode 100644 index 0000000000..34bf32f921 --- /dev/null +++ b/src/com/amazon/ion/impl/IonTokenReader.java @@ -0,0 +1,1623 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import static com.amazon.ion.util.IonTextUtils.isDigit; +import static com.amazon.ion.util.IonTextUtils.isOperatorPart; +import static com.amazon.ion.util.IonTextUtils.isWhitespace; +import static com.amazon.ion.util.IonTextUtils.printCodePointAsString; + +import com.amazon.ion.Decimal; +import com.amazon.ion.IonException; +import com.amazon.ion.Timestamp; +import com.amazon.ion.UnexpectedEofException; +import com.amazon.ion.impl._Private_IonConstants.HighNibble; +import com.amazon.ion.util.IonTextUtils; +import java.io.IOException; +import java.io.PushbackReader; +import java.io.Reader; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Stack; + +/** + * This class is responsible for breaking the input stream into Tokens. + * It does both the token recognition (aka the scanner) and the state + * management needed to use the value underlying some of these tokens. + */ +final class IonTokenReader +{ + // TODO clean up, many of these are unused. + public static int isPunctuation = 0x0001; + public static int isKeyword = 0x0002; + public static int isTypeName = 0x0004; + public static int isConstant = 0x0008; + private static int isPosInt = 0x0010; + private static int isNegInt = 0x0020; + public static int isFloat = 0x0040; + public static int isDecimal = 0x0080; + public static int isTag = 0x0100; + + static public enum Type { + + eof (isPunctuation, "" ), + tOpenParen (isPunctuation, "(" ), + tCloseParen (isPunctuation, ")" ), + tOpenSquare (isPunctuation, "[" ), + tCloseSquare (isPunctuation, "[" ), + tOpenCurly (isPunctuation, "{" ), + tCloseCurly (isPunctuation, "}" ), + tOpenDoubleCurly (isPunctuation, "{{" ), + //tCloseDoubleCurly (isPunctuation, "}}" ), // only valid at end of blob, not recognized in token read + tSingleQuote (isPunctuation, "'" ), + tDoubleQuote (isPunctuation, "\"" ), + //tColon (isPunctuation, ":" ), // filled in during identifier scan with lookahead + //tDoubleColon (isPunctuation, "::" ), // ditto + tComma (isPunctuation, "," ), + + kwTrue ((isConstant + isTag + isKeyword), "true", HighNibble.hnBoolean), + kwFalse ((isConstant + isTag + isKeyword), "false", HighNibble.hnBoolean), + kwNull ((isConstant + isTag + isKeyword), "null", HighNibble.hnNull), + + kwNullNull ((isConstant + isTag + isKeyword), "null.null", HighNibble.hnNull), + kwNullInt ((isConstant + isTag + isKeyword), "null.int", HighNibble.hnPosInt), + kwNullList ((isConstant + isTag + isKeyword), "null.list", HighNibble.hnList), + kwNullSexp ((isConstant + isTag + isKeyword), "null.sexp", HighNibble.hnSexp), + kwNullFloat ((isConstant + isTag + isKeyword), "null.float", HighNibble.hnFloat), + kwNullBlob ((isConstant + isTag + isKeyword), "null.blob", HighNibble.hnBlob), + kwNullClob ((isConstant + isTag + isKeyword), "null.clob", HighNibble.hnClob), + kwNullString ((isConstant + isTag + isKeyword), "null.string", HighNibble.hnString), + kwNullStruct ((isConstant + isTag + isKeyword), "null.struct", HighNibble.hnStruct), // LN + kwNullSymbol ((isConstant + isTag + isKeyword), "null.symbol", HighNibble.hnSymbol), + kwNullBoolean ((isConstant + isTag + isKeyword), "null.bool", HighNibble.hnBoolean), + kwNullDecimal ((isConstant + isTag + isKeyword), "null.decimal", HighNibble.hnDecimal), + kwNullTimestamp ((isConstant + isTag + isKeyword), "null.timestamp", HighNibble.hnTimestamp), + + kwNan ((isConstant + isKeyword), "nan", HighNibble.hnFloat), + kwPosInf ((isConstant + isKeyword), "+inf", HighNibble.hnFloat), + kwNegInf ((isConstant + isKeyword), "-inf", HighNibble.hnFloat), + + constNegInt ((isConstant + isNegInt), "cNegInt", HighNibble.hnNegInt), + constPosInt ((isConstant + isPosInt), "cPosInt", HighNibble.hnPosInt), + constFloat ((isConstant + isFloat), "cFloat", HighNibble.hnFloat), + constDecimal ((isConstant + isDecimal), "cDec", HighNibble.hnDecimal), + constTime ((isConstant), "cTime", HighNibble.hnTimestamp), + constString ((isConstant + isTag), "cString", HighNibble.hnString), + constSymbol ((isConstant + isTag), "cSymbol", HighNibble.hnSymbol), + constMemberName ((isConstant + isTag), "cMemberName", HighNibble.hnSymbol), + constUserTypeDecl ((isConstant + isTag), "cUserTypeDecl", HighNibble.hnSymbol), + + none(0); + + private int flags; + private String image; + private HighNibble highNibble; + + Type() {} + Type(int v) { + this.flags = v; + } + Type(int v, String name) { + flags = v; + image = name; + } + Type(int v, String name, HighNibble ln) { + flags = v; + image = name; + highNibble = ln; + } + + /** + * TODO why this class exists at all. We should store + * this stuff directly in IonTimestampImpl as a BigDecimal (time) and + * int (offset, -1==unknown) and avoid constructing more objects. + *

    + * Also, we probably don't need the DateFormats. The parser already + * matches against a regex, so we should be able to pull the data + * straight from the string to compute the value. + */ + public static class timeinfo { // TODO remove vestigial class timeinfo + + static public Timestamp parse(String s) { + Timestamp t = null; + s = s.trim(); // TODO why is this necessary? + try { + t = Timestamp.valueOf(s); // TODO should Timestamp just throw an IonException? + } + catch (IllegalArgumentException e) { + throw new IonException(e); + } + return t; + } + + } + + // TODO move out of this class into TR. + public Type setNumericValue(IonTokenReader tr, String s) { + switch (this) { + case kwNan: + tr.doubleValue = Double.NaN; + return this; + case kwPosInf: + tr.doubleValue = Double.POSITIVE_INFINITY; + return this; + case kwNegInf: + tr.doubleValue = Double.NEGATIVE_INFINITY; + return this; + case constNegInt: + case constPosInt: + if (NumberType.NT_HEX.equals(tr.numberType)) { + tr.intValue = new BigInteger(s, 16); + // In hex case we've discarded the prefix [+-]?0x so + // reconstruct the sign + if (this == constNegInt) tr.intValue = tr.intValue.negate(); + } + else { + tr.intValue = new BigInteger(s, 10); + + // Make sure that sign aligns with type. + // Note that this allows negative zero. + assert (BigInteger.ZERO.equals(tr.intValue) + ? true + : this == (tr.intValue.signum() < 0 ? constNegInt : constPosInt)); + } + return this; + case constFloat: + tr.doubleValue = Double.parseDouble(s); + return this; + case constDecimal: + // BigDecimal parses using e instead of d or D + String eFormat = s.replace('d', 'e'); + if (eFormat == s) { + // No match for 'd' but look for 'D' + eFormat = s.replace('D', 'e'); + } + tr.decimalValue = Decimal.valueOf(eFormat); + return this; + case constTime: + tr.dateValue = timeinfo.parse(s); + return this; + default: + throw new AssertionError("Unknown op for numeric case: " + this); + } + } + + public boolean isKeyword() { return ((flags & isKeyword) != 0); } + public boolean isConstant() { return ((flags & isConstant) != 0); } + public boolean isNumeric() { return ((flags & (isPosInt + isNegInt + isFloat + isDecimal)) != 0); } + + public String getImage() { return (image == null) ? this.name() : image; } + + public HighNibble getHighNibble() { return highNibble; } + + @Override + public String toString() { + if (this.getImage() != null) return this.getImage(); + return super.toString(); + } + } + + // + // this is a temp reader we use when converting blob's into + // blob values + // + static class LocalReader extends Reader { + + IonTokenReader _tr; + int _sboffset; + int _sbavailable; + + LocalReader(IonTokenReader tr) { + _tr = tr; + } + + @Override + public void close() throws IOException { + _tr = null; + return; + } + + @Override + public void reset() { + _sboffset = 0; + _sbavailable = _tr.value.length(); + } + + @Override + public int read() throws IOException { + int c = -1; + + if (_sbavailable > 0) { + c = _tr.value.charAt(_sboffset++); + _sbavailable--; + } + else { + c = _tr.read(); + } + return c; + } + + @Override + public int read(char[] dst, int dstoffset, int len) throws IOException { + int needed = len; + + while (needed-- > 0) { + int c = this.read(); + if (c < 0) break; + dst[dstoffset++] = (char)c; + } + return len - needed; + } + } + + // easy to use number types + static enum NumberType { + NT_POSINT, + NT_NEGINT, + NT_HEX, // pos or neg + NT_FLOAT, + NT_DECIMAL, + NT_DECIMAL_NEGATIVE_ZERO + } + + /** + * Magic "character" to represent an escape sequence with an empty expansion. + * (or, in other words, a non-eof character that should be ignored) + */ + static final int EMPTY_ESCAPE_SEQUENCE = -2; + + + //-------------------------------------------------------------------- + // + // this is the parsing context to manage the in list, in struct, + // waiting for identifier tag (usertypedecl or member name) vs value + // + // + static public enum Context { + NONE, + STRING, + BLOB, + CLOB, + EXPRESSION, + DATALIST, + STRUCT; + } + public Stack contextStack = new Stack(); + public Context context = Context.NONE; + + public void pushContext(Context newcontext) { + contextStack.push(newcontext); + context = newcontext; + } + public Context popContext() { + context = contextStack.pop(); + return context; + } + + //-------------------------------------------------------------------- + // + // state that keeps us on track ... + // + + private IonCharacterReader in; + private LocalReader localReader; + private PushbackReader pushbackReader; + + public boolean inQuotedContent; + public boolean isIncomplete; + public boolean isLongString; + public boolean quotedIdentifier; + public int embeddedSlash; + public int endquote; + + + public Type t = Type.none; + public Type keyword = Type.none; + public StringBuilder value = new StringBuilder(); + + public String stringValue; + public Double doubleValue; + public BigInteger intValue; + + public Timestamp dateValue; + public BigDecimal decimalValue; + public boolean isNegative; + public NumberType numberType; + + public IonTokenReader(final Reader r) { + this.in = new IonCharacterReader( r ); + } + + public long getConsumedAmount() { + return in.getConsumedAmount(); + } + + public int getLineNumber() { + return in.getLineNumber(); + } + + public int getColumn() { + return in.getColumn(); + } + + public String position() { + return "line " + this.getLineNumber() + " column " + this.getColumn(); + } + + public String getValueString(boolean is_in_expression) throws IOException { + if (this.isIncomplete) { + finishScanString(is_in_expression); + stringValue = value.toString(); // TODO combine with below? + this.inQuotedContent = false; + } + else if (stringValue == null) { + stringValue = value.toString(); + } + return stringValue; + } + + void resetValue() { + isIncomplete = false; + stringValue = null; + doubleValue = null; + intValue = null; + dateValue = null; + decimalValue = null; + isNegative = false; + numberType = null; + t = null; + value.setLength(0); + } + + public PushbackReader getPushbackReader() { + if (localReader == null) { + localReader = new LocalReader(this); + pushbackReader = + new PushbackReader(localReader, + _Private_Utils.MAX_LOOKAHEAD_UTF16); + } + localReader.reset(); + return pushbackReader ; + } + + /** + * Reads the next character using the underlying stream. + * @return -1 on end of stream. + * @throws IOException + */ + final int read() throws IOException { + final int ch = in.read(); + assert ch != '\r'; + return ch; + } + + + int readIgnoreWhitespace() throws IOException { + assert !inQuotedContent; + + int c; + + // read through comments - this detects and rejects double slash + // and slash star style comments and their contents + for (;;) { + c = read(); + + if (c == '/') { // possibly a start of a comment + int c2 = read(); + + if (c2 == '/') { + // we have a // comment, scan for the terminating new line + while (c2 != '\n' && c2 != -1) { + c2 = read(); + } + c = c2; + } + else if (c2 == '*') { + // we have a /* */ comment scan for closing */ + scancomment: + while (c2 != -1) { + c2 = read(); + + if (c2 == '*') { + c2 = read(); + + if (c2 == '/') { + break scancomment; + } + unread(c2); // in case this was the '*' we're looking for + } + } + c = read(); + } + else { + // it wasn't a comment start, throw it back + unread(c2); + } + } + if (/* inContent ||*/ !isWhitespace(c)) { + break; + } + } + return c; + } + void unread(int c) throws IOException { + this.in.unread(c); + } + + // TODO clone the body of next(c) here, later , for perf, we + // don't really want TWO methods calls per character, or + // mostly we don't want an extra one. (and I doubt that + // Java handles the tail optimization) + public Type next(boolean is_in_expression) throws IOException { + inQuotedContent = false; + int c = this.readIgnoreWhitespace(); + return next(c, is_in_expression); + } + + private Type next(int c, boolean is_in_expression) throws IOException { + int c2; + t = Type.none; + + isIncomplete = false; + switch (c) { + case -1: + return (t = Type.eof); + case '{': + c2 = read(); + if (c2 == '{') { + return (t = Type.tOpenDoubleCurly); + } + unread(c2); + return (t = Type.tOpenCurly); + case '}': + return (t = Type.tCloseCurly); + case '[': + return (t = Type.tOpenSquare); + case ']': + return (t = Type.tCloseSquare); + case '(': + return (t = Type.tOpenParen); + case ')': + return (t = Type.tCloseParen); + case ',': + return (t = Type.tComma); + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + return readNumber(c); + case '\"': + inQuotedContent = true; + this.keyword = Type.none; // anything in quotes isn't a keyword + return scanString(c, _Private_IonConstants.lnIsVarLen - 1); + case '\'': + c2 = read(); + if (c2 == '\'') { + c2 = read(); + if (c2 == '\'') { + return scanLongString(); + } + this.unread(c2); + c2 = '\''; // restore c2 for the next unread + } + this.unread(c2); + inQuotedContent = true; + return scanIdentifier(c); + case '+': + c2 = read(); + if (c2 == 'i') { + c2 = read(); + if (c2 == 'n') { + c2 = read(); + if (c2 == 'f') { + return (t = Type.kwPosInf); + } + this.unread(c2); + c2 = 'n'; + } + this.unread(c2); + c2 = 'i'; + } + this.unread(c2); + if (is_in_expression) { + return scanOperator(c); + } + break; // break to error + case '-': + c2 = read(); + if (c2 >= '0' && c2 <= '9') { + this.unread(c2); + return readNumber(c); + } + if (c2 == 'i') { + c2 = read(); + if (c2 == 'n') { + c2 = read(); + if (c2 == 'f') { + return (t = Type.kwNegInf); + } + this.unread(c2); + c2 = 'n'; + } + this.unread(c2); + c2 = 'i'; + } + this.unread(c2); + if (is_in_expression) { + return scanOperator(c); + } + break; // break to error + default: + if (IonTextUtils.isIdentifierStart(c)) { + return scanIdentifier(c); + } + if (is_in_expression && isOperatorPart(c)) { + return scanOperator(c); + } + } + + String message = + "Unexpected character " + printCodePointAsString(c) + + " encountered at line " + this.getLineNumber() + + " column " + this.getColumn(); + throw new IonException(message); + } + + + public Type scanIdentifier(int c) throws IOException + { + // reset our local value buffer + resetValue(); + + // we don't have an identifier type any longer, just string + this.t = Type.constSymbol; + + // some strings are keywords, most are not + // mostly we'll guess that it's not a keyword + this.keyword = null; + + // first we read the identifier content into our value buffer + // if the value is quoted, use the escape char loop, otherwise + // look for the non-identifier character (and throw it back) + + if (!readIdentifierContents(c)) { // not quoted (true is quoted) + // anything in quotes (even single quotes) is not a keyword + // and here we're not in quoted content (that is handled above) + // so see if it's also a keyword + this.keyword = IonTokenReader.matchKeyword(value, 0, value.length()); + if (this.keyword != null) { + if (this.keyword == Type.kwNull) { + c = this.read(); + if (c == '.') { + int dot = value.length(); + value.append((char)c); + c = this.read(); + int added = readIdentifierContents(c, IonTokenConstsX.TN_MAX_NAME_LENGTH + 1); // +1 is "enough roap" so if there are extra letters at the end of the keyword we keep at least 1 + int kw = IonTokenConstsX.keyword(value, dot+1, dot+added+1); + switch (kw) { + case IonTokenConstsX.KEYWORD_NULL: + case IonTokenConstsX.KEYWORD_BOOL: + case IonTokenConstsX.KEYWORD_INT: + case IonTokenConstsX.KEYWORD_FLOAT: + case IonTokenConstsX.KEYWORD_DECIMAL: + case IonTokenConstsX.KEYWORD_TIMESTAMP: + case IonTokenConstsX.KEYWORD_SYMBOL: + case IonTokenConstsX.KEYWORD_STRING: + case IonTokenConstsX.KEYWORD_BLOB: + case IonTokenConstsX.KEYWORD_CLOB: + case IonTokenConstsX.KEYWORD_LIST: + case IonTokenConstsX.KEYWORD_SEXP: + case IonTokenConstsX.KEYWORD_STRUCT: + this.keyword = setNullType(value, dot + 1, value.length() - dot - 1); + break; + default: + // not a valid type name - so we have to unread back to the dot + for (int ii=value.length(); ii>dot; ) { + ii--; + char uc = value.charAt(ii); + unread(uc); + } + + String message = + position() + + ": Expected Ion type after 'null.' but found: " + + value; + throw new IonException(message); + } + } + else { + unread(c); + } + } + this.t = this.keyword; + return this.t; + } + } + + // see if we're a user type of a member name + c = this.readIgnoreWhitespace(); + if (c != ':') { + unread(c); + } + else { + c = read(); + if (c != ':') { + unread(c); + this.t = Type.constMemberName; + } + else { + this.t = Type.constUserTypeDecl; + } + } + return this.t; + } + + boolean readIdentifierContents(int c) throws IOException { + + int quote = c; + inQuotedContent = (quote == '\'' || quote == '\"'); + + if ((quotedIdentifier = inQuotedContent) == true) { + for (;;) { + c = read(); + if (c < 0 || c == quote) break; + if (c == '\\') { + c = readEscapedCharacter(this.in, false /*not clob*/); + } + if (c != EMPTY_ESCAPE_SEQUENCE) { + value.appendCodePoint(c); + } + } + if (c == -1) { // TODO throw UnexpectedEofException + throw new IonException("end encountered before closing quote '\\" + (char)endquote+ "'"); + } + // c, at this point is a single quote, which we don't append + inQuotedContent = false; + } + else { + value.append((char)c); + for (;;) { + c = read(); + if (!IonTextUtils.isIdentifierPart(c)) { + break; + } + value.append((char)c); + } + unread(c); // we throw back our terminator here + } + return quotedIdentifier; + } + int readIdentifierContents(int c, int max_length) throws IOException { + assert(c != '\'' && c != '\"'); + + value.append((char)c); + int count = 1; + + while (count < max_length) { + c = read(); + if (!IonTextUtils.isIdentifierPart(c)) { + unread(c); // we throw back our terminator here + break; + } + value.append((char)c); + count++; + } + + return count; + } + + static Type matchKeyword(StringBuilder sb, int pos, int valuelen) throws IOException + { + Type keyword = null; + + switch (sb.charAt(pos++)) { // there has to be at least 1 chacter or we wouldn't be here + case 'f': + if (valuelen == 5 // "f" + && sb.charAt(pos++) == 'a' + && sb.charAt(pos++) == 'l' + && sb.charAt(pos++) == 's' + && sb.charAt(pos++) == 'e' + ) { + keyword = Type.kwFalse; + } + break; + case 'n': + if (valuelen == 4 // 'n' + && sb.charAt(pos++) == 'u' + && sb.charAt(pos++) == 'l' + && sb.charAt(pos++) == 'l' + ) { + keyword = Type.kwNull; + } + else if (valuelen == 3 // 'n' + && sb.charAt(pos++) == 'a' + && sb.charAt(pos++) == 'n' + ) { + keyword = Type.kwNan; + } + break; + case 't': + if (valuelen == 4 // "t" + && sb.charAt(pos++) == 'r' + && sb.charAt(pos++) == 'u' + && sb.charAt(pos++) == 'e' + ) { + keyword = Type.kwTrue; + } + break; + } + + return keyword; + } + + public Type setNullType(StringBuilder sb, int pos, int valuelen) + { + switch (valuelen) { + case 3: + //int + if (sb.charAt(pos++) == 'i' + && sb.charAt(pos++) == 'n' + && sb.charAt(pos++) == 't' + ) { + return Type.kwNullInt; + } + break; + case 4: + //bool, blob, list, null, clob, sexp + switch (sb.charAt(pos++)) { + case 'b': + //bool, blob + switch (sb.charAt(pos++)) { + case 'o': + // bool + if (sb.charAt(pos++) == 'o' + && sb.charAt(pos++) == 'l' + ) { + return Type.kwNullBoolean; + } + break; + case 'l': + //blob + if (sb.charAt(pos++) == 'o' + && sb.charAt(pos++) == 'b' + ) { + return Type.kwNullBlob; + } + break; + } + break; + case 'l': + if (sb.charAt(pos++) == 'i' + && sb.charAt(pos++) == 's' + && sb.charAt(pos++) == 't' + ) { + return Type.kwNullList; + } + break; + case 'n': + if (sb.charAt(pos++) == 'u' + && sb.charAt(pos++) == 'l' + && sb.charAt(pos++) == 'l' + ) { + return Type.kwNullNull; + } + break; + case 'c': + //clob + if (sb.charAt(pos++) == 'l' + && sb.charAt(pos++) == 'o' + && sb.charAt(pos++) == 'b' + ) { + return Type.kwNullClob; + } + break; + case 's': + //sexp + if (sb.charAt(pos++) == 'e' + && sb.charAt(pos++) == 'x' + && sb.charAt(pos++) == 'p' + ) { + return Type.kwNullSexp; + } + break; + } + break; + case 5: + //float + if (sb.charAt(pos++) == 'f' + && sb.charAt(pos++) == 'l' + && sb.charAt(pos++) == 'o' + && sb.charAt(pos++) == 'a' + && sb.charAt(pos++) == 't' + ) { + return Type.kwNullFloat; + } + break; + case 6: + switch (sb.charAt(pos++)) { + case 's': + //string + //struct + //symbol + switch(sb.charAt(pos++)) { + case 't': + if (sb.charAt(pos++) == 'r') { + switch (sb.charAt(pos++)) { + case 'i': + if (sb.charAt(pos++) == 'n' && sb.charAt(pos++) == 'g') { + return Type.kwNullString; + } + break; + case 'u': + if (sb.charAt(pos++) == 'c' && sb.charAt(pos++) == 't') { + return Type.kwNullStruct; + } + break; + } + } + break; + case 'y': + if (sb.charAt(pos++) == 'm' + && sb.charAt(pos++) == 'b' + && sb.charAt(pos++) == 'o' + && sb.charAt(pos++) == 'l' + ) { + return Type.kwNullSymbol; + } + break; + } + } + break; + case 7: + //decimal + if (sb.charAt(pos++) == 'd' + && sb.charAt(pos++) == 'e' + && sb.charAt(pos++) == 'c' + && sb.charAt(pos++) == 'i' + && sb.charAt(pos++) == 'm' + && sb.charAt(pos++) == 'a' + && sb.charAt(pos++) == 'l' + ) { + return Type.kwNullDecimal; + } + break; + case 9: + //timestamp + if (sb.charAt(pos++) == 't' + && sb.charAt(pos++) == 'i' + && sb.charAt(pos++) == 'm' + && sb.charAt(pos++) == 'e' + && sb.charAt(pos++) == 's' + && sb.charAt(pos++) == 't' + && sb.charAt(pos++) == 'a' + && sb.charAt(pos++) == 'm' + && sb.charAt(pos++) == 'p' + ) { + return Type.kwNullTimestamp; + } + break; + } + String nullimage = sb.toString(); + throw new IonException("invalid null value '"+nullimage+"' at " + this.position()); + } + + public Type scanOperator(int c) throws IOException + { + // reset our local value buffer + resetValue(); + + // we don't have an identifier type any longer, just string + this.t = Type.constSymbol; + + // some strings are keywords, most are not + // mostly we'll guess that it's not a keyword + this.keyword = null; + + // first we read the identifier content into our value buffer + // if the value is quoted, use the escape char loop, otherwise + // look for the non-identifier character (and throw it back) + + value.append((char)c); + for (;;) { + c = read(); + if (!IonTextUtils.isOperatorPart(c)) { + break; + } + value.append((char)c); + } + unread(c); // we throw back our terminator here + + return this.t; + } + + + public Type scanString(int c, int maxlookahead) throws IOException + { + // reset out local value buffer + resetValue(); + + if (c != '\"') { + throw new IonException("Programmer error! Only a quote should get you here."); + } + endquote = '\"'; + + sizedloop: + while (maxlookahead-- > 0) { + switch ((c = this.read())) { + case -1: break sizedloop; // TODO deoptimize, throw exception + case '\"': break sizedloop; + case '\n': + throw new IonException("unexpected line terminator encountered in quoted string"); + case '\\': + c = readEscapedCharacter(this.in, false /*not clob*/); + // throws UnexpectedEofException on EOF + if (c != EMPTY_ESCAPE_SEQUENCE) { + value.appendCodePoint(c); + } + break; + default: + // c could be part of a surrogate pair so we can't use + // appendCodePoint() + value.append((char)c); + break; + } + } + + if (maxlookahead != -1 && c == '\"') { + // this is the normal, non-longline case so we're just done + closeString(); + } + else { + // we're not at the closing quote, so this is an incomplete + // string which will have to be read to the end ... later + leaveOpenString(c, false); + } + return Type.constString; + } + void leaveOpenString(int c, boolean longstring) { + if (c == -1) { + throw new UnexpectedEofException(); + } + + this.isIncomplete = true; + this.inQuotedContent = true; + this.isLongString = longstring; + } + + /** + * Scans the rest of the string through {@link #endquote}. + *

    + XXX WARNING XXX + * Almost identical logic is found in + * {@link IonBinary#appendToLongValue(int, boolean, boolean, PushbackReader)} + * + * @param is_in_expression + * @throws UnexpectedEofException if we hit EOF before the endquote. + */ + void finishScanString(boolean is_in_expression) throws IOException + { + assert isIncomplete; + assert inQuotedContent; + + for (;;) + { + int c = this.read(); + if (c == -1) { + throw new UnexpectedEofException(); + } + if (c == endquote) break; // endquote == -1 during long strings + if (c == '\\') { + c = readEscapedCharacter(this.in, false /*not clob*/); + if (c != EMPTY_ESCAPE_SEQUENCE) { + value.appendCodePoint(c); + } + } + else if (c == '\'' && isLongString) { + // This happens while handling triple-quote field names + if (twoMoreSingleQuotes()) { + // At the end of a long string. Look for another one. + inQuotedContent = false; + c = readIgnoreWhitespace(); + + if (c == '\'' && twoMoreSingleQuotes()) { + // Yep, another triple-quote. We've consumed all three + // so just keep reading. + inQuotedContent = true; + } + else { + // Something else, we are done. + unread(c); + break; + } + } + else { + // Still inside the long string + value.append((char)c); + } + } + else if (!isLongString && (c == '\n')) { + throw new IonException("unexpected line terminator encountered in quoted string"); + } + else { + value.append((char)c); + } + } + return; + } + + /** + * If two single quotes are next on the input, consume them and return + * true. Otherwise, leave them on the input and return false. + * @return true when there are two pending single quotes. + * @throws IOException + */ + private boolean twoMoreSingleQuotes() throws IOException + { + int c = read(); + if (c == '\'') { + int c2 = read(); + if (c2 == '\'') { + return true; + } + unread(c2); + } + unread(c); + return false; + } + + + void closeString() { + this.isIncomplete = false; + this.inQuotedContent = false; + this.isLongString = false; + } + + public Type scanLongString() throws IOException + { + // reset out local value buffer + resetValue(); + + endquote = -1; + + // we're not at the closing quote, so this is an incomplete + // string which will have to be read to the end ... later + leaveOpenString(EMPTY_ESCAPE_SEQUENCE, true); + + return Type.constString; + } + /** + * Closes parsing of a triple-quote string, remembering that there may be + * more to come. + */ + void closeLongString() { + this.isIncomplete = true; + this.inQuotedContent = false; + this.isLongString = true; + } + + + /** + * @return the translated escape sequence. Will not return -1 since EOF is + * not legal in this context. May return {@link #EMPTY_ESCAPE_SEQUENCE} to + * indicate a zero-length sequence such as BS-NL. + * @throws UnexpectedEofException if the EOF is encountered in the middle + * of the escape sequence. + * */ + public static int readEscapedCharacter(PushbackReader r, boolean inClob) + throws IOException, UnexpectedEofException + { + // \\ The backslash character + // \0 The character 0 (nul terminator char, for some) + // \xhh The character with hexadecimal value 0xhh + // \ u hhhh The character with hexadecimal value 0xhhhh + // \t The tab character ('\u0009') + // \n The newline (line feed) character ('\ u 000A') + // \v The vertical tab character ('\u000B') + // \r The carriage-return character ('\ u 000D') + // \f The form-feed character ('\ u 000C') + // \a The alert (bell) character ('\ u 0007') + // \" The double quote character + // \' The single quote character + // \? The question mark character + + int c2 = 0, c = r.read(); + switch (c) { + case -1: + throw new UnexpectedEofException(); + case 't': return '\t'; + case 'n': return '\n'; + case 'v': return '\u000B'; + case 'r': return '\r'; + case 'f': return '\f'; + case 'b': return '\u0008'; + case 'a': return '\u0007'; + case '\\': return '\\'; + case '\"': return '\"'; + case '\'': return '\''; + case '/': return '/'; + case '?': return '?'; + + case 'U': + // Expecting 8 hex digits + if (inClob) { + throw new IonException("Unicode escapes \\U not allowed in clob"); + } + c = readDigit(r, 16, true); + if (c < 0) break; // TODO throw UnexpectedEofException + c2 = c << 28; + c = readDigit(r, 16, true); + if (c < 0) break; + c2 += c << 24; + c = readDigit(r, 16, true); + if (c < 0) break; + c2 += c << 20; + c = readDigit(r, 16, true); + if (c < 0) break; + c2 += c << 16; + // ... fall through... + case 'u': + // Expecting 4 hex digits + if (inClob) { + throw new IonException("Unicode escapes \\u not allowed in clob"); + } + c = readDigit(r, 16, true); + if (c < 0) break; + c2 += c << 12; + c = readDigit(r, 16, true); + if (c < 0) break; + c2 += c << 8; + // ... fall through... + case 'x': + // Expecting 2 hex digits + c = readDigit(r, 16, true); + if (c < 0) break; + c2 += c << 4; + c = readDigit(r, 16, true); + if (c < 0) break; + return c2 + c; + + case '0': + return 0; + case '\n': + return EMPTY_ESCAPE_SEQUENCE; + default: + break; + } + throw new IonException("invalid escape sequence \"\\" + + (char) c + "\" [" + c + "]"); + } + + public static int readDigit(/*EscapedCharacterReader*/PushbackReader r, int radix, boolean isRequired) throws IOException { + int c = r.read(); + if (c < 0) { + r.unread(c); + return -1; + } + + int c2 = Character.digit(c, radix); + if (c2 < 0) { + if (isRequired) { + throw new IonException("bad digit in escaped character '"+((char)c)+"'"); + } + // if it's not a required digit, we just throw it back + r.unread(c); + return -1; + } + return c2; + } + + + private void checkAndUnreadNumericStopper(int c) throws IOException + { + // Ignore EOF, so we don't "unread" it - EOF is a terminator + if (c != -1) { + if ( ! this.isValueTerminatingCharacter(c) ) { + final String message = + position() + ": Numeric value followed by invalid character " + + printCodePointAsString(c); + throw new IonException(message); + } + this.unread(c); + } + } + + private final boolean isValueTerminatingCharacter(int c) throws IOException + { + boolean isTerminator; + + if (c == '/') { + // this is terminating only if it starts a comment of some sort + c = this.read(); + this.unread(c); // we never "keep" this character + isTerminator = (c == '/' || c == '*'); + } + else { + isTerminator = IonTextUtils.isNumericStop(c); + } + + return isTerminator; + } + + public Type readNumber(int c) throws IOException { + // clear out our string buffer + resetValue(); + + // process the initial sign character, if its there. + boolean explicitPlusSign = false; + switch (c) { + case '-': + value.append((char)c); + c = this.read(); + t = Type.constNegInt; + this.numberType = NumberType.NT_NEGINT; + this.isNegative = true; + break; + case '+': + // we eat the plus sign, but remember we saw one + explicitPlusSign = true; + c = this.read(); + t = Type.constPosInt; + this.isNegative = false; + this.numberType = NumberType.NT_POSINT; + break; + default: + t = Type.constPosInt; + this.numberType = NumberType.NT_POSINT; + this.isNegative = false; + break; + } + + // process the initial digit. Caller has checked that it's a digit. + assert isDigit(c, 10); + value.append((char)c); + boolean isZero = (c == '0'); + boolean leadingZero = isZero; + + // process the next character after the initial digit. + switch((c = this.read())) { + case 'x': + case 'X': + if (!isZero) { + throw new IonException("badly formed number encountered at " + position()); + } + this.numberType = NumberType.NT_HEX; + // We don't need to append the char because we're going to wipe + // the value anyway to accumulate just the hex digits. + return scanHexNumber(); + case '.': + case 'd': + case 'D': + t = Type.constDecimal; + this.numberType = NumberType.NT_DECIMAL; + value.append((char)c); + break; + case 'e': + case 'E': + t = Type.constFloat; + this.numberType = NumberType.NT_FLOAT; + value.append((char)c); + break; + default: + if (!isDigit(c, 10)) { + checkAndUnreadNumericStopper(c); + if (isZero && NumberType.NT_NEGINT.equals(this.numberType)) { + t = Type.constPosInt; + this.numberType = NumberType.NT_POSINT; + } + return t; + } + value.append((char)c); + isZero &= (c == '0'); + break; + } + + // see if see can continue find leading digits + if (NumberType.NT_NEGINT.equals(numberType) + || NumberType.NT_POSINT.equals(numberType) + ) { + // We've now scanned at least two digits. + // read in the remaining whole number digits + for (;;) { + c = this.read(); + if (!isDigit(c, 10)) break; + value.append((char)c); + isZero &= (c == '0'); + } + + // and see what we ran aground on .. + switch (c) { + case '.': + case 'd': + case 'D': + if (leadingZero) { + throw new IonException(position() + ": Invalid leading zero on numeric"); + } + t = Type.constDecimal; + this.numberType = NumberType.NT_DECIMAL; + value.append((char)c); + break; + case 'e': + case 'E': + if (leadingZero) { + throw new IonException(position() + ": Invalid leading zero on numeric"); + } + t = Type.constFloat; + this.numberType = NumberType.NT_FLOAT; + value.append((char)c); + break; + case 'T': // same as '-' it's a timestamp + case '-': + if (NumberType.NT_POSINT.equals(this.numberType) && !explicitPlusSign) { + return scanTimestamp(c); + } + // otherwise fall through to "all other" processing + default: + // Hit the end of our integer. Make sure its a valid stopper. + checkAndUnreadNumericStopper(c); + + if (leadingZero && !isZero) { + throw new IonException(position() + ": Invalid leading zero on numeric"); + } + if (isZero && NumberType.NT_NEGINT.equals(this.numberType)) { + t = Type.constPosInt; + this.numberType = NumberType.NT_POSINT; + } + return t; + } + } + + // now we must have just decimal or float + // if it's decimal we ended on a DOT, so get the trailing + // (fractional) digits + if (NumberType.NT_DECIMAL.equals(this.numberType)) { + for (;;) { + c = this.read(); + if (!isDigit(c, 10)) break; + value.append((char)c); + } + // and see what we ran aground on this time.. + switch (c) { + case '-': + case '+': + // We'll handle exponent below. + this.unread(c); + break; + case 'e': + case 'E': + // this is the only viable option now + t = Type.constFloat; + this.numberType = NumberType.NT_FLOAT; + value.append((char)c); + break; + case 'd': + case 'D': + // this is the only viable option now + assert t == Type.constDecimal; + assert NumberType.NT_DECIMAL.equals(this.numberType); + value.append((char)c); + break; + default: + checkAndUnreadNumericStopper(c); + return t; + } + } + + // now we have the exponent part to read + // reading the first character of the exponent, it can be + // a sign character or a digit (this is the only place + // the sign is valid) and we're for sure a float at this point + switch ((c = this.read())) { + case '-': + value.append((char)c); + break; + case '+': + // we eat the plus sign + break; + default: + if (!isDigit(c, 10)) { + // this they said 'e' then they better put something valid after it! + throw new IonException("badly formed number encountered at " + position()); + } + this.unread(c); + // we'll re-read this in just a bit, but we had to check + break; + } + + // read in the remaining whole number digits and then quit + for (;;) { + c = this.read(); + if (!isDigit(c, 10)) break; + value.append((char)c); + } + + checkAndUnreadNumericStopper(c); + + return t; + } + + Type scanHexNumber() throws IOException + { + boolean anydigits = false; + int c; + + boolean isZero = true; + + // Leave the 0x in the buffer since we'll need it to parse the content. + // first get clear out our string buffer, we're starting over + value.setLength(0); // resetValue() wipes out too much + + // read in the remaining whole number digits and then quit + for (;;) { + c = this.read(); + if (!isDigit(c, 16)) break; + anydigits = true; + isZero &= (c == '0'); + value.append((char)c); + } + if (!anydigits) { + throw new IonException("badly formed hexadecimal number encountered at " + position()); + } + + checkAndUnreadNumericStopper(c); + if (isZero && t == Type.constNegInt) { + t = Type.constPosInt; + } + + // as long as we saw any digit, we're good + return t; + } + + /** + * Scans a timestamp after reading yyyy-. + * + * We can be a little lenient here since the result will be reparsed and + * validated more thoroughly by {@link Timestamp#valueOf(CharSequence)}. + * + * @param c the last character scanned; must be '-'. + * @return {@link Type#constTime} + */ + Type scanTimestamp(int c) throws IOException { + +endofdate: + for (;;) { // fake for loop to create a label we can jump out of, + // because 4 or 5 levels of nested if's is just ugly + + // at this point we will have read leading digits and exactly 1 dash + // in other words, we'll have read the year + if (c == 'T') { + // yearT is a valid timestamp value + value.append((char)c); + c = this.read(); // because we'll unread it before we return + break endofdate; + } + if (c != '-') { + // not a dash or a T after the year - so this is a bad value + throw new IllegalStateException("invalid timestamp, expecting a dash here at " + this.position()); + } + + // append the dash and then read the month field + value.append((char)c); // so append it, because we haven't already + c = readDigits(2, "month"); + if (c == 'T') { + // year-monthT is a valid timestamp value + value.append((char)c); + c = this.read(); // because we'll unread it before we return + break endofdate; + } + if (c != '-') { + // if the month isn't followed by a dash or a T it's an invalid month + throw new IonException("invalid timestamp, expecting month at " + this.position()); + } + + // append the dash and read the day (or day-of-month) field + value.append((char)c); + c = readDigits(2, "day of month"); + if (c == 'T') { + +check4timezone: + for (;;) { // another fake label/ for=goto + + // attach the 'T' to the value we're collecting + value.append((char)c); + + // we're going to "watch" how many digits we read in the hours + // field. It's 0 that's actually ok, since we can end at the + // 'T' we just read + int length_before_reading_hours = value.length(); + // so read the hours + c = readDigits(2, "hours"); + if (length_before_reading_hours == value.length()) { + // FIXME I don't think there should be a timezone here + break check4timezone; + } + if (c != ':') { + throw new IonException("invalid timestamp, expecting hours at " + this.position()); + } + value.append((char)c); + // so read the minutes + c = readDigits(2, "minutes"); + if (c != ':') { + if (c == '-' || c == '+' || c == 'Z') { + break check4timezone; + } + break endofdate; + } + value.append((char)c); + // so read the seconds + c = readDigits(2, "seconds"); + if (c != '.') { + if (c == '-' || c == '+' || c == 'Z') { + break check4timezone; + } + break endofdate; + } + value.append((char)c); + // so read the fractional seconds + c = readDigits(32, "fractional seconds"); + break check4timezone; + }//check4timezone + + // now check to see if it's a timezone offset we're looking at + if (c == '-' || c == '+') { + value.append((char)c); + // so read the timezone offset + c = readDigits(2, "timezone offset"); + if (c != ':') break endofdate; + value.append((char)c); + c = readDigits(2, "timezone offset"); + } + else if (c == 'Z') { + value.append((char)c); + c = this.read(); // because we'll unread it before we return + } + } + break endofdate; + }//endofdate + + checkAndUnreadNumericStopper(c); + + return Type.constTime; + } + + int readDigits(int limit, String field) throws IOException { + int c, len = 0; + // read in the remaining whole number digits and then quit + for (;;) { + c = this.read(); + if (!isDigit(c, 10)) break; + len++; + if (len > limit) { + throw new IonException("invalid format " + field + " too long at " + this.position()); + } + value.append((char)c); + } + return c; + } + + public boolean makeValidNumeric(Type castto) throws IOException { + String s = getValueString(false); + + try + { + this.t = castto.setNumericValue(this, s); + } + catch (NumberFormatException e) + { + throw new IonException(this.position() + ": invalid numeric value " + s, e); + } + + return this.t.isNumeric(); + } + +} diff --git a/src/software/amazon/ion/impl/IonUTF8.java b/src/com/amazon/ion/impl/IonUTF8.java similarity index 98% rename from src/software/amazon/ion/impl/IonUTF8.java rename to src/com/amazon/ion/impl/IonUTF8.java index 14d12ebfcc..4b1c718bc0 100644 --- a/src/software/amazon/ion/impl/IonUTF8.java +++ b/src/com/amazon/ion/impl/IonUTF8.java @@ -1,23 +1,24 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; +import com.amazon.ion.IonException; import java.io.Closeable; import java.io.IOException; import java.io.OutputStream; -import software.amazon.ion.IonException; /** * this class holds the various constants and helper functions Ion uses diff --git a/src/software/amazon/ion/impl/IonWriterSystem.java b/src/com/amazon/ion/impl/IonWriterSystem.java similarity index 79% rename from src/software/amazon/ion/impl/IonWriterSystem.java rename to src/com/amazon/ion/impl/IonWriterSystem.java index 6237a7c58a..02890c0a85 100644 --- a/src/software/amazon/ion/impl/IonWriterSystem.java +++ b/src/com/amazon/ion/impl/IonWriterSystem.java @@ -1,35 +1,38 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; -import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; -import static software.amazon.ion.impl.PrivateUtils.newSymbolToken; -import static software.amazon.ion.impl.PrivateUtils.newSymbolTokens; +import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; +import static com.amazon.ion.impl._Private_Utils.newSymbolToken; +import static com.amazon.ion.impl._Private_Utils.newSymbolTokens; +import com.amazon.ion.IonException; +import com.amazon.ion.IonType; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.SystemSymbols; +import com.amazon.ion.UnknownSymbolException; +import com.amazon.ion.system.IonWriterBuilder.InitialIvmHandling; +import com.amazon.ion.system.IonWriterBuilder.IvmMinimizing; import java.io.IOException; -import software.amazon.ion.IonType; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.SystemSymbols; -import software.amazon.ion.UnknownSymbolException; -import software.amazon.ion.system.IonWriterBuilder.InitialIvmHandling; -import software.amazon.ion.system.IonWriterBuilder.IvmMinimizing; + abstract class IonWriterSystem - extends PrivateIonWriterBase + extends _Private_IonWriterBase { /** * The system symtab used when resetting the stream. @@ -119,7 +122,7 @@ public final SymbolTable getSymbolTable() public final void setSymbolTable(SymbolTable symbols) throws IOException { - if (symbols == null || PrivateUtils.symtabIsSharedNotSystem(symbols)) { + if (symbols == null || _Private_Utils.symtabIsSharedNotSystem(symbols)) { throw new IllegalArgumentException("symbol table must be local or system to be set, or reset"); } if (getDepth() > 0) { @@ -136,20 +139,20 @@ boolean shouldWriteIvm() } if (_initial_ivm_handling == InitialIvmHandling.SUPPRESS) { - // TODO amzn/ion-java#24 Must write IVM if given system != 1.0 + // TODO amzn/ion-java/issues/24 Must write IVM if given system != 1.0 return false; } - // TODO amzn/ion-java#24 Add SUPPRESS_ALL to suppress non 1.0 IVMs + // TODO amzn/ion-java/issues/24 Add SUPPRESS_ALL to suppress non 1.0 IVMs if (_ivm_minimizing == IvmMinimizing.ADJACENT) { - // TODO amzn/ion-java#24 Write IVM if current system version != given system + // TODO amzn/ion-java/issues/24 Write IVM if current system version != given system // For now we assume that it's the same since we only support 1.0 return ! _previous_value_was_ivm; } if (_ivm_minimizing == IvmMinimizing.DISTANT) { - // TODO amzn/ion-java#24 Write IVM if current system version != given system + // TODO amzn/ion-java/issues/24 Write IVM if current system version != given system // For now we assume that it's the same since we only support 1.0 return ! _anything_written; } @@ -288,7 +291,7 @@ final void writeSymbol(int symbolId) throws IOException && getDepth() == 0 && _annotation_count == 0) { // $ion_1_0 is written as an IVM only if it is not annotated - // TODO amzn/ion-java#24 Make sure to get the right symtab, default may differ. + // TODO amzn/ion-java/issues/24 Make sure to get the right symtab, default may differ. writeIonVersionMarker(); } else @@ -303,7 +306,7 @@ public final void writeSymbol(String value) throws IOException && getDepth() == 0 && _annotation_count == 0) { // $ion_1_0 is written as an IVM only if it is not annotated - // TODO amzn/ion-java#24 Make sure to get the right symtab, default may differ. + // TODO amzn/ion-java/issues/24 Make sure to get the right symtab, default may differ. writeIonVersionMarker(); } else { @@ -343,7 +346,7 @@ public final boolean isFieldNameSet() case STRING: return _field_name != null; case INT: - return _field_name_sid > 0; + return _field_name_sid >= 0; default: break; } @@ -399,6 +402,40 @@ public final void setFieldNameSymbol(SymbolToken name) } } + /** + * Returns the symbol id of the current field name, if the field name + * has been set. If the name has not been set, either as either a String + * or a symbol id value, this returns -1 (undefined symbol). + * @return symbol id of the name of the field about to be written or -1 if + * it is not set + */ + final int getFieldId() + { + int id; + + if (_field_name_type == null) { + throw new IllegalStateException("the field has not be set"); + } + switch (_field_name_type) { + case STRING: + try { + id = add_symbol(_field_name); + } + catch (IOException e) { + throw new IonException(e); + } + // TODO cache the sid? + break; + case INT: + id = _field_name_sid; + break; + default: + throw new IllegalStateException("the field has not be set"); + } + + return id; + } + final SymbolToken assumeFieldNameSymbol() { if (_field_name_type == null) { @@ -435,11 +472,38 @@ final void ensureAnnotationCapacity(int length) { } + final int[] internAnnotationsAndGetSids() throws IOException + { + int count = _annotation_count; + if (count == 0) return _Private_Utils.EMPTY_INT_ARRAY; + + int[] sids = new int[count]; + for (int i = 0; i < count; i++) + { + SymbolToken sym = _annotations[i]; + int sid = sym.getSid(); + if (sid == UNKNOWN_SYMBOL_ID) + { + String text = sym.getText(); + sid = add_symbol(text); + _annotations[i] = new SymbolTokenImpl(text, sid); + } + sids[i] = sid; + } + return sids; + } + + final boolean hasAnnotations() { return _annotation_count != 0; } + final int annotationCount() + { + return _annotation_count; + } + final void clearAnnotations() { _annotation_count = 0; @@ -487,7 +551,7 @@ public final void setTypeAnnotationSymbols(SymbolToken... annotations) if (sym.getText() == null) { validateSymbolId(sym.getSid()); } - sym = PrivateUtils.localize(symtab, sym); + sym = _Private_Utils.localize(symtab, sym); _annotations[i] = sym; } _annotation_count = count; @@ -497,7 +561,7 @@ public final void setTypeAnnotationSymbols(SymbolToken... annotations) @Override final String[] getTypeAnnotations() { - return PrivateUtils.toStrings(_annotations, _annotation_count); + return _Private_Utils.toStrings(_annotations, _annotation_count); } public final void setTypeAnnotations(String... annotations) @@ -529,6 +593,12 @@ public final void addTypeAnnotation(String annotation) @Override final int[] getTypeAnnotationIds() { - return PrivateUtils.toSids(_annotations, _annotation_count); + return _Private_Utils.toSids(_annotations, _annotation_count); + } + + public T asFacet(Class facetType) + { + // This implementation has no facets. + return null; } } diff --git a/src/com/amazon/ion/impl/IonWriterSystemBinary.java b/src/com/amazon/ion/impl/IonWriterSystemBinary.java new file mode 100644 index 0000000000..51073cfb31 --- /dev/null +++ b/src/com/amazon/ion/impl/IonWriterSystemBinary.java @@ -0,0 +1,1050 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import static com.amazon.ion.impl._Private_IonConstants.tidDATAGRAM; +import static com.amazon.ion.impl._Private_IonConstants.tidList; +import static com.amazon.ion.impl._Private_IonConstants.tidSexp; +import static com.amazon.ion.impl._Private_IonConstants.tidStruct; +import static com.amazon.ion.impl.lite._Private_LiteDomTrampoline.reverseEncode; + +import com.amazon.ion.IonException; +import com.amazon.ion.IonType; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.Timestamp; +import com.amazon.ion.impl.BlockedBuffer.BlockedByteInputStream; +import com.amazon.ion.impl.IonBinary.BufferManager; +import com.amazon.ion.system.IonWriterBuilder.InitialIvmHandling; +import com.amazon.ion.system.IonWriterBuilder.IvmMinimizing; +import java.io.IOException; +import java.io.OutputStream; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.LinkedList; +import java.util.Queue; + + +final class IonWriterSystemBinary + extends IonWriterSystem + implements _Private_ListWriter +{ + + /** + * Implements patching of the internal OutputStream with actual value + * lengths and types. Every instance of this class represents a container + * (including the datagram). Information about the children of the container + * is stored in _types/_positions/_lengths arrays. + * + * The patching mechanism works as follows. For every primitive value, we store + * its type, position in internal OutputStream and the serialized lengths + * in this class. With every value insertion, its length is then summed to the + * length of the @_parent. Upon seeing a container, a new PatchedValue is created + * and inserted into @_children queue, then all subsequent values are written + * as described above. As the container is finished, its @_parent becomes active + * and values are continued to be written. In the end, a general tree is created + * with information about all the values in the datagram. The values themselves + * are serialized into internal OutputStream + * + * To finalize the internal OutputStream into a user stream, the arrays in + * PatchedValues are traversed from the beginning. For every value in + * in @_types[i], it's type descriptor is written into user stream. If the type + * is primitive, content of the internal OutputStream is copied starting + * from @_positions[i] with length @_lengths[i]. If @_types[i] is a container, + * a child is popped from the @_children queue and recursively written as + * described above. Whenever TID_SYMBOL_TABLE_PATCH is seen in @_types, the + * current symbol table gets reset with the one on the top of @_symtabs queue. + */ + static class PatchedValues { + + private final static int DEFAULT_PATCH_COUNT = 10; + + /** first free position in the _types/_positions/_lengths arrays */ + int _freePos; + /** types of the container */ + int[] _types; + /** positions where values start in buffer */ + int[] _positions; + /** + * lengths of the values. The high 32bits of this value store the length + * of the field name, the low 32bits store the actual value length. We need the + * field name length to properly adjust the total @_parent length in @endPatch + */ + long[] _lengths; + // the parent + PatchedValues _parent; + Queue _children; + Queue _symtabs; + + PatchedValues() { + _freePos = -1; + _types = new int[DEFAULT_PATCH_COUNT]; + _positions = new int[DEFAULT_PATCH_COUNT]; + _lengths = new long[DEFAULT_PATCH_COUNT]; + } + + void reset() { + _freePos = -1; + _children = null; + _symtabs = null; + } + + /** + * Add a new PatchedValues instance to _children and return it + */ + PatchedValues addChild() { + PatchedValues pv = new PatchedValues(); + pv._parent = this; + if (_children == null) { + _children = new LinkedList(); + } + _children.add(pv); + return pv; + } + + /** + * Inject a symbol table at the specified position in internal OutputStream. Most of the + * times, the injection will happen upon seeing the first non-system symbol. At that point + * the information about the symbol is already stored here by @startPatch call, so we need + * to be able to inject a symtab before this value (injectBeforeCurrent) + * + * @param st + * @param injectBeforeCurrent Flags if symbol table must be injected before the current value + * which essentially shifts the array by 1 + * @throws IllegalStateException if the PatchedValues is not a top-level one + */ + void injectSymbolTable(SymbolTable st, boolean injectBeforeCurrent) { + if (_parent != null) { + // we're not on top-level + throw new IllegalStateException("Cannot inject a symbol table when not on top-level"); + } + if (_symtabs == null) { + _symtabs = new LinkedList(); + } + ++_freePos; + if (_freePos == _positions.length) { + grow(); + } + if (injectBeforeCurrent) { + // move current value to the right + _types[_freePos] = _types[_freePos - 1]; + _lengths[_freePos] = _lengths[_freePos - 1]; + // set previous value to symbol table type + _types[_freePos - 1] = TID_SYMBOL_TABLE_PATCH; + _lengths[_freePos - 1] = 0; + } else { + // add symbol table to the next free position as a usual value + _types[_freePos] = TID_SYMBOL_TABLE_PATCH; + _lengths[_freePos] = 0; + } + // inject the symbol table to the current position on the top + _symtabs.add(st); + } + + int getType() { + return _types[_freePos]; + } + + PatchedValues getParent() { + return _parent; + } + + /** Start the patch for given @type at given @pos */ + void startPatch(int type, int pos) { + ++_freePos; + if (_freePos == _positions.length) { + grow(); + } + _types[_freePos] = type; + _lengths[_freePos] = 0; + _positions[_freePos] = pos; + } + + /** Set the field name length as high 32bits of @_lengths item */ + void patchFieldName(int fieldNameLength) { + _lengths[_freePos] = ((long)fieldNameLength) << 32; + } + + /** + * Set the value length as low 32bits of @_lengths item. The low 32bits will be + * summed with the given value (say, it's a list that is in struct) + * @param len + */ + void patchValue(int len) { + long memLen = (_lengths[_freePos] & 0xFFFFFFFF00000000L); + long curLen = (_lengths[_freePos] & 0x00000000FFFFFFFFL); + _lengths[_freePos] = memLen | (curLen + len); + } + + /** + * End this patch. If there's a @_parent available, it's @_length will be summed + * with the length of current patched value + */ + void endPatch() { + if (_parent != null) { + int memberLen = (int)(_lengths[_freePos] >> 32); + int valueLen = (int)(_lengths[_freePos] & 0xFFFFFFFF); + int totalLen = memberLen + valueLen; + switch (_types[_freePos]) { + case TID_SYMBOL_TABLE_PATCH: + case TID_RAW: + break; + case TID_ANNOTATION_PATCH: + totalLen += IonBinary.lenVarUInt(valueLen); + break; + default: + // add the type if it's specified + ++totalLen; + // add actual length of @totalLen :) + if (valueLen >= _Private_IonConstants.lnIsVarLen) { + totalLen += IonBinary.lenVarUInt(valueLen); + } + } + _parent.patchValue(totalLen); + } + } + + // grow all related arrays + private void grow() { + int newSize = _positions.length * 2; + _types = growOne(_types, newSize); + _positions = growOne(_positions, newSize); + _lengths = growOne(_lengths, newSize); + } + // grow single array and return the result + static int[] growOne(int[] source, int newSize) { + int[] dest = new int[newSize]; + System.arraycopy(source, 0, dest, 0, source.length); + return dest; + } + // grow single array and return the result + static long[] growOne(long[] source, int newSize) { + long[] dest = new long[newSize]; + System.arraycopy(source, 0, dest, 0, source.length); + return dest; + } + } + + // top-level patch + PatchedValues _patch = new PatchedValues(); + + private final static int TID_ANNOTATION_PATCH = _Private_IonConstants.tidDATAGRAM + 1; + private final static int TID_SYMBOL_TABLE_PATCH = _Private_IonConstants.tidDATAGRAM + 2; + private final static int TID_RAW = _Private_IonConstants.tidDATAGRAM + 3; + + // private static final boolean _verbose_debug = false; + + static final int UNKNOWN_LENGTH = -1; + + BufferManager _manager; + IonBinary.Writer _writer; + + /** Not null */ + private final OutputStream _user_output_stream; + + /** + * Do we {@link #flush()} after each top-level value? + * @see #closeValue() + */ + private final boolean _auto_flush; + + boolean _in_struct; + + /** Ensure we don't use a closed {@link #output} stream. */ + private boolean _closed; + + private final static int TID_FOR_SYMBOL_TABLE_PATCH = _Private_IonConstants.tidDATAGRAM + 1; + private final static int DEFAULT_PATCH_COUNT = 10; + private final static int DEFAULT_PATCH_DEPTH = 10; + private final static int NOT_A_SYMBOL_TABLE_IDX = -1; + + // Patch: + // offset in data stream + // accumulated length -- combine w offset in long? offset:len (allows len+=more) + // type of data (should this be in the data stream? + // patched value's parent is struct flag (low nibble in data stream?) + // + // the patches are the accumulated list of patch points and are + // in position order (which is conveniently the order they are + // encountered and created). + int _patch_count = 0; + int [] _patch_lengths = new int[DEFAULT_PATCH_COUNT]; // TODO: should these be merged? (since array access is expensive) + int [] _patch_offsets = new int[DEFAULT_PATCH_COUNT]; // should patch lengths and patch offsets be longs? + int [] _patch_table_idx = new int[DEFAULT_PATCH_COUNT]; + int [] _patch_types = new int[DEFAULT_PATCH_COUNT]; + boolean [] _patch_in_struct = new boolean[DEFAULT_PATCH_COUNT]; + + // this is only loaded by the User writer, but it is read + // by the "get byte" operations and must be coordinated + // with the patch list that the system writer maintains here. + int _patch_symbol_table_count = 0; + SymbolTable[] _patch_symbol_tables = new SymbolTable[DEFAULT_PATCH_COUNT]; + + // the patch stack is the list of patch points that currently + // need updating. The value is the index into the patch arrays. + // As a value requiring patches is closed its patch idx is removed + // from the stack. + int _top; + int [] _patch_stack = new int[DEFAULT_PATCH_DEPTH]; + + /** + * This is the depth as seen by the user. Since there are cases where we + * don't push onto the patch stack and cases where we push non-user + * containers onto the patch stack we compute this separately during + * stepIn and stepOut. + */ + private int _user_depth; + + + /** + * @param out OutputStream the users output byte stream, if specified + * @param autoFlush when true the writer flushes to the output stream + * between top level values + * @param ensureInitialIvm when true, an initial IVM will be emitted even + * when the user doesn't explicitly write one. When false, an initial IVM + * won't be emitted unless the user does it. That can result in an invalid + * Ion stream if not used carefully. + * @throws NullPointerException if any parameter is null. + */ + IonWriterSystemBinary(SymbolTable defaultSystemSymtab, + OutputStream out, + boolean autoFlush, + boolean ensureInitialIvm) + { + super(defaultSystemSymtab, + (ensureInitialIvm ? InitialIvmHandling.ENSURE : null), + IvmMinimizing.ADJACENT); + + out.getClass(); // Efficient null check + _user_output_stream = out; + + // the buffer manager and writer + // are used to hold the buffered + // binary values pending flush(). + _manager = new BufferManager(); + _writer = _manager.openWriter(); + _auto_flush = autoFlush; + } + + /** + * Empty our buffers, assuming it is safe to do so. + * This is called by {@link #flush()} and {@link #finish()}. + */ + private void writeAllBufferedData() + throws IOException + { + writeBytes(_user_output_stream); + + clearFieldName(); + clearAnnotations(); + + _in_struct = false; + _patch_count = 0; + _patch_symbol_table_count = 0; + _top = 0; + try { + _writer.setPosition(0); + _writer.truncate(); + } + catch (IOException e) { + throw new IonException(e); + } + } + + @Override + public void finish() throws IOException + { + if (getDepth() != 0) { + throw new IllegalStateException(ERROR_FINISH_NOT_AT_TOP_LEVEL); + } + + writeAllBufferedData(); + super.finish(); + } + + final OutputStream getOutputStream() + { + return _user_output_stream; + } + + public final boolean isInStruct() + { + return _in_struct; + } + + private final boolean topInStruct() { + if (_top == 0) return false; + boolean in_struct = _patch_in_struct[_patch_stack[_top - 1]]; + return in_struct; + } + protected final boolean atDatagramLevel() + { + return (topType() == _Private_IonConstants.tidDATAGRAM); +// return is_datagram; + } + + @Override + public final int getDepth() + { + return _user_depth; + } + + protected final IonType getContainer() + { + IonType type; + int tid = parentType(); + switch (tid) { + case tidList: + type = IonType.LIST; + break; + case tidSexp: + type = IonType.SEXP; + break; + case tidStruct: + type = IonType.STRUCT; + break; + case tidDATAGRAM: + type = IonType.DATAGRAM; + break; + default: + throw new IonException("unexpected parent type "+tid+" does not represent a container"); + } + return type; + } + + @Override + final SymbolTable inject_local_symbol_table() throws IOException + { + SymbolTable symbols = super.inject_local_symbol_table(); + PatchedValues top; + // find the parent + for (top = _patch; top.getParent() != null; top = top.getParent()) {} + // inject the symbol table; if @_patch is not the top element, then inject + // the symtab before this top-level value begins + super.startValue(); + top.injectSymbolTable(symbols, _patch.getParent() != null); + super.endValue(); + return symbols; + } + + private final int topLength() { + return _patch_lengths[_patch_stack[_top - 1]]; + } + + private final int topType() { + if (_top == 0) return _Private_IonConstants.tidDATAGRAM; + return _patch_types[_patch_stack[_top - 1]]; + } + + private final int parentType() { + int ii = _top - 2; + while (ii >= 0) { + int type = _patch_types[_patch_stack[ii]]; + if (type != _Private_IonConstants.tidTypedecl) return type; + ii--; + } + return _Private_IonConstants.tidDATAGRAM; + } + + private final void startValue(int value_type) + throws IOException + { + super.startValue(); + + int[] sids = null; + int sid_count = annotationCount(); + if (sid_count > 0) { + // prepare the SIDs of annotations before doing the patch as this may + // fail and leave the Writer in undefined state + sids = super.internAnnotationsAndGetSids(); + _patch.startPatch(_Private_IonConstants.tidTypedecl, _writer.position()); + } else { + _patch.startPatch(value_type, _writer.position()); + } + // write field name + if (_in_struct) { + if (!isFieldNameSet()) { + throw new IllegalStateException(ERROR_MISSING_FIELD_NAME); + } + int sid = super.getFieldId(); + if (sid < 0) { + throw new UnsupportedOperationException("symbol resolution must be handled by the user writer"); + } + int fieldNameLength = _writer.writeVarUIntValue(sid, true); + _patch.patchFieldName(fieldNameLength); + clearFieldName(); + } + + // write annotations + if (sid_count > 0) { + _patch = _patch.addChild(); + // add all annotations as if it's a value with type = -1 + _patch.startPatch(TID_ANNOTATION_PATCH, _writer.position()); + int len = 0; + for (int ii=0; ii + * The {@link OutputStream} spec is mum regarding the behavior of flush on + * a closed stream, so we shouldn't assume that our stream can handle that. + */ + public final void flush() throws IOException + { + if (! _closed) + { + if (atDatagramLevel() && ! hasAnnotations()) + { + SymbolTable symtab = getSymbolTable(); + + if (symtab != null && + symtab.isReadOnly() && + symtab.isLocalTable()) + { + // It's no longer possible to add more symbols to the local + // symtab, so we can safely write everything out. + writeAllBufferedData(); + } + } + + _user_output_stream.flush(); + } + } + + public final void close() throws IOException + { + if (! _closed) { + try + { + if (getDepth() == 0) { + finish(); + } + } + finally + { + // Do this first so we are closed even if the call below throws. + _closed = true; + + _user_output_stream.close(); + } + } + } + + + + @Override + void writeIonVersionMarkerAsIs(SymbolTable systemSymtab) + throws IOException + { + if (_user_depth != 0) { + throw new IllegalStateException("IVM not on top-level"); + } + super.startValue(); + _patch.startPatch(TID_RAW, _writer.position()); + _patch.patchValue(4); + _writer.write(_Private_IonConstants.BINARY_VERSION_MARKER_1_0); + _patch.endPatch(); + super.endValue(); + } + + @Override + void writeLocalSymtab(SymbolTable symbols) + throws IOException + { + // this method *should* be called when @_patch is a top-level value, but + // we cannot be sure, so try to find the top-level anyway + PatchedValues top; + // find the parent + for (top = _patch; top.getParent() != null; top = top.getParent()) {} + super.startValue(); + top.injectSymbolTable(symbols, _patch.getParent() != null); + super.endValue(); + super.writeLocalSymtab(symbols); + } + + public final void stepIn(IonType containerType) throws IOException + { + int tid; + switch (containerType) + { + case LIST: tid = tidList; break; + case SEXP: tid = tidSexp; break; + case STRUCT: tid = tidStruct; break; + default: + throw new IllegalArgumentException(); + } + startValue(tid); + _patch = _patch.addChild(); + _in_struct = (tid == tidStruct); + ++_user_depth; + } + + public final void stepOut() throws IOException + { + if (_patch.getParent() == null) { + throw new IllegalStateException(IonMessages.CANNOT_STEP_OUT); + } + // container is over, getting out + _patch = _patch.getParent(); + // now close current value + closeValue(); + if (_patch.getParent() == null) { + _in_struct = false; + // we're on top-level + if (_auto_flush) { + flush(); + } + } else { + _in_struct = (_patch.getParent().getType() == _Private_IonConstants.tidStruct); + } + --_user_depth; + } + + + public void writeNull(IonType type) throws IOException + { + int tid; + switch (type) { + case NULL: tid = _Private_IonConstants.tidNull; break; + case BOOL: tid = _Private_IonConstants.tidBoolean; break; + case INT: tid = _Private_IonConstants.tidPosInt; break; + case FLOAT: tid = _Private_IonConstants.tidFloat; break; + case DECIMAL: tid = _Private_IonConstants.tidDecimal; break; + case TIMESTAMP: tid = _Private_IonConstants.tidTimestamp; break; + case SYMBOL: tid = _Private_IonConstants.tidSymbol; break; + case STRING: tid = _Private_IonConstants.tidString; break; + case BLOB: tid = _Private_IonConstants.tidBlob; break; + case CLOB: tid = _Private_IonConstants.tidClob; break; + case SEXP: tid = _Private_IonConstants.tidSexp; break; + case LIST: tid = _Private_IonConstants.tidList; break; + case STRUCT: tid = _Private_IonConstants.tidStruct; break; + default: + throw new IllegalArgumentException("Invalid type: " + type); + } + startValue(TID_RAW); + _writer.write((tid << 4) | _Private_IonConstants.lnIsNull); + _patch.patchValue(1); + closeValue(); + } + + public void writeBool(boolean value) throws IOException + { + int ln = value ? _Private_IonConstants.lnBooleanTrue : _Private_IonConstants.lnBooleanFalse; + startValue(TID_RAW); + _writer.write((_Private_IonConstants.tidBoolean << 4) | ln); + _patch.patchValue(1); + closeValue(); + } + public void writeInt(long value) throws IOException + { + int len; + if (value < 0) { + startValue(_Private_IonConstants.tidNegInt); + len = _writer.writeUIntValue(-value); + } else { + startValue(_Private_IonConstants.tidPosInt); + len = _writer.writeUIntValue(value); + } + _patch.patchValue(len); + closeValue(); + } + + public void writeInt(BigInteger value) throws IOException + { + if (value == null) { + writeNull(IonType.INT); + return; + } + + boolean is_negative = (value.signum() < 0); + BigInteger positive = value; + + if (is_negative) { + positive = value.negate(); + } + + int len = IonBinary.lenIonInt(positive); + startValue(is_negative ? _Private_IonConstants.tidNegInt : _Private_IonConstants.tidPosInt); + _writer.writeUIntValue(positive, len); + _patch.patchValue(len); + + closeValue(); + } + + public void writeFloat(double value) throws IOException + { + int len = IonBinary.lenIonFloat(value); + startValue(_Private_IonConstants.tidFloat); // int's are always less than varlen long + len = _writer.writeFloatValue(value); + _patch.patchValue(len); + closeValue(); + } + + @Override + public void writeDecimal(BigDecimal value) throws IOException + { + if (value == null) { + writeNull(IonType.DECIMAL); + return; + } + startValue(_Private_IonConstants.tidDecimal); + int len = _writer.writeDecimalContent(value); + _patch.patchValue(len); + closeValue(); + } + + public void writeTimestamp(Timestamp value) throws IOException + { + if (value == null) { + writeNull(IonType.TIMESTAMP); + return; + } + startValue(_Private_IonConstants.tidTimestamp); + int len = _writer.writeTimestamp(value); + _patch.patchValue(len); + closeValue(); + } + + public void writeString(String value) throws IOException + { + if (value == null) { + writeNull(IonType.STRING); + return; + } + startValue(_Private_IonConstants.tidString); + int len = _writer.writeStringData(value); + _patch.patchValue(len); + closeValue(); + } + + @Override + void writeSymbolAsIs(int symbolId) throws IOException + { + startValue(_Private_IonConstants.tidSymbol); + int len = _writer.writeUIntValue(symbolId); + _patch.patchValue(len); + closeValue(); + } + + @Override + public void writeSymbolAsIs(String value) throws IOException + { + if (value == null) { + writeNull(IonType.SYMBOL); + return; + } + int sid = add_symbol(value); + writeSymbolAsIs(sid); + } + + public void writeClob(byte[] value, int start, int len) throws IOException + { + if (value == null) { + writeNull(IonType.CLOB); + return; + } + if (start < 0 || len < 0 || start+len > value.length) { + throw new IllegalArgumentException("the start and len must be contained in the byte array"); + } + startValue(_Private_IonConstants.tidClob); + _writer.write(value, start, len); + _patch.patchValue(len); + closeValue(); + } + + public void writeBlob(byte[] value, int start, int len) throws IOException + { + if (value == null) { + writeNull(IonType.BLOB); + return; + } + if (start < 0 || len < 0 || start+len > value.length) { + throw new IllegalArgumentException("the start and len must be contained in the byte array"); + } + startValue(_Private_IonConstants.tidBlob); + _writer.write(value, start, len); + _patch.patchValue(len); + closeValue(); + } + + // just transfer the bytes into the current patch as 'proper' ion binary serialization + public void writeRaw(byte[] value, int start, int len) throws IOException + { + startValue(TID_RAW); + _writer.write(value, start, len); + _patch.patchValue(len); + closeValue(); + } + + public void writeBoolList(boolean[] values) throws IOException + { + stepIn(IonType.LIST); + for (boolean b : values) { + writeBool(b); + } + stepOut(); + } + + public void writeIntList(byte[] values) throws IOException + { + stepIn(IonType.LIST); + for (byte b : values) { + writeInt(b); + } + stepOut(); + } + + public void writeIntList(short[] values) throws IOException + { + stepIn(IonType.LIST); + for (short s : values) { + writeInt(s); + } + stepOut(); + } + + public void writeIntList(int[] values) throws IOException + { + stepIn(IonType.LIST); + for (int i : values) { + writeInt(i); + } + stepOut(); + } + + public void writeIntList(long[] values) throws IOException + { + stepIn(IonType.LIST); + for (long l : values) { + writeInt(l); + } + stepOut(); + } + + public void writeFloatList(float[] values) throws IOException + { + stepIn(IonType.LIST); + for (float f : values) { + writeFloat(f); + } + stepOut(); + } + + public void writeFloatList(double[] values) throws IOException + { + stepIn(IonType.LIST); + for (double d : values) { + writeFloat(d); + } + stepOut(); + } + + public void writeStringList(String[] values) throws IOException + { + stepIn(IonType.LIST); + for (String s : values) { + writeString(s); + } + stepOut(); + } + + // TODO make private after IonBinaryWriter is removed + /** + * Writes everything we've got into the output stream, performing all + * necessary patches along the way. + * + * This implements {@link com.amazon.ion.IonBinaryWriter#writeBytes(OutputStream)} + * via our subclass {@link IonWriterBinaryCompatibility.System}. + */ + int writeBytes(OutputStream userstream) throws IOException { + if (_patch.getParent() != null) { + throw new IllegalStateException("Tried to flush while not on top-level"); + } + + try { + BlockedByteInputStream datastream = + new BlockedByteInputStream(_manager.buffer()); + int size = writeRecursive(datastream, userstream, _patch); + return size; + } finally { + _patch.reset(); + } + } + + int writeRecursive(BlockedByteInputStream datastream, OutputStream userstream, PatchedValues p) throws IOException { + int totalSize = 0; + for (int i = 0; i <= p._freePos; ++i) { + int type = p._types[i]; + int pos = p._positions[i]; + int fnlen = (int)(p._lengths[i] >> 32); + int vallen = (int)(p._lengths[i] & 0xFFFFFFFF); + if (p.getParent() == null) { + if (pos > totalSize) { + // write whatever data that we have in the datastream (eg external data) + datastream.writeTo(userstream, pos - totalSize); + totalSize = pos; + } + totalSize += fnlen + vallen; + } + // write member name + if (fnlen > 0) { + datastream.writeTo(userstream, fnlen); + } + switch (type) { + case TID_ANNOTATION_PATCH: + IonBinary.writeVarUInt(userstream, vallen); + datastream.writeTo(userstream, vallen); + break; + + case TID_SYMBOL_TABLE_PATCH: + SymbolTable symtab = p._symtabs.remove(); + if (!symtab.isSystemTable()) { + byte[] symtabBytes = reverseEncode(1024, symtab); + userstream.write(symtabBytes); + totalSize += symtabBytes.length; + } + break; + + case TID_RAW: + datastream.writeTo(userstream, vallen); + break; + + default: + // write type + if (vallen >= _Private_IonConstants.lnIsVarLen) { + int typeByte = (type << 4) | _Private_IonConstants.lnIsVarLen; + userstream.write(typeByte); + IonBinary.writeVarUInt(userstream, vallen); + } else { + int typeByte = (type << 4) | vallen; + userstream.write(typeByte); + } + switch (type) { + case _Private_IonConstants.tidList: + case _Private_IonConstants.tidSexp: + case _Private_IonConstants.tidStruct: + case _Private_IonConstants.tidTypedecl: + // write the container + assert p._children != null; + writeRecursive(datastream, userstream, p._children.remove()); + break; + + default: + datastream.writeTo(userstream, vallen); + } + } + } + return totalSize; + } + + static class CountingStream extends OutputStream + { + private final OutputStream _wrapped; + private int _written; + + CountingStream(OutputStream userstream) { + _wrapped = userstream; + } + + public int getBytesWritten() { + return _written; + } + + @Override + public void write(int b) throws IOException + { + _wrapped.write(b); + _written++; + } + @Override + public void write(byte[] bytes) throws IOException + { + _wrapped.write(bytes); + _written += bytes.length; + } + @Override + public void write(byte[] bytes, int off, int len) throws IOException + { + _wrapped.write(bytes, off, len); + _written += len; + } + + } + + protected int write_symbol_table(OutputStream userstream, + SymbolTable symtab) throws IOException + { + CountingStream cs = new CountingStream(userstream); + // TODO this is assuming the symtab needed here, broken for open content. + IonWriterSystemBinary writer = + new IonWriterSystemBinary(_default_system_symbol_table, + cs, + false /* autoflush */ , + false /* ensureInitialIvm */); + symtab.writeTo(writer); + writer.finish(); + int symtab_len = cs.getBytesWritten(); + return symtab_len; + } + + protected int XXX_get_pending_length_with_no_symbol_tables() + { + int buffer_length = _manager.buffer().size(); + int patch_amount = 0; + + for (int patch_idx = 0; patch_idx < _patch_count; patch_idx ++) { + // int vlen = _patch_list[patch_idx + IonBinaryWriter.POSITION_OFFSET]; + int vlen = _patch_lengths[patch_idx]; + if (vlen >= _Private_IonConstants.lnIsVarLen) { + int ln = IonBinary.lenVarUInt(vlen); + patch_amount += ln; + } + } + + int symbol_table_length = 0; + + int total_length = 0; + total_length += buffer_length + + patch_amount + + symbol_table_length; + + return total_length; + } +} diff --git a/src/software/amazon/ion/impl/IonWriterSystemText.java b/src/com/amazon/ion/impl/IonWriterSystemText.java similarity index 90% rename from src/software/amazon/ion/impl/IonWriterSystemText.java rename to src/com/amazon/ion/impl/IonWriterSystemText.java index ea30350ab5..0e78e95ccd 100644 --- a/src/software/amazon/ion/impl/IonWriterSystemText.java +++ b/src/com/amazon/ion/impl/IonWriterSystemText.java @@ -1,48 +1,50 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; - -import static software.amazon.ion.SystemSymbols.SYMBOLS; -import static software.amazon.ion.impl.PrivateIonConstants.tidList; -import static software.amazon.ion.impl.PrivateIonConstants.tidSexp; -import static software.amazon.ion.impl.PrivateIonConstants.tidStruct; - +package com.amazon.ion.impl; + +import static com.amazon.ion.SystemSymbols.SYMBOLS; +import static com.amazon.ion.impl._Private_IonConstants.tidList; +import static com.amazon.ion.impl._Private_IonConstants.tidSexp; +import static com.amazon.ion.impl._Private_IonConstants.tidStruct; + +import com.amazon.ion.IonException; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonType; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.Timestamp; +import com.amazon.ion.system.IonTextWriterBuilder.LstMinimizing; +import com.amazon.ion.util.IonTextUtils; +import com.amazon.ion.util.IonTextUtils.SymbolVariant; +import com.amazon.ion.util._Private_FastAppendable; import java.io.IOException; import java.io.OutputStream; import java.math.BigDecimal; import java.math.BigInteger; -import software.amazon.ion.IonException; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonType; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.Timestamp; -import software.amazon.ion.system.IonTextWriterBuilder.LstMinimizing; -import software.amazon.ion.util.IonTextUtils; -import software.amazon.ion.util.PrivateFastAppendable; -import software.amazon.ion.util.IonTextUtils.SymbolVariant; + class IonWriterSystemText extends IonWriterSystem { /** Not null. */ - private final PrivateIonTextWriterBuilder _options; + private final _Private_IonTextWriterBuilder _options; /** At least one. */ private final int _long_string_threshold; - private final PrivateIonTextAppender _output; + private final _Private_IonTextAppender _output; /** Ensure we don't use a closed {@link #output} stream. */ private boolean _closed; @@ -75,15 +77,15 @@ class IonWriterSystemText * @throws NullPointerException if any parameter is null. */ protected IonWriterSystemText(SymbolTable defaultSystemSymtab, - PrivateIonTextWriterBuilder options, - PrivateFastAppendable out) + _Private_IonTextWriterBuilder options, + _Private_FastAppendable out) { super(defaultSystemSymtab, options.getInitialIvmHandling(), options.getIvmMinimizing()); _output = - PrivateIonTextAppender.forFastAppendable(out, + _Private_IonTextAppender.forFastAppendable(out, options.getCharset()); _options = options; @@ -100,7 +102,7 @@ protected IonWriterSystemText(SymbolTable defaultSystemSymtab, } - PrivateIonTextWriterBuilder getBuilder() + _Private_IonTextWriterBuilder getBuilder() { return _options; } @@ -122,16 +124,16 @@ protected IonType getContainer() } else { switch(_stack_parent_type[_top-1]) { - case PrivateIonConstants.tidDATAGRAM: + case _Private_IonConstants.tidDATAGRAM: container = IonType.DATAGRAM; break; - case PrivateIonConstants.tidSexp: + case _Private_IonConstants.tidSexp: container = IonType.SEXP; break; - case PrivateIonConstants.tidList: + case _Private_IonConstants.tidList: container = IonType.LIST; break; - case PrivateIonConstants.tidStruct: + case _Private_IonConstants.tidStruct: container = IonType.STRUCT; break; default: @@ -148,11 +150,11 @@ void push(int typeid) _stack_parent_type[_top] = typeid; _stack_pending_comma[_top] = _pending_separator; switch (typeid) { - case PrivateIonConstants.tidSexp: + case _Private_IonConstants.tidSexp: _separator_character = ' '; break; - case PrivateIonConstants.tidList: - case PrivateIonConstants.tidStruct: + case _Private_IonConstants.tidList: + case _Private_IonConstants.tidStruct: _separator_character = ','; break; default: @@ -180,15 +182,15 @@ int pop() { int parentid = (_top > 0) ? _stack_parent_type[_top - 1] : -1; switch (parentid) { case -1: - case PrivateIonConstants.tidSexp: + case _Private_IonConstants.tidSexp: _in_struct = false; _separator_character = ' '; break; - case PrivateIonConstants.tidList: + case _Private_IonConstants.tidList: _in_struct = false; _separator_character = ','; break; - case PrivateIonConstants.tidStruct: + case _Private_IonConstants.tidStruct: _in_struct = true; _separator_character = ','; break; @@ -660,7 +662,7 @@ public void writeString(String value) && ! _following_long_string && _long_string_threshold < value.length()) { - // TODO amzn/ion-java#57 This can lead to mixed newlines in the output. + // TODO amzn/ion-java/issues/57 This can lead to mixed newlines in the output. // It assumes NL line separators, but _options could use CR+NL _output.printLongString(value); diff --git a/src/com/amazon/ion/impl/IonWriterSystemTextMarkup.java b/src/com/amazon/ion/impl/IonWriterSystemTextMarkup.java new file mode 100644 index 0000000000..0925429566 --- /dev/null +++ b/src/com/amazon/ion/impl/IonWriterSystemTextMarkup.java @@ -0,0 +1,266 @@ + +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import com.amazon.ion.IonType; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.Timestamp; +import com.amazon.ion.util._Private_FastAppendable; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; + +class IonWriterSystemTextMarkup extends IonWriterSystemText { + + /** + * myTypeBeingWritten is used to communicate the type of the data that was + * requested to be written via a write to the subsequent methods that + * need to know the type for calling callbacks. + * It is not used when writing containers, or separators, since the type of + * container is passed in by the user, or can be obtained via + * getContainer(). + */ + private IonType myTypeBeingWritten; + private final _Private_MarkupCallback myCallback; + + public IonWriterSystemTextMarkup(SymbolTable defaultSystemSymtab, + _Private_IonTextWriterBuilder options, + _Private_FastAppendable out) + { + this(defaultSystemSymtab, options, out, options.getCallbackBuilder()); + } + + private IonWriterSystemTextMarkup(SymbolTable defaultSystemSymtab, + _Private_IonTextWriterBuilder options, + _Private_FastAppendable out, + _Private_CallbackBuilder builder) + { + this(defaultSystemSymtab, options, builder.build(out)); + } + + private IonWriterSystemTextMarkup(SymbolTable defaultSystemSymtab, + _Private_IonTextWriterBuilder options, + _Private_MarkupCallback callback) + { + super(defaultSystemSymtab, options, callback.getAppendable()); + myCallback = callback; + } + + @Override + void startValue() + throws IOException + { + super.startValue(); + myCallback.beforeValue(myTypeBeingWritten); + } + + @Override + void closeValue() + throws IOException + { + myCallback.afterValue(myTypeBeingWritten); + super.closeValue(); + } + + @Override + protected void writeFieldNameToken(SymbolToken sym) + throws IOException + { + myCallback.beforeFieldName(myTypeBeingWritten, sym); + super.writeFieldNameToken(sym); + myCallback.afterFieldName(myTypeBeingWritten, sym); + } + + @Override + protected void writeAnnotations(SymbolToken[] annotations) + throws IOException + { + myCallback.beforeAnnotations(myTypeBeingWritten); + super.writeAnnotations(annotations); + myCallback.afterAnnotations(myTypeBeingWritten); + } + + @Override + protected void writeAnnotationToken(SymbolToken sym) + throws IOException + { + myCallback.beforeEachAnnotation(myTypeBeingWritten, sym); + super.writeAnnotationToken(sym); + myCallback.afterEachAnnotation(myTypeBeingWritten, sym); + } + + @Override + protected boolean writeSeparator(boolean followingLongString) + throws IOException + { + // Determine the type of the container. + IonType containerType = getContainer(); + // Don't set beingWritten, because writeSeparator is called during the + // writing of another value, and it would mess up the other callbacks + + if (_pending_separator) { + myCallback.beforeSeparator(containerType); + } + + followingLongString = super.writeSeparator(followingLongString); + + if (_pending_separator) { + myCallback.afterSeparator(containerType); + } + return followingLongString; + } + + @Override + public void stepIn(IonType containerType) + throws IOException + { + myTypeBeingWritten = containerType; + super.stepIn(containerType); + myCallback.afterStepIn(containerType); + myTypeBeingWritten = null; + } + + @Override + public void stepOut() + throws IOException + { + myTypeBeingWritten = getContainer(); + myCallback.beforeStepOut(myTypeBeingWritten); + super.stepOut(); + myTypeBeingWritten = null; + } + + // ================ Customer Methods for writing Ion data. ================ + + @Override + public void writeBlob(byte[] value, int start, int len) + throws IOException + { + myTypeBeingWritten = IonType.BLOB; + super.writeBlob(value, start, len); + myTypeBeingWritten = null; + } + + @Override + public void writeBool(boolean value) + throws IOException + { + myTypeBeingWritten = IonType.BOOL; + super.writeBool(value); + myTypeBeingWritten = null; + } + + @Override + public void writeClob(byte[] value, int start, int len) + throws IOException + { + myTypeBeingWritten = IonType.CLOB; + super.writeClob(value, start, len); + myTypeBeingWritten = null; + } + + @Override + public void writeDecimal(BigDecimal value) + throws IOException + { + myTypeBeingWritten = IonType.DECIMAL; + super.writeDecimal(value); + myTypeBeingWritten = null; + } + + @Override + public void writeFloat(double value) + throws IOException + { + myTypeBeingWritten = IonType.FLOAT; + super.writeFloat(value); + myTypeBeingWritten = null; + } + + @Override + public void writeInt(long value) + throws IOException + { + myTypeBeingWritten = IonType.INT; + super.writeInt(value); + myTypeBeingWritten = null; + } + + @Override + public void writeInt(BigInteger value) + throws IOException + { + myTypeBeingWritten = IonType.INT; + super.writeInt(value); + myTypeBeingWritten = null; + } + + @Override + public void writeNull() + throws IOException + { + myTypeBeingWritten = IonType.NULL; + super.writeNull(); + myTypeBeingWritten = null; + } + + @Override + public void writeNull(IonType type) + throws IOException + { + myTypeBeingWritten = type; + super.writeNull(type); + myTypeBeingWritten = null; + } + + @Override + public void writeString(String value) + throws IOException + { + myTypeBeingWritten = IonType.STRING; + super.writeString(value); + myTypeBeingWritten = null; + } + + @Override + public void writeSymbolAsIs(String value) + throws IOException + { + myTypeBeingWritten = IonType.SYMBOL; + super.writeSymbolAsIs(value); + myTypeBeingWritten = null; + } + + @Override + public void writeSymbolAsIs(int value) + throws IOException + { + myTypeBeingWritten = IonType.SYMBOL; + super.writeSymbolAsIs(value); + myTypeBeingWritten = null; + } + + @Override + public void writeTimestamp(Timestamp value) + throws IOException + { + myTypeBeingWritten = IonType.TIMESTAMP; + super.writeTimestamp(value); + myTypeBeingWritten = null; + } +} diff --git a/src/software/amazon/ion/impl/IonWriterSystemTree.java b/src/com/amazon/ion/impl/IonWriterSystemTree.java similarity index 81% rename from src/software/amazon/ion/impl/IonWriterSystemTree.java rename to src/com/amazon/ion/impl/IonWriterSystemTree.java index 663008a768..ff3bf30d14 100644 --- a/src/software/amazon/ion/impl/IonWriterSystemTree.java +++ b/src/com/amazon/ion/impl/IonWriterSystemTree.java @@ -1,45 +1,46 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; - -import static software.amazon.ion.impl.PrivateUtils.valueIsLocalSymbolTable; - +package com.amazon.ion.impl; + +import static com.amazon.ion.impl._Private_Utils.valueIsLocalSymbolTable; + +import com.amazon.ion.IonBlob; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonClob; +import com.amazon.ion.IonContainer; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonDecimal; +import com.amazon.ion.IonException; +import com.amazon.ion.IonFloat; +import com.amazon.ion.IonSequence; +import com.amazon.ion.IonString; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSymbol; +import com.amazon.ion.IonTimestamp; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.Timestamp; +import com.amazon.ion.ValueFactory; +import com.amazon.ion.system.IonWriterBuilder.InitialIvmHandling; +import com.amazon.ion.system.IonWriterBuilder.IvmMinimizing; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; -import software.amazon.ion.IonBlob; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonClob; -import software.amazon.ion.IonContainer; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonDecimal; -import software.amazon.ion.IonException; -import software.amazon.ion.IonFloat; -import software.amazon.ion.IonSequence; -import software.amazon.ion.IonString; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonTimestamp; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.Timestamp; -import software.amazon.ion.ValueFactory; -import software.amazon.ion.system.IonWriterBuilder.InitialIvmHandling; -import software.amazon.ion.system.IonWriterBuilder.IvmMinimizing; /** @@ -51,9 +52,8 @@ final class IonWriterSystemTree extends IonWriterSystem { - /** Factory for constructing local symbol tables. Not null. */ - private final LocalSymbolTableAsStruct.Factory _lst_factory; - + /** Factory for constructing local symbol tables. Not null. */ + private final LocalSymbolTableAsStruct.Factory _lst_factory; private final ValueFactory _factory; @@ -87,7 +87,7 @@ protected IonWriterSystemTree(SymbolTable defaultSystemSymbolTable, IvmMinimizing.ADJACENT); _factory = rootContainer.getSystem(); - _lst_factory = (LocalSymbolTableAsStruct.Factory)((PrivateValueFactory)_factory).getLstFactory(); + _lst_factory = (LocalSymbolTableAsStruct.Factory)((_Private_ValueFactory)_factory).getLstFactory(); _catalog = catalog; _current_parent = rootContainer; _in_struct = (_current_parent instanceof IonStruct); @@ -186,7 +186,7 @@ private void append(IonValue value) if (hasAnnotations()) { SymbolToken[] annotations = getTypeAnnotationSymbols(); // TODO this makes an extra copy of the array - ((PrivateIonValue)value).setTypeAnnotationSymbols(annotations); + ((_Private_IonValue)value).setTypeAnnotationSymbols(annotations); this.clearAnnotations(); } @@ -219,7 +219,7 @@ public void stepIn(IonType containerType) throws IOException public void stepOut() throws IOException { - PrivateIonValue prior = (PrivateIonValue)_current_parent; + _Private_IonValue prior = (_Private_IonValue)_current_parent; popParent(); if (_current_parent instanceof IonDatagram @@ -238,7 +238,7 @@ void writeIonVersionMarkerAsIs(SymbolTable systemSymtab) { startValue(); IonValue root = get_root(); - ((PrivateIonDatagram)root).appendTrailingSymbolTable(systemSymtab); + ((_Private_IonDatagram)root).appendTrailingSymbolTable(systemSymtab); endValue(); } @@ -248,7 +248,7 @@ void writeLocalSymtab(SymbolTable symtab) throws IOException { IonValue root = get_root(); - ((PrivateIonDatagram)root).appendTrailingSymbolTable(symtab); + ((_Private_IonDatagram)root).appendTrailingSymbolTable(symtab); super.writeLocalSymtab(symtab); } diff --git a/src/software/amazon/ion/impl/IonWriterUser.java b/src/com/amazon/ion/impl/IonWriterUser.java similarity index 91% rename from src/software/amazon/ion/impl/IonWriterUser.java rename to src/com/amazon/ion/impl/IonWriterUser.java index b5b3fd2dca..22acb7eac7 100644 --- a/src/software/amazon/ion/impl/IonWriterUser.java +++ b/src/com/amazon/ion/impl/IonWriterUser.java @@ -1,43 +1,49 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonException; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonType; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.Timestamp; +import com.amazon.ion.ValueFactory; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonException; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonType; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.Timestamp; -import software.amazon.ion.ValueFactory; /** + *

    * This writer handles the symbol table processing and * provides default implementations for the list forms * of the write methods as often the list form is not * susceptible to optimization. + *

    + * *

    * This writer has a ({@link #_system_writer}) to which the actual data is * written, but data flows through the {@link #_current_writer} most of the * time. + *

    + * *

    * The critical responsibility here is the recognition of IVMs and local symbol * tables. When the user starts writing a local symtab, the stream is diverted @@ -45,10 +51,11 @@ * collects the symtab data into an {@link IonStruct} instance. When that * struct is stepped-out, the diversion is stopped and the new * {@link SymbolTable} is installed. + *

    */ class IonWriterUser - extends PrivateIonWriterBase - implements PrivateIonWriter + extends _Private_IonWriterBase + implements _Private_IonWriter { /** Factory for constructing the DOM of local symtabs. Not null. */ private final ValueFactory _symtab_value_factory; @@ -216,16 +223,20 @@ private boolean symbol_table_being_collected() } /** + *

    * Diverts the data stream to a temporary tree writer which collects * local symtab data into an IonStruct from which we'll later construct a * {@link SymbolTable} instance. + *

    *

    * Once the value image of the symbol table is complete (which * happens when the caller steps out of the containing struct) * the diverted stream is abandonded and the symbol table gets constructed. + *

    *

    * If there was a makeSymbolTable(Reader) this copy might be, * at least partially, avoided. + *

    */ private void open_local_symbol_table_copy() { @@ -255,7 +266,7 @@ private void close_local_symbol_table_copy() throws IOException // convert the struct we just wrote with the TreeWriter to a // local symbol table LocalSymbolTableAsStruct.Factory lstFactory = - (LocalSymbolTableAsStruct.Factory)((PrivateValueFactory)_symtab_value_factory).getLstFactory(); + (LocalSymbolTableAsStruct.Factory)((_Private_ValueFactory)_symtab_value_factory).getLstFactory(); SymbolTable symtab = lstFactory.newLocalSymtab(_catalog, _symbol_table_value); _symbol_table_value = null; @@ -271,7 +282,7 @@ public final void setSymbolTable(SymbolTable symbols) throws IOException { if (symbols == null || - PrivateUtils.symtabIsSharedNotSystem(symbols)) + _Private_Utils.symtabIsSharedNotSystem(symbols)) { String message = "symbol table must be local or system to be set, or reset"; diff --git a/src/com/amazon/ion/impl/IonWriterUserBinary.java b/src/com/amazon/ion/impl/IonWriterUserBinary.java new file mode 100644 index 0000000000..8d9a7823dd --- /dev/null +++ b/src/com/amazon/ion/impl/IonWriterUserBinary.java @@ -0,0 +1,158 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import static com.amazon.ion.impl._Private_Utils.isNonSymbolScalar; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonType; +import com.amazon.ion.util.IonStreamUtils; +import java.io.IOException; + + +class IonWriterUserBinary + extends IonWriterUser + implements _Private_ListWriter +{ + /** + * This is null if the writer is not stream copy optimized. + */ + private final _Private_SymtabExtendsCache mySymtabExtendsCache; + + /** + * This is null if the writer is not stream copy optimized. + */ + private final _Private_ByteTransferSink myCopySink; + + // If we wanted to we could keep an extra reference to the + // system writer which was correctly typed as an + // IonBinaryWriter and avoid the casting in the 3 "overridden" + // methods. However those are sufficiently expensive that + // the cost of the cast should be lost in the noise. + + IonWriterUserBinary(_Private_IonBinaryWriterBuilder options, + IonWriterSystemBinary systemWriter) + { + super(options.getCatalog(), + options.getSymtabValueFactory(), + systemWriter, + options.buildContextSymbolTable()); + + if (options.isStreamCopyOptimized()) + { + mySymtabExtendsCache = new _Private_SymtabExtendsCache(); + myCopySink = new _Private_ByteTransferSink() + { + public void writeBytes(byte[] data, int off, int len) throws IOException + { + ((IonWriterSystemBinary) _current_writer).writeRaw(data, off, len); + } + }; + } + else + { + mySymtabExtendsCache = null; + myCopySink = null; + } + } + + + @Override + public boolean isStreamCopyOptimized() + { + return mySymtabExtendsCache != null; + } + + @Override + public void writeValue(IonReader reader) + throws IOException + { + // If reader is not on a value, type is null, and NPE will be thrown + // by calls below + IonType type = reader.getType(); + + // See if we can copy bytes directly from the source. This test should + // only happen at the outermost call, not recursively down the tree. + + if (isStreamCopyOptimized() && + _current_writer instanceof IonWriterSystemBinary) + { + _Private_ByteTransferReader transfer = + reader.asFacet(_Private_ByteTransferReader.class); + + if (transfer != null && + (isNonSymbolScalar(type) || + mySymtabExtendsCache.symtabsCompat(getSymbolTable(), + reader.getSymbolTable()))) + { + // TODO amzn/ion-java/issues/16 Doesn't copy annotations or field names. + transfer.transferCurrentValue(myCopySink); + return; + } + } + + // From here on, we won't call back into this method, so we won't + // bother doing all those checks again. + writeValueRecursively(type, reader); + } + + + public void writeBoolList(boolean[] values) throws IOException + { + IonStreamUtils.writeBoolList(_current_writer, values); + } + + + public void writeFloatList(float[] values) throws IOException + { + IonStreamUtils.writeFloatList(_current_writer, values); + } + + + public void writeFloatList(double[] values) throws IOException + { + IonStreamUtils.writeFloatList(_current_writer, values); + } + + + public void writeIntList(byte[] values) throws IOException + { + IonStreamUtils.writeIntList(_current_writer, values); + } + + + public void writeIntList(short[] values) throws IOException + { + IonStreamUtils.writeIntList(_current_writer, values); + } + + + public void writeIntList(int[] values) throws IOException + { + IonStreamUtils.writeIntList(_current_writer, values); + } + + + public void writeIntList(long[] values) throws IOException + { + IonStreamUtils.writeIntList(_current_writer, values); + } + + + public void writeStringList(String[] values) throws IOException + { + IonStreamUtils.writeStringList(_current_writer, values); + } +} diff --git a/src/software/amazon/ion/impl/LocalSymbolTable.java b/src/com/amazon/ion/impl/LocalSymbolTable.java similarity index 92% rename from src/software/amazon/ion/impl/LocalSymbolTable.java rename to src/com/amazon/ion/impl/LocalSymbolTable.java index d221661400..dde4bb8d67 100644 --- a/src/software/amazon/ion/impl/LocalSymbolTable.java +++ b/src/com/amazon/ion/impl/LocalSymbolTable.java @@ -1,31 +1,41 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; - -import static software.amazon.ion.SystemSymbols.IMPORTS; -import static software.amazon.ion.SystemSymbols.IMPORTS_SID; -import static software.amazon.ion.SystemSymbols.ION; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; -import static software.amazon.ion.SystemSymbols.MAX_ID_SID; -import static software.amazon.ion.SystemSymbols.NAME_SID; -import static software.amazon.ion.SystemSymbols.SYMBOLS_SID; -import static software.amazon.ion.SystemSymbols.VERSION_SID; -import static software.amazon.ion.impl.PrivateUtils.copyOf; -import static software.amazon.ion.impl.PrivateUtils.getSidForSymbolTableField; -import static software.amazon.ion.impl.PrivateUtils.safeEquals; - +package com.amazon.ion.impl; + +import static com.amazon.ion.SystemSymbols.IMPORTS; +import static com.amazon.ion.SystemSymbols.IMPORTS_SID; +import static com.amazon.ion.SystemSymbols.ION; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; +import static com.amazon.ion.SystemSymbols.MAX_ID_SID; +import static com.amazon.ion.SystemSymbols.NAME_SID; +import static com.amazon.ion.SystemSymbols.SYMBOLS_SID; +import static com.amazon.ion.SystemSymbols.VERSION_SID; +import static com.amazon.ion.impl._Private_Utils.copyOf; +import static com.amazon.ion.impl._Private_Utils.getSidForSymbolTableField; +import static com.amazon.ion.impl._Private_Utils.safeEquals; + +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonException; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.ReadOnlyValueException; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.util.IonTextUtils; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -34,15 +44,6 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonException; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonType; -import software.amazon.ion.IonWriter; -import software.amazon.ion.ReadOnlyValueException; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.util.IonTextUtils; /** * A local symbol table. @@ -53,12 +54,11 @@ class LocalSymbolTable implements SymbolTable { - static class Factory implements PrivateLocalSymbolTableFactory + static class Factory implements _Private_LocalSymbolTableFactory { private Factory(){} // Should be accessed through the singleton - @Override public SymbolTable newLocalSymtab(IonCatalog catalog, IonReader reader, boolean alreadyInStruct) @@ -67,11 +67,11 @@ public SymbolTable newLocalSymtab(IonCatalog catalog, LocalSymbolTableImports imports = readLocalSymbolTable(reader, catalog, alreadyInStruct, - symbolsList); + symbolsList, + reader.getSymbolTable()); return new LocalSymbolTable(imports, symbolsList); } - @Override public SymbolTable newLocalSymtab(SymbolTable defaultSystemSymtab, SymbolTable... imports) { @@ -86,7 +86,6 @@ public SymbolTable newLocalSymtab(SymbolTable defaultSystemSymtab, static final Factory DEFAULT_LST_FACTORY = new Factory(); - /** * The initial length of {@link #mySymbolNames}. */ @@ -157,7 +156,7 @@ protected LocalSymbolTable(LocalSymbolTableImports imports, List symbols if (symbolsList == null || symbolsList.isEmpty()) { mySymbolsCount = 0; - mySymbolNames = PrivateUtils.EMPTY_STRING_ARRAY; + mySymbolNames = _Private_Utils.EMPTY_STRING_ARRAY; } else { @@ -202,7 +201,8 @@ protected LocalSymbolTable(LocalSymbolTable other, int maxId) protected static LocalSymbolTableImports readLocalSymbolTable(IonReader reader, IonCatalog catalog, boolean isOnStruct, - List symbolsListOut) + List symbolsListOut, + SymbolTable symbolTable) { if (! isOnStruct) { @@ -242,7 +242,7 @@ protected static LocalSymbolTableImports readLocalSymbolTable(IonReader reader, sid = getSidForSymbolTableField(fieldName); } - // TODO amzn/ion-java#36 Switching over SIDs doesn't cover the case + // TODO amzn/ion-java/issues/36 Switching over SIDs doesn't cover the case // where the relevant field names are defined by a prev LST; // the prev LST could have 'symbols' defined locally with a // different SID! @@ -291,7 +291,7 @@ protected static LocalSymbolTableImports readLocalSymbolTable(IonReader reader, else if (fieldType == IonType.SYMBOL) { // trying to import the current table - if(reader.getSymbolTable().isLocalTable() && ION_SYMBOL_TABLE.equals(reader.stringValue())) + if(symbolTable.isLocalTable() && ION_SYMBOL_TABLE.equals(reader.stringValue())) { SymbolTable currentSymbolTable = reader.getSymbolTable(); importsList.addAll(Arrays.asList(currentSymbolTable.getImportedTables())); @@ -533,6 +533,7 @@ private static final void validateSymbol(String name) */ int putSymbol(String symbolName) { + if (isReadOnly) { throw new ReadOnlyValueException(SymbolTable.class); @@ -825,7 +826,7 @@ boolean symtabExtends(SymbolTable other) // Superset must have same/more known symbols than subset. if (getMaxId() < subset.getMaxId()) return false; - // TODO amzn/ion-java#18 Currently, we check imports by their refs. which + // TODO amzn/ion-java/issues/18 Currently, we check imports by their refs. which // might be overly strict; imports which are not the same ref. // but have the same semantic states fails the extension check. if (! myImportsList.equalImports(subset.myImportsList)) diff --git a/src/software/amazon/ion/impl/LocalSymbolTableAsStruct.java b/src/com/amazon/ion/impl/LocalSymbolTableAsStruct.java similarity index 86% rename from src/software/amazon/ion/impl/LocalSymbolTableAsStruct.java rename to src/com/amazon/ion/impl/LocalSymbolTableAsStruct.java index 7cc6f3021e..3ac9e674ae 100644 --- a/src/software/amazon/ion/impl/LocalSymbolTableAsStruct.java +++ b/src/com/amazon/ion/impl/LocalSymbolTableAsStruct.java @@ -1,36 +1,37 @@ /* - * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; - -import static software.amazon.ion.SystemSymbols.IMPORTS; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; -import static software.amazon.ion.SystemSymbols.MAX_ID; -import static software.amazon.ion.SystemSymbols.NAME; -import static software.amazon.ion.SystemSymbols.SYMBOLS; -import static software.amazon.ion.SystemSymbols.VERSION; - +package com.amazon.ion.impl; + +import static com.amazon.ion.SystemSymbols.IMPORTS; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; +import static com.amazon.ion.SystemSymbols.MAX_ID; +import static com.amazon.ion.SystemSymbols.NAME; +import static com.amazon.ion.SystemSymbols.SYMBOLS; +import static com.amazon.ion.SystemSymbols.VERSION; + +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonList; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.ValueFactory; import java.util.ArrayList; import java.util.List; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonList; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.ValueFactory; /** * A LocalSymbolTable that memoizes its IonStruct representation. @@ -42,7 +43,7 @@ class LocalSymbolTableAsStruct extends LocalSymbolTable { - static class Factory implements PrivateLocalSymbolTableFactory + static class Factory implements _Private_LocalSymbolTableFactory { private final ValueFactory imageFactory; @@ -56,7 +57,6 @@ public Factory(ValueFactory imageFactory) this.imageFactory = imageFactory; } - @Override public SymbolTable newLocalSymtab(IonCatalog catalog, IonReader reader, boolean alreadyInStruct) @@ -65,11 +65,11 @@ public SymbolTable newLocalSymtab(IonCatalog catalog, LocalSymbolTableImports imports = readLocalSymbolTable(reader, catalog, alreadyInStruct, - symbolsList); + symbolsList, + reader.getSymbolTable()); return new LocalSymbolTableAsStruct(imageFactory, imports, symbolsList); } - @Override public SymbolTable newLocalSymtab(SymbolTable defaultSystemSymtab, SymbolTable... imports) { @@ -101,7 +101,8 @@ public SymbolTable newLocalSymtab(IonCatalog catalog, LocalSymbolTableImports imports = readLocalSymbolTable(reader, catalog, false, - symbolsList); + symbolsList, + ionRep.getSymbolTable()); LocalSymbolTableAsStruct table = new LocalSymbolTableAsStruct(imageFactory, imports, diff --git a/src/software/amazon/ion/impl/LocalSymbolTableImports.java b/src/com/amazon/ion/impl/LocalSymbolTableImports.java similarity index 93% rename from src/software/amazon/ion/impl/LocalSymbolTableImports.java rename to src/com/amazon/ion/impl/LocalSymbolTableImports.java index ca78b9c322..3d9bfe7e58 100644 --- a/src/software/amazon/ion/impl/LocalSymbolTableImports.java +++ b/src/com/amazon/ion/impl/LocalSymbolTableImports.java @@ -1,26 +1,26 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; -import java.util.ArrayList; -import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; +import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; import java.util.Arrays; import java.util.List; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; /** * This class manages the system symbol table and any shared symbol table(s) @@ -29,7 +29,7 @@ *

    * This class is immutable, and hence safe for use by multiple threads. */ -// TODO amzn/ion-java#37 Create specialized class to handle the common case where +// TODO amzn/ion-java/issues/37 Create specialized class to handle the common case where // there are zero or one imported non-system shared symtab(s). final class LocalSymbolTableImports { diff --git a/src/software/amazon/ion/impl/OutputStreamFastAppendable.java b/src/com/amazon/ion/impl/OutputStreamFastAppendable.java similarity index 89% rename from src/software/amazon/ion/impl/OutputStreamFastAppendable.java rename to src/com/amazon/ion/impl/OutputStreamFastAppendable.java index c3335f7553..9da49d08bf 100644 --- a/src/software/amazon/ion/impl/OutputStreamFastAppendable.java +++ b/src/com/amazon/ion/impl/OutputStreamFastAppendable.java @@ -1,33 +1,34 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; -import static software.amazon.ion.impl.PrivateIonConstants.makeUnicodeScalar; +import static com.amazon.ion.impl._Private_IonConstants.makeUnicodeScalar; +import com.amazon.ion.util._Private_FastAppendable; import java.io.Closeable; import java.io.Flushable; import java.io.IOException; import java.io.OutputStream; -import software.amazon.ion.util.PrivateFastAppendable; /** - * Adapts an {@link OutputStream} to implement {@link PrivateFastAppendable}. + * Adapts an {@link OutputStream} to implement {@link _Private_FastAppendable}. * This always outputs UTF-8! */ final class OutputStreamFastAppendable - implements PrivateFastAppendable, Closeable, Flushable + implements _Private_FastAppendable, Closeable, Flushable { private static final int MAX_BYTES_LEN = 4096; diff --git a/src/software/amazon/ion/impl/SharedSymbolTable.java b/src/com/amazon/ion/impl/SharedSymbolTable.java similarity index 91% rename from src/software/amazon/ion/impl/SharedSymbolTable.java rename to src/com/amazon/ion/impl/SharedSymbolTable.java index ae476d73d6..dedb85985f 100644 --- a/src/software/amazon/ion/impl/SharedSymbolTable.java +++ b/src/com/amazon/ion/impl/SharedSymbolTable.java @@ -1,26 +1,37 @@ /* - * Copyright 2013-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; - -import static software.amazon.ion.SystemSymbols.ION; -import static software.amazon.ion.SystemSymbols.ION_1_0; -import static software.amazon.ion.SystemSymbols.NAME_SID; -import static software.amazon.ion.SystemSymbols.SYMBOLS_SID; -import static software.amazon.ion.SystemSymbols.VERSION_SID; -import static software.amazon.ion.impl.PrivateUtils.getSidForSymbolTableField; - +package com.amazon.ion.impl; + +import static com.amazon.ion.SystemSymbols.ION; +import static com.amazon.ion.SystemSymbols.ION_1_0; +import static com.amazon.ion.SystemSymbols.NAME_SID; +import static com.amazon.ion.SystemSymbols.SYMBOLS_SID; +import static com.amazon.ion.SystemSymbols.VERSION_SID; +import static com.amazon.ion.impl._Private_Utils.getSidForSymbolTableField; + +import com.amazon.ion.IonException; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.ReadOnlyValueException; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.SystemSymbols; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -29,16 +40,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import software.amazon.ion.IonException; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonType; -import software.amazon.ion.IonWriter; -import software.amazon.ion.ReadOnlyValueException; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.SystemSymbols; /** * An immutable shared symbol table, supporting (non-system) shared @@ -68,7 +69,7 @@ final class SharedSymbolTable /** * The singleton instance of Ion 1.0 system symbol table. *

    - * TODO amzn/ion-java#34 Optimize system symtabs by using our custom backing impl. + * TODO amzn/ion-java/issues/34 Optimize system symtabs by using our custom backing impl. */ private static final SymbolTable ION_1_0_SYSTEM_SYMTAB; static @@ -237,9 +238,9 @@ static SymbolTable newSharedSymbolTable(IonReader reader, sid = getSidForSymbolTableField(fieldName); } - // TODO amzn/ion-java#35 If there's more than one 'symbols' or 'imports' + // TODO amzn/ion-java/issues/35 If there's more than one 'symbols' or 'imports' // field, they will be merged together. - // TODO amzn/ion-java#36 Switching over SIDs doesn't cover the case + // TODO amzn/ion-java/issues/36 Switching over SIDs doesn't cover the case // where the relevant field names are defined by a prev LST; // the prev LST could have 'symbols' defined locally with a // different SID! @@ -397,7 +398,7 @@ private static void putToMapIfNotThere(Map symbolsMap, while (symbols.hasNext()) { String text = symbols.next(); - // TODO amzn/ion-java#12 What about empty symbols? + // TODO amzn/ion-java/issues/12 What about empty symbols? if (symbolsMap.get(text) == null) { putToMapIfNotThere(symbolsMap, text, sid); diff --git a/src/com/amazon/ion/impl/SimpleByteBuffer.java b/src/com/amazon/ion/impl/SimpleByteBuffer.java new file mode 100644 index 0000000000..0eb06a97a4 --- /dev/null +++ b/src/com/amazon/ion/impl/SimpleByteBuffer.java @@ -0,0 +1,1386 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import com.amazon.ion.Decimal; +import com.amazon.ion.IonException; +import com.amazon.ion.Timestamp; +import com.amazon.ion.Timestamp.Precision; +import java.io.IOException; +import java.io.OutputStream; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.MathContext; + +/** + * Manages a very simple byte buffer that is a single contiguous + * byte array, without resize ability. + */ +final class SimpleByteBuffer + implements ByteBuffer +{ + byte[] _bytes; + int _start; + int _eob; + boolean _is_read_only; + + + /** + * Creates a read-write buffer. + * + * @param bytes assumed to be owned by this new instance. + */ + public SimpleByteBuffer(byte[] bytes) { + this(bytes, 0, bytes.length, false); + } + + /** + * + * @param bytes assumed to be owned by this new instance. + */ + public SimpleByteBuffer(byte[] bytes, boolean isReadOnly) { + this(bytes, 0, bytes.length, isReadOnly); + } + + /** + * Creates a read-write buffer. + * + * @param bytes assumed to be owned by this new instance. + */ + public SimpleByteBuffer(byte[] bytes, int start, int length) { + this(bytes, start, length, false); + } + + /** + * + * @param bytes assumed to be owned by this new instance. + */ + public SimpleByteBuffer(byte[] bytes, int start, int length, boolean isReadOnly) { + if (bytes == null || start < 0 || start > bytes.length || length < 0 || start + length > bytes.length) { + throw new IllegalArgumentException(); + } + _bytes = bytes; + _start = start; + _eob = start + length; + _is_read_only = isReadOnly; + } + + public int getLength() + { + int length = _eob - _start; + return length; + } + + /** + * Makes a copy of the internal byte array. + */ + public byte[] getBytes() + { + int length = _eob - _start; + byte[] copy = new byte[length]; + System.arraycopy(_bytes, _start, copy, 0, length); + return copy; + } + + public int getBytes(byte[] buffer, int offset, int length) + { + if (buffer == null || offset < 0 || offset > buffer.length || length < 0 || offset + length > buffer.length) { + throw new IllegalArgumentException(); + } + + int datalength = _eob - _start; + if (datalength > length) { + throw new IllegalArgumentException("insufficient space in destination buffer"); + } + + System.arraycopy(_bytes, _start, buffer, offset, datalength); + + return datalength; + + } + public ByteReader getReader() + { + ByteReader reader = new SimpleByteReader(this); + return reader; + } + + public ByteWriter getWriter() + { + if (_is_read_only) { + throw new IllegalStateException("this buffer is read only"); + } + SimpleByteWriter writer = new SimpleByteWriter(this); + return writer; + } + + public void writeBytes(OutputStream out) throws IOException + { + int length = _eob - _start; + out.write(_bytes, _start, length); + } + + static final class SimpleByteReader implements ByteReader + { + SimpleByteBuffer _buffer; + int _position; + + SimpleByteReader(SimpleByteBuffer bytebuffer) { + _buffer = bytebuffer; + _position = bytebuffer._start; + } + + public int position() + { + int pos = _position - _buffer._start; + return pos; + } + + public void position(int newPosition) + { + if (newPosition < 0) { + throw new IllegalArgumentException("position must be non-negative"); + } + int pos = newPosition + _buffer._start; + if (pos > _buffer._eob) { + throw new IllegalArgumentException("position is past end of buffer"); + } + _position = pos; + } + + public void skip(int length) + { + if (length < 0) { + throw new IllegalArgumentException("length to skip must be non-negative"); + } + int pos = _position + length; + if (pos > _buffer._eob) { + throw new IllegalArgumentException("skip would skip past end of buffer"); + } + _position = pos; + } + + public int read() + { + if (_position >= _buffer._eob) return EOF; + int b = _buffer._bytes[_position++]; + return (b & 0xff); + } + + public int read(byte[] dst, int start, int len) + { + if (dst == null || start < 0 || len < 0 || start + len > dst.length) { + // no need to test this start >= dst.length || + // since we test start+len > dst.length which is the correct test + throw new IllegalArgumentException(); + } + + if (_position >= _buffer._eob) return 0; + int readlen = len; + if (readlen + _position > _buffer._eob) readlen = _buffer._eob - _position; + System.arraycopy(_buffer._bytes, _position, dst, start, readlen); + _position += readlen; + return readlen; + } + + public int readTypeDesc() + { + return read(); + } + + public long readULong(int len) throws IOException + { + long retvalue = 0; + int b; + + switch (len) { + default: + throw new IonException("value too large for Java long"); + case 8: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 7: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 6: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 5: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 4: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 3: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 2: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 1: + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 8) | b; + case 0: + // do nothing, it's just a 0 length is a 0 value + } + return retvalue; + } + + + public int readVarInt() throws IOException + { + int retvalue = 0; + boolean is_negative = false; + int b; + + // synthetic label "done" (yuck) +done: for (;;) { + // read the first byte - it has the sign bit + if ((b = read()) < 0) throwUnexpectedEOFException(); + if ((b & 0x40) != 0) { + is_negative = true; + } + retvalue = (b & 0x3F); + if ((b & 0x80) != 0) break done; + + // for the second byte we shift our eariler bits just as much, + // but there are fewer of them there to shift + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break done; + + // for the rest, they're all the same + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break done; + + // for the rest, they're all the same + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break done; + + // for the rest, they're all the same + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break done; + + // if we get here we have more bits than we have room for :( + throwIntOverflowExeption(); + } + if (is_negative) { + retvalue = -retvalue; + } + return retvalue; + } + + public long readVarLong() throws IOException + { + long retvalue = 0; + boolean is_negative = false; + int b; + + // synthetic label "done" (yuck) +done: for (;;) { + // read the first byte - it has the sign bit + if ((b = read()) < 0) throwUnexpectedEOFException(); + if ((b & 0x40) != 0) { + is_negative = true; + } + retvalue = (b & 0x3F); + if ((b & 0x80) != 0) break done; + + // for the second byte we shift our eariler bits just as much, + // but there are fewer of them there to shift + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break done; + + // for the rest, they're all the same + for (;;) { + if ((b = read()) < 0) throwUnexpectedEOFException(); + if ((retvalue & 0xFE00000000000000L) != 0) throwIntOverflowExeption(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break done; + } + } + if (is_negative) { + retvalue = -retvalue; + } + return retvalue; + } + + /** + * Reads an integer value, returning null to mean -0. + * @throws IOException + */ + public Integer readVarInteger() throws IOException + { + int retvalue = 0; + boolean is_negative = false; + int b; + + // sythetic label "done" (yuck) +done: for (;;) { + // read the first byte - it has the sign bit + if ((b = read()) < 0) throwUnexpectedEOFException(); + if ((b & 0x40) != 0) { + is_negative = true; + } + retvalue = (b & 0x3F); + if ((b & 0x80) != 0) break done; + + // for the second byte we shift our eariler bits just as much, + // but there are fewer of them there to shift + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break done; + + // for the rest, they're all the same + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break done; + + // for the rest, they're all the same + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break done; + + // for the rest, they're all the same + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break done; + + // if we get here we have more bits than we have room for :( + throwIntOverflowExeption(); + } + + Integer retInteger = null; + if (is_negative) { + if (retvalue != 0) { + retInteger = new Integer(-retvalue); + } + } + else { + retInteger = new Integer(retvalue); + } + return retInteger; + } + + public int readVarUInt() throws IOException + { + int retvalue = 0; + int b; + + for (;;) { // fake loop to create a "goto done" + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break; + + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break; + + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break; + + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break; + + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break; + + // if we get here we have more bits than we have room for :( + throwIntOverflowExeption(); + } + return retvalue; + } + + public double readFloat(int len) throws IOException + { + if (len == 0) + { + // special case, return pos zero + return 0.0d; + } + + if (len != 8) + { + throw new IOException("Length of float read must be 0 or 8"); + } + + long dBits = this.readULong(len); + return Double.longBitsToDouble(dBits); + } + + public long readVarULong() throws IOException + { + long retvalue = 0; + int b; + + for (;;) { + if ((b = read()) < 0) throwUnexpectedEOFException(); + if ((retvalue & 0xFE00000000000000L) != 0) throwIntOverflowExeption(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break; + } + return retvalue; + } + + /** + * Near clone of {@link IonReaderBinaryRawX#readDecimal(int)} + * and {@link IonBinary.Reader#readDecimalValue(IonDecimalImpl, int)} + * so keep them in sync! + */ + public Decimal readDecimal(int len) throws IOException + { + MathContext mathContext = MathContext.UNLIMITED; + + Decimal bd; + + // we only write out the '0' value as the nibble 0 + if (len == 0) { + bd = Decimal.valueOf(0, mathContext); + } + else { + // otherwise we to it the hard way .... + int startpos = this.position(); + int exponent = this.readVarInt(); + int bitlen = len - (this.position() - startpos); + + BigInteger value; + int signum; + if (bitlen > 0) + { + byte[] bits = new byte[bitlen]; + this.read(bits, 0, bitlen); + + signum = 1; + if (bits[0] < 0) + { + // value is negative, clear the sign + bits[0] &= 0x7F; + signum = -1; + } + value = new BigInteger(signum, bits); + } + else { + signum = 0; + value = BigInteger.ZERO; + } + + // Ion stores exponent, BigDecimal uses the negation "scale" + int scale = -exponent; + if (value.signum() == 0 && signum == -1) + { + assert value.equals(BigInteger.ZERO); + bd = Decimal.negativeZero(scale, mathContext); + } + else + { + bd = Decimal.valueOf(value, scale, mathContext); + } + } + return bd; + } + + public Timestamp readTimestamp(int len) throws IOException + { + if (len < 1) { + // nothing to do here - and the timestamp will be NULL + return null; + } + + Timestamp val; + Precision p = null; // FIXME remove + Integer offset = null; + int year = 0, month = 0, day = 0, hour = 0, minute = 0, second = 0; + BigDecimal frac = null; + int remaining, end = this.position() + len; + + // first up is the offset, which requires a special int reader + // to return the -0 as a null Integer + offset = this.readVarInteger(); + + // now we'll read the struct values from the input stream + assert position() < end; + if (position() < end) { // FIXME remove + // year is from 0001 to 9999 + // or 0x1 to 0x270F or 14 bits - 1 or 2 bytes + year = readVarUInt(); + p = Precision.YEAR; // our lowest significant option + + // now we look for hours and minutes + if (position() < end) { + month = readVarUInt(); + p = Precision.MONTH; + + // now we look for hours and minutes + if (position() < end) { + day = readVarUInt(); + p = Precision.DAY; // our lowest significant option + + // now we look for hours and minutes + if (position() < end) { + hour = readVarUInt(); + minute = readVarUInt(); + p = Precision.MINUTE; + + if (position() < end) { + second = readVarUInt(); + p = Precision.SECOND; + + remaining = end - position(); + if (remaining > 0) { + // now we read in our actual "milliseconds since the epoch" + frac = this.readDecimal(remaining); + } + } + } + } + } + } + + // now we let timestamp put it all together + val = Timestamp.createFromUtcFields(p, year, month, day, hour, minute, second, frac, offset); + return val; + } + + public String readString(int len) throws IOException + { + // len is bytes, which is greater than or equal to java + // chars even after utf8 to utf16 decoding nonsense + // the char array is way faster than using string buffer + char[] chars = new char[len]; + int ii = 0; + int c; + int endPosition = this.position() + len; + + while (this.position() < endPosition) { + c = readUnicodeScalar(); + if (c < 0) throwUnexpectedEOFException(); + if (c < 0x10000) { + chars[ii++] = (char)c; + } + else { // when c is >= 0x10000 we need surrogate encoding + chars[ii++] = (char)_Private_IonConstants.makeHighSurrogate(c); + chars[ii++] = (char)_Private_IonConstants.makeLowSurrogate(c); + } + } + if (this.position() < endPosition) throwUnexpectedEOFException(); + + return new String(chars, 0, ii); + } + + public final int readUnicodeScalar() throws IOException { + int b; + + b = read(); + // ascii is all good, even -1 (eof) + if (b >= 0x80) { + b = readUnicodeScalar_helper(b); + + } + return b; + } + private final int readUnicodeScalar_helper(int b) throws IOException + { + int c = -1; + + // now we start gluing the multi-byte value together + if ((b & 0xe0) == 0xc0) { + // for values from 0x80 to 0x7FF (all legal) + c = (b & ~0xe0); + b = read(); + if ((b & 0xc0) != 0x80) throwUTF8Exception(); + c <<= 6; + c |= (b & ~0x80); + } + else if ((b & 0xf0) == 0xe0) { + // for values from 0x800 to 0xFFFFF (NOT all legal) + c = (b & ~0xf0); + b = read(); + if ((b & 0xc0) != 0x80) throwUTF8Exception(); + c <<= 6; + c |= (b & ~0x80); + b = read(); + if ((b & 0xc0) != 0x80) throwUTF8Exception(); + c <<= 6; + c |= (b & ~0x80); + if (c > 0x00D7FF && c < 0x00E000) { + throw new IonException("illegal surrogate value encountered in input utf-8 stream"); + } + } + else if ((b & 0xf8) == 0xf0) { + // for values from 0x010000 to 0x1FFFFF (NOT all legal) + c = (b & ~0xf8); + b = read(); + if ((b & 0xc0) != 0x80) throwUTF8Exception(); + c <<= 6; + c |= (b & ~0x80); + b = read(); + if ((b & 0xc0) != 0x80) throwUTF8Exception(); + c <<= 6; + c |= (b & ~0x80); + b = read(); + if ((b & 0xc0) != 0x80) throwUTF8Exception(); + c <<= 6; + c |= (b & ~0x80); + if (c > 0x10FFFF) { + throw new IonException("illegal utf value encountered in input utf-8 stream"); + } + } + else { + throwUTF8Exception(); + } + return c; + } + void throwUTF8Exception() throws IOException + { + throw new IOException("Invalid UTF-8 character encounter in a string at pos " + this.position()); + } + void throwUnexpectedEOFException() throws IOException { + throw new IOException("unexpected EOF in value at offset " + this.position()); + } + void throwIntOverflowExeption() throws IOException { + throw new IOException("int in stream is too long for a Java int 32 use readLong()"); + } + } + + static final class UserByteWriter extends OutputStream implements ByteWriter + { + SimpleByteWriter _simple_writer; + OutputStream _user_stream; + int _position; + int _limit; + int _buffer_size; + + // constants used for the various write routines to verify + // sufficient room in the output buffer for writing + private final static int MAX_UINT7_BINARY_LENGTH = 5; // (1 + (32 / 7)) + private final static int MAX_FLOAT_BINARY_LENGTH = 8; // ieee 64 bit float + + private final static int REQUIRED_BUFFER_SPACE = 8; // max of the required lengths + + UserByteWriter(OutputStream userOuputStream, byte[] buf) + { + if (buf == null || buf.length < REQUIRED_BUFFER_SPACE) { + throw new IllegalArgumentException("requires a buffer at least "+REQUIRED_BUFFER_SPACE+" bytes long"); + } + + SimpleByteBuffer bytebuffer = new SimpleByteBuffer(buf); + _simple_writer = new SimpleByteWriter(bytebuffer); + _user_stream = userOuputStream; + _buffer_size = buf.length; + _limit = _buffer_size; + } + + private final void checkForSpace(int needed) { + if (_position + needed > _limit) { + flush(); + } + } + + /** + * this flushes the current buffer to the users output + * stream and resets the buffer. It is called automatically + * as the buffer fills. It can be called at any time if more + * frequent flushing is desired. + */ + @Override + public void flush() { + // is there anything to flush? + if (_position + _buffer_size > _limit) { + try { + _simple_writer.flushTo(_user_stream); + } + catch (IOException e) { + throw new IonException(e); + } + _limit = _position + _buffer_size; + } + } + + public void insert(int length) + { + throw new UnsupportedOperationException("use a SimpleByteWriter if you need to insert"); + } + + public int position() + { + return _position; + } + + public void position(int newPosition) + { + throw new UnsupportedOperationException("use a SimpleByteWriter if you need to set your position"); + } + + public void remove(int length) + { + throw new UnsupportedOperationException("use a SimpleByteWriter if you need to remove bytes"); + } + + @Override + public void write(int b) throws IOException + { + write((byte)b); + } + + public void write(byte b) throws IOException + { + checkForSpace(1); + _simple_writer.write(b); + _position++; + } + + // cloned from SimpleByteBuffer as we need to check the + // length and this may require flushing more often than + // the default writeDecimal supports. + public int writeDecimal(BigDecimal value) throws IOException + { + int returnlen = _simple_writer.writeDecimal(value, this); + return returnlen; + } + + public int writeFloat(double value) throws IOException + { + checkForSpace(MAX_FLOAT_BINARY_LENGTH); + int returnlen = _simple_writer.writeFloat(value); + return returnlen; + } + + public int writeIonInt(long value, int len) throws IOException + { + checkForSpace(len); + int returnlen = _simple_writer.writeIonInt(value, len); + return returnlen; + } + + public int writeIonInt(int value, int len) throws IOException + { + checkForSpace(len); + int returnlen = _simple_writer.writeIonInt(value, len); + return returnlen; + } + + public int writeString(String value) throws IOException + { + int returnlen = _simple_writer.writeString(value, this); + return returnlen; + } + + public void writeTypeDesc(int typeDescByte) throws IOException + { + checkForSpace(1); + _simple_writer.writeTypeDesc(typeDescByte); + } + + public int writeTypeDescWithLength(int typeid, int lenOfLength, + int valueLength) throws IOException + { + checkForSpace(1 + MAX_UINT7_BINARY_LENGTH); + int returnlen = _simple_writer.writeTypeDescWithLength(typeid, lenOfLength, valueLength); + return returnlen; + } + + public int writeTypeDescWithLength(int typeid, int valueLength) throws IOException + { + checkForSpace(1 + MAX_UINT7_BINARY_LENGTH); + int returnlen = _simple_writer.writeTypeDescWithLength(typeid, valueLength); + return returnlen; + } + + public int writeVarInt(long value, int len, boolean forceZeroWrite) + throws IOException + { + checkForSpace(len); + int returnlen = _simple_writer.writeVarInt(value, len, forceZeroWrite); + return returnlen; + } + + public int writeVarInt(int value, int len, boolean forceZeroWrite) + throws IOException + { + checkForSpace(len); + int returnlen = _simple_writer.writeVarInt(value, len, forceZeroWrite); + return returnlen; + } + + public int writeVarUInt(int value, int len, boolean forceZeroWrite) + throws IOException + { + checkForSpace(len); + int returnlen = _simple_writer.writeVarUInt(value, len, forceZeroWrite); + return returnlen; + } + + public int writeVarUInt(long value, int len, boolean forceZeroWrite) + throws IOException + { + checkForSpace(len); + int returnlen = _simple_writer.writeVarUInt(value, len, forceZeroWrite); + return returnlen; + } + } + + static final class SimpleByteWriter extends OutputStream implements ByteWriter + { + private static final int _ib_FLOAT64_LEN = 8; + private static final Double DOUBLE_POS_ZERO = Double.valueOf(0.0); + + SimpleByteBuffer _buffer; + int _position; + + SimpleByteWriter(SimpleByteBuffer bytebuffer) { + _buffer = bytebuffer; + _position = bytebuffer._start; + } + + protected void flushTo(OutputStream userOutput) throws IOException { + _buffer.writeBytes(userOutput); + _position = 0; + } + + public int position() + { + return _position - _buffer._start; + } + + public void position(int newPosition) + { + if (newPosition < 0) { + throw new IllegalArgumentException("position must be non-negative"); + } + int pos = newPosition + _buffer._start; + if (pos > _buffer._eob) { + throw new IllegalArgumentException("position is past end of buffer"); + } + _position = pos; + } + + public void insert(int length) + { + if (length < 0) { + throw new IllegalArgumentException("insert length must be non negative"); + } + int remaining = _buffer._eob - _position; + System.arraycopy(_buffer._bytes, _position, _buffer._bytes, _position + length, remaining); + _buffer._eob += length; + } + + public void remove(int length) + { + if (length < 0) { + throw new IllegalArgumentException("remove length must be non negative"); + } + int remaining = _buffer._eob - _position; + System.arraycopy(_buffer._bytes, _position + length, _buffer._bytes, _position, remaining); + _buffer._eob -= length; + } + + @Override + final public void write(int arg0) + throws IOException + { + write((byte)arg0); + } + + final public void write(byte b) + { + _buffer._bytes[_position++] = b; + if (_position > _buffer._eob) _buffer._eob = _position; + } + + @Override + public void write(byte[] bytes, int start, int len) + { + if (bytes == null || start < 0 || start >= bytes.length || len < 0 || start + len > bytes.length) { + throw new IllegalArgumentException(); + } + + System.arraycopy(bytes, start, _buffer._bytes, _position, len); + _position += len; + if (_position > _buffer._eob) _buffer._eob = _position; + + return; + } + + public void writeTypeDesc(int typeDescByte) + { + write((byte)(typeDescByte & 0xff)); + } + + public int writeTypeDescWithLength(int typeid, int lenOfLength, int valueLength) + { + int written_len = 1; + + int td = ((typeid & 0xf) << 4); + if (valueLength >= _Private_IonConstants.lnIsVarLen) { + td |= _Private_IonConstants.lnIsVarLen; + writeTypeDesc(td); + written_len += writeVarUInt(valueLength, lenOfLength, true); + } + else { + td |= (valueLength & 0xf); + writeTypeDesc(td); + } + return written_len; + } + + public int writeTypeDescWithLength(int typeid, int valueLength) + { + int written_len = 1; + + int td = ((typeid & 0xf) << 4); + if (valueLength >= _Private_IonConstants.lnIsVarLen) { + td |= _Private_IonConstants.lnIsVarLen; + writeTypeDesc(td); + int lenOfLength = IonBinary.lenVarUInt(valueLength); + written_len += writeVarUInt(valueLength, lenOfLength, true); + } + else { + td |= (valueLength & 0xf); + writeTypeDesc(td); + } + return written_len; + } + + public int writeVarInt(int value, int len, boolean force_zero_write) + { + // int len = 0; + + if (value != 0) { + int mask = 0x7F; + boolean is_negative = false; + + assert len == IonBinary.lenVarInt(value); + if (is_negative = (value < 0)) { + value = -value; + } + + // we write the first "byte" separately as it has the sign + // changed shift operator from >> to >>>, we don't want to + // sign extend, this is only a problem with MIN_VALUE where + // negating the value (above) still results in a negative number + int b = (byte)((value >>> (7*(len-1))) & mask); + if (is_negative) b |= 0x40; // the sign bit only on the first byte + if (len == 1) b |= 0x80; // the terminator in case the first "byte" is the last + write((byte)b); + + // write the rest + switch (len) { // we already wrote 1 byte + case 5: write((byte)((value >> (7*3)) & mask)); + case 4: write((byte)((value >> (7*2)) & mask)); + case 3: write((byte)((value >> (7*1)) & mask)); + case 2: write((byte)((value & mask) | 0x80)); + case 1: // do nothing + } + } + else if (force_zero_write) { + write((byte)0x80); + assert len == 1; + + } + else { + assert len == 0; + } + return len; + } + + public int writeVarInt(int value, boolean force_zero_write) + { + int len = IonBinary.lenVarInt(value); + len = writeVarInt(value, len, force_zero_write); + return len; + } + + public int writeVarUInt(int value, int len, boolean force_zero_write) + { + int mask = 0x7F; + if (value < 0) { + throw new IllegalArgumentException("signed int where unsigned (>= 0) was expected"); + } + assert len == IonBinary.lenVarUInt(value); + + switch (len - 1) { + case 4: write((byte)((value >> (7*4)) & mask)); + case 3: write((byte)((value >> (7*3)) & mask)); + case 2: write((byte)((value >> (7*2)) & mask)); + case 1: write((byte)((value >> (7*1)) & mask)); + case 0: write((byte)((value & mask) | 0x80L)); + break; + case -1: // or 0 + if (force_zero_write) { + write((byte)0x80); + len = 1; + } + break; + } + return len; + } + + public int writeVarUInt(int value, boolean force_zero_write) + { + int len = IonBinary.lenVarUInt(value); + len = writeVarUInt(value, len, force_zero_write); + return len; + } + + public int writeIonInt(int value, int len) + { + return writeIonInt((long)value, len); + } + + public int writeIonInt(long value, int len) + { + // we shouldn't be writing out 0's as an Ion int value + if (value == 0) { + assert len == 0; + return len; // aka 0 + } + + // figure out how many we have bytes we have to write out + long mask = 0xffL; + boolean is_negative = (value < 0); + + assert len == IonBinary.lenIonInt(value); + + if (is_negative) { + value = -value; + // note for Long.MIN_VALUE the negation returns + // itself as a value, but that's also the correct + // "positive" value to write out anyway, so it + // all works out + } + + // write the rest + switch (len) { // we already wrote 1 byte + case 8: write((byte)((value >> (8*7)) & mask)); + case 7: write((byte)((value >> (8*6)) & mask)); + case 6: write((byte)((value >> (8*5)) & mask)); + case 5: write((byte)((value >> (8*4)) & mask)); + case 4: write((byte)((value >> (8*3)) & mask)); + case 3: write((byte)((value >> (8*2)) & mask)); + case 2: write((byte)((value >> (8*1)) & mask)); + case 1: write((byte)(value & mask)); + } + + return len; + } + + public int writeVarInt(long value, int len, boolean forceZeroWrite) + { + //int len = 0; + + if (value != 0) { + long mask = 0x7fL; + assert len == IonBinary.lenInt(value); + int b; + if (value < 0) { + value = -value; + // we write the first "byte" separately as it has the sign + // and we have to deal with the oddball MIN_VALUE case + if (value == Long.MIN_VALUE) { + // we use the shift without sign extension as this + // represents a positive value (even though it's neg) + b = (byte)((value >>> (7*len)) & mask); + // len must be greater than 1 so we don't need to set the high bit + } + else { + // here, and hereafter, we don't care about sign extension + b = (byte)((value >>> (7*len)) & mask); + if (len == 1) b |= 0x80; // the terminator in case the first "byte" is the last + } + // we don't worry about stepping on a data bit here as + // we have extra bits at the max size and below that we've + // already taken the sign bit into account + b |= 0x40; // the sign bit + write((byte)b); + } + else { + // we write the first "byte" separately as it has the sign + b = (byte)((value >>> (7*len)) & mask); + if (len == 1) b |= 0x80; // the terminator in case the first "byte" is the last + write((byte)b); + } + + // write the rest + switch (len - 1) { // we already wrote 1 byte + case 9: write((byte)((value >>> (7*9)) & mask)); + case 8: write((byte)((value >> (7*8)) & mask)); + case 7: write((byte)((value >> (7*7)) & mask)); + case 6: write((byte)((value >> (7*6)) & mask)); + case 5: write((byte)((value >> (7*5)) & mask)); + case 4: write((byte)((value >> (7*4)) & mask)); + case 3: write((byte)((value >> (7*3)) & mask)); + case 2: write((byte)((value >> (7*2)) & mask)); + case 1: write((byte)((value >> (7*1)) & mask)); + case 0: write((byte)((value & mask) | 0x80L)); + } + } + else if (forceZeroWrite) { + write((byte)0x80); + assert len == 1; + } + else { + assert len == 0; + } + return len; + } + + public int writeVarUInt(long value, int len, boolean force_zero_write) + { + int mask = 0x7F; + assert len == IonBinary.lenVarUInt(value); + assert value > 0; + + switch (len - 1) { + case 9: write((byte)((value >> (7*9)) & mask)); + case 8: write((byte)((value >> (7*8)) & mask)); + case 7: write((byte)((value >> (7*7)) & mask)); + case 6: write((byte)((value >> (7*6)) & mask)); + case 5: write((byte)((value >> (7*5)) & mask)); + case 4: write((byte)((value >> (7*4)) & mask)); + case 3: write((byte)((value >> (7*3)) & mask)); + case 2: write((byte)((value >> (7*2)) & mask)); + case 1: write((byte)((value >> (7*1)) & mask)); + case 0: write((byte)((value & mask) | 0x80L)); + break; + case -1: // or len == 0 + if (force_zero_write) { + write((byte)0x80); + assert len == 1; + } + else { + assert len == 0; + } + break; + } + return len; + } + + public int writeULong(long value, int lenToWrite) throws IOException + { + switch (lenToWrite) { + case 8: write((byte)((value >> 56) & 0xffL)); + case 7: write((byte)((value >> 48) & 0xffL)); + case 6: write((byte)((value >> 40) & 0xffL)); + case 5: write((byte)((value >> 32) & 0xffL)); + case 4: write((byte)((value >> 24) & 0xffL)); + case 3: write((byte)((value >> 16) & 0xffL)); + case 2: write((byte)((value >> 8) & 0xffL)); + case 1: write((byte)((value >> 0) & 0xffL)); + } + return lenToWrite; + } + + public int writeFloat(double value) throws IOException + { + if (Double.valueOf(value).equals(DOUBLE_POS_ZERO)) + { + // pos zero special case + return 0; + } + + // TODO write "custom" serialization or verify that + // the java routine is doing the right thing + long dBits = Double.doubleToRawLongBits(value); + return this.writeULong(dBits, _ib_FLOAT64_LEN); + } + + public int writeDecimal(BigDecimal value) throws IOException + { + int returnlen = writeDecimal(value, null); + return returnlen; + } + + // this is a private version that supports both the buffer write + // and the output to the user stream. This is slower (since there + // are extra tests, an extra call, and an extra paramenter), but + // it avoids duplicating this code. And decimals are generally + // expensive anyway. + private int writeDecimal(BigDecimal value, UserByteWriter userWriter) throws IOException + { + int returnlen = 0; + // we only write out the '0' value as the nibble 0 + if (value != null && !BigDecimal.ZERO.equals(value)) { + // otherwise we do it the hard way .... + BigInteger mantissa = value.unscaledValue(); + + boolean isNegative = (mantissa.compareTo(BigInteger.ZERO) < 0); + if (isNegative) { + mantissa = mantissa.negate(); + } + + byte[] bits = mantissa.toByteArray(); + int scale = value.scale(); + + // Ion stores exponent, BigDecimal uses the negation "scale" + int exponent = -scale; + if (userWriter != null) { + returnlen += userWriter.writeIonInt(exponent, IonBinary.lenVarUInt(exponent)); + } + else { + returnlen += this.writeIonInt(exponent, IonBinary.lenVarUInt(exponent)); + } + + // If the first bit is set, we can't use it for the sign, + // and we need to write an extra byte to hold it. + boolean needExtraByteForSign = ((bits[0] & 0x80) != 0); + if (needExtraByteForSign) + { + if (userWriter != null) { + userWriter.write((byte)(isNegative ? 0x80 : 0x00)); + } + else { + this.write((byte)(isNegative ? 0x80 : 0x00)); + } + returnlen++; + } + else if (isNegative) + { + bits[0] |= 0x80; + } + // if we have a userWriter to write to, we really don't care about + // the value in our local buffer. + if (userWriter != null) { + userWriter.write(bits, 0, bits.length); + } + else { + this.write(bits, 0, bits.length); + } + returnlen += bits.length; + } + return returnlen; + } + + public int writeTimestamp(Timestamp di) + throws IOException + { + if (di == null) return 0; + + int returnlen = 0; + Precision precision = di.getPrecision(); + + Integer offset = di.getLocalOffset(); + if (offset == null) { + // TODO don't use magic numbers! + this.write((byte)(0xff & (0x80 | 0x40))); // negative 0 (no timezone) + returnlen ++; + } + else { + int value = offset.intValue(); + returnlen += this.writeVarInt(value, true); + } + + // now the date - year, month, day as varUint7's + // if we have a non-null value we have at least the date + if (precision.includes(Precision.YEAR)) { + returnlen += this.writeVarUInt(di.getZYear(), true); + } + if (precision.includes(Precision.MONTH)) { + returnlen += this.writeVarUInt(di.getZMonth(), true); + } + if (precision.includes(Precision.DAY)) { + returnlen += this.writeVarUInt(di.getZDay(), true); + } + + // now the time part + if (precision.includes(Precision.MINUTE)) { + returnlen += this.writeVarUInt(di.getZHour(), true); + returnlen += this.writeVarUInt(di.getZMinute(), true); + } + if (precision.includes(Precision.SECOND)) { + returnlen += this.writeVarUInt(di.getZSecond(), true); + BigDecimal fraction = di.getZFractionalSecond(); + if (fraction != null) { + // and, finally, any fractional component that is known + returnlen += this.writeDecimal(di.getZFractionalSecond()); + } + } + return returnlen; + } + + final public int writeString(String value) throws IOException + { + int returnlen = writeString(value, null); + return returnlen; + } + + final private int writeString(String value, UserByteWriter userWriter) throws IOException + { + int len = 0; + + for (int ii=0; ii 127) { + if (c >= 0xD800 && c <= 0xDFFF) { + if (_Private_IonConstants.isHighSurrogate(c)) { + ii++; + // houston we have a high surrogate (let's hope it has a partner + if (ii >= value.length()) { + throw new IonException("invalid string, unpaired high surrogate character"); + } + int c2 = value.charAt(ii); + if (!_Private_IonConstants.isLowSurrogate(c2)) { + throw new IonException("invalid string, unpaired high surrogate character"); + } + c = _Private_IonConstants.makeUnicodeScalar(c, c2); + } + else if (_Private_IonConstants.isLowSurrogate(c)) { + // it's a loner low surrogate - that's an error + throw new IonException("invalid string, unpaired low surrogate character"); + } + // from 0xE000 up the _writeUnicodeScalar will check for us + } + c = IonBinary.makeUTF8IntFromScalar(c); + } + + // write it here - try to test userWriter as little as possible + if (userWriter == null) { + for (;;) { + write((byte)(c & 0xff)); + len++; + if ((c & 0xffffff00) == 0) { + break; + } + c = c >>> 8; + } + } + else { + for (;;) { + userWriter.write((byte)(c & 0xff)); + len++; + if ((c & 0xffffff00) == 0) { + break; + } + c = c >>> 8; + } + } + } + return len; + } + void throwUTF8Exception() throws IOException + { + throwException("Invalid UTF-8 character encounter in a string at pos " + this.position()); + } + void throwException(String msg) throws IOException { + throw new IOException(msg); + } + } +} diff --git a/src/software/amazon/ion/impl/SubstituteSymbolTable.java b/src/com/amazon/ion/impl/SubstituteSymbolTable.java similarity index 90% rename from src/software/amazon/ion/impl/SubstituteSymbolTable.java rename to src/com/amazon/ion/impl/SubstituteSymbolTable.java index 1189200206..368cfcc1ae 100644 --- a/src/software/amazon/ion/impl/SubstituteSymbolTable.java +++ b/src/com/amazon/ion/impl/SubstituteSymbolTable.java @@ -1,27 +1,28 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonWriter; +import com.amazon.ion.ReadOnlyValueException; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; import java.io.IOException; import java.util.Collections; import java.util.Iterator; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonWriter; -import software.amazon.ion.ReadOnlyValueException; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; /** * A symbol table used for an import where no exact match could be found. diff --git a/src/software/amazon/ion/impl/SymbolTableReader.java b/src/com/amazon/ion/impl/SymbolTableReader.java similarity index 93% rename from src/software/amazon/ion/impl/SymbolTableReader.java rename to src/com/amazon/ion/impl/SymbolTableReader.java index ba5ab1db8e..08afa1fcbd 100644 --- a/src/software/amazon/ion/impl/SymbolTableReader.java +++ b/src/com/amazon/ion/impl/SymbolTableReader.java @@ -1,50 +1,51 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; - - -import static software.amazon.ion.SystemSymbols.IMPORTS; -import static software.amazon.ion.SystemSymbols.IMPORTS_SID; -import static software.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE; -import static software.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE_SID; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; -import static software.amazon.ion.SystemSymbols.MAX_ID; -import static software.amazon.ion.SystemSymbols.MAX_ID_SID; -import static software.amazon.ion.SystemSymbols.NAME; -import static software.amazon.ion.SystemSymbols.NAME_SID; -import static software.amazon.ion.SystemSymbols.SYMBOLS; -import static software.amazon.ion.SystemSymbols.SYMBOLS_SID; -import static software.amazon.ion.SystemSymbols.VERSION; -import static software.amazon.ion.SystemSymbols.VERSION_SID; -import static software.amazon.ion.impl.PrivateUtils.newSymbolToken; - +package com.amazon.ion.impl; + + +import static com.amazon.ion.SystemSymbols.IMPORTS; +import static com.amazon.ion.SystemSymbols.IMPORTS_SID; +import static com.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE; +import static com.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE_SID; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; +import static com.amazon.ion.SystemSymbols.MAX_ID; +import static com.amazon.ion.SystemSymbols.MAX_ID_SID; +import static com.amazon.ion.SystemSymbols.NAME; +import static com.amazon.ion.SystemSymbols.NAME_SID; +import static com.amazon.ion.SystemSymbols.SYMBOLS; +import static com.amazon.ion.SystemSymbols.SYMBOLS_SID; +import static com.amazon.ion.SystemSymbols.VERSION; +import static com.amazon.ion.SystemSymbols.VERSION_SID; +import static com.amazon.ion.impl._Private_Utils.newSymbolToken; + +import com.amazon.ion.Decimal; +import com.amazon.ion.IntegerSize; +import com.amazon.ion.IonException; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonType; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.Timestamp; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.util.Arrays; import java.util.Date; import java.util.Iterator; -import software.amazon.ion.Decimal; -import software.amazon.ion.IntegerSize; -import software.amazon.ion.IonException; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonType; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.Timestamp; /** * This is a reader that traverses a {@link SymbolTable} @@ -547,7 +548,14 @@ final boolean hasLocalSymbols() { return flag_state; } - private final boolean has_next() + + public boolean hasNext() + { + boolean has_next = has_next_helper(); + return has_next; + } + + private final boolean has_next_helper() { // this just tells us whether or not we have more // value coming at our current scanning depth @@ -752,7 +760,7 @@ private final int stateFollowingLocalSymbols() */ public IonType next() { - if (has_next() == false) { + if (has_next_helper() == false) { return null; } int new_state; @@ -1072,7 +1080,7 @@ public String[] getTypeAnnotations() } return new String[] { ION_SHARED_SYMBOL_TABLE }; } - return PrivateUtils.EMPTY_STRING_ARRAY; + return _Private_Utils.EMPTY_STRING_ARRAY; } private static final SymbolToken ION_SYMBOL_TABLE_SYM = @@ -1104,7 +1112,51 @@ public SymbolToken[] getTypeAnnotationSymbols() public Iterator iterateTypeAnnotations() { String[] annotations = getTypeAnnotations(); - return PrivateUtils.stringIterator(annotations); + return _Private_Utils.stringIterator(annotations); + } + + + public int getFieldId() + { + + switch (_current_state) + { + case S_STRUCT: + case S_IN_STRUCT: + case S_IN_IMPORTS: + case S_IMPORT_STRUCT: + case S_IN_IMPORT_STRUCT: + case S_IMPORT_STRUCT_CLOSE: + case S_IMPORT_LIST_CLOSE: + case S_AFTER_IMPORT_LIST: + case S_IN_SYMBOLS: + case S_SYMBOL: + case S_SYMBOL_LIST_CLOSE: + case S_STRUCT_CLOSE: + case S_EOF: + return SymbolTable.UNKNOWN_SYMBOL_ID; + + case S_NAME: + case S_IMPORT_NAME: + return NAME_SID; + + case S_VERSION: + case S_IMPORT_VERSION: + return VERSION_SID; + + case S_MAX_ID: + case S_IMPORT_MAX_ID: + return MAX_ID_SID; + + case S_IMPORT_LIST: + return IMPORTS_SID; + + case S_SYMBOL_LIST: + return SYMBOLS_SID; + + default: + throw new IonException("Internal error: UnifiedSymbolTableReader is in an unrecognized state: "+_current_state); + } } public String getFieldName() diff --git a/src/com/amazon/ion/impl/SymbolTokenImpl.java b/src/com/amazon/ion/impl/SymbolTokenImpl.java new file mode 100644 index 0000000000..91abb3b8b0 --- /dev/null +++ b/src/com/amazon/ion/impl/SymbolTokenImpl.java @@ -0,0 +1,93 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import static com.amazon.ion.util.IonTextUtils.printString; + +import com.amazon.ion.SymbolToken; +import com.amazon.ion.UnknownSymbolException; + + +final class SymbolTokenImpl + implements _Private_SymbolToken +{ + private final String myText; + private final int mySid; + + SymbolTokenImpl(String text, int sid) + { + assert text != null || sid >= 0 : "Neither text nor sid is defined"; + + myText = text; + mySid = sid; + } + + SymbolTokenImpl(int sid) + { + assert sid >= 0 : "sid is undefined"; + + myText = null; + mySid = sid; + } + + + public String getText() + { + return myText; + } + + public String assumeText() + { + if (myText == null) throw new UnknownSymbolException(mySid); + return myText; + } + + public int getSid() + { + return mySid; + } + + @Override + public String toString() + { + return "SymbolToken::{text:" + myText + ",id:" + mySid + "}"; + } + + /* + * TODO amzn/ion-java#126 + *Equals and hashCode must be symmetric. + *Two symboltokens are only equal when text1 equals text2 (including null == null) + *This is an incomplete solution, needs to be updated as symboltokens are fleshed out. + */ + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + SymbolTokenImpl other = (SymbolTokenImpl) o; + if(myText == null || other.myText == null){ + return myText == other.myText; + } + return myText.equals(other.myText); + } + + @Override + public int hashCode() { + if(myText != null) return myText.hashCode(); + return 0; + } +} diff --git a/src/com/amazon/ion/impl/SystemValueIterator.java b/src/com/amazon/ion/impl/SystemValueIterator.java new file mode 100644 index 0000000000..452b4bd6da --- /dev/null +++ b/src/com/amazon/ion/impl/SystemValueIterator.java @@ -0,0 +1,88 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonValue; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.impl.IonBinary.BufferManager; +import java.io.Closeable; +import java.io.IOException; +import java.util.Iterator; + + +interface SystemValueIterator + extends Iterator, Closeable +{ + /******************************************************************** + * + * Iterator + * + */ + public boolean hasNext(); + public IonValue next(); + public void remove(); + + /******************************************************************** + * + * SystemReader + * + */ + + // constructors in original SystemReader: + /* make these static + public SystemReader makeSystemReader(IonSystemImpl system, String s); + public SystemReader makeSystemReader(IonSystemImpl system, + IonCatalog catalog, + Reader input); + public SystemReader makeSystemReader(IonSystemImpl system, + IonCatalog catalog, + SymbolTable initialSymboltable, + Reader input); + @Deprecated + public SystemReader makeSystemReader(IonSystemImpl system, + IonCatalog catalog, + BufferManager buffer); + + public SystemReader makeSystemReader(IonSystemImpl system, + IonCatalog catalog, + InputStream stream); + */ + + public IonSystem getSystem(); + public IonCatalog getCatalog(); + + /** + * Returns the current symtab, either system or local. + * @return may be null. + */ + public SymbolTable getSymbolTable(); + + /** + * Gets the current symtab if its local, otherwise creates a local symtab + * and makes it current. + * @return not null. + */ + public SymbolTable getLocalSymbolTable(); + + public boolean currentIsHidden(); +// public boolean canSetLocalSymbolTable(); +// public void setLocalSymbolTable(SymbolTable symbolTable); + public BufferManager getBuffer(); + public void resetBuffer(); + public void close() throws IOException; +} diff --git a/src/software/amazon/ion/impl/UnifiedDataPageX.java b/src/com/amazon/ion/impl/UnifiedDataPageX.java similarity index 95% rename from src/software/amazon/ion/impl/UnifiedDataPageX.java rename to src/com/amazon/ion/impl/UnifiedDataPageX.java index 48324c956b..769f4933d4 100644 --- a/src/software/amazon/ion/impl/UnifiedDataPageX.java +++ b/src/com/amazon/ion/impl/UnifiedDataPageX.java @@ -1,18 +1,19 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; import java.io.IOException; import java.io.InputStream; diff --git a/src/software/amazon/ion/impl/UnifiedInputBufferX.java b/src/com/amazon/ion/impl/UnifiedInputBufferX.java similarity index 96% rename from src/software/amazon/ion/impl/UnifiedInputBufferX.java rename to src/com/amazon/ion/impl/UnifiedInputBufferX.java index c0b9854a1e..c8444883cb 100644 --- a/src/software/amazon/ion/impl/UnifiedInputBufferX.java +++ b/src/com/amazon/ion/impl/UnifiedInputBufferX.java @@ -1,18 +1,19 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; abstract class UnifiedInputBufferX diff --git a/src/software/amazon/ion/impl/UnifiedInputStreamX.java b/src/com/amazon/ion/impl/UnifiedInputStreamX.java similarity index 97% rename from src/software/amazon/ion/impl/UnifiedInputStreamX.java rename to src/com/amazon/ion/impl/UnifiedInputStreamX.java index b78e789e79..f962946191 100644 --- a/src/software/amazon/ion/impl/UnifiedInputStreamX.java +++ b/src/com/amazon/ion/impl/UnifiedInputStreamX.java @@ -1,25 +1,26 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; +import com.amazon.ion.impl.IonReaderTextRawTokensX.IonReaderTextTokenException; +import com.amazon.ion.impl.UnifiedSavePointManagerX.SavePoint; import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.io.Reader; -import software.amazon.ion.impl.IonReaderTextRawTokensX.IonReaderTextTokenException; -import software.amazon.ion.impl.UnifiedSavePointManagerX.SavePoint; /** * This is a local stream abstraction, and implementation, that diff --git a/src/software/amazon/ion/impl/UnifiedSavePointManagerX.java b/src/com/amazon/ion/impl/UnifiedSavePointManagerX.java similarity index 97% rename from src/software/amazon/ion/impl/UnifiedSavePointManagerX.java rename to src/com/amazon/ion/impl/UnifiedSavePointManagerX.java index 46beb6ba2d..aa8bddf0de 100644 --- a/src/software/amazon/ion/impl/UnifiedSavePointManagerX.java +++ b/src/com/amazon/ion/impl/UnifiedSavePointManagerX.java @@ -1,18 +1,19 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; /** diff --git a/src/com/amazon/ion/impl/_Private_ByteTransferReader.java b/src/com/amazon/ion/impl/_Private_ByteTransferReader.java new file mode 100644 index 0000000000..b1b9d95d35 --- /dev/null +++ b/src/com/amazon/ion/impl/_Private_ByteTransferReader.java @@ -0,0 +1,29 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import com.amazon.ion.IonReader; +import java.io.IOException; + +/** + * An {@link IonReader} {@linkplain com.amazon.ion.facet facet} that can rapidly bulk-copy + * Ion binary data under certain circumstances. + */ +public interface _Private_ByteTransferReader +{ + public void transferCurrentValue(_Private_ByteTransferSink writer) + throws IOException; +} diff --git a/src/software/amazon/ion/impl/PrivateByteTransferSink.java b/src/com/amazon/ion/impl/_Private_ByteTransferSink.java similarity index 53% rename from src/software/amazon/ion/impl/PrivateByteTransferSink.java rename to src/com/amazon/ion/impl/_Private_ByteTransferSink.java index 6838f02ccb..1f26745e49 100644 --- a/src/software/amazon/ion/impl/PrivateByteTransferSink.java +++ b/src/com/amazon/ion/impl/_Private_ByteTransferSink.java @@ -1,30 +1,28 @@ /* - * Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; import java.io.IOException; /** - * A destination sink that can be fed bytes. The typical usage is a {@link PrivateByteTransferReader} that funnels data + * A destination sink that can be fed bytes. The typical usage is a {@link _Private_ByteTransferReader} that funnels data * to an binary Ion target. - * - * @deprecated This is an internal API that is subject to change without notice. */ -@Deprecated -public interface PrivateByteTransferSink +public interface _Private_ByteTransferSink { /** * Writes the given data to the sink. diff --git a/src/com/amazon/ion/impl/_Private_CallbackBuilder.java b/src/com/amazon/ion/impl/_Private_CallbackBuilder.java new file mode 100644 index 0000000000..2184d87829 --- /dev/null +++ b/src/com/amazon/ion/impl/_Private_CallbackBuilder.java @@ -0,0 +1,30 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import com.amazon.ion.util._Private_FastAppendable; + +/** + * NOT FOR APPLICATION USE! + *

    + * Interface used by {@link _Private_IonTextWriterBuilder} to allow + * instantiation of a new {@link _Private_MarkupCallback} during every + * IonTextWriterBuilder.build; + */ +public interface _Private_CallbackBuilder +{ + _Private_MarkupCallback build(_Private_FastAppendable rawOutput); +} diff --git a/src/software/amazon/ion/impl/PrivateCommandLine.java b/src/com/amazon/ion/impl/_Private_CommandLine.java similarity index 72% rename from src/software/amazon/ion/impl/PrivateCommandLine.java rename to src/com/amazon/ion/impl/_Private_CommandLine.java index bbe46287e6..2296d400e6 100644 --- a/src/software/amazon/ion/impl/PrivateCommandLine.java +++ b/src/com/amazon/ion/impl/_Private_CommandLine.java @@ -1,30 +1,30 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.system.IonTextWriterBuilder; +import com.amazon.ion.util.JarInfo; import java.io.IOException; -import software.amazon.ion.IonType; -import software.amazon.ion.IonWriter; -import software.amazon.ion.system.IonTextWriterBuilder; -import software.amazon.ion.util.JarInfo; /** - * @deprecated This is an internal API that is subject to change without notice. + * NOT FOR APPLICATION USE! */ -@Deprecated -public final class PrivateCommandLine +public final class _Private_CommandLine { static final int argid_VERSION = 2; static final int argid_HELP = 3; @@ -47,6 +47,7 @@ public static void main(String[] args) throws IOException process_command_line(args); info = new JarInfo(); + if (printVersion) { doPrintVersion(); } @@ -79,11 +80,23 @@ private static void process_command_line(String[] args) } private static int getArgumentId(String arg) { - if (arg.equals("help")) { - return argid_HELP; + if (arg.startsWith("-") && arg.length() == 2) { + switch (arg.charAt(1)) { + case 'h': case '?': + return argid_HELP; + case 'v': + return argid_VERSION; + default: + return argid_INVALID; + } } - if (arg.equals("version")) { - return argid_VERSION; + if (arg.startsWith("--") && arg.length() > 2) { + if (arg.equals("help")) { + return argid_HELP; + } + if (arg.equals("version")) { + return argid_VERSION; + } } return argid_INVALID; } diff --git a/src/software/amazon/ion/impl/PrivateCurriedValueFactory.java b/src/com/amazon/ion/impl/_Private_CurriedValueFactory.java similarity index 82% rename from src/software/amazon/ion/impl/PrivateCurriedValueFactory.java rename to src/com/amazon/ion/impl/_Private_CurriedValueFactory.java index 1e3c1e2f8f..eb492f5dc7 100644 --- a/src/software/amazon/ion/impl/PrivateCurriedValueFactory.java +++ b/src/com/amazon/ion/impl/_Private_CurriedValueFactory.java @@ -1,51 +1,52 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; - +package com.amazon.ion.impl; + +import com.amazon.ion.ContainedValueException; +import com.amazon.ion.IonBlob; +import com.amazon.ion.IonBool; +import com.amazon.ion.IonClob; +import com.amazon.ion.IonDecimal; +import com.amazon.ion.IonException; +import com.amazon.ion.IonFloat; +import com.amazon.ion.IonInt; +import com.amazon.ion.IonList; +import com.amazon.ion.IonNull; +import com.amazon.ion.IonSequence; +import com.amazon.ion.IonSexp; +import com.amazon.ion.IonString; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSymbol; +import com.amazon.ion.IonTimestamp; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.Timestamp; +import com.amazon.ion.ValueFactory; import java.math.BigDecimal; import java.math.BigInteger; -import software.amazon.ion.ContainedValueException; -import software.amazon.ion.IonBlob; -import software.amazon.ion.IonBool; -import software.amazon.ion.IonClob; -import software.amazon.ion.IonDecimal; -import software.amazon.ion.IonException; -import software.amazon.ion.IonFloat; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonList; -import software.amazon.ion.IonNull; -import software.amazon.ion.IonSequence; -import software.amazon.ion.IonSexp; -import software.amazon.ion.IonString; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonTimestamp; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.Timestamp; -import software.amazon.ion.ValueFactory; +import java.util.Collection; /** + * NOT FOR APPLICATION USE! + *

    * Helper for implementing curried container insertion methods such as * {@link IonStruct#put(String)}. - * - * @deprecated This is an internal API that is subject to change without notice. */ -@Deprecated -public abstract class PrivateCurriedValueFactory +public abstract class _Private_CurriedValueFactory implements ValueFactory { private final ValueFactory myFactory; @@ -53,7 +54,7 @@ public abstract class PrivateCurriedValueFactory /** * @param factory must not be null. */ - protected PrivateCurriedValueFactory(ValueFactory factory) + protected _Private_CurriedValueFactory(ValueFactory factory) { myFactory = factory; } @@ -241,6 +242,15 @@ public IonList newEmptyList() return v; } + @Deprecated + public IonList newList(Collection values) + throws ContainedValueException, NullPointerException + { + IonList v = myFactory.newList(values); + handle(v); + return v; + } + public IonList newList(IonSequence firstChild) throws ContainedValueException, NullPointerException { @@ -303,6 +313,15 @@ public IonSexp newEmptySexp() return v; } + @Deprecated + public IonSexp newSexp(Collection values) + throws ContainedValueException, NullPointerException + { + IonSexp v = myFactory.newSexp(values); + handle(v); + return v; + } + public IonSexp newSexp(IonSequence firstChild) throws ContainedValueException, NullPointerException { diff --git a/src/software/amazon/ion/impl/PrivateFastAppendableDecorator.java b/src/com/amazon/ion/impl/_Private_FastAppendableDecorator.java similarity index 69% rename from src/software/amazon/ion/impl/PrivateFastAppendableDecorator.java rename to src/com/amazon/ion/impl/_Private_FastAppendableDecorator.java index a501523744..ac1246e4de 100644 --- a/src/software/amazon/ion/impl/PrivateFastAppendableDecorator.java +++ b/src/com/amazon/ion/impl/_Private_FastAppendableDecorator.java @@ -1,34 +1,34 @@ /* - * Copyright 2013-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; +import com.amazon.ion.util._Private_FastAppendable; import java.io.Closeable; import java.io.Flushable; import java.io.IOException; -import software.amazon.ion.util.PrivateFastAppendable; /** - * @deprecated This is an internal API that is subject to change without notice. + * NOT FOR APPLICATION USE! */ -@Deprecated -public abstract class PrivateFastAppendableDecorator - implements PrivateFastAppendable, Closeable, Flushable +public abstract class _Private_FastAppendableDecorator + implements _Private_FastAppendable, Closeable, Flushable { - private final PrivateFastAppendable myOutput; + private final _Private_FastAppendable myOutput; - public PrivateFastAppendableDecorator(PrivateFastAppendable output) { + public _Private_FastAppendableDecorator(_Private_FastAppendable output) { myOutput = output; } diff --git a/src/com/amazon/ion/impl/_Private_FastAppendableTrampoline.java b/src/com/amazon/ion/impl/_Private_FastAppendableTrampoline.java new file mode 100644 index 0000000000..4440c889d6 --- /dev/null +++ b/src/com/amazon/ion/impl/_Private_FastAppendableTrampoline.java @@ -0,0 +1,38 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import com.amazon.ion.util._Private_FastAppendable; +import java.io.OutputStream; + + +/** + * NOT FOR APPLICATION USE! + */ +public final class _Private_FastAppendableTrampoline +{ + + public static _Private_FastAppendable forAppendable(Appendable appendable) + { + return new AppendableFastAppendable(appendable); + } + + public static _Private_FastAppendable forOutputStream( + OutputStream outputStream) + { + return new OutputStreamFastAppendable(outputStream); + } +} diff --git a/src/software/amazon/ion/impl/PrivateIonBinaryWriterBuilder.java b/src/com/amazon/ion/impl/_Private_IonBinaryWriterBuilder.java similarity index 56% rename from src/software/amazon/ion/impl/PrivateIonBinaryWriterBuilder.java rename to src/com/amazon/ion/impl/_Private_IonBinaryWriterBuilder.java index 96b48e60ba..c92da72b29 100644 --- a/src/software/amazon/ion/impl/PrivateIonBinaryWriterBuilder.java +++ b/src/com/amazon/ion/impl/_Private_IonBinaryWriterBuilder.java @@ -1,52 +1,57 @@ /* - * Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; - +package com.amazon.ion.impl; + +import static com.amazon.ion.impl._Private_Utils.initialSymtab; + +import com.amazon.ion.IonBinaryWriter; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonException; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SubstituteSymbolTableException; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.ValueFactory; +import com.amazon.ion.impl.BlockedBuffer.BufferedOutputStream; +import com.amazon.ion.impl.bin._Private_IonManagedBinaryWriterBuilder; +import com.amazon.ion.impl.bin._Private_IonManagedBinaryWriterBuilder.AllocatorMode; +import com.amazon.ion.system.IonBinaryWriterBuilder; +import com.amazon.ion.system.IonSystemBuilder; import java.io.IOException; import java.io.OutputStream; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonException; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SubstituteSymbolTableException; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.ValueFactory; -import software.amazon.ion.impl.bin.PrivateIonManagedBinaryWriterBuilder; -import software.amazon.ion.impl.bin.PrivateIonManagedBinaryWriterBuilder.AllocatorMode; -import software.amazon.ion.system.IonBinaryWriterBuilder; -import software.amazon.ion.system.IonSystemBuilder; /** - * @deprecated This is an internal API that is subject to change without notice. + * NOT FOR APPLICATION USE! */ -@Deprecated -public class PrivateIonBinaryWriterBuilder +@SuppressWarnings("deprecation") +public class _Private_IonBinaryWriterBuilder extends IonBinaryWriterBuilder { - // amzn/ion-java#59 expose configuration points properly and figure out deprecation path for the old writer. - private final PrivateIonManagedBinaryWriterBuilder myBinaryWriterBuilder; + // amzn/ion-java/issues/59 expose configuration points properly and figure out deprecation path for the old writer. + private final _Private_IonManagedBinaryWriterBuilder myBinaryWriterBuilder; private ValueFactory mySymtabValueFactory; /** System or local */ private SymbolTable myInitialSymbolTable; - private PrivateIonBinaryWriterBuilder() + private _Private_IonBinaryWriterBuilder() { myBinaryWriterBuilder = - PrivateIonManagedBinaryWriterBuilder + _Private_IonManagedBinaryWriterBuilder .create(AllocatorMode.POOLED) .withPaddedLengthPreallocation(0) ; @@ -54,7 +59,7 @@ private PrivateIonBinaryWriterBuilder() private - PrivateIonBinaryWriterBuilder(PrivateIonBinaryWriterBuilder that) + _Private_IonBinaryWriterBuilder(_Private_IonBinaryWriterBuilder that) { super(that); @@ -67,9 +72,9 @@ private PrivateIonBinaryWriterBuilder() /** * @return a new mutable builder. */ - public static PrivateIonBinaryWriterBuilder standard() + public static _Private_IonBinaryWriterBuilder standard() { - return new PrivateIonBinaryWriterBuilder.Mutable(); + return new _Private_IonBinaryWriterBuilder.Mutable(); } @@ -77,19 +82,19 @@ public static PrivateIonBinaryWriterBuilder standard() @Override - public final PrivateIonBinaryWriterBuilder copy() + public final _Private_IonBinaryWriterBuilder copy() { return new Mutable(this); } @Override - public PrivateIonBinaryWriterBuilder immutable() + public _Private_IonBinaryWriterBuilder immutable() { return this; } @Override - public PrivateIonBinaryWriterBuilder mutable() + public _Private_IonBinaryWriterBuilder mutable() { return copy(); } @@ -112,10 +117,10 @@ public void setSymtabValueFactory(ValueFactory factory) mySymtabValueFactory = factory; } - public PrivateIonBinaryWriterBuilder + public _Private_IonBinaryWriterBuilder withSymtabValueFactory(ValueFactory factory) { - PrivateIonBinaryWriterBuilder b = mutable(); + _Private_IonBinaryWriterBuilder b = mutable(); b.setSymtabValueFactory(factory); return b; } @@ -181,9 +186,9 @@ else if (! symtab.isSystemTable()) */ @Override public - PrivateIonBinaryWriterBuilder withInitialSymbolTable(SymbolTable symtab) + _Private_IonBinaryWriterBuilder withInitialSymbolTable(SymbolTable symtab) { - PrivateIonBinaryWriterBuilder b = mutable(); + _Private_IonBinaryWriterBuilder b = mutable(); b.setInitialSymbolTable(symtab); return b; } @@ -203,16 +208,16 @@ public void setIsFloatBinary32Enabled(boolean enabled) { @Override public - PrivateIonBinaryWriterBuilder withFloatBinary32Enabled() { - PrivateIonBinaryWriterBuilder b = mutable(); + _Private_IonBinaryWriterBuilder withFloatBinary32Enabled() { + _Private_IonBinaryWriterBuilder b = mutable(); b.setIsFloatBinary32Enabled(true); return b; } @Override public - PrivateIonBinaryWriterBuilder withFloatBinary32Disabled() { - PrivateIonBinaryWriterBuilder b = mutable(); + _Private_IonBinaryWriterBuilder withFloatBinary32Disabled() { + _Private_IonBinaryWriterBuilder b = mutable(); b.setIsFloatBinary32Enabled(false); return b; } @@ -244,10 +249,10 @@ public void setStreamCopyOptimized(final boolean optimized) /** * Fills all properties and returns an immutable builder. */ - private PrivateIonBinaryWriterBuilder fillDefaults() + private _Private_IonBinaryWriterBuilder fillDefaults() { // Ensure that we don't modify the user's builder. - PrivateIonBinaryWriterBuilder b = copy(); + _Private_IonBinaryWriterBuilder b = copy(); if (b.getSymtabValueFactory() == null) { @@ -258,6 +263,54 @@ private PrivateIonBinaryWriterBuilder fillDefaults() return b.immutable(); } + /** + * Fills all properties and returns an immutable builder. + */ + private _Private_IonBinaryWriterBuilder fillLegacyDefaults() + { + // amzn/ion-java/issues/59 Fix this to use the new writer or eliminate it + + // Ensure that we don't modify the user's builder. + _Private_IonBinaryWriterBuilder b = copy(); + + if (b.getSymtabValueFactory() == null) + { + IonSystem system = IonSystemBuilder.standard().build(); + b.setSymtabValueFactory(system); + } + + SymbolTable initialSymtab = b.getInitialSymbolTable(); + if (initialSymtab == null) + { + initialSymtab = initialSymtab(LocalSymbolTable.DEFAULT_LST_FACTORY, + _Private_Utils.systemSymtab(1), + b.getImports()); + b.setInitialSymbolTable(initialSymtab); + } + else if (initialSymtab.isSystemTable()) + { + initialSymtab = initialSymtab(LocalSymbolTable.DEFAULT_LST_FACTORY, + initialSymtab, + b.getImports()); + b.setInitialSymbolTable(initialSymtab); + } + + return b.immutable(); + } + + + private IonWriterSystemBinary buildSystemWriter(OutputStream out) + { + SymbolTable defaultSystemSymtab = + myInitialSymbolTable.getSystemSymbolTable(); + + return new IonWriterSystemBinary(defaultSystemSymtab, + out, + false /* autoFlush */, + true /* ensureInitialIvm */); + } + + /** * Returns a symtab usable in a local context. * This copies {@link #myInitialSymbolTable} if symbols have been added to @@ -277,7 +330,7 @@ SymbolTable buildContextSymbolTable() @Override public final IonWriter build(OutputStream out) { - PrivateIonBinaryWriterBuilder b = fillDefaults(); + _Private_IonBinaryWriterBuilder b = fillDefaults(); try { return b.myBinaryWriterBuilder.newWriter(out); @@ -288,27 +341,41 @@ public final IonWriter build(OutputStream out) } } + + @Deprecated + public final IonBinaryWriter buildLegacy() + { + // amzn/ion-java/issues/59 Fix this to use the new writer or eliminate it + _Private_IonBinaryWriterBuilder b = fillLegacyDefaults(); + + IonWriterSystemBinary systemWriter = + b.buildSystemWriter(new BufferedOutputStream()); + + return new _Private_IonBinaryWriterImpl(b, systemWriter); + } + + //========================================================================= private static final class Mutable - extends PrivateIonBinaryWriterBuilder + extends _Private_IonBinaryWriterBuilder { private Mutable() { } - private Mutable(PrivateIonBinaryWriterBuilder that) + private Mutable(_Private_IonBinaryWriterBuilder that) { super(that); } @Override - public PrivateIonBinaryWriterBuilder immutable() + public _Private_IonBinaryWriterBuilder immutable() { - return new PrivateIonBinaryWriterBuilder(this); + return new _Private_IonBinaryWriterBuilder(this); } @Override - public PrivateIonBinaryWriterBuilder mutable() + public _Private_IonBinaryWriterBuilder mutable() { return this; } diff --git a/src/com/amazon/ion/impl/_Private_IonBinaryWriterImpl.java b/src/com/amazon/ion/impl/_Private_IonBinaryWriterImpl.java new file mode 100644 index 0000000000..a5570866a9 --- /dev/null +++ b/src/com/amazon/ion/impl/_Private_IonBinaryWriterImpl.java @@ -0,0 +1,84 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import com.amazon.ion.IonBinaryWriter; +import com.amazon.ion.IonException; +import com.amazon.ion.IonWriter; +import com.amazon.ion.impl.BlockedBuffer.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +/** + * NOT FOR APPLICATION USE! + *

    + * Adapts the binary {@link IonWriter} implementation to the deprecated + * {@link IonBinaryWriter} interface. + */ +@Deprecated +public final class _Private_IonBinaryWriterImpl + extends IonWriterUserBinary + implements IonBinaryWriter +{ + _Private_IonBinaryWriterImpl(_Private_IonBinaryWriterBuilder options, + IonWriterSystemBinary systemWriter) + { + super(options, systemWriter); + } + + + private BufferedOutputStream getOutputStream() + { + IonWriterSystemBinary systemWriter = + (IonWriterSystemBinary)_system_writer; + return (BufferedOutputStream) systemWriter.getOutputStream(); + } + + + public int byteSize() + { + try { + finish(); + } + catch (IOException e) { + throw new IonException(e); + } + int size = getOutputStream().byteSize(); + return size; + } + + public byte[] getBytes() throws IOException + { + finish(); + byte[] bytes = getOutputStream().getBytes(); + return bytes; + } + + public int getBytes(byte[] bytes, int offset, int len) + throws IOException + { + finish(); + int written = getOutputStream().getBytes(bytes, offset, len); + return written; + } + + public int writeBytes(OutputStream userstream) throws IOException + { + finish(); + int written = getOutputStream().writeBytes(userstream); + return written; + } +} diff --git a/src/software/amazon/ion/impl/PrivateIonConstants.java b/src/com/amazon/ion/impl/_Private_IonConstants.java similarity index 92% rename from src/software/amazon/ion/impl/PrivateIonConstants.java rename to src/com/amazon/ion/impl/_Private_IonConstants.java index 505c32695b..38b9b1ee97 100644 --- a/src/software/amazon/ion/impl/PrivateIonConstants.java +++ b/src/com/amazon/ion/impl/_Private_IonConstants.java @@ -1,28 +1,28 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; -import software.amazon.ion.IonException; +import com.amazon.ion.IonException; /** - * @deprecated This is an internal API that is subject to change without notice. + * NOT FOR APPLICATION USE! */ -@Deprecated -public final class PrivateIonConstants +public final class _Private_IonConstants { - private PrivateIonConstants() { } + private _Private_IonConstants() { } public final static int BB_TOKEN_LEN = 1; @@ -268,12 +268,12 @@ public static final int getLowNibble(int td) } public static final int True = - makeTypeDescriptor(PrivateIonConstants.tidBoolean, - PrivateIonConstants.lnBooleanTrue); + makeTypeDescriptor(_Private_IonConstants.tidBoolean, + _Private_IonConstants.lnBooleanTrue); public static final int False = - makeTypeDescriptor(PrivateIonConstants.tidBoolean, - PrivateIonConstants.lnBooleanFalse); + makeTypeDescriptor(_Private_IonConstants.tidBoolean, + _Private_IonConstants.lnBooleanFalse); /** * Prefix string used in IonStructs' equality checks. @@ -291,7 +291,7 @@ public static final int getLowNibble(int td) * * *

    - * TODO amzn/ion-java#23 However, there is still a potential failure if one of the + * TODO amzn/ion-java/issues/23 However, there is still a potential failure if one of the * IonStruct's nested value has a field name with text * {@code " -- UNKNOWN SYMBOL TEXT -- $123"}, and that another nested value * of an IonStruct has a field name with unknown text and sid 123, these diff --git a/src/com/amazon/ion/impl/_Private_IonContainer.java b/src/com/amazon/ion/impl/_Private_IonContainer.java new file mode 100644 index 0000000000..a77ab51a81 --- /dev/null +++ b/src/com/amazon/ion/impl/_Private_IonContainer.java @@ -0,0 +1,32 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import com.amazon.ion.IonContainer; +import com.amazon.ion.IonValue; + +/** + * NOT FOR APPLICATION USE! + *

    + * Internal, private, interfaces for manipulating + * the base child collection of IonContainer + */ +public interface _Private_IonContainer + extends IonContainer +{ + public int get_child_count(); + public IonValue get_child(int idx); +} diff --git a/src/com/amazon/ion/impl/_Private_IonDatagram.java b/src/com/amazon/ion/impl/_Private_IonDatagram.java new file mode 100644 index 0000000000..2a9f15f5c5 --- /dev/null +++ b/src/com/amazon/ion/impl/_Private_IonDatagram.java @@ -0,0 +1,28 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import com.amazon.ion.IonDatagram; +import com.amazon.ion.SymbolTable; + +/** + * NOT FOR APPLICATION USE! + */ +public interface _Private_IonDatagram + extends _Private_IonValue, IonDatagram +{ + void appendTrailingSymbolTable(SymbolTable symtab); +} diff --git a/src/software/amazon/ion/impl/PrivateIonReaderFactory.java b/src/com/amazon/ion/impl/_Private_IonReaderFactory.java similarity index 75% rename from src/software/amazon/ion/impl/PrivateIonReaderFactory.java rename to src/com/amazon/ion/impl/_Private_IonReaderFactory.java index 28f7cc4c31..8239e3ff50 100644 --- a/src/software/amazon/ion/impl/PrivateIonReaderFactory.java +++ b/src/com/amazon/ion/impl/_Private_IonReaderFactory.java @@ -1,41 +1,44 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; -import static software.amazon.ion.impl.PrivateIonConstants.BINARY_VERSION_MARKER_SIZE; -import static software.amazon.ion.impl.UnifiedInputStreamX.makeStream; -import static software.amazon.ion.util.IonStreamUtils.isIonBinary; +import static com.amazon.ion.impl.UnifiedInputStreamX.makeStream; +import static com.amazon.ion.impl._Private_IonConstants.BINARY_VERSION_MARKER_SIZE; +import static com.amazon.ion.util.IonStreamUtils.isIonBinary; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonException; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonTextReader; +import com.amazon.ion.IonValue; +import com.amazon.ion.util.IonStreamUtils; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.Reader; import java.util.zip.GZIPInputStream; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonException; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonValue; -import software.amazon.ion.util.IonStreamUtils; /** - * @deprecated This is an internal API that is subject to change without notice. + * NOT FOR APPLICATION USE! */ -@Deprecated -public final class PrivateIonReaderFactory +@SuppressWarnings("deprecation") +public final class _Private_IonReaderFactory { + public static final IonReader makeReader(IonCatalog catalog, byte[] bytes) { @@ -44,10 +47,10 @@ public static final IonReader makeReader(IonCatalog catalog, public static final IonReader makeReader(IonCatalog catalog, byte[] bytes, - PrivateLocalSymbolTableFactory lstFactory) - { - return makeReader(catalog, bytes, 0, bytes.length, lstFactory); - } + _Private_LocalSymbolTableFactory lstFactory) + { + return makeReader(catalog, bytes, 0, bytes.length, lstFactory); + } public static IonReader makeSystemReader(byte[] bytes) { @@ -75,7 +78,7 @@ public static final IonReader makeReader(IonCatalog catalog, byte[] bytes, int offset, int length, - PrivateLocalSymbolTableFactory lstFactory) + _Private_LocalSymbolTableFactory lstFactory) { try { @@ -95,7 +98,7 @@ public static IonReader makeSystemReader(byte[] bytes, try { UnifiedInputStreamX uis = makeUnifiedStream(bytes, offset, length); - return makeSystemReader(uis, offset); + return makeSystemReader(uis); } catch (IOException e) { @@ -104,8 +107,8 @@ public static IonReader makeSystemReader(byte[] bytes, } - public static final IonReader makeReader(IonCatalog catalog, - char[] chars) + public static final IonTextReader makeReader(IonCatalog catalog, + char[] chars) { return makeReader(catalog, chars, 0, chars.length); } @@ -117,10 +120,10 @@ public static final IonReader makeSystemReader(char[] chars) } - public static final IonReader makeReader(IonCatalog catalog, - char[] chars, - int offset, - int length) + public static final IonTextReader makeReader(IonCatalog catalog, + char[] chars, + int offset, + int length) { UnifiedInputStreamX in = makeStream(chars, offset, length); return new IonReaderTextUserX(catalog, LocalSymbolTable.DEFAULT_LST_FACTORY, in, offset); @@ -135,15 +138,15 @@ public static final IonReader makeSystemReader(char[] chars, } - public static final IonReader makeReader(IonCatalog catalog, - CharSequence chars) + public static final IonTextReader makeReader(IonCatalog catalog, + CharSequence chars) { return makeReader(catalog, chars, LocalSymbolTable.DEFAULT_LST_FACTORY); } - public static final IonReader makeReader(IonCatalog catalog, - CharSequence chars, - PrivateLocalSymbolTableFactory lstFactory) + public static final IonTextReader makeReader(IonCatalog catalog, + CharSequence chars, + _Private_LocalSymbolTableFactory lstFactory) { UnifiedInputStreamX in = makeStream(chars); return new IonReaderTextUserX(catalog, lstFactory, in); @@ -156,10 +159,10 @@ public static final IonReader makeSystemReader(CharSequence chars) } - public static final IonReader makeReader(IonCatalog catalog, - CharSequence chars, - int offset, - int length) + public static final IonTextReader makeReader(IonCatalog catalog, + CharSequence chars, + int offset, + int length) { UnifiedInputStreamX in = makeStream(chars, offset, length); return new IonReaderTextUserX(catalog, LocalSymbolTable.DEFAULT_LST_FACTORY, in, offset); @@ -173,6 +176,7 @@ public static final IonReader makeSystemReader(CharSequence chars, return new IonReaderTextSystemX(in); } + public static final IonReader makeReader(IonCatalog catalog, InputStream is) { @@ -181,7 +185,7 @@ public static final IonReader makeReader(IonCatalog catalog, public static final IonReader makeReader(IonCatalog catalog, InputStream is, - PrivateLocalSymbolTableFactory lstFactory) + _Private_LocalSymbolTableFactory lstFactory) { try { UnifiedInputStreamX uis = makeUnifiedStream(is); @@ -196,22 +200,23 @@ public static IonReader makeSystemReader(InputStream is) { try { UnifiedInputStreamX uis = makeUnifiedStream(is); - return makeSystemReader(uis, 0); + return makeSystemReader(uis); } catch (IOException e) { throw new IonException(e); } } - public static final IonReader makeReader(IonCatalog catalog, - Reader chars) + + public static final IonTextReader makeReader(IonCatalog catalog, + Reader chars) { return makeReader(catalog, chars, LocalSymbolTable.DEFAULT_LST_FACTORY); } - public static final IonReader makeReader(IonCatalog catalog, - Reader chars, - PrivateLocalSymbolTableFactory lstFactory) + public static final IonTextReader makeReader(IonCatalog catalog, + Reader chars, + _Private_LocalSymbolTableFactory lstFactory) { try { UnifiedInputStreamX in = makeStream(chars); @@ -233,7 +238,6 @@ public static final IonReader makeSystemReader(Reader chars) } } - public static final IonReader makeReader(IonCatalog catalog, IonValue value) { @@ -257,7 +261,7 @@ public static final IonReader makeSystemReader(IonSystem system, private static IonReader makeReader(IonCatalog catalog, UnifiedInputStreamX uis, int offset, - PrivateLocalSymbolTableFactory lstFactory) + _Private_LocalSymbolTableFactory lstFactory) throws IOException { IonReader r; @@ -270,8 +274,7 @@ private static IonReader makeReader(IonCatalog catalog, return r; } - private static IonReader makeSystemReader(UnifiedInputStreamX uis, - int offset) + private static IonReader makeSystemReader(UnifiedInputStreamX uis) throws IOException { IonReader r; diff --git a/src/software/amazon/ion/impl/PrivateIonSymbol.java b/src/com/amazon/ion/impl/_Private_IonSymbol.java similarity index 50% rename from src/software/amazon/ion/impl/PrivateIonSymbol.java rename to src/com/amazon/ion/impl/_Private_IonSymbol.java index b3c8f77dd6..6e0dbc1cd6 100644 --- a/src/software/amazon/ion/impl/PrivateIonSymbol.java +++ b/src/com/amazon/ion/impl/_Private_IonSymbol.java @@ -1,28 +1,28 @@ /* - * Copyright 2016-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.impl.PrivateIonValue.SymbolTableProvider; +import com.amazon.ion.IonSymbol; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.impl._Private_IonValue.SymbolTableProvider; /** - * @deprecated This is an internal API that is subject to change without notice. + * NOT FOR APPLICATION USE! */ -@Deprecated -public interface PrivateIonSymbol +public interface _Private_IonSymbol extends IonSymbol { /** diff --git a/src/software/amazon/ion/impl/PrivateIonSystem.java b/src/com/amazon/ion/impl/_Private_IonSystem.java similarity index 58% rename from src/software/amazon/ion/impl/PrivateIonSystem.java rename to src/com/amazon/ion/impl/_Private_IonSystem.java index f2d55b2f60..d9024761f8 100644 --- a/src/software/amazon/ion/impl/PrivateIonSystem.java +++ b/src/com/amazon/ion/impl/_Private_IonSystem.java @@ -1,47 +1,47 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; +import com.amazon.ion.IonContainer; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.system.IonSystemBuilder; import java.io.InputStream; import java.io.Reader; import java.util.Iterator; -import software.amazon.ion.IonContainer; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonValue; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.system.IonSystemBuilder; /** - * @deprecated This is an internal API that is subject to change without notice. + * NOT FOR APPLICATION USE! */ -@Deprecated -public interface PrivateIonSystem +public interface _Private_IonSystem extends IonSystem { public SymbolTable newSharedSymbolTable(IonStruct ionRep); /** - * TODO Must correct amzn/ion-java#14 before exposing this or using from public API. + * TODO Must correct amzn/ion-java/issues/63 before exposing this or using from public API. */ public Iterator systemIterate(String ionText); /** - * TODO Must correct amzn/ion-java#14 before exposing this or using from public API. + * TODO Must correct amzn/ion-java/issues/63 before exposing this or using from public API. */ public Iterator systemIterate(Reader ionText); diff --git a/src/software/amazon/ion/impl/PrivateIonTextAppender.java b/src/com/amazon/ion/impl/_Private_IonTextAppender.java similarity index 92% rename from src/software/amazon/ion/impl/PrivateIonTextAppender.java rename to src/com/amazon/ion/impl/_Private_IonTextAppender.java index b251c95b96..4e6f952615 100644 --- a/src/software/amazon/ion/impl/PrivateIonTextAppender.java +++ b/src/com/amazon/ion/impl/_Private_IonTextAppender.java @@ -1,24 +1,29 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; -import static software.amazon.ion.impl.PrivateIonConstants.MAX_LONG_TEXT_SIZE; -import static software.amazon.ion.impl.PrivateIonConstants.isHighSurrogate; -import static software.amazon.ion.impl.PrivateIonConstants.isLowSurrogate; -import static software.amazon.ion.impl.PrivateIonConstants.makeUnicodeScalar; +import static com.amazon.ion.impl._Private_IonConstants.MAX_LONG_TEXT_SIZE; +import static com.amazon.ion.impl._Private_IonConstants.isHighSurrogate; +import static com.amazon.ion.impl._Private_IonConstants.isLowSurrogate; +import static com.amazon.ion.impl._Private_IonConstants.makeUnicodeScalar; +import com.amazon.ion.Decimal; +import com.amazon.ion.impl.Base64Encoder.TextStream; +import com.amazon.ion.system.IonTextWriterBuilder; +import com.amazon.ion.util._Private_FastAppendable; import java.io.ByteArrayInputStream; import java.io.Closeable; import java.io.Flushable; @@ -28,19 +33,14 @@ import java.math.BigInteger; import java.nio.CharBuffer; import java.nio.charset.Charset; -import software.amazon.ion.Decimal; -import software.amazon.ion.impl.Base64Encoder.TextStream; -import software.amazon.ion.system.IonTextWriterBuilder; -import software.amazon.ion.util.PrivateFastAppendable; /** + * NOT FOR APPLICATION USE! + *

    * Generic text sink that enables optimized output of both ASCII and UTF-16. - * - * @deprecated This is an internal API that is subject to change without notice. */ -@Deprecated -public final class PrivateIonTextAppender +public final class _Private_IonTextAppender implements Closeable, Flushable { private static boolean is8bitValue(int v) @@ -220,22 +220,22 @@ public static boolean isOperatorPart(int codePoint) { //========================================================================= - private final PrivateFastAppendable myAppendable; + private final _Private_FastAppendable myAppendable; private final boolean escapeNonAscii; - PrivateIonTextAppender(PrivateFastAppendable out, boolean escapeNonAscii) + _Private_IonTextAppender(_Private_FastAppendable out, boolean escapeNonAscii) { this.myAppendable = out; this.escapeNonAscii = escapeNonAscii; } - public static PrivateIonTextAppender - forFastAppendable(PrivateFastAppendable out, Charset charset) + public static _Private_IonTextAppender + forFastAppendable(_Private_FastAppendable out, Charset charset) { - boolean escapeNonAscii = charset.equals(PrivateUtils.ASCII_CHARSET); - return new PrivateIonTextAppender(out, escapeNonAscii); + boolean escapeNonAscii = charset.equals(_Private_Utils.ASCII_CHARSET); + return new _Private_IonTextAppender(out, escapeNonAscii); } @@ -245,10 +245,10 @@ public static boolean isOperatorPart(int codePoint) { * characters will be escaped. Otherwise, only select code points will be * escaped. */ - public static PrivateIonTextAppender forAppendable(Appendable out, + public static _Private_IonTextAppender forAppendable(Appendable out, Charset charset) { - PrivateFastAppendable fast = new AppendableFastAppendable(out); + _Private_FastAppendable fast = new AppendableFastAppendable(out); return forFastAppendable(fast, charset); } @@ -256,11 +256,11 @@ public static PrivateIonTextAppender forAppendable(Appendable out, /** * Doesn't escape non-ASCII characters. */ - public static PrivateIonTextAppender forAppendable(Appendable out) + public static _Private_IonTextAppender forAppendable(Appendable out) { - PrivateFastAppendable fast = new AppendableFastAppendable(out); + _Private_FastAppendable fast = new AppendableFastAppendable(out); boolean escapeNonAscii = false; - return new PrivateIonTextAppender(fast, escapeNonAscii); + return new _Private_IonTextAppender(fast, escapeNonAscii); } @@ -270,10 +270,10 @@ public static PrivateIonTextAppender forAppendable(Appendable out) * characters will be escaped. Otherwise, only select code points will be * escaped. */ - public static PrivateIonTextAppender forOutputStream(OutputStream out, + public static _Private_IonTextAppender forOutputStream(OutputStream out, Charset charset) { - PrivateFastAppendable fast = new OutputStreamFastAppendable(out); + _Private_FastAppendable fast = new OutputStreamFastAppendable(out); return forFastAppendable(fast, charset); } @@ -716,7 +716,7 @@ public void printInt(BigInteger value) } - public void printDecimal(PrivateIonTextWriterBuilder _options, + public void printDecimal(_Private_IonTextWriterBuilder _options, BigDecimal value) throws IOException { @@ -864,7 +864,7 @@ public void printFloat(Double value) // LOBs - public void printBlob(PrivateIonTextWriterBuilder _options, + public void printBlob(_Private_IonTextWriterBuilder _options, byte[] value, int start, int len) throws IOException { @@ -935,7 +935,7 @@ private void printClobBytes(byte[] value, int start, int end, } - public void printClob(PrivateIonTextWriterBuilder _options, + public void printClob(_Private_IonTextWriterBuilder _options, byte[] value, int start, int len) throws IOException { diff --git a/src/com/amazon/ion/impl/_Private_IonTextWriterBuilder.java b/src/com/amazon/ion/impl/_Private_IonTextWriterBuilder.java new file mode 100644 index 0000000000..5c94c62556 --- /dev/null +++ b/src/com/amazon/ion/impl/_Private_IonTextWriterBuilder.java @@ -0,0 +1,305 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import static com.amazon.ion.impl._Private_Utils.initialSymtab; + +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.system.IonSystemBuilder; +import com.amazon.ion.system.IonTextWriterBuilder; +import com.amazon.ion.system.SimpleCatalog; +import com.amazon.ion.util._Private_FastAppendable; +import java.io.OutputStream; + +/** + * NOT FOR APPLICATION USE! + */ +public class _Private_IonTextWriterBuilder + extends IonTextWriterBuilder +{ + private final static CharSequence SPACE_CHARACTER = " "; + // TODO amzn/ion-java/issues/57 decide if this should be platform-specific + private final static CharSequence LINE_SEPARATOR = + System.getProperty("line.separator"); + + + public static _Private_IonTextWriterBuilder standard() + { + return new _Private_IonTextWriterBuilder.Mutable(); + } + + public static _Private_IonTextWriterBuilder STANDARD = + standard().immutable(); + + + //========================================================================= + + private boolean _pretty_print; + public boolean _blob_as_string; + public boolean _clob_as_string; + public boolean _decimal_as_float; + public boolean _sexp_as_list; + public boolean _skip_annotations; + public boolean _string_as_json; + public boolean _symbol_as_string; + public boolean _timestamp_as_millis; + public boolean _timestamp_as_string; + public boolean _untyped_nulls; + private _Private_CallbackBuilder _callback_builder; + + + private _Private_IonTextWriterBuilder() + { + super(); + } + + private _Private_IonTextWriterBuilder(_Private_IonTextWriterBuilder that) + { + super(that); + this._callback_builder = that._callback_builder ; + this._pretty_print = that._pretty_print ; + this._blob_as_string = that._blob_as_string ; + this._clob_as_string = that._clob_as_string ; + this._decimal_as_float = that._decimal_as_float ; + this._sexp_as_list = that._sexp_as_list ; + this._skip_annotations = that._skip_annotations ; + this._string_as_json = that._string_as_json ; + this._symbol_as_string = that._symbol_as_string ; + this._timestamp_as_millis = that._timestamp_as_millis; + this._timestamp_as_string = that._timestamp_as_string; + this._untyped_nulls = that._untyped_nulls ; + } + + + @Override + public final _Private_IonTextWriterBuilder copy() + { + return new Mutable(this); + } + + @Override + public _Private_IonTextWriterBuilder immutable() + { + return this; + } + + @Override + public _Private_IonTextWriterBuilder mutable() + { + return copy(); + } + + + //========================================================================= + + @Override + public final IonTextWriterBuilder withPrettyPrinting() + { + _Private_IonTextWriterBuilder b = mutable(); + b._pretty_print = true; + return b; + } + + @Override + public final IonTextWriterBuilder withJsonDowngrade() + { + _Private_IonTextWriterBuilder b = mutable(); + + b.withMinimalSystemData(); + + _blob_as_string = true; + _clob_as_string = true; + // datagramAsList = true; // TODO + _decimal_as_float = true; + _sexp_as_list = true; + _skip_annotations = true; + // skipSystemValues = true; // TODO + _string_as_json = true; + _symbol_as_string = true; + _timestamp_as_string = true; // TODO different from Printer + _timestamp_as_millis = false; + _untyped_nulls = true; + + return b; + } + + + final boolean isPrettyPrintOn() + { + return _pretty_print; + } + + final CharSequence lineSeparator() + { + if (_pretty_print) { + return LINE_SEPARATOR; + } + else { + return SPACE_CHARACTER; + } + } + + + //========================================================================= + + private _Private_IonTextWriterBuilder fillDefaults() + { + // Ensure that we don't modify the user's builder. + IonTextWriterBuilder b = copy(); + + if (b.getCatalog() == null) + { + b.setCatalog(new SimpleCatalog()); + } + + if (b.getCharset() == null) + { + b.setCharset(UTF8); + } + + return (_Private_IonTextWriterBuilder) b.immutable(); + } + + + /** Assumes that {@link #fillDefaults()} has been called. */ + private IonWriter build(_Private_FastAppendable appender) + { + IonCatalog catalog = getCatalog(); + SymbolTable[] imports = getImports(); + + // TODO We shouldn't need a system here + IonSystem system = + IonSystemBuilder.standard().withCatalog(catalog).build(); + + SymbolTable defaultSystemSymtab = system.getSystemSymbolTable(); + + IonWriterSystemText systemWriter = + (getCallbackBuilder() == null + ? new IonWriterSystemText(defaultSystemSymtab, + this, + appender) + : new IonWriterSystemTextMarkup(defaultSystemSymtab, + this, + appender)); + + SymbolTable initialSymtab = + initialSymtab(((_Private_ValueFactory)system).getLstFactory(), defaultSystemSymtab, imports); + + return new IonWriterUser(catalog, system, systemWriter, initialSymtab); + } + + + @Override + public final IonWriter build(Appendable out) + { + _Private_IonTextWriterBuilder b = fillDefaults(); + + _Private_FastAppendable fast = new AppendableFastAppendable(out); + + return b.build(fast); + } + + + @Override + public final IonWriter build(OutputStream out) + { + _Private_IonTextWriterBuilder b = fillDefaults(); + + _Private_FastAppendable fast = new OutputStreamFastAppendable(out); + + return b.build(fast); + } + + //========================================================================= + + private static final class Mutable + extends _Private_IonTextWriterBuilder + { + private Mutable() { } + + private Mutable(_Private_IonTextWriterBuilder that) + { + super(that); + } + + @Override + public _Private_IonTextWriterBuilder immutable() + { + return new _Private_IonTextWriterBuilder(this); + } + + @Override + public _Private_IonTextWriterBuilder mutable() + { + return this; + } + + @Override + protected void mutationCheck() + { + } + } + + //------------------------------------------------------------------------- + + /** + * Gets the {@link _Private_CallbackBuilder} that will be used to create a + * {@link _Private_MarkupCallback} when a new writer is built. + * @return The builder that will be used to build a new MarkupCallback. + * @see #setCallbackBuilder(_Private_CallbackBuilder) + * @see #withCallbackBuilder(_Private_CallbackBuilder) + */ + public final _Private_CallbackBuilder getCallbackBuilder() + { + return this._callback_builder; + } + + /** + * Sets the {@link _Private_CallbackBuilder} that will be used to create a + * {@link _Private_MarkupCallback} when a new writer is built. + * @param builder + * The builder that will be used to build a new MarkupCallback. + * @see #getCallbackBuilder() + * @see #withCallbackBuilder(_Private_CallbackBuilder) + * @throws UnsupportedOperationException + * if this is immutable. + */ + public void setCallbackBuilder(_Private_CallbackBuilder builder) + { + mutationCheck(); + this._callback_builder = builder; + } + + /** + * Declares the {@link _Private_CallbackBuilder} to use when building. + * @param builder + * The builder that will be used to build a new MarkupCallback. + * @return this instance, if mutable; otherwise a mutable copy of this + * instance. + * @see #getCallbackBuilder() + * @see #setCallbackBuilder(_Private_CallbackBuilder) + */ + public final _Private_IonTextWriterBuilder + withCallbackBuilder(_Private_CallbackBuilder builder) + { + _Private_IonTextWriterBuilder b = mutable(); + b.setCallbackBuilder(builder); + return b; + } +} diff --git a/src/software/amazon/ion/impl/PrivateIonValue.java b/src/com/amazon/ion/impl/_Private_IonValue.java similarity index 80% rename from src/software/amazon/ion/impl/PrivateIonValue.java rename to src/com/amazon/ion/impl/_Private_IonValue.java index cce85f4188..c81648e350 100644 --- a/src/software/amazon/ion/impl/PrivateIonValue.java +++ b/src/com/amazon/ion/impl/_Private_IonValue.java @@ -1,30 +1,30 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonValue; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; import java.io.PrintWriter; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonValue; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; /** - * @deprecated This is an internal API that is subject to change without notice. + * NOT FOR APPLICATION USE! */ -@Deprecated -public interface PrivateIonValue +public interface _Private_IonValue extends IonValue { diff --git a/src/software/amazon/ion/impl/PrivateIonWriter.java b/src/com/amazon/ion/impl/_Private_IonWriter.java similarity index 65% rename from src/software/amazon/ion/impl/PrivateIonWriter.java rename to src/com/amazon/ion/impl/_Private_IonWriter.java index d3a48fd423..519a14f3d1 100644 --- a/src/software/amazon/ion/impl/PrivateIonWriter.java +++ b/src/com/amazon/ion/impl/_Private_IonWriter.java @@ -1,37 +1,37 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonWriter; import java.io.IOException; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonWriter; /** - * @deprecated This is an internal API that is subject to change without notice. + * NOT FOR APPLICATION USE! */ -@Deprecated -public interface PrivateIonWriter +public interface _Private_IonWriter extends IonWriter { /** Mostly for testing at this point, but could be useful public API. */ IonCatalog getCatalog(); /** - * Returns true if the field name has been set either through setFieldName. - * This is generally more efficient than calling getFieldName and - * checking the return type as it does not need to resolve the + * Returns true if the field name has been set either through setFieldName or + * setFieldId. This is generally more efficient than calling getFieldName or + * getFieldId and checking the return type as it does not need to resolve the * name through a symbol table. This returns false if the field name has not * been set. * @@ -60,6 +60,6 @@ public interface PrivateIonWriter */ void writeIonVersionMarker() throws IOException; - /** Indicates whether the writer is stream copy optimized through {@link #writeValue(software.amazon.ion.IonReader)}. */ + /** Indicates whether the writer is stream copy optimized through {@link #writeValue(com.amazon.ion.IonReader)}. */ public boolean isStreamCopyOptimized(); } diff --git a/src/software/amazon/ion/impl/PrivateIonWriterBase.java b/src/com/amazon/ion/impl/_Private_IonWriterBase.java similarity index 89% rename from src/software/amazon/ion/impl/PrivateIonWriterBase.java rename to src/com/amazon/ion/impl/_Private_IonWriterBase.java index a5cdc96716..b04e4e5ef7 100644 --- a/src/software/amazon/ion/impl/PrivateIonWriterBase.java +++ b/src/com/amazon/ion/impl/_Private_IonWriterBase.java @@ -1,39 +1,42 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.Timestamp; +import com.amazon.ion.UnknownSymbolException; import java.io.IOException; import java.math.BigDecimal; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonType; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.UnknownSymbolException; +import java.util.Date; /** + * NOT FOR APPLICATION USE! + *

    * Base type for Ion writers. This handles the writeIonEvents and provides default * handlers for the list forms of write. This also resolves symbols if a symbol * table is available (which it will not be if the underlying writer is a system * writer). - * - * @deprecated This is an internal API that is subject to change without notice. */ -@Deprecated -public abstract class PrivateIonWriterBase - implements IonWriter, PrivateReaderWriter +public abstract class _Private_IonWriterBase + implements IonWriter, _Private_ReaderWriter { protected static final String ERROR_MISSING_FIELD_NAME = "IonWriter.setFieldName() must be called before writing a value into a struct."; @@ -111,9 +114,9 @@ public abstract void setSymbolTable(SymbolTable symbols) /** - * Returns true if the field name has been set either through setFieldName. - * This is generally more efficient than calling getFieldName and - * checking the return type as it does not need to resolve the + * Returns true if the field name has been set either through setFieldName or + * setFieldId. This is generally more efficient than calling getFieldName or + * getFieldId and checking the return type as it does not need to resolve the * name through a symbol table. This returns false if the field name has not * been set. * @return true if a field name has been set false otherwise @@ -246,12 +249,29 @@ public final void writeSymbolToken(SymbolToken tok) } + public void writeTimestampUTC(Date value) throws IOException + { + Timestamp time = Timestamp.forDateZ(value); + writeTimestamp(time); + } + + + // // default value and reader implementations. // note that these could be optimized, especially // the reader versions, when the reader is of the // same format as the writer. // + @Deprecated + public void writeValue(IonValue value) throws IOException + { + if (value != null) + { + value.writeTo(this); + } + } + public void writeValues(IonReader reader) throws IOException { if (reader.getDepth() == 0) { @@ -260,10 +280,10 @@ public void writeValues(IonReader reader) throws IOException if (reader.getType() == null) reader.next(); - if (getDepth() == 0 && reader instanceof PrivateReaderWriter) { + if (getDepth() == 0 && reader instanceof _Private_ReaderWriter) { // Optimize symbol table copying - PrivateReaderWriter private_reader = - (PrivateReaderWriter)reader; + _Private_ReaderWriter private_reader = + (_Private_ReaderWriter)reader; while (reader.getType() != null) { transfer_symbol_tables(private_reader); writeValue(reader); @@ -278,7 +298,7 @@ public void writeValues(IonReader reader) throws IOException } } - private final void transfer_symbol_tables(PrivateReaderWriter reader) + private final void transfer_symbol_tables(_Private_ReaderWriter reader) throws IOException { SymbolTable reader_symbols = reader.pop_passed_symbol_table(); @@ -468,4 +488,10 @@ public final SymbolTable pop_passed_symbol_table() _symbol_table_stack[_symbol_table_top] = null; return symbols; } + + public T asFacet(Class facetType) + { + // This implementation has no facets. + return null; + } } diff --git a/src/software/amazon/ion/impl/PrivateIonWriterFactory.java b/src/com/amazon/ion/impl/_Private_IonWriterFactory.java similarity index 71% rename from src/software/amazon/ion/impl/PrivateIonWriterFactory.java rename to src/com/amazon/ion/impl/_Private_IonWriterFactory.java index 8aaeabcf27..b02b8dfc94 100644 --- a/src/software/amazon/ion/impl/PrivateIonWriterFactory.java +++ b/src/com/amazon/ion/impl/_Private_IonWriterFactory.java @@ -1,33 +1,33 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonContainer; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.system.IonWriterBuilder.InitialIvmHandling; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonContainer; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.system.IonWriterBuilder.InitialIvmHandling; /** + * NOT FOR APPLICATION USE! + *

    * This is the factory class for constructing writers with various capabilities. - * - * @deprecated This is an internal API that is subject to change without notice. */ -@Deprecated -public final class PrivateIonWriterFactory +public final class _Private_IonWriterFactory { /** * @param container must not be null. diff --git a/src/software/amazon/ion/impl/PrivateListWriter.java b/src/com/amazon/ion/impl/_Private_ListWriter.java similarity index 60% rename from src/software/amazon/ion/impl/PrivateListWriter.java rename to src/com/amazon/ion/impl/_Private_ListWriter.java index 5d65340c08..71eb97c1a8 100644 --- a/src/software/amazon/ion/impl/PrivateListWriter.java +++ b/src/com/amazon/ion/impl/_Private_ListWriter.java @@ -1,29 +1,29 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; +import com.amazon.ion.IonWriter; import java.io.IOException; -import software.amazon.ion.IonWriter; /** + * NOT FOR APPLICATION USE! + *

    * An IonWriter that has optimized list-writing. - * - * @deprecated This is an internal API that is subject to change without notice. */ -@Deprecated -public interface PrivateListWriter +public interface _Private_ListWriter extends IonWriter { public void writeBoolList(boolean[] values)throws IOException; diff --git a/src/software/amazon/ion/impl/PrivateLocalSymbolTableFactory.java b/src/com/amazon/ion/impl/_Private_LocalSymbolTableFactory.java similarity index 81% rename from src/software/amazon/ion/impl/PrivateLocalSymbolTableFactory.java rename to src/com/amazon/ion/impl/_Private_LocalSymbolTableFactory.java index 2e1121f699..ad6c5f47c4 100644 --- a/src/software/amazon/ion/impl/PrivateLocalSymbolTableFactory.java +++ b/src/com/amazon/ion/impl/_Private_LocalSymbolTableFactory.java @@ -1,23 +1,24 @@ /* - * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonType; -import software.amazon.ion.SymbolTable; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonType; +import com.amazon.ion.SymbolTable; /** * NOT FOR APPLICATION USE @@ -33,7 +34,7 @@ * too. */ @SuppressWarnings("javadoc") -public interface PrivateLocalSymbolTableFactory +public interface _Private_LocalSymbolTableFactory { /** * Constructs a new local symbol table represented by the current value of @@ -77,4 +78,4 @@ public SymbolTable newLocalSymtab(IonCatalog catalog, */ public SymbolTable newLocalSymtab(SymbolTable defaultSystemSymtab, SymbolTable... imports); -} +} \ No newline at end of file diff --git a/src/com/amazon/ion/impl/_Private_MarkupCallback.java b/src/com/amazon/ion/impl/_Private_MarkupCallback.java new file mode 100644 index 0000000000..dabbd4e205 --- /dev/null +++ b/src/com/amazon/ion/impl/_Private_MarkupCallback.java @@ -0,0 +1,352 @@ + +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import com.amazon.ion.IonType; +import com.amazon.ion.SymbolToken; + +import com.amazon.ion.system.IonTextWriterBuilder; +import com.amazon.ion.util._Private_FastAppendable; +import java.io.IOException; + +/** + * NOT FOR APPLICATION USE! + * + * + * Callback for giving users the ability to inject markup into their Ion + * documents. + *

    + * Customers who want to inject markup into their Ion documents will need to + * extend {@link _Private_MarkupCallback}, implement {@link _Private_CallbackBuilder} to build new + * instances, and pass an instance of their {@link _Private_CallbackBuilder} into + * {@link IonTextWriterBuilder#setCallbackBuilder(_Private_CallbackBuilder)} or + * {@link IonTextWriterBuilder#withCallbackBuilder(_Private_CallbackBuilder)} + *

    + *

    + * Note: It is only necessary for subclasses to implement methods they're using + * to inject markup. + *

    + *

    + * When all method are called, they are called with (at least) an + * {@link IonType} that represents either the type of data being written or the + * container type that the writer is writing, or currently inside. + * before/afterFieldName and before/afterEachAnnotation get, respectively, the + * field name and annotation that is being written. Methods can use + * {@link #myAppendable} access the output stream. + *

    + *

    Example Calls

    + *

    + * Here are some example call patterns, using the following syntax to show where + * calls will be executed (whitespace is used to increase readability): + * + *

    + * <methodName TYPE>
    + * 
    + * + *

    + *

    + * Input: + * + *

    + * { cookies:"Chocolate Chip" }
    + * 
    + * + * Output: + * + *
    + * <beforeData STRUCT>{<afterStepIn STRUCT>
    + *     <beforeFieldName STRING "cookies"> cookies <afterFieldName STRING "cookies">:<beforeData STRING> "Chocolate Chip" <afterData STRING>
    + * <beforeStepOut STRUCT>}<afterData STRUCT>
    + * 
    + * + *

    + *

    + * Input: + * + *

    + * anno1::anno2::{
    + *   fname:"John",
    + *   lname:"Smith",
    + *   age:32.785
    + * }
    + * 
    + * + * Output: + * + *
    + * <beforeAnnotations STRUCT>
    + *   <beforeEachAnnotation STRUCT "anno1">anno1<afterEachAnnotation STRUCT "anno1">::
    + *   <beforeEachAnnotation STRUCT "anno2">anno2<afterEachAnnotation STRUCT "anno2">::
    + * <afterAnnotations STRUCT>
    + * <beforeData STRUCT>{<afterStepIn STRUCT>
    + *     <beforeFieldName STRING "fname">fname<afterFieldName STRING "fname">:<beforeData STRING>"John"<afterData STRING><beforeSeparator STRUCT>,<afterSeparator STRUCT>
    + *     <beforeFieldName STRING "lname">lname<afterFieldName STRING "lname">:<beforeData STRING>"Smith"<afterData STRING><beforeSeparator STRUCT>,<afterSeparator STRUCT>
    + *     <beforeFieldName DECIMAL "age">age<afterFieldName DECIMAL "age">:<beforeData DECIMAL>32.785<afterData DECIMAL>
    + * <beforeStepOut STRUCT>}<afterData STRUCT>
    + * 
    + * + *

    + *

    + * Input: + * + *

    + * (where (field (this) result) (== (field (curr) majorVersionString) "1.0"))
    + * 
    + * + * Output: + * + *
    + * <beforeData SEXP>(<afterStepIn SEXP>
    + *     <beforeData SYMBOL>where<afterData SYMBOL><beforeSeparator SEXP> <afterSeparator SEXP>
    + *     <beforeData SEXP>(<afterStepIn SEXP>
    + *         <beforeData SYMBOL>field<afterData SYMBOL><beforeSeparator SEXP> <afterSeparator SEXP>
    + *         <beforeData SEXP>(<afterStepIn SEXP>
    + *             <beforeData SYMBOL>this<afterData SYMBOL>
    + *         <beforeStepOut SEXP>)<afterData SEXP><beforeSeparator SEXP> <afterSeparator SEXP>
    + *         <beforeData SYMBOL>result<afterData SYMBOL>
    + *     <beforeStepOut SEXP>)<afterData SEXP><beforeSeparator SEXP> <afterSeparator SEXP>
    + *     <beforeData SEXP>(<afterStepIn SEXP>
    + *         <beforeData SYMBOL>==<afterData SYMBOL><beforeSeparator SEXP> <afterSeparator SEXP>
    + *         <beforeData SEXP>(<afterStepIn SEXP>
    + *             <beforeData SYMBOL>field<afterData SYMBOL><beforeSeparator SEXP> <afterSeparator SEXP>
    + *             <beforeData SEXP>(<afterStepIn SEXP>
    + *                 <beforeData SYMBOL>curr<afterData SYMBOL>
    + *             <beforeStepOut SEXP>)<afterData SEXP><beforeSeparator SEXP> <afterSeparator SEXP>
    + *             <beforeData SYMBOL>majorVersionString<afterData SYMBOL>
    + *         <beforeStepOut SEXP>)<afterData SEXP><beforeSeparator SEXP> <afterSeparator SEXP>
    + *         <beforeData STRING>"1.0"<afterData STRING>
    + *     <beforeStepOut SEXP>)<afterData SEXP>
    + * <beforeStepOut SEXP>)<afterData SEXP>
    + * 
    + * + *

    + *

    + * Input: + * + *

    + * 1 5 "Cheesecake" 3.2 true null 'baby tigers' 47e1
    + * 
    + * + * Output: + * + *
    + * <beforeData INT>1<afterData INT><beforeSeparator DATAGRAM> <afterSeparator DATAGRAM>
    + * <beforeData INT>5<afterData INT><beforeSeparator DATAGRAM> <afterSeparator DATAGRAM>
    + * <beforeData STRING>"Cheesecake"<afterData STRING><beforeSeparator DATAGRAM> <afterSeparator DATAGRAM>
    + * <beforeData DECIMAL>3.2<afterData DECIMAL><beforeSeparator DATAGRAM> <afterSeparator DATAGRAM>
    + * <beforeData BOOL>true<afterData BOOL><beforeSeparator DATAGRAM> <afterSeparator DATAGRAM>
    + * <beforeData NULL>null<afterData NULL><beforeSeparator DATAGRAM> <afterSeparator DATAGRAM>
    + * <beforeData SYMBOL>'baby tigers'<afterData SYMBOL><beforeSeparator DATAGRAM> <afterSeparator DATAGRAM>
    + * <beforeData FLOAT>470.0e0<afterData FLOAT>
    + * 
    + * + *

    + *

    + * Input: + * + *

    + * [true, 3.4, 3d6, 2.3e8, "string", '''multi-''' '''string''',Symbol, 'qSymbol',
    + *     {{"clob data"}}, {{YmxvYiBkYXRh}}, 1970-06-06, null.struct]
    + * 
    + * + * Output: + * + *
    + * <beforeData LIST>[<afterStepIn LIST>
    + *     <beforeData BOOL>true<afterData BOOL><beforeSeparator LIST>,<afterSeparator LIST>
    + *     <beforeData DECIMAL>3.4<afterData DECIMAL><beforeSeparator LIST>,<afterSeparator LIST>
    + *     <beforeData DECIMAL>3d6<afterData DECIMAL><beforeSeparator LIST>,<afterSeparator LIST>
    + *     <beforeData FLOAT>2.3E8<afterData FLOAT><beforeSeparator LIST>,<afterSeparator LIST>
    + *     <beforeData STRING>"string"<afterData STRING><beforeSeparator LIST>,<afterSeparator LIST>
    + *     <beforeData STRING>"multi-string"<afterData STRING><beforeSeparator LIST>,<afterSeparator LIST>
    + *     <beforeData SYMBOL>Symbol<afterData SYMBOL><beforeSeparator LIST>,<afterSeparator LIST>
    + *     <beforeData SYMBOL>qSymbol<afterData SYMBOL><beforeSeparator LIST>,<afterSeparator LIST>
    + *     <beforeData CLOB>{{"clob data"}}<afterData CLOB><beforeSeparator LIST>,<afterSeparator LIST>
    + *     <beforeData BLOB>{{YmxvYiBkYXRh}}<afterData BLOB><beforeSeparator LIST>,<afterSeparator LIST>
    + *     <beforeData TIMESTAMP>1970-06-06<afterData TIMESTAMP><beforeSeparator LIST>,<afterSeparator LIST>
    + *     <beforeData NULL>null.struct<afterData NULL>
    + * <beforeStepOut LIST>]<afterData LIST>
    + * 
    + */ +public abstract class _Private_MarkupCallback +{ + private final _Private_FastAppendable myAppendable; + + public _Private_MarkupCallback(_Private_FastAppendable appendable) + { + this.myAppendable = appendable; + } + + /** + * Gets the {@link _Private_FastAppendable} that IonWriter will use to write its + * output. + */ + public final _Private_FastAppendable getAppendable() + { + return myAppendable; + } + + /** + * Callback to be executed before an Ion value is written. If iType is a + * container type, this is executed before the container's opening + * delimiter. To write data after the opening delimiter use + * {@link #afterStepIn(IonType)}. + * @param iType + * The type of data that will be written. + */ + public void beforeValue(IonType iType) + throws IOException + { + } + + /** + * Callback to be executed after an Ion value is written. If iType is a + * container type, this is executed after the container's closing delimiter. + * To write data before the closing delimiter, use + * {@link #beforeStepOut(IonType)}. + * @param iType + * The type of data that was written. + */ + public void afterValue(IonType iType) + throws IOException + { + } + + /** + * Callback to be executed before a field name is written. + * @param iType + * The type of data in the field name's corresponding value. + * @param name + * The field name that is being written. + */ + public void beforeFieldName(IonType iType, SymbolToken name) + throws IOException + { + } + + /** + * Callback to be executed after a field name is written. + * @param iType + * The type of data in the field name's corresponding value. + * @param name + * The field name that is being written. + */ + public void afterFieldName(IonType iType, SymbolToken name) + throws IOException + { + } + + /** + * Callback to be executed after the opening delimiter of a container is + * written. To write data before the opening delimiter, use + * {@link #beforeValue(IonType)}. + * @param containerType + * The type of container that was just stepped into. + */ + public void afterStepIn(IonType containerType) + throws IOException + { + } + + /** + * Callback to be executed before the closing delimiter of a container is + * written. To write data after the closing delimiter, use + * {@link #afterValue(IonType)}. + * @param containerType + * The type of container that is about to be stepped out of. + */ + public void beforeStepOut(IonType containerType) + throws IOException + { + } + + /** + * Callback to be executed before a separator is written. Called after the + * data has been written, and before the separator. It is called when inside + * all container types, including Sexp, and when not inside a container. It + * is not called after the last element in a container. + * @param containerType + * The type of container the writer is currently in. When writing + * top-level values, the type is DATAGRAM. + */ + public void beforeSeparator(IonType containerType) + throws IOException + { + } + + /** + * Callback to be executed after a separator has been written. Called just + * after the separator is written. It is called when inside all container + * types, including Sexp, and when not inside a container. It is not called + * after the last element in a container. + * @param containerType + * The type of container the writer is currently in. When writing + * top-level values, the type is DATAGRAM. + */ + public void afterSeparator(IonType containerType) + throws IOException + { + } + + /** + * Callback to be executed before annotations are written. It is executed + * once per set of annotations. + * @param iType + * The type of the data whose annotations are being written. + */ + public void beforeAnnotations(IonType iType) + throws IOException + { + } + + /** + * Callback to be executed after annotations are written. It is executed + * once per set of annotations. + * @param iType + * The type of the data whose annotations are being written. + */ + public void afterAnnotations(IonType iType) + throws IOException + { + } + + /** + * Callback to be executed before each annotation is written. It is executed + * once per annotation. + * @param iType + * The type of the data whose annotation is being written. + * @param annotation + * The annotation that is being written. + */ + public void beforeEachAnnotation(IonType iType, SymbolToken annotation) + throws IOException + { + } + + /** + * Callback to be executed after each annotation is written. It is executed + * once per annotation, before the delimiter. + * @param iType + * The type of the data whose annotation is being written. + * @param annotation + * The annotation that is being written. + */ + public void afterEachAnnotation(IonType iType, SymbolToken annotation) + throws IOException + { + } +} diff --git a/src/com/amazon/ion/impl/_Private_ReaderWriter.java b/src/com/amazon/ion/impl/_Private_ReaderWriter.java new file mode 100644 index 0000000000..aa91f56298 --- /dev/null +++ b/src/com/amazon/ion/impl/_Private_ReaderWriter.java @@ -0,0 +1,27 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import com.amazon.ion.SymbolTable; + +/** + * NOT FOR APPLICATION USE! + */ +public interface _Private_ReaderWriter +{ + public SymbolTable pop_passed_symbol_table(); +} + diff --git a/src/software/amazon/ion/impl/PrivateScalarConversions.java b/src/com/amazon/ion/impl/_Private_ScalarConversions.java similarity index 97% rename from src/software/amazon/ion/impl/PrivateScalarConversions.java rename to src/com/amazon/ion/impl/_Private_ScalarConversions.java index 5c71c1586a..b5cf96aa5b 100644 --- a/src/software/amazon/ion/impl/PrivateScalarConversions.java +++ b/src/com/amazon/ion/impl/_Private_ScalarConversions.java @@ -1,33 +1,33 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; +import com.amazon.ion.Decimal; +import com.amazon.ion.IntegerSize; +import com.amazon.ion.IonException; +import com.amazon.ion.IonType; +import com.amazon.ion.Timestamp; import java.math.BigDecimal; import java.math.BigInteger; import java.util.Date; -import software.amazon.ion.Decimal; -import software.amazon.ion.IntegerSize; -import software.amazon.ion.IonException; -import software.amazon.ion.IonType; -import software.amazon.ion.Timestamp; /** - * @deprecated This is an internal API that is subject to change without notice. + * NOT FOR APPLICATION USE! */ -@Deprecated -public class PrivateScalarConversions +public class _Private_ScalarConversions { public static final class AS_TYPE { @@ -605,7 +605,7 @@ public final boolean can_convert(int new_type) { } public final int get_conversion_fnid(int new_type) { - return PrivateScalarConversions.getConversionFnid(_authoritative_type_idx, new_type); + return _Private_ScalarConversions.getConversionFnid(_authoritative_type_idx, new_type); } public final void cast(int castfnid) { @@ -868,7 +868,7 @@ private final void fn_from_timestamp_to_date() { add_value_type(AS_TYPE.date_value); } private final void fn_from_date_to_timestamp() { - _timestamp_value = Timestamp.forMillis(_date_value.getTime(), null); + _timestamp_value = new Timestamp(_date_value.getTime(), null); add_value_type(AS_TYPE.timestamp_value); } } diff --git a/src/com/amazon/ion/impl/_Private_SymbolToken.java b/src/com/amazon/ion/impl/_Private_SymbolToken.java new file mode 100644 index 0000000000..5a2f5930bc --- /dev/null +++ b/src/com/amazon/ion/impl/_Private_SymbolToken.java @@ -0,0 +1,31 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import com.amazon.ion.SymbolToken; + +public interface _Private_SymbolToken + extends SymbolToken{ + /** + * Hashes the symboltoken for bucketing. + */ + public int hashCode(); + + /** + * Compares symboltokens for equality. + */ + public boolean equals(Object other); +} diff --git a/src/software/amazon/ion/impl/PrivateSymtabExtendsCache.java b/src/com/amazon/ion/impl/_Private_SymtabExtendsCache.java similarity index 69% rename from src/software/amazon/ion/impl/PrivateSymtabExtendsCache.java rename to src/com/amazon/ion/impl/_Private_SymtabExtendsCache.java index c333b8be4c..1fc47c9572 100644 --- a/src/software/amazon/ion/impl/PrivateSymtabExtendsCache.java +++ b/src/com/amazon/ion/impl/_Private_SymtabExtendsCache.java @@ -1,31 +1,30 @@ /* - * Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; -import static software.amazon.ion.impl.PrivateUtils.symtabExtends; +import static com.amazon.ion.impl._Private_Utils.symtabExtends; -import software.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolTable; /** * Cache to reduce unnecessary calls to - * {@link PrivateUtils#symtabExtends(SymbolTable, SymbolTable)}. This is + * {@link _Private_Utils#symtabExtends(SymbolTable, SymbolTable)}. This is * only used if the writer is stream copy optimized. - * @deprecated This is an internal API that is subject to change without notice. */ -@Deprecated -public final class PrivateSymtabExtendsCache +public final class _Private_SymtabExtendsCache { private SymbolTable myWriterSymtab; private SymbolTable myReaderSymtab; diff --git a/src/software/amazon/ion/impl/PrivateUtils.java b/src/com/amazon/ion/impl/_Private_Utils.java similarity index 90% rename from src/software/amazon/ion/impl/PrivateUtils.java rename to src/com/amazon/ion/impl/_Private_Utils.java index f047a87a52..62785fa277 100644 --- a/src/software/amazon/ion/impl/PrivateUtils.java +++ b/src/com/amazon/ion/impl/_Private_Utils.java @@ -1,33 +1,47 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; - -import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; -import static software.amazon.ion.SystemSymbols.IMPORTS; -import static software.amazon.ion.SystemSymbols.IMPORTS_SID; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; -import static software.amazon.ion.SystemSymbols.MAX_ID; -import static software.amazon.ion.SystemSymbols.MAX_ID_SID; -import static software.amazon.ion.SystemSymbols.NAME; -import static software.amazon.ion.SystemSymbols.NAME_SID; -import static software.amazon.ion.SystemSymbols.SYMBOLS; -import static software.amazon.ion.SystemSymbols.SYMBOLS_SID; -import static software.amazon.ion.SystemSymbols.VERSION; -import static software.amazon.ion.SystemSymbols.VERSION_SID; -import static software.amazon.ion.util.IonStreamUtils.isIonBinary; - +package com.amazon.ion.impl; + +import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; +import static com.amazon.ion.SystemSymbols.IMPORTS; +import static com.amazon.ion.SystemSymbols.IMPORTS_SID; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; +import static com.amazon.ion.SystemSymbols.MAX_ID; +import static com.amazon.ion.SystemSymbols.MAX_ID_SID; +import static com.amazon.ion.SystemSymbols.NAME; +import static com.amazon.ion.SystemSymbols.NAME_SID; +import static com.amazon.ion.SystemSymbols.SYMBOLS; +import static com.amazon.ion.SystemSymbols.SYMBOLS_SID; +import static com.amazon.ion.SystemSymbols.VERSION; +import static com.amazon.ion.SystemSymbols.VERSION_SID; +import static com.amazon.ion.util.IonStreamUtils.isIonBinary; + +import com.amazon.ion.IonException; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.SubstituteSymbolTableException; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.UnknownSymbolException; +import com.amazon.ion.ValueFactory; +import com.amazon.ion.impl.IonBinary.BufferManager; +import com.amazon.ion.impl.IonBinary.Reader; import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -45,24 +59,18 @@ import java.util.ListIterator; import java.util.NoSuchElementException; import java.util.TimeZone; -import software.amazon.ion.IonException; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.SubstituteSymbolTableException; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.UnknownSymbolException; -import software.amazon.ion.ValueFactory; /** - * @deprecated This is an internal API that is subject to change without notice. + * NOT FOR APPLICATION USE! */ -@Deprecated -public final class PrivateUtils +public final class _Private_Utils { + /** + * Marker for code points relevant to removal of IonReader.hasNext(). + */ + public static final boolean READER_HASNEXT_REMOVED = false; + + /** Just a zero-length byte array, used to avoid allocation. */ public static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; @@ -72,6 +80,17 @@ public final class PrivateUtils /** Just a zero-length int array, used to avoid allocation. */ public final static int[] EMPTY_INT_ARRAY = new int[0]; + /** + * (null.timestamp) requires 11 ASCII chars to distinguish from + * (null.timestamps) aka (null '.' 'timestamps') + * + * @see IonCharacterReader#DEFAULT_BUFFER_SIZE + * @see IonCharacterReader#BUFFER_PADDING + */ + public static final int MAX_LOOKAHEAD_UTF16 = 11; + + + public static final String ASCII_CHARSET_NAME = "US-ASCII"; public static final Charset ASCII_CHARSET = @@ -189,7 +208,7 @@ public static SymbolTokenImpl newSymbolToken(int sid) public static SymbolToken newSymbolToken(SymbolTable symtab, String text) { - // TODO amzn/ion-java#21 symtab should not be null + // TODO amzn/ion-java/issues/21 symtab should not be null text.getClass(); // quick null check SymbolToken tok = (symtab == null ? null : symtab.find(text)); @@ -208,7 +227,7 @@ public static SymbolToken newSymbolToken(SymbolTable symtab, { if (sid < 1) throw new IllegalArgumentException(); - // TODO amzn/ion-java#21 symtab should not be null + // TODO amzn/ion-java/issues/21 symtab should not be null String text = (symtab == null ? null : symtab.findKnownSymbol(sid)); return new SymbolTokenImpl(text, sid); } @@ -269,7 +288,7 @@ public static SymbolToken localize(SymbolTable symtab, String text = sym.getText(); int sid = sym.getSid(); - if (symtab != null) // TODO amzn/ion-java#21 require symtab + if (symtab != null) // TODO amzn/ion-java/issues/21 require symtab { if (text == null) { @@ -338,7 +357,7 @@ public static void localize(SymbolTable symtab, */ public static String[] toStrings(SymbolToken[] symbols, int count) { - if (count == 0) return PrivateUtils.EMPTY_STRING_ARRAY; + if (count == 0) return _Private_Utils.EMPTY_STRING_ARRAY; String[] annotations = new String[count]; for (int i = 0; i < count; i++) @@ -356,7 +375,7 @@ public static String[] toStrings(SymbolToken[] symbols, int count) public static int[] toSids(SymbolToken[] symbols, int count) { - if (count == 0) return PrivateUtils.EMPTY_INT_ARRAY; + if (count == 0) return _Private_Utils.EMPTY_INT_ARRAY; int[] sids = new int[count]; for (int i = 0; i < count; i++) @@ -584,11 +603,22 @@ public static byte[] loadFileBytes(File file) public static String utf8FileToString(File file) throws IonException, IOException { - byte[] utf8Bytes = PrivateUtils.loadFileBytes(file); + byte[] utf8Bytes = _Private_Utils.loadFileBytes(file); String s = utf8(utf8Bytes); return s; } + public static byte[] loadStreamBytes(InputStream in) + throws IOException + { + BufferManager buffer = new BufferManager(in); + Reader bufReader = buffer.reader(); + bufReader.sync(); + bufReader.setPosition(0); + byte[] bytes = bufReader.getBytes(); + return bytes; + } + public static String loadReader(java.io.Reader in) throws IOException @@ -611,10 +641,10 @@ public static boolean streamIsIonBinary(PushbackInputStream pushback) throws IonException, IOException { boolean isBinary = false; - byte[] cookie = new byte[PrivateIonConstants.BINARY_VERSION_MARKER_SIZE]; + byte[] cookie = new byte[_Private_IonConstants.BINARY_VERSION_MARKER_SIZE]; int len = readFully(pushback, cookie); - if (len == PrivateIonConstants.BINARY_VERSION_MARKER_SIZE) { + if (len == _Private_IonConstants.BINARY_VERSION_MARKER_SIZE) { isBinary = isIonBinary(cookie); } if (len > 0) { @@ -644,7 +674,7 @@ public static Iterator iterate(ValueFactory valueFactory, * * @return boolean true if v can be a local symbol table otherwise false */ - public static boolean valueIsLocalSymbolTable(PrivateIonValue v) + public static boolean valueIsLocalSymbolTable(_Private_IonValue v) { return (v instanceof IonStruct && v.findTypeAnnotation(ION_SYMBOL_TABLE) == 0); @@ -737,13 +767,6 @@ public static SymbolTable newSharedSymtab(String name, symbols); } - public static SymbolTable newSubstituteSymtab(SymbolTable original, - int version, - int maxId) - { - return new SubstituteSymbolTable(original, version, maxId); - } - /** * Creates a mutable copy of this local symbol table. The cloned table @@ -805,13 +828,13 @@ public static SymbolTable copyLocalSymbolTable(SymbolTable symtab) * {@value LocalSymbolTable#DEFAULT_LST_FACTORY}. */ @Deprecated - public static PrivateLocalSymbolTableFactory newLocalSymbolTableAsStructFactory(ValueFactory imageFactory) + public static _Private_LocalSymbolTableFactory newLocalSymbolTableAsStructFactory(ValueFactory imageFactory) { return new LocalSymbolTableAsStruct.Factory(imageFactory); } /** - * Returns a minimal symtab, either system or local depending on the + * Returns a minimal symtab that, either system or local depending on the * given values, that supports representation as an IonStruct. If the * imports are empty, the default system symtab is returned. * @@ -825,7 +848,7 @@ public static PrivateLocalSymbolTableFactory newLocalSymbolTableAsStructFactory( * The first (and only the first) may be a system table, in which case the * {@code defaultSystemSymtab} is ignored. */ - public static SymbolTable initialSymtab(PrivateLocalSymbolTableFactory lstFactory, + public static SymbolTable initialSymtab(_Private_LocalSymbolTableFactory lstFactory, SymbolTable defaultSystemSymtab, SymbolTable... imports) { @@ -843,6 +866,7 @@ public static SymbolTable initialSymtab(PrivateLocalSymbolTableFactory lstFactor } + @SuppressWarnings("deprecation") /** * Trampoline to * {@link LocalSymbolTableAsStruct#getIonRepresentation()}; @@ -935,7 +959,7 @@ public static boolean symtabExtends(SymbolTable superset, SymbolTable subset) // If the subset's symtab is a system symtab, the superset's is always // an extension of the subset's as system symtab-ness is irrelevant to // the conditions for copy opt. to be safe. - // TODO amzn/ion-java#24 System symtab-ness ARE relevant if there's multiple + // TODO amzn/ion-java/issues/24 System symtab-ness ARE relevant if there's multiple // versions. if (subset.isSystemTable()) return true; @@ -1055,7 +1079,7 @@ public static final Iterator stringIterator(String... values) { if (values == null || values.length == 0) { - return PrivateUtils.emptyIterator(); + return _Private_Utils.emptyIterator(); } return new StringIterator(values, values.length); } @@ -1064,7 +1088,7 @@ public static final Iterator stringIterator(String[] values, int len) { if (values == null || values.length == 0 || len == 0) { - return PrivateUtils.emptyIterator(); + return _Private_Utils.emptyIterator(); } return new StringIterator(values, len); } @@ -1104,7 +1128,7 @@ public static final Iterator intIterator(int... values) { if (values == null || values.length == 0) { - return PrivateUtils.emptyIterator(); + return _Private_Utils.emptyIterator(); } return new IntIterator(values); } @@ -1113,15 +1137,11 @@ public static final Iterator intIterator(int[] values, int len) { if (values == null || values.length == 0 || len == 0) { - return PrivateUtils.emptyIterator(); + return _Private_Utils.emptyIterator(); } return new IntIterator(values, 0, len); } - - //======================================================================== - - public static void writeAsBase64(InputStream byteStream, Appendable out) throws IOException { @@ -1133,4 +1153,12 @@ public static void writeAsBase64(InputStream byteStream, Appendable out) out.append((char) c); } } + + public static SymbolTable newSubstituteSymtab(SymbolTable original, + int version, + int maxId) + { + return new SubstituteSymbolTable(original, version, maxId); + } + } diff --git a/src/software/amazon/ion/impl/PrivateValueFactory.java b/src/com/amazon/ion/impl/_Private_ValueFactory.java similarity index 50% rename from src/software/amazon/ion/impl/PrivateValueFactory.java rename to src/com/amazon/ion/impl/_Private_ValueFactory.java index e4e2517ee5..304123f99b 100644 --- a/src/software/amazon/ion/impl/PrivateValueFactory.java +++ b/src/com/amazon/ion/impl/_Private_ValueFactory.java @@ -1,23 +1,24 @@ /* - * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; -import software.amazon.ion.IonStruct; -import software.amazon.ion.ValueFactory; +import com.amazon.ion.IonStruct; +import com.amazon.ion.ValueFactory; -public interface PrivateValueFactory extends ValueFactory +public interface _Private_ValueFactory extends ValueFactory { /** * Gets the {@link LocalSymbolTableAsStruct.Factory} associated with this @@ -28,5 +29,5 @@ public interface PrivateValueFactory extends ValueFactory * @return a LocalSymbolTableAsStruct.Factory; never null. */ @SuppressWarnings("javadoc") - public PrivateLocalSymbolTableFactory getLstFactory(); + public _Private_LocalSymbolTableFactory getLstFactory(); } diff --git a/src/software/amazon/ion/impl/bin/AbstractIonWriter.java b/src/com/amazon/ion/impl/bin/AbstractIonWriter.java similarity index 66% rename from src/software/amazon/ion/impl/bin/AbstractIonWriter.java rename to src/com/amazon/ion/impl/bin/AbstractIonWriter.java index 2728ee05d6..0fe57ed210 100644 --- a/src/software/amazon/ion/impl/bin/AbstractIonWriter.java +++ b/src/com/amazon/ion/impl/bin/AbstractIonWriter.java @@ -1,37 +1,39 @@ /* - * Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.bin; - +package com.amazon.ion.impl.bin; + +import com.amazon.ion.Decimal; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.Timestamp; +import com.amazon.ion.impl._Private_ByteTransferReader; +import com.amazon.ion.impl._Private_ByteTransferSink; +import com.amazon.ion.impl._Private_IonWriter; +import com.amazon.ion.impl._Private_SymtabExtendsCache; +import com.amazon.ion.impl._Private_Utils; import java.io.IOException; import java.math.BigInteger; -import software.amazon.ion.Decimal; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.Timestamp; -import software.amazon.ion.impl.PrivateByteTransferReader; -import software.amazon.ion.impl.PrivateByteTransferSink; -import software.amazon.ion.impl.PrivateIonWriter; -import software.amazon.ion.impl.PrivateSymtabExtendsCache; -import software.amazon.ion.impl.PrivateUtils; +import java.util.Date; /** Common adapter for binary {@link IonWriter} implementations. */ -/*package*/ abstract class AbstractIonWriter implements PrivateIonWriter, PrivateByteTransferSink +/*package*/ abstract class AbstractIonWriter implements _Private_IonWriter, _Private_ByteTransferSink { /*package*/ enum WriteValueOptimization { @@ -40,12 +42,12 @@ } /** The cache for copy optimization checks--null if not copy optimized. */ - private final PrivateSymtabExtendsCache symtabExtendsCache; + private final _Private_SymtabExtendsCache symtabExtendsCache; /*package*/ AbstractIonWriter(final WriteValueOptimization optimization) { this.symtabExtendsCache = optimization == WriteValueOptimization.COPY_OPTIMIZED - ? new PrivateSymtabExtendsCache() : null; + ? new _Private_SymtabExtendsCache() : null; } public final void writeValue(final IonValue value) throws IOException @@ -54,7 +56,7 @@ public final void writeValue(final IonValue value) throws IOException { if (value instanceof IonDatagram) { - // XXX this is a hack to make the writer consistent with the legacy implementations and flush out an IVM + // XXX this is a hack to make the writer consistent with legacy implementations and flush out an IVM finish(); } value.writeTo(this); @@ -67,11 +69,11 @@ public final void writeValue(final IonReader reader) throws IOException if (isStreamCopyOptimized()) { - final PrivateByteTransferReader transferReader = - reader.asFacet(PrivateByteTransferReader.class); + final _Private_ByteTransferReader transferReader = + reader.asFacet(_Private_ByteTransferReader.class); if (transferReader != null - && (PrivateUtils.isNonSymbolScalar(type) + && (_Private_Utils.isNonSymbolScalar(type) || symtabExtendsCache.symtabsCompat(getSymbolTable(), reader.getSymbolTable()))) { // we have something we can pipe over @@ -87,9 +89,6 @@ public final void writeValueRecursive(final IonReader reader) throws IOException { final IonType type = reader.getType(); - // TODO amzn/ion-java#45 make sure the plumbing symbol tokens do the right thing for - // different symbol contexts in the reader and this writer - final SymbolToken fieldName = reader.getFieldNameSymbol(); if (fieldName != null && !isFieldNameSet() && isInStruct()) { @@ -144,8 +143,8 @@ public final void writeValueRecursive(final IonReader reader) throws IOException writeTimestamp(timestampValue); break; case SYMBOL: - final SymbolToken symbolValue = reader.symbolValue(); - writeSymbolToken(symbolValue); + final SymbolToken symbolToken = reader.symbolValue(); + writeSymbolToken(symbolToken); break; case STRING: final String stringValue = reader.stringValue(); @@ -187,8 +186,38 @@ public final void writeValues(final IonReader reader) throws IOException } } + public final void writeTimestampUTC(final Date value) throws IOException + { + writeTimestamp(Timestamp.forDateZ(value)); + } + public final boolean isStreamCopyOptimized() { return symtabExtendsCache != null; } + + @SuppressWarnings("deprecation") + public T asFacet(Class facetType) + { + if (facetType == _Private_IonManagedWriter.class) + { + return facetType.cast(this); + } + return null; // Consistent with readers' behavior when requested facet isn't supported + } + + /** + * Writes a portion of the byte array out as an IonString value. This + * copies the portion of the byte array that is written. + * + * @param data bytes to be written. + * May be {@code null} to represent {@code null.string}. + * @param offset offset of the first byte in value to write + * @param length number of bytes to write from value + * @see IonWriter#writeClob(byte[], int, int) + * @see IonWriter#writeBlob(byte[], int, int) + */ + public abstract void writeString(byte[] data, int offset, int length) + throws IOException; + } diff --git a/src/software/amazon/ion/impl/bin/AbstractSymbolTable.java b/src/com/amazon/ion/impl/bin/AbstractSymbolTable.java similarity index 73% rename from src/software/amazon/ion/impl/bin/AbstractSymbolTable.java rename to src/com/amazon/ion/impl/bin/AbstractSymbolTable.java index 6eb1266ac8..a014440c57 100644 --- a/src/software/amazon/ion/impl/bin/AbstractSymbolTable.java +++ b/src/com/amazon/ion/impl/bin/AbstractSymbolTable.java @@ -1,36 +1,36 @@ /* - * Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.bin; - -import static software.amazon.ion.IonType.LIST; -import static software.amazon.ion.IonType.STRUCT; -import static software.amazon.ion.SystemSymbols.IMPORTS_SID; -import static software.amazon.ion.SystemSymbols.ION_1_0; -import static software.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE_SID; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; -import static software.amazon.ion.SystemSymbols.MAX_ID_SID; -import static software.amazon.ion.SystemSymbols.NAME_SID; -import static software.amazon.ion.SystemSymbols.SYMBOLS_SID; -import static software.amazon.ion.SystemSymbols.VERSION_SID; -import static software.amazon.ion.impl.bin.Symbols.systemSymbol; +package com.amazon.ion.impl.bin; +import static com.amazon.ion.IonType.LIST; +import static com.amazon.ion.IonType.STRUCT; +import static com.amazon.ion.SystemSymbols.IMPORTS_SID; +import static com.amazon.ion.SystemSymbols.ION_1_0; +import static com.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE_SID; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; +import static com.amazon.ion.SystemSymbols.MAX_ID_SID; +import static com.amazon.ion.SystemSymbols.NAME_SID; +import static com.amazon.ion.SystemSymbols.SYMBOLS_SID; +import static com.amazon.ion.SystemSymbols.VERSION_SID; +import static com.amazon.ion.impl.bin.Symbols.systemSymbol; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; import java.io.IOException; import java.util.Iterator; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; /** * Provides the basic implementation bits for {@link SymbolTable}. diff --git a/src/software/amazon/ion/impl/bin/Block.java b/src/com/amazon/ion/impl/bin/Block.java similarity index 79% rename from src/software/amazon/ion/impl/bin/Block.java rename to src/com/amazon/ion/impl/bin/Block.java index 75a33c3c8e..9f03d2c252 100644 --- a/src/software/amazon/ion/impl/bin/Block.java +++ b/src/com/amazon/ion/impl/bin/Block.java @@ -1,18 +1,19 @@ /* - * Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.bin; +package com.amazon.ion.impl.bin; import java.io.Closeable; diff --git a/src/software/amazon/ion/impl/bin/BlockAllocator.java b/src/com/amazon/ion/impl/bin/BlockAllocator.java similarity index 77% rename from src/software/amazon/ion/impl/bin/BlockAllocator.java rename to src/com/amazon/ion/impl/bin/BlockAllocator.java index 43b5d7f6b0..fe6cf1bb13 100644 --- a/src/software/amazon/ion/impl/bin/BlockAllocator.java +++ b/src/com/amazon/ion/impl/bin/BlockAllocator.java @@ -1,18 +1,19 @@ /* - * Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.bin; +package com.amazon.ion.impl.bin; import java.io.Closeable; diff --git a/src/software/amazon/ion/impl/bin/BlockAllocatorProvider.java b/src/com/amazon/ion/impl/bin/BlockAllocatorProvider.java similarity index 72% rename from src/software/amazon/ion/impl/bin/BlockAllocatorProvider.java rename to src/com/amazon/ion/impl/bin/BlockAllocatorProvider.java index 140c7987df..8e6ee59691 100644 --- a/src/software/amazon/ion/impl/bin/BlockAllocatorProvider.java +++ b/src/com/amazon/ion/impl/bin/BlockAllocatorProvider.java @@ -1,18 +1,19 @@ /* - * Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.bin; +package com.amazon.ion.impl.bin; /** * Constructs {@link BlockAllocator} instances. Such instances returned by {@link #vendAllocator(int)} diff --git a/src/software/amazon/ion/impl/bin/BlockAllocatorProviders.java b/src/com/amazon/ion/impl/bin/BlockAllocatorProviders.java similarity index 78% rename from src/software/amazon/ion/impl/bin/BlockAllocatorProviders.java rename to src/com/amazon/ion/impl/bin/BlockAllocatorProviders.java index 8f0636a0d9..d9a17423e4 100644 --- a/src/software/amazon/ion/impl/bin/BlockAllocatorProviders.java +++ b/src/com/amazon/ion/impl/bin/BlockAllocatorProviders.java @@ -1,18 +1,19 @@ /* - * Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.bin; +package com.amazon.ion.impl.bin; /** * Utility implementations of {@link BlockAllocatorProvider}. diff --git a/src/com/amazon/ion/impl/bin/IonBinaryWriterAdapter.java b/src/com/amazon/ion/impl/bin/IonBinaryWriterAdapter.java new file mode 100644 index 0000000000..3a5f3e9abf --- /dev/null +++ b/src/com/amazon/ion/impl/bin/IonBinaryWriterAdapter.java @@ -0,0 +1,268 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl.bin; + +import com.amazon.ion.IonBinaryWriter; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.Timestamp; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Date; + +/** + * Adapts arbitrary {@link IonWriter} implementations to an {@link IonBinaryWriter} instances. + *

    + * This class is provided as a shim for compatibility to allow binary writer implementations to be adaptable to + * the legacy interface. + */ +@Deprecated +/*package*/ final class IonBinaryWriterAdapter implements IonBinaryWriter +{ + /** + * Simple interface for constructing an {@link IonWriter} from an output stream. + * Essentially used as a polymorphic constructor. + */ + public interface Factory + { + IonWriter create(final OutputStream out) throws IOException; + } + + /** Internal {@link ByteArrayOutputStream} implementation to get access to the buffer. */ + private static class InternalByteArrayOutputStream extends ByteArrayOutputStream + { + public byte[] bytes() + { + return buf; + } + } + + private final InternalByteArrayOutputStream buffer; + private final IonWriter delegate; + + public IonBinaryWriterAdapter(final Factory factory) throws IOException + { + this.buffer = new InternalByteArrayOutputStream(); + this.delegate = factory.create(buffer); + } + + /*package*/ IonWriter getDelegate() + { + return delegate; + } + + /*package*/ void reset() + { + buffer.reset(); + } + + // Adapter Methods + + public int byteSize() + { + return buffer.size(); + } + + public byte[] getBytes() throws IOException + { + return buffer.toByteArray(); + } + + public int getBytes(byte[] bytes, int offset, int maxlen) throws IOException + { + final int amount = Math.min(maxlen, buffer.size()); + System.arraycopy(buffer.bytes(), 0, bytes, offset, amount); + return amount; + } + + public int writeBytes(OutputStream userstream) throws IOException + { + buffer.writeTo(userstream); + return buffer.size(); + } + + // Delegates + + public SymbolTable getSymbolTable() + { + return delegate.getSymbolTable(); + } + + public void flush() throws IOException + { + delegate.flush(); + } + + public void finish() throws IOException + { + delegate.finish(); + } + + public void close() throws IOException + { + delegate.close(); + } + + public void setFieldName(String name) + { + delegate.setFieldName(name); + } + + public void setFieldNameSymbol(SymbolToken name) + { + delegate.setFieldNameSymbol(name); + } + + public void setTypeAnnotations(String... annotations) + { + delegate.setTypeAnnotations(annotations); + } + + public void setTypeAnnotationSymbols(SymbolToken... annotations) + { + delegate.setTypeAnnotationSymbols(annotations); + } + + public void addTypeAnnotation(String annotation) + { + delegate.addTypeAnnotation(annotation); + } + + public void stepIn(IonType containerType) throws IOException + { + delegate.stepIn(containerType); + } + + public void stepOut() throws IOException + { + delegate.stepOut(); + } + + public boolean isInStruct() + { + return delegate.isInStruct(); + } + + // Write Methods + + public void writeValue(IonValue value) throws IOException + { + delegate.writeValue(value); + } + + public void writeValue(IonReader reader) throws IOException + { + delegate.writeValue(reader); + } + + public void writeValues(IonReader reader) throws IOException + { + delegate.writeValues(reader); + } + + public void writeNull() throws IOException + { + delegate.writeNull(); + } + + public void writeNull(IonType type) throws IOException + { + delegate.writeNull(type); + } + + public void writeBool(boolean value) throws IOException + { + delegate.writeBool(value); + } + + public void writeInt(long value) throws IOException + { + delegate.writeInt(value); + } + + public void writeInt(BigInteger value) throws IOException + { + delegate.writeInt(value); + } + + public void writeFloat(double value) throws IOException + { + delegate.writeFloat(value); + } + + public void writeDecimal(BigDecimal value) throws IOException + { + delegate.writeDecimal(value); + } + + public void writeTimestamp(Timestamp value) throws IOException + { + delegate.writeTimestamp(value); + } + + public void writeTimestampUTC(Date value) throws IOException + { + delegate.writeTimestampUTC(value); + } + + public void writeSymbol(String content) throws IOException + { + delegate.writeSymbol(content); + } + + public void writeSymbolToken(SymbolToken content) throws IOException + { + delegate.writeSymbolToken(content); + } + + public void writeString(String value) throws IOException + { + delegate.writeString(value); + } + + public void writeClob(byte[] value) throws IOException + { + delegate.writeClob(value); + } + + public void writeClob(byte[] value, int start, int len) throws IOException + { + delegate.writeClob(value, start, len); + } + + public void writeBlob(byte[] value) throws IOException + { + delegate.writeBlob(value); + } + + public void writeBlob(byte[] value, int start, int len) throws IOException + { + delegate.writeBlob(value, start, len); + } + + public T asFacet(Class facetType) + { + // This implementation has no facets. + return null; + } +} diff --git a/src/software/amazon/ion/impl/bin/IonManagedBinaryWriter.java b/src/com/amazon/ion/impl/bin/IonManagedBinaryWriter.java similarity index 92% rename from src/software/amazon/ion/impl/bin/IonManagedBinaryWriter.java rename to src/com/amazon/ion/impl/bin/IonManagedBinaryWriter.java index 61d77afbc7..e898f2a16a 100644 --- a/src/software/amazon/ion/impl/bin/IonManagedBinaryWriter.java +++ b/src/com/amazon/ion/impl/bin/IonManagedBinaryWriter.java @@ -1,35 +1,44 @@ /* - * Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.bin; - +package com.amazon.ion.impl.bin; + +import static com.amazon.ion.IonType.LIST; +import static com.amazon.ion.IonType.STRUCT; +import static com.amazon.ion.SystemSymbols.IMPORTS_SID; +import static com.amazon.ion.SystemSymbols.ION_1_0_MAX_ID; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; +import static com.amazon.ion.SystemSymbols.MAX_ID_SID; +import static com.amazon.ion.SystemSymbols.NAME_SID; +import static com.amazon.ion.SystemSymbols.SYMBOLS_SID; +import static com.amazon.ion.SystemSymbols.VERSION_SID; +import static com.amazon.ion.impl.bin.Symbols.symbol; +import static com.amazon.ion.impl.bin.Symbols.systemSymbol; +import static com.amazon.ion.impl.bin.Symbols.systemSymbolTable; +import static com.amazon.ion.impl.bin.Symbols.systemSymbols; import static java.util.Collections.unmodifiableList; -import static software.amazon.ion.IonType.LIST; -import static software.amazon.ion.IonType.STRUCT; -import static software.amazon.ion.SystemSymbols.IMPORTS_SID; -import static software.amazon.ion.SystemSymbols.ION_1_0_MAX_ID; -import static software.amazon.ion.SystemSymbols.ION_1_0_SID; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; -import static software.amazon.ion.SystemSymbols.MAX_ID_SID; -import static software.amazon.ion.SystemSymbols.NAME_SID; -import static software.amazon.ion.SystemSymbols.SYMBOLS_SID; -import static software.amazon.ion.SystemSymbols.VERSION_SID; -import static software.amazon.ion.impl.bin.Symbols.symbol; -import static software.amazon.ion.impl.bin.Symbols.systemSymbol; -import static software.amazon.ion.impl.bin.Symbols.systemSymbolTable; -import static software.amazon.ion.impl.bin.Symbols.systemSymbols; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonException; +import com.amazon.ion.IonType; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.Timestamp; +import com.amazon.ion.UnknownSymbolException; +import com.amazon.ion.impl.bin.IonRawBinaryWriter.StreamCloseMode; +import com.amazon.ion.impl.bin.IonRawBinaryWriter.StreamFlushMode; import java.io.IOException; import java.io.OutputStream; import java.math.BigDecimal; @@ -42,19 +51,10 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonException; -import software.amazon.ion.IonType; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.Timestamp; -import software.amazon.ion.UnknownSymbolException; -import software.amazon.ion.impl.PrivateUtils; -import software.amazon.ion.impl.bin.IonRawBinaryWriter.StreamCloseMode; -import software.amazon.ion.impl.bin.IonRawBinaryWriter.StreamFlushMode; /** Wraps {@link IonRawBinaryWriter} with symbol table management. */ -/*package*/ final class IonManagedBinaryWriter extends AbstractIonWriter +@SuppressWarnings("deprecation") +/*package*/ final class IonManagedBinaryWriter extends AbstractIonWriter implements _Private_IonManagedWriter { private interface SymbolResolver { @@ -439,13 +439,9 @@ public void afterStepOut(final IonManagedBinaryWriter self) // in case this is intentional symbols = Symbols.unknownSharedSymbolTable(desc.name, desc.version, desc.maxId); } - final boolean hasDeclaredMaxId = desc.maxId != -1; - final boolean declaredMaxIdMatches = desc.maxId == symbols.getMaxId(); - final boolean declaredVersionMatches = desc.version == symbols.getVersion(); - if (hasDeclaredMaxId && (!declaredMaxIdMatches || !declaredVersionMatches)) + if (desc.maxId != -1 && desc.maxId != symbols.getMaxId()) { - // the max ID doesn't match, so we need a substitute - symbols = PrivateUtils.newSubstituteSymtab(symbols, desc.version, desc.maxId); + throw new IllegalArgumentException("Import doesn't match Max ID: " + desc); } self.userImports.add(symbols); } @@ -641,10 +637,9 @@ public void makeReadOnly() private final List userSymbols; private final ImportDescriptor userCurrentImport; - private boolean forceSystemOutput; private boolean closed; - /*package*/ IonManagedBinaryWriter(final PrivateIonManagedBinaryWriterBuilder builder, + /*package*/ IonManagedBinaryWriter(final _Private_IonManagedBinaryWriterBuilder builder, final OutputStream out) throws IOException { @@ -677,8 +672,6 @@ public void makeReadOnly() this.localsLocked = false; this.localSymbolTableView = new LocalSymbolTableView(); this.symbolState = SymbolState.SYSTEM_SYMBOLS; - - this.forceSystemOutput = false; this.closed = false; this.userState = UserState.NORMAL; @@ -715,6 +708,11 @@ public void makeReadOnly() } } + public _Private_IonRawWriter getRawWriter() + { + return user; + } + // Compatibility with Implementation Writer Interface public IonCatalog getCatalog() @@ -889,6 +887,11 @@ public void setFieldNameSymbol(SymbolToken token) user.setFieldNameSymbol(token); } + public void requireLocalSymbolTable() throws IOException + { + startLocalSymbolTableIfNeeded(true); + } + public void setTypeAnnotations(final String... annotations) { if (annotations == null) @@ -993,14 +996,11 @@ public void writeTimestamp(final Timestamp value) throws IOException public void writeSymbol(String content) throws IOException { - final SymbolToken token = intern(content); - writeSymbolToken(token); + writeSymbolToken(intern(content)); } - public void writeSymbolToken(SymbolToken token) throws IOException - { - token = intern(token); - if (token != null && token.getSid() == ION_1_0_SID && user.getDepth() == 0 && !user.hasAnnotations()) + private boolean handleIVM(int sid) throws IOException { + if (user.isIVM(sid)) { if (user.hasWrittenValuesSinceFinished()) { @@ -1009,12 +1009,22 @@ public void writeSymbolToken(SymbolToken token) throws IOException } else { - // TODO determine if redundant IVM writes need to actually be surfaced - // we need to signal that we need to write out the IVM even if nothing else is written - forceSystemOutput = true; + // no-op the write--we already wrote the IVM for the user + // TODO should this not no-op when called multiple times with IVMs? + // TODO integrate with all that IVM configuration-fu } + return true; + } + return false; + } + + public void writeSymbolToken(SymbolToken token) throws IOException + { + if (token != null && handleIVM(token.getSid())) + { return; } + token = intern(token); user.writeSymbolToken(token); } @@ -1044,6 +1054,12 @@ public void writeBlob(final byte[] data, final int offset, final int length) thr user.writeBlob(data, offset, length); } + @Override + public void writeString(byte[] data, int offset, int length) throws IOException + { + user.writeString(data, offset, length); + } + public void writeBytes(byte[] data, int off, int len) throws IOException { // this is a raw transfer--we basically have to dump the symbol table since we don't have much context @@ -1063,7 +1079,7 @@ public void flush() throws IOException private void unsafeFlush() throws IOException { - if (user.hasWrittenValuesSinceFinished() || forceSystemOutput) + if (user.hasWrittenValuesSinceFinished()) { // this implies that we have a local symbol table of some sort and the user locked it symbolState.closeTable(symbols); @@ -1071,7 +1087,6 @@ private void unsafeFlush() throws IOException // make sure that until the local symbol state changes we no-op the table closing routine symbolState = SymbolState.LOCAL_SYMBOLS_FLUSHED; - forceSystemOutput = false; // push the data out symbols.finish(); user.finish(); @@ -1119,4 +1134,5 @@ public void close() throws IOException } } } + } diff --git a/src/software/amazon/ion/impl/bin/IonRawBinaryWriter.java b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java similarity index 90% rename from src/software/amazon/ion/impl/bin/IonRawBinaryWriter.java rename to src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java index b8e13d68aa..25d4e145a5 100644 --- a/src/software/amazon/ion/impl/bin/IonRawBinaryWriter.java +++ b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java @@ -1,42 +1,51 @@ /* - * Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.bin; - +package com.amazon.ion.impl.bin; + +import static com.amazon.ion.Decimal.isNegativeZero; +import static com.amazon.ion.IonType.BLOB; +import static com.amazon.ion.IonType.BOOL; +import static com.amazon.ion.IonType.CLOB; +import static com.amazon.ion.IonType.DECIMAL; +import static com.amazon.ion.IonType.FLOAT; +import static com.amazon.ion.IonType.INT; +import static com.amazon.ion.IonType.LIST; +import static com.amazon.ion.IonType.NULL; +import static com.amazon.ion.IonType.SEXP; +import static com.amazon.ion.IonType.STRING; +import static com.amazon.ion.IonType.STRUCT; +import static com.amazon.ion.IonType.SYMBOL; +import static com.amazon.ion.IonType.TIMESTAMP; +import static com.amazon.ion.IonType.isContainer; +import static com.amazon.ion.SystemSymbols.ION_1_0_SID; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; +import static com.amazon.ion.Timestamp.Precision.DAY; +import static com.amazon.ion.Timestamp.Precision.MINUTE; +import static com.amazon.ion.Timestamp.Precision.MONTH; +import static com.amazon.ion.Timestamp.Precision.SECOND; import static java.lang.Double.doubleToRawLongBits; import static java.lang.Float.floatToRawIntBits; -import static software.amazon.ion.Decimal.isNegativeZero; -import static software.amazon.ion.IonType.BLOB; -import static software.amazon.ion.IonType.BOOL; -import static software.amazon.ion.IonType.CLOB; -import static software.amazon.ion.IonType.DECIMAL; -import static software.amazon.ion.IonType.FLOAT; -import static software.amazon.ion.IonType.INT; -import static software.amazon.ion.IonType.LIST; -import static software.amazon.ion.IonType.NULL; -import static software.amazon.ion.IonType.SEXP; -import static software.amazon.ion.IonType.STRING; -import static software.amazon.ion.IonType.STRUCT; -import static software.amazon.ion.IonType.SYMBOL; -import static software.amazon.ion.IonType.TIMESTAMP; -import static software.amazon.ion.IonType.isContainer; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; -import static software.amazon.ion.Timestamp.Precision.DAY; -import static software.amazon.ion.Timestamp.Precision.MINUTE; -import static software.amazon.ion.Timestamp.Precision.MONTH; -import static software.amazon.ion.Timestamp.Precision.SECOND; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonException; +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.Timestamp; import java.io.IOException; import java.io.OutputStream; import java.math.BigDecimal; @@ -46,18 +55,12 @@ import java.util.LinkedList; import java.util.List; import java.util.NoSuchElementException; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonException; -import software.amazon.ion.IonType; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.Timestamp; /** * Low-level binary {@link IonWriter} that understands encoding concerns but doesn't operate with any sense of symbol table management. */ -/*package*/ final class IonRawBinaryWriter extends AbstractIonWriter +@SuppressWarnings("deprecation") +/*package*/ final class IonRawBinaryWriter extends AbstractIonWriter implements _Private_IonRawWriter { /** short-hand for array of bytes--useful for static definitions. */ private static byte[] bytes(int... vals) { @@ -468,8 +471,8 @@ public String toString() private boolean hasWrittenValuesSinceFinished; private boolean hasWrittenValuesSinceConstructed; - private SymbolToken currentFieldName; - private final List currentAnnotations; + private Integer currentFieldSid; + private final List currentAnnotationSids; // XXX this is for managed detection of TLV that is a LST--this is easier to track here than at the managed level private boolean hasTopLevelSymbolTableAnnotation; @@ -504,8 +507,8 @@ public String toString() this.hasWrittenValuesSinceFinished = false; this.hasWrittenValuesSinceConstructed = false; - this.currentFieldName = null; - this.currentAnnotations = new ArrayList(); + this.currentFieldSid = null; + this.currentAnnotationSids = new ArrayList(); this.hasTopLevelSymbolTableAnnotation = false; this.closed = false; @@ -525,12 +528,17 @@ public void setFieldName(final String name) } public void setFieldNameSymbol(final SymbolToken name) + { + setFieldNameSymbol(name.getSid()); + } + + public void setFieldNameSymbol(int sid) { if (!isInStruct()) { throw new IonException("Cannot set field name outside of struct context"); } - currentFieldName = name; + currentFieldSid = sid; } public void setTypeAnnotations(final String... annotations) @@ -538,15 +546,32 @@ public void setTypeAnnotations(final String... annotations) throw new UnsupportedOperationException("Cannot set annotations on a low-level binary writer via string"); } - public void setTypeAnnotationSymbols(final SymbolToken... annotations) + private void clearAnnotations() { - currentAnnotations.clear(); + currentAnnotationSids.clear(); hasTopLevelSymbolTableAnnotation = false; + } + + public void setTypeAnnotationSymbols(final SymbolToken... annotations) + { + clearAnnotations(); if (annotations != null) { for (final SymbolToken annotation : annotations) { - addTypeAnnotationSymbol(annotation); + addTypeAnnotationSymbol(annotation.getSid()); + } + } + } + + public void setTypeAnnotationSymbols(int... sids) + { + clearAnnotations(); + if (sids != null) + { + for (final int sid : sids) + { + addTypeAnnotationSymbol(sid); } } } @@ -560,16 +585,22 @@ public void addTypeAnnotation(final String annotation) /*package*/ void addTypeAnnotationSymbol(final SymbolToken annotation) { - if (depth == 0 && annotation.getSid() == ION_SYMBOL_TABLE_SID) + addTypeAnnotationSymbol(annotation.getSid()); + + } + + public void addTypeAnnotationSymbol(int sid) + { + if (depth == 0 && sid == ION_SYMBOL_TABLE_SID) { hasTopLevelSymbolTableAnnotation = true; } - currentAnnotations.add(annotation); + currentAnnotationSids.add(sid); } /*package*/ boolean hasAnnotations() { - return !currentAnnotations.isEmpty(); + return !currentAnnotationSids.isEmpty(); } /** Returns true if a value has been written since construction or {@link #finish()}. */ @@ -591,7 +622,7 @@ public void addTypeAnnotation(final String annotation) /*package*/ int getFieldId() { - return currentFieldName.getSid(); + return currentFieldSid; } // Compatibility with Implementation Writer Interface @@ -603,7 +634,7 @@ public IonCatalog getCatalog() public boolean isFieldNameSet() { - return currentFieldName != null; + return currentFieldSid != null; } public void writeIonVersionMarker() throws IOException @@ -737,31 +768,30 @@ private void writeVarInt(final long value) updateLength(len); } - private static int checkSid(SymbolToken symbol) + private static void checkSid(int sid) { - final int sid = symbol.getSid(); if (sid < 0) { - throw new IllegalArgumentException("Invalid symbol: " + symbol.getText() + " SID: " + sid); + throw new IllegalArgumentException("Invalid symbol with SID: " + sid); } - return sid; } /** prepare to write values with field name and annotations. */ private void prepareValue() { - if (isInStruct() && currentFieldName == null) + if (isInStruct() && currentFieldSid == null) { throw new IllegalStateException("IonWriter.setFieldName() must be called before writing a value into a struct."); } - if (currentFieldName != null) + if (currentFieldSid != null) { - writeVarUInt(checkSid(currentFieldName)); + checkSid(currentFieldSid); + writeVarUInt(currentFieldSid); // clear out field name - currentFieldName = null; + currentFieldSid = null; } - if (!currentAnnotations.isEmpty()) + if (!currentAnnotationSids.isEmpty()) { // we have to push a container context for annotations updateLength(preallocationMode.typedLength); @@ -771,16 +801,16 @@ private void prepareValue() final long annotationsLengthPosition = buffer.position(); buffer.writeVarUInt(0L); int annotationsLength = 0; - for (final SymbolToken symbol : currentAnnotations) + for (final int symbol : currentAnnotationSids) { - final int sid = checkSid(symbol); - final int symbolLength = buffer.writeVarUInt(sid); + checkSid(symbol); + final int symbolLength = buffer.writeVarUInt(symbol); annotationsLength += symbolLength; } if (annotationsLength > MAX_ANNOTATION_LENGTH) { // TODO deal with side patching if we want to support > 32 4-byte symbols annotations... seems excessive - throw new IonException("Annotations too large: " + currentAnnotations); + throw new IonException("Annotations too large: " + currentAnnotationSids); } // update the annotations size @@ -789,7 +819,7 @@ private void prepareValue() buffer.writeVarUIntDirect1At(annotationsLengthPosition, annotationsLength); // clear out annotations - currentAnnotations.clear(); + currentAnnotationSids.clear(); hasTopLevelSymbolTableAnnotation = false; } } @@ -824,11 +854,11 @@ public void stepIn(final IonType containerType) throws IOException public void stepOut() throws IOException { - if (currentFieldName != null) + if (currentFieldSid != null) { throw new IonException("Cannot step out with field name set"); } - if (!currentAnnotations.isEmpty()) + if (!currentAnnotationSids.isEmpty()) { throw new IonException("Cannot step out with field name set"); } @@ -1193,7 +1223,6 @@ public void writeDecimal(final BigDecimal value) throws IOException finishValue(); } - @SuppressWarnings("deprecation") public void writeTimestamp(final Timestamp value) throws IOException { if (value == null) @@ -1249,11 +1278,13 @@ public void writeTimestamp(final Timestamp value) throws IOException { final int second = value.getZSecond(); writeVarUInt(second); - final BigDecimal fraction = value.getZFractionalSecond(); - if (fraction != null && !BigDecimal.ZERO.equals(fraction) && fraction.scale() > -1) - { - writeDecimalValue(fraction); + if (fraction != null) { + final BigInteger mantissaBigInt = fraction.unscaledValue(); + final int exponent = -fraction.scale(); + if (!(mantissaBigInt.equals(BigInteger.ZERO) && exponent > -1)) { + writeDecimalValue(fraction); + } } } @@ -1275,7 +1306,26 @@ public void writeSymbolToken(final SymbolToken content) throws IOException writeNull(IonType.SYMBOL); return; } - final int sid = checkSid(content); + writeSymbolToken(content.getSid()); + } + + boolean isIVM(int sid) + { + // When SID 2 occurs at the top level with no annotations, it has the + // special properties of an IVM. Otherwise, it's considered a normal + // symbol value. + // TODO amzn/ion-java/issues/88 requires this behavior to be changed, + // such that top-level SID 2 is treated as a symbol value, not an IVM. + return depth == 0 && sid == ION_1_0_SID && !hasAnnotations(); + } + + public void writeSymbolToken(int sid) throws IOException + { + if (isIVM(sid)) + { + throw new IonException("Direct writing of IVM is not supported in low-level binary writer"); + } + checkSid(sid); prepareValue(); writeTypedUInt(SYMBOL_TYPE, sid); finishValue(); @@ -1397,6 +1447,19 @@ public void writeBlob(final byte[] data, final int offset, final int length) thr finishValue(); } + @Override + public void writeString(byte[] data, int offset, int length) throws IOException + { + if (data == null) + { + writeNull(IonType.STRING); + return; + } + prepareValue(); + writeTypedBytes(STRING_TYPE, data, offset, length); + finishValue(); + } + /** * Writes a raw value into the buffer, updating lengths appropriately. *

    @@ -1508,4 +1571,5 @@ public void close() throws IOException } } } + } diff --git a/src/software/amazon/ion/impl/bin/PooledBlockAllocatorProvider.java b/src/com/amazon/ion/impl/bin/PooledBlockAllocatorProvider.java similarity index 86% rename from src/software/amazon/ion/impl/bin/PooledBlockAllocatorProvider.java rename to src/com/amazon/ion/impl/bin/PooledBlockAllocatorProvider.java index 4cc61f491e..831cb4d2ce 100644 --- a/src/software/amazon/ion/impl/bin/PooledBlockAllocatorProvider.java +++ b/src/com/amazon/ion/impl/bin/PooledBlockAllocatorProvider.java @@ -1,18 +1,19 @@ /* - * Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.bin; +package com.amazon.ion.impl.bin; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; diff --git a/src/software/amazon/ion/impl/bin/Symbols.java b/src/com/amazon/ion/impl/bin/Symbols.java similarity index 83% rename from src/software/amazon/ion/impl/bin/Symbols.java rename to src/com/amazon/ion/impl/bin/Symbols.java index ec298cf224..77540c9ae7 100644 --- a/src/software/amazon/ion/impl/bin/Symbols.java +++ b/src/com/amazon/ion/impl/bin/Symbols.java @@ -1,51 +1,52 @@ /* - * Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.bin; - +package com.amazon.ion.impl.bin; + +import static com.amazon.ion.SystemSymbols.IMPORTS; +import static com.amazon.ion.SystemSymbols.IMPORTS_SID; +import static com.amazon.ion.SystemSymbols.ION; +import static com.amazon.ion.SystemSymbols.ION_1_0; +import static com.amazon.ion.SystemSymbols.ION_1_0_MAX_ID; +import static com.amazon.ion.SystemSymbols.ION_1_0_SID; +import static com.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE; +import static com.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE_SID; +import static com.amazon.ion.SystemSymbols.ION_SID; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; +import static com.amazon.ion.SystemSymbols.MAX_ID; +import static com.amazon.ion.SystemSymbols.MAX_ID_SID; +import static com.amazon.ion.SystemSymbols.NAME; +import static com.amazon.ion.SystemSymbols.NAME_SID; +import static com.amazon.ion.SystemSymbols.SYMBOLS; +import static com.amazon.ion.SystemSymbols.SYMBOLS_SID; +import static com.amazon.ion.SystemSymbols.VERSION; +import static com.amazon.ion.SystemSymbols.VERSION_SID; import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; import static java.util.Collections.unmodifiableMap; -import static software.amazon.ion.SystemSymbols.IMPORTS; -import static software.amazon.ion.SystemSymbols.IMPORTS_SID; -import static software.amazon.ion.SystemSymbols.ION; -import static software.amazon.ion.SystemSymbols.ION_1_0; -import static software.amazon.ion.SystemSymbols.ION_1_0_MAX_ID; -import static software.amazon.ion.SystemSymbols.ION_1_0_SID; -import static software.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE; -import static software.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE_SID; -import static software.amazon.ion.SystemSymbols.ION_SID; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; -import static software.amazon.ion.SystemSymbols.MAX_ID; -import static software.amazon.ion.SystemSymbols.MAX_ID_SID; -import static software.amazon.ion.SystemSymbols.NAME; -import static software.amazon.ion.SystemSymbols.NAME_SID; -import static software.amazon.ion.SystemSymbols.SYMBOLS; -import static software.amazon.ion.SystemSymbols.SYMBOLS_SID; -import static software.amazon.ion.SystemSymbols.VERSION; -import static software.amazon.ion.SystemSymbols.VERSION_SID; +import com.amazon.ion.IonException; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; -import software.amazon.ion.IonException; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; /** * Utilities for dealing with {@link SymbolToken} and {@link SymbolTable}. diff --git a/src/software/amazon/ion/impl/bin/WriteBuffer.java b/src/com/amazon/ion/impl/bin/WriteBuffer.java similarity index 98% rename from src/software/amazon/ion/impl/bin/WriteBuffer.java rename to src/com/amazon/ion/impl/bin/WriteBuffer.java index b5ddb05b72..e3973c56e4 100644 --- a/src/software/amazon/ion/impl/bin/WriteBuffer.java +++ b/src/com/amazon/ion/impl/bin/WriteBuffer.java @@ -1,18 +1,19 @@ /* - * Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.bin; +package com.amazon.ion.impl.bin; import java.io.Closeable; import java.io.IOException; @@ -1073,8 +1074,8 @@ public void writeTo(final OutputStream out) throws IOException { for (int i = 0; i <= index; i++) { - Block block = blocks.get(i); - out.write(block.data, 0, block.limit); + Block block = blocks.get(i); + out.write(block.data, 0, block.limit); } } diff --git a/src/com/amazon/ion/impl/bin/_PrivateIon_HashTrampoline.java b/src/com/amazon/ion/impl/bin/_PrivateIon_HashTrampoline.java new file mode 100644 index 0000000000..a54da159a1 --- /dev/null +++ b/src/com/amazon/ion/impl/bin/_PrivateIon_HashTrampoline.java @@ -0,0 +1,44 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl.bin; + +import com.amazon.ion.IonWriter; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +/** + * NOT FOR APPLICATION USE! + * + * Exposes {@link IonRawBinaryWriter} functionality for use when creating Ion hashes. + */ +@Deprecated +public class _PrivateIon_HashTrampoline +{ + public static IonWriter newIonWriter(ByteArrayOutputStream baos) throws IOException + { + return new IonRawBinaryWriter( + new PooledBlockAllocatorProvider(), + _Private_IonManagedBinaryWriterBuilder.DEFAULT_BLOCK_SIZE, + baos, + AbstractIonWriter.WriteValueOptimization.NONE, + IonRawBinaryWriter.StreamCloseMode.CLOSE, + IonRawBinaryWriter.StreamFlushMode.FLUSH, + IonRawBinaryWriter.PreallocationMode.PREALLOCATE_0, + false // force floats to be encoded as binary64 + ); + } +} diff --git a/src/software/amazon/ion/impl/bin/PrivateIonManagedBinaryWriterBuilder.java b/src/com/amazon/ion/impl/bin/_Private_IonManagedBinaryWriterBuilder.java similarity index 62% rename from src/software/amazon/ion/impl/bin/PrivateIonManagedBinaryWriterBuilder.java rename to src/com/amazon/ion/impl/bin/_Private_IonManagedBinaryWriterBuilder.java index 2443beeb0a..8c1dd6aab9 100644 --- a/src/software/amazon/ion/impl/bin/PrivateIonManagedBinaryWriterBuilder.java +++ b/src/com/amazon/ion/impl/bin/_Private_IonManagedBinaryWriterBuilder.java @@ -1,35 +1,39 @@ /* - * Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.bin; +package com.amazon.ion.impl.bin; -import static software.amazon.ion.impl.bin.IonManagedBinaryWriter.ONLY_SYSTEM_IMPORTS; +import static com.amazon.ion.impl.bin.IonManagedBinaryWriter.ONLY_SYSTEM_IMPORTS; +import com.amazon.ion.IonBinaryWriter; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonException; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SubstituteSymbolTableException; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SystemSymbols; +import com.amazon.ion.impl.bin.AbstractIonWriter.WriteValueOptimization; +import com.amazon.ion.impl.bin.IonBinaryWriterAdapter.Factory; +import com.amazon.ion.impl.bin.IonManagedBinaryWriter.ImportedSymbolContext; +import com.amazon.ion.impl.bin.IonManagedBinaryWriter.ImportedSymbolResolverMode; +import com.amazon.ion.impl.bin.IonRawBinaryWriter.PreallocationMode; +import com.amazon.ion.system.SimpleCatalog; import java.io.IOException; import java.io.OutputStream; import java.util.Arrays; import java.util.List; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SubstituteSymbolTableException; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SystemSymbols; -import software.amazon.ion.impl.bin.AbstractIonWriter.WriteValueOptimization; -import software.amazon.ion.impl.bin.IonManagedBinaryWriter.ImportedSymbolContext; -import software.amazon.ion.impl.bin.IonManagedBinaryWriter.ImportedSymbolResolverMode; -import software.amazon.ion.impl.bin.IonRawBinaryWriter.PreallocationMode; -import software.amazon.ion.system.SimpleCatalog; // TODO unify this with the IonWriter builder APIs @@ -37,11 +41,9 @@ * Constructs instances of binary {@link IonWriter}. *

    * This class is thread-safe. - * - * @deprecated This is an internal API that is subject to change without notice. */ -@Deprecated -public final class PrivateIonManagedBinaryWriterBuilder +@SuppressWarnings("deprecation") +public final class _Private_IonManagedBinaryWriterBuilder { public enum AllocatorMode { @@ -77,7 +79,7 @@ BlockAllocatorProvider createAllocatorProvider() /*package*/ volatile SymbolTable initialSymbolTable; /*package*/ volatile boolean isFloatBinary32Enabled; - private PrivateIonManagedBinaryWriterBuilder(final BlockAllocatorProvider provider) + private _Private_IonManagedBinaryWriterBuilder(final BlockAllocatorProvider provider) { this.provider = provider; this.symbolsBlockSize = DEFAULT_BLOCK_SIZE; @@ -89,7 +91,7 @@ private PrivateIonManagedBinaryWriterBuilder(final BlockAllocatorProvider provid this.isFloatBinary32Enabled = false; } - private PrivateIonManagedBinaryWriterBuilder(final PrivateIonManagedBinaryWriterBuilder other) + private _Private_IonManagedBinaryWriterBuilder(final _Private_IonManagedBinaryWriterBuilder other) { this.provider = other.provider; this.symbolsBlockSize = other.symbolsBlockSize; @@ -102,14 +104,14 @@ private PrivateIonManagedBinaryWriterBuilder(final PrivateIonManagedBinaryWriter this.isFloatBinary32Enabled = other.isFloatBinary32Enabled; } - public PrivateIonManagedBinaryWriterBuilder copy() + public _Private_IonManagedBinaryWriterBuilder copy() { - return new PrivateIonManagedBinaryWriterBuilder(this); + return new _Private_IonManagedBinaryWriterBuilder(this); } // Parameter Setting Methods - public PrivateIonManagedBinaryWriterBuilder withSymbolsBlockSize(final int blockSize) + public _Private_IonManagedBinaryWriterBuilder withSymbolsBlockSize(final int blockSize) { if (blockSize < 1) { @@ -119,7 +121,7 @@ public PrivateIonManagedBinaryWriterBuilder withSymbolsBlockSize(final int block return this; } - public PrivateIonManagedBinaryWriterBuilder withUserBlockSize(final int blockSize) + public _Private_IonManagedBinaryWriterBuilder withUserBlockSize(final int blockSize) { if (blockSize < 1) { @@ -129,7 +131,7 @@ public PrivateIonManagedBinaryWriterBuilder withUserBlockSize(final int blockSiz return this; } - public PrivateIonManagedBinaryWriterBuilder withImports(final SymbolTable... tables) + public _Private_IonManagedBinaryWriterBuilder withImports(final SymbolTable... tables) { if (tables != null) { @@ -138,7 +140,7 @@ public PrivateIonManagedBinaryWriterBuilder withImports(final SymbolTable... tab return this; } - public PrivateIonManagedBinaryWriterBuilder withImports(final List tables) + public _Private_IonManagedBinaryWriterBuilder withImports(final List tables) { return withImports(ImportedSymbolResolverMode.DELEGATE, tables); } @@ -147,7 +149,7 @@ public PrivateIonManagedBinaryWriterBuilder withImports(final List * Adds imports, flattening them to make lookup more efficient. This is particularly useful * when a builder instance is long lived. */ - public PrivateIonManagedBinaryWriterBuilder withFlatImports(final SymbolTable... tables) + public _Private_IonManagedBinaryWriterBuilder withFlatImports(final SymbolTable... tables) { if (tables != null) { @@ -157,51 +159,51 @@ public PrivateIonManagedBinaryWriterBuilder withFlatImports(final SymbolTable... } /** @see #withFlatImports(SymbolTable...) */ - public PrivateIonManagedBinaryWriterBuilder withFlatImports(final List tables) + public _Private_IonManagedBinaryWriterBuilder withFlatImports(final List tables) { return withImports(ImportedSymbolResolverMode.FLAT, tables); } - /*package*/ PrivateIonManagedBinaryWriterBuilder withImports(final ImportedSymbolResolverMode mode, final List tables) { + /*package*/ _Private_IonManagedBinaryWriterBuilder withImports(final ImportedSymbolResolverMode mode, final List tables) { imports = new ImportedSymbolContext(mode, tables); return this; } - /*package*/ PrivateIonManagedBinaryWriterBuilder withPreallocationMode(final PreallocationMode preallocationMode) + /*package*/ _Private_IonManagedBinaryWriterBuilder withPreallocationMode(final PreallocationMode preallocationMode) { this.preallocationMode = preallocationMode; return this; } - public PrivateIonManagedBinaryWriterBuilder withPaddedLengthPreallocation(final int pad) + public _Private_IonManagedBinaryWriterBuilder withPaddedLengthPreallocation(final int pad) { this.preallocationMode = PreallocationMode.withPadSize(pad); return this; } - public PrivateIonManagedBinaryWriterBuilder withCatalog(final IonCatalog catalog) + public _Private_IonManagedBinaryWriterBuilder withCatalog(final IonCatalog catalog) { this.catalog = catalog; return this; } - public PrivateIonManagedBinaryWriterBuilder withStreamCopyOptimization(boolean optimized) + public _Private_IonManagedBinaryWriterBuilder withStreamCopyOptimization(boolean optimized) { this.optimization = optimized ? WriteValueOptimization.COPY_OPTIMIZED : WriteValueOptimization.NONE; return this; } - public PrivateIonManagedBinaryWriterBuilder withFloatBinary32Enabled() { + public _Private_IonManagedBinaryWriterBuilder withFloatBinary32Enabled() { isFloatBinary32Enabled = true; return this; } - public PrivateIonManagedBinaryWriterBuilder withFloatBinary32Disabled() { + public _Private_IonManagedBinaryWriterBuilder withFloatBinary32Disabled() { isFloatBinary32Enabled = false; return this; } - public PrivateIonManagedBinaryWriterBuilder withInitialSymbolTable(SymbolTable symbolTable) + public _Private_IonManagedBinaryWriterBuilder withInitialSymbolTable(SymbolTable symbolTable) { if (symbolTable != null) { @@ -243,6 +245,26 @@ public IonWriter newWriter(final OutputStream out) throws IOException return new IonManagedBinaryWriter(this, out); } + public IonBinaryWriter newLegacyWriter() + { + try + { + return new IonBinaryWriterAdapter( + new Factory() + { + public IonWriter create(final OutputStream out) throws IOException + { + return newWriter(out); + } + } + ); + } + catch (final IOException e) + { + throw new IonException("I/O error", e); + } + } + // Static Factories /** @@ -251,8 +273,8 @@ public IonWriter newWriter(final OutputStream out) throws IOException * Builders generally bind to an allocation pool as defined by {@link AllocatorMode}, so applications should reuse * them as much as possible. */ - public static PrivateIonManagedBinaryWriterBuilder create(final AllocatorMode allocatorMode) + public static _Private_IonManagedBinaryWriterBuilder create(final AllocatorMode allocatorMode) { - return new PrivateIonManagedBinaryWriterBuilder(allocatorMode.createAllocatorProvider()); + return new _Private_IonManagedBinaryWriterBuilder(allocatorMode.createAllocatorProvider()); } } diff --git a/src/com/amazon/ion/impl/bin/_Private_IonManagedWriter.java b/src/com/amazon/ion/impl/bin/_Private_IonManagedWriter.java new file mode 100644 index 0000000000..b836315bae --- /dev/null +++ b/src/com/amazon/ion/impl/bin/_Private_IonManagedWriter.java @@ -0,0 +1,64 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl.bin; + +import com.amazon.ion.IonWriter; +import java.io.IOException; + +/** + * An {@link IonWriter} that manages local symbol tables, while providing access + * to its underlying IonRawWriter. + * + * @deprecated This is a private API subject to change without notice. + */ +@Deprecated +public interface _Private_IonManagedWriter extends IonWriter +{ + /** + * Get the underlying raw user value writer. This may be used to directly + * write user values, field names, and annotations, bypassing any symbol + * table management performed by this IonManagedWriter. + * @return the {@link _Private_IonRawWriter} responsible for writing this + * IonManagedWriter's user values. + */ + _Private_IonRawWriter getRawWriter(); + + /** + * If a local symbol table has not already been started by this writer since + * it was constructed or last finished, start writing a local symbol table + * by opening the struct and declaring the imports. Any local symbols + * subsequently written using this IonManagedWriter will be appended as they + * are encountered. + * + * When using the raw writer, a call to this API is required unless one of + * the following applies: + *

      + *
    • The user won't be writing any symbols.
    • + *
    • The user has manually written the symbol table struct to the + * correct raw writer. In this case, making this call would start a + * new (and probably redundant) symbol table context, which is + * wasteful but not harmful.
    • + *
    • The user definitely will be writing more symbols in this context + * using the managed writer.
    • + *
    + * Failing to call this when required will leave the system symbol table as + * the active symbol table context for the subsequently written values, which + * will lead to missing symbol ID mappings. + * @throws IOException + */ + void requireLocalSymbolTable() throws IOException; + +} diff --git a/src/com/amazon/ion/impl/bin/_Private_IonRawWriter.java b/src/com/amazon/ion/impl/bin/_Private_IonRawWriter.java new file mode 100644 index 0000000000..23ac04186e --- /dev/null +++ b/src/com/amazon/ion/impl/bin/_Private_IonRawWriter.java @@ -0,0 +1,84 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl.bin; + +import com.amazon.ion.IonWriter; +import java.io.IOException; + +/** + * An {@link IonWriter} with no symbol table management. + * + * @deprecated This is a private API subject to change without notice. + */ +@Deprecated +public interface _Private_IonRawWriter extends IonWriter +{ + /** + * Sets the current field name to the given symbol ID. It is up to the + * caller to make sure the given symbol ID has a mapping in the current + * context's symbol table. The pending field name symbol is cleared when the + * current value is written via stepIn() or one of the write*() methods. + * @param sid - a symbol ID + */ + public void setFieldNameSymbol(int sid); + + /** + * Sets the full list of pending annotations to the given symbol IDs. Any + * pending annotation symbol IDs are cleared. It is up to the caller to make + * sure the given symbol IDs have mappings in the current context's symbol + * table. The contents of the given array are copied into this writer, so + * the caller does not need to preserve the array. The list of pending + * annotation symbol IDs is cleared when the current value is written via + * stepIn() or one of the write*() methods. + * @param sids - symbol IDs representing the annotation symbols for the + * current value + */ + public void setTypeAnnotationSymbols(int... sids); + + /** + * Adds the given symbol ID to the list of pending annotation symbol IDs. It + * is up to the caller to make sure the given symbol ID has a mapping in the + * current context's symbol table. The list of pending annotation symbol IDs + * is cleared when the current value is written via stepIn() or one of the + * write*() methods. + * @param sid - a symbol ID + */ + public void addTypeAnnotationSymbol(int sid); + + /** + * Directly write the given symbol ID to represent a symbol value. It is up + * to the caller to make sure the given symbol ID has a mapping in the + * current context's symbol table. + * @param sid - a symbol ID + * @throws IOException + */ + public void writeSymbolToken(int sid) throws IOException; + + /** + * Writes a portion of the byte array out as an IonString value. This + * copies the portion of the byte array that is written. + * + * @param data well-formed UTF-8-encoded bytes to be written. + * May be {@code null} to represent {@code null.string}. + * @param offset offset of the first byte in value to write + * @param length number of bytes to write from value + * @see IonWriter#writeClob(byte[], int, int) + * @see IonWriter#writeBlob(byte[], int, int) + */ + public void writeString(byte[] data, int offset, int length) + throws IOException; + +} diff --git a/src/com/amazon/ion/impl/bin/package-info.java b/src/com/amazon/ion/impl/bin/package-info.java new file mode 100644 index 0000000000..14ec59635e --- /dev/null +++ b/src/com/amazon/ion/impl/bin/package-info.java @@ -0,0 +1,51 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +/** + * Provides the implementation for the second-generation Ion binary implementation. + * At this time, this is limited to a binary {@link com.amazon.ion.IonWriter}. + * + *

    + * This package limits most of its APIs to package-level access, the public API of note is contained within + * the {@link com.amazon.ion.impl.bin._Private_IonManagedBinaryWriterBuilder} which builds instances of + * {@link com.amazon.ion.impl.bin.IonManagedBinaryWriter}. See the below section for what Managed means + * in this context. + * + *

    Block API

    + * A generalized interface for blocks of heap memory are provided via the {@link com.amazon.ion.impl.bin.Block} API. + * There are two factory type APIs to actually get a {@link com.amazon.ion.impl.bin.Block} instance: + * {@link com.amazon.ion.impl.bin.BlockAllocator} which vend blocks of a particular fixed size + * and {@link com.amazon.ion.impl.bin.BlockAllocatorProvider} which creates {@link com.amazon.ion.impl.bin.BlockAllocator} + * instances. + *

    + * The primary reason for this level of indirection is flexibility for the underlying implementations of {@link com.amazon.ion.impl.bin.Block} + * and {@link com.amazon.ion.impl.bin.BlockAllocator}. These APIs are not required to be thread-safe, whereas + * {@link com.amazon.ion.impl.bin.BlockAllocatorProvider} is required to be thread-safe. + *

    + * The APIs for {@link com.amazon.ion.impl.bin.BlockAllocator} and {@link com.amazon.ion.impl.bin.Block} + * follow the resource pattern (similar in principle to I/O streams), and should be closed when no longer needed + * to allow implementation resources to be released or re-used. + * + *

    Raw Binary Ion Writer

    + * The {@link com.amazon.ion.impl.bin.IonRawBinaryWriter} deals with the low-level encoding considerations of the + * Ion format. The {@link com.amazon.ion.impl.bin.WriteBuffer} is used closely with this implementation to + * deal with the Ion sub-field encodings (e.g. VarInt, VarUInt, and UTF-8). + * + *

    Managed Binary Ion Writer

    + * The {@link com.amazon.ion.impl.bin.IonManagedBinaryWriter} is layered on top of the {@link com.amazon.ion.impl.bin.IonRawBinaryWriter}. + * In particular, it intercepts symbol, annotation, field names and handles the mechanics of symbol table management + * transparently to the user. + */ +package com.amazon.ion.impl.bin; \ No newline at end of file diff --git a/src/software/amazon/ion/impl/lite/ContainerlessContext.java b/src/com/amazon/ion/impl/lite/ContainerlessContext.java similarity index 72% rename from src/software/amazon/ion/impl/lite/ContainerlessContext.java rename to src/com/amazon/ion/impl/lite/ContainerlessContext.java index 5e41b5a28d..f71fb21726 100644 --- a/src/software/amazon/ion/impl/lite/ContainerlessContext.java +++ b/src/com/amazon/ion/impl/lite/ContainerlessContext.java @@ -1,20 +1,21 @@ /* - * Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; +package com.amazon.ion.impl.lite; -import software.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolTable; /** * Context for IonValues that are not contained in any Container or Datagram diff --git a/src/software/amazon/ion/impl/lite/IonBlobLite.java b/src/com/amazon/ion/impl/lite/IonBlobLite.java similarity index 73% rename from src/software/amazon/ion/impl/lite/IonBlobLite.java rename to src/com/amazon/ion/impl/lite/IonBlobLite.java index a7f899964a..4373d6dcf2 100644 --- a/src/software/amazon/ion/impl/lite/IonBlobLite.java +++ b/src/com/amazon/ion/impl/lite/IonBlobLite.java @@ -1,26 +1,28 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; +package com.amazon.ion.impl.lite; +import com.amazon.ion.IonBlob; +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.ValueVisitor; +import com.amazon.ion.impl._Private_Utils; import java.io.IOException; import java.io.InputStream; -import software.amazon.ion.IonBlob; -import software.amazon.ion.IonType; -import software.amazon.ion.IonWriter; -import software.amazon.ion.ValueVisitor; -import software.amazon.ion.impl.PrivateUtils; + final class IonBlobLite extends IonLobLite @@ -73,7 +75,7 @@ public void printBase64(Appendable out) InputStream byteStream = newInputStream(); try { - PrivateUtils.writeAsBase64(byteStream, out); + _Private_Utils.writeAsBase64(byteStream, out); } finally { diff --git a/src/software/amazon/ion/impl/lite/IonBoolLite.java b/src/com/amazon/ion/impl/lite/IonBoolLite.java similarity index 81% rename from src/software/amazon/ion/impl/lite/IonBoolLite.java rename to src/com/amazon/ion/impl/lite/IonBoolLite.java index fe85e2165a..7c62a943c1 100644 --- a/src/software/amazon/ion/impl/lite/IonBoolLite.java +++ b/src/com/amazon/ion/impl/lite/IonBoolLite.java @@ -1,25 +1,27 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; +package com.amazon.ion.impl.lite; +import com.amazon.ion.IonBool; +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.NullValueException; +import com.amazon.ion.ValueVisitor; import java.io.IOException; -import software.amazon.ion.IonBool; -import software.amazon.ion.IonType; -import software.amazon.ion.IonWriter; -import software.amazon.ion.NullValueException; -import software.amazon.ion.ValueVisitor; + final class IonBoolLite extends IonValueLite diff --git a/src/software/amazon/ion/impl/lite/IonClobLite.java b/src/com/amazon/ion/impl/lite/IonClobLite.java similarity index 75% rename from src/software/amazon/ion/impl/lite/IonClobLite.java rename to src/com/amazon/ion/impl/lite/IonClobLite.java index f5186d9b16..38d7ec1d16 100644 --- a/src/software/amazon/ion/impl/lite/IonClobLite.java +++ b/src/com/amazon/ion/impl/lite/IonClobLite.java @@ -1,29 +1,31 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; +package com.amazon.ion.impl.lite; +import com.amazon.ion.IonClob; +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.ValueVisitor; +import com.amazon.ion.impl._Private_Utils; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; import java.nio.charset.Charset; -import software.amazon.ion.IonClob; -import software.amazon.ion.IonType; -import software.amazon.ion.IonWriter; -import software.amazon.ion.ValueVisitor; -import software.amazon.ion.impl.PrivateUtils; + final class IonClobLite extends IonLobLite @@ -84,7 +86,7 @@ public String stringValue(Charset cs) byte[] bytes = getBytes(); if (bytes == null) return null; - return PrivateUtils.decode(bytes, cs); + return _Private_Utils.decode(bytes, cs); } @Override diff --git a/src/software/amazon/ion/impl/lite/IonContainerLite.java b/src/com/amazon/ion/impl/lite/IonContainerLite.java similarity index 91% rename from src/software/amazon/ion/impl/lite/IonContainerLite.java rename to src/com/amazon/ion/impl/lite/IonContainerLite.java index d8fe748df1..4efdd23dbb 100644 --- a/src/software/amazon/ion/impl/lite/IonContainerLite.java +++ b/src/com/amazon/ion/impl/lite/IonContainerLite.java @@ -1,39 +1,40 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; - +package com.amazon.ion.impl.lite; + +import com.amazon.ion.ContainedValueException; +import com.amazon.ion.IonContainer; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonException; +import com.amazon.ion.IonValue; +import com.amazon.ion.NullValueException; +import com.amazon.ion.ReadOnlyValueException; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.ValueVisitor; +import com.amazon.ion.impl._Private_IonConstants; +import com.amazon.ion.impl._Private_IonContainer; +import com.amazon.ion.impl._Private_Utils; import java.util.HashMap; import java.util.Iterator; import java.util.ListIterator; import java.util.NoSuchElementException; -import software.amazon.ion.ContainedValueException; -import software.amazon.ion.IonContainer; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonException; -import software.amazon.ion.IonValue; -import software.amazon.ion.NullValueException; -import software.amazon.ion.ReadOnlyValueException; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.ValueVisitor; -import software.amazon.ion.impl.PrivateIonConstants; -import software.amazon.ion.impl.PrivateIonContainer; -import software.amazon.ion.impl.PrivateUtils; abstract class IonContainerLite extends IonValueLite - implements PrivateIonContainer, IonContext + implements _Private_IonContainer, IonContext { protected int _child_count; @@ -153,7 +154,7 @@ public ListIterator listIterator(int index) if (isNullValue()) { if (index != 0) throw new IndexOutOfBoundsException(); - return PrivateUtils. emptyIterator(); + return _Private_Utils. emptyIterator(); } return new SequenceContentIterator(index, isReadOnly()); @@ -409,6 +410,24 @@ void makeReadOnlyInternal() _isLocked(true); } + + + + /** + * methods from IonValue + * + * public void deepMaterialize() + * public IonContainer getContainer() + * public int getFieldId() + * public String getFieldName() + * public String[] getTypeAnnotations() + * public boolean hasTypeAnnotation(String annotation) + * public boolean isNullValue() + * public boolean isReadOnly() + * public void removeTypeAnnotation(String annotation) + */ + + /* * IonContext methods * @@ -553,20 +572,20 @@ void add(int index, IonValueLite child) */ static final int[] INITIAL_SIZE = make_initial_size_array(); static int[] make_initial_size_array() { - int[] sizes = new int[PrivateIonConstants.tidDATAGRAM + 1]; - sizes[PrivateIonConstants.tidList] = 1; - sizes[PrivateIonConstants.tidSexp] = 4; - sizes[PrivateIonConstants.tidStruct] = 5; - sizes[PrivateIonConstants.tidDATAGRAM] = 3; + int[] sizes = new int[_Private_IonConstants.tidDATAGRAM + 1]; + sizes[_Private_IonConstants.tidList] = 1; + sizes[_Private_IonConstants.tidSexp] = 4; + sizes[_Private_IonConstants.tidStruct] = 5; + sizes[_Private_IonConstants.tidDATAGRAM] = 3; return sizes; } static final int[] NEXT_SIZE = make_next_size_array(); static int[] make_next_size_array() { - int[] sizes = new int[PrivateIonConstants.tidDATAGRAM + 1]; - sizes[PrivateIonConstants.tidList] = 4; - sizes[PrivateIonConstants.tidSexp] = 8; - sizes[PrivateIonConstants.tidStruct] = 8; - sizes[PrivateIonConstants.tidDATAGRAM] = 10; + int[] sizes = new int[_Private_IonConstants.tidDATAGRAM + 1]; + sizes[_Private_IonConstants.tidList] = 4; + sizes[_Private_IonConstants.tidSexp] = 8; + sizes[_Private_IonConstants.tidStruct] = 8; + sizes[_Private_IonConstants.tidDATAGRAM] = 10; return sizes; } final protected int initialSize() diff --git a/src/software/amazon/ion/impl/lite/IonContext.java b/src/com/amazon/ion/impl/lite/IonContext.java similarity index 80% rename from src/software/amazon/ion/impl/lite/IonContext.java rename to src/com/amazon/ion/impl/lite/IonContext.java index f701a80ee6..2f7089f1ef 100644 --- a/src/software/amazon/ion/impl/lite/IonContext.java +++ b/src/com/amazon/ion/impl/lite/IonContext.java @@ -1,21 +1,22 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; +package com.amazon.ion.impl.lite; -import software.amazon.ion.IonContainer; -import software.amazon.ion.SymbolTable; +import com.amazon.ion.IonContainer; +import com.amazon.ion.SymbolTable; /** * Provides the parent, system and symbol table definitions that are shared diff --git a/src/software/amazon/ion/impl/lite/IonDatagramLite.java b/src/com/amazon/ion/impl/lite/IonDatagramLite.java similarity index 94% rename from src/software/amazon/ion/impl/lite/IonDatagramLite.java rename to src/com/amazon/ion/impl/lite/IonDatagramLite.java index cdc6c62ebd..75645fd862 100644 --- a/src/software/amazon/ion/impl/lite/IonDatagramLite.java +++ b/src/com/amazon/ion/impl/lite/IonDatagramLite.java @@ -1,44 +1,45 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; - -import static software.amazon.ion.SystemSymbols.ION_1_0; - +package com.amazon.ion.impl.lite; + +import static com.amazon.ion.SystemSymbols.ION_1_0; + +import com.amazon.ion.ContainedValueException; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonException; +import com.amazon.ion.IonSymbol; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.SystemSymbols; +import com.amazon.ion.ValueFactory; +import com.amazon.ion.ValueVisitor; +import com.amazon.ion.impl._Private_CurriedValueFactory; +import com.amazon.ion.impl._Private_IonDatagram; +import com.amazon.ion.impl._Private_Utils; import java.io.IOException; import java.io.OutputStream; import java.lang.reflect.Array; import java.util.Collection; import java.util.ListIterator; import java.util.NoSuchElementException; -import software.amazon.ion.ContainedValueException; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonException; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.SystemSymbols; -import software.amazon.ion.ValueFactory; -import software.amazon.ion.ValueVisitor; -import software.amazon.ion.impl.PrivateCurriedValueFactory; -import software.amazon.ion.impl.PrivateIonDatagram; -import software.amazon.ion.impl.PrivateUtils; /** * The datagram generally behaves as an IonSexp. A list with space @@ -77,7 +78,7 @@ final class IonDatagramLite extends IonSequenceLite - implements IonDatagram, IonContext, PrivateIonDatagram + implements IonDatagram, IonContext, _Private_IonDatagram { private static final int HASH_SIGNATURE = @@ -213,7 +214,7 @@ public boolean add(IonValue child) @Override public ValueFactory add() { - return new PrivateCurriedValueFactory(this.getSystem()) + return new _Private_CurriedValueFactory(this.getSystem()) { @Override protected void handle(IonValue newValue) @@ -255,7 +256,7 @@ public void add(int index, IonValue element) @Override public ValueFactory add(final int index) { - return new PrivateCurriedValueFactory(getSystem()) + return new _Private_CurriedValueFactory(getSystem()) { @Override protected void handle(IonValue newValue) @@ -461,6 +462,22 @@ public byte[] getBytes() throws IonException return encoder.toNewByteArray(); } + public int getBytes(byte[] dst) throws IonException + { + ReverseBinaryEncoder encoder = + new ReverseBinaryEncoder(REVERSE_BINARY_ENCODER_INITIAL_SIZE); + encoder.serialize(this); + return encoder.toNewByteArray(dst); + } + + public int getBytes(byte[] dst, int offset) throws IonException + { + ReverseBinaryEncoder encoder = + new ReverseBinaryEncoder(REVERSE_BINARY_ENCODER_INITIAL_SIZE); + encoder.serialize(this); + return encoder.toNewByteArray(dst, offset); + } + public int getBytes(OutputStream out) throws IOException, IonException { ReverseBinaryEncoder encoder = @@ -524,6 +541,11 @@ void setSymbolTableAtIndex(int elementid, SymbolTable symbols) } } + public byte[] toBytes() throws IonException + { + return getBytes(); + } + ////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////// @@ -1017,7 +1039,8 @@ void load_current_symbol_table(IonValueLite prev_user_value) rep = __iterator.get_datagram_ivm(); } else { - rep = PrivateUtils.symtabTree(new_symbol_table); + IonSystem sys = __iterator.get_datagram_system(); + rep = _Private_Utils.symtabTree(new_symbol_table); } assert(rep != null && __iterator.get_datagram_system() == rep.getSystem()); @@ -1124,7 +1147,7 @@ private static int count_system_values(IonSystem sys, int count = 0; while (curr.isLocalTable()) { count++; - curr = PrivateUtils.symtabTree(curr).getSymbolTable(); + curr = _Private_Utils.symtabTree(curr).getSymbolTable(); } // we should terminate when the symbol tables symbol table is the system symbol table assert(curr != null); diff --git a/src/software/amazon/ion/impl/lite/IonDecimalLite.java b/src/com/amazon/ion/impl/lite/IonDecimalLite.java similarity index 86% rename from src/software/amazon/ion/impl/lite/IonDecimalLite.java rename to src/com/amazon/ion/impl/lite/IonDecimalLite.java index 1795226b9a..4a8d71e7a1 100644 --- a/src/software/amazon/ion/impl/lite/IonDecimalLite.java +++ b/src/com/amazon/ion/impl/lite/IonDecimalLite.java @@ -1,27 +1,29 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; +package com.amazon.ion.impl.lite; +import com.amazon.ion.Decimal; +import com.amazon.ion.IonDecimal; +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.NullValueException; +import com.amazon.ion.ValueVisitor; import java.io.IOException; import java.math.BigDecimal; -import software.amazon.ion.Decimal; -import software.amazon.ion.IonDecimal; -import software.amazon.ion.IonType; -import software.amazon.ion.IonWriter; -import software.amazon.ion.NullValueException; -import software.amazon.ion.ValueVisitor; + final class IonDecimalLite extends IonValueLite diff --git a/src/software/amazon/ion/impl/lite/IonFloatLite.java b/src/com/amazon/ion/impl/lite/IonFloatLite.java similarity index 83% rename from src/software/amazon/ion/impl/lite/IonFloatLite.java rename to src/com/amazon/ion/impl/lite/IonFloatLite.java index fdb048d71a..66118c291d 100644 --- a/src/software/amazon/ion/impl/lite/IonFloatLite.java +++ b/src/com/amazon/ion/impl/lite/IonFloatLite.java @@ -1,27 +1,29 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; +package com.amazon.ion.impl.lite; +import com.amazon.ion.Decimal; +import com.amazon.ion.IonFloat; +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.NullValueException; +import com.amazon.ion.ValueVisitor; import java.io.IOException; import java.math.BigDecimal; -import software.amazon.ion.Decimal; -import software.amazon.ion.IonFloat; -import software.amazon.ion.IonType; -import software.amazon.ion.IonWriter; -import software.amazon.ion.NullValueException; -import software.amazon.ion.ValueVisitor; + final class IonFloatLite extends IonValueLite diff --git a/src/software/amazon/ion/impl/lite/IonIntLite.java b/src/com/amazon/ion/impl/lite/IonIntLite.java similarity index 90% rename from src/software/amazon/ion/impl/lite/IonIntLite.java rename to src/com/amazon/ion/impl/lite/IonIntLite.java index 8d2ce15237..3fce4810aa 100644 --- a/src/software/amazon/ion/impl/lite/IonIntLite.java +++ b/src/com/amazon/ion/impl/lite/IonIntLite.java @@ -1,28 +1,30 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; +package com.amazon.ion.impl.lite; +import com.amazon.ion.IntegerSize; +import com.amazon.ion.IonInt; +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.NullValueException; +import com.amazon.ion.ValueVisitor; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; -import software.amazon.ion.IntegerSize; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonType; -import software.amazon.ion.IonWriter; -import software.amazon.ion.NullValueException; -import software.amazon.ion.ValueVisitor; + final class IonIntLite @@ -243,7 +245,6 @@ public void accept(ValueVisitor visitor) throws Exception visitor.visit(this); } - @Override public IntegerSize getIntegerSize() { if (isNullValue()) diff --git a/src/software/amazon/ion/impl/lite/IonListLite.java b/src/com/amazon/ion/impl/lite/IonListLite.java similarity index 75% rename from src/software/amazon/ion/impl/lite/IonListLite.java rename to src/com/amazon/ion/impl/lite/IonListLite.java index 41f0a1a435..3d8ad0d0b0 100644 --- a/src/software/amazon/ion/impl/lite/IonListLite.java +++ b/src/com/amazon/ion/impl/lite/IonListLite.java @@ -1,25 +1,27 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; +package com.amazon.ion.impl.lite; +import com.amazon.ion.ContainedValueException; +import com.amazon.ion.IonList; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.ValueVisitor; import java.util.Collection; -import software.amazon.ion.ContainedValueException; -import software.amazon.ion.IonList; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.ValueVisitor; + final class IonListLite extends IonSequenceLite @@ -46,7 +48,7 @@ final class IonListLite } /** - * Constructs a list value not backed by binary. + * Constructs a list value * * @param elements * the initial set of child elements. If null, then the new diff --git a/src/software/amazon/ion/impl/lite/IonLoaderLite.java b/src/com/amazon/ion/impl/lite/IonLoaderLite.java similarity index 77% rename from src/software/amazon/ion/impl/lite/IonLoaderLite.java rename to src/com/amazon/ion/impl/lite/IonLoaderLite.java index 8dd6d5821e..62e963de4c 100644 --- a/src/software/amazon/ion/impl/lite/IonLoaderLite.java +++ b/src/com/amazon/ion/impl/lite/IonLoaderLite.java @@ -1,35 +1,37 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; +package com.amazon.ion.impl.lite; -import static software.amazon.ion.impl.PrivateIonReaderFactory.makeReader; +import static com.amazon.ion.impl._Private_IonReaderFactory.makeReader; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonException; +import com.amazon.ion.IonLoader; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonWriter; +import com.amazon.ion.impl._Private_IonWriterFactory; +import com.amazon.ion.impl._Private_LocalSymbolTableFactory; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.Reader; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonException; -import software.amazon.ion.IonLoader; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonWriter; -import software.amazon.ion.impl.PrivateIonWriterFactory; -import software.amazon.ion.impl.PrivateLocalSymbolTableFactory; + final class IonLoaderLite implements IonLoader @@ -39,7 +41,7 @@ final class IonLoaderLite /** Not null. */ private final IonCatalog _catalog; - private final PrivateLocalSymbolTableFactory _lstFactory; + private final _Private_LocalSymbolTableFactory _lstFactory; /** * @param system must not be null. @@ -52,7 +54,7 @@ public IonLoaderLite(IonSystemLite system, IonCatalog catalog) _system = system; _catalog = catalog; - _lstFactory = system.getLstFactory(); + _lstFactory = _system.getLstFactory(); } public IonSystem getSystem() @@ -75,7 +77,7 @@ private IonDatagramLite load_helper(IonReader reader) throws IOException { IonDatagramLite datagram = new IonDatagramLite(_system, _catalog); - IonWriter writer = PrivateIonWriterFactory.makeWriter(datagram); + IonWriter writer = _Private_IonWriterFactory.makeWriter(datagram); writer.writeValues(reader); return datagram; } diff --git a/src/software/amazon/ion/impl/lite/IonLobLite.java b/src/com/amazon/ion/impl/lite/IonLobLite.java similarity index 87% rename from src/software/amazon/ion/impl/lite/IonLobLite.java rename to src/com/amazon/ion/impl/lite/IonLobLite.java index 2e7c8872c4..b74033c168 100644 --- a/src/software/amazon/ion/impl/lite/IonLobLite.java +++ b/src/com/amazon/ion/impl/lite/IonLobLite.java @@ -1,23 +1,25 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; +package com.amazon.ion.impl.lite; +import com.amazon.ion.IonLob; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.util.zip.CRC32; -import software.amazon.ion.IonLob; + abstract class IonLobLite extends IonValueLite diff --git a/src/software/amazon/ion/impl/lite/IonNullLite.java b/src/com/amazon/ion/impl/lite/IonNullLite.java similarity index 72% rename from src/software/amazon/ion/impl/lite/IonNullLite.java rename to src/com/amazon/ion/impl/lite/IonNullLite.java index 34da74bfc5..ce6abefabe 100644 --- a/src/software/amazon/ion/impl/lite/IonNullLite.java +++ b/src/com/amazon/ion/impl/lite/IonNullLite.java @@ -1,24 +1,26 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; +package com.amazon.ion.impl.lite; +import com.amazon.ion.IonNull; +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.ValueVisitor; import java.io.IOException; -import software.amazon.ion.IonNull; -import software.amazon.ion.IonType; -import software.amazon.ion.IonWriter; -import software.amazon.ion.ValueVisitor; + final class IonNullLite extends IonValueLite diff --git a/src/software/amazon/ion/impl/lite/IonSequenceLite.java b/src/com/amazon/ion/impl/lite/IonSequenceLite.java similarity index 95% rename from src/software/amazon/ion/impl/lite/IonSequenceLite.java rename to src/com/amazon/ion/impl/lite/IonSequenceLite.java index 9fb5db65be..5214b79ec3 100644 --- a/src/software/amazon/ion/impl/lite/IonSequenceLite.java +++ b/src/com/amazon/ion/impl/lite/IonSequenceLite.java @@ -1,19 +1,28 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; +package com.amazon.ion.impl.lite; +import com.amazon.ion.ContainedValueException; +import com.amazon.ion.IonSequence; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.ValueFactory; +import com.amazon.ion.impl._Private_CurriedValueFactory; +import com.amazon.ion.impl._Private_IonValue; import java.io.IOException; import java.lang.reflect.Array; import java.util.ArrayList; @@ -24,14 +33,7 @@ import java.util.List; import java.util.ListIterator; import java.util.Map; -import software.amazon.ion.ContainedValueException; -import software.amazon.ion.IonSequence; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.IonWriter; -import software.amazon.ion.ValueFactory; -import software.amazon.ion.impl.PrivateCurriedValueFactory; -import software.amazon.ion.impl.PrivateIonValue; + abstract class IonSequenceLite extends IonContainerLite @@ -163,7 +165,7 @@ public boolean addAll(int index, Collection c) public ValueFactory add() { - return new PrivateCurriedValueFactory(this.getSystem()) + return new _Private_CurriedValueFactory(this.getSystem()) { @Override protected void handle(IonValue newValue) @@ -182,7 +184,7 @@ public void add(int index, IonValue element) public ValueFactory add(final int index) { - return new PrivateCurriedValueFactory(getSystem()) + return new _Private_CurriedValueFactory(getSystem()) { @Override protected void handle(IonValue newValue) @@ -326,7 +328,7 @@ public int indexOf(Object o) if (o == null) { throw new NullPointerException(); } - PrivateIonValue v = (PrivateIonValue) o; + _Private_IonValue v = (_Private_IonValue) o; if (this != v.getContainer()) return -1; return v.getElementId(); } diff --git a/src/software/amazon/ion/impl/lite/IonSexpLite.java b/src/com/amazon/ion/impl/lite/IonSexpLite.java similarity index 73% rename from src/software/amazon/ion/impl/lite/IonSexpLite.java rename to src/com/amazon/ion/impl/lite/IonSexpLite.java index d1a61fe56d..42a5625865 100644 --- a/src/software/amazon/ion/impl/lite/IonSexpLite.java +++ b/src/com/amazon/ion/impl/lite/IonSexpLite.java @@ -1,25 +1,27 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; +package com.amazon.ion.impl.lite; +import com.amazon.ion.ContainedValueException; +import com.amazon.ion.IonSexp; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.ValueVisitor; import java.util.Collection; -import software.amazon.ion.ContainedValueException; -import software.amazon.ion.IonSexp; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.ValueVisitor; + final class IonSexpLite extends IonSequenceLite @@ -39,7 +41,7 @@ final class IonSexpLite } /** - * Constructs a sexp value not backed by binary. + * Constructs a sexp value * * @param elements * the initial set of child elements. If null, then the new diff --git a/src/software/amazon/ion/impl/lite/IonStringLite.java b/src/com/amazon/ion/impl/lite/IonStringLite.java similarity index 75% rename from src/software/amazon/ion/impl/lite/IonStringLite.java rename to src/com/amazon/ion/impl/lite/IonStringLite.java index 87083af9be..e7bd0e945b 100644 --- a/src/software/amazon/ion/impl/lite/IonStringLite.java +++ b/src/com/amazon/ion/impl/lite/IonStringLite.java @@ -1,24 +1,26 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; +package com.amazon.ion.impl.lite; +import com.amazon.ion.IonString; +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.ValueVisitor; import java.io.IOException; -import software.amazon.ion.IonString; -import software.amazon.ion.IonType; -import software.amazon.ion.IonWriter; -import software.amazon.ion.ValueVisitor; + final class IonStringLite extends IonTextLite diff --git a/src/software/amazon/ion/impl/lite/IonStructLite.java b/src/com/amazon/ion/impl/lite/IonStructLite.java similarity index 95% rename from src/software/amazon/ion/impl/lite/IonStructLite.java rename to src/com/amazon/ion/impl/lite/IonStructLite.java index 8d7cb0d814..2f0afd334c 100644 --- a/src/software/amazon/ion/impl/lite/IonStructLite.java +++ b/src/com/amazon/ion/impl/lite/IonStructLite.java @@ -1,19 +1,31 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; - +package com.amazon.ion.impl.lite; + +import com.amazon.ion.ContainedValueException; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.ValueFactory; +import com.amazon.ion.ValueVisitor; +import com.amazon.ion.impl._Private_CurriedValueFactory; +import com.amazon.ion.util.Equivalence; +import com.amazon.ion.UnknownSymbolException; import java.io.IOException; import java.io.PrintWriter; import java.util.Arrays; @@ -23,18 +35,8 @@ import java.util.ListIterator; import java.util.Map; import java.util.Map.Entry; -import software.amazon.ion.ContainedValueException; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.ValueFactory; -import software.amazon.ion.ValueVisitor; -import software.amazon.ion.impl.PrivateCurriedValueFactory; -import software.amazon.ion.util.Equivalence; import java.util.Set; -import software.amazon.ion.UnknownSymbolException; + final class IonStructLite extends IonContainerLite @@ -42,7 +44,7 @@ final class IonStructLite { private static final int HASH_SIGNATURE = IonType.STRUCT.toString().hashCode(); - // TODO amzn/ion-java#41: add support for _isOrdered + // TODO amzn/ion-java/issues/41: add support for _isOrdered IonStructLite(ContainerlessContext context, boolean isNull) { @@ -458,7 +460,7 @@ public boolean add(IonValue child) public ValueFactory add(final String fieldName) { - return new PrivateCurriedValueFactory(_context.getSystem()) + return new _Private_CurriedValueFactory(_context.getSystem()) { @Override protected void handle(IonValue newValue) @@ -529,7 +531,7 @@ public void add(SymbolToken fieldName, IonValue child) public ValueFactory put(final String fieldName) { - return new PrivateCurriedValueFactory(_context.getSystem()) + return new _Private_CurriedValueFactory(_context.getSystem()) { @Override protected void handle(IonValue newValue) diff --git a/src/software/amazon/ion/impl/lite/IonSymbolLite.java b/src/com/amazon/ion/impl/lite/IonSymbolLite.java similarity index 77% rename from src/software/amazon/ion/impl/lite/IonSymbolLite.java rename to src/com/amazon/ion/impl/lite/IonSymbolLite.java index 8f13f8eb0a..88ba206756 100644 --- a/src/software/amazon/ion/impl/lite/IonSymbolLite.java +++ b/src/com/amazon/ion/impl/lite/IonSymbolLite.java @@ -1,36 +1,39 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; - -import software.amazon.ion.IonType; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; -import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; -import software.amazon.ion.SymbolToken; -import static software.amazon.ion.SystemSymbols.ION_1_0; -import static software.amazon.ion.SystemSymbols.ION_1_0_SID; - +package com.amazon.ion.impl.lite; + +import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; +import static com.amazon.ion.SystemSymbols.ION_1_0; +import static com.amazon.ion.SystemSymbols.ION_1_0_SID; + +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.NullValueException; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.UnknownSymbolException; +import com.amazon.ion.ValueVisitor; +import com.amazon.ion.impl._Private_IonSymbol; +import com.amazon.ion.impl._Private_Utils; import java.io.IOException; -import software.amazon.ion.UnknownSymbolException; -import software.amazon.ion.ValueVisitor; -import software.amazon.ion.impl.PrivateIonSymbol; -import software.amazon.ion.impl.PrivateUtils; + final class IonSymbolLite extends IonTextLite - implements PrivateIonSymbol + implements _Private_IonSymbol { private static final int HASH_SIGNATURE = IonType.SYMBOL.toString().hashCode(); @@ -62,11 +65,11 @@ final class IonSymbolLite if (text != null) { super.setValue(text); - // TODO [ION-320] - needs consistent handling, when to retain SID's vs ignore + // TODO [amzn/ion-java/issues/27] - needs consistent handling, when to retain SID's vs ignore } else { - // TODO [IONJAVA-579] - needs consistent handling, resolution against context symbol table + // TODO [amzn/ion-java/issues/223] - needs consistent handling, resolution against context symbol table _sid = sid; // there *is* an encoding present so we must update _isSymbolIdPresent(true); @@ -128,51 +131,16 @@ public IonType getType() return IonType.SYMBOL; } - - /** - * Get's the text of this NON-NULL symbol, finding it from our symbol - * table if it's not yet known (and caching the result if possible). - *

    - * Caller must check {@link #isNullValue()} - * - * @return null if symbol text is unknown. - */ - private String _stringValue() + @Deprecated + public int getSymbolId() + throws NullValueException { - return _stringValue(new LazySymbolTableProvider(this)); + return getSymbolId(null); } - private String _stringValue(SymbolTableProvider symbolTableProvider) + private int getSymbolId(SymbolTableProvider symbolTableProvider) + throws NullValueException { - String name = _get_value(); - if (name == null) - { - assert _sid >= 0; - - SymbolTable symbols = symbolTableProvider.getSymbolTable(); - name = symbols.findKnownSymbol(_sid); - if (name != null) - { - // if this is a mutable value we'll hang onto - // our now known symbol table so we don't have - // to look it up again. - // If the value is immutable, honor that contract. - if (_isLocked() == false) { - _set_value(name); - } - } - } - return name; - } - - public SymbolToken symbolValue() - { - return symbolValue(new LazySymbolTableProvider(this)); - } - - private int resolveSymbolId(SymbolTableProvider symbolTableProvider) - { - validateThisNotNull(); if (_sid != UNKNOWN_SYMBOL_ID || isReadOnly()) { @@ -180,15 +148,15 @@ private int resolveSymbolId(SymbolTableProvider symbolTableProvider) } SymbolTable symtab = - symbolTableProvider != null ? symbolTableProvider.getSymbolTable() - : getSymbolTable(); + symbolTableProvider != null ? symbolTableProvider.getSymbolTable() + : getSymbolTable(); if (symtab == null) { symtab = getSystem().getSystemSymbolTable(); } assert(symtab != null); String name = _get_value(); - // TODO [ION-320] - needs consistent handling, when to retain SID's vs ignore (here memoizing SID on read) + // TODO [amzn/ion-java/issues/27] - needs consistent handling, when to retain SID's vs ignore (here memoizing SID on read) if (!symtab.isLocalTable()) { setSID(symtab.findSymbol(name)); @@ -214,13 +182,55 @@ private void setSID(int sid) } } + + /** + * Get's the text of this NON-NULL symbol, finding it from our symbol + * table if it's not yet known (and caching the result if possible). + *

    + * Caller must check {@link #isNullValue()} + * + * @return null if symbol text is unknown. + */ + private String _stringValue() + { + return _stringValue(new LazySymbolTableProvider(this)); + } + + private String _stringValue(SymbolTableProvider symbolTableProvider) + { + String name = _get_value(); + if (name == null) + { + assert _sid >= 0; + + SymbolTable symbols = symbolTableProvider.getSymbolTable(); + name = symbols.findKnownSymbol(_sid); + if (name != null) + { + // if this is a mutable value we'll hang onto + // our now known symbol table so we don't have + // to look it up again. + // If the value is immutable, honor that contract. + if (_isLocked() == false) { + _set_value(name); + } + } + } + return name; + } + + public SymbolToken symbolValue() + { + return symbolValue(new LazySymbolTableProvider(this)); + } + public SymbolToken symbolValue(SymbolTableProvider symbolTableProvider) { if (isNullValue()) return null; - int sid = resolveSymbolId(symbolTableProvider); + int sid = getSymbolId(symbolTableProvider); String text = _stringValue(symbolTableProvider); - return PrivateUtils.newSymbolToken(text, sid); + return _Private_Utils.newSymbolToken(text, sid); } @@ -252,7 +262,7 @@ else if (_stringValue() != null) } else { - // TODO [IONJAVA-579] - needs consistent handling, resolution against context symbol table + // TODO [amzn/ion-java/issues/223] - needs consistent handling, resolution against context symbol table // there is not text, so we can't clear the SID. allSymbolIDsCleared = false; } @@ -271,16 +281,17 @@ protected void setIsIonVersionMarker(boolean isIVM) _sid = ION_1_0_SID; } + @Override final void writeBodyTo(IonWriter writer, SymbolTableProvider symbolTableProvider) throws IOException { - // TODO amzn/ion-java#27 Fix symbol handling + // TODO amzn/ion-java/issues/27 Fix symbol handling // A million-dollar question is - if text is missing, do // we throw (cannot serialize) or do we pass the sid thru??? - SymbolToken symbol = symbolValue(symbolTableProvider); - writer.writeSymbolToken(symbol); + // NB! This will throw if symbol is not set + writer.writeSymbolToken(symbolValue(symbolTableProvider)); } @Override @@ -297,7 +308,7 @@ private String stringValue(SymbolTableProvider symbolTableProvider) return null; } String name = _stringValue(symbolTableProvider); - if (name == null) { + if (name == null && _sid != 0) { assert(_sid > 0); throw new UnknownSymbolException(_sid); } diff --git a/src/software/amazon/ion/impl/lite/IonSystemLite.java b/src/com/amazon/ion/impl/lite/IonSystemLite.java similarity index 85% rename from src/software/amazon/ion/impl/lite/IonSystemLite.java rename to src/com/amazon/ion/impl/lite/IonSystemLite.java index db49fd6688..1e16f7392b 100644 --- a/src/software/amazon/ion/impl/lite/IonSystemLite.java +++ b/src/com/amazon/ion/impl/lite/IonSystemLite.java @@ -1,29 +1,54 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; - -import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; -import static software.amazon.ion.SystemSymbols.ION_1_0; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; -import static software.amazon.ion.impl.PrivateIonReaderFactory.makeReader; -import static software.amazon.ion.impl.PrivateIonReaderFactory.makeSystemReader; -import static software.amazon.ion.impl.PrivateUtils.addAllNonNull; -import static software.amazon.ion.impl.PrivateUtils.initialSymtab; -import static software.amazon.ion.impl.PrivateUtils.newSymbolToken; -import static software.amazon.ion.util.IonTextUtils.printString; - +package com.amazon.ion.impl.lite; + +import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; +import static com.amazon.ion.SystemSymbols.ION_1_0; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; +import static com.amazon.ion.impl._Private_IonReaderFactory.makeReader; +import static com.amazon.ion.impl._Private_IonReaderFactory.makeSystemReader; +import static com.amazon.ion.impl._Private_Utils.addAllNonNull; +import static com.amazon.ion.impl._Private_Utils.initialSymtab; +import static com.amazon.ion.impl._Private_Utils.newSymbolToken; +import static com.amazon.ion.util.IonTextUtils.printString; + +import com.amazon.ion.IonBinaryWriter; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonContainer; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonException; +import com.amazon.ion.IonLoader; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonTextReader; +import com.amazon.ion.IonTimestamp; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.UnexpectedEofException; +import com.amazon.ion.UnsupportedIonVersionException; +import com.amazon.ion.impl._Private_IonBinaryWriterBuilder; +import com.amazon.ion.impl._Private_IonSystem; +import com.amazon.ion.impl._Private_IonWriterFactory; +import com.amazon.ion.impl._Private_ScalarConversions.CantConvertException; +import com.amazon.ion.impl._Private_Utils; +import com.amazon.ion.system.IonReaderBuilder; +import com.amazon.ion.system.IonTextWriterBuilder; import java.io.Closeable; import java.io.IOException; import java.io.InputStream; @@ -33,31 +58,12 @@ import java.util.Date; import java.util.Iterator; import java.util.NoSuchElementException; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonContainer; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonException; -import software.amazon.ion.IonLoader; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonTimestamp; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.UnexpectedEofException; -import software.amazon.ion.UnsupportedIonVersionException; -import software.amazon.ion.impl.PrivateIonBinaryWriterBuilder; -import software.amazon.ion.impl.PrivateIonSystem; -import software.amazon.ion.impl.PrivateIonWriterFactory; -import software.amazon.ion.impl.PrivateUtils; -import software.amazon.ion.system.IonReaderBuilder; -import software.amazon.ion.system.IonTextWriterBuilder; + +@SuppressWarnings("deprecation") final class IonSystemLite extends ValueFactoryLite - implements PrivateIonSystem + implements _Private_IonSystem { private final SymbolTable _system_symbol_table; @@ -68,13 +74,14 @@ final class IonSystemLite /** Immutable. */ private final IonTextWriterBuilder myTextWriterBuilder; /** Immutable. */ - private final PrivateIonBinaryWriterBuilder myBinaryWriterBuilder; + + private final _Private_IonBinaryWriterBuilder myBinaryWriterBuilder; /** Immutable. **/ private final IonReaderBuilder myReaderBuilder; public IonSystemLite(IonTextWriterBuilder twb, - PrivateIonBinaryWriterBuilder bwb, - IonReaderBuilder rb) + _Private_IonBinaryWriterBuilder bwb, + IonReaderBuilder rb) { IonCatalog catalog = twb.getCatalog(); assert catalog != null; @@ -117,7 +124,7 @@ public T clone(T value) throws IonException if (value instanceof IonDatagram) { IonDatagram datagram = newDatagram(); - IonWriter writer = PrivateIonWriterFactory.makeWriter(datagram); + IonWriter writer = _Private_IonWriterFactory.makeWriter(datagram); IonReader reader = makeSystemReader(value.getSystem(), value); try { @@ -204,6 +211,22 @@ public Iterator iterate(IonReader reader) return iterator; } + @Deprecated + public IonBinaryWriter newBinaryWriter() + { + _Private_IonBinaryWriterBuilder b = myBinaryWriterBuilder; + return b.buildLegacy(); + } + + @Deprecated + public IonBinaryWriter newBinaryWriter(SymbolTable... imports) + { + _Private_IonBinaryWriterBuilder b = (_Private_IonBinaryWriterBuilder) + myBinaryWriterBuilder.withImports(imports); + return b.buildLegacy(); + } + + public IonWriter newBinaryWriter(OutputStream out, SymbolTable... imports) { return myBinaryWriterBuilder.withImports(imports).build(out); @@ -239,18 +262,18 @@ public SymbolTable newLocalSymbolTable(SymbolTable... imports) public SymbolTable newSharedSymbolTable(IonStruct ionRep) { - return PrivateUtils.newSharedSymtab(ionRep); + return _Private_Utils.newSharedSymtab(ionRep); } public SymbolTable newSharedSymbolTable(IonReader reader) { - return PrivateUtils.newSharedSymtab(reader, false); + return _Private_Utils.newSharedSymtab(reader, false); } public SymbolTable newSharedSymbolTable(IonReader reader, boolean isOnStruct) { - return PrivateUtils.newSharedSymtab(reader, isOnStruct); + return _Private_Utils.newSharedSymtab(reader, isOnStruct); } public SymbolTable newSharedSymbolTable(String name, @@ -284,7 +307,7 @@ public SymbolTable newSharedSymbolTable(String name, addAllNonNull(syms, newSymbols); SymbolTable st = - PrivateUtils.newSharedSymtab(name, version, prior, + _Private_Utils.newSharedSymtab(name, version, prior, syms.iterator()); return st; } @@ -316,7 +339,7 @@ private IonValueLite load_value_helper(IonReader reader, boolean isTopLevel) v = newBool(reader.booleanValue()); break; case INT: - // TODO amzn/ion-java#9 Inefficient since we can't determine the size + // TODO amzn/ion-java/issues/9 Inefficient since we can't determine the size // of the integer in order to avoid making BigIntegers. v = newInt(reader.bigIntegerValue()); break; @@ -473,7 +496,7 @@ IonValueLite newValue(IonType valueType) public IonWriter newWriter(IonContainer container) { - IonWriter writer = PrivateIonWriterFactory.makeWriter(container); + IonWriter writer = _Private_IonWriterFactory.makeWriter(container); return writer; } @@ -692,7 +715,7 @@ public IonReader newSystemReader(byte[] ionData, int offset, int len) } - public IonReader newReader(String ionText) + public IonTextReader newReader(String ionText) { return myReaderBuilder.build(ionText); } @@ -713,23 +736,25 @@ public IonReader newSystemReader(InputStream ionData) return makeSystemReader(ionData); } + + //========================================================================== + // methods in IonSystemImpl (now declared in IonSystemPrivate) + //========================================================================== + public IonReader newReader(Reader ionText) { return myReaderBuilder.build(ionText); } - public IonReader newReader(IonValue value) + public IonReader newSystemReader(Reader ionText) { - return myReaderBuilder.build(value); + return makeSystemReader(ionText); } - //========================================================================== - // methods in _Private_IonSystem - //========================================================================== - public IonReader newSystemReader(Reader ionText) + public IonReader newReader(IonValue value) { - return makeSystemReader(ionText); + return myReaderBuilder.build(value); } public IonReader newSystemReader(IonValue value) @@ -747,7 +772,7 @@ public IonReader newSystemReader(IonValue value) */ public IonWriter newTreeSystemWriter(IonContainer container) { - IonWriter writer = PrivateIonWriterFactory.makeSystemWriter(container); + IonWriter writer = _Private_IonWriterFactory.makeSystemWriter(container); return writer; } @@ -756,7 +781,7 @@ public IonWriter newTreeSystemWriter(IonContainer container) */ public IonWriter newTreeWriter(IonContainer container) { - IonWriter writer = PrivateIonWriterFactory.makeWriter(container); + IonWriter writer = _Private_IonWriterFactory.makeWriter(container); return writer; } @@ -764,18 +789,18 @@ public IonWriter newTreeWriter(IonContainer container) public Iterator systemIterate(Reader ionText) { IonReader ir = newSystemReader(ionText); - return PrivateUtils.iterate(this, ir); + return _Private_Utils.iterate(this, ir); } public Iterator systemIterate(String ionText) { IonReader ir = newSystemReader(ionText); - return PrivateUtils.iterate(this, ir); + return _Private_Utils.iterate(this, ir); } public Iterator systemIterate(IonReader reader) { - return PrivateUtils.iterate(this, reader); + return _Private_Utils.iterate(this, reader); } @@ -788,4 +813,5 @@ public boolean valueIsSharedSymbolTable(IonValue value) } return false; } + } diff --git a/src/software/amazon/ion/impl/lite/IonTextLite.java b/src/com/amazon/ion/impl/lite/IonTextLite.java similarity index 74% rename from src/software/amazon/ion/impl/lite/IonTextLite.java rename to src/com/amazon/ion/impl/lite/IonTextLite.java index 0383ffd969..8001300c9f 100644 --- a/src/software/amazon/ion/impl/lite/IonTextLite.java +++ b/src/com/amazon/ion/impl/lite/IonTextLite.java @@ -1,20 +1,22 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; +package com.amazon.ion.impl.lite; + +import com.amazon.ion.IonText; -import software.amazon.ion.IonText; abstract class IonTextLite extends IonValueLite diff --git a/src/software/amazon/ion/impl/lite/IonTimestampLite.java b/src/com/amazon/ion/impl/lite/IonTimestampLite.java similarity index 87% rename from src/software/amazon/ion/impl/lite/IonTimestampLite.java rename to src/com/amazon/ion/impl/lite/IonTimestampLite.java index fc3bddd891..7696d10f4b 100644 --- a/src/software/amazon/ion/impl/lite/IonTimestampLite.java +++ b/src/com/amazon/ion/impl/lite/IonTimestampLite.java @@ -1,28 +1,30 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; +package com.amazon.ion.impl.lite; +import com.amazon.ion.IonTimestamp; +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.NullValueException; +import com.amazon.ion.Timestamp; +import com.amazon.ion.ValueVisitor; import java.io.IOException; import java.math.BigDecimal; import java.util.Date; -import software.amazon.ion.IonTimestamp; -import software.amazon.ion.IonType; -import software.amazon.ion.IonWriter; -import software.amazon.ion.NullValueException; -import software.amazon.ion.Timestamp; -import software.amazon.ion.ValueVisitor; + final class IonTimestampLite extends IonValueLite @@ -132,12 +134,12 @@ public void setValue(Timestamp timestamp) public void setValue(BigDecimal millis, Integer localOffset) { - setValue(Timestamp.forMillis(millis, localOffset)); + setValue(new Timestamp(millis, localOffset)); } public void setValue(long millis, Integer localOffset) { - setValue(Timestamp.forMillis(millis, localOffset)); + setValue(new Timestamp(millis, localOffset)); } public void setTime(Date value) diff --git a/src/software/amazon/ion/impl/lite/IonValueLite.java b/src/com/amazon/ion/impl/lite/IonValueLite.java similarity index 92% rename from src/software/amazon/ion/impl/lite/IonValueLite.java rename to src/com/amazon/ion/impl/lite/IonValueLite.java index c0245cbcf9..8c58a0dd7b 100644 --- a/src/software/amazon/ion/impl/lite/IonValueLite.java +++ b/src/com/amazon/ion/impl/lite/IonValueLite.java @@ -1,61 +1,54 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; - -import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; -import static software.amazon.ion.impl.PrivateUtils.EMPTY_STRING_ARRAY; -import static software.amazon.ion.impl.PrivateUtils.newSymbolToken; -import static software.amazon.ion.util.Equivalence.ionEquals; - +package com.amazon.ion.impl.lite; + +import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; +import static com.amazon.ion.impl._Private_Utils.EMPTY_STRING_ARRAY; +import static com.amazon.ion.impl._Private_Utils.newSymbolToken; +import static com.amazon.ion.util.Equivalence.ionEquals; + +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonException; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.NullValueException; +import com.amazon.ion.ReadOnlyValueException; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.UnknownSymbolException; +import com.amazon.ion.ValueVisitor; +import com.amazon.ion.impl._Private_IonValue; +import com.amazon.ion.impl._Private_IonWriter; +import com.amazon.ion.impl._Private_Utils; +import com.amazon.ion.system.IonTextWriterBuilder; +import com.amazon.ion.util.Printer; import java.io.IOException; import java.io.PrintWriter; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonException; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.IonWriter; -import software.amazon.ion.NullValueException; -import software.amazon.ion.ReadOnlyValueException; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.UnknownSymbolException; -import software.amazon.ion.ValueVisitor; -import software.amazon.ion.impl.PrivateIonValue; -import software.amazon.ion.impl.PrivateIonWriter; -import software.amazon.ion.impl.PrivateUtils; -import software.amazon.ion.system.IonTextWriterBuilder; /** * Base class of the light weight implementation of * Ion values. - * - * This implementation is not backed by a buffer - * and is therefore fully materialized. If you need - * only a few values from a large datagram this - * implementation may be more expensive than the - * original implementation. */ abstract class IonValueLite - implements PrivateIonValue + implements _Private_IonValue { private static final int TYPE_ANNOTATION_HASH_SIGNATURE = "TYPE ANNOTATION".hashCode(); - private static final IonTextWriterBuilder TO_STRING_TEXT_WRITER_BUILDER = - IonTextWriterBuilder.standard().withCharsetAscii().immutable(); - /** * this hold all the various boolean flags we have * in a single int. Use set_flag(), clear_flag(), is_true() @@ -76,8 +69,7 @@ abstract class IonValueLite * (e.g. if it is a {@link IonContainerLite} based sub type) contains one or more defined * Symbol ID's (SID)'s. This flag is used to track lifecycle, such that operations that require SID's are purged * from the IonValueLite and it's contained sub-DOM can conduct a fast evaluation rather than having to do a full - * contained graph traversal on each and every invocation. For more detail on impact see - * IONPT-8 + * contained graph traversal on each and every invocation. */ protected static final int IS_SYMBOL_ID_PRESENT = 0x80; @@ -316,9 +308,9 @@ public SymbolTable getSymbolTable() String text = existingToken.getText(); if (text != null) { this._annotations[i] = - PrivateUtils.newSymbolToken(text, UNKNOWN_SYMBOL_ID); + _Private_Utils.newSymbolToken(text, UNKNOWN_SYMBOL_ID); } else { - // TODO - IONJAVA-579 needs consistent handling, should attempt to resolve and if it cant; fail + // TODO - amzn/ion-java/issues/223 needs consistent handling, should attempt to resolve and if it cant; fail this._annotations[i] = existing._annotations[i]; hasSIDsRetained |= this._annotations[i].getSid() > UNKNOWN_SYMBOL_ID; } @@ -333,7 +325,7 @@ public SymbolTable getSymbolTable() // existing 'read only' flag - we force the deep-copy back to being mutable clear_flag(IS_LOCKED); // whilst the clone *should* guarantee symbol context is purged, the annotation behavior existing above - // under the TO DO for IONJAVA-579 does mean that SID context can be propogated through a clone, therefore + // under the TO DO for amzn/ion-java/issues/223 does mean that SID context can be propogated through a clone, therefore // the encoding flag has to reflect this reality _isSymbolIdPresent(hasSIDsRetained); } @@ -459,9 +451,23 @@ public final int getElementId() return this._elementid(); } + + public final int getFieldId() + { + if (_fieldId != UNKNOWN_SYMBOL_ID || _fieldName == null) + { + return _fieldId; + } + + SymbolToken tok = getSymbolTable().find(_fieldName); + + return (tok != null ? tok.getSid() : UNKNOWN_SYMBOL_ID); + } + + public SymbolToken getFieldNameSymbol() { - // TODO amzn/ion-java#27 We should memoize the results of symtab lookups. + // TODO amzn/ion-java/issues/27 We should memoize the results of symtab lookups. // BUT: that could cause thread-safety problems for read-only values. // I think makeReadOnly should populate the tokens fully // so that we only need to lookup from mutable instances. @@ -493,7 +499,7 @@ else if(sid != 0){ // not a struct field return null; } - return PrivateUtils.newSymbolToken(text, sid); + return _Private_Utils.newSymbolToken(text, sid); } public final SymbolToken getKnownFieldNameSymbol() @@ -607,10 +613,15 @@ public final String getFieldName() if (_fieldName != null) return _fieldName; if (_fieldId <= 0) return null; - // TODO amzn/ion-java#27 why no symtab lookup, like getFieldNameSymbol()? + // TODO amzn/ion-java/issues/27 why no symtab lookup, like getFieldNameSymbol()? throw new UnknownSymbolException(_fieldId); } + public final int getFieldNameId() + { + return getFieldId(); + } + /** * @return not null, in conflict with the public documentation. */ @@ -672,7 +683,7 @@ public final SymbolToken[] getTypeAnnotationSymbols(SymbolTableProvider symbolTa String text = token.getText(); if (text != null && token.getSid() == UNKNOWN_SYMBOL_ID) { - // TODO amzn/ion-java#27 We should memoize the result of symtab lookups + // TODO amzn/ion-java/issues/27 We should memoize the result of symtab lookups // into _annotations. // See getFieldNameSymbol() for challenges doing so. @@ -735,14 +746,14 @@ public final String[] getTypeAnnotations() return EMPTY_STRING_ARRAY; } - return PrivateUtils.toStrings(_annotations, count); + return _Private_Utils.toStrings(_annotations, count); } public void setTypeAnnotations(String... annotations) { checkForLock(); - _annotations = PrivateUtils.newSymbolTokens(getSymbolTable(), + _annotations = _Private_Utils.newSymbolTokens(getSymbolTable(), annotations); } @@ -757,7 +768,6 @@ public final boolean hasTypeAnnotation(String annotation) return false; } - @Override public final int findTypeAnnotation(String annotation) { assert(annotation != null && annotation.length() > 0); @@ -815,7 +825,7 @@ protected int hashTypeAnnotations(final int original, SymbolTableProvider symbol * Implements equality over values. * This is currently defined using the Equivalence class. * - * @see software.amazon.ion.util.Equivalence + * @see com.amazon.ion.util.Equivalence * * @param other The value to compare with. * @@ -913,7 +923,17 @@ public void removeTypeAnnotation(String annotation) @Override public String toString() { - return toString(TO_STRING_TEXT_WRITER_BUILDER); + StringBuilder buf = new StringBuilder(1024); + try + { + Printer p = new Printer(); + p.print(this, buf); + } + catch (IOException e) + { + throw new IonException(e); + } + return buf.toString(); } public String toString(IonTextWriterBuilder writerBuilder) @@ -971,7 +991,7 @@ final void writeChildren(IonWriter writer, Iterable container, final void writeTo(IonWriter writer, SymbolTableProvider symbolTableProvider) { if (writer.isInStruct() - && ! ((PrivateIonWriter) writer).isFieldNameSet()) + && ! ((_Private_IonWriter) writer).isFieldNameSet()) { SymbolToken tok = getFieldNameSymbol(symbolTableProvider); if (tok == null) @@ -1176,4 +1196,3 @@ public String validate() * private SymbolTable _symbolTable; * */ - diff --git a/src/software/amazon/ion/impl/lite/ReverseBinaryEncoder.java b/src/com/amazon/ion/impl/lite/ReverseBinaryEncoder.java similarity index 90% rename from src/software/amazon/ion/impl/lite/ReverseBinaryEncoder.java rename to src/com/amazon/ion/impl/lite/ReverseBinaryEncoder.java index 439128c552..250ca1e0f9 100644 --- a/src/software/amazon/ion/impl/lite/ReverseBinaryEncoder.java +++ b/src/com/amazon/ion/impl/lite/ReverseBinaryEncoder.java @@ -1,74 +1,75 @@ /* - * Copyright 2013-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; - -import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; -import static software.amazon.ion.SystemSymbols.IMPORTS_SID; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; -import static software.amazon.ion.SystemSymbols.MAX_ID_SID; -import static software.amazon.ion.SystemSymbols.NAME_SID; -import static software.amazon.ion.SystemSymbols.SYMBOLS_SID; -import static software.amazon.ion.SystemSymbols.VERSION_SID; -import static software.amazon.ion.impl.PrivateIonConstants.BINARY_VERSION_MARKER_1_0; -import static software.amazon.ion.impl.PrivateIonConstants.lnBooleanFalse; -import static software.amazon.ion.impl.PrivateIonConstants.lnBooleanTrue; -import static software.amazon.ion.impl.PrivateIonConstants.lnIsNull; -import static software.amazon.ion.impl.PrivateIonConstants.lnIsVarLen; -import static software.amazon.ion.impl.PrivateIonConstants.tidBlob; -import static software.amazon.ion.impl.PrivateIonConstants.tidBoolean; -import static software.amazon.ion.impl.PrivateIonConstants.tidClob; -import static software.amazon.ion.impl.PrivateIonConstants.tidDecimal; -import static software.amazon.ion.impl.PrivateIonConstants.tidFloat; -import static software.amazon.ion.impl.PrivateIonConstants.tidList; -import static software.amazon.ion.impl.PrivateIonConstants.tidNegInt; -import static software.amazon.ion.impl.PrivateIonConstants.tidNull; -import static software.amazon.ion.impl.PrivateIonConstants.tidPosInt; -import static software.amazon.ion.impl.PrivateIonConstants.tidSexp; -import static software.amazon.ion.impl.PrivateIonConstants.tidString; -import static software.amazon.ion.impl.PrivateIonConstants.tidStruct; -import static software.amazon.ion.impl.PrivateIonConstants.tidSymbol; -import static software.amazon.ion.impl.PrivateIonConstants.tidTimestamp; -import static software.amazon.ion.impl.PrivateIonConstants.tidTypedecl; - +package com.amazon.ion.impl.lite; + +import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; +import static com.amazon.ion.SystemSymbols.IMPORTS_SID; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; +import static com.amazon.ion.SystemSymbols.MAX_ID_SID; +import static com.amazon.ion.SystemSymbols.NAME_SID; +import static com.amazon.ion.SystemSymbols.SYMBOLS_SID; +import static com.amazon.ion.SystemSymbols.VERSION_SID; +import static com.amazon.ion.impl._Private_IonConstants.BINARY_VERSION_MARKER_1_0; +import static com.amazon.ion.impl._Private_IonConstants.lnBooleanFalse; +import static com.amazon.ion.impl._Private_IonConstants.lnBooleanTrue; +import static com.amazon.ion.impl._Private_IonConstants.lnIsNull; +import static com.amazon.ion.impl._Private_IonConstants.lnIsVarLen; +import static com.amazon.ion.impl._Private_IonConstants.tidBlob; +import static com.amazon.ion.impl._Private_IonConstants.tidBoolean; +import static com.amazon.ion.impl._Private_IonConstants.tidClob; +import static com.amazon.ion.impl._Private_IonConstants.tidDecimal; +import static com.amazon.ion.impl._Private_IonConstants.tidFloat; +import static com.amazon.ion.impl._Private_IonConstants.tidList; +import static com.amazon.ion.impl._Private_IonConstants.tidNegInt; +import static com.amazon.ion.impl._Private_IonConstants.tidNull; +import static com.amazon.ion.impl._Private_IonConstants.tidPosInt; +import static com.amazon.ion.impl._Private_IonConstants.tidSexp; +import static com.amazon.ion.impl._Private_IonConstants.tidString; +import static com.amazon.ion.impl._Private_IonConstants.tidStruct; +import static com.amazon.ion.impl._Private_IonConstants.tidSymbol; +import static com.amazon.ion.impl._Private_IonConstants.tidTimestamp; +import static com.amazon.ion.impl._Private_IonConstants.tidTypedecl; + +import com.amazon.ion.Decimal; +import com.amazon.ion.IonBlob; +import com.amazon.ion.IonBool; +import com.amazon.ion.IonClob; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonDecimal; +import com.amazon.ion.IonException; +import com.amazon.ion.IonFloat; +import com.amazon.ion.IonInt; +import com.amazon.ion.IonList; +import com.amazon.ion.IonSequence; +import com.amazon.ion.IonSexp; +import com.amazon.ion.IonString; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSymbol; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonTimestamp; +import com.amazon.ion.IonValue; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.Timestamp; import java.io.IOException; import java.io.OutputStream; import java.math.BigDecimal; import java.math.BigInteger; import java.util.ArrayList; import java.util.ListIterator; -import software.amazon.ion.Decimal; -import software.amazon.ion.IonBlob; -import software.amazon.ion.IonBool; -import software.amazon.ion.IonClob; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonDecimal; -import software.amazon.ion.IonException; -import software.amazon.ion.IonFloat; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonList; -import software.amazon.ion.IonSequence; -import software.amazon.ion.IonSexp; -import software.amazon.ion.IonString; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonTimestamp; -import software.amazon.ion.IonValue; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.Timestamp; /** @@ -167,6 +168,55 @@ byte[] toNewByteArray() return bytes; } + /** + * Copies the current contents of the Ion binary-encoded byte array into a + * a given byte array. + *

    + * The given array must be large enough to contain all the bytes of the + * Ion binary-encoded byte array. + *

    + * This makes an unchecked assumption that {{@link #serialize(IonDatagram)} + * is already called. + *

    + * TODO To be deprecated along with {@link IonDatagram#getBytes(byte[])} + * + * @param dst the byte array into which bytes are to be written + * + * @return the number of bytes copied into {@code dst} + */ + int toNewByteArray(byte[] dst) + { + int length = myBuffer.length - myOffset; + System.arraycopy(myBuffer, myOffset, dst, 0, length); + return length; + } + + /** + * Copies the current contents of the Ion binary-encoded byte array into a + * a given byte sub-array. + *

    + * The given sub-array must be large enough to contain all the bytes of the + * Ion binary-encoded byte array. + *

    + * This makes an unchecked assumption that {{@link #serialize(IonDatagram)} + * is already called. + *

    + * TODO To be deprecated along with {@link IonDatagram#getBytes(byte[], int)} + * + * @param dst the byte sub-array into which bytes are to be written + * @param offset the offset within the sub-array of the first byte to be + * written; must be non-negative and no larger + * than {@code dst.length} + * + * @return the number of bytes copied into {@code dst} + */ + int toNewByteArray(byte[] dst, int offset) + { + int length = myBuffer.length - myOffset; + System.arraycopy(myBuffer, myOffset, dst, offset, length); + return length; + } + /** * Copies the current contents of the Ion binary-encoded byte array to a * specified stream. @@ -644,6 +694,10 @@ private void writeIonBoolContent(IonBool val) writeByte(encoded); } + /** + * @see IonBinary#writeUIntValue(BigInteger value, int len) + * @param val + */ private void writeIonIntContent(IonInt val) { if (val.isNullValue()) @@ -747,7 +801,7 @@ private void writeIonFloatContent(IonFloat val) private static final byte[] positiveZeroBitArray = new byte[0]; /** - * @see software.amazon.ion.impl.IonBinary.Writer#writeDecimalContent + * @see com.amazon.ion.impl.IonBinary.Writer#writeDecimalContent */ private void writeIonDecimalContent(BigDecimal bd) { @@ -821,6 +875,7 @@ private void writeIonTimestampContent(IonTimestamp val) switch (t.getPrecision()) { // Fall through each case - by design + case FRACTION: case SECOND: { BigDecimal fraction = t.getZFractionalSecond(); @@ -1106,7 +1161,7 @@ private void writeIonStructContent(IonStruct val) { final int originalOffset = myBuffer.length - myOffset; - // TODO amzn/ion-java#31 should not preserve the ordering of fields + // TODO amzn/ion-java/issues/31 should not preserve the ordering of fields ArrayList values = new ArrayList(); // Fill ArrayList with IonValues, the add() just copies the @@ -1127,7 +1182,7 @@ private void writeIonStructContent(IonStruct val) writeVarUInt(sid); } - // TODO amzn/ion-java#41 Detect if the struct fields are sorted in ascending + // TODO amzn/ion-java/issues/41 Detect if the struct fields are sorted in ascending // order of Sids. If so, 1 should be written into 'length' field. // Note that this 'length' field is not the same as the four-bit // length L in the type descriptor octet. @@ -1218,7 +1273,7 @@ private int findSid(SymbolToken symToken) * have different Ion versions. * * - * TODO amzn/ion-java#25 Currently, {@link IonDatagram#systemIterator()} doesn't + * TODO amzn/ion-java/issues/25 Currently, {@link IonDatagram#systemIterator()} doesn't * retain information about interspersed IVMs within the IonDatagram. * As such, we cannot obtain the location of interspersed IVMs, if any. * diff --git a/src/software/amazon/ion/impl/lite/TopLevelContext.java b/src/com/amazon/ion/impl/lite/TopLevelContext.java similarity index 76% rename from src/software/amazon/ion/impl/lite/TopLevelContext.java rename to src/com/amazon/ion/impl/lite/TopLevelContext.java index 51beee8349..797ff6c9cc 100644 --- a/src/software/amazon/ion/impl/lite/TopLevelContext.java +++ b/src/com/amazon/ion/impl/lite/TopLevelContext.java @@ -1,20 +1,21 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; +package com.amazon.ion.impl.lite; -import software.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolTable; /** @@ -39,7 +40,7 @@ final class TopLevelContext * symbol table and following the parent/owning_context * chain will lead to a system object. *

    - * TODO amzn/ion-java#19 we cannot assume that the IonSystem knows the proper IVM + * TODO amzn/ion-java/issues/19 we cannot assume that the IonSystem knows the proper IVM * in this context */ private final SymbolTable _symbols; diff --git a/src/software/amazon/ion/impl/lite/ValueFactoryLite.java b/src/com/amazon/ion/impl/lite/ValueFactoryLite.java similarity index 90% rename from src/software/amazon/ion/impl/lite/ValueFactoryLite.java rename to src/com/amazon/ion/impl/lite/ValueFactoryLite.java index a0609d0888..9d9800f9b9 100644 --- a/src/software/amazon/ion/impl/lite/ValueFactoryLite.java +++ b/src/com/amazon/ion/impl/lite/ValueFactoryLite.java @@ -1,36 +1,37 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; - +package com.amazon.ion.impl.lite; + +import com.amazon.ion.ContainedValueException; +import com.amazon.ion.Decimal; +import com.amazon.ion.IonSequence; +import com.amazon.ion.IonSexp; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.Timestamp; +import com.amazon.ion.impl._Private_LocalSymbolTableFactory; +import com.amazon.ion.impl._Private_Utils; +import com.amazon.ion.impl._Private_ValueFactory; import java.math.BigDecimal; import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; -import software.amazon.ion.ContainedValueException; -import software.amazon.ion.Decimal; -import software.amazon.ion.IonSequence; -import software.amazon.ion.IonSexp; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.Timestamp; -import software.amazon.ion.impl.PrivateLocalSymbolTableFactory; -import software.amazon.ion.impl.PrivateUtils; -import software.amazon.ion.impl.PrivateValueFactory; /** * This class handles all of the IonValueLite @@ -38,14 +39,14 @@ * */ abstract class ValueFactoryLite - implements PrivateValueFactory + implements _Private_ValueFactory { - protected final PrivateLocalSymbolTableFactory _lstFactory; + protected final _Private_LocalSymbolTableFactory _lstFactory; private ContainerlessContext _context; ValueFactoryLite() { - _lstFactory = PrivateUtils.newLocalSymbolTableAsStructFactory(this); + _lstFactory = _Private_Utils.newLocalSymbolTableAsStructFactory(this); } protected void set_system(IonSystemLite system) { @@ -445,8 +446,7 @@ private ArrayList newInts(long[] elements) return e; } - @Override - public PrivateLocalSymbolTableFactory getLstFactory() + public _Private_LocalSymbolTableFactory getLstFactory() { return _lstFactory; } diff --git a/src/software/amazon/ion/impl/lite/PrivateLiteDomTrampoline.java b/src/com/amazon/ion/impl/lite/_Private_LiteDomTrampoline.java similarity index 50% rename from src/software/amazon/ion/impl/lite/PrivateLiteDomTrampoline.java rename to src/com/amazon/ion/impl/lite/_Private_LiteDomTrampoline.java index eeeddab03b..d9749aa3ac 100644 --- a/src/software/amazon/ion/impl/lite/PrivateLiteDomTrampoline.java +++ b/src/com/amazon/ion/impl/lite/_Private_LiteDomTrampoline.java @@ -1,36 +1,36 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; +package com.amazon.ion.impl.lite; -import software.amazon.ion.IonSystem; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.impl.PrivateIonBinaryWriterBuilder; -import software.amazon.ion.system.IonReaderBuilder; -import software.amazon.ion.system.IonTextWriterBuilder; +import com.amazon.ion.IonSystem; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.impl._Private_IonBinaryWriterBuilder; +import com.amazon.ion.system.IonReaderBuilder; +import com.amazon.ion.system.IonTextWriterBuilder; /** + * NOT FOR APPLICATION USE! + *

    * Isolates private APIs that are needed from other packages in this library. - * - * @deprecated This is an internal API that is subject to change without notice. + * The leading _ reduces the chance of somebody finding this via autocomplete. */ -@Deprecated -public final class PrivateLiteDomTrampoline +public final class _Private_LiteDomTrampoline { - public static IonSystem newLiteSystem(IonTextWriterBuilder twb, - PrivateIonBinaryWriterBuilder bwb, + _Private_IonBinaryWriterBuilder bwb, IonReaderBuilder rb) { return new IonSystemLite(twb, bwb, rb); diff --git a/src/software/amazon/ion/overview.html b/src/com/amazon/ion/overview.html similarity index 65% rename from src/software/amazon/ion/overview.html rename to src/com/amazon/ion/overview.html index c62f1e4d7a..cf81a4bda3 100644 --- a/src/software/amazon/ion/overview.html +++ b/src/com/amazon/ion/overview.html @@ -1,44 +1,68 @@ - + + -Amazon Ion Java is the reference implementation of the -Ion data notation for the -JavaTM Platform Standard Edition 8 +ion-java is the reference implementation of the +Ion data notation for the +JavaTM 2 Platform Standard Edition 5.0 and above.

    This document describes all APIs intended for public consumption, limited to the packages listed here. The ion-java library distribution includes other -packages not documented here; use of those packages is not supported. +packages not documented here; use of those packages is not supported, +so don't use them!

    More generally: Any behavior or features not present in this API documentation is unsupported, probably untested, and subject to change without notice.

    In addition, unless otherwise noted, interfaces and classes listed here -should not be implemented or extended by code outside of this library. We +should not be implemented or extended by code outside of this library! We may add or remove API(s) of existing interfaces or classes in future releases. +If your application does not observe these warnings, it will impede the release +cycle of this library.

    Start at IonSystem

    -The central interface in ion-java is {@link software.amazon.ion.IonSystem}, which is +The central interface in ion-java is {@link com.amazon.ion.IonSystem}, which is the main factory and facade for all Ion processing. The intended architectural pattern is for your application to build a single system instance and use it throughout the application. The {@code IonSystem} interface provides access to all other components, including the capability to construct -{@link software.amazon.ion.IonValue IonValue} hierarchies. +{@link com.amazon.ion.IonValue IonValue} hierarchies.

    What all this means is that your first task is acquiring a system instance, and for that we turn to -{@link software.amazon.ion.system.IonSystemBuilder IonSystemBuilder}. +{@link com.amazon.ion.system.IonSystemBuilder IonSystemBuilder}. Here's the easiest way to bootstrap:

         IonSystem ion = IonSystemBuilder.standard().build();
     
    That should be sufficient for many, but not all, applications. Long-running services will probably want to use a non-default -{@linkplain software.amazon.ion.IonCatalog catalog} by configuring the builder -before calling {@link software.amazon.ion.system.IonSystemBuilder#build() build()}. +{@linkplain com.amazon.ion.IonCatalog catalog} by configuring the builder +before calling {@link com.amazon.ion.system.IonSystemBuilder#build() build()}. + +

    SystemFactory is Deprecated

    + +As of early 2011, the {@link com.amazon.ion.system.SystemFactory SystemFactory} +class has been deprecated in favor of {@code IonSystemBuilder}. +This should be a straightforward application change and we strongly recommend +that all applications update.

    An Important Caveat

    Objects returned by one {@code IonSystem} cannot by mixed with objects returned @@ -55,7 +79,7 @@

    An Important Caveat

    Given any {@code IonValue} instance it is possible to retrieve the relevant -system via {@link software.amazon.ion.IonValue#getSystem()}. +system via {@link com.amazon.ion.IonValue#getSystem()}. This is generally the best way to ensure that you're using the correct system while modifying existing trees. You can also use the "Curried" insertion methods to add new values to @@ -70,14 +94,14 @@

    Getting Data In

    This release defines three mechanisms for accepting Ion data:
      -
    • {@link software.amazon.ion.IonReader IonReader} scans an input stream using a +
    • {@link com.amazon.ion.IonReader IonReader} scans an input stream using a "pull parsing" paradigm. This is a low-level, high-performance API, and the other mechanisms are built on top of it.
    • Iteration reads an input stream by iterating over its top-level elements. This "one at a time" input mechanism is intended for stream-oriented applications. -
    • {@link software.amazon.ion.IonLoader IonLoader} loads an entire input stream - into a single {@linkplain software.amazon.ion.IonDatagram datagram}. +
    • {@link com.amazon.ion.IonLoader IonLoader} loads an entire input stream + into a single {@linkplain com.amazon.ion.IonDatagram datagram}. This "all at once" input mechanism is intended for document-oriented applications.
    @@ -87,42 +111,42 @@

    Getting Data In

    To construct an {@code IonReader}, call one of the {@code newReader} methods on {@code IonSystem}; for example -{@link software.amazon.ion.IonSystem#newReader(InputStream)}. +{@link com.amazon.ion.IonSystem#newReader(InputStream)}. You can then pull data from the reader. Don't forget to -{@link software.amazon.ion.IonReader#close() close} it when you're done! +{@link com.amazon.ion.IonReader#close() close} it when you're done!

    Ion iterators are extensions of {@link java.util.Iterator} so they are used once and then discarded. Use the various {@code iterate} methods on {@code IonSystem} to create them; for example -{@link software.amazon.ion.IonSystem#iterate(InputStream)}. +{@link com.amazon.ion.IonSystem#iterate(InputStream)}.

    To construct an {@code IonLoader}, call -{@link software.amazon.ion.IonSystem#newLoader()} and configure it as necessary. +{@link com.amazon.ion.IonSystem#newLoader()} and configure it as necessary. {@code IonLoaders} are safe for use by multiple threads. The {@code IonSystem} also maintains a "default loader" so you don't have to pass one around, see -{@link software.amazon.ion.IonSystem#getLoader()}. +{@link com.amazon.ion.IonSystem#getLoader()}.

    Getting Data Out

    There's also several mechanisms for generating Ion data:
      -
    • {@link software.amazon.ion.IonWriter IonWriter} is the low-level API for +
    • {@link com.amazon.ion.IonWriter IonWriter} is the low-level API for generating Ion data in some form. It's agnostic to the output format; in theory the actual output could be some other format entirely. -
    • {@link software.amazon.ion.IonValue#toString() IonValue.toString()} will also +
    • {@link com.amazon.ion.IonValue#toString() IonValue.toString()} will also generate Ion text, but it's primarily intended for debugging purposes and cannot be customized. The particular layout is not specified by contract, so don't assume that it will always output the same thing! -
    • {@link software.amazon.ion.IonValue#writeTo(IonWriter)} outputs Ion data in +
    • {@link com.amazon.ion.IonValue#writeTo(IonWriter)} outputs Ion data in the writer's format. This is the best way to output the data model. -
    • From an {@link software.amazon.ion.IonDatagram IonDatagram} you can call - {@link software.amazon.ion.IonDatagram#getBytes() getBytes()} to get Ion +
    • From an {@link com.amazon.ion.IonDatagram IonDatagram} you can call + {@link com.amazon.ion.IonDatagram#getBytes() getBytes()} to get Ion binary data.
    You can create {@code IonWriter}s using methods on {@code IonSystem}, but the -{@link software.amazon.ion.system.IonTextWriterBuilder} provides more flexibility. +{@link com.amazon.ion.system.IonTextWriterBuilder} provides more flexibility.

    No Canonical Serialization

    @@ -141,25 +165,26 @@

    No Canonical Serialization

    assertEquals(expectedIonValue.toString(), actualIonValue.toString()); The same goes for output via any other API, including -{@link software.amazon.ion.IonWriter IonWriter}s. +{@link com.amazon.ion.IonWriter IonWriter}s.

    The correct approach to performing semantic equivalence checks over Ion data is to use documented equivalence APIs such as -{@link software.amazon.ion.IonValue#equals(Object) IonValue.equals()}. +{@link com.amazon.ion.IonValue#equals(Object) IonValue.equals()}.

    JSON Integration

    The Ion text format is a superset of JSON, so JSON data is Ion data. This means that you can read JSON data as-is using the Ion libraries, with a caveat: -
    • JSON numbers with exponents are decoded as Ion float, while those with fractions (but not exponents) are decoded as Ion decimal.

    To output JSON with this library, Ion data can be "downconverted" to JSON format using an -{@link software.amazon.ion.system.IonTextWriterBuilder IonTextWriterBuilder}. +{@link com.amazon.ion.system.IonTextWriterBuilder IonTextWriterBuilder}. This replaces Ion-only datatypes with more-or-less equivalent JSON values. +(The {@link com.amazon.ion.util.Printer Printer} can also be used, but it is +no longer recommended and will be deprecated.)

    Thread Safety

    diff --git a/src/com/amazon/ion/package-info.java b/src/com/amazon/ion/package-info.java new file mode 100644 index 0000000000..9923ef22b4 --- /dev/null +++ b/src/com/amazon/ion/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +/** + * Public interfaces of the core Ion system. + */ +package com.amazon.ion; diff --git a/src/software/amazon/ion/system/IonBinaryWriterBuilder.java b/src/com/amazon/ion/system/IonBinaryWriterBuilder.java similarity index 92% rename from src/software/amazon/ion/system/IonBinaryWriterBuilder.java rename to src/com/amazon/ion/system/IonBinaryWriterBuilder.java index eddc25b362..c6d6a0e545 100644 --- a/src/software/amazon/ion/system/IonBinaryWriterBuilder.java +++ b/src/com/amazon/ion/system/IonBinaryWriterBuilder.java @@ -1,25 +1,26 @@ /* - * Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.system; +package com.amazon.ion.system; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SubstituteSymbolTableException; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.impl.PrivateIonBinaryWriterBuilder; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SubstituteSymbolTableException; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.impl._Private_IonBinaryWriterBuilder; /** @@ -33,6 +34,7 @@ *

    * Instances of this class are not not safe for use by multiple threads * unless they are {@linkplain #immutable() immutable}. + * */ public abstract class IonBinaryWriterBuilder extends IonWriterBuilderBase @@ -64,7 +66,7 @@ protected IonBinaryWriterBuilder(IonBinaryWriterBuilder that) */ public static IonBinaryWriterBuilder standard() { - return PrivateIonBinaryWriterBuilder.standard(); + return _Private_IonBinaryWriterBuilder.standard(); } diff --git a/src/software/amazon/ion/system/IonReaderBuilder.java b/src/com/amazon/ion/system/IonReaderBuilder.java similarity index 91% rename from src/software/amazon/ion/system/IonReaderBuilder.java rename to src/com/amazon/ion/system/IonReaderBuilder.java index 7366ba52d9..895933b43d 100644 --- a/src/software/amazon/ion/system/IonReaderBuilder.java +++ b/src/com/amazon/ion/system/IonReaderBuilder.java @@ -1,30 +1,32 @@ /* - * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.system; +package com.amazon.ion.system; -import static software.amazon.ion.impl.PrivateIonReaderFactory.makeReader; +import static com.amazon.ion.impl._Private_IonReaderFactory.makeReader; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonException; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonTextReader; +import com.amazon.ion.IonValue; import java.io.IOException; import java.io.InputStream; import java.io.Reader; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonException; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonValue; /** * Build a new {@link IonReader} from the given {@link IonCatalog} and data @@ -268,7 +270,7 @@ public IonReader build(IonValue value) * * @see IonSystem#newReader(String) */ - public IonReader build(String ionText) + public IonTextReader build(String ionText) { return makeReader(validateCatalog(), ionText); } diff --git a/src/software/amazon/ion/system/IonSystemBuilder.java b/src/com/amazon/ion/system/IonSystemBuilder.java similarity index 89% rename from src/software/amazon/ion/system/IonSystemBuilder.java rename to src/com/amazon/ion/system/IonSystemBuilder.java index 982f251d1f..22141e83b7 100644 --- a/src/software/amazon/ion/system/IonSystemBuilder.java +++ b/src/com/amazon/ion/system/IonSystemBuilder.java @@ -1,28 +1,29 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.system; +package com.amazon.ion.system; -import static software.amazon.ion.impl.lite.PrivateLiteDomTrampoline.newLiteSystem; +import static com.amazon.ion.impl.lite._Private_LiteDomTrampoline.newLiteSystem; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.impl.PrivateIonBinaryWriterBuilder; -import software.amazon.ion.impl.PrivateUtils; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.impl._Private_IonBinaryWriterBuilder; +import com.amazon.ion.impl._Private_Utils; /** * The builder for creating {@link IonSystem}s. @@ -103,7 +104,10 @@ public static IonSystemBuilder standard() /** You no touchy. */ - private IonSystemBuilder() {} + private IonSystemBuilder() + { + // empty + } private IonSystemBuilder(IonSystemBuilder that) { @@ -127,6 +131,7 @@ public final IonSystemBuilder copy() * * @return this instance, if immutable; * otherwise an immutable copy of this instance. + * */ public IonSystemBuilder immutable() { @@ -138,6 +143,8 @@ public IonSystemBuilder immutable() * * @return this instance, if mutable; * otherwise a mutable copy of this instance. + * + */ public IonSystemBuilder mutable() { @@ -213,6 +220,8 @@ public final IonSystemBuilder withCatalog(IonCatalog catalog) * * @see #setStreamCopyOptimized(boolean) * @see #withStreamCopyOptimized(boolean) + * + */ public final boolean isStreamCopyOptimized() { @@ -231,6 +240,8 @@ public final boolean isStreamCopyOptimized() * * @see #isStreamCopyOptimized() * @see #withStreamCopyOptimized(boolean) + * + */ public final void setStreamCopyOptimized(boolean optimized) { @@ -248,6 +259,8 @@ public final void setStreamCopyOptimized(boolean optimized) * * @see #isStreamCopyOptimized() * @see #setStreamCopyOptimized(boolean) + * + */ public final IonSystemBuilder withStreamCopyOptimized(boolean optimized) { @@ -273,21 +286,22 @@ public final IonSystem build() IonTextWriterBuilder.standard().withCharsetAscii(); twb.setCatalog(catalog); - PrivateIonBinaryWriterBuilder bwb = - PrivateIonBinaryWriterBuilder.standard(); + _Private_IonBinaryWriterBuilder bwb = + _Private_IonBinaryWriterBuilder.standard(); bwb.setCatalog(catalog); bwb.setStreamCopyOptimized(myStreamCopyOptimized); // TODO Would be nice to remove this since it's implied by the BWB. // However that currently causes problems in the IonSystem // constructors (which get a null initialSymtab). - SymbolTable systemSymtab = PrivateUtils.systemSymtab(1); + SymbolTable systemSymtab = _Private_Utils.systemSymtab(1); bwb.setInitialSymbolTable(systemSymtab); // This is what we need, more or less. // bwb = bwb.fillDefaults(); - IonReaderBuilder rb = IonReaderBuilder.standard().withCatalog(catalog); - return newLiteSystem(twb, bwb, rb); + IonSystem sys = newLiteSystem(twb, bwb, rb); + + return sys; } //========================================================================= diff --git a/src/software/amazon/ion/system/IonTextWriterBuilder.java b/src/com/amazon/ion/system/IonTextWriterBuilder.java similarity index 94% rename from src/software/amazon/ion/system/IonTextWriterBuilder.java rename to src/com/amazon/ion/system/IonTextWriterBuilder.java index 19035470ca..daa90ff689 100644 --- a/src/software/amazon/ion/system/IonTextWriterBuilder.java +++ b/src/com/amazon/ion/system/IonTextWriterBuilder.java @@ -1,28 +1,29 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.system; +package com.amazon.ion.system; -import static software.amazon.ion.system.IonWriterBuilder.InitialIvmHandling.SUPPRESS; +import static com.amazon.ion.system.IonWriterBuilder.InitialIvmHandling.SUPPRESS; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.impl._Private_IonTextWriterBuilder; +import com.amazon.ion.impl._Private_Utils; import java.io.OutputStream; import java.nio.charset.Charset; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.impl.PrivateIonTextWriterBuilder; -import software.amazon.ion.impl.PrivateUtils; /** * The builder for creating {@link IonWriter}s emitting the Ion text syntax. @@ -65,8 +66,9 @@ *

    * Currently, there is no configuration point available to disable the * auto-flushing mechanism. Please vote on - * issue amzn/ion-java#32 + * issue amzn/ion-java/issues/32 * if you require it. + * */ public abstract class IonTextWriterBuilder extends IonWriterBuilderBase @@ -75,6 +77,8 @@ public abstract class IonTextWriterBuilder * A strategy for minimizing the output of local symbol tables. * By default, no minimization takes place and the writer outputs all data * as-is. + * + */ public enum LstMinimizing { @@ -91,7 +95,7 @@ public enum LstMinimizing /** * Discards everything, collapsing the LST to an IVM. - * If {@link software.amazon.ion.system.IonWriterBuilder.IvmMinimizing} + * If {@link com.amazon.ion.system.IonWriterBuilder.IvmMinimizing} * is also in effect, then even that IVM may be suppressed. * * @see IonTextWriterBuilder#setIvmMinimizing(IonWriterBuilder.IvmMinimizing) @@ -102,12 +106,12 @@ public enum LstMinimizing /** * The {@code "US-ASCII"} charset. */ - public static final Charset ASCII = PrivateUtils.ASCII_CHARSET; + public static final Charset ASCII = _Private_Utils.ASCII_CHARSET; /** * The {@code "UTF-8"} charset. */ - public static final Charset UTF8 = PrivateUtils.UTF8_CHARSET; + public static final Charset UTF8 = _Private_Utils.UTF8_CHARSET; /** @@ -124,7 +128,7 @@ public enum LstMinimizing */ public static IonTextWriterBuilder standard() { - return PrivateIonTextWriterBuilder.standard(); + return _Private_IonTextWriterBuilder.standard(); } /** @@ -134,6 +138,8 @@ public static IonTextWriterBuilder standard() * @return a new, mutable builder instance. * * @see #withMinimalSystemData() + * + */ public static IonTextWriterBuilder minimal() { @@ -322,6 +328,8 @@ public final IonTextWriterBuilder withCharsetAscii() * * @return this instance, if mutable; * otherwise a mutable copy of this instance. + * + */ public final IonTextWriterBuilder withMinimalSystemData() { @@ -444,6 +452,8 @@ public void setInitialIvmHandling(InitialIvmHandling handling) * * @see #setIvmMinimizing(IonWriterBuilder.IvmMinimizing) * @see #withIvmMinimizing(IonWriterBuilder.IvmMinimizing) + * + */ @Override public final IvmMinimizing getIvmMinimizing() @@ -462,6 +472,8 @@ public final IvmMinimizing getIvmMinimizing() * @see #withIvmMinimizing(IonWriterBuilder.IvmMinimizing) * * @throws UnsupportedOperationException if this is immutable. + * + */ public void setIvmMinimizing(IvmMinimizing minimizing) { @@ -482,6 +494,8 @@ public void setIvmMinimizing(IvmMinimizing minimizing) * * @see #setIvmMinimizing(IonWriterBuilder.IvmMinimizing) * @see #getIvmMinimizing() + * + */ public final IonTextWriterBuilder withIvmMinimizing(IvmMinimizing minimizing) @@ -500,6 +514,8 @@ public void setIvmMinimizing(IvmMinimizing minimizing) * * @see #setLstMinimizing(LstMinimizing) * @see #withLstMinimizing(LstMinimizing) + * + */ public final LstMinimizing getLstMinimizing() { @@ -518,6 +534,8 @@ public final LstMinimizing getLstMinimizing() * @see #withLstMinimizing(LstMinimizing) * * @throws UnsupportedOperationException if this is immutable. + * + */ public void setLstMinimizing(LstMinimizing minimizing) { @@ -538,6 +556,8 @@ public void setLstMinimizing(LstMinimizing minimizing) * * @see #getLstMinimizing() * @see #setLstMinimizing(LstMinimizing) + * + */ public final IonTextWriterBuilder withLstMinimizing(LstMinimizing minimizing) diff --git a/src/software/amazon/ion/system/IonWriterBuilder.java b/src/com/amazon/ion/system/IonWriterBuilder.java similarity index 86% rename from src/software/amazon/ion/system/IonWriterBuilder.java rename to src/com/amazon/ion/system/IonWriterBuilder.java index d99221e299..f61f170822 100644 --- a/src/software/amazon/ion/system/IonWriterBuilder.java +++ b/src/com/amazon/ion/system/IonWriterBuilder.java @@ -1,27 +1,30 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.system; +package com.amazon.ion.system; import java.io.OutputStream; -import software.amazon.ion.IonWriter; +import com.amazon.ion.IonWriter; /** * Common options for writing Ion data streams of any form. *

    * WARNING: This class should not be extended by code outside of * this library. + *

    + * */ public abstract class IonWriterBuilder { @@ -37,6 +40,8 @@ public enum InitialIvmHandling * Always emits an initial IVM, even when the user hasn't explicitly * written one. If the user does write one, this won't * cause an extra to be emitted. + * + */ ENSURE, @@ -55,6 +60,8 @@ public enum InitialIvmHandling * stream; that's the job of {@link InitialIvmHandling}. * * @see IonTextWriterBuilder#setIvmMinimizing(IonWriterBuilder.IvmMinimizing) + * + */ public enum IvmMinimizing { @@ -97,6 +104,8 @@ public enum IvmMinimizing * stream; that's the job of {@link InitialIvmHandling}. * * @return the IVM minimizing strategy. + * + */ public abstract IvmMinimizing getIvmMinimizing(); diff --git a/src/software/amazon/ion/system/IonWriterBuilderBase.java b/src/com/amazon/ion/system/IonWriterBuilderBase.java similarity index 91% rename from src/software/amazon/ion/system/IonWriterBuilderBase.java rename to src/com/amazon/ion/system/IonWriterBuilderBase.java index 9ada86f8b1..a36601e845 100644 --- a/src/software/amazon/ion/system/IonWriterBuilderBase.java +++ b/src/com/amazon/ion/system/IonWriterBuilderBase.java @@ -1,22 +1,24 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.system; +package com.amazon.ion.system; + +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; abstract class IonWriterBuilderBase diff --git a/src/software/amazon/ion/system/SimpleCatalog.java b/src/com/amazon/ion/system/SimpleCatalog.java similarity index 92% rename from src/software/amazon/ion/system/SimpleCatalog.java rename to src/com/amazon/ion/system/SimpleCatalog.java index 557ef3487d..7ce65756f0 100644 --- a/src/software/amazon/ion/system/SimpleCatalog.java +++ b/src/com/amazon/ion/system/SimpleCatalog.java @@ -1,28 +1,29 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.system; +package com.amazon.ion.system; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonMutableCatalog; +import com.amazon.ion.SymbolTable; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.TreeMap; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonMutableCatalog; -import software.amazon.ion.SymbolTable; /** diff --git a/src/com/amazon/ion/system/SystemFactory.java b/src/com/amazon/ion/system/SystemFactory.java new file mode 100644 index 0000000000..fe4d209f5f --- /dev/null +++ b/src/com/amazon/ion/system/SystemFactory.java @@ -0,0 +1,66 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.system; + +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonSystem; + +/** + * The factory for creating {@link IonSystem}s. + * Most applications will only have one or two system instances; + * see {@link IonSystem} for important constraints. + *

    + * Most long-lived applications will want to provide a custom + * {@link IonCatalog} implementation rather than using the default + * {@link SimpleCatalog}. + * + * @deprecated Use the more configurable {@link IonSystemBuilder} instead. + */ +@Deprecated +public final class SystemFactory +{ + /** + * Constructs a new system instance with a default configuration. + *

    + * The catalog used by the new instance will be a {@link SimpleCatalog} + * with no initial entries, so please be aware of the limitations of that + * class. + * + * @return a new {@link IonSystem} instance; not null. + * + * @deprecated Use {@link IonSystemBuilder IonSystemBuilder.standard().build()} instead. + */ + @Deprecated + public static IonSystem newSystem() + { + return IonSystemBuilder.standard().build(); + } + + /** + * Constructs a new system instance with the given catalog. + * + * @param catalog the catalog to use in the new system. + * If null, a new {@link SimpleCatalog} will be used. + * @return a new {@link IonSystem} instance; not null. + * + * @deprecated Use {@link IonSystemBuilder IonSystemBuilder.standard().withCatalog(catalog).build()} instead. + */ + @Deprecated + public static IonSystem newSystem(IonCatalog catalog) + { + return IonSystemBuilder.standard().withCatalog(catalog).build(); + } +} diff --git a/src/com/amazon/ion/system/package-info.java b/src/com/amazon/ion/system/package-info.java new file mode 100644 index 0000000000..c7fc56c746 --- /dev/null +++ b/src/com/amazon/ion/system/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +/** + * Public implementation of the core Ion system. + */ +package com.amazon.ion.system; diff --git a/src/software/amazon/ion/util/AbstractValueVisitor.java b/src/com/amazon/ion/util/AbstractValueVisitor.java similarity index 67% rename from src/software/amazon/ion/util/AbstractValueVisitor.java rename to src/com/amazon/ion/util/AbstractValueVisitor.java index b0a52754ca..246e8ad8b7 100644 --- a/src/software/amazon/ion/util/AbstractValueVisitor.java +++ b/src/com/amazon/ion/util/AbstractValueVisitor.java @@ -1,25 +1,36 @@ /* - * Copyright (c) 2007-2013 Amazon.com, Inc. All rights reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.util; - -import software.amazon.ion.IonBlob; -import software.amazon.ion.IonBool; -import software.amazon.ion.IonClob; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonDecimal; -import software.amazon.ion.IonFloat; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonList; -import software.amazon.ion.IonNull; -import software.amazon.ion.IonSexp; -import software.amazon.ion.IonString; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonTimestamp; -import software.amazon.ion.IonValue; -import software.amazon.ion.ValueVisitor; +package com.amazon.ion.util; + +import com.amazon.ion.IonBlob; +import com.amazon.ion.IonBool; +import com.amazon.ion.IonClob; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonDecimal; +import com.amazon.ion.IonFloat; +import com.amazon.ion.IonInt; +import com.amazon.ion.IonList; +import com.amazon.ion.IonNull; +import com.amazon.ion.IonSexp; +import com.amazon.ion.IonString; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSymbol; +import com.amazon.ion.IonTimestamp; +import com.amazon.ion.IonValue; +import com.amazon.ion.ValueVisitor; /** * A base class for extending Ion {@link ValueVisitor}s. diff --git a/src/software/amazon/ion/util/Equivalence.java b/src/com/amazon/ion/util/Equivalence.java similarity index 92% rename from src/software/amazon/ion/util/Equivalence.java rename to src/com/amazon/ion/util/Equivalence.java index 644225ddc0..dbbd391b2c 100644 --- a/src/software/amazon/ion/util/Equivalence.java +++ b/src/com/amazon/ion/util/Equivalence.java @@ -1,42 +1,43 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.util; - -import static software.amazon.ion.impl.PrivateIonConstants.UNKNOWN_SYMBOL_TEXT_PREFIX; - +package com.amazon.ion.util; + +import static com.amazon.ion.impl._Private_IonConstants.UNKNOWN_SYMBOL_TEXT_PREFIX; + +import com.amazon.ion.Decimal; +import com.amazon.ion.IonBool; +import com.amazon.ion.IonDecimal; +import com.amazon.ion.IonException; +import com.amazon.ion.IonFloat; +import com.amazon.ion.IonInt; +import com.amazon.ion.IonLob; +import com.amazon.ion.IonSequence; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSymbol; +import com.amazon.ion.IonText; +import com.amazon.ion.IonTimestamp; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.SymbolToken; import java.io.IOException; import java.io.InputStream; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Set; -import software.amazon.ion.Decimal; -import software.amazon.ion.IonBool; -import software.amazon.ion.IonDecimal; -import software.amazon.ion.IonException; -import software.amazon.ion.IonFloat; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonLob; -import software.amazon.ion.IonSequence; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonText; -import software.amazon.ion.IonTimestamp; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.SymbolToken; /** * Provides equivalence comparisons between two {@link IonValue}s, following @@ -48,13 +49,13 @@ *

      *    IonValue v1 = ...;
      *    IonValue v2 = ...;
    - *    software.amazon.ion.util.Equivalence.ionEquals( v1, v2 );
    + *    com.amazon.ion.util.Equivalence.ionEquals( v1, v2 );
      *
    * * More likely, a static import would make using this class easier. * *
    - *    import static software.amazon.ion.util.Equivalence.ionEquals;
    + *    import static com.amazon.ion.util.Equivalence.ionEquals;
      *    ...
      *    boolean equivalent = ionEquals( v1, v2 );
      *
    @@ -103,11 +104,12 @@ * (A, V), where A is an ordered list of annotations, and V is an Ion Primitive * Data or Ion Complex Data value. The list of annotations, A is an tuple of Ion * Symbols (a specific type of Ion Primitive). + */ public final class Equivalence { /** - * TODO amzn/ion-java#26 Marker for code that needs to be altered in order to + * TODO amzn/ion-java/issues/26 Marker for code that needs to be altered in order to * support a public comparison API to determine ordering of values, not * just equality. */ @@ -335,7 +337,7 @@ static class Field { SymbolToken tok = value.getFieldNameSymbol(); String name = tok.getText(); if (name == null) { - // TODO amzn/ion-java#23 Problematic with unknown field names. + // TODO amzn/ion-java/issues/23 Problematic with unknown field names. name = UNKNOWN_SYMBOL_TEXT_PREFIX + tok.getSid(); } this.name = name; @@ -349,7 +351,7 @@ static class Field { @Override public int hashCode() { return name.hashCode(); - // TODO amzn/ion-java#58 : implement hash code such that it respects + // TODO amzn/ion-java/issues/58 : implement hash code such that it respects // 'strict'. The prevously attempted fix is commented out below but // is not sufficient because value.hasCode will always include // type annotations in the hash computation. Type annotations @@ -433,14 +435,14 @@ private static int ionCompareToImpl(final IonValue v1, ((IonFloat) v2).doubleValue()); break; case DECIMAL: - assert !PUBLIC_COMPARISON_API; // TODO amzn/ion-java#26 + assert !PUBLIC_COMPARISON_API; // TODO amzn/ion-java/issues/26 result = Decimal.equals(((IonDecimal) v1).decimalValue(), ((IonDecimal) v2).decimalValue()) ? 0 : 1; break; case TIMESTAMP: if (strict) { - assert !PUBLIC_COMPARISON_API; // TODO amzn/ion-java#26 + assert !PUBLIC_COMPARISON_API; // TODO amzn/ion-java/issues/26 result = (((IonTimestamp) v1).timestampValue().equals( ((IonTimestamp) v2).timestampValue()) ? 0 : 1); @@ -468,7 +470,7 @@ private static int ionCompareToImpl(final IonValue v1, result = compareLobContents((IonLob) v1, (IonLob) v2); break; case STRUCT: - assert !PUBLIC_COMPARISON_API; // TODO amzn/ion-java#26 + assert !PUBLIC_COMPARISON_API; // TODO amzn/ion-java/issues/26 result = compareStructs((IonStruct) v1, (IonStruct) v2, strict); diff --git a/src/software/amazon/ion/util/GzipOrRawInputStream.java b/src/com/amazon/ion/util/GzipOrRawInputStream.java similarity index 87% rename from src/software/amazon/ion/util/GzipOrRawInputStream.java rename to src/com/amazon/ion/util/GzipOrRawInputStream.java index 0a510c97e1..68b60f1e83 100644 --- a/src/software/amazon/ion/util/GzipOrRawInputStream.java +++ b/src/com/amazon/ion/util/GzipOrRawInputStream.java @@ -1,18 +1,19 @@ /* - * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.util; +package com.amazon.ion.util; import java.io.FilterInputStream; import java.io.IOException; diff --git a/src/software/amazon/ion/util/IonStreamUtils.java b/src/com/amazon/ion/util/IonStreamUtils.java similarity index 84% rename from src/software/amazon/ion/util/IonStreamUtils.java rename to src/com/amazon/ion/util/IonStreamUtils.java index 24945a61d8..798fcfc62a 100644 --- a/src/software/amazon/ion/util/IonStreamUtils.java +++ b/src/com/amazon/ion/util/IonStreamUtils.java @@ -1,28 +1,29 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.util; +package com.amazon.ion.util; -import static software.amazon.ion.impl.PrivateIonConstants.BINARY_VERSION_MARKER_1_0; -import static software.amazon.ion.util.GzipOrRawInputStream.GZIP_HEADER; +import static com.amazon.ion.impl._Private_IonConstants.BINARY_VERSION_MARKER_1_0; +import static com.amazon.ion.util.GzipOrRawInputStream.GZIP_HEADER; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.impl._Private_ListWriter; import java.io.IOException; import java.io.InputStream; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonType; -import software.amazon.ion.IonWriter; -import software.amazon.ion.impl.PrivateListWriter; /** * Utility methods for working with the Ion streaming interfaces, @@ -82,6 +83,7 @@ public static boolean isIonBinary(byte[] buffer, int offset, int length) * * @return {@code true} if the buffer contains GZIPped data; {@code false} * if the buffer is null or if the {@code length} is too short. + * */ public static boolean isGzip(byte[] buffer, int offset, int length) { @@ -113,6 +115,7 @@ private static boolean cookieMatches(byte[] cookie, /** * Returns a stream that decompresses a stream if it contains GZIPped data, * otherwise has no effect on the stream (but may wrap it). + * */ public static InputStream unGzip(InputStream in) throws IOException @@ -135,8 +138,8 @@ public static InputStream unGzip(InputStream in) public static void writeBoolList(IonWriter writer, boolean[] values) throws IOException { - if (writer instanceof PrivateListWriter) { - ((PrivateListWriter)writer).writeBoolList(values); + if (writer instanceof _Private_ListWriter) { + ((_Private_ListWriter)writer).writeBoolList(values); return; } @@ -160,8 +163,8 @@ public static void writeBoolList(IonWriter writer, boolean[] values) public static void writeFloatList(IonWriter writer, float[] values) throws IOException { - if (writer instanceof PrivateListWriter) { - ((PrivateListWriter)writer).writeFloatList(values); + if (writer instanceof _Private_ListWriter) { + ((_Private_ListWriter)writer).writeFloatList(values); return; } @@ -183,8 +186,8 @@ public static void writeFloatList(IonWriter writer, float[] values) public static void writeFloatList(IonWriter writer, double[] values) throws IOException { - if (writer instanceof PrivateListWriter) { - ((PrivateListWriter)writer).writeFloatList(values); + if (writer instanceof _Private_ListWriter) { + ((_Private_ListWriter)writer).writeFloatList(values); return; } @@ -207,8 +210,8 @@ public static void writeFloatList(IonWriter writer, double[] values) public static void writeIntList(IonWriter writer, byte[] values) throws IOException { - if (writer instanceof PrivateListWriter) { - ((PrivateListWriter)writer).writeIntList(values); + if (writer instanceof _Private_ListWriter) { + ((_Private_ListWriter)writer).writeIntList(values); return; } @@ -230,8 +233,8 @@ public static void writeIntList(IonWriter writer, byte[] values) public static void writeIntList(IonWriter writer, short[] values) throws IOException { - if (writer instanceof PrivateListWriter) { - ((PrivateListWriter)writer).writeIntList(values); + if (writer instanceof _Private_ListWriter) { + ((_Private_ListWriter)writer).writeIntList(values); return; } @@ -253,8 +256,8 @@ public static void writeIntList(IonWriter writer, short[] values) public static void writeIntList(IonWriter writer, int[] values) throws IOException { - if (writer instanceof PrivateListWriter) { - ((PrivateListWriter)writer).writeIntList(values); + if (writer instanceof _Private_ListWriter) { + ((_Private_ListWriter)writer).writeIntList(values); return; } @@ -276,8 +279,8 @@ public static void writeIntList(IonWriter writer, int[] values) public static void writeIntList(IonWriter writer, long[] values) throws IOException { - if (writer instanceof PrivateListWriter) { - ((PrivateListWriter)writer).writeIntList(values); + if (writer instanceof _Private_ListWriter) { + ((_Private_ListWriter)writer).writeIntList(values); return; } @@ -300,8 +303,8 @@ public static void writeIntList(IonWriter writer, long[] values) public static void writeStringList(IonWriter writer, String[] values) throws IOException { - if (writer instanceof PrivateListWriter) { - ((PrivateListWriter)writer).writeStringList(values); + if (writer instanceof _Private_ListWriter) { + ((_Private_ListWriter)writer).writeStringList(values); return; } diff --git a/src/software/amazon/ion/util/IonTextUtils.java b/src/com/amazon/ion/util/IonTextUtils.java similarity index 93% rename from src/software/amazon/ion/util/IonTextUtils.java rename to src/com/amazon/ion/util/IonTextUtils.java index b86ab29043..47e0d17326 100644 --- a/src/software/amazon/ion/util/IonTextUtils.java +++ b/src/com/amazon/ion/util/IonTextUtils.java @@ -1,30 +1,32 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.util; +package com.amazon.ion.util; -import static software.amazon.ion.impl.PrivateIonConstants.isHighSurrogate; -import static software.amazon.ion.impl.PrivateIonConstants.isLowSurrogate; -import static software.amazon.ion.impl.PrivateIonConstants.makeUnicodeScalar; -import static software.amazon.ion.impl.PrivateIonTextAppender.ZERO_PADDING; -import static software.amazon.ion.impl.PrivateIonTextAppender.isIdentifierKeyword; -import static software.amazon.ion.impl.PrivateIonTextAppender.symbolNeedsQuoting; -import static software.amazon.ion.impl.PrivateIonTextWriterBuilder.STANDARD; +import static com.amazon.ion.impl._Private_IonConstants.isHighSurrogate; +import static com.amazon.ion.impl._Private_IonConstants.isLowSurrogate; +import static com.amazon.ion.impl._Private_IonConstants.makeUnicodeScalar; +import static com.amazon.ion.impl._Private_IonTextAppender.ZERO_PADDING; +import static com.amazon.ion.impl._Private_IonTextAppender.isIdentifierKeyword; +import static com.amazon.ion.impl._Private_IonTextAppender.symbolNeedsQuoting; +import static com.amazon.ion.impl._Private_IonTextWriterBuilder.STANDARD; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.impl._Private_IonTextAppender; import java.io.IOException; import java.math.BigDecimal; -import software.amazon.ion.impl.PrivateIonTextAppender; /** @@ -116,15 +118,15 @@ public static boolean isDigit(int codePoint, int radix) } public static boolean isIdentifierStart(int codePoint) { - return PrivateIonTextAppender.isIdentifierStart(codePoint); + return _Private_IonTextAppender.isIdentifierStart(codePoint); } public static boolean isIdentifierPart(int codePoint) { - return PrivateIonTextAppender.isIdentifierPart(codePoint); + return _Private_IonTextAppender.isIdentifierPart(codePoint); } public static boolean isOperatorPart(int codePoint) { - return PrivateIonTextAppender.isOperatorPart(codePoint); + return _Private_IonTextAppender.isOperatorPart(codePoint); } @@ -622,6 +624,16 @@ public static String printSymbol(CharSequence text) } + /** + * @param token the symbolToken to be printed. + * + * @return a string representing the symboltoken. + */ + public static String printSymbol(SymbolToken token){ + return String.format("{$%s:%s}", token.getText(), token.getSid()); + } + + /** * Prints text as a single-quoted Ion symbol. * If the {@code text} is null, this prints {@code null.symbol}. @@ -812,8 +824,8 @@ else if (isLowSurrogate(c)) public static void printDecimal(Appendable out, BigDecimal decimal) throws IOException { - PrivateIonTextAppender appender = - PrivateIonTextAppender.forAppendable(out); + _Private_IonTextAppender appender = + _Private_IonTextAppender.forAppendable(out); appender.printDecimal(STANDARD, decimal); } @@ -841,8 +853,8 @@ public static String printDecimal(BigDecimal decimal) public static void printFloat(Appendable out, double value) throws IOException { - PrivateIonTextAppender appender = - PrivateIonTextAppender.forAppendable(out); + _Private_IonTextAppender appender = + _Private_IonTextAppender.forAppendable(out); appender.printFloat(value); } @@ -865,8 +877,8 @@ public static String printFloat(double value) public static void printFloat(Appendable out, Double value) throws IOException { - PrivateIonTextAppender appender = - PrivateIonTextAppender.forAppendable(out); + _Private_IonTextAppender appender = + _Private_IonTextAppender.forAppendable(out); appender.printFloat(value); } @@ -894,8 +906,8 @@ public static void printBlob(Appendable out, byte[] value) } else { - PrivateIonTextAppender appender = - PrivateIonTextAppender.forAppendable(out); + _Private_IonTextAppender appender = + _Private_IonTextAppender.forAppendable(out); appender.printBlob(STANDARD, value, 0, value.length); } } @@ -930,8 +942,8 @@ public static void printClob(Appendable out, byte[] value) } else { - PrivateIonTextAppender appender = - PrivateIonTextAppender.forAppendable(out); + _Private_IonTextAppender appender = + _Private_IonTextAppender.forAppendable(out); appender.printClob(STANDARD, value, 0, value.length); } } diff --git a/src/software/amazon/ion/util/IonValueUtils.java b/src/com/amazon/ion/util/IonValueUtils.java similarity index 59% rename from src/software/amazon/ion/util/IonValueUtils.java rename to src/com/amazon/ion/util/IonValueUtils.java index 05278793dc..bd1591cca9 100644 --- a/src/software/amazon/ion/util/IonValueUtils.java +++ b/src/com/amazon/ion/util/IonValueUtils.java @@ -1,20 +1,21 @@ /* - * Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.util; +package com.amazon.ion.util; -import software.amazon.ion.IonValue; +import com.amazon.ion.IonValue; /** * Utility methods for working with {@link IonValue}s. diff --git a/src/software/amazon/ion/util/JarInfo.java b/src/com/amazon/ion/util/JarInfo.java similarity index 89% rename from src/software/amazon/ion/util/JarInfo.java rename to src/com/amazon/ion/util/JarInfo.java index 2b8f7786d1..1f0ba4807f 100644 --- a/src/software/amazon/ion/util/JarInfo.java +++ b/src/com/amazon/ion/util/JarInfo.java @@ -1,18 +1,19 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.util; +package com.amazon.ion.util; import java.io.IOException; import java.net.URL; @@ -21,8 +22,8 @@ import java.util.List; import java.util.jar.Attributes; import java.util.jar.Manifest; -import software.amazon.ion.IonException; -import software.amazon.ion.Timestamp; +import com.amazon.ion.IonException; +import com.amazon.ion.Timestamp; /** @@ -146,4 +147,4 @@ private boolean tryLoadBuildProperties(Manifest manifest) } return true; } -} +} \ No newline at end of file diff --git a/src/com/amazon/ion/util/Printer.java b/src/com/amazon/ion/util/Printer.java new file mode 100644 index 0000000000..cc717b1d7e --- /dev/null +++ b/src/com/amazon/ion/util/Printer.java @@ -0,0 +1,1160 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.util; + +import static com.amazon.ion.SystemSymbols.IMPORTS; +import static com.amazon.ion.SystemSymbols.ION_1_0_SID; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; +import static com.amazon.ion.SystemSymbols.SYMBOLS; + +import com.amazon.ion.Decimal; +import com.amazon.ion.IonBlob; +import com.amazon.ion.IonBool; +import com.amazon.ion.IonClob; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonDecimal; +import com.amazon.ion.IonFloat; +import com.amazon.ion.IonInt; +import com.amazon.ion.IonList; +import com.amazon.ion.IonNull; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonSequence; +import com.amazon.ion.IonSexp; +import com.amazon.ion.IonString; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSymbol; +import com.amazon.ion.IonTimestamp; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.Timestamp; +import com.amazon.ion.impl._Private_IonSymbol; +import com.amazon.ion.impl._Private_IonSystem; +import com.amazon.ion.impl._Private_IonTextWriterBuilder; +import com.amazon.ion.impl._Private_IonValue; +import com.amazon.ion.impl._Private_IonValue.SymbolTableProvider; +import com.amazon.ion.system.IonTextWriterBuilder; +import com.amazon.ion.system.IonTextWriterBuilder.LstMinimizing; +import com.amazon.ion.system.IonWriterBuilder.IvmMinimizing; +import com.amazon.ion.util.IonTextUtils.SymbolVariant; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; +import java.util.Iterator; + + +/** + * Renders {@link IonValue}s to text. + *

    + * By default, output is in a compact format with minimal whitespace. + * For example: + *

    + *    annot::{f1:["hello","goodbye"],'another field':long::0}
    + *
    + * The format can be tuned through various properties on the Printer instance, + * as well as through the {@link Printer.Options} structure. + *

    + * Instances of this class are safe for use by multiple threads. + *

    + * While printers are inexpensive to create, their configuration facilities + * make them useful as shared resources. Changes to configuration settings + * (e.g., {@link #setJsonMode()}) do not affect concurrently-running + * calls to {@link #print}. + * + * @see IonWriter + * @see IonTextWriterBuilder + */ +public class Printer +{ + public class Options + implements Cloneable + { + public boolean blobAsString; + public boolean clobAsString; + public boolean datagramAsList; + public boolean decimalAsFloat; + public boolean sexpAsList; + public boolean skipAnnotations; + public boolean skipSystemValues; + public boolean simplifySystemValues; + public boolean stringAsJson; + public boolean symbolAsString; + public boolean timestampAsString; + public boolean timestampAsMillis; + public boolean untypedNulls; + + + @Override + public Options clone() + { + try + { + return (Options) super.clone(); + } + catch (CloneNotSupportedException e) + { + throw new InternalError(); + } + } + } + + + protected Options myOptions = new Options(); + + public Printer() + { + myOptions = new Options(); + } + + public Printer(Options options) + { + myOptions = options.clone(); + } + + + //========================================================================= + // Options + + /* Potential printing options: + * + * - Render all times with a specific offset (what about unknowns?) + * - Render all times with long offset (no Z) + * - control over decimal-point placement in float and decimal. + * (render 12.00 or 1200d-2) + */ + + + /** + * Indicates whether this printer skips (i.e., doesn't print) + * system IDs and local symbol tables. + * By default, this property is false. + */ + public synchronized boolean getSkipSystemValues() + { + return myOptions.skipSystemValues; + } + + /** + * Sets whether this printer skips (i.e., doesn't print) + * system IDs and local symbol tables. + * By default, this property is false. + */ + public synchronized void setSkipSystemValues(boolean skip) + { + myOptions.skipSystemValues = skip; + } + + + /** + * Indicates whether this printer skips (i.e., doesn't print) + * annotations. + * By default, this property is false. + */ + public synchronized boolean getSkipAnnotations() + { + return myOptions.skipAnnotations; + } + + /** + * Sets whether this printer skips (i.e., doesn't print) + * annotations. + * By default, this property is false. + */ + public synchronized void setSkipAnnotations(boolean skip) + { + myOptions.skipAnnotations = skip; + } + + + /** + * Indicates whether this printer renders blobs as Base64 strings. + * By default, this is false. + */ + public synchronized boolean getPrintBlobAsString() + { + return myOptions.blobAsString; + } + + /** + * Sets whether this printer renders blobs as Base64 strings. + * By default, this is false. + */ + public synchronized void setPrintBlobAsString(boolean blobAsString) + { + myOptions.blobAsString = blobAsString; + } + + + /** + * Indicates whether this printer renders clobs as ASCII strings. + * By default, this is false. + */ + public synchronized boolean getPrintClobAsString() + { + return myOptions.clobAsString; + } + + /** + * Sets whether this printer renders clobs as ASCII strings. + * By default, this is false. + */ + public synchronized void setPrintClobAsString(boolean clobAsString) + { + myOptions.clobAsString = clobAsString; + } + + + /** + * Indicates whether this printer renders datagrams as lists. + * By default, this property is false. + */ + public synchronized boolean getPrintDatagramAsList() + { + return myOptions.datagramAsList; + } + + /** + * Sets whether this printer renders datagrams as lists. + * By default, this property is false. + */ + public synchronized void setPrintDatagramAsList(boolean datagramAsList) + { + myOptions.datagramAsList = datagramAsList; + } + + + /** + * Indicates whether this printer renders decimals as floats, thus using 'e' + * notation for all real values. + * By default, this is false. + */ + public synchronized boolean getPrintDecimalAsFloat() + { + return myOptions.decimalAsFloat; + } + + /** + * Sets whether this printer renders decimals as floats, thus using 'e' + * notation for all real values. + * By default, this is false. + */ + public synchronized void setPrintDecimalAsFloat(boolean decimalAsFloat) + { + myOptions.decimalAsFloat = decimalAsFloat; + } + + + /** + * Indicates whether this printer renders sexps as lists. + * By default, this is false. + */ + public synchronized boolean getPrintSexpAsList() + { + return myOptions.sexpAsList; + } + + /** + * Sets whether this printer renders sexps as lists. + * By default, this is false. + */ + public synchronized void setPrintSexpAsList(boolean sexpAsList) + { + myOptions.sexpAsList = sexpAsList; + } + + + /** + * Indicates whether this printer renders strings using JSON escapes. + * By default, this is false. + */ + public synchronized boolean getPrintStringAsJson() + { + return myOptions.stringAsJson; + } + + /** + * Sets whether this printer renders strings using JSON escapes. + * By default, this is false. + */ + public synchronized void setPrintStringAsJson(boolean stringAsJson) + { + myOptions.stringAsJson = stringAsJson; + } + + + /** + * Indicates whether this printer renders symbols as strings. + * By default, this is false. + */ + public synchronized boolean getPrintSymbolAsString() + { + return myOptions.symbolAsString; + } + + /** + * Sets whether this printer renders symbols as strings. + * By default, this is false. + */ + public synchronized void setPrintSymbolAsString(boolean symbolAsString) + { + myOptions.symbolAsString = symbolAsString; + } + + + /** + * Indicates whether this printer renders timestamps as millisecond values. + * By default, this is false. + */ + public synchronized boolean getPrintTimestampAsMillis() + { + return myOptions.timestampAsMillis; + } + + /** + * Sets whether this printer renders timestamps as millisecond values. + * By default, this is false. + */ + public synchronized void setPrintTimestampAsMillis(boolean timestampAsMillis) + { + myOptions.timestampAsMillis = timestampAsMillis; + } + + + /** + * Indicates whether this printer renders timestamps as strings. + * By default, this is false. + */ + public synchronized boolean getPrintTimestampAsString() + { + return myOptions.timestampAsString; + } + + /** + * Sets whether this printer renders timestamps as strings. + * By default, this is false. + */ + public synchronized void setPrintTimestampAsString(boolean timestampAsString) + { + myOptions.timestampAsString = timestampAsString; + } + + + /** + * Indicates whether this printer renders all null values as {@code null} + * (i.e., the same as an {@link IonNull}). + * By default, this is false. + */ + public synchronized boolean getPrintUntypedNulls() + { + return myOptions.untypedNulls; + } + + /** + * Sets whether this printer renders all null values as {@code null} + * (i.e., the same as an {@link IonNull}). + * By default, this is false. + */ + public synchronized void setPrintUntypedNulls(boolean untypedNulls) + { + myOptions.untypedNulls = untypedNulls; + } + + + /** + * Configures this printer's options to render legal JSON text. + * The following options are modified so that: + *

      + *
    • {@link Options#blobAsString} is {@code true}
    • + *
    • {@link Options#clobAsString} is {@code true}
    • + *
    • {@link Options#datagramAsList} is {@code true}
    • + *
    • {@link Options#decimalAsFloat} is {@code true}
    • + *
    • {@link Options#sexpAsList} is {@code true}
    • + *
    • {@link Options#skipAnnotations} is {@code true}
    • + *
    • {@link Options#skipSystemValues} is {@code true}
    • + *
    • {@link Options#stringAsJson} is {@code true}
    • + *
    • {@link Options#symbolAsString} is {@code true}
    • + *
    • {@link Options#timestampAsString} is {@code false}
    • + *
    • {@link Options#timestampAsMillis} is {@code true}
    • + *
    • {@link Options#untypedNulls} is {@code true}
    • + *
    + * All other options are left as is. + */ + public synchronized void setJsonMode() + { + myOptions.blobAsString = true; + myOptions.clobAsString = true; + myOptions.datagramAsList = true; + myOptions.decimalAsFloat = true; + myOptions.sexpAsList = true; + myOptions.skipAnnotations = true; + myOptions.skipSystemValues = true; + myOptions.stringAsJson = true; + myOptions.symbolAsString = true; + myOptions.timestampAsString = false; + myOptions.timestampAsMillis = true; + myOptions.untypedNulls = true; + } + + + //========================================================================= + // Print methods + + + public void print(IonValue value, Appendable out) + throws IOException + { + // Copy the options so visitor won't see changes made while printing. + Options options; + synchronized (this) // So we don't clone in the midst of changes + { + options = myOptions.clone(); + } + + if (true) + { + _print(value, makeVisitor(options, out)); + } + else + { + // Bridge to the configurable text writer. This is here for + // testing purposes. It *almost* works except for printing + // datagram as list. + + boolean dg = value instanceof IonDatagram; + + _Private_IonTextWriterBuilder o = + _Private_IonTextWriterBuilder.standard(); + o.setCharset(IonTextWriterBuilder.ASCII); + if (dg) + { + if (options.skipSystemValues) + { + o.withMinimalSystemData(); + } + else if (options.simplifySystemValues) { + + o.setIvmMinimizing(IvmMinimizing.DISTANT); + o.setLstMinimizing(LstMinimizing.LOCALS); + } + } + + o._blob_as_string = options.blobAsString; + o._clob_as_string = options.clobAsString; + o._decimal_as_float = options.decimalAsFloat; + // TODO datagram as list + o._sexp_as_list = options.sexpAsList; + o._skip_annotations = options.skipAnnotations; + o._string_as_json = options.stringAsJson; + o._symbol_as_string = options.symbolAsString; + o._timestamp_as_millis = options.timestampAsMillis; + o._timestamp_as_string = options.timestampAsString; + o._untyped_nulls = options.untypedNulls; + + IonWriter writer = o.build(out); + // TODO doesn't work for datagram since it skips system values + // value.writeTo(writer); + _Private_IonSystem system = (_Private_IonSystem) value.getSystem(); + IonReader reader = system.newSystemReader(value); + writer.writeValues(reader); + writer.finish(); + } + } + + private void _print(IonValue value, PrinterVisitor pv) + throws IOException + { + try + { + if (! (value instanceof IonDatagram)) + { + pv.setSymbolTableProvider(new BasicSymbolTableProvider(value.getSymbolTable())); + } + value.accept(pv); + } + catch (IOException e) + { + throw e; + } + catch (RuntimeException e) + { + throw e; + } + catch (Exception e) + { + // Shouldn't happen. + throw new RuntimeException(e); + } + } + + + /** + * Subclasses can override this if they wish to construct a specialization + * of the {@link PrinterVisitor}. + * + * @param options is a fresh copy of the Printer's options instance, + * not null. + * @param out is not null. + * @return the visitor to invoke for printing. + */ + protected PrinterVisitor makeVisitor(Options options, Appendable out) + { + return new PrinterVisitor(options, out); + } + + + private static class BasicSymbolTableProvider + implements SymbolTableProvider + { + private final SymbolTable symbolTable; + + public BasicSymbolTableProvider(SymbolTable symbolTable) + { + this.symbolTable = symbolTable; + } + + public SymbolTable getSymbolTable() + { + return symbolTable; + } + + } + + //========================================================================= + // Print methods + + public static class PrinterVisitor + extends AbstractValueVisitor + { + + final protected Options myOptions; + final protected Appendable myOut; + + /** + * Should we quote operators at the current level of the hierarchy? + * As we recurse down into containers, this value is pushed on the + * stack by {@link #writeChild(IonValue, boolean)}. + */ + private boolean myQuoteOperators = true; + + private SymbolTableProvider mySymbolTableProvider = null; + + + //--------------------------------------------------------------------- + + public PrinterVisitor(Options options, Appendable out) + { + myOptions = options; + myOut = out; + } + + void setSymbolTableProvider(SymbolTableProvider symbolTableProvider) + { + mySymbolTableProvider = symbolTableProvider; + } + + /** + * Recurse down into a container, we push the current value of + * {@link #myQuoteOperators} onto the stack and replace it with + * the given value. + * + * @param value + * @param quoteOperators replaces the current value of + * {@link #myQuoteOperators} during the recursive visitation. + * @throws Exception propagated from visitation of value. + * @throws NullPointerException if value is null. + */ + protected void writeChild(IonValue value, boolean quoteOperators) + throws Exception + { + boolean oldQuoteOperators = myQuoteOperators; + myQuoteOperators = quoteOperators; + value.accept(this); + myQuoteOperators = oldQuoteOperators; + } + + + public void writeAnnotations(IonValue value) throws IOException + { + if (! myOptions.skipAnnotations) + { + SymbolToken[] anns = ((_Private_IonValue)value).getTypeAnnotationSymbols(mySymbolTableProvider); + if (anns != null) + { + for (SymbolToken ann : anns) { + String text = ann.getText(); + if (text == null) { + myOut.append('$'); + myOut.append(Integer.toString(ann.getSid())); + } + else { + IonTextUtils.printSymbol(myOut, text); + } + myOut.append("::"); + } + } + } + } + + public void writeNull(String type) throws IOException + { + if (myOptions.untypedNulls) + { + myOut.append("null"); + } + else + { + myOut.append("null."); + myOut.append(type); + } + } + + + public void writeSequenceContent(IonSequence value, + boolean quoteOperators, + char open, + char separator, + char close) + throws IOException, Exception + { + myOut.append(open); + + boolean hitOne = false; + for (IonValue child : value) + { + if (hitOne) + { + myOut.append(separator); + } + hitOne = true; + + writeChild(child, quoteOperators); + } + + myOut.append(close); + } + + public void writeSymbolToken(SymbolToken sym) throws IOException + { + String text = sym.getText(); + if (text != null) + { + writeSymbol(text); + } + else + { + int sid = sym.getSid(); + if (sid < 0) + { + throw new IllegalArgumentException("Bad SID " + sid); + } + + text = "$" + sym.getSid(); + if (myOptions.symbolAsString) + { + writeString(text); + } + else + { + myOut.append(text); // SID literal is never quoted + } + } + } + + public void writeSymbol(String text) throws IOException + { + if (myOptions.symbolAsString) + { + writeString(text); + } + else + { + SymbolVariant variant = IonTextUtils.symbolVariant(text); + switch (variant) + { + case IDENTIFIER: + myOut.append(text); + break; + case OPERATOR: + if (! myQuoteOperators) + { + myOut.append(text); + break; + } + // else fall through... + case QUOTED: + IonTextUtils.printQuotedSymbol(myOut, text); + break; + } + } + } + + + /** + * @param text may be null + */ + public void writeString(String text) throws IOException + { + if (myOptions.stringAsJson) + { + IonTextUtils.printJsonString(myOut, text); + } + else + { + IonTextUtils.printString(myOut, text); + } + } + + + //--------------------------------------------------------------------- + // AbstractValueVisitor overrides + + @Override + protected void defaultVisit(IonValue value) + { + String message = "cannot print " + value.getClass().getName(); + throw new UnsupportedOperationException(message); + } + + @Override + public void visit(IonBlob value) throws IOException + { + writeAnnotations(value); + + if (value.isNullValue()) + { + writeNull("blob"); + } + else + { + myOut.append(myOptions.blobAsString ? "\"" : "{{"); + value.printBase64(myOut); + myOut.append(myOptions.blobAsString ? "\"" : "}}"); + } + } + + @Override + public void visit(IonBool value) + throws IOException + { + writeAnnotations(value); + + if (value.isNullValue()) + { + writeNull("bool"); + } + else + { + myOut.append(value.booleanValue() ? "true" : "false"); + } + } + + @Override + public void visit(IonClob value) throws IOException + { + writeAnnotations(value); + + if (value.isNullValue()) + { + writeNull("clob"); + } + else + { + if (! myOptions.clobAsString) + { + myOut.append("{{"); + } + myOut.append('"'); + + InputStream byteStream = value.newInputStream(); + try + { + int c; + // if-statement hoisted above loop for efficiency + if (myOptions.stringAsJson) + { + while ((c = byteStream.read()) != -1) + { + IonTextUtils.printJsonCodePoint(myOut, c); + } + } + else + { + while ((c = byteStream.read()) != -1) + { + IonTextUtils.printStringCodePoint(myOut, c); + } + } + } + finally + { + byteStream.close(); + } + + myOut.append('"'); + if (! myOptions.clobAsString) + { + myOut.append("}}"); + } + } + } + + @Override + public void visit(IonDatagram value) throws IOException, Exception + { + Iterator i = (myOptions.skipSystemValues + ? value.iterator() + : value.systemIterator()); + + final boolean asList = myOptions.datagramAsList; + if (asList) + { + myOut.append('['); + } + + boolean hitOne = false; + + // If we're skipping system values at the iterator level, + // we don't need to bother trying to simplify them. + final boolean simplify_system_values = + myOptions.simplifySystemValues && ! myOptions.skipSystemValues; + + SymbolTable previous_symbols = null; + + while (i.hasNext()) + { + IonValue child = i.next(); + SymbolTable childSymbolTable = child.getSymbolTable(); + mySymbolTableProvider = new BasicSymbolTableProvider(childSymbolTable); //children of datagrams are top-level values + if (simplify_system_values) + { + child = simplify(child, previous_symbols); + previous_symbols = childSymbolTable; + } + + if (child != null) + { + if (hitOne) + { + myOut.append(asList ? ',' : ' '); + } + writeChild(child, true); + hitOne = true; // we've only "hit one" if we wrote it + } + } + + if (asList) + { + myOut.append(']'); + } + } + + private final IonValue simplify(IonValue child, + SymbolTable previous_symbols) + { + IonType t = child.getType(); + switch (t) { + case STRUCT: + if (child.hasTypeAnnotation(ION_SYMBOL_TABLE)) { + if (symbol_table_struct_has_imports(child)) + { + return ((IonStruct)child).cloneAndRemove(SYMBOLS); + } + return null; + } + // fall through to default (print the value) + break; + case SYMBOL: + if (((IonSymbol)child).getSymbolId() == ION_1_0_SID) { + if (previous_symbols != null && previous_symbols.isSystemTable()) { + return null; + } + // fall through to default (print the value) + } + // fall through to default (print the value) + break; + default: + break; + } + return child; + } + + static final private boolean symbol_table_struct_has_imports(IonValue child) { + IonStruct struct = (IonStruct)child; + IonValue imports = struct.get(IMPORTS); + if (imports instanceof IonList) { + return ((IonList)imports).size() != 0; + } + return false; + } + + @Override + public void visit(IonDecimal value) throws IOException + { + writeAnnotations(value); + + if (value.isNullValue()) + { + writeNull("decimal"); + } + else + { + Decimal decimal = value.decimalValue(); + BigInteger unscaled = decimal.unscaledValue(); + + int signum = decimal.signum(); + if (signum < 0) + { + myOut.append('-'); + unscaled = unscaled.negate(); + } + else if (signum == 0 && decimal.isNegativeZero()) + { + // for the various forms of negative zero we have to + // write the sign ourselves, since neither BigInteger + // nor BigDecimal recognize negative zero, but Ion does. + myOut.append('-'); + } + + + final String unscaledText = unscaled.toString(); + final int significantDigits = unscaledText.length(); + + final int scale = decimal.scale(); + final int exponent = -scale; + + if (myOptions.decimalAsFloat) + { + myOut.append(unscaledText); + myOut.append('e'); + myOut.append(Integer.toString(exponent)); + } + else if (exponent == 0) + { + myOut.append(unscaledText); + myOut.append('.'); + } + else if (0 < scale) + { + int wholeDigits; + int remainingScale; + if (significantDigits > scale) + { + wholeDigits = significantDigits - scale; + remainingScale = 0; + } + else + { + wholeDigits = 1; + remainingScale = scale - significantDigits + 1; + } + + myOut.append(unscaledText, 0, wholeDigits); + if (wholeDigits < significantDigits) + { + myOut.append('.'); + myOut.append(unscaledText, wholeDigits, + significantDigits); + } + + if (remainingScale != 0) + { + myOut.append("d-"); + myOut.append(Integer.toString(remainingScale)); + } + } + else // (exponent > 0) + { + // We cannot move the decimal point to the right, adding + // rightmost zeros, because that would alter the precision. + myOut.append(unscaledText); + myOut.append('d'); + myOut.append(Integer.toString(exponent)); + } + } + } + + @Override + public void visit(IonFloat value) throws IOException + { + writeAnnotations(value); + + if (value.isNullValue()) + { + writeNull("float"); + } + else + { + double real = value.doubleValue(); + IonTextUtils.printFloat(myOut, real); + } + } + + @Override + public void visit(IonInt value) throws IOException + { + writeAnnotations(value); + + if (value.isNullValue()) + { + writeNull("int"); + } + else + { + myOut.append(value.bigIntegerValue().toString(10)); + } + } + + @Override + public void visit(IonList value) throws IOException, Exception + { + writeAnnotations(value); + + if (value.isNullValue()) + { + writeNull("list"); + } + else + { + writeSequenceContent(value, true, '[', ',', ']'); + } + } + + @Override + public void visit(IonNull value) throws IOException + { + writeAnnotations(value); + myOut.append("null"); + } + + + @Override + public void visit(IonSexp value) throws IOException, Exception + { + writeAnnotations(value); + + if (value.isNullValue()) + { + writeNull("sexp"); + } + else if (myOptions.sexpAsList) + { + writeSequenceContent(value, true, '[', ',', ']'); + } + else + { + writeSequenceContent(value, false, '(', ' ', ')'); + } + } + + + @Override + public void visit(IonString value) throws IOException + { + writeAnnotations(value); + + if (value.isNullValue()) + { + writeNull("string"); + } + else + { + writeString(value.stringValue()); + } + } + + + @Override + public void visit(IonStruct value) throws IOException, Exception + { + writeAnnotations(value); + + if (value.isNullValue()) + { + writeNull("struct"); + } + else + { + myOut.append('{'); + + boolean hitOne = false; + for (IonValue child : value) + { + if (hitOne) + { + myOut.append(','); + } + hitOne = true; + + SymbolToken sym = ((_Private_IonValue)child).getFieldNameSymbol(mySymbolTableProvider); + writeSymbolToken(sym); + myOut.append(':'); + writeChild(child, true); + } + myOut.append('}'); + } + } + + + @Override + public void visit(IonSymbol value) throws IOException + { + writeAnnotations(value); + + SymbolToken is = ((_Private_IonSymbol)value).symbolValue(mySymbolTableProvider); + if (is == null) + { + writeNull("symbol"); + } + else + { + writeSymbolToken(is); + } + } + + + @Override + public void visit(IonTimestamp value) throws IOException + { + writeAnnotations(value); + + if (value.isNullValue()) + { + writeNull("timestamp"); + } + else if (myOptions.timestampAsMillis) + { + myOut.append(Long.toString(value.getMillis())); + } + else + { + Timestamp ts = value.timestampValue(); + + if (myOptions.timestampAsString) + { + myOut.append('"'); + ts.print(myOut); + myOut.append('"'); + } + else + { + ts.print(myOut); + } + } + } + } +} diff --git a/src/software/amazon/ion/util/Spans.java b/src/com/amazon/ion/util/Spans.java similarity index 71% rename from src/software/amazon/ion/util/Spans.java rename to src/com/amazon/ion/util/Spans.java index f22cc83c22..4d65750c1a 100644 --- a/src/software/amazon/ion/util/Spans.java +++ b/src/com/amazon/ion/util/Spans.java @@ -1,23 +1,24 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.util; +package com.amazon.ion.util; -import static software.amazon.ion.facet.Facets.asFacet; +import static com.amazon.ion.facet.Facets.asFacet; -import software.amazon.ion.Span; -import software.amazon.ion.SpanProvider; +import com.amazon.ion.Span; +import com.amazon.ion.SpanProvider; /** * Utility methods for working with {@link Span}s. diff --git a/src/software/amazon/ion/util/PrivateFastAppendable.java b/src/com/amazon/ion/util/_Private_FastAppendable.java similarity index 76% rename from src/software/amazon/ion/util/PrivateFastAppendable.java rename to src/com/amazon/ion/util/_Private_FastAppendable.java index 9369ee100b..136bdaf43b 100644 --- a/src/software/amazon/ion/util/PrivateFastAppendable.java +++ b/src/com/amazon/ion/util/_Private_FastAppendable.java @@ -1,26 +1,24 @@ /* - * Copyright 2013-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.util; +package com.amazon.ion.util; import java.io.IOException; -/** - * @deprecated This is an internal API that is subject to change without notice. - */ -@Deprecated -public interface PrivateFastAppendable + +public interface _Private_FastAppendable extends Appendable { /** diff --git a/src/com/amazon/ion/util/package-info.java b/src/com/amazon/ion/util/package-info.java new file mode 100644 index 0000000000..4cda041699 --- /dev/null +++ b/src/com/amazon/ion/util/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +/** + * Various utilites for working with Ion data. + */ +package com.amazon.ion.util; diff --git a/src/software/amazon/ion/ContainedValueException.java b/src/software/amazon/ion/ContainedValueException.java deleted file mode 100644 index 34ef2c09cb..0000000000 --- a/src/software/amazon/ion/ContainedValueException.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2007 Amazon.com, Inc. All rights reserved. - */ - -package software.amazon.ion; - -/** - * An error caused by adding an {@link IonValue} into a container when it's - * already contained elsewhere. - */ -public class ContainedValueException - extends IonException -{ - private static final long serialVersionUID = 1L; - - public ContainedValueException() - { - super(); - } - - /** - * @param message - */ - public ContainedValueException(String message) - { - super(message); - } -} diff --git a/src/software/amazon/ion/EmptySymbolException.java b/src/software/amazon/ion/EmptySymbolException.java deleted file mode 100644 index 8934475a78..0000000000 --- a/src/software/amazon/ion/EmptySymbolException.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2007-2013 Amazon.com, Inc. All rights reserved. - */ - -package software.amazon.ion; - -/** - * An error caused by a symbol not containing at least one character for - * its text. - * - * @deprecated this exception is not used as empty symbols are valid. In cases where null is used as the symbol value it was - * replaced by {@link NullPointerException} - */ -@Deprecated -public class EmptySymbolException - extends IonException -{ - private static final long serialVersionUID = -7801632953459636349L; - - public EmptySymbolException() - { - super("Symbols must contain at least one character."); - } -} diff --git a/src/software/amazon/ion/InvalidSystemSymbolException.java b/src/software/amazon/ion/InvalidSystemSymbolException.java deleted file mode 100644 index 3563628cc3..0000000000 --- a/src/software/amazon/ion/InvalidSystemSymbolException.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2007-2013 Amazon.com, Inc. All rights reserved. - */ - -package software.amazon.ion; - - -/** - * An error caused by use of an invalid symbol starting with - * "$ion_". - */ -public class InvalidSystemSymbolException - extends IonException -{ - private static final long serialVersionUID = 2206499395645594047L; - - private String myBadSymbol; - - - public InvalidSystemSymbolException(String badSymbol) - { - super("Invalid system symbol '" + badSymbol + "'"); - myBadSymbol = badSymbol; - } - - - public String getBadSymbol() - { - return myBadSymbol; - } -} diff --git a/src/software/amazon/ion/impl/IonMessages.java b/src/software/amazon/ion/impl/IonMessages.java deleted file mode 100644 index c7594ebdaf..0000000000 --- a/src/software/amazon/ion/impl/IonMessages.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at: - * - * http://aws.amazon.com/apache2.0/ - * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. - */ - -package software.amazon.ion.impl; - -final class IonMessages -{ - static final String CANNOT_STEP_OUT = - "Cannot stepOut any further, already at top level."; - -} diff --git a/src/software/amazon/ion/impl/PrivateByteTransferReader.java b/src/software/amazon/ion/impl/PrivateByteTransferReader.java deleted file mode 100644 index 64a5d282fc..0000000000 --- a/src/software/amazon/ion/impl/PrivateByteTransferReader.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at: - * - * http://aws.amazon.com/apache2.0/ - * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. - */ - -package software.amazon.ion.impl; - -import java.io.IOException; -import software.amazon.ion.IonReader; - -/** - * An {@link IonReader} {@linkplain software.amazon.ion.facet facet} that can rapidly bulk-copy - * Ion binary data under certain circumstances. - * - * @deprecated This is an internal API that is subject to change without notice. - */ -@Deprecated -public interface PrivateByteTransferReader -{ - public void transferCurrentValue(PrivateByteTransferSink writer) - throws IOException; -} diff --git a/src/software/amazon/ion/impl/PrivateFastAppendableTrampoline.java b/src/software/amazon/ion/impl/PrivateFastAppendableTrampoline.java deleted file mode 100644 index 40da4c788e..0000000000 --- a/src/software/amazon/ion/impl/PrivateFastAppendableTrampoline.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2016-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at: - * - * http://aws.amazon.com/apache2.0/ - * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. - */ - -package software.amazon.ion.impl; - -import java.io.OutputStream; -import software.amazon.ion.util.PrivateFastAppendable; - - -/** - * @deprecated This is an internal API that is subject to change without notice. - */ -@Deprecated -public final class PrivateFastAppendableTrampoline -{ - - public static PrivateFastAppendable forAppendable(Appendable appendable) - { - return new AppendableFastAppendable(appendable); - } - - public static PrivateFastAppendable forOutputStream( - OutputStream outputStream) - { - return new OutputStreamFastAppendable(outputStream); - } -} diff --git a/src/software/amazon/ion/impl/PrivateIonContainer.java b/src/software/amazon/ion/impl/PrivateIonContainer.java deleted file mode 100644 index 7c83b6e886..0000000000 --- a/src/software/amazon/ion/impl/PrivateIonContainer.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at: - * - * http://aws.amazon.com/apache2.0/ - * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. - */ - -package software.amazon.ion.impl; - -import software.amazon.ion.IonContainer; -import software.amazon.ion.IonValue; - -/** - * Internal, private, interfaces for manipulating - * the base child collection of IonContainer - * - * @deprecated This is an internal API that is subject to change without notice. - */ -@Deprecated -public interface PrivateIonContainer - extends IonContainer -{ - public int get_child_count(); - public IonValue get_child(int idx); -} diff --git a/src/software/amazon/ion/impl/PrivateIonDatagram.java b/src/software/amazon/ion/impl/PrivateIonDatagram.java deleted file mode 100644 index 8c5c623b4e..0000000000 --- a/src/software/amazon/ion/impl/PrivateIonDatagram.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at: - * - * http://aws.amazon.com/apache2.0/ - * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. - */ - -package software.amazon.ion.impl; - -import software.amazon.ion.IonDatagram; -import software.amazon.ion.SymbolTable; - -/** - * @deprecated This is an internal API that is subject to change without notice. - */ -@Deprecated -public interface PrivateIonDatagram - extends PrivateIonValue, IonDatagram -{ - void appendTrailingSymbolTable(SymbolTable symtab); -} diff --git a/src/software/amazon/ion/impl/PrivateIonTextWriterBuilder.java b/src/software/amazon/ion/impl/PrivateIonTextWriterBuilder.java deleted file mode 100644 index fb3245e420..0000000000 --- a/src/software/amazon/ion/impl/PrivateIonTextWriterBuilder.java +++ /dev/null @@ -1,250 +0,0 @@ -/* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at: - * - * http://aws.amazon.com/apache2.0/ - * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. - */ - -package software.amazon.ion.impl; - -import static software.amazon.ion.impl.PrivateUtils.initialSymtab; - -import java.io.OutputStream; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.system.IonSystemBuilder; -import software.amazon.ion.system.IonTextWriterBuilder; -import software.amazon.ion.system.SimpleCatalog; -import software.amazon.ion.util.PrivateFastAppendable; - -/** - * @deprecated This is an internal API that is subject to change without notice. - */ -@Deprecated -public class PrivateIonTextWriterBuilder - extends IonTextWriterBuilder -{ - private final static CharSequence SPACE_CHARACTER = " "; - // TODO amzn/ion-java#57 decide if this should be platform-specific - private final static CharSequence LINE_SEPARATOR = - System.getProperty("line.separator"); - - - public static PrivateIonTextWriterBuilder standard() - { - return new PrivateIonTextWriterBuilder.Mutable(); - } - - public static PrivateIonTextWriterBuilder STANDARD = - standard().immutable(); - - - //========================================================================= - - private boolean _pretty_print; - public boolean _blob_as_string; - public boolean _clob_as_string; - public boolean _decimal_as_float; - public boolean _sexp_as_list; - public boolean _skip_annotations; - public boolean _string_as_json; - public boolean _symbol_as_string; - public boolean _timestamp_as_millis; - public boolean _timestamp_as_string; - public boolean _untyped_nulls; - - private PrivateIonTextWriterBuilder() - { - super(); - } - - private PrivateIonTextWriterBuilder(PrivateIonTextWriterBuilder that) - { - super(that); - this._pretty_print = that._pretty_print ; - this._blob_as_string = that._blob_as_string ; - this._clob_as_string = that._clob_as_string ; - this._decimal_as_float = that._decimal_as_float ; - this._sexp_as_list = that._sexp_as_list ; - this._skip_annotations = that._skip_annotations ; - this._string_as_json = that._string_as_json ; - this._symbol_as_string = that._symbol_as_string ; - this._timestamp_as_millis = that._timestamp_as_millis; - this._timestamp_as_string = that._timestamp_as_string; - this._untyped_nulls = that._untyped_nulls ; - } - - - @Override - public final PrivateIonTextWriterBuilder copy() - { - return new Mutable(this); - } - - @Override - public PrivateIonTextWriterBuilder immutable() - { - return this; - } - - @Override - public PrivateIonTextWriterBuilder mutable() - { - return copy(); - } - - - //========================================================================= - - @Override - public final IonTextWriterBuilder withPrettyPrinting() - { - PrivateIonTextWriterBuilder b = mutable(); - b._pretty_print = true; - return b; - } - - @Override - public final IonTextWriterBuilder withJsonDowngrade() - { - PrivateIonTextWriterBuilder b = mutable(); - - b.withMinimalSystemData(); - - _blob_as_string = true; - _clob_as_string = true; - // datagramAsList = true; // TODO - _decimal_as_float = true; - _sexp_as_list = true; - _skip_annotations = true; - // skipSystemValues = true; // TODO - _string_as_json = true; - _symbol_as_string = true; - _timestamp_as_string = true; // TODO different from Printer - _timestamp_as_millis = false; - _untyped_nulls = true; - - return b; - } - - - final boolean isPrettyPrintOn() - { - return _pretty_print; - } - - final CharSequence lineSeparator() - { - if (_pretty_print) { - return LINE_SEPARATOR; - } - else { - return SPACE_CHARACTER; - } - } - - - //========================================================================= - - private PrivateIonTextWriterBuilder fillDefaults() - { - // Ensure that we don't modify the user's builder. - IonTextWriterBuilder b = copy(); - - if (b.getCatalog() == null) - { - b.setCatalog(new SimpleCatalog()); - } - - if (b.getCharset() == null) - { - b.setCharset(UTF8); - } - - return (PrivateIonTextWriterBuilder) b.immutable(); - } - - - /** Assumes that {@link #fillDefaults()} has been called. */ - private IonWriter build(PrivateFastAppendable appender) - { - IonCatalog catalog = getCatalog(); - SymbolTable[] imports = getImports(); - - // TODO We shouldn't need a system here - IonSystem system = - IonSystemBuilder.standard().withCatalog(catalog).build(); - - SymbolTable defaultSystemSymtab = system.getSystemSymbolTable(); - - IonWriterSystemText systemWriter = new IonWriterSystemText(defaultSystemSymtab, - this, - appender); - - SymbolTable initialSymtab = - initialSymtab(((PrivateValueFactory)system).getLstFactory(), defaultSystemSymtab, imports); - - return new IonWriterUser(catalog, system, systemWriter, initialSymtab); - } - - - @Override - public final IonWriter build(Appendable out) - { - PrivateIonTextWriterBuilder b = fillDefaults(); - - PrivateFastAppendable fast = new AppendableFastAppendable(out); - - return b.build(fast); - } - - - @Override - public final IonWriter build(OutputStream out) - { - PrivateIonTextWriterBuilder b = fillDefaults(); - - PrivateFastAppendable fast = new OutputStreamFastAppendable(out); - - return b.build(fast); - } - - //========================================================================= - - private static final class Mutable - extends PrivateIonTextWriterBuilder - { - private Mutable() { } - - private Mutable(PrivateIonTextWriterBuilder that) - { - super(that); - } - - @Override - public PrivateIonTextWriterBuilder immutable() - { - return new PrivateIonTextWriterBuilder(this); - } - - @Override - public PrivateIonTextWriterBuilder mutable() - { - return this; - } - - @Override - protected void mutationCheck() - { - } - } -} diff --git a/src/software/amazon/ion/impl/PrivateReaderWriter.java b/src/software/amazon/ion/impl/PrivateReaderWriter.java deleted file mode 100644 index 5b6b4031bf..0000000000 --- a/src/software/amazon/ion/impl/PrivateReaderWriter.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at: - * - * http://aws.amazon.com/apache2.0/ - * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. - */ - -package software.amazon.ion.impl; - -import software.amazon.ion.SymbolTable; - -/** - * @deprecated This is an internal API that is subject to change without notice. - */ -@Deprecated -public interface PrivateReaderWriter -{ - public SymbolTable pop_passed_symbol_table(); -} - diff --git a/src/software/amazon/ion/impl/SymbolTokenImpl.java b/src/software/amazon/ion/impl/SymbolTokenImpl.java deleted file mode 100644 index bd0fc11b41..0000000000 --- a/src/software/amazon/ion/impl/SymbolTokenImpl.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at: - * - * http://aws.amazon.com/apache2.0/ - * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. - */ - -package software.amazon.ion.impl; - -import static software.amazon.ion.util.IonTextUtils.printString; - -import software.amazon.ion.SymbolToken; -import software.amazon.ion.UnknownSymbolException; - - -final class SymbolTokenImpl - implements SymbolToken -{ - private final String myText; - private final int mySid; - - SymbolTokenImpl(String text, int sid) - { - assert text != null || sid >= 0 : "Neither text nor sid is defined"; - - myText = text; - mySid = sid; - } - - SymbolTokenImpl(int sid) - { - assert sid >= 0 : "sid is undefined"; - - myText = null; - mySid = sid; - } - - - public String getText() - { - return myText; - } - - public String assumeText() - { - if (myText == null) throw new UnknownSymbolException(mySid); - return myText; - } - - public int getSid() - { - return mySid; - } - - @Override - public String toString() - { - String text = (myText == null ? null : printString(myText)); - return "SymbolToken::{text:" + text + ",id:" + mySid + "}"; - } -} diff --git a/src/software/amazon/ion/impl/bin/PrivateIonHashTrampoline.java b/src/software/amazon/ion/impl/bin/PrivateIonHashTrampoline.java deleted file mode 100644 index 7440807e22..0000000000 --- a/src/software/amazon/ion/impl/bin/PrivateIonHashTrampoline.java +++ /dev/null @@ -1,29 +0,0 @@ -package software.amazon.ion.impl.bin; - -import software.amazon.ion.IonWriter; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; - -/** - * NOT FOR APPLICATION USE! - * - * Exposes {@link IonRawBinaryWriter} functionality for use when creating Ion hashes. - */ -@Deprecated -public final class PrivateIonHashTrampoline -{ - public static IonWriter newIonWriter(ByteArrayOutputStream baos) throws IOException - { - return new IonRawBinaryWriter( - new PooledBlockAllocatorProvider(), - PrivateIonManagedBinaryWriterBuilder.DEFAULT_BLOCK_SIZE, - baos, - AbstractIonWriter.WriteValueOptimization.NONE, - IonRawBinaryWriter.StreamCloseMode.CLOSE, - IonRawBinaryWriter.StreamFlushMode.FLUSH, - IonRawBinaryWriter.PreallocationMode.PREALLOCATE_0, - false // force floats to be encoded as binary64 - ); - } -} diff --git a/src/software/amazon/ion/impl/bin/package-info.java b/src/software/amazon/ion/impl/bin/package-info.java deleted file mode 100644 index 8e2cade484..0000000000 --- a/src/software/amazon/ion/impl/bin/package-info.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2016-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at: - * - * http://aws.amazon.com/apache2.0/ - * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. - */ - -/** - * Provides the implementation for the second-generation Ion binary implementation. - * At this time, this is limited to a binary {@link software.amazon.ion.IonWriter}. - * - *

    - * This package limits most of its APIs to package-level access, the public API of note is contained within - * the {@link software.amazon.ion.impl.bin.PrivateIonManagedBinaryWriterBuilder} which builds instances of - * {@link software.amazon.ion.impl.bin.IonManagedBinaryWriter}. See the below section for what Managed means - * in this context. - * - *

    Block API

    - * A generalized interface for blocks of heap memory are provided via the {@link software.amazon.ion.impl.bin.Block} API. - * There are two factory type APIs to actually get a {@link software.amazon.ion.impl.bin.Block} instance: - * {@link software.amazon.ion.impl.bin.BlockAllocator} which vend blocks of a particular fixed size - * and {@link software.amazon.ion.impl.bin.BlockAllocatorProvider} which creates {@link software.amazon.ion.impl.bin.BlockAllocator} - * instances. - *

    - * The primary reason for this level of indirection is flexibility for the underlying implementations of {@link software.amazon.ion.impl.bin.Block} - * and {@link software.amazon.ion.impl.bin.BlockAllocator}. These APIs are not required to be thread-safe, whereas - * {@link software.amazon.ion.impl.bin.BlockAllocatorProvider} is required to be thread-safe. - *

    - * The APIs for {@link software.amazon.ion.impl.bin.BlockAllocator} and {@link software.amazon.ion.impl.bin.Block} - * follow the resource pattern (similar in principle to I/O streams), and should be closed when no longer needed - * to allow implementation resources to be released or re-used. - * - *

    Raw Binary Ion Writer

    - * The {@link software.amazon.ion.impl.bin.IonRawBinaryWriter} deals with the low-level encoding considerations of the - * Ion format. The {@link software.amazon.ion.impl.bin.WriteBuffer} is used closely with this implementation to - * deal with the Ion sub-field encodings (e.g. VarInt, VarUInt, and UTF-8). - * - *

    Managed Binary Ion Writer

    - * The {@link software.amazon.ion.impl.bin.IonManagedBinaryWriter} is layered on top of the {@link software.amazon.ion.impl.bin.IonRawBinaryWriter}. - * In particular, it intercepts symbol, annotation, field names and handles the mechanics of symbol table management - * transparently to the user. - */ -package software.amazon.ion.impl.bin; \ No newline at end of file diff --git a/src/software/amazon/ion/package-info.java b/src/software/amazon/ion/package-info.java deleted file mode 100644 index cdd31421bc..0000000000 --- a/src/software/amazon/ion/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at: - * - * http://aws.amazon.com/apache2.0/ - * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. - */ - -/** - * Public interfaces of the core Ion system. - */ -package software.amazon.ion; diff --git a/src/software/amazon/ion/system/package-info.java b/src/software/amazon/ion/system/package-info.java deleted file mode 100644 index 51df4ab8c2..0000000000 --- a/src/software/amazon/ion/system/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at: - * - * http://aws.amazon.com/apache2.0/ - * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. - */ - -/** - * Public implementation of the core Ion system. - */ -package software.amazon.ion.system; diff --git a/src/software/amazon/ion/util/package-info.java b/src/software/amazon/ion/util/package-info.java deleted file mode 100644 index 07c2d717e7..0000000000 --- a/src/software/amazon/ion/util/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at: - * - * http://aws.amazon.com/apache2.0/ - * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. - */ - -/** - * Various utilites for working with Ion data. - */ -package software.amazon.ion.util; diff --git a/test/AllTests.java b/test/AllTests.java index 609b86e896..326611b69a 100644 --- a/test/AllTests.java +++ b/test/AllTests.java @@ -1,97 +1,107 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ +import com.amazon.ion.AnnotationEscapesTest; +import com.amazon.ion.AssertionsEnabledTest; +import com.amazon.ion.BadIonTest; +import com.amazon.ion.BinaryReaderWrappedValueLengthTest; +import com.amazon.ion.BinaryTest; +import com.amazon.ion.BlobTest; +import com.amazon.ion.BoolTest; +import com.amazon.ion.ClobTest; +import com.amazon.ion.CloneTest; +import com.amazon.ion.DatagramTest; +import com.amazon.ion.DecimalTest; +import com.amazon.ion.EquivTimelineTest; +import com.amazon.ion.EquivsTest; +import com.amazon.ion.ExtendedDecimalTest; +import com.amazon.ion.FieldNameEscapesTest; +import com.amazon.ion.FloatTest; +import com.amazon.ion.GoodIonTest; +import com.amazon.ion.HashCodeCorrectnessTest; +import com.amazon.ion.HashCodeDeltaCollisionTest; +import com.amazon.ion.HashCodeDistributionTest; +import com.amazon.ion.IntTest; +import com.amazon.ion.IonExceptionTest; +import com.amazon.ion.IonRawWriterBasicTest; +import com.amazon.ion.IonRawWriterSymbolsTest; +import com.amazon.ion.IonReaderToIonValueTest; +import com.amazon.ion.IonSystemTest; +import com.amazon.ion.IonValueTest; +import com.amazon.ion.JavaNumericsTest; +import com.amazon.ion.ListTest; +import com.amazon.ion.LoaderTest; +import com.amazon.ion.LongStringTest; +import com.amazon.ion.NonEquivsTest; +import com.amazon.ion.NopPaddingTest; +import com.amazon.ion.NullTest; +import com.amazon.ion.RawValueSpanReaderBasicTest; +import com.amazon.ion.impl.RawValueSpanReaderTest; +import com.amazon.ion.RoundTripTest; +import com.amazon.ion.SexpTest; +import com.amazon.ion.StringFieldNameEscapesTest; +import com.amazon.ion.StringTest; +import com.amazon.ion.StructTest; +import com.amazon.ion.SurrogateEscapeTest; +import com.amazon.ion.SymbolTest; +import com.amazon.ion.SystemProcessingTests; +import com.amazon.ion.TimestampBadTest; +import com.amazon.ion.TimestampGoodTest; +import com.amazon.ion.TimestampTest; +import com.amazon.ion.ValueFactorySequenceTest; +import com.amazon.ion.facet.FacetsTest; +import com.amazon.ion.impl.ByteBufferTest; +import com.amazon.ion.impl.CharacterReaderTest; +import com.amazon.ion.impl.IonImplUtilsTest; +import com.amazon.ion.impl.IonMarkupWriterFilesTest; +import com.amazon.ion.impl.IonMarkupWriterTest; +import com.amazon.ion.impl.IonWriterTests; +import com.amazon.ion.impl.IterationTest; +import com.amazon.ion.impl.LocalSymbolTableTest; +import com.amazon.ion.impl.SharedSymbolTableTest; +import com.amazon.ion.impl.SymbolTableTest; +import com.amazon.ion.impl.TreeReaderTest; +import com.amazon.ion.impl.bin.IonManagedBinaryWriterTest; +import com.amazon.ion.impl.bin.IonRawBinaryWriterTest; +import com.amazon.ion.impl.bin.PooledBlockAllocatorProviderTest; +import com.amazon.ion.impl.bin.WriteBufferTest; +import com.amazon.ion.impl.lite.IonContextTest; +import com.amazon.ion.impl.lite.SIDPresentLifecycleTest; +import com.amazon.ion.streaming.BadIonStreamingTest; +import com.amazon.ion.streaming.BinaryStreamingTest; +import com.amazon.ion.streaming.GoodIonStreamingTest; +import com.amazon.ion.streaming.InputStreamReaderTest; +import com.amazon.ion.streaming.MiscStreamingTest; +import com.amazon.ion.streaming.ReaderDomCopyTest; +import com.amazon.ion.streaming.ReaderIntegerSizeTest; +import com.amazon.ion.streaming.ReaderSkippingTest; +import com.amazon.ion.streaming.ReaderTest; +import com.amazon.ion.streaming.RoundTripStreamingTest; +import com.amazon.ion.streaming.SpanTests; +import com.amazon.ion.system.IonBinaryWriterBuilderTest; +import com.amazon.ion.system.IonReaderBuilderTest; +import com.amazon.ion.system.IonSystemBuilderTest; +import com.amazon.ion.system.IonTextWriterBuilderTest; +import com.amazon.ion.system.SimpleCatalogTest; +import com.amazon.ion.util.EquivalenceTest; +import com.amazon.ion.util.IonStreamUtilsTest; +import com.amazon.ion.util.JarInfoTest; +import com.amazon.ion.util.PrinterTest; +import com.amazon.ion.util.TextTest; import org.junit.runner.RunWith; import org.junit.runners.Suite; -import software.amazon.ion.AnnotationEscapesTest; -import software.amazon.ion.AssertionsEnabledTest; -import software.amazon.ion.BadIonTest; -import software.amazon.ion.BinaryReaderWrappedValueLengthTest; -import software.amazon.ion.BinaryTest; -import software.amazon.ion.BlobTest; -import software.amazon.ion.BoolTest; -import software.amazon.ion.ClobTest; -import software.amazon.ion.CloneTest; -import software.amazon.ion.DatagramTest; -import software.amazon.ion.DecimalTest; -import software.amazon.ion.EquivTimelineTest; -import software.amazon.ion.EquivsTest; -import software.amazon.ion.ExtendedDecimalTest; -import software.amazon.ion.FieldNameEscapesTest; -import software.amazon.ion.FloatTest; -import software.amazon.ion.GoodIonTest; -import software.amazon.ion.HashCodeCorrectnessTest; -import software.amazon.ion.HashCodeDeltaCollisionTest; -import software.amazon.ion.HashCodeDistributionTest; -import software.amazon.ion.IntTest; -import software.amazon.ion.IonExceptionTest; -import software.amazon.ion.IonReaderToIonValueTest; -import software.amazon.ion.IonSystemTest; -import software.amazon.ion.IonValueTest; -import software.amazon.ion.JavaNumericsTest; -import software.amazon.ion.ListTest; -import software.amazon.ion.LoaderTest; -import software.amazon.ion.LongStringTest; -import software.amazon.ion.NonEquivsTest; -import software.amazon.ion.NopPaddingTest; -import software.amazon.ion.NullTest; -import software.amazon.ion.RoundTripTest; -import software.amazon.ion.SexpTest; -import software.amazon.ion.StringFieldNameEscapesTest; -import software.amazon.ion.StringTest; -import software.amazon.ion.StructTest; -import software.amazon.ion.SurrogateEscapeTest; -import software.amazon.ion.SymbolTest; -import software.amazon.ion.SystemProcessingTests; -import software.amazon.ion.TimestampBadTest; -import software.amazon.ion.TimestampGoodTest; -import software.amazon.ion.TimestampTest; -import software.amazon.ion.ValueFactorySequenceTest; -import software.amazon.ion.facet.FacetsTest; -import software.amazon.ion.impl.IonImplUtilsTest; -import software.amazon.ion.impl.IonWriterTests; -import software.amazon.ion.impl.IterationTest; -import software.amazon.ion.impl.LocalSymbolTableTest; -import software.amazon.ion.impl.SharedSymbolTableTest; -import software.amazon.ion.impl.SymbolTableTest; -import software.amazon.ion.impl.TreeReaderTest; -import software.amazon.ion.impl.bin.IonManagedBinaryWriterTest; -import software.amazon.ion.impl.bin.IonRawBinaryWriterTest; -import software.amazon.ion.impl.bin.PooledBlockAllocatorProviderTest; -import software.amazon.ion.impl.bin.WriteBufferTest; -import software.amazon.ion.impl.lite.IonContextTest; -import software.amazon.ion.impl.lite.SIDPresentLifecycleTest; -import software.amazon.ion.streaming.BadIonStreamingTest; -import software.amazon.ion.streaming.BinaryStreamingTest; -import software.amazon.ion.streaming.GoodIonStreamingTest; -import software.amazon.ion.streaming.InputStreamReaderTest; -import software.amazon.ion.streaming.MiscStreamingTest; -import software.amazon.ion.streaming.ReaderDomCopyTest; -import software.amazon.ion.streaming.ReaderIntegerSizeTest; -import software.amazon.ion.streaming.ReaderSkippingTest; -import software.amazon.ion.streaming.ReaderTest; -import software.amazon.ion.streaming.RoundTripStreamingTest; -import software.amazon.ion.streaming.SpanTests; -import software.amazon.ion.system.IonBinaryWriterBuilderTest; -import software.amazon.ion.system.IonReaderBuilderTest; -import software.amazon.ion.system.IonSystemBuilderTest; -import software.amazon.ion.system.IonTextWriterBuilderTest; -import software.amazon.ion.system.SimpleCatalogTest; -import software.amazon.ion.util.EquivalenceTest; -import software.amazon.ion.util.IonStreamUtilsTest; -import software.amazon.ion.util.JarInfoTest; -import software.amazon.ion.util.TextTest; /** @@ -103,7 +113,9 @@ AssertionsEnabledTest.class, IonExceptionTest.class, FacetsTest.class, + ByteBufferTest.class, TextTest.class, + CharacterReaderTest.class, JavaNumericsTest.class, ExtendedDecimalTest.class, IonImplUtilsTest.class, @@ -137,6 +149,10 @@ StringFieldNameEscapesTest.class, SurrogateEscapeTest.class, + // Markup tests + IonMarkupWriterTest.class, + IonMarkupWriterFilesTest.class, + NopPaddingTest.class, // Binary format tests @@ -146,6 +162,7 @@ JarInfoTest.class, LoaderTest.class, IterationTest.class, + PrinterTest.class, SymbolTableTest.class, SharedSymbolTableTest.class, LocalSymbolTableTest.class, @@ -201,9 +218,13 @@ HashCodeDistributionTest.class, HashCodeDeltaCollisionTest.class, + IonRawWriterBasicTest.class, + IonRawWriterSymbolsTest.class, + RawValueSpanReaderBasicTest.class, + RawValueSpanReaderTest.class, + // DOM Lifecycle / mode tests - SIDPresentLifecycleTest.class, - HashCodeDeltaCollisionTest.class + SIDPresentLifecycleTest.class }) public class AllTests { diff --git a/test/software/amazon/ion/AnnotationEscapesTest.java b/test/com/amazon/ion/AnnotationEscapesTest.java similarity index 55% rename from test/software/amazon/ion/AnnotationEscapesTest.java rename to test/com/amazon/ion/AnnotationEscapesTest.java index b68ea239e6..672c3a4a5c 100644 --- a/test/software/amazon/ion/AnnotationEscapesTest.java +++ b/test/com/amazon/ion/AnnotationEscapesTest.java @@ -1,20 +1,20 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import software.amazon.ion.IonValue; public class AnnotationEscapesTest extends TextTestCase diff --git a/test/software/amazon/ion/AssertionsEnabledTest.java b/test/com/amazon/ion/AssertionsEnabledTest.java similarity index 66% rename from test/software/amazon/ion/AssertionsEnabledTest.java rename to test/com/amazon/ion/AssertionsEnabledTest.java index ee782d54fb..786cd8eec0 100644 --- a/test/software/amazon/ion/AssertionsEnabledTest.java +++ b/test/com/amazon/ion/AssertionsEnabledTest.java @@ -1,24 +1,26 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import org.junit.Test; + public class AssertionsEnabledTest { /** diff --git a/test/software/amazon/ion/BadIonTest.java b/test/com/amazon/ion/BadIonTest.java similarity index 75% rename from test/software/amazon/ion/BadIonTest.java rename to test/com/amazon/ion/BadIonTest.java index 326066842a..28b3602e2c 100644 --- a/test/software/amazon/ion/BadIonTest.java +++ b/test/com/amazon/ion/BadIonTest.java @@ -1,22 +1,26 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; -import static software.amazon.ion.TestUtils.BAD_IONTESTS_FILES; -import static software.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; -import static software.amazon.ion.TestUtils.testdataFiles; +package com.amazon.ion; +import static com.amazon.ion.TestUtils.BAD_IONTESTS_FILES; +import static com.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; +import static com.amazon.ion.TestUtils.testdataFiles; + +import com.amazon.ion.impl._Private_Utils; +import com.amazon.ion.junit.Injected.Inject; import java.io.File; import java.io.FileReader; import java.io.IOException; @@ -24,8 +28,6 @@ import java.nio.charset.Charset; import java.nio.charset.CharsetEncoder; import org.junit.Test; -import software.amazon.ion.impl.PrivateUtils; -import software.amazon.ion.junit.Injected.Inject; public class BadIonTest @@ -73,7 +75,7 @@ public void testLoadString() { // This will fail if the file has bad UTF-8 data. // That's OK, the other test will still do the right thing. - ionText = PrivateUtils.utf8FileToString(myTestFile); + ionText = _Private_Utils.utf8FileToString(myTestFile); } catch (IonException e) { diff --git a/test/software/amazon/ion/BinaryByteArrayIteratorSystemProcessingTest.java b/test/com/amazon/ion/BinaryByteArrayIteratorSystemProcessingTest.java similarity index 66% rename from test/software/amazon/ion/BinaryByteArrayIteratorSystemProcessingTest.java rename to test/com/amazon/ion/BinaryByteArrayIteratorSystemProcessingTest.java index 7602be6b37..1bc2fac1a7 100644 --- a/test/software/amazon/ion/BinaryByteArrayIteratorSystemProcessingTest.java +++ b/test/com/amazon/ion/BinaryByteArrayIteratorSystemProcessingTest.java @@ -1,21 +1,21 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.util.Iterator; -import software.amazon.ion.IonValue; public class BinaryByteArrayIteratorSystemProcessingTest diff --git a/test/software/amazon/ion/BinaryReaderSystemProcessingTest.java b/test/com/amazon/ion/BinaryReaderSystemProcessingTest.java similarity index 63% rename from test/software/amazon/ion/BinaryReaderSystemProcessingTest.java rename to test/com/amazon/ion/BinaryReaderSystemProcessingTest.java index a07a0d6259..c1894984bb 100644 --- a/test/software/amazon/ion/BinaryReaderSystemProcessingTest.java +++ b/test/com/amazon/ion/BinaryReaderSystemProcessingTest.java @@ -1,22 +1,22 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; + + -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonLoader; -import software.amazon.ion.IonReader; public class BinaryReaderSystemProcessingTest extends ReaderSystemProcessingTestCase diff --git a/test/software/amazon/ion/BinaryReaderWrappedValueLengthTest.java b/test/com/amazon/ion/BinaryReaderWrappedValueLengthTest.java similarity index 76% rename from test/software/amazon/ion/BinaryReaderWrappedValueLengthTest.java rename to test/com/amazon/ion/BinaryReaderWrappedValueLengthTest.java index 6b870bf72b..c1c5324abd 100644 --- a/test/software/amazon/ion/BinaryReaderWrappedValueLengthTest.java +++ b/test/com/amazon/ion/BinaryReaderWrappedValueLengthTest.java @@ -1,29 +1,27 @@ /* - * Copyright 2016-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; +import static com.amazon.ion.BitUtils.bytes; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; -import static software.amazon.ion.BitUtils.bytes; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import software.amazon.ion.IonException; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonType; public class BinaryReaderWrappedValueLengthTest diff --git a/test/software/amazon/ion/BinaryStreamIteratorSystemProcessingTest.java b/test/com/amazon/ion/BinaryStreamIteratorSystemProcessingTest.java similarity index 69% rename from test/software/amazon/ion/BinaryStreamIteratorSystemProcessingTest.java rename to test/com/amazon/ion/BinaryStreamIteratorSystemProcessingTest.java index 9188d92388..a0986d1201 100644 --- a/test/software/amazon/ion/BinaryStreamIteratorSystemProcessingTest.java +++ b/test/com/amazon/ion/BinaryStreamIteratorSystemProcessingTest.java @@ -1,23 +1,23 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.util.Iterator; -import software.amazon.ion.IonValue; public class BinaryStreamIteratorSystemProcessingTest diff --git a/test/software/amazon/ion/BinaryTest.java b/test/com/amazon/ion/BinaryTest.java similarity index 91% rename from test/software/amazon/ion/BinaryTest.java rename to test/com/amazon/ion/BinaryTest.java index bde270f1e7..e7641ae01f 100644 --- a/test/software/amazon/ion/BinaryTest.java +++ b/test/com/amazon/ion/BinaryTest.java @@ -1,27 +1,24 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; import java.util.Arrays; import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonFloat; -import software.amazon.ion.IonNull; -import software.amazon.ion.IonValue; public class BinaryTest extends IonTestCase { diff --git a/test/software/amazon/ion/BitUtils.java b/test/com/amazon/ion/BitUtils.java similarity index 74% rename from test/software/amazon/ion/BitUtils.java rename to test/com/amazon/ion/BitUtils.java index fc54acca62..c9a1ce72cf 100644 --- a/test/software/amazon/ion/BitUtils.java +++ b/test/com/amazon/ion/BitUtils.java @@ -1,18 +1,19 @@ /* - * Copyright 2016-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; public class BitUtils { diff --git a/test/software/amazon/ion/BlobTest.java b/test/com/amazon/ion/BlobTest.java similarity index 92% rename from test/software/amazon/ion/BlobTest.java rename to test/com/amazon/ion/BlobTest.java index c19a8c978f..f39fcc210b 100644 --- a/test/software/amazon/ion/BlobTest.java +++ b/test/com/amazon/ion/BlobTest.java @@ -1,30 +1,26 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; - -import static software.amazon.ion.TestUtils.US_ASCII_CHARSET; -import static software.amazon.ion.impl.PrivateUtils.encode; +package com.amazon.ion; +import static com.amazon.ion.TestUtils.US_ASCII_CHARSET; +import static com.amazon.ion.impl._Private_Utils.encode; +import com.amazon.ion.impl._Private_Utils; import java.io.IOException; import java.io.InputStream; import org.junit.Test; -import software.amazon.ion.IonBlob; -import software.amazon.ion.IonType; -import software.amazon.ion.NullValueException; -import software.amazon.ion.UnexpectedEofException; -import software.amazon.ion.impl.PrivateUtils; public class BlobTest @@ -219,7 +215,7 @@ public TestData(int[] bytes, String base64) private static byte[] EncodeAscii(String ascii) { - return PrivateUtils.encode(ascii, US_ASCII_CHARSET); + return _Private_Utils.encode(ascii, US_ASCII_CHARSET); } diff --git a/test/software/amazon/ion/BoolTest.java b/test/com/amazon/ion/BoolTest.java similarity index 86% rename from test/software/amazon/ion/BoolTest.java rename to test/com/amazon/ion/BoolTest.java index 3966352782..208c8724ae 100644 --- a/test/software/amazon/ion/BoolTest.java +++ b/test/com/amazon/ion/BoolTest.java @@ -1,23 +1,21 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; + +package com.amazon.ion; import org.junit.Test; -import software.amazon.ion.IonBool; -import software.amazon.ion.IonType; -import software.amazon.ion.NullValueException; -import software.amazon.ion.ReadOnlyValueException; diff --git a/test/software/amazon/ion/Checker.java b/test/com/amazon/ion/Checker.java similarity index 65% rename from test/software/amazon/ion/Checker.java rename to test/com/amazon/ion/Checker.java index d27ab1093b..7e41ffa178 100644 --- a/test/software/amazon/ion/Checker.java +++ b/test/com/amazon/ion/Checker.java @@ -1,18 +1,20 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; + public interface Checker { diff --git a/test/software/amazon/ion/ClobTest.java b/test/com/amazon/ion/ClobTest.java similarity index 88% rename from test/software/amazon/ion/ClobTest.java rename to test/com/amazon/ion/ClobTest.java index 93bdbea901..07ef4ab300 100644 --- a/test/software/amazon/ion/ClobTest.java +++ b/test/com/amazon/ion/ClobTest.java @@ -1,30 +1,28 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static software.amazon.ion.impl.PrivateUtils.UTF8_CHARSET; +import static com.amazon.ion.impl._Private_Utils.UTF8_CHARSET; + +import com.amazon.ion.impl._Private_Utils; import java.io.IOException; import java.io.InputStream; import java.io.Reader; import org.junit.Test; -import software.amazon.ion.IonClob; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.NullValueException; -import software.amazon.ion.impl.PrivateUtils; @@ -33,7 +31,7 @@ public class ClobTest { public static final String SAMPLE_ASCII = "Wow!"; public static final byte[] SAMPLE_ASCII_AS_UTF8 = - PrivateUtils.utf8(SAMPLE_ASCII); + _Private_Utils.utf8(SAMPLE_ASCII); public void checkNullClob(IonClob value) diff --git a/test/software/amazon/ion/CloneTest.java b/test/com/amazon/ion/CloneTest.java similarity index 80% rename from test/software/amazon/ion/CloneTest.java rename to test/com/amazon/ion/CloneTest.java index 0e94379515..b9cb52d253 100644 --- a/test/software/amazon/ion/CloneTest.java +++ b/test/com/amazon/ion/CloneTest.java @@ -1,32 +1,26 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static software.amazon.ion.impl.PrivateUtils.newSymbolToken; +import static com.amazon.ion.impl._Private_Utils.newSymbolToken; +import com.amazon.ion.system.SimpleCatalog; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonValue; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.UnknownSymbolException; -import software.amazon.ion.system.SimpleCatalog; public class CloneTest @@ -66,7 +60,7 @@ public void testDifferentValueFactoryCloneWithUnknownSymbolText() SymbolToken tok = newSymbolToken(99); IonSymbol original = system().newSymbol(tok); - // TODO amzn/ion-java#30 An UnknownSymbolException is expected here, but + // TODO amzn/ion-java/issues/30 An UnknownSymbolException is expected here, but // it isn't thrown. IonSymbol copy = otherSystem.clone(original); @@ -105,7 +99,7 @@ public void testDifferentValueFactoryCloneWithUnknownAnnotationText() IonInt original = system().newInt(5); original.setTypeAnnotationSymbols(tok); - // TODO amzn/ion-java#30 An UnknownSymbolException is expected here, but + // TODO amzn/ion-java/issues/30 An UnknownSymbolException is expected here, but // it isn't thrown. IonInt copy = otherSystem.clone(original); @@ -159,7 +153,7 @@ public void testDifferentValueFactoryCloneWithUnknownFieldNameText() // This works since the cloned child doesn't retain its field name. otherSystem.clone(child); - // TODO amzn/ion-java#30 An UnknownSymbolException is expected here, but + // TODO amzn/ion-java/issues/30 An UnknownSymbolException is expected here, but // it isn't thrown. IonStruct copy = otherSystem.clone(original); diff --git a/test/software/amazon/ion/ContainerTestCase.java b/test/com/amazon/ion/ContainerTestCase.java similarity index 90% rename from test/software/amazon/ion/ContainerTestCase.java rename to test/com/amazon/ion/ContainerTestCase.java index ad3b7ff1c1..6ca1b7626a 100644 --- a/test/software/amazon/ion/ContainerTestCase.java +++ b/test/com/amazon/ion/ContainerTestCase.java @@ -1,37 +1,24 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; +import com.amazon.ion.system.SimpleCatalog; import java.util.Iterator; import org.junit.Ignore; import org.junit.Test; -import software.amazon.ion.IonContainer; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonException; -import software.amazon.ion.IonList; -import software.amazon.ion.IonNull; -import software.amazon.ion.IonString; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.NullValueException; -import software.amazon.ion.ReadOnlyValueException; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.system.SimpleCatalog; @@ -255,7 +242,7 @@ public void testRemoveFromContainer() } - /** TODO amzn/ion-java#8 datagram is lazy creating local symtabs */ + /** TODO amzn/ion-java/issues/8 datagram is lazy creating local symtabs */ @Test @Ignore public void testDetachHasDifferentSymtab() { @@ -365,7 +352,7 @@ public void testRemovingReadOnlyChild() assertSame(n, c.iterator().next()); } - /** TODO amzn/ion-java#6 */ + /** TODO amzn/ion-java/issues/6 */ @Test @Ignore public void testSelfContainment() { diff --git a/test/software/amazon/ion/DatagramIteratorSystemProcessingTest.java b/test/com/amazon/ion/DatagramIteratorSystemProcessingTest.java similarity index 56% rename from test/software/amazon/ion/DatagramIteratorSystemProcessingTest.java rename to test/com/amazon/ion/DatagramIteratorSystemProcessingTest.java index da4da769c4..5831dd9c42 100644 --- a/test/software/amazon/ion/DatagramIteratorSystemProcessingTest.java +++ b/test/com/amazon/ion/DatagramIteratorSystemProcessingTest.java @@ -1,13 +1,22 @@ /* - * Copyright (c) 2008 Amazon.com, Inc. All rights reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.util.Iterator; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonLoader; -import software.amazon.ion.IonValue; + public class DatagramIteratorSystemProcessingTest extends IteratorSystemProcessingTestCase diff --git a/test/software/amazon/ion/DatagramMaker.java b/test/com/amazon/ion/DatagramMaker.java similarity index 80% rename from test/software/amazon/ion/DatagramMaker.java rename to test/com/amazon/ion/DatagramMaker.java index 552fd91aca..7f2ef82f05 100644 --- a/test/software/amazon/ion/DatagramMaker.java +++ b/test/com/amazon/ion/DatagramMaker.java @@ -1,28 +1,26 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static software.amazon.ion.TestUtils.ensureBinary; -import static software.amazon.ion.TestUtils.ensureText; +import static com.amazon.ion.TestUtils.ensureBinary; +import static com.amazon.ion.TestUtils.ensureText; +import com.amazon.ion.impl._Private_Utils; import java.util.ArrayList; import java.util.Arrays; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonLoader; -import software.amazon.ion.IonSystem; -import software.amazon.ion.impl.PrivateUtils; /** * Abstracts the various ways that {@link IonDatagram}s can be created, so test @@ -97,7 +95,7 @@ public boolean sourceIsBinary() public IonDatagram newDatagram(IonSystem system, String ionText) { - byte[] utf8 = PrivateUtils.utf8(ionText); + byte[] utf8 = _Private_Utils.utf8(ionText); return newDatagram(system, utf8); } diff --git a/test/software/amazon/ion/DatagramTest.java b/test/com/amazon/ion/DatagramTest.java similarity index 88% rename from test/software/amazon/ion/DatagramTest.java rename to test/com/amazon/ion/DatagramTest.java index 6491d6fd97..9193cfabac 100644 --- a/test/software/amazon/ion/DatagramTest.java +++ b/test/com/amazon/ion/DatagramTest.java @@ -1,51 +1,37 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; -import static software.amazon.ion.SystemSymbols.ION_1_0; -import static software.amazon.ion.SystemSymbols.ION_1_0_SID; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; -import static software.amazon.ion.SystemSymbols.SYMBOLS; -import static software.amazon.ion.impl.Symtabs.FRED_MAX_IDS; -import static software.amazon.ion.junit.IonAssert.assertIonEquals; +import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; +import static com.amazon.ion.SystemSymbols.ION_1_0; +import static com.amazon.ion.SystemSymbols.ION_1_0_SID; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; +import static com.amazon.ion.SystemSymbols.SYMBOLS; +import static com.amazon.ion.impl.Symtabs.FRED_MAX_IDS; +import static com.amazon.ion.junit.IonAssert.assertIonEquals; +import com.amazon.ion.impl.Symtabs; +import com.amazon.ion.impl._Private_IonSystem; +import com.amazon.ion.impl._Private_IonValue; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Collection; import java.util.Iterator; import org.junit.Before; import org.junit.Test; -import software.amazon.ion.IonBool; -import software.amazon.ion.IonContainer; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonList; -import software.amazon.ion.IonLoader; -import software.amazon.ion.IonNull; -import software.amazon.ion.IonSequence; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.ReadOnlyValueException; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.impl.PrivateIonSystem; -import software.amazon.ion.impl.PrivateIonValue; -import software.amazon.ion.impl.Symtabs; public class DatagramTest @@ -144,7 +130,7 @@ public IonDatagram roundTrip(String text) public void testAutomaticIVM() throws Exception { - PrivateIonSystem system = system(); + _Private_IonSystem system = system(); SymbolTable systemSymtab_1_0 = system.getSystemSymbolTable(ION_1_0); IonDatagram dg = system.newDatagram(); @@ -165,7 +151,7 @@ public void testAutomaticIVM() public void testManualIVM() throws Exception { - PrivateIonSystem system = system(); + _Private_IonSystem system = system(); SymbolTable systemSymtab_1_0 = system.getSystemSymbolTable(ION_1_0); IonDatagram dg = system.newDatagram(); @@ -178,7 +164,7 @@ public void testManualIVM() assertSame(systemSymtab_1_0, sysId.getSymbolTable()); - // TODO amzn/ion-java#20 + // TODO amzn/ion-java/issues/20 // assertEquals(0, dg.size()); // TODO adding $ion_1_1 should fail: unsupported version @@ -225,7 +211,7 @@ public void testBinaryData() assertSame(symbol, datagram1.get(0)); SymbolTable symtab = symbol.getSymbolTable(); - assertEquals(symbol.symbolValue().getSid(), symtab.getMaxId()); + assertEquals(symbol.getSymbolId(), symtab.getMaxId()); // TODO if we keep max_id in the struct, should validate it here. } @@ -280,6 +266,34 @@ public void testGetBytes() // check strict data equivalence IonDatagram dg2 = myLoader.load(bytes2); assertIonEquals(dg, dg2); + + // now check extraction into sub-array + final int OFFSET = 5; + assertTrue(bytes1.length > OFFSET); + bytes2 = new byte[size + OFFSET]; + + outLen = dg.getBytes(bytes2, OFFSET); + assertEquals(size, outLen); + + for (int i = 0; i < bytes1.length; i++) + { + if (bytes1[i] != bytes2[i + OFFSET]) + { + fail("Binary data differs at index " + i); + } + } + + try { + dg.getBytes(new byte[3]); + fail("Expected IndexOutOfBoundsException"); + } + catch (IndexOutOfBoundsException e) { /* good */ } + + try { + dg.getBytes(new byte[size + OFFSET - 1], OFFSET); + fail("Expected IndexOutOfBoundsException"); + } + catch (IndexOutOfBoundsException e) { /* good */ } } @@ -401,7 +415,7 @@ public void testAddingDatagramToDatagram() } catch (IllegalArgumentException e) { } - // Cannot insert a datagram // TODO amzn/ion-java#48 + // Cannot insert a datagram // TODO amzn/ion-java/issues/48 // try // { // dg1.add(1, dg2); @@ -484,7 +498,7 @@ public void testNewDatagramWithImports() public void testEmptyDatagram() { IonDatagram dg = loader().load(""); -// testEmptySequence(dg); // TODO amzn/ion-java#48 implement add(int,v) +// testEmptySequence(dg); // TODO amzn/ion-java/issues/48 implement add(int,v) dg.add().newInt(1); testClearContainer(dg); } @@ -494,7 +508,7 @@ public void testEmptyDatagram() @Test public void testRemoveViaIteratorThenDirect() { - // TODO amzn/ion-java#51 implement remove on datagram iterator + // TODO amzn/ion-java/issues/51 implement remove on datagram iterator } @@ -556,7 +570,7 @@ public void testClearAfterClone() @Test public void testToString() { - // TODO amzn/ion-java#8 I think this is wrong, the datagram has injected IVM + // TODO amzn/ion-java/issues/8 I think this is wrong, the datagram has injected IVM IonDatagram dg = loader().load("1"); assertEquals("$ion_1_0 1", dg.toString()); @@ -568,7 +582,7 @@ public void testToString() text.endsWith(" {a:b}")); // Just force symtab analysis and make sure output is still okay - dg.getBytes(); + dg.getBytes(new byte[dg.byteSize()]); text = dg.toString(); assertTrue("missing version marker", text.startsWith(ION_1_0 + ' ')); @@ -592,6 +606,7 @@ public void testToStringWithSymbols() dg.getBytes(); // Force encoding and symtab construction String result = dg.toString(); + assertEquals(ION_1_0 + " x", result); } @@ -652,11 +667,11 @@ public void testGetTopLevelValueOfDatagram() public void testGetAssignedSymbolTable() { IonDatagram dg = system().newDatagram(); - ((PrivateIonValue)dg).getAssignedSymbolTable(); + ((_Private_IonValue)dg).getAssignedSymbolTable(); } /** - * TODO amzn/ion-java#50 Datagram.set() should work, but it's documented to throw. + * TODO amzn/ion-java/issues/50 Datagram.set() should work, but it's documented to throw */ @Test(expected = UnsupportedOperationException.class) public void testSet() diff --git a/test/software/amazon/ion/DatagramTreeReaderSystemProcessingTest.java b/test/com/amazon/ion/DatagramTreeReaderSystemProcessingTest.java similarity index 77% rename from test/software/amazon/ion/DatagramTreeReaderSystemProcessingTest.java rename to test/com/amazon/ion/DatagramTreeReaderSystemProcessingTest.java index 98ff79ffcd..f8077abb08 100644 --- a/test/software/amazon/ion/DatagramTreeReaderSystemProcessingTest.java +++ b/test/com/amazon/ion/DatagramTreeReaderSystemProcessingTest.java @@ -1,25 +1,25 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static software.amazon.ion.DatagramMaker.FROM_BYTES_TEXT; +import static com.amazon.ion.DatagramMaker.FROM_BYTES_TEXT; +import com.amazon.ion.junit.Injected.Inject; import org.junit.After; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonReader; -import software.amazon.ion.junit.Injected.Inject; + public class DatagramTreeReaderSystemProcessingTest diff --git a/test/software/amazon/ion/DecimalTest.java b/test/com/amazon/ion/DecimalTest.java similarity index 91% rename from test/software/amazon/ion/DecimalTest.java rename to test/com/amazon/ion/DecimalTest.java index 6d53a15f75..6d79dacfe7 100644 --- a/test/software/amazon/ion/DecimalTest.java +++ b/test/com/amazon/ion/DecimalTest.java @@ -1,25 +1,22 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; + +package com.amazon.ion; import java.math.BigDecimal; import org.junit.Test; -import software.amazon.ion.Decimal; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonDecimal; -import software.amazon.ion.IonType; -import software.amazon.ion.NullValueException; @@ -200,24 +197,24 @@ public void testPrinting() testPrinting("123d0", "123."); testPrinting("123d-1", "12.3"); testPrinting("123d-2", "1.23"); - testPrinting("123d-3", "0.123"); - testPrinting("123d-4", "0.0123"); + testPrinting("123d-3", "1.23d-1"); + testPrinting("123d-4", "1.23d-2"); testPrinting("-123d3", "-123d3"); testPrinting("-123d1", "-123d1"); testPrinting("-123d0", "-123."); testPrinting("-123d-1", "-12.3"); testPrinting("-123d-2", "-1.23"); - testPrinting("-123d-3", "-0.123"); - testPrinting("-123d-4", "-0.0123"); + testPrinting("-123d-3", "-1.23d-1"); + testPrinting("-123d-4", "-1.23d-2"); // Zeros are a bit trickier // testPrinting("0.0", "0.0"); // testPrinting("0.00", "0.00"); testPrinting("0d1", "0d1"); - testPrinting("0d-1", "0.0"); + testPrinting("0d-1", "0d-1"); testPrinting("0d2", "0d2"); - testPrinting("0d-2", "0.00"); // should be 0.00 or 0.0d-1 ? + testPrinting("0d-2", "0d-2"); // should be 0.00 or 0.0d-1 ? } public void testPrinting(String input, String output) diff --git a/test/software/amazon/ion/EquivTimelineTest.java b/test/com/amazon/ion/EquivTimelineTest.java similarity index 75% rename from test/software/amazon/ion/EquivTimelineTest.java rename to test/com/amazon/ion/EquivTimelineTest.java index c0e4ca614e..58a5b78aa1 100644 --- a/test/software/amazon/ion/EquivTimelineTest.java +++ b/test/com/amazon/ion/EquivTimelineTest.java @@ -1,31 +1,28 @@ /* - * Copyright 2013-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static software.amazon.ion.TestUtils.EQUIVS_TIMESTAMP_IONTESTS_FILES; -import static software.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; -import static software.amazon.ion.TestUtils.testdataFiles; +import static com.amazon.ion.TestUtils.EQUIVS_TIMESTAMP_IONTESTS_FILES; +import static com.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; +import static com.amazon.ion.TestUtils.testdataFiles; +import com.amazon.ion.junit.Injected.Inject; import java.io.File; import java.io.IOException; -import software.amazon.ion.IonTimestamp; -import software.amazon.ion.IonValue; -import software.amazon.ion.Timestamp; -import software.amazon.ion.junit.Injected.Inject; - public class EquivTimelineTest extends EquivsTest { diff --git a/test/com/amazon/ion/EquivsTest.java b/test/com/amazon/ion/EquivsTest.java new file mode 100644 index 0000000000..4a295b7dc6 --- /dev/null +++ b/test/com/amazon/ion/EquivsTest.java @@ -0,0 +1,34 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion; + +import static com.amazon.ion.TestUtils.EQUIVS_IONTESTS_FILES; +import static com.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; + +import com.amazon.ion.junit.Injected.Inject; +import java.io.File; + +public class EquivsTest + extends EquivsTestCase +{ + @Inject("testFile") + public static final File[] FILES = TestUtils.testdataFiles(GLOBAL_SKIP_LIST, EQUIVS_IONTESTS_FILES); + + public EquivsTest() + { + super(true); + } +} diff --git a/test/software/amazon/ion/EquivsTestCase.java b/test/com/amazon/ion/EquivsTestCase.java similarity index 90% rename from test/software/amazon/ion/EquivsTestCase.java rename to test/com/amazon/ion/EquivsTestCase.java index 2c3742625b..7bdce6bb9e 100644 --- a/test/software/amazon/ion/EquivsTestCase.java +++ b/test/com/amazon/ion/EquivsTestCase.java @@ -1,31 +1,35 @@ /* - * Copyright 2013-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static software.amazon.ion.IonType.DATAGRAM; +import static com.amazon.ion.IonType.DATAGRAM; +import com.amazon.ion.impl._Private_Utils; +import com.amazon.ion.junit.IonAssert; +import java.io.File; import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonSequence; -import software.amazon.ion.IonString; -import software.amazon.ion.IonValue; -import software.amazon.ion.impl.PrivateUtils; -import software.amazon.ion.junit.IonAssert; -import software.amazon.ion.system.IonBinaryWriterBuilder; -import software.amazon.ion.system.IonSystemBuilder; -import software.amazon.ion.system.IonTextWriterBuilder; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonSequence; +import com.amazon.ion.IonString; +import com.amazon.ion.IonValue; +import com.amazon.ion.impl._Private_Utils; +import com.amazon.ion.junit.IonAssert; +import com.amazon.ion.system.IonBinaryWriterBuilder; +import com.amazon.ion.system.IonSystemBuilder; +import com.amazon.ion.system.IonTextWriterBuilder; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; @@ -245,7 +249,7 @@ public void testEquivsOverString() { if (myTestFile.getName().endsWith(".ion")) { - String ionText = PrivateUtils.utf8FileToString(myTestFile); + String ionText = _Private_Utils.utf8FileToString(myTestFile); IonDatagram dg = loader().load(ionText); runEquivalenceChecks(dg, myExpectedEquality); } diff --git a/test/software/amazon/ion/ExtendedDecimalTest.java b/test/com/amazon/ion/ExtendedDecimalTest.java similarity index 93% rename from test/software/amazon/ion/ExtendedDecimalTest.java rename to test/com/amazon/ion/ExtendedDecimalTest.java index ab369a24bf..7e95771899 100644 --- a/test/software/amazon/ion/ExtendedDecimalTest.java +++ b/test/com/amazon/ion/ExtendedDecimalTest.java @@ -1,32 +1,32 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; +import static com.amazon.ion.Decimal.negativeZero; import static java.math.MathContext.DECIMAL64; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static software.amazon.ion.Decimal.negativeZero; import java.math.BigDecimal; import java.math.BigInteger; import java.math.MathContext; import org.junit.Test; -import software.amazon.ion.Decimal; /** diff --git a/test/software/amazon/ion/FakeSymbolToken.java b/test/com/amazon/ion/FakeSymbolToken.java similarity index 62% rename from test/software/amazon/ion/FakeSymbolToken.java rename to test/com/amazon/ion/FakeSymbolToken.java index de6e4e5567..426e95a40e 100644 --- a/test/software/amazon/ion/FakeSymbolToken.java +++ b/test/com/amazon/ion/FakeSymbolToken.java @@ -1,21 +1,19 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; - -import software.amazon.ion.SymbolToken; -import software.amazon.ion.UnknownSymbolException; +package com.amazon.ion; /** * NOT SUITABLE FOR PUBLIC USE since it doesn't enforce correctness. diff --git a/test/software/amazon/ion/FieldNameEscapesTest.java b/test/com/amazon/ion/FieldNameEscapesTest.java similarity index 54% rename from test/software/amazon/ion/FieldNameEscapesTest.java rename to test/com/amazon/ion/FieldNameEscapesTest.java index ec4e35049a..708f700bd6 100644 --- a/test/software/amazon/ion/FieldNameEscapesTest.java +++ b/test/com/amazon/ion/FieldNameEscapesTest.java @@ -1,21 +1,20 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonValue; public class FieldNameEscapesTest extends TextTestCase diff --git a/test/software/amazon/ion/FloatTest.java b/test/com/amazon/ion/FloatTest.java similarity index 92% rename from test/software/amazon/ion/FloatTest.java rename to test/com/amazon/ion/FloatTest.java index 5073344135..da7c273918 100644 --- a/test/software/amazon/ion/FloatTest.java +++ b/test/com/amazon/ion/FloatTest.java @@ -1,24 +1,23 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import org.junit.Ignore; import org.junit.Test; -import software.amazon.ion.IonFloat; -import software.amazon.ion.IonType; -import software.amazon.ion.NullValueException; + diff --git a/test/software/amazon/ion/GoodIonTest.java b/test/com/amazon/ion/GoodIonTest.java similarity index 70% rename from test/software/amazon/ion/GoodIonTest.java rename to test/com/amazon/ion/GoodIonTest.java index 0e5406c451..9ee8505c95 100644 --- a/test/software/amazon/ion/GoodIonTest.java +++ b/test/com/amazon/ion/GoodIonTest.java @@ -1,44 +1,38 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static software.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; -import static software.amazon.ion.TestUtils.GOOD_IONTESTS_FILES; -import static software.amazon.ion.TestUtils.testdataFiles; -import static software.amazon.ion.junit.IonAssert.assertIonIteratorEquals; +import static com.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; +import static com.amazon.ion.TestUtils.GOOD_IONTESTS_FILES; +import static com.amazon.ion.TestUtils.testdataFiles; +import static com.amazon.ion.junit.IonAssert.assertIonIteratorEquals; +import com.amazon.ion.impl._Private_Utils; +import com.amazon.ion.junit.Injected.Inject; +import com.amazon.ion.streaming.ReaderCompare; import java.io.File; import java.io.FileInputStream; import java.util.Iterator; import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonValue; -import software.amazon.ion.impl.PrivateUtils; -import software.amazon.ion.junit.Injected.Inject; -import software.amazon.ion.junit.IonAssert; -import software.amazon.ion.streaming.ReaderCompare; public class GoodIonTest extends IonTestCase { @Inject("testFile") - public static final File[] FILES = - testdataFiles(GLOBAL_SKIP_LIST, - GOOD_IONTESTS_FILES); - + public static final File[] FILES = testdataFiles(GLOBAL_SKIP_LIST, GOOD_IONTESTS_FILES); private File myTestFile; private boolean myFileIsBinary; @@ -102,8 +96,11 @@ public void testLoadString() { if (! myFileIsBinary) { - String ionText = PrivateUtils.utf8FileToString(myTestFile); - loader().load(ionText); + String ionText = _Private_Utils.utf8FileToString(myTestFile); + IonDatagram dg = loader().load(ionText); + + // Flush out any encoding problems in the data. + forceDeepMaterialization(dg); } } @@ -111,7 +108,7 @@ public void testLoadString() public void testIterateByteArray() throws Exception { - byte[] bytes = PrivateUtils.loadFileBytes(myTestFile); + byte[] bytes = _Private_Utils.loadFileBytes(myTestFile); Iterator i = system().iterate(bytes); while (i.hasNext()) @@ -123,12 +120,13 @@ public void testIterateByteArray() /** * Test files containing values with unknown text for symbols. */ - private static final String[] FILES_WITH_UNKNOWN_SYMBOL_TEXT = { "good" + File.separator + "item1.10n" }; + private static final String[] FILES_WITH_UNKNOWN_SYMBOL_TEXT = + { "good" + File.separator + "item1.10n", "good" + File.separator + "symbols.ion" }; /** * Skipping test files with unknown text for symbols. * This is okay because the appropriate test is covered in - * {@link SymbolTest#testClone()}. + * SymbolTest */ @Test public void testClone() diff --git a/test/software/amazon/ion/HashCodeCorrectnessTest.java b/test/com/amazon/ion/HashCodeCorrectnessTest.java similarity index 97% rename from test/software/amazon/ion/HashCodeCorrectnessTest.java rename to test/com/amazon/ion/HashCodeCorrectnessTest.java index 1a48f634fa..d603cea2c7 100644 --- a/test/software/amazon/ion/HashCodeCorrectnessTest.java +++ b/test/com/amazon/ion/HashCodeCorrectnessTest.java @@ -1,33 +1,23 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.util.HashSet; import java.util.Set; import org.junit.Test; -import software.amazon.ion.IonBool; -import software.amazon.ion.IonDecimal; -import software.amazon.ion.IonFloat; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonLob; -import software.amazon.ion.IonSequence; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonText; -import software.amazon.ion.IonTimestamp; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; /** * Test cases for {@link IonValue#hashCode()} implementations. diff --git a/test/software/amazon/ion/HashCodeDeltaCollisionTest.java b/test/com/amazon/ion/HashCodeDeltaCollisionTest.java similarity index 89% rename from test/software/amazon/ion/HashCodeDeltaCollisionTest.java rename to test/com/amazon/ion/HashCodeDeltaCollisionTest.java index 479065b779..51e9394292 100644 --- a/test/software/amazon/ion/HashCodeDeltaCollisionTest.java +++ b/test/com/amazon/ion/HashCodeDeltaCollisionTest.java @@ -1,40 +1,33 @@ /* - * Copyright 2013-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static software.amazon.ion.Timestamp.UTC_OFFSET; +import static com.amazon.ion.Timestamp.UTC_OFFSET; +import com.amazon.ion.IonValueDeltaGenerator.IonDecimalDeltaType; +import com.amazon.ion.IonValueDeltaGenerator.IonFloatDeltaType; +import com.amazon.ion.IonValueDeltaGenerator.IonIntDeltaType; +import com.amazon.ion.IonValueDeltaGenerator.IonSymbolDeltaType; +import com.amazon.ion.IonValueDeltaGenerator.IonTimestampDeltaType; import java.math.BigDecimal; import java.util.HashSet; import java.util.Random; import java.util.Set; import org.junit.BeforeClass; import org.junit.Test; -import software.amazon.ion.IonDecimal; -import software.amazon.ion.IonFloat; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonTimestamp; -import software.amazon.ion.IonValue; -import software.amazon.ion.Timestamp; -import software.amazon.ion.IonValueDeltaGenerator.IonDecimalDeltaType; -import software.amazon.ion.IonValueDeltaGenerator.IonFloatDeltaType; -import software.amazon.ion.IonValueDeltaGenerator.IonIntDeltaType; -import software.amazon.ion.IonValueDeltaGenerator.IonSymbolDeltaType; -import software.amazon.ion.IonValueDeltaGenerator.IonTimestampDeltaType; public class HashCodeDeltaCollisionTest diff --git a/test/software/amazon/ion/HashCodeDistributionTest.java b/test/com/amazon/ion/HashCodeDistributionTest.java similarity index 94% rename from test/software/amazon/ion/HashCodeDistributionTest.java rename to test/com/amazon/ion/HashCodeDistributionTest.java index 0ff016ede9..ea36abc6cb 100644 --- a/test/software/amazon/ion/HashCodeDistributionTest.java +++ b/test/com/amazon/ion/HashCodeDistributionTest.java @@ -1,24 +1,25 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static software.amazon.ion.IonType.LIST; -import static software.amazon.ion.IonType.SEXP; -import static software.amazon.ion.IonType.STRING; -import static software.amazon.ion.IonType.STRUCT; -import static software.amazon.ion.IonType.SYMBOL; +import static com.amazon.ion.IonType.LIST; +import static com.amazon.ion.IonType.SEXP; +import static com.amazon.ion.IonType.STRING; +import static com.amazon.ion.IonType.STRUCT; +import static com.amazon.ion.IonType.SYMBOL; import java.io.BufferedInputStream; import java.io.File; @@ -34,10 +35,6 @@ import java.util.concurrent.ConcurrentHashMap; import org.junit.Ignore; import org.junit.Test; -import software.amazon.ion.IonContainer; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; /** Chi-square test for {@link IonValue#hashCode()} implementations. * Using selected test data, determine Chi^2 value for various hash table diff --git a/test/software/amazon/ion/InputStreamWrapper.java b/test/com/amazon/ion/InputStreamWrapper.java similarity index 53% rename from test/software/amazon/ion/InputStreamWrapper.java rename to test/com/amazon/ion/InputStreamWrapper.java index b547c998c5..5284d40d7d 100644 --- a/test/software/amazon/ion/InputStreamWrapper.java +++ b/test/com/amazon/ion/InputStreamWrapper.java @@ -1,18 +1,19 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.io.IOException; import java.io.InputStream; diff --git a/test/software/amazon/ion/IntTest.java b/test/com/amazon/ion/IntTest.java similarity index 97% rename from test/software/amazon/ion/IntTest.java rename to test/com/amazon/ion/IntTest.java index 50d18a85bd..d21e29042c 100644 --- a/test/software/amazon/ion/IntTest.java +++ b/test/com/amazon/ion/IntTest.java @@ -1,17 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; + +package com.amazon.ion; import java.math.BigDecimal; import java.math.BigInteger; diff --git a/test/software/amazon/ion/IonExceptionTest.java b/test/com/amazon/ion/IonExceptionTest.java similarity index 86% rename from test/software/amazon/ion/IonExceptionTest.java rename to test/com/amazon/ion/IonExceptionTest.java index a7dc7a20a3..dd7c18b089 100644 --- a/test/software/amazon/ion/IonExceptionTest.java +++ b/test/com/amazon/ion/IonExceptionTest.java @@ -1,18 +1,19 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; @@ -20,7 +21,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import org.junit.Test; -import software.amazon.ion.IonException; + public class IonExceptionTest { diff --git a/test/com/amazon/ion/IonRawWriterBasicTest.java b/test/com/amazon/ion/IonRawWriterBasicTest.java new file mode 100644 index 0000000000..2e66170fa1 --- /dev/null +++ b/test/com/amazon/ion/IonRawWriterBasicTest.java @@ -0,0 +1,255 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import com.amazon.ion.facet.Facets; +import com.amazon.ion.impl.bin._Private_IonManagedWriter; +import com.amazon.ion.impl.bin._Private_IonRawWriter; +import com.amazon.ion.system.IonReaderBuilder; +import com.amazon.ion.system.IonSystemBuilder; +import com.amazon.ion.system.SimpleCatalog; +import java.io.ByteArrayOutputStream; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +/** + * Tests the raw string writing APIs and basic behavior of the raw symbol ID + * writing APIs provided by the {@link _Private_IonRawWriter} and {@link IonBinaryWriter} + * facets. + * @see IonRawWriterSymbolsTest + */ +@SuppressWarnings({"deprecation", "javadoc"}) +public class IonRawWriterBasicTest +{ + private static final IonSystem system = IonSystemBuilder.standard().build(); + + private static IonWriter writer(OutputStream out, SymbolTable...imports) + { + // Only the binary writer supports the managed/raw facets. + return system.newBinaryWriter(out, imports); + } + + static List getStrings(String file) + { + List strings = new ArrayList(); + IonReader reader; + try + { + reader = IonReaderBuilder.standard().build(new FileInputStream(IonTestCase.getTestdataFile(file))); + } + catch (FileNotFoundException e) + { + // convert to unchecked exception so this plays well in static initializers + throw new RuntimeException(e); + } + while (reader.next() != null) + { + try + { + strings.add(reader.stringValue()); + } + catch (UnknownSymbolException e) + { + // If the input file contains unknown symbol text, e.g. $123, + // just skip it. + } + } + return strings; + } + + static abstract class Roundtrip { + + private final SymbolTable[] imports; + private final SimpleCatalog catalog; + + public Roundtrip(SymbolTable...imports) + { + this.imports = imports; + catalog = new SimpleCatalog(); + for (SymbolTable imported : imports) + { + catalog.putTable(imported); + } + } + + public void test() throws IOException + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IonWriter writer = writer(out, imports); + _Private_IonManagedWriter managedWriter = writer.asFacet(_Private_IonManagedWriter.class); + _Private_IonRawWriter rawWriter = managedWriter.getRawWriter(); + write(managedWriter, rawWriter); + writer.close(); + read(IonReaderBuilder.standard().withCatalog(catalog).build(out.toByteArray())); + } + + abstract void write(_Private_IonManagedWriter managedWriter, _Private_IonRawWriter rawWriter) throws IOException; + abstract void read(IonReader reader); + } + + private void testWriteEncodedString(final String str, final int padding) throws IOException + { + new Roundtrip() + { + int written = 0; + + @Override + void write(_Private_IonManagedWriter managedWriter, _Private_IonRawWriter rawWriter) + throws IOException + { + byte[] encoded = str.getBytes("UTF-8"); + int encodedLength = encoded.length; + + rawWriter.writeString(encoded, 0, encodedLength); + written++; + + byte[] block = new byte[encodedLength + padding]; + for (int i = 0; i < 3; i++) + { + // Write from buffer with 'padding' extra bytes. 'shift' indicates + // the position at which the actual data starts. + int shift = padding * i / 2; + System.arraycopy(encoded, 0, block, shift, encodedLength); + rawWriter.writeString(block, shift, encodedLength); + written++; + } + } + + @Override + void read(IonReader reader) + { + int read = 0; + while (reader.next() != null) + { + assertEquals(IonType.STRING, reader.getType()); + assertEquals(str, reader.stringValue()); + read++; + } + assertEquals(written, read); + } + + }.test(); + } + + @Test + public void testWriteEncodedStrings() throws IOException + { + int i = 0; + for (String str : getStrings("good/strings.ion")) + { + testWriteEncodedString(str, i * 2); + i++; + } + } + + @Test + public void testWriteNullString() throws IOException + { + new Roundtrip() + { + + @Override + void write(_Private_IonManagedWriter managedWriter, _Private_IonRawWriter rawWriter) + throws IOException + { + rawWriter.writeString(null, 1, 2); // position arguments ignored + } + + @Override + void read(IonReader reader) + { + assertEquals(IonType.STRING, reader.next()); + assertTrue(reader.isNullValue()); + assertNull(reader.stringValue()); + } + + }.test(); + } + + private _Private_IonRawWriter basicRawWriter() + { + IonWriter writer = writer(new ByteArrayOutputStream()); + _Private_IonManagedWriter managedWriter = Facets.assumeFacet(_Private_IonManagedWriter.class, writer); + return managedWriter.getRawWriter(); + } + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test + public void testRawWriteIVMFails() throws IOException + { + _Private_IonRawWriter rawWriter = basicRawWriter(); + rawWriter.addTypeAnnotationSymbol(10); + rawWriter.writeSymbolToken(2); // Symbol 2 with an annotation is not IVM + rawWriter.stepIn(IonType.STRUCT); + rawWriter.setFieldNameSymbol(11); + rawWriter.writeSymbolToken(2); // Symbol 2 below top level is not IVM + rawWriter.stepOut(); + thrown.expect(IonException.class); + rawWriter.writeSymbolToken(2); // Top-level symbol 2 is IVM + } + + @Test + public void testRawSetFieldNameOutsideStructFails() + { + _Private_IonRawWriter rawWriter = basicRawWriter(); + thrown.expect(IonException.class); + rawWriter.setFieldNameSymbol(10); + } + + @Test + public void testRawSetTypeAnnotationSymbolsOverwrites() throws IOException + { + new Roundtrip() + { + + @Override + void write(_Private_IonManagedWriter managedWriter, _Private_IonRawWriter rawWriter) + throws IOException + { + managedWriter.requireLocalSymbolTable(); + rawWriter.addTypeAnnotationSymbol(4); + rawWriter.addTypeAnnotationSymbol(5); + rawWriter.setTypeAnnotationSymbols(6, 7); + rawWriter.writeInt(42); // dummy value + } + + @Override + void read(IonReader reader) + { + reader.next(); + SymbolToken[] annotations = reader.getTypeAnnotationSymbols(); + assertEquals(2, annotations.length); + int[] annotationSids = { annotations[0].getSid(), annotations[1].getSid() }; + assertArrayEquals(new int[]{6, 7}, annotationSids); + } + + }.test(); + } +} diff --git a/test/com/amazon/ion/IonRawWriterSymbolsTest.java b/test/com/amazon/ion/IonRawWriterSymbolsTest.java new file mode 100644 index 0000000000..d9767d6857 --- /dev/null +++ b/test/com/amazon/ion/IonRawWriterSymbolsTest.java @@ -0,0 +1,477 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion; + +import static com.amazon.ion.IonRawWriterBasicTest.getStrings; +import static org.junit.Assert.assertEquals; + +import com.amazon.ion.IonRawWriterBasicTest.Roundtrip; +import com.amazon.ion.impl.bin._Private_IonManagedWriter; +import com.amazon.ion.impl.bin._Private_IonRawWriter; +import com.amazon.ion.junit.Injected; +import com.amazon.ion.junit.Injected.Inject; +import com.amazon.ion.system.IonSystemBuilder; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Tests the raw symbol ID writing APIs of the {@link _Private_IonRawWriter} and + * {@link IonBinaryWriter} facets. + * @see IonRawWriterBasicTest + */ +@SuppressWarnings({"deprecation", "javadoc"}) +@RunWith(Injected.class) +public class IonRawWriterSymbolsTest +{ + + private static final IonSystem system = IonSystemBuilder.standard().build(); + + private enum SymbolCombo + { + + /** + * Seralize and deserialize all symbols as symbol values. + */ + VALUES + { + + @Override + public List serialize(_Private_IonRawWriter writer) throws IOException + { + List writtenSymbols = new ArrayList(); + for (int i = 0; i < UNIQUE_SYMBOLS; i++) + { + writtenSymbols.addAll(serializeNext(writer, null)); + } + return writtenSymbols; + } + + @Override + public void deserialize(IonReader reader, List expectedSymbols) + { + Iterator expectedSymbolsIterator = expectedSymbols.iterator(); + for (int i = 0; i < UNIQUE_SYMBOLS; i++) + { + assertEquals(IonType.SYMBOL, reader.next()); + deserializeNext(reader, expectedSymbolsIterator, null); + } + } + + @Override + public List serializeNext(_Private_IonRawWriter writer, SymbolCombo delegate) throws IOException + { + assert delegate == null; + String symbol = circularSymbolsList.next(); + writer.writeSymbolToken(getLocalSid(symbol)); + return Collections.singletonList(symbol); + } + + @Override + public void deserializeNext(IonReader reader, Iterator expectedSymbols, SymbolCombo delegate) + { + assert delegate == null; + assertEquals(expectedSymbols.next(), reader.symbolValue().getText()); + } + + }, + /** + * Serialize and deserialize all symbols as field names. + */ + FIELDS + { + + @Override + public List serialize(_Private_IonRawWriter writer) + throws IOException + { + List writtenSymbols = new ArrayList(); + writer.stepIn(IonType.STRUCT); + for (int i = 0; i < UNIQUE_SYMBOLS; i++) + { + writtenSymbols.addAll(serializeNext(writer, null)); + } + writer.stepOut(); + return writtenSymbols; + } + + @Override + public void deserialize(IonReader reader, + List expectedSymbols) + { + reader.next(); + reader.stepIn(); + Iterator expectedSymbolsIterator = expectedSymbols.iterator(); + for (int i = 0; i < UNIQUE_SYMBOLS; i++) + { + reader.next(); + deserializeNext(reader, expectedSymbolsIterator, null); + } + reader.stepOut(); + } + + @Override + protected List serializeNext(_Private_IonRawWriter writer, + SymbolCombo delegate) + throws IOException + { + List writtenSymbols = new ArrayList(); + String symbol = circularSymbolsList.next(); + writtenSymbols.add(symbol); + writer.setFieldNameSymbol(getLocalSid(symbol)); + if (delegate == null) { + writer.writeInt(42); // dummy value + } + else + { + writtenSymbols.addAll(delegate.serializeNext(writer, null)); + } + return writtenSymbols; + } + + @Override + public void deserializeNext(IonReader reader, Iterator expectedSymbols, SymbolCombo delegate) + { + assertEquals(expectedSymbols.next(), reader.getFieldName()); + if (delegate != null) + { + delegate.deserializeNext(reader, expectedSymbols, null); + } + } + + }, + /** + * Serialize and deserialize all symbols as annotations. + */ + ANNOTATIONS + { + + @Override + public List serialize(_Private_IonRawWriter writer) + throws IOException + { + List writtenSymbols = new ArrayList(); + for (int numAnnotations = 1; numAnnotations < UNIQUE_SYMBOLS; numAnnotations++) + { + if (numAnnotations <= 2) + { + // Tests addTypeAnnotationSymbol called alone and repetitively + for (int i = 0; i < numAnnotations; i++) + { + String symbol = circularSymbolsList.next(); + writtenSymbols.add(symbol); + writer.addTypeAnnotationSymbol(getLocalSid(symbol)); + } + } + else + { + // tests setTypeAnnotationSymbols + int[] symbolIds = new int[numAnnotations]; + for (int i = 0; i < numAnnotations; i++) + { + String symbol = circularSymbolsList.next(); + writtenSymbols.add(symbol); + symbolIds[i] = getLocalSid(symbol); + writer.setTypeAnnotationSymbols(symbolIds); + } + } + writer.writeInt(numAnnotations); // dummy value + } + return writtenSymbols; + } + + @Override + public void deserialize(IonReader reader, + List expectedSymbols) + { + Iterator expectedSymbolsIterator = expectedSymbols.iterator(); + for (int numAnnotations = 1; numAnnotations < UNIQUE_SYMBOLS; numAnnotations++) + { + reader.next(); + for (String annotation : reader.getTypeAnnotations()) + { + assertEquals(expectedSymbolsIterator.next(), annotation); + } + } + } + + // NOTE: for ANNOTATIONS, the *Next methods only add a single + // annotation, for a cleaner interface. Multiple annotations on + // a single value are tested by the *serialize methods. + + @Override + protected List serializeNext(_Private_IonRawWriter writer, + SymbolCombo delegate) + throws IOException + { + List writtenSymbols = new ArrayList(); + String symbol = circularSymbolsList.next(); + writtenSymbols.add(symbol); + writer.addTypeAnnotationSymbol(getLocalSid(symbol)); + if (delegate == null) + { + writer.writeInt(42); //dummy value + } + else + { + writtenSymbols.addAll(delegate.serializeNext(writer, null)); + } + return writtenSymbols; + } + + @Override + public void deserializeNext(IonReader reader, Iterator expectedSymbols, SymbolCombo delegate) + { + assertEquals(expectedSymbols.next(), reader.getTypeAnnotations()[0]); + if (delegate != null) + { + delegate.deserializeNext(reader, expectedSymbols, null); + } + } + + }, + /** + * Serialize and deserialize symbols as symbol values with annotations. + */ + ANNOTATIONS_AND_VALUES + { + + @Override + public List serialize(_Private_IonRawWriter writer) + throws IOException + { + List writtenSymbols = new ArrayList(); + for (int i = 0; i < UNIQUE_SYMBOLS; i++) + { + writtenSymbols.addAll(serializeNext(writer, null)); + } + return writtenSymbols; + } + + @Override + public void deserialize(IonReader reader, + List expectedSymbols) + { + Iterator expectedSymbolsIterator = expectedSymbols.iterator(); + for (int i = 0; i < UNIQUE_SYMBOLS; i++) + { + reader.next(); + deserializeNext(reader, expectedSymbolsIterator, null); + } + } + + @Override + protected List serializeNext(_Private_IonRawWriter writer, + SymbolCombo delegate) + throws IOException + { + + return ANNOTATIONS.serializeNext(writer, VALUES); + } + + @Override + public void deserializeNext(IonReader reader, Iterator expectedSymbols, SymbolCombo delegate) + { + ANNOTATIONS.deserializeNext(reader, expectedSymbols, VALUES); + } + + }, + /** + * Serialize and deserialize symbols as symbol values within a struct + * (i.e. with field name symbols). + */ + FIELDS_AND_VALUES + { + + @Override + public List serialize(_Private_IonRawWriter writer) + throws IOException + { + List writtenSymbols = new ArrayList(); + writer.stepIn(IonType.STRUCT); + for (int i = 0; i < UNIQUE_SYMBOLS; i++) + { + writtenSymbols.addAll(serializeNext(writer, null)); + } + writer.stepOut(); + return writtenSymbols; + } + + @Override + public void deserialize(IonReader reader, + List expectedSymbols) + { + reader.next(); + reader.stepIn(); + Iterator expectedSymbolsIterator = expectedSymbols.iterator(); + for (int i = 0; i < UNIQUE_SYMBOLS; i++) + { + reader.next(); + deserializeNext(reader, expectedSymbolsIterator, null); + } + reader.stepOut(); + } + + @Override + protected List serializeNext(_Private_IonRawWriter writer, + SymbolCombo delegate) + throws IOException + { + return FIELDS.serializeNext(writer, VALUES); + } + + @Override + protected void deserializeNext(IonReader reader, + Iterator expectedSymbols, + SymbolCombo delegate) + { + FIELDS.deserializeNext(reader, expectedSymbols, VALUES); + } + + }, + /** + * Serialize and deserialize symbols as symbol values with annotations + * within a struct (i.e. with field name symbols). + */ + FIELDS_AND_ANNOTATIONS_AND_VALUES + { + + @Override + public List serialize(_Private_IonRawWriter writer) + throws IOException + { + List writtenSymbols = new ArrayList(); + writer.stepIn(IonType.STRUCT); + for (int i = 0; i < UNIQUE_SYMBOLS; i++) + { + writtenSymbols.addAll(serializeNext(writer, null)); + } + writer.stepOut(); + return writtenSymbols; + } + + @Override + public void deserialize(IonReader reader, + List expectedSymbols) + { + reader.next(); + reader.stepIn(); + Iterator expectedSymbolsIterator = expectedSymbols.iterator(); + for (int i = 0; i < UNIQUE_SYMBOLS; i++) + { + reader.next(); + deserializeNext(reader, expectedSymbolsIterator, null); + } + reader.stepOut(); + } + + @Override + protected List serializeNext(_Private_IonRawWriter writer, + SymbolCombo delegate) + throws IOException + { + return FIELDS.serializeNext(writer, ANNOTATIONS_AND_VALUES); + } + + @Override + protected void deserializeNext(IonReader reader, + Iterator expectedSymbols, + SymbolCombo delegate) + { + FIELDS.deserializeNext(reader, expectedSymbols, ANNOTATIONS_AND_VALUES); + } + + }; + + // An endless and centralized supply of symbols. Useful for chaining. + private static final Iterator circularSymbolsList = new Iterator() + { + + private int i = 0; + + public boolean hasNext() + { + return true; + } + + public String next() + { + return symbolsList.get(i++ % symbolsList.size()); + } + + public void remove() + { + throw new UnsupportedOperationException(); + } + + }; + + public abstract List serialize(_Private_IonRawWriter writer) throws IOException; + public abstract void deserialize(IonReader reader, List expectedSymbols); + protected abstract List serializeNext(_Private_IonRawWriter writer, SymbolCombo delegate) throws IOException; + protected abstract void deserializeNext(IonReader reader, Iterator expectedSymbols, SymbolCombo delegate); + } + + private static final List symbolsList = getStrings("good/symbols.ion"); + private static final int UNIQUE_SYMBOLS = symbolsList.size(); + private static final SymbolTable sharedTable = system.newSharedSymbolTable("shared", 1, symbolsList.iterator()); + + private static int getLocalSid(String symbolText) + { + return sharedTable.findSymbol(symbolText) + system.getSystemSymbolTable().getMaxId(); + } + + @Inject("symbolCombo") + public static SymbolCombo[] SYMBOL_COMBOS = SymbolCombo.values(); + + private SymbolCombo mySymbolCombo; + + public void setSymbolCombo(SymbolCombo symbolCombo) + { + mySymbolCombo = symbolCombo; + } + + @Test + public void testDirectWriteSymbolId() throws Exception + { + new Roundtrip(sharedTable) + { + + List expectedSymbols = null; + + @Override + void write(_Private_IonManagedWriter managedWriter, _Private_IonRawWriter rawWriter) + throws IOException + { + managedWriter.requireLocalSymbolTable(); // starts the LST + expectedSymbols = mySymbolCombo.serialize(rawWriter); + } + + @Override + void read(IonReader reader) + { + mySymbolCombo.deserialize(reader, expectedSymbols); + } + + }.test(); + + } + +} diff --git a/test/software/amazon/ion/IonReaderToIonValueTest.java b/test/com/amazon/ion/IonReaderToIonValueTest.java similarity index 62% rename from test/software/amazon/ion/IonReaderToIonValueTest.java rename to test/com/amazon/ion/IonReaderToIonValueTest.java index 25637d0ceb..c6789c925a 100644 --- a/test/software/amazon/ion/IonReaderToIonValueTest.java +++ b/test/com/amazon/ion/IonReaderToIonValueTest.java @@ -1,24 +1,21 @@ /* - * Copyright 2016-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import org.junit.Test; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonType; public class IonReaderToIonValueTest extends IonTestCase { @Test diff --git a/test/software/amazon/ion/IonSystemTest.java b/test/com/amazon/ion/IonSystemTest.java similarity index 89% rename from test/software/amazon/ion/IonSystemTest.java rename to test/com/amazon/ion/IonSystemTest.java index f0170c82c9..9aa5df6ce5 100644 --- a/test/software/amazon/ion/IonSystemTest.java +++ b/test/com/amazon/ion/IonSystemTest.java @@ -1,21 +1,24 @@ /* - * Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static software.amazon.ion.impl.PrivateUtils.EMPTY_BYTE_ARRAY; +import static com.amazon.ion.impl._Private_Utils.EMPTY_BYTE_ARRAY; +import com.amazon.ion.impl._Private_Utils; +import com.amazon.ion.system.SimpleCatalog; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -27,22 +30,7 @@ import java.util.Iterator; import java.util.zip.GZIPOutputStream; import org.junit.Test; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonException; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonLoader; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonString; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.UnexpectedEofException; -import software.amazon.ion.impl.PrivateUtils; -import software.amazon.ion.system.SimpleCatalog; + public class IonSystemTest extends IonTestCase @@ -181,8 +169,8 @@ public void testCloneDetachesFromDatagramOnDiffSystemClone() { IonValue original = system().singleValue("some_symbol"); - IonValue clone = - newSystem(new SimpleCatalog()).clone(original); + // Test on ValueFactory.clone() with different ValueFactory (and DOM impls) + IonValue clone = newSystem(new SimpleCatalog()).clone(original); assertNull("Cloned value should not have a container (parent)", clone.getContainer()); @@ -329,7 +317,7 @@ public void testGzipDetection() throws Exception { String ionText = "1234"; - byte[] textBytes = PrivateUtils.utf8(ionText); + byte[] textBytes = _Private_Utils.utf8(ionText); byte[] gzipTextBytes = gzip(textBytes); checkGzipDetection(textBytes); @@ -357,7 +345,7 @@ private byte[] toBinaryIon(String text) throws IOException public void singleValueTextGzip() throws IOException { String ionText = "1234"; - byte[] textBytes = PrivateUtils.utf8(ionText); + byte[] textBytes = _Private_Utils.utf8(ionText); byte[] gzipTextBytes = gzip(textBytes); IonInt ionValue = (IonInt) system().singleValue(gzipTextBytes); diff --git a/test/software/amazon/ion/IonTestCase.java b/test/com/amazon/ion/IonTestCase.java similarity index 86% rename from test/software/amazon/ion/IonTestCase.java rename to test/com/amazon/ion/IonTestCase.java index dfb1939663..9c96dc2635 100644 --- a/test/software/amazon/ion/IonTestCase.java +++ b/test/com/amazon/ion/IonTestCase.java @@ -1,64 +1,43 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; -import static software.amazon.ion.SystemSymbols.ION_1_0; +import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; +import static com.amazon.ion.SystemSymbols.ION_1_0; -import java.io.ByteArrayOutputStream; +import com.amazon.ion.impl._Private_IonSystem; +import com.amazon.ion.junit.Injected; +import com.amazon.ion.junit.Injected.Inject; +import com.amazon.ion.junit.IonAssert; +import com.amazon.ion.system.IonSystemBuilder; +import com.amazon.ion.system.SimpleCatalog; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; import java.math.BigDecimal; import java.math.BigInteger; import java.util.Iterator; +import java.util.Map; +import java.util.Properties; import org.junit.After; import org.junit.Assert; import org.junit.Rule; import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; -import software.amazon.ion.Decimal; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonDecimal; -import software.amazon.ion.IonException; -import software.amazon.ion.IonFloat; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonLoader; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonSexp; -import software.amazon.ion.IonString; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonTimestamp; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.IonWriter; -import software.amazon.ion.ReadOnlyValueException; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.SystemSymbols; -import software.amazon.ion.Timestamp; -import software.amazon.ion.UnknownSymbolException; -import software.amazon.ion.ValueFactory; -import software.amazon.ion.impl.PrivateIonSystem; -import software.amazon.ion.junit.Injected; -import software.amazon.ion.junit.IonAssert; -import software.amazon.ion.system.IonSystemBuilder; -import software.amazon.ion.system.SimpleCatalog; /** * Base class with helpers for Ion unit tests. @@ -79,9 +58,14 @@ public abstract class IonTestCase } } + protected enum StreamingMode { OLD_STREAMING, NEW_STREAMING } + // Using an enum makes the test names more understandable than a boolean. protected enum StreamCopySpeed { COPY_OPTIMIZED, COPY_NON_OPTIMIZED } + @Inject("streamingMode") + public static final StreamingMode[] STREAMING_DIMENSION = { StreamingMode.NEW_STREAMING }; + /** * Flag on whether IonSystems generated is * {@link IonSystemBuilder#isStreamCopyOptimized()}. @@ -89,9 +73,11 @@ protected enum StreamCopySpeed { COPY_OPTIMIZED, COPY_NON_OPTIMIZED } * This is false by default. Keep this in sync with {@link IonSystemBuilder}! */ private boolean myStreamCopyOptimized = false; + private StreamingMode myStreamingMode; + private static boolean ourSystemPropertiesLoaded = false; protected SimpleCatalog myCatalog; - protected PrivateIonSystem mySystem; + protected _Private_IonSystem mySystem; protected IonLoader myLoader; @Rule @@ -109,6 +95,16 @@ public void tearDown() //========================================================================= // Setters/Getters for injected values + public StreamingMode getStreamingMode() + { + return myStreamingMode; + } + + public void setStreamingMode(StreamingMode mode) + { + myStreamingMode = mode; + } + public boolean isStreamCopyOptimized() { return myStreamCopyOptimized; @@ -119,6 +115,71 @@ public void setCopySpeed(StreamCopySpeed speed) myStreamCopyOptimized = (speed == StreamCopySpeed.COPY_OPTIMIZED); } + //========================================================================= + // Access to test data + + public static synchronized void loadSystemProperties() + { + if (! ourSystemPropertiesLoaded) + { + InputStream stream = + IonTestCase.class.getResourceAsStream("/system.properties"); + + if (stream != null) + { + Properties props = new Properties(); + + try + { + props.load(stream); + + Iterator entries = props.entrySet().iterator(); + while (entries.hasNext()) + { + Map.Entry entry = (Map.Entry) entries.next(); + + String key = (String) entry.getKey(); + + // Command-line values override system.properties + if (System.getProperty(key) == null) + { + System.setProperty(key, (String) entry.getValue()); + } + } + } + catch (IOException e) + { + synchronized (System.out) + { + System.out.println("Caught exception while loading system.properties:"); + e.printStackTrace(System.out); + } + } + } + + ourSystemPropertiesLoaded = true; + } + } + + + public static String requireSystemProperty(String prop) + { + loadSystemProperties(); + + String value = System.getProperty(prop); + if (value == null) + { + value = System.getenv(prop); + } + if (value == null) { + String message = "Missing required system property: " + prop; + throw new IllegalStateException(message); + } + return value; + } + + + public static File getProjectHome() { String basedir = System.getProperty("user.dir"); @@ -131,7 +192,7 @@ public static File getProjectFile(String path) } /** - * Gets a {@link File} contained in the ion-tests "iontestdata" data + * Gets a {@link File} contained in the IonTests "iontestdata" data * directory. * * @param path is relative to the "iontestdata" directory. @@ -142,7 +203,7 @@ public static File getTestdataFile(String path) } /** - * Gets a {@link File} contained in the ion-tests "bulk" data directory. + * Gets a {@link File} contained in the IonTests "bulk" data directory. * * @param path is relative to the "bulk" directory */ @@ -172,18 +233,29 @@ public IonDatagram load(File ionFile) IonLoader loader = loader(); IonDatagram dg = loader.load(ionFile); + // Flush out any encoding problems in the data. + forceDeepMaterialization(dg); + return dg; } + /** + * No op, removal pending the binary backed impl removal + */ + public void forceDeepMaterialization(IonValue value) + { + // No op + } + //========================================================================= // Fixture Helpers /** * @return - * the singleton IonSystem, stream-copy optimized depending - * on the injected {@link #myStreamCopyOptimized}. + * the singleton IonSystem, stream-copy optimized depending on the injected + * {@link #myStreamCopyOptimized}. */ - protected PrivateIonSystem system() + protected _Private_IonSystem system() { if (mySystem == null) { @@ -193,18 +265,24 @@ protected PrivateIonSystem system() } /** + * Returns a new IonSystem for each call, using the passed in IonCatalog + * to build the IonSystem. + * + * @param catalog + * the catalog to use when building the IonSystem + * * @return - * a new IonSystem instance, stream-copy optimized depending on - * {@link #myStreamCopyOptimized}. + * a new IonSystem instance, stream-copy optimized depending on the + * injected {@link #myStreamCopyOptimized}. */ - protected PrivateIonSystem newSystem(IonCatalog catalog) + protected _Private_IonSystem newSystem(IonCatalog catalog) { IonSystemBuilder b = IonSystemBuilder.standard().withCatalog(catalog); b.withStreamCopyOptimized(myStreamCopyOptimized); IonSystem system = b.build(); - return (PrivateIonSystem) system; + return (_Private_IonSystem) system; } protected SimpleCatalog catalog() @@ -728,9 +806,9 @@ public static void checkSymbol(String name, int id, IonValue value) } // just so we can set a break point on this before we lose all context - int sid = sym.symbolValue().getSid(); + int sid = sym.getSymbolId(); if (sid != id) { - assertEquals("symbol id", id, sym.symbolValue().getSid()); + assertEquals("symbol id", id, sym.getSymbolId()); } checkSymbol(name, id, sym.symbolValue()); @@ -948,7 +1026,7 @@ public void testCloneVariants(IonValue original) // Test on ValueFactory.clone() with the same ValueFactory testValueFactoryClone(original, original.getSystem()); - // Test on ValueFactory.clone() with different ValueFactory + // Test on ValueFactory.clone() with different ValueFactory (and DOM impls) testValueFactoryClone(original, newSystem(new SimpleCatalog())); } @@ -998,7 +1076,7 @@ public void testValueFactoryClone(IonValue original, IonValue clone = newFactory.clone(original); IonAssert.assertIonEquals(original, clone); - // TODO amzn/ion-java#30 IonSystemLite.clone() on a value that is in IonSystemImpl + // TODO amzn/ion-java/issues/30 IonSystemLite.clone() on a value that is in IonSystemImpl // doesn't seem to copy over local symbol tables. // assertEquals(original.toString(), clone.toString()); @@ -1012,20 +1090,6 @@ public void testValueFactoryClone(IonValue original, assertFalse("Cloned value should be modifiable", clone.isReadOnly()); } - public byte[] writeBinaryBytes(IonReader reader, SymbolTable... imports) throws IOException { - ByteArrayOutputStream buf = new ByteArrayOutputStream(); - IonWriter writer = system().newBinaryWriter(buf, imports); - try - { - writer.writeValues(reader); - } - finally - { - writer.close(); - } - return buf.toByteArray(); - } - public void logSkippedTest() { System.out.println("WARNING: skipped a test in " + getClass().getName()); diff --git a/test/software/amazon/ion/IonValueChecker.java b/test/com/amazon/ion/IonValueChecker.java similarity index 81% rename from test/software/amazon/ion/IonValueChecker.java rename to test/com/amazon/ion/IonValueChecker.java index aa8a1ef7a7..27b6a7da7b 100644 --- a/test/software/amazon/ion/IonValueChecker.java +++ b/test/com/amazon/ion/IonValueChecker.java @@ -1,29 +1,26 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; +import static com.amazon.ion.IonTestCase.checkSymbol; +import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -import static software.amazon.ion.IonTestCase.checkSymbol; -import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; -import software.amazon.ion.IonValue; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.UnknownSymbolException; public class IonValueChecker implements Checker @@ -56,6 +53,9 @@ public IonValueChecker fieldName(String expectedText, int expectedSid) expectedText, myCurrentValue.getFieldName()); } + assertEquals("IonValue.getFieldId", + expectedSid, myCurrentValue.getFieldId()); + SymbolToken sym = myCurrentValue.getFieldNameSymbol(); checkSymbol(expectedText, expectedSid, sym); diff --git a/test/software/amazon/ion/IonValueDeltaGenerator.java b/test/com/amazon/ion/IonValueDeltaGenerator.java similarity index 90% rename from test/software/amazon/ion/IonValueDeltaGenerator.java rename to test/com/amazon/ion/IonValueDeltaGenerator.java index 9f27271c19..fa8394cdaf 100644 --- a/test/software/amazon/ion/IonValueDeltaGenerator.java +++ b/test/com/amazon/ion/IonValueDeltaGenerator.java @@ -1,22 +1,23 @@ /* - * Copyright 2013-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static junit.framework.Assert.assertTrue; +import com.amazon.ion.Timestamp.Precision; import java.math.BigDecimal; import java.math.BigInteger; import java.util.ArrayList; @@ -24,16 +25,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import software.amazon.ion.IonDecimal; -import software.amazon.ion.IonFloat; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonTimestamp; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.Timestamp; -import software.amazon.ion.Timestamp.Precision; +import junit.framework.Assert; /** * Class that generates {@link IonValue} based off configured properties. @@ -118,7 +110,7 @@ public Builder() { } /** - * Configures the IonSystem that is the entry point for the DOM. + * Configures the IonSystem that is the entry point for DOMs. */ public Builder ionSystem(IonSystem val) { this.ionSystem = val; @@ -228,7 +220,7 @@ private void validateIonInt(IonInt val) throws IllegalArgumentException (baseInt.compareTo(LONG_MAX_VALUE) < 0) && ((IonIntDeltaType) deltaType).equals(IonIntDeltaType.BIGINTEGER)) { - fail("IonInt is tested on the BigInteger dimension but the " + + Assert.fail("IonInt is tested on the BigInteger dimension but the " + "value isn't sufficiently large to be backed by a " + "BigInteger"); } @@ -263,7 +255,7 @@ public Set generateValues() case TIMESTAMP: return generateIonTimestamps(); default: - fail("not supported: " + valueType); + Assert.fail("not supported: " + valueType); return null; } } @@ -294,7 +286,7 @@ private Set generateIonDecimals() scale = 16; break; default: - fail("not supported: " + deltaType); + Assert.fail("not supported: " + deltaType); } BigDecimal decimalDelta = BigDecimal.valueOf(delta, scale); @@ -328,7 +320,7 @@ private Set generateIonFloats() floatDelta = delta * 1.0d; break; default: - fail("not supported: " + deltaType); + Assert.fail("not supported: " + deltaType); } double curr = ((IonFloat) baseValue).doubleValue(); @@ -356,7 +348,7 @@ private Set generateIonInts() intDelta = BigInteger.valueOf(delta); break; default: - fail("not supported: " + deltaType); + Assert.fail("not supported: " + deltaType); } IonInt curr = (IonInt) baseValue; @@ -390,7 +382,7 @@ private Set generateIonSymbols() break; case TEXT: // TODO How should we test for text deltas? default: - fail("not supported: " + deltaType); + Assert.fail("not supported: " + deltaType); } IonSymbol next = ionSystem.newSymbol(nextToken); @@ -448,7 +440,7 @@ private Set generateIonTimestamps() precisionToDelta = Precision.SECOND; break; default: - fail("not supported: " + deltaType); + Assert.fail("not supported: " + deltaType); } IonTimestamp curr = (IonTimestamp) baseValue; @@ -505,7 +497,7 @@ private IonTimestamp getIonTimestampWithDelta(Timestamp currTimestamp, cal.add(Calendar.MILLISECOND, delta); break; default: - fail("not supported: " + precisionToDelta); + Assert.fail("not supported: " + precisionToDelta); } switch (precisionToSet) @@ -523,11 +515,11 @@ private IonTimestamp getIonTimestampWithDelta(Timestamp currTimestamp, cal.clear(Calendar.MILLISECOND); break; default: - fail("not supported: " + precisionToSet); + Assert.fail("not supported: " + precisionToSet); } - Timestamp nextTimestamp = Timestamp.forCalendar(cal); - assertTrue(nextTimestamp.getPrecision().equals(precisionToSet)); + Timestamp nextTimestamp = new Timestamp(cal); + Assert.assertTrue(nextTimestamp.getPrecision().equals(precisionToSet)); return ionSystem.newTimestamp(nextTimestamp); } diff --git a/test/software/amazon/ion/IonValueTest.java b/test/com/amazon/ion/IonValueTest.java similarity index 88% rename from test/software/amazon/ion/IonValueTest.java rename to test/com/amazon/ion/IonValueTest.java index c6e92c4087..eb2a6bb111 100644 --- a/test/software/amazon/ion/IonValueTest.java +++ b/test/com/amazon/ion/IonValueTest.java @@ -1,35 +1,29 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; +import static com.amazon.ion.junit.IonAssert.assertAnnotations; import static java.lang.String.format; -import static software.amazon.ion.junit.IonAssert.assertAnnotations; +import com.amazon.ion.junit.IonAssert; +import com.amazon.ion.system.IonTextWriterBuilder; +import com.amazon.ion.system.IonWriterBuilder.InitialIvmHandling; import org.junit.Ignore; import org.junit.Test; -import software.amazon.ion.IonList; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonValue; -import software.amazon.ion.ReadOnlyValueException; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.SystemSymbols; -import software.amazon.ion.junit.IonAssert; -import software.amazon.ion.system.IonTextWriterBuilder; -import software.amazon.ion.system.IonWriterBuilder.InitialIvmHandling; + public class IonValueTest extends IonTestCase @@ -150,7 +144,7 @@ public void testSetTypeAnnotationInterning() IonValue v = system().newNull(); v.setTypeAnnotations(nameCopy); - // TODO amzn/ion-java#21 fails because v doesn't have any symbol table at all + // TODO amzn/ion-java/issues/21 fails because v doesn't have any symbol table at all // assertSame(nameOrig, v.getTypeAnnotations()[0]); // assertSame(nameOrig, v.getTypeAnnotationSymbols()[0].getText()); } @@ -295,7 +289,7 @@ public void testCustomToString() v.toString(IonTextWriterBuilder.standard() .withInitialIvmHandling(InitialIvmHandling.ENSURE))); - // TODO amzn/ion-java#57 determine if these really should be platform independent newlines + // TODO amzn/ion-java/issues/57 determine if these really should be platform independent newlines final String pretty = format("%n[%n hello,%n a::12%n]"); assertEquals(pretty, v.toString(IonTextWriterBuilder.pretty())); @@ -314,6 +308,7 @@ public void testCloningSystemLookingValue() IonValue v2 = system().clone(v); IonAssert.assertIonEquals(v, v2); + // Try cross-product of system implementations. IonSystem system2 = newSystem(null); v2 = system2.clone(v); IonAssert.assertIonEquals(v, v2); diff --git a/test/software/amazon/ion/IteratorSystemProcessingTestCase.java b/test/com/amazon/ion/IteratorSystemProcessingTestCase.java similarity index 84% rename from test/software/amazon/ion/IteratorSystemProcessingTestCase.java rename to test/com/amazon/ion/IteratorSystemProcessingTestCase.java index 1aaee61c34..5ac1a21d8a 100644 --- a/test/software/amazon/ion/IteratorSystemProcessingTestCase.java +++ b/test/com/amazon/ion/IteratorSystemProcessingTestCase.java @@ -1,25 +1,22 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.util.Iterator; -import software.amazon.ion.IonContainer; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.Timestamp; + abstract class IteratorSystemProcessingTestCase extends SystemProcessingTestCase diff --git a/test/software/amazon/ion/JavaNumericsTest.java b/test/com/amazon/ion/JavaNumericsTest.java similarity index 91% rename from test/software/amazon/ion/JavaNumericsTest.java rename to test/com/amazon/ion/JavaNumericsTest.java index a1d4d46739..381174f1be 100644 --- a/test/software/amazon/ion/JavaNumericsTest.java +++ b/test/com/amazon/ion/JavaNumericsTest.java @@ -1,18 +1,19 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.math.BigDecimal; import java.math.BigInteger; diff --git a/test/software/amazon/ion/JavaReaderIteratorSystemProcessingTest.java b/test/com/amazon/ion/JavaReaderIteratorSystemProcessingTest.java similarity index 65% rename from test/software/amazon/ion/JavaReaderIteratorSystemProcessingTest.java rename to test/com/amazon/ion/JavaReaderIteratorSystemProcessingTest.java index 0dc5d59f5c..5b044d2af5 100644 --- a/test/software/amazon/ion/JavaReaderIteratorSystemProcessingTest.java +++ b/test/com/amazon/ion/JavaReaderIteratorSystemProcessingTest.java @@ -1,23 +1,23 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.io.Reader; import java.io.StringReader; import java.util.Iterator; -import software.amazon.ion.IonValue; public class JavaReaderIteratorSystemProcessingTest diff --git a/test/software/amazon/ion/ListTest.java b/test/com/amazon/ion/ListTest.java similarity index 89% rename from test/software/amazon/ion/ListTest.java rename to test/com/amazon/ion/ListTest.java index 5910a088fc..43c74fa408 100644 --- a/test/software/amazon/ion/ListTest.java +++ b/test/com/amazon/ion/ListTest.java @@ -1,30 +1,25 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; import java.util.List; import org.junit.Test; -import software.amazon.ion.ContainedValueException; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonList; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; public class ListTest @@ -45,9 +40,7 @@ protected IonList makeEmpty() @Override protected IonList newSequence(Collection children) { - IonList list = system().newEmptyList(); - list.addAll(children); - return list; + return system().newList(children); } @Override @@ -231,26 +224,22 @@ public void testCreatingListFromCollection() IonSystem system = system(); List elements = null; - IonList v; - try { - v = newSequence(elements); - fail("Expected NullPointerException"); - } - catch (NullPointerException e) {} + IonList v = system.newList(elements); + testFreshNullSequence(v); elements = new ArrayList(); - v = newSequence(elements); + v = system.newList(elements); testEmptySequence(v); elements.add(system.newString("hi")); elements.add(system.newInt(1776)); - v = newSequence(elements); + v = system.newList(elements); assertEquals(2, v.size()); checkString("hi", v.get(0)); checkInt(1776, v.get(1)); try { - v = newSequence(elements); + v = system.newList(elements); fail("Expected ContainedValueException"); } catch (ContainedValueException e) { } @@ -259,7 +248,7 @@ public void testCreatingListFromCollection() elements.add(system.newInt(1776)); elements.add(null); try { - newSequence(elements); + system.newList(elements); fail("Expected NullPointerException"); } catch (NullPointerException e) { } diff --git a/test/software/amazon/ion/LoadBinaryBytesSystemProcessingTest.java b/test/com/amazon/ion/LoadBinaryBytesSystemProcessingTest.java similarity index 61% rename from test/software/amazon/ion/LoadBinaryBytesSystemProcessingTest.java rename to test/com/amazon/ion/LoadBinaryBytesSystemProcessingTest.java index ff8574f1f8..d90dffb0e2 100644 --- a/test/software/amazon/ion/LoadBinaryBytesSystemProcessingTest.java +++ b/test/com/amazon/ion/LoadBinaryBytesSystemProcessingTest.java @@ -1,21 +1,22 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; + + -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonLoader; public class LoadBinaryBytesSystemProcessingTest extends DatagramIteratorSystemProcessingTest diff --git a/test/software/amazon/ion/LoadBinaryStreamSystemProcessingTest.java b/test/com/amazon/ion/LoadBinaryStreamSystemProcessingTest.java similarity index 64% rename from test/software/amazon/ion/LoadBinaryStreamSystemProcessingTest.java rename to test/com/amazon/ion/LoadBinaryStreamSystemProcessingTest.java index 35d72e276a..d938fd9a3b 100644 --- a/test/software/amazon/ion/LoadBinaryStreamSystemProcessingTest.java +++ b/test/com/amazon/ion/LoadBinaryStreamSystemProcessingTest.java @@ -1,23 +1,23 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.io.ByteArrayInputStream; import java.io.InputStream; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonLoader; + public class LoadBinaryStreamSystemProcessingTest extends DatagramIteratorSystemProcessingTest diff --git a/test/software/amazon/ion/LoadTextBytesSystemProcessingTest.java b/test/com/amazon/ion/LoadTextBytesSystemProcessingTest.java similarity index 53% rename from test/software/amazon/ion/LoadTextBytesSystemProcessingTest.java rename to test/com/amazon/ion/LoadTextBytesSystemProcessingTest.java index bea8fd1b79..c8b432d5d6 100644 --- a/test/software/amazon/ion/LoadTextBytesSystemProcessingTest.java +++ b/test/com/amazon/ion/LoadTextBytesSystemProcessingTest.java @@ -1,22 +1,22 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; + +import com.amazon.ion.impl._Private_Utils; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonLoader; -import software.amazon.ion.impl.PrivateUtils; public class LoadTextBytesSystemProcessingTest @@ -28,7 +28,7 @@ public class LoadTextBytesSystemProcessingTest protected void prepare(String text) throws Exception { - myBytes = PrivateUtils.convertUtf16UnitsToUtf8(text); + myBytes = _Private_Utils.convertUtf16UnitsToUtf8(text); } @Override diff --git a/test/software/amazon/ion/LoadTextStreamSystemProcessingTest.java b/test/com/amazon/ion/LoadTextStreamSystemProcessingTest.java similarity index 57% rename from test/software/amazon/ion/LoadTextStreamSystemProcessingTest.java rename to test/com/amazon/ion/LoadTextStreamSystemProcessingTest.java index ab9eac0897..4d3ed3c2b2 100644 --- a/test/software/amazon/ion/LoadTextStreamSystemProcessingTest.java +++ b/test/com/amazon/ion/LoadTextStreamSystemProcessingTest.java @@ -1,24 +1,24 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; +import com.amazon.ion.impl._Private_Utils; import java.io.ByteArrayInputStream; import java.io.InputStream; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonLoader; -import software.amazon.ion.impl.PrivateUtils; + public class LoadTextStreamSystemProcessingTest @@ -30,7 +30,7 @@ public class LoadTextStreamSystemProcessingTest protected void prepare(String text) throws Exception { - myBytes = PrivateUtils.convertUtf16UnitsToUtf8(text); + myBytes = _Private_Utils.convertUtf16UnitsToUtf8(text); } @Override diff --git a/test/software/amazon/ion/LoaderTest.java b/test/com/amazon/ion/LoaderTest.java similarity index 93% rename from test/software/amazon/ion/LoaderTest.java rename to test/com/amazon/ion/LoaderTest.java index 18e9a76c89..e00685d27f 100644 --- a/test/software/amazon/ion/LoaderTest.java +++ b/test/com/amazon/ion/LoaderTest.java @@ -1,23 +1,26 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static software.amazon.ion.SystemSymbols.ION_1_0; -import static software.amazon.ion.impl.PrivateUtils.UTF8_CHARSET; -import static software.amazon.ion.impl.PrivateUtils.utf8; +import static com.amazon.ion.SystemSymbols.ION_1_0; +import static com.amazon.ion.impl._Private_Utils.UTF8_CHARSET; +import static com.amazon.ion.impl._Private_Utils.utf8; +import com.amazon.ion.impl.Symtabs; +import com.amazon.ion.system.SimpleCatalog; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; @@ -30,20 +33,7 @@ import java.util.Arrays; import java.util.Iterator; import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonException; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonList; -import software.amazon.ion.IonLoader; -import software.amazon.ion.IonMutableCatalog; -import software.amazon.ion.IonNull; -import software.amazon.ion.IonString; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonValue; -import software.amazon.ion.impl.Symtabs; -import software.amazon.ion.system.SimpleCatalog; + public class LoaderTest extends IonTestCase diff --git a/test/software/amazon/ion/LongStringTest.java b/test/com/amazon/ion/LongStringTest.java similarity index 56% rename from test/software/amazon/ion/LongStringTest.java rename to test/com/amazon/ion/LongStringTest.java index 491b6f20b0..f7454932ef 100644 --- a/test/software/amazon/ion/LongStringTest.java +++ b/test/com/amazon/ion/LongStringTest.java @@ -1,18 +1,20 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; + public class LongStringTest extends TextTestCase diff --git a/test/software/amazon/ion/NewDatagramIteratorSystemProcessingTest.java b/test/com/amazon/ion/NewDatagramIteratorSystemProcessingTest.java similarity index 79% rename from test/software/amazon/ion/NewDatagramIteratorSystemProcessingTest.java rename to test/com/amazon/ion/NewDatagramIteratorSystemProcessingTest.java index dc41683c23..3674699150 100644 --- a/test/software/amazon/ion/NewDatagramIteratorSystemProcessingTest.java +++ b/test/com/amazon/ion/NewDatagramIteratorSystemProcessingTest.java @@ -1,24 +1,21 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.util.Iterator; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonValue; -import software.amazon.ion.IonWriter; /** * TODO replicates other tests in this hierarchy diff --git a/test/com/amazon/ion/NonEquivsTest.java b/test/com/amazon/ion/NonEquivsTest.java new file mode 100644 index 0000000000..f8a38ab7ea --- /dev/null +++ b/test/com/amazon/ion/NonEquivsTest.java @@ -0,0 +1,35 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion; + +import static com.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; +import static com.amazon.ion.TestUtils.NON_EQUIVS_IONTESTS_FILEs; + +import com.amazon.ion.junit.Injected.Inject; +import java.io.File; + +public class NonEquivsTest + extends EquivsTestCase +{ + @Inject("testFile") + public static final File[] FILES = + TestUtils.testdataFiles(GLOBAL_SKIP_LIST, NON_EQUIVS_IONTESTS_FILEs); + + public NonEquivsTest() + { + super(false); + } +} diff --git a/test/software/amazon/ion/NopPaddingTest.java b/test/com/amazon/ion/NopPaddingTest.java similarity index 91% rename from test/software/amazon/ion/NopPaddingTest.java rename to test/com/amazon/ion/NopPaddingTest.java index 57ca9ee411..2fd7fa3555 100644 --- a/test/software/amazon/ion/NopPaddingTest.java +++ b/test/com/amazon/ion/NopPaddingTest.java @@ -1,17 +1,18 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import org.junit.Test; diff --git a/test/software/amazon/ion/NullTest.java b/test/com/amazon/ion/NullTest.java similarity index 78% rename from test/software/amazon/ion/NullTest.java rename to test/com/amazon/ion/NullTest.java index 48f3944cbb..aa0c4d770e 100644 --- a/test/software/amazon/ion/NullTest.java +++ b/test/com/amazon/ion/NullTest.java @@ -1,25 +1,22 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; + +package com.amazon.ion; import java.util.Iterator; import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonList; -import software.amazon.ion.IonNull; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; diff --git a/test/com/amazon/ion/RawValueSpanReaderBasicTest.java b/test/com/amazon/ion/RawValueSpanReaderBasicTest.java new file mode 100644 index 0000000000..f209214c13 --- /dev/null +++ b/test/com/amazon/ion/RawValueSpanReaderBasicTest.java @@ -0,0 +1,154 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; + +import com.amazon.ion.system.IonReaderBuilder; +import java.io.ByteArrayInputStream; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +/** + * Tests basic behavior of the {@link RawValueSpanProvider} reader facet, which + * provides access to the reader's underlying byte buffer and vends OffsetSpans + * that provide positions of the current value in that buffer. + * @see RawValueSpanReaderTest + */ +@SuppressWarnings({"deprecation", "javadoc"}) +public class RawValueSpanReaderBasicTest +{ + + private byte[] dummyData; + + @Before + public void setup() + { + // Trivial binary Ion. IVM followed by a list with a single element (int 0). + dummyData = new byte[]{(byte)0xE0, (byte)0x01, (byte)0x00, (byte)0xEA, (byte)0xB1, (byte)0x20}; + } + + @Test + public void testTextReaderReturnsNullFacet() + { + // Only the binary reader currently provides this facet. + IonReader reader = IonReaderBuilder.standard().build("text"); + assertNull(reader.asFacet(RawValueSpanProvider.class)); + } + + @Test + public void testNonByteBackedReaderNotSupported() + { + IonReader reader = IonReaderBuilder.standard().build(new ByteArrayInputStream(dummyData)); + assertNull(reader.asFacet(RawValueSpanProvider.class)); + assertNull(reader.asFacet(SeekableReader.class)); + } + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test + public void testSeekToNullSpanFails() + { + IonReader reader = IonReaderBuilder.standard().build(dummyData); + SeekableReader seekable = reader.asFacet(SeekableReader.class); + thrown.expect(IllegalArgumentException.class); + seekable.hoist(null); + } + + @Test + public void testSeekToIncompatibleSpanFails() + { + IonReader reader = IonReaderBuilder.standard().build(dummyData); + SeekableReader seekable = reader.asFacet(SeekableReader.class); + thrown.expect(IllegalArgumentException.class); + seekable.hoist(new Span(){ + + public T asFacet(Class facetType) + { + return null; + } + + }); + } + + @Test + public void testSeekToValueSpanFails() + { + IonReader reader = IonReaderBuilder.standard().build(dummyData); + RawValueSpanProvider valueSpanProvider = reader.asFacet(RawValueSpanProvider.class); + reader.next(); // position reader on list + Span valueSpan = valueSpanProvider.valueSpan(); + reader.next(); // advance past list + SeekableReader seekable = reader.asFacet(SeekableReader.class); + thrown.expect(IllegalArgumentException.class); + // seeking to a value span is not allowed because it's positioned + // past the TID and length bytes. + seekable.hoist(valueSpan); + } + + @Test + public void testGetBufferReturnsSame() + { + IonReader reader = IonReaderBuilder.standard().build(dummyData); + assertSame(dummyData, reader.asFacet(RawValueSpanProvider.class).buffer()); + } + + @Test + public void testGetSpanNotAtTopLevelValueFails() + { + IonReader reader = IonReaderBuilder.standard().build(dummyData); + RawValueSpanProvider spanProvider = reader.asFacet(RawValueSpanProvider.class); + thrown.expect(IllegalStateException.class); + spanProvider.valueSpan(); + } + + @Test + public void testGetSpanNotAtContainerValueFails() + { + IonReader reader = IonReaderBuilder.standard().build(dummyData); + RawValueSpanProvider spanProvider = reader.asFacet(RawValueSpanProvider.class); + reader.next(); + reader.stepIn(); // inside list + thrown.expect(IllegalStateException.class); + spanProvider.valueSpan(); + } + + @Test + public void testSingleOctetValueSpan() + { + IonReader reader = IonReaderBuilder.standard().build(dummyData); + SpanProvider seekableSpanProvider = reader.asFacet(SpanProvider.class); + RawValueSpanProvider valueSpanProvider = reader.asFacet(RawValueSpanProvider.class); + reader.next(); + reader.stepIn(); // inside list + assertEquals(IonType.INT, reader.next()); + Span seekableSpan = seekableSpanProvider.currentSpan(); + OffsetSpan valueSpan = (OffsetSpan)valueSpanProvider.valueSpan(); + // int 0 is fully represented by the type ID octet. + assertEquals(valueSpan.getStartOffset(), valueSpan.getFinishOffset()); + reader.stepOut(); // go logically past the value + SeekableReader seekable = reader.asFacet(SeekableReader.class); + seekable.hoist(seekableSpan); // seek back + reader.next(); //required after any seek + assertEquals(0, reader.intValue()); + } + +} diff --git a/test/software/amazon/ion/ReaderChecker.java b/test/com/amazon/ion/ReaderChecker.java similarity index 85% rename from test/software/amazon/ion/ReaderChecker.java rename to test/com/amazon/ion/ReaderChecker.java index da80590c8f..7905173d80 100644 --- a/test/software/amazon/ion/ReaderChecker.java +++ b/test/com/amazon/ion/ReaderChecker.java @@ -1,33 +1,32 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; +import static com.amazon.ion.IonTestCase.checkSymbol; +import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; +import static com.amazon.ion.impl._Private_Utils.stringIterator; +import static com.amazon.ion.junit.IonAssert.assertIteratorEquals; +import static com.amazon.ion.junit.IonAssert.assertSymbolEquals; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; -import static software.amazon.ion.IonTestCase.checkSymbol; -import static software.amazon.ion.impl.PrivateUtils.stringIterator; -import static software.amazon.ion.junit.IonAssert.assertIteratorEquals; -import static software.amazon.ion.junit.IonAssert.assertSymbolEquals; import java.util.Iterator; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonType; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.UnknownSymbolException; + public class ReaderChecker implements Checker @@ -54,6 +53,9 @@ public ReaderChecker noFieldName() assertEquals("IonReader.getFieldName()", null, myReader.getFieldName()); + assertEquals("IonReader.getFieldId()", + UNKNOWN_SYMBOL_ID, myReader.getFieldId()); + assertEquals("IonReader.getFieldNameSymbol()", null, myReader.getFieldNameSymbol()); @@ -79,6 +81,9 @@ public ReaderChecker fieldName(String expectedText, int expectedSid) expectedText, myReader.getFieldName()); } + assertEquals("IonReader.getFieldId()", + expectedSid, myReader.getFieldId()); + SymbolToken sym = myReader.getFieldNameSymbol(); checkSymbol(expectedText, expectedSid, sym); diff --git a/test/software/amazon/ion/ReaderMaker.java b/test/com/amazon/ion/ReaderMaker.java similarity index 90% rename from test/software/amazon/ion/ReaderMaker.java rename to test/com/amazon/ion/ReaderMaker.java index 6b5bb78a80..ff5cc88fb7 100644 --- a/test/software/amazon/ion/ReaderMaker.java +++ b/test/com/amazon/ion/ReaderMaker.java @@ -1,22 +1,24 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static software.amazon.ion.TestUtils.ensureBinary; -import static software.amazon.ion.TestUtils.ensureText; +import static com.amazon.ion.TestUtils.ensureBinary; +import static com.amazon.ion.TestUtils.ensureText; +import com.amazon.ion.impl._Private_Utils; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -25,10 +27,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonSystem; -import software.amazon.ion.impl.PrivateUtils; /** * Abstracts the various ways that {@link IonReader}s can be created, so test @@ -231,7 +229,7 @@ public int getOffset() public IonReader newReader(IonSystem system, String ionText) { - byte[] utf8 = PrivateUtils.utf8(ionText); + byte[] utf8 = _Private_Utils.utf8(ionText); return newReader(system, utf8); } @@ -252,6 +250,14 @@ public IonReader newReader(IonSystem system, byte[] ionData, } + public IonReader newReader(IonSystem system, InputStream ionData) + throws IOException + { + byte[] bytes = _Private_Utils.loadStreamBytes(ionData); + return newReader(system, bytes); + } + + public static ReaderMaker[] valuesExcluding(ReaderMaker... exclusions) { ReaderMaker[] all = values(); diff --git a/test/software/amazon/ion/ReaderSystemProcessingTestCase.java b/test/com/amazon/ion/ReaderSystemProcessingTestCase.java similarity index 91% rename from test/software/amazon/ion/ReaderSystemProcessingTestCase.java rename to test/com/amazon/ion/ReaderSystemProcessingTestCase.java index 5c7335af53..de9b468221 100644 --- a/test/software/amazon/ion/ReaderSystemProcessingTestCase.java +++ b/test/com/amazon/ion/ReaderSystemProcessingTestCase.java @@ -1,34 +1,30 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static software.amazon.ion.impl.PrivateUtils.EMPTY_STRING_ARRAY; -import static software.amazon.ion.junit.IonAssert.assertNoCurrentValue; -import static software.amazon.ion.junit.IonAssert.expectNextField; +import static com.amazon.ion.impl._Private_Utils.EMPTY_STRING_ARRAY; +import static com.amazon.ion.junit.IonAssert.assertNoCurrentValue; +import static com.amazon.ion.junit.IonAssert.expectNextField; +import com.amazon.ion.junit.IonAssert; import java.util.Date; import org.junit.After; import org.junit.Assert; import org.junit.Test; -import software.amazon.ion.Decimal; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonType; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.Timestamp; -import software.amazon.ion.junit.IonAssert; + public abstract class ReaderSystemProcessingTestCase @@ -240,6 +236,10 @@ public void testIsInStruct() assertFalse(myReader.isInStruct()); assertEquals(0, myReader.getDepth()); + assertTrue(myReader.hasNext()); + assertFalse(myReader.isInStruct()); + assertEquals(0, myReader.getDepth()); + assertEquals(IonType.STRUCT, myReader.next()); assertFalse(myReader.isInStruct()); assertEquals(0, myReader.getDepth()); @@ -249,6 +249,10 @@ public void testIsInStruct() assertTrue(myReader.isInStruct()); assertEquals(1, myReader.getDepth()); + assertTrue(myReader.hasNext()); // List is coming up + assertTrue(myReader.isInStruct()); // but we're still at struct level + assertEquals(1, myReader.getDepth()); + assertSame(IonType.LIST, myReader.next()); assertTrue(myReader.isInStruct()); // still in struct until we stepIn() assertEquals(1, myReader.getDepth()); @@ -289,8 +293,10 @@ public void testHasNextLeavesCurrentData() String text = "hello 2"; startIteration(text); + assertTrue(myReader.hasNext()); assertEquals(IonType.SYMBOL, myReader.next()); assertEquals(IonType.SYMBOL, myReader.getType()); + assertTrue(myReader.hasNext()); // FIXed ME text reader was broken, now fixed // really the binary readers were returning the @@ -382,7 +388,6 @@ public void testStepOutInMiddle() expectNextField(r, "d"); } - @Test public void testSkippingFieldsNoQuote() throws Exception diff --git a/test/software/amazon/ion/RoundTripTest.java b/test/com/amazon/ion/RoundTripTest.java similarity index 79% rename from test/software/amazon/ion/RoundTripTest.java rename to test/com/amazon/ion/RoundTripTest.java index 098dcd1a93..7d228a7c14 100644 --- a/test/software/amazon/ion/RoundTripTest.java +++ b/test/com/amazon/ion/RoundTripTest.java @@ -1,25 +1,33 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; - -import static software.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; -import static software.amazon.ion.TestUtils.GOOD_IONTESTS_FILES; -import static software.amazon.ion.TestUtils.TEXT_ONLY_FILTER; -import static software.amazon.ion.TestUtils.testdataFiles; -import static software.amazon.ion.junit.IonAssert.assertIonEquals; - +package com.amazon.ion; + +import static com.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; +import static com.amazon.ion.TestUtils.GOOD_IONTESTS_FILES; +import static com.amazon.ion.TestUtils.TEXT_ONLY_FILTER; +import static com.amazon.ion.TestUtils.testdataFiles; +import static com.amazon.ion.junit.IonAssert.assertIonEquals; + +import com.amazon.ion.TestUtils.And; +import com.amazon.ion.TestUtils.FileIsNot; +import com.amazon.ion.junit.Injected.Inject; +import com.amazon.ion.streaming.ReaderCompare; +import com.amazon.ion.streaming.RoundTripStreamingTest; +import com.amazon.ion.system.IonTextWriterBuilder; +import com.amazon.ion.util.Printer; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; @@ -29,31 +37,18 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonException; -import software.amazon.ion.IonLoader; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonValue; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SystemSymbols; -import software.amazon.ion.TestUtils.And; -import software.amazon.ion.TestUtils.FileIsNot; -import software.amazon.ion.junit.Injected.Inject; -import software.amazon.ion.streaming.ReaderCompare; -import software.amazon.ion.streaming.RoundTripStreamingTest; -import software.amazon.ion.system.IonTextWriterBuilder; /** * Processes all text files in the "good" suite, transforming between text and * binary twice to ensure that the process is equivalent. * - * TODO amzn/ion-java#29 Refactor this test class, possible duplicate test coverage in + * TODO amzn/ion-java/issues/29 Refactor this test class, possible duplicate test coverage in * {@link RoundTripStreamingTest}. */ public class RoundTripTest extends IonTestCase { - // TODO amzn/ion-java#27 Writing IonSymbol to bytes using IonSymbol.writeTo(IonWriter) + // TODO amzn/ion-java/issues/27 Writing IonSymbol to bytes using IonSymbol.writeTo(IonWriter) // will throw UnknownSymbolException if symbol text is unknown public static final FilenameFilter ROUND_TRIP_TEST_SKIP_LIST = new FileIsNot("good/symbols.ion"); @@ -74,11 +69,7 @@ private enum BinaryEncoderType * encodings that won't round-trip. */ @Inject("testFile") - public static final File[] FILES = - testdataFiles(new And(TEXT_ONLY_FILTER, - GLOBAL_SKIP_LIST, - ROUND_TRIP_TEST_SKIP_LIST), - GOOD_IONTESTS_FILES); + public static final File[] FILES = testdataFiles(new And(TEXT_ONLY_FILTER, GLOBAL_SKIP_LIST, ROUND_TRIP_TEST_SKIP_LIST), GOOD_IONTESTS_FILES); @Inject("copySpeed") public static final StreamCopySpeed[] STREAM_COPY_SPEEDS = @@ -88,6 +79,7 @@ private enum BinaryEncoderType public static final BinaryEncoderType[] BINARY_ENCODER_TYPES = BinaryEncoderType.values(); + private Printer myPrinter; private StringBuilder myBuilder; private File myTestFile; private BinaryEncoderType myBinaryEncoderType; @@ -108,6 +100,7 @@ public void setUp() throws Exception { super.setUp(); + myPrinter = new Printer(); myBuilder = new StringBuilder(); } @@ -116,6 +109,7 @@ public void setUp() public void tearDown() throws Exception { + myPrinter = null; myBuilder = null; super.tearDown(); } @@ -124,16 +118,11 @@ public void tearDown() private String renderSystemView(IonDatagram datagram) throws IOException { + // TODO use Printer in raw mode. for (Iterator i = datagram.systemIterator(); i.hasNext();) { IonValue value = (IonValue) i.next(); - IonWriter writer = IonTextWriterBuilder.standard().build(myBuilder); - try { - value.writeTo(writer); - } - finally { - writer.close(); - } + myPrinter.print(value, myBuilder); myBuilder.append('\n'); } @@ -145,16 +134,11 @@ private String renderSystemView(IonDatagram datagram) private String renderUserView(IonDatagram datagram) throws IOException { + // TODO use Printer in raw mode. for (Iterator i = datagram.iterator(); i.hasNext();) { IonValue value = (IonValue) i.next(); - IonWriter writer = IonTextWriterBuilder.standard().build(myBuilder); - try { - value.writeTo(writer); - } - finally { - writer.close(); - } + myPrinter.print(value, myBuilder); myBuilder.append('\n'); } diff --git a/test/software/amazon/ion/SequenceTestCase.java b/test/com/amazon/ion/SequenceTestCase.java similarity index 95% rename from test/software/amazon/ion/SequenceTestCase.java rename to test/com/amazon/ion/SequenceTestCase.java index c30e12520d..41e8a2af6a 100644 --- a/test/software/amazon/ion/SequenceTestCase.java +++ b/test/com/amazon/ion/SequenceTestCase.java @@ -1,18 +1,19 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import java.util.ArrayList; import java.util.Arrays; @@ -22,20 +23,6 @@ import java.util.ListIterator; import java.util.NoSuchElementException; import org.junit.Test; -import software.amazon.ion.ContainedValueException; -import software.amazon.ion.IonBool; -import software.amazon.ion.IonContainer; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonList; -import software.amazon.ion.IonNull; -import software.amazon.ion.IonSequence; -import software.amazon.ion.IonString; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.NullValueException; -import software.amazon.ion.ReadOnlyValueException; @@ -583,7 +570,7 @@ public void testRetainAll() (Object)nullValue2); IonSequence seq = makeEmpty(); - // TODO amzn/ion-java#49 implement IonDatagram.retainAll + // TODO amzn/ion-java/issues/49 implement IonDatagram.retainAll if (seq.getType() == IonType.DATAGRAM) return; assertFalse(seq.retainAll(empty)); @@ -725,6 +712,8 @@ public void testAddOfClone() s.add(v2); v2.addTypeAnnotation("foo"); + forceDeepMaterialization(v1); + if (s.getType() != IonType.DATAGRAM) { IonDatagram dg = system().newDatagram(s); diff --git a/test/software/amazon/ion/SexpTest.java b/test/com/amazon/ion/SexpTest.java similarity index 89% rename from test/software/amazon/ion/SexpTest.java rename to test/com/amazon/ion/SexpTest.java index 08b7578a34..18ceca29e6 100644 --- a/test/software/amazon/ion/SexpTest.java +++ b/test/com/amazon/ion/SexpTest.java @@ -1,29 +1,25 @@ + /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; + +package com.amazon.ion; import java.util.ArrayList; import java.util.Collection; import java.util.List; import org.junit.Test; -import software.amazon.ion.ContainedValueException; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonSexp; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; @@ -45,9 +41,7 @@ protected IonSexp makeEmpty() @Override protected IonSexp newSequence(Collection children) { - IonSexp sexp = system().newEmptySexp(); - sexp.addAll(children); - return sexp; + return system().newSexp(children); } @Override @@ -205,26 +199,22 @@ public void testCreatingSexpFromCollection() IonSystem system = system(); List elements = null; - IonSexp v; - try { - v = newSequence(elements); - fail("Expected NullPointerException"); - } - catch (NullPointerException e) {} + IonSexp v = system.newSexp(elements); + testFreshNullSequence(v); elements = new ArrayList(); - v = newSequence(elements); + v = system.newSexp(elements); testEmptySequence(v); elements.add(system.newString("hi")); elements.add(system.newInt(1776)); - v = newSequence(elements); + v = system.newSexp(elements); assertEquals(2, v.size()); checkString("hi", v.get(0)); checkInt(1776, v.get(1)); try { - v = newSequence(elements); + v = system.newSexp(elements); fail("Expected ContainedValueException"); } catch (ContainedValueException e) { } @@ -233,7 +223,7 @@ public void testCreatingSexpFromCollection() elements.add(system.newInt(1776)); elements.add(null); try { - v = newSequence(elements); + v = system.newSexp(elements); fail("Expected NullPointerException"); } catch (NullPointerException e) { } diff --git a/test/software/amazon/ion/SharedSymtabMaker.java b/test/com/amazon/ion/SharedSymtabMaker.java similarity index 68% rename from test/software/amazon/ion/SharedSymtabMaker.java rename to test/com/amazon/ion/SharedSymtabMaker.java index 554854f6bb..6b5e031690 100644 --- a/test/software/amazon/ion/SharedSymtabMaker.java +++ b/test/com/amazon/ion/SharedSymtabMaker.java @@ -1,28 +1,24 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; +import static com.amazon.ion.impl.Symtabs.sharedSymtabStruct; import static junit.framework.Assert.assertSame; -import static software.amazon.ion.impl.Symtabs.sharedSymtabStruct; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonType; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.impl.PrivateIonSystem; +import com.amazon.ion.impl._Private_IonSystem; /** * Abstracts the various ways that a shared {@link SymbolTable} can be created @@ -36,7 +32,7 @@ public enum SharedSymtabMaker FROM_READER { @Override - public SymbolTable newSharedSymtab(PrivateIonSystem system, + public SymbolTable newSharedSymtab(_Private_IonSystem system, IonReader reader) { return system.newSharedSymbolTable(reader); @@ -50,7 +46,7 @@ public SymbolTable newSharedSymtab(PrivateIonSystem system, FROM_READER_ON_STRUCT { @Override - public SymbolTable newSharedSymtab(PrivateIonSystem system, + public SymbolTable newSharedSymtab(_Private_IonSystem system, IonReader reader) { assertSame(IonType.STRUCT, reader.next()); @@ -65,7 +61,7 @@ public SymbolTable newSharedSymtab(PrivateIonSystem system, FROM_READER_NOT_ON_STRUCT { @Override - public SymbolTable newSharedSymtab(PrivateIonSystem system, + public SymbolTable newSharedSymtab(_Private_IonSystem system, IonReader reader) { return system.newSharedSymbolTable(reader, false); @@ -78,7 +74,7 @@ public SymbolTable newSharedSymtab(PrivateIonSystem system, FROM_STRUCT { @Override - public SymbolTable newSharedSymtab(PrivateIonSystem system, + public SymbolTable newSharedSymtab(_Private_IonSystem system, IonStruct struct) { return system.newSharedSymbolTable(struct); @@ -86,7 +82,7 @@ public SymbolTable newSharedSymtab(PrivateIonSystem system, }; - public SymbolTable newSharedSymtab(PrivateIonSystem system, + public SymbolTable newSharedSymtab(_Private_IonSystem system, String name, int version, String... symbols) @@ -98,7 +94,7 @@ public SymbolTable newSharedSymtab(PrivateIonSystem system, /** * @param reader must be positioned JUST BEFORE the symtab struct. */ - public SymbolTable newSharedSymtab(PrivateIonSystem system, + public SymbolTable newSharedSymtab(_Private_IonSystem system, IonReader reader) { assertSame(IonType.STRUCT, reader.next()); @@ -106,7 +102,7 @@ public SymbolTable newSharedSymtab(PrivateIonSystem system, return newSharedSymtab(system, struct); } - public SymbolTable newSharedSymtab(PrivateIonSystem system, + public SymbolTable newSharedSymtab(_Private_IonSystem system, IonStruct struct) { IonReader reader = system.newReader(struct); diff --git a/test/software/amazon/ion/StringFieldNameEscapesTest.java b/test/com/amazon/ion/StringFieldNameEscapesTest.java similarity index 59% rename from test/software/amazon/ion/StringFieldNameEscapesTest.java rename to test/com/amazon/ion/StringFieldNameEscapesTest.java index 50bcf732c0..96e3f0e387 100644 --- a/test/software/amazon/ion/StringFieldNameEscapesTest.java +++ b/test/com/amazon/ion/StringFieldNameEscapesTest.java @@ -1,21 +1,20 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonValue; public class StringFieldNameEscapesTest extends TextTestCase diff --git a/test/software/amazon/ion/StringTest.java b/test/com/amazon/ion/StringTest.java similarity index 94% rename from test/software/amazon/ion/StringTest.java rename to test/com/amazon/ion/StringTest.java index 5e40d6c388..79dff34f14 100644 --- a/test/software/amazon/ion/StringTest.java +++ b/test/com/amazon/ion/StringTest.java @@ -1,24 +1,21 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import org.junit.Test; -import software.amazon.ion.IonString; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.UnexpectedEofException; diff --git a/test/software/amazon/ion/StructTest.java b/test/com/amazon/ion/StructTest.java similarity index 97% rename from test/software/amazon/ion/StructTest.java rename to test/com/amazon/ion/StructTest.java index 7c1210d33b..88861772ae 100644 --- a/test/software/amazon/ion/StructTest.java +++ b/test/com/amazon/ion/StructTest.java @@ -1,21 +1,24 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; +import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; +import com.amazon.ion.impl._Private_IonValue; +import com.amazon.ion.impl._Private_Utils; import java.io.PrintWriter; import java.util.ArrayList; import java.util.HashMap; @@ -25,25 +28,6 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import software.amazon.ion.ContainedValueException; -import software.amazon.ion.IonBool; -import software.amazon.ion.IonContainer; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonList; -import software.amazon.ion.IonNull; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonString; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.ReadOnlyValueException; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.UnknownSymbolException; -import software.amazon.ion.impl.PrivateIonValue; -import software.amazon.ion.impl.PrivateUtils; public class StructTest @@ -639,7 +623,7 @@ public void testAddSymbolTokenWithBadSid() IonStruct struct = system().newNullStruct(); IonBool nullBool = system().newNullBool(); - SymbolToken is = PrivateUtils.newSymbolToken("f", 1); + SymbolToken is = _Private_Utils.newSymbolToken("f", 1); struct.add(is, nullBool); is = nullBool.getFieldNameSymbol(); checkSymbol("f", UNKNOWN_SYMBOL_ID, is); @@ -669,7 +653,7 @@ public void testBadAddSymbolToken() } catch (NullPointerException e) { } - SymbolToken is = PrivateUtils.newSymbolToken("f", 1); + SymbolToken is = _Private_Utils.newSymbolToken("f", 1); try { value.add(is, null); fail("Expected NullPointerException"); @@ -826,7 +810,7 @@ public void testRemoveClearsFieldName(IonStruct s) f.removeFromContainer(); assertEquals(null, f.getFieldName()); - assertEquals(null, f.getFieldNameSymbol()); + assertEquals(SymbolTable.UNKNOWN_SYMBOL_ID, f.getFieldId()); } @Test @@ -1532,7 +1516,7 @@ public void testRandomChanges() } } void dump(IonStruct s1_temp, ArrayList s2) { - PrivateIonValue s1 = ((PrivateIonValue)s1_temp); + _Private_IonValue s1 = ((_Private_IonValue)s1_temp); s1.dump(new PrintWriter(System.out)); System.out.println("array: "+s2.toString()); } @@ -1761,8 +1745,8 @@ void compare_field_lists(IonStruct s1, ArrayList s2, long seed) errors += "extra field in struct "+v; errors += "\n"; } - if (v instanceof PrivateIonValue) { - int eid = ((PrivateIonValue)v).getElementId(); + if (v instanceof _Private_IonValue) { + int eid = ((_Private_IonValue)v).getElementId(); if (eid != struct_idx) { difference = true; errors += "index of struct field "+struct_idx+" doesn't match array index "+eid+": "+v; @@ -1785,7 +1769,7 @@ void compare_field_lists(IonStruct s1, ArrayList s2, long seed) if (_debug_print_flag) { // now check the map, if there is one - PrivateIonValue l = (PrivateIonValue)s1; + _Private_IonValue l = (_Private_IonValue)s1; String map_error = l.validate(); if (map_error != null) { errors += map_error; diff --git a/test/software/amazon/ion/SurrogateEscapeTest.java b/test/com/amazon/ion/SurrogateEscapeTest.java similarity index 74% rename from test/software/amazon/ion/SurrogateEscapeTest.java rename to test/com/amazon/ion/SurrogateEscapeTest.java index 4c5c6ac405..86170ae471 100644 --- a/test/software/amazon/ion/SurrogateEscapeTest.java +++ b/test/com/amazon/ion/SurrogateEscapeTest.java @@ -1,38 +1,35 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; +import com.amazon.ion.impl._Private_Utils; import org.junit.After; import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonString; -import software.amazon.ion.IonType; -import software.amazon.ion.impl.PrivateUtils; public class SurrogateEscapeTest extends IonTestCase { private StringBuilder buf = new StringBuilder(); private IonDatagram load() { - byte[] utf8 = PrivateUtils.utf8(buf.toString()); + byte[] utf8 = _Private_Utils.utf8(buf.toString()); return loader().load(utf8); } private IonReader reader() { - byte[] utf8 = PrivateUtils.utf8(buf.toString()); + byte[] utf8 = _Private_Utils.utf8(buf.toString()); return system().newReader(utf8); } @@ -53,6 +50,9 @@ private void assertSingletonCodePoint(final int expectedCode) { assertSingleCodePoint(expectedCode, ((IonString) dg.get(0)).stringValue()); final IonReader reader = reader(); + if (! _Private_Utils.READER_HASNEXT_REMOVED) { + assertTrue(reader.hasNext()); + } assertEquals(IonType.STRING, reader.next()); assertSingleCodePoint(expectedCode, reader.stringValue()); } diff --git a/test/software/amazon/ion/SymbolTest.java b/test/com/amazon/ion/SymbolTest.java similarity index 87% rename from test/software/amazon/ion/SymbolTest.java rename to test/com/amazon/ion/SymbolTest.java index 853aafd3bb..9a44d1d3ad 100644 --- a/test/software/amazon/ion/SymbolTest.java +++ b/test/com/amazon/ion/SymbolTest.java @@ -1,31 +1,24 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; -import static software.amazon.ion.impl.PrivateUtils.newSymbolToken; +import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; +import static com.amazon.ion.impl._Private_Utils.newSymbolToken; import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonList; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.UnsupportedIonVersionException; -import software.amazon.ion.ValueFactory; @@ -39,6 +32,12 @@ public static void checkNullSymbol(IonSymbol value) assertTrue("isNullValue() is false", value.isNullValue()); assertNull("stringValue() isn't null", value.stringValue()); assertNull("symbolValue() isn't null", value.symbolValue()); + + try { + value.getSymbolId(); + fail("Expected NullValueException"); + } + catch (NullValueException e) { } } public void modifySymbol(IonSymbol value) @@ -55,7 +54,7 @@ public void modifySymbol(IonSymbol value) String modValue1 = "dude!"; value.setValue(modValue1); checkSymbol(modValue1, value); - int sid = value.symbolValue().getSid(); + int sid = value.getSymbolId(); checkSymbol(modValue1, sid, value); diff --git a/test/software/amazon/ion/SystemProcessingTestCase.java b/test/com/amazon/ion/SystemProcessingTestCase.java similarity index 96% rename from test/software/amazon/ion/SystemProcessingTestCase.java rename to test/com/amazon/ion/SystemProcessingTestCase.java index c72e8ffa22..6fce2cee24 100644 --- a/test/software/amazon/ion/SystemProcessingTestCase.java +++ b/test/com/amazon/ion/SystemProcessingTestCase.java @@ -1,36 +1,34 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; -import static software.amazon.ion.SystemSymbols.ION_1_0; -import static software.amazon.ion.SystemSymbols.ION_1_0_SID; -import static software.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE; -import static software.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE_SID; -import static software.amazon.ion.TestUtils.FERMATA; -import static software.amazon.ion.impl.Symtabs.LocalSymbolTablePrefix; +import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; +import static com.amazon.ion.SystemSymbols.ION_1_0; +import static com.amazon.ion.SystemSymbols.ION_1_0_SID; +import static com.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE; +import static com.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE_SID; +import static com.amazon.ion.TestUtils.FERMATA; +import static com.amazon.ion.impl.Symtabs.LocalSymbolTablePrefix; +import com.amazon.ion.impl.SymbolTableTest; +import com.amazon.ion.impl.Symtabs; +import com.amazon.ion.system.SimpleCatalog; import org.junit.Ignore; import org.junit.Test; -import software.amazon.ion.IonException; -import software.amazon.ion.IonType; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.Timestamp; -import software.amazon.ion.impl.SymbolTableTest; -import software.amazon.ion.impl.Symtabs; -import software.amazon.ion.system.SimpleCatalog; + public abstract class SystemProcessingTestCase @@ -194,7 +192,7 @@ protected abstract void checkEof() //========================================================================= - /** TODO amzn/ion-java#8 This is broken for loaders which are now more lazy */ + /** TODO amzn/ion-java/issues/8 This is broken for loaders which are now more lazy */ @Test @Ignore public void testLocalTableResetting() throws Exception @@ -1043,7 +1041,7 @@ public void testSystemIterationShowsIvm() checkEof(); } - // TODO amzn/ion-java#25 test for interspersed IVMs - testSystemIterationShowsInterspersedIvm + // TODO amzn/ion-java/issues/25 test for interspersed IVMs - testSystemIterationShowsInterspersedIvm @Test public void testHighUnicodeDirectInBlob() diff --git a/test/software/amazon/ion/SystemProcessingTests.java b/test/com/amazon/ion/SystemProcessingTests.java similarity index 74% rename from test/software/amazon/ion/SystemProcessingTests.java rename to test/com/amazon/ion/SystemProcessingTests.java index fb20ce16f2..64e77e7f40 100644 --- a/test/software/amazon/ion/SystemProcessingTests.java +++ b/test/com/amazon/ion/SystemProcessingTests.java @@ -1,22 +1,24 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import org.junit.runner.RunWith; import org.junit.runners.Suite; + @RunWith(Suite.class) @Suite.SuiteClasses({ TextIteratorSystemProcessingTest.class, diff --git a/test/software/amazon/ion/TestUtils.java b/test/com/amazon/ion/TestUtils.java similarity index 90% rename from test/software/amazon/ion/TestUtils.java rename to test/com/amazon/ion/TestUtils.java index a2b5cb68ac..c5fbfb2e76 100644 --- a/test/software/amazon/ion/TestUtils.java +++ b/test/com/amazon/ion/TestUtils.java @@ -1,22 +1,24 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import software.amazon.ion.impl.PrivateUtils; -import software.amazon.ion.util.IonStreamUtils; +import static com.amazon.ion.impl._Private_Utils.READER_HASNEXT_REMOVED; +import com.amazon.ion.impl._Private_Utils; +import com.amazon.ion.util.IonStreamUtils; import java.io.File; import java.io.FilenameFilter; import java.math.BigDecimal; @@ -27,6 +29,7 @@ import java.util.Iterator; import java.util.List; + public class TestUtils { public static final String US_ASCII_CHARSET_NAME = "US-ASCII"; @@ -129,15 +132,14 @@ public boolean accept(File dir, String name) public static final FilenameFilter GLOBAL_SKIP_LIST = new FileIsNot( - "bad/clobWithNullCharacter.ion" // TODO amzn/ion-java#43 - , "bad/emptyAnnotatedInt.10n" // TODO amzn/ion-java#55 - , "equivs/paddedInts.10n" // TODO amzn/ion-java#54 - , "good/subfieldVarUInt32bit.ion" // TODO amzn/ion-java#62 - , "good/utf16.ion" // TODO amzn/ion-java#61 - , "good/utf32.ion" // TODO amzn/ion-java#61 - , "good/whitespace.ion" // TODO amzn/ion-java#104 - , "good/item1.10n" // TODO amzn/ion-java#126 (roundtrip symbols with unknown text) - ); + "bad/clobWithNullCharacter.ion" // TODO amzn/ion-java/43 + ,"bad/emptyAnnotatedInt.10n" // TODO amzn/ion-java/55 + ,"good/subfieldVarUInt32bit.ion" // TODO amzn/ion-java/62 + ,"good/utf16.ion" // TODO amzn/ion-java/61 + ,"good/utf32.ion" // TODO amzn/ion-java/61 + ,"good/whitespace.ion" + , "good/item1.10n" // TODO amzn/ion-java#126 (roundtrip symbols with unknown text) + ); private static void testdataFiles(FilenameFilter filter, @@ -228,7 +230,7 @@ public static byte[] ensureText(IonSystem system, byte[] ionData) IonDatagram dg = system.getLoader().load(ionData); String ionText = dg.toString(); - return PrivateUtils.utf8(ionText); + return _Private_Utils.utf8(ionText); } @@ -306,9 +308,19 @@ public static void deepRead(IonReader reader, boolean flgMaterializeScalars) } } + @SuppressWarnings("deprecation") private static IonType doNext(IonReader reader) { - return reader.next(); + boolean hasnext = true; + IonType t = null; + + if (! READER_HASNEXT_REMOVED) { + hasnext = reader.hasNext(); + } + if (hasnext) { + t = reader.next(); + } + return t; } @@ -371,7 +383,7 @@ private static void materializeScalar(IonReader reader) public static String hexDump(final String str) { - final byte[] utf16Bytes = PrivateUtils.encode(str, UTF16BE_CHARSET); + final byte[] utf16Bytes = _Private_Utils.encode(str, UTF16BE_CHARSET); StringBuilder buf = new StringBuilder(utf16Bytes.length * 4); for (byte b : utf16Bytes) { buf.append(Integer.toString(0x00FF & b, 16)); @@ -527,7 +539,7 @@ public static boolean symbolTableEquals(final SymbolTable first, final SymbolTab static { - if (! PrivateUtils.utf8(FERMATA_UTF8).equals(FERMATA)) + if (! _Private_Utils.utf8(FERMATA_UTF8).equals(FERMATA)) { throw new AssertionError("Broken encoding"); } diff --git a/test/software/amazon/ion/TextByteArrayIteratorSystemProcessingTest.java b/test/com/amazon/ion/TextByteArrayIteratorSystemProcessingTest.java similarity index 58% rename from test/software/amazon/ion/TextByteArrayIteratorSystemProcessingTest.java rename to test/com/amazon/ion/TextByteArrayIteratorSystemProcessingTest.java index 2115881ff4..be8ae0ef70 100644 --- a/test/software/amazon/ion/TextByteArrayIteratorSystemProcessingTest.java +++ b/test/com/amazon/ion/TextByteArrayIteratorSystemProcessingTest.java @@ -1,22 +1,22 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; +import com.amazon.ion.impl._Private_Utils; import java.util.Iterator; -import software.amazon.ion.IonValue; -import software.amazon.ion.impl.PrivateUtils; public class TextByteArrayIteratorSystemProcessingTest @@ -28,7 +28,7 @@ public class TextByteArrayIteratorSystemProcessingTest protected void prepare(String text) throws Exception { - myBytes = PrivateUtils.convertUtf16UnitsToUtf8(text); + myBytes = _Private_Utils.convertUtf16UnitsToUtf8(text); } @Override diff --git a/test/software/amazon/ion/TextIteratorSystemProcessingTest.java b/test/com/amazon/ion/TextIteratorSystemProcessingTest.java similarity index 61% rename from test/software/amazon/ion/TextIteratorSystemProcessingTest.java rename to test/com/amazon/ion/TextIteratorSystemProcessingTest.java index 32f490273e..f16da9245d 100644 --- a/test/software/amazon/ion/TextIteratorSystemProcessingTest.java +++ b/test/com/amazon/ion/TextIteratorSystemProcessingTest.java @@ -1,22 +1,22 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; +import com.amazon.ion.impl._Private_IonSystem; import java.util.Iterator; -import software.amazon.ion.IonValue; -import software.amazon.ion.impl.PrivateIonSystem; public class TextIteratorSystemProcessingTest @@ -36,7 +36,7 @@ protected Iterator iterate() protected Iterator systemIterate() throws Exception { - PrivateIonSystem sys = system(); + _Private_IonSystem sys = system(); Iterator it = sys.systemIterate(myText); return it; } diff --git a/test/software/amazon/ion/TextReaderSystemProcessingTest.java b/test/com/amazon/ion/TextReaderSystemProcessingTest.java similarity index 61% rename from test/software/amazon/ion/TextReaderSystemProcessingTest.java rename to test/com/amazon/ion/TextReaderSystemProcessingTest.java index 9901334745..2e04b88abf 100644 --- a/test/software/amazon/ion/TextReaderSystemProcessingTest.java +++ b/test/com/amazon/ion/TextReaderSystemProcessingTest.java @@ -1,20 +1,23 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; + + + -import software.amazon.ion.IonReader; public class TextReaderSystemProcessingTest extends ReaderSystemProcessingTestCase diff --git a/test/software/amazon/ion/TextStreamIteratorSystemProcessingTest.java b/test/com/amazon/ion/TextStreamIteratorSystemProcessingTest.java similarity index 62% rename from test/software/amazon/ion/TextStreamIteratorSystemProcessingTest.java rename to test/com/amazon/ion/TextStreamIteratorSystemProcessingTest.java index a716a80ef7..87a92b25da 100644 --- a/test/software/amazon/ion/TextStreamIteratorSystemProcessingTest.java +++ b/test/com/amazon/ion/TextStreamIteratorSystemProcessingTest.java @@ -1,24 +1,24 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; +import com.amazon.ion.impl._Private_Utils; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.util.Iterator; -import software.amazon.ion.IonValue; -import software.amazon.ion.impl.PrivateUtils; public class TextStreamIteratorSystemProcessingTest @@ -31,7 +31,7 @@ public class TextStreamIteratorSystemProcessingTest protected void prepare(String text) throws Exception { - myBytes = PrivateUtils.convertUtf16UnitsToUtf8(text); + myBytes = _Private_Utils.convertUtf16UnitsToUtf8(text); myStream = new ByteArrayInputStream(myBytes); } diff --git a/test/software/amazon/ion/TextTestCase.java b/test/com/amazon/ion/TextTestCase.java similarity index 79% rename from test/software/amazon/ion/TextTestCase.java rename to test/com/amazon/ion/TextTestCase.java index 6e332c7a7f..6f5e3927b7 100644 --- a/test/software/amazon/ion/TextTestCase.java +++ b/test/com/amazon/ion/TextTestCase.java @@ -1,21 +1,23 @@ + /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; + +package com.amazon.ion; import org.junit.Test; -import software.amazon.ion.IonText; -import software.amazon.ion.IonValue; + public abstract class TextTestCase extends IonTestCase diff --git a/test/software/amazon/ion/TimestampBadTest.java b/test/com/amazon/ion/TimestampBadTest.java similarity index 60% rename from test/software/amazon/ion/TimestampBadTest.java rename to test/com/amazon/ion/TimestampBadTest.java index c3a133ba88..ede4ad4144 100644 --- a/test/software/amazon/ion/TimestampBadTest.java +++ b/test/com/amazon/ion/TimestampBadTest.java @@ -1,32 +1,32 @@ /* - * Copyright 2013-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static software.amazon.ion.TestUtils.BAD_TIMESTAMP_IONTESTS_FILES; -import static software.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; -import static software.amazon.ion.TestUtils.testdataFiles; +import static com.amazon.ion.TestUtils.BAD_TIMESTAMP_IONTESTS_FILES; +import static com.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; +import static com.amazon.ion.TestUtils.testdataFiles; +import com.amazon.ion.impl._Private_Utils; +import com.amazon.ion.junit.Injected.Inject; import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.StringReader; import org.junit.Test; -import software.amazon.ion.IonException; -import software.amazon.ion.Timestamp; -import software.amazon.ion.impl.PrivateUtils; -import software.amazon.ion.junit.Injected.Inject; + public class TimestampBadTest extends IonTestCase @@ -50,7 +50,7 @@ public void testValueOf() String tsText; try { - tsText = PrivateUtils.utf8FileToString(myTestFile); + tsText = _Private_Utils.utf8FileToString(myTestFile); } catch (IonException e) { diff --git a/test/software/amazon/ion/TimestampGoodTest.java b/test/com/amazon/ion/TimestampGoodTest.java similarity index 68% rename from test/software/amazon/ion/TimestampGoodTest.java rename to test/com/amazon/ion/TimestampGoodTest.java index c01c37ee8b..e5c2167fc2 100644 --- a/test/software/amazon/ion/TimestampGoodTest.java +++ b/test/com/amazon/ion/TimestampGoodTest.java @@ -1,35 +1,33 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; -import static software.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; -import static software.amazon.ion.TestUtils.GOOD_TIMESTAMP_IONTESTS_FILES; -import static software.amazon.ion.TestUtils.testdataFiles; +import static com.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; +import static com.amazon.ion.TestUtils.GOOD_TIMESTAMP_IONTESTS_FILES; +import static com.amazon.ion.TestUtils.testdataFiles; +import com.amazon.ion.impl._Private_Utils; +import com.amazon.ion.junit.Injected.Inject; import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.StringReader; import java.util.Iterator; import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonTimestamp; -import software.amazon.ion.IonValue; -import software.amazon.ion.Timestamp; -import software.amazon.ion.impl.PrivateUtils; -import software.amazon.ion.junit.Injected.Inject; + public class TimestampGoodTest extends IonTestCase @@ -70,7 +68,7 @@ public void testValueOf() { if (! myTestFile.getName().endsWith(".ion")) return; - String fileText = PrivateUtils.utf8FileToString(myTestFile); + String fileText = _Private_Utils.utf8FileToString(myTestFile); BufferedReader reader = new BufferedReader(new StringReader(fileText)); String line = reader.readLine(); diff --git a/test/software/amazon/ion/TimestampTest.java b/test/com/amazon/ion/TimestampTest.java similarity index 93% rename from test/software/amazon/ion/TimestampTest.java rename to test/com/amazon/ion/TimestampTest.java index 322898aef8..eef97cb463 100644 --- a/test/software/amazon/ion/TimestampTest.java +++ b/test/com/amazon/ion/TimestampTest.java @@ -1,40 +1,41 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; - -import static software.amazon.ion.Decimal.NEGATIVE_ZERO; -import static software.amazon.ion.Decimal.negativeZero; -import static software.amazon.ion.Timestamp.MAXIMUM_ALLOWED_TIMESTAMP_IN_MILLIS_DECIMAL; -import static software.amazon.ion.Timestamp.MINIMUM_TIMESTAMP_IN_MILLIS; -import static software.amazon.ion.Timestamp.MINIMUM_TIMESTAMP_IN_MILLIS_DECIMAL; -import static software.amazon.ion.Timestamp.UNKNOWN_OFFSET; -import static software.amazon.ion.Timestamp.UTC_OFFSET; -import static software.amazon.ion.Timestamp.createFromUtcFields; -import static software.amazon.ion.Timestamp.Precision.DAY; -import static software.amazon.ion.Timestamp.Precision.MINUTE; -import static software.amazon.ion.Timestamp.Precision.MONTH; -import static software.amazon.ion.Timestamp.Precision.SECOND; -import static software.amazon.ion.Timestamp.Precision.YEAR; -import static software.amazon.ion.impl.PrivateUtils.UTC; - +package com.amazon.ion; + +import static com.amazon.ion.Decimal.NEGATIVE_ZERO; +import static com.amazon.ion.Decimal.negativeZero; +import static com.amazon.ion.Timestamp.MAXIMUM_ALLOWED_TIMESTAMP_IN_MILLIS_DECIMAL; +import static com.amazon.ion.Timestamp.MINIMUM_TIMESTAMP_IN_MILLIS; +import static com.amazon.ion.Timestamp.MINIMUM_TIMESTAMP_IN_MILLIS_DECIMAL; +import static com.amazon.ion.Timestamp.UNKNOWN_OFFSET; +import static com.amazon.ion.Timestamp.UTC_OFFSET; +import static com.amazon.ion.Timestamp.createFromUtcFields; +import static com.amazon.ion.Timestamp.Precision.DAY; +import static com.amazon.ion.Timestamp.Precision.FRACTION; +import static com.amazon.ion.Timestamp.Precision.MINUTE; +import static com.amazon.ion.Timestamp.Precision.MONTH; +import static com.amazon.ion.Timestamp.Precision.SECOND; +import static com.amazon.ion.Timestamp.Precision.YEAR; +import static com.amazon.ion.impl._Private_Utils.UTC; + +import com.amazon.ion.Timestamp.Precision; import java.math.BigDecimal; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.time.Instant; -import java.time.ZoneId; import java.util.Calendar; import java.util.Date; import java.util.GregorianCalendar; @@ -42,7 +43,6 @@ import org.junit.Ignore; import org.junit.Test; -import software.amazon.ion.Timestamp.Precision; /** * Validates Ion date parsing, specified as per W3C but with requiring @@ -155,7 +155,8 @@ private void checkFields(int expectedYear, int expectedMonth, int expectedDay, expectedFraction, expectedOffset, ts); - assertEquals(expectedPrecision, ts.getPrecision()); + assertTrue(expectedPrecision.equals(ts.getPrecision()) + || expectedPrecision.includes(Precision.SECOND) == ts.getPrecision().includes(Precision.SECOND)); @SuppressWarnings("deprecation") Timestamp ts2 = @@ -173,25 +174,20 @@ private void checkFields(int expectedYear, int expectedMonth, int expectedDay, Integer expectedOffset, Timestamp ts) { - BigDecimal expectedDecimalSeconds = BigDecimal.valueOf(expectedSecond); - if (expectedFraction != null) { - expectedDecimalSeconds = expectedDecimalSeconds.add(expectedFraction); - } - assertEquals("year", expectedYear, ts.getYear()); assertEquals("month", expectedMonth, ts.getMonth()); assertEquals("day", expectedDay, ts.getDay()); assertEquals("hour", expectedHour, ts.getHour()); assertEquals("minute", expectedMinute, ts.getMinute()); assertEquals("second", expectedSecond, ts.getSecond()); - assertEquals("fraction", expectedDecimalSeconds, ts.getDecimalSecond()); + assertEquals("fraction", expectedFraction, ts.getFractionalSecond()); assertEquals("offset", expectedOffset, ts.getLocalOffset()); } private void checkTime(int expectedYear, int expectedMonth, int expectedDay, String expectedText, String ionText) { - Timestamp ts = Timestamp.forDay(expectedYear, expectedMonth, expectedDay); + Timestamp ts = new Timestamp(expectedYear, expectedMonth, expectedDay); checkFields(expectedYear, expectedMonth, expectedDay, 0, 0, 0, null, UNKNOWN_OFFSET, DAY, ts); assertEquals(expectedText, ts.toString()); @@ -364,7 +360,7 @@ public void testTimestampConstants() assertEquals("0001-01-01", EARLIEST_ION_TIMESTAMP.toString()); assertEquals(-62135769600000L, EARLIEST_ION_TIMESTAMP.getMillis()); - checkFields(1970, 1, 1, 0, 0, 0, new BigDecimal("0.000"), 0, SECOND, UNIX_EPOCH_TIMESTAMP); + checkFields(1970, 1, 1, 0, 0, 0, new BigDecimal("0.000"), 0, FRACTION, UNIX_EPOCH_TIMESTAMP); assertEquals("1970-01-01T00:00:00.000Z", UNIX_EPOCH_TIMESTAMP.toZString()); assertEquals("1970-01-01T00:00:00.000Z", UNIX_EPOCH_TIMESTAMP.toString()); assertEquals(0L, UNIX_EPOCH_TIMESTAMP.getMillis()); @@ -721,65 +717,83 @@ public void testMonthBoundaries() public void testNewTimestamp() { // ===== Test on Timestamp(...) date precise constructor ===== - Timestamp ts = Timestamp.forDay(2010, 2, 1); + Timestamp ts = new Timestamp(2010, 2, 1); checkFields(2010, 2, 1, 0, 0, 0, null, UNKNOWN_OFFSET, DAY, ts); assertEquals("2010-02-01", ts.toString()); assertEquals("2010-02-01", ts.toZString()); // ===== Test on Timestamp(...) minute precise constructor ===== - ts = Timestamp.forMinute(2010, 2, 1, 10, 11, PST_OFFSET); + ts = new Timestamp(2010, 2, 1, 10, 11, PST_OFFSET); checkFields(2010, 2, 1, 10, 11, 0, null, PST_OFFSET, MINUTE, ts); assertEquals("2010-02-01T10:11-08:00", ts.toString()); assertEquals("2010-02-01T18:11Z", ts.toZString()); - ts = Timestamp.forMinute(2010, 2, 1, 10, 11, null); + ts = new Timestamp(2010, 2, 1, 10, 11, null); checkFields(2010, 2, 1, 10, 11, 0, null, null, MINUTE, ts); assertEquals("2010-02-01T10:11-00:00", ts.toString()); assertEquals("2010-02-01T10:11Z", ts.toZString()); // ===== Test on Timestamp(...) second precise constructor ===== - ts = Timestamp.forSecond(2010, 2, 1, 10, 11, 12, PST_OFFSET); + ts = new Timestamp(2010, 2, 1, 10, 11, 12, PST_OFFSET); checkFields(2010, 2, 1, 10, 11, 12, null, PST_OFFSET, SECOND, ts); assertEquals("2010-02-01T10:11:12-08:00", ts.toString()); assertEquals("2010-02-01T18:11:12Z", ts.toZString()); - ts = Timestamp.forSecond(2010, 2, 1, 10, 11, 12, null); + ts = new Timestamp(2010, 2, 1, 10, 11, 12, null); checkFields(2010, 2, 1, 10, 11, 12, null, null, SECOND, ts); assertEquals("2010-02-01T10:11:12-00:00", ts.toString()); assertEquals("2010-02-01T10:11:12Z", ts.toZString()); - BigDecimal frac = new BigDecimal("0.34"); - BigDecimal decimalSeconds = new BigDecimal("12.34"); - ts = Timestamp.forSecond(2010, 2, 1, 10, 11, decimalSeconds, PST_OFFSET); - checkFields(2010, 2, 1, 10, 11, 12, frac, PST_OFFSET, SECOND, ts); + // ===== Test on Timestamp(...) fractional second precise constructor ===== + BigDecimal fraction = new BigDecimal(".34"); + ts = new Timestamp(2010, 2, 1, 10, 11, 12, fraction, PST_OFFSET); + checkFields(2010, 2, 1, 10, 11, 12, fraction, PST_OFFSET, FRACTION, ts); + assertEquals("2010-02-01T10:11:12.34-08:00", ts.toString()); + assertEquals("2010-02-01T18:11:12.34Z", ts.toZString()); + + ts = new Timestamp(2010, 2, 1, 10, 11, 12, fraction, null); + checkFields(2010, 2, 1, 10, 11, 12, fraction, null, FRACTION, ts); + assertEquals("2010-02-01T10:11:12.34-00:00", ts.toString()); + assertEquals("2010-02-01T10:11:12.34Z", ts.toZString()); + + // This was broken. It had FRACTION precision, but no + // fractional data, and didn't equal the same value created other ways. + ts = new Timestamp(2010, 2, 1, 10, 11, 12, BigDecimal.ZERO, 0); + assertEquals(Timestamp.valueOf("2010-02-01T10:11:12Z"), ts); + checkFields(2010, 2, 1, 10, 11, 12, null, 0, SECOND, ts); + + // New static method unifies decimal seconds + BigDecimal second = new BigDecimal("12.34"); + ts = Timestamp.forSecond(2010, 2, 1, 10, 11, second, PST_OFFSET); + checkFields(2010, 2, 1, 10, 11, 12, fraction, PST_OFFSET, FRACTION, ts); assertEquals("2010-02-01T10:11:12.34-08:00", ts.toString()); assertEquals("2010-02-01T18:11:12.34Z", ts.toZString()); - ts = Timestamp.forSecond(2010, 2, 1, 10, 11, decimalSeconds, null); - checkFields(2010, 2, 1, 10, 11, 12, frac, null, SECOND, ts); + ts = Timestamp.forSecond(2010, 2, 1, 10, 11, second, null); + checkFields(2010, 2, 1, 10, 11, 12, fraction, null, FRACTION, ts); assertEquals("2010-02-01T10:11:12.34-00:00", ts.toString()); assertEquals("2010-02-01T10:11:12.34Z", ts.toZString()); - decimalSeconds = new BigDecimal("12."); - ts = Timestamp.forSecond(2010, 2, 1, 10, 11, decimalSeconds, PST_OFFSET); + second = new BigDecimal("12."); + ts = Timestamp.forSecond(2010, 2, 1, 10, 11, second, PST_OFFSET); checkFields(2010, 2, 1, 10, 11, 12, null, PST_OFFSET, SECOND, ts); assertEquals("2010-02-01T10:11:12-08:00", ts.toString()); assertEquals("2010-02-01T18:11:12Z", ts.toZString()); } - /** Test for {@link Timestamp#forMillis(BigDecimal, Integer)} */ + /** Test for {@link Timestamp#Timestamp(BigDecimal, Integer)} */ @Test public void testNewTimestampFromBigDecimal() { BigDecimal bigDec = new BigDecimal("1325419950555.123"); - Timestamp ts = Timestamp.forMillis(bigDec, PST_OFFSET); - checkFields(2012, 1, 1, 4, 12, 30, new BigDecimal("0.555123"), PST_OFFSET, SECOND, ts); + Timestamp ts = new Timestamp(bigDec, PST_OFFSET); + checkFields(2012, 1, 1, 4, 12, 30, new BigDecimal("0.555123"), PST_OFFSET, FRACTION, ts); assertEquals("2012-01-01T04:12:30.555123-08:00", ts.toString()); assertEquals("2012-01-01T12:12:30.555123Z", ts.toZString()); - ts = Timestamp.forMillis(bigDec, null); - checkFields(2012, 1, 1, 12, 12, 30, new BigDecimal("0.555123"), null, SECOND, ts); + ts = new Timestamp(bigDec, null); + checkFields(2012, 1, 1, 12, 12, 30, new BigDecimal("0.555123"), null, FRACTION, ts); assertEquals("2012-01-01T12:12:30.555123-00:00", ts.toString()); assertEquals("2012-01-01T12:12:30.555123Z", ts.toZString()); } @@ -809,7 +823,7 @@ public void testNewMinimumTimestampFromStringAndMillisIsSame() { public void testNewTimestampFromBigDecimalWithMaximumAllowedMillis() { Timestamp ts = Timestamp.forMillis(MAXIMUM_ALLOWED_TIMESTAMP_IN_MILLIS_DECIMAL.add(BigDecimal.ONE.negate()), PST_OFFSET); - checkFields(9999, 12, 31, 15, 59, 59, new BigDecimal("0.999"), PST_OFFSET, SECOND, ts); + checkFields(9999, 12, 31, 15, 59, 59, new BigDecimal("0.999"), PST_OFFSET, FRACTION, ts); assertEquals("9999-12-31T15:59:59.999-08:00", ts.toString()); assertEquals("9999-12-31T23:59:59.999Z", ts.toZString()); } @@ -853,7 +867,7 @@ public void testNewTimestampFromBigDecimalWithScaleTooBigPositive() } /** - * Test for {@link Timestamp#forMillis(BigDecimal, Integer)}, + * Test for {@link Timestamp#Timestamp(BigDecimal, Integer)}, * ensuring that varying BigDecimals with different scales produce * Timestamps (with correct precision of second/fractional seconds) as * expected. @@ -881,44 +895,44 @@ public void testNewTimestampFromBigDecimalWithDifferentScales() Timestamp ts; - ts = Timestamp.forMillis(decScaleNegFour, null); + ts = new Timestamp(decScaleNegFour, null); checkFields(2012, 1, 1, 12, 12, 30, null, null, SECOND, ts); - ts = Timestamp.forMillis(decScaleNegThree, null); + ts = new Timestamp(decScaleNegThree, null); checkFields(2012, 1, 1, 12, 12, 30, null, null, SECOND, ts); - ts = Timestamp.forMillis(decScaleNegTwo, null); - checkFields(2012, 1, 1, 12, 12, 30, new BigDecimal("0.5"), null, SECOND, ts); + ts = new Timestamp(decScaleNegTwo, null); + checkFields(2012, 1, 1, 12, 12, 30, new BigDecimal("0.5"), null, FRACTION, ts); - ts = Timestamp.forMillis(decScaleNegOne, null); - checkFields(2012, 1, 1, 12, 12, 30, new BigDecimal("0.55"), null, SECOND, ts); + ts = new Timestamp(decScaleNegOne, null); + checkFields(2012, 1, 1, 12, 12, 30, new BigDecimal("0.55"), null, FRACTION, ts); - ts = Timestamp.forMillis(decScaleZero, null); - checkFields(2012, 1, 1, 12, 12, 30, new BigDecimal("0.555"), null, SECOND, ts); + ts = new Timestamp(decScaleZero, null); + checkFields(2012, 1, 1, 12, 12, 30, new BigDecimal("0.555"), null, FRACTION, ts); - ts = Timestamp.forMillis(decScalePosOne, null); - checkFields(2012, 1, 1, 12, 12, 30, new BigDecimal("0.5555"), null, SECOND, ts); + ts = new Timestamp(decScalePosOne, null); + checkFields(2012, 1, 1, 12, 12, 30, new BigDecimal("0.5555"), null, FRACTION, ts); - ts = Timestamp.forMillis(new BigDecimal("0e3"), null); + ts = new Timestamp(new BigDecimal("0e3"), null); checkFields(1970, 1, 1, 0, 0, 0, null, null, SECOND, ts); - ts = Timestamp.forMillis(new BigDecimal("1e3"), null); + ts = new Timestamp(new BigDecimal("1e3"), null); checkFields(1970, 1, 1, 0, 0, 1, null, null, SECOND, ts); } - /** Test for {@link Timestamp#forMillis(long, Integer)} */ + /** Test for {@link Timestamp#Timestamp(long, Integer)} */ @Test public void testNewTimestampFromLong() { long actualMillis = 1265019072340L; - Timestamp ts = Timestamp.forMillis(actualMillis, PST_OFFSET); - checkFields(2010, 2, 1, 2, 11, 12, new BigDecimal("0.340"), PST_OFFSET, SECOND, ts); + Timestamp ts = new Timestamp(actualMillis, PST_OFFSET); + checkFields(2010, 2, 1, 2, 11, 12, new BigDecimal("0.340"), PST_OFFSET, FRACTION, ts); assertEquals("2010-02-01T02:11:12.340-08:00", ts.toString()); assertEquals("2010-02-01T10:11:12.340Z", ts.toZString()); - ts = Timestamp.forMillis(actualMillis, null); - checkFields(2010, 2, 1, 10, 11, 12, new BigDecimal("0.340"), null, SECOND, ts); + ts = new Timestamp(actualMillis, null); + checkFields(2010, 2, 1, 10, 11, 12, new BigDecimal("0.340"), null, FRACTION, ts); assertEquals("2010-02-01T10:11:12.340-00:00", ts.toString()); assertEquals("2010-02-01T10:11:12.340Z", ts.toZString()); } @@ -967,6 +981,11 @@ public void testNewTimestampFromUtcFieldsWithDifferentPrecisions() p = SECOND; ts = Timestamp.createFromUtcFields(p, zyear, zmonth, zday, zhour, zminute, zsecond, zfrac, offset); + // 2012-02-03T04:05:06-00:00 + checkFields(zyear, zmonth, zday, zhour, zminute, zsecond, zfrac, offset, p, ts); + + p = FRACTION; + ts = Timestamp.createFromUtcFields(p, zyear, zmonth, zday, zhour, zminute, zsecond, zfrac, offset); // 2012-02-03T04:05:06.007-00:00 checkFields(zyear, zmonth, zday, zhour, zminute, zsecond, zfrac, offset, p, ts); @@ -989,7 +1008,7 @@ public void testNewTimestampFromUtcFieldsWithDifferentOffsets() int zminute = 5; int zsecond = 6; BigDecimal zfrac = new BigDecimal("0.007"); - Precision p = SECOND; + Precision p = FRACTION; // Varying local offsets (in minutes) Integer offset; @@ -1041,7 +1060,7 @@ public void testNewTimestampFromCalendar() cal.set(Calendar.YEAR, 2009); assertFalse(cal.isSet(Calendar.MONTH)); - Timestamp ts = Timestamp.forCalendar(cal); + Timestamp ts = new Timestamp(cal); checkFields(2009, 1, 1, 0, 0, 0, null, UNKNOWN_OFFSET, YEAR, ts); assertEquals("2009T", ts.toString()); assertEquals("2009T", ts.toZString()); @@ -1052,7 +1071,7 @@ public void testNewTimestampFromCalendar() cal.set(Calendar.MONTH, 2); assertFalse(cal.isSet(Calendar.HOUR_OF_DAY)); - ts = Timestamp.forCalendar(cal); + ts = new Timestamp(cal); checkFields(2009, 3, 1, 0, 0, 0, null, UNKNOWN_OFFSET, MONTH, ts); assertEquals("2009-03T", ts.toString()); assertEquals("2009-03T", ts.toZString()); @@ -1063,7 +1082,7 @@ public void testNewTimestampFromCalendar() assertFalse(cal.isSet(Calendar.HOUR_OF_DAY) && cal.isSet(Calendar.MINUTE)); - ts = Timestamp.forCalendar(cal); + ts = new Timestamp(cal); checkFields(2009, 3, 18, 0, 0, 0, null, UNKNOWN_OFFSET, DAY, ts); assertEquals("2009-03-18", ts.toString()); assertEquals("2009-03-18", ts.toZString()); @@ -1073,7 +1092,7 @@ public void testNewTimestampFromCalendar() cal.set(2009, 1, 1, 10, 11); assertFalse(cal.isSet(Calendar.SECOND)); - ts = Timestamp.forCalendar(cal); + ts = new Timestamp(cal); checkFields(2009, 2, 1, 10, 11, 0, null, 0, MINUTE, ts); assertEquals("2009-02-01T10:11Z", ts.toString()); assertEquals("2009-02-01T10:11Z", ts.toZString()); @@ -1083,7 +1102,7 @@ public void testNewTimestampFromCalendar() cal.set(2009, 1, 1, 10, 11, 12); assertFalse(cal.isSet(Calendar.MILLISECOND)); - ts = Timestamp.forCalendar(cal); + ts = new Timestamp(cal); checkFields(2009, 2, 1, 10, 11, 12, null, 0, SECOND, ts); assertEquals("2009-02-01T10:11:12Z", ts.toString()); assertEquals("2009-02-01T10:11:12Z", ts.toZString()); @@ -1094,8 +1113,8 @@ public void testNewTimestampFromCalendar() cal.set(Calendar.MILLISECOND, 345); assertTrue(cal.isSet(Calendar.MILLISECOND)); - ts = Timestamp.forCalendar(cal); - checkFields(2009, 2, 1, 10, 11, 12, new BigDecimal("0.345"), 0, SECOND, ts); + ts = new Timestamp(cal); + checkFields(2009, 2, 1, 10, 11, 12, new BigDecimal("0.345"), 0, FRACTION, ts); assertEquals("2009-02-01T10:11:12.345Z", ts.toString()); assertEquals("2009-02-01T10:11:12.345Z", ts.toZString()); @@ -1106,8 +1125,8 @@ public void testNewTimestampFromCalendar() cal.setTimeZone(TimeZone.getTimeZone("PST")); assertEquals(TimeZone.getTimeZone("PST"), cal.getTimeZone()); - ts = Timestamp.forCalendar(cal); - checkFields(2009, 2, 1, 10, 11, 12, new BigDecimal("0.345"), PST_OFFSET, SECOND, ts); + ts = new Timestamp(cal); + checkFields(2009, 2, 1, 10, 11, 12, new BigDecimal("0.345"), PST_OFFSET, FRACTION, ts); assertEquals("2009-02-01T10:11:12.345-08:00", ts.toString()); assertEquals("2009-02-01T18:11:12.345Z", ts.toZString()); @@ -1117,8 +1136,8 @@ public void testNewTimestampFromCalendar() cal.set(Calendar.MILLISECOND, 345); cal.set(Calendar.ZONE_OFFSET, -28800000); - ts = Timestamp.forCalendar(cal); - checkFields(2009, 2, 1, 10, 11, 12, new BigDecimal("0.345"), PST_OFFSET, SECOND, ts); + ts = new Timestamp(cal); + checkFields(2009, 2, 1, 10, 11, 12, new BigDecimal("0.345"), PST_OFFSET, FRACTION, ts); assertEquals("2009-02-01T10:11:12.345-08:00", ts.toString()); assertEquals("2009-02-01T18:11:12.345Z", ts.toZString()); } @@ -1132,14 +1151,14 @@ public void testFromCalendarWithDaylightSavingsTime() cal.set(2012, 2, 9, 10, 11, 12); cal.set(Calendar.MILLISECOND, 345); cal.setTimeZone(TimeZone.getTimeZone("PST")); - Timestamp ts = Timestamp.forCalendar(cal); - checkFields(2012, 3, 9, 10, 11, 12, new BigDecimal("0.345"), PST_OFFSET, SECOND, ts); + Timestamp ts = new Timestamp(cal); + checkFields(2012, 3, 9, 10, 11, 12, new BigDecimal("0.345"), PST_OFFSET, FRACTION, ts); assertEquals("2012-03-09T10:11:12.345-08:00", ts.toString()); // Move forward into daylight savings cal.add(Calendar.DAY_OF_MONTH, 5); - ts = Timestamp.forCalendar(cal); - checkFields(2012, 3, 14, 10, 11, 12, new BigDecimal("0.345"), -420, SECOND, ts); + ts = new Timestamp(cal); + checkFields(2012, 3, 14, 10, 11, 12, new BigDecimal("0.345"), -420, FRACTION, ts); assertEquals("2012-03-14T10:11:12.345-07:00", ts.toString()); } @@ -1149,24 +1168,36 @@ public void testNewTimestampFromClearCalendar() { Calendar cal = makeUtcCalendar(); cal.clear(); - Timestamp.forCalendar(cal); + new Timestamp(cal); } - @SuppressWarnings("deprecation") + + @Test + public void testTimestampWithNegativeFraction() + { + BigDecimal frac = Decimal.negativeZero(3); + + Timestamp ts = new Timestamp(2000, 11, 14, 17, 30, 12, frac, 0); + Timestamp expected = Timestamp.valueOf("2000-11-14T17:30:12.000Z"); + assertEquals(expected, ts); + assertEquals(expected.hashCode(), ts.hashCode()); + + frac = new BigDecimal("-0.123"); + ts = new Timestamp(2000, 11, 14, 17, 30, 12, frac, 0); + assertEquals("2000-11-14T17:30:12.123Z", ts.toString()); + } + + private void checkFraction(String textOfFrac, BigDecimal frac) { Timestamp expected = Timestamp.valueOf("2000-11-14T17:30:12" + textOfFrac + "Z"); - // negative fractions are not meaningful for checking second construction - if (frac.signum() >= 0) { - BigDecimal seconds = BigDecimal.valueOf(12).add(frac); - Timestamp ts = Timestamp.forSecond(2000, 11, 14, 17, 30, seconds, 0); - assertEquals(expected, ts); - assertEquals("hash code", expected.hashCode(), ts.hashCode()); - } + Timestamp ts = new Timestamp(2000, 11, 14, 17, 30, 12, frac, 0); + assertEquals(expected, ts); + assertEquals("hash code", expected.hashCode(), ts.hashCode()); - Timestamp ts = createFromUtcFields(SECOND, 2000, 11, 14, 17, 30, 12, frac, + ts = createFromUtcFields(FRACTION, 2000, 11, 14, 17, 30, 12, frac, UTC_OFFSET); assertEquals(expected, ts); assertEquals("hash code", expected.hashCode(), ts.hashCode()); @@ -1189,9 +1220,14 @@ public void testTimestampWithDecimalFraction() public void testNewTimestampWithLargeFraction() { BigDecimal frac = new BigDecimal("1.23"); + try { + new Timestamp(2000, 11, 14, 17, 30, 12, frac, 0); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { } try { - Timestamp.createFromUtcFields(SECOND, + Timestamp.createFromUtcFields(FRACTION, 2000, 11, 14, 17, 30, 12, frac, 0); fail("Expected exception"); } @@ -1345,8 +1381,12 @@ public void testForSqlTimestampZ() assertEquals(now.getTime(), ts.getMillis()); assertEquals(Timestamp.UTC_OFFSET, ts.getLocalOffset()); + BigDecimal frac = ts.getFractionalSecond(); + frac = frac.movePointRight(9); // Convert to nanos + assertEquals("nanos", now.getNanos(), frac.intValue()); + BigDecimal second = BigDecimal.valueOf(ts.getSecond()); - BigDecimal frac = ts.getDecimalSecond().subtract(second); + frac = ts.getDecimalSecond().subtract(second); frac = frac.movePointRight(9); // Convert to nanos assertEquals("nanos", now.getNanos(), frac.intValue()); @@ -1396,7 +1436,6 @@ public void testLocalYearBoundaryNegative() { } @Test - @SuppressWarnings("deprecation") public void testTimestampCopyConstructor() { BigDecimal second = new BigDecimal("12.3456789"); @@ -1406,18 +1445,19 @@ public void testTimestampCopyConstructor() expectedTs.toString()); //===== Test on Timestamp(...) fractional precision copy-constructor ===== - Timestamp actualTs = Timestamp.forSecond(expectedTs.getYear(), - expectedTs.getMonth(), - expectedTs.getDay(), - expectedTs.getHour(), - expectedTs.getMinute(), - expectedTs.getDecimalSecond(), - expectedTs.getLocalOffset()); + Timestamp actualTs = new Timestamp(expectedTs.getYear(), + expectedTs.getMonth(), + expectedTs.getDay(), + expectedTs.getHour(), + expectedTs.getMinute(), + expectedTs.getSecond(), + expectedTs.getFractionalSecond(), + expectedTs.getLocalOffset()); assertEquals(expectedTs, actualTs); //===== Test on Timestamp.createFromUtcFields(...) copy-constructor ===== actualTs = Timestamp - .createFromUtcFields(expectedTs.getPrecision(), + .createFromUtcFields(Precision.FRACTION, expectedTs.getZYear(), expectedTs.getZMonth(), expectedTs.getZDay(), @@ -1448,10 +1488,10 @@ public void testNegativeEpochWithFractionalSeconds() new BigDecimal("-9223372036854.775808"); Timestamp expectedTs = - Timestamp.forMillis(negativeEpochDecimalMillis, UTC_OFFSET); + new Timestamp(negativeEpochDecimalMillis, UTC_OFFSET); long expectedMillis = expectedTs.getMillis(); Timestamp expectedTsMillis = - Timestamp.forMillis(expectedMillis, UTC_OFFSET); + new Timestamp(expectedMillis, UTC_OFFSET); Timestamp expectedTsMillisWithNanosPrecision = Timestamp.valueOf("1677-09-21T00:12:43.145000000Z"); @@ -1480,12 +1520,12 @@ public void testNegativeEpochWithFractionalSeconds() //===== Test on Timestamp(BigDecimal, Integer) constructor ===== - Timestamp actualTs = Timestamp.forMillis(negativeEpochDecimalMillis, UTC_OFFSET); + Timestamp actualTs = new Timestamp(negativeEpochDecimalMillis, UTC_OFFSET); assertEquals(expectedTs, actualTs); //===== Test on Timestamp(long, Integer) constructor ===== - actualTs = Timestamp.forMillis(expectedMillis, UTC_OFFSET); + actualTs = new Timestamp(expectedMillis, UTC_OFFSET); assertEquals(expectedTsMillis, actualTs); @@ -1736,7 +1776,6 @@ public void testAdjustYearOutsideMinRange() public void testAddMonth() { TimestampArithmeticInvoker invoker = new TimestampArithmeticInvoker() { - @Override public Timestamp invoke(Timestamp input, Number amount) { return input.addMonth(amount.intValue()); } @@ -1771,7 +1810,6 @@ public Timestamp invoke(Timestamp input, Number amount) { public void testAdjustMonth() { TimestampArithmeticInvoker invoker = new TimestampArithmeticInvoker() { - @Override public Timestamp invoke(Timestamp input, Number amount) { return input.adjustMonth(amount.intValue()); } @@ -2906,9 +2944,10 @@ public void testMillisIsIndepedentOfOffset() throws ParseException { assertEquals(t1.getDecimalMillis(), t2.getDecimalMillis()); } - @Ignore // TODO ion-java#165 + @Ignore @Test public void testInstantVsTimestampMillis() { + /* If ion-java migrates to Java 8, which added Instant, this test becomes valid. // addresses: https://github.com/amzn/ion-java/issues/165 String tsText = "0001-01-01T00:00:00.000Z"; // Instant extends the Gregorian calendar system all the way back to the beginning, whereas Timestamp uses @@ -2916,7 +2955,7 @@ public void testInstantVsTimestampMillis() { // result, the two map from milliseconds to date differently before 1582, as demonstrated below. long millisFromInstant = Instant.parse(tsText).toEpochMilli(); long millisFromTimestamp = Timestamp.valueOf(tsText).getMillis(); - assertTrue(millisFromInstant != millisFromTimestamp); + assertNotEquals(millisFromInstant, millisFromTimestamp); // However, the discrepancy can be avoided by using Timestamp.forCalendar, which always respects the given // Calendar's method for determining leap years. Timestamp ts1 = Timestamp.forCalendar(GregorianCalendar.from(Instant.parse(tsText).atZone(ZoneId.of("UTC")))); @@ -2928,5 +2967,6 @@ public void testInstantVsTimestampMillis() { // values are roundtripped unchanged. assertEquals(millisFromInstant, ts1.getMillis()); assertEquals(millisFromTimestamp, ts2.getMillis()); + */ } } diff --git a/test/software/amazon/ion/TrBwBrProcessingTest.java b/test/com/amazon/ion/TrBwBrProcessingTest.java similarity index 66% rename from test/software/amazon/ion/TrBwBrProcessingTest.java rename to test/com/amazon/ion/TrBwBrProcessingTest.java index 42d87d35ac..f49cfca0d5 100644 --- a/test/software/amazon/ion/TrBwBrProcessingTest.java +++ b/test/com/amazon/ion/TrBwBrProcessingTest.java @@ -1,22 +1,22 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; import org.junit.Ignore; import org.junit.Test; -import software.amazon.ion.IonReader; /** * Tests TextReader - BinaryWriter - BinaryReader @@ -31,7 +31,9 @@ protected void prepare(String text) myMissingSymbolTokensHaveText = false; IonReader textReader = system().newSystemReader(text); - myBytes = writeBinaryBytes(textReader); + IonBinaryWriter binaryWriter = system().newBinaryWriter(); + binaryWriter.writeValues(textReader); + myBytes = binaryWriter.getBytes(); } @Override diff --git a/test/software/amazon/ion/TrueSequenceTestCase.java b/test/com/amazon/ion/TrueSequenceTestCase.java similarity index 91% rename from test/software/amazon/ion/TrueSequenceTestCase.java rename to test/com/amazon/ion/TrueSequenceTestCase.java index b9d1e0bb63..22ef0a8708 100644 --- a/test/software/amazon/ion/TrueSequenceTestCase.java +++ b/test/com/amazon/ion/TrueSequenceTestCase.java @@ -1,32 +1,26 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; +import com.amazon.ion.impl._Private_IonValue; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import org.junit.Test; -import software.amazon.ion.ContainedValueException; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonNull; -import software.amazon.ion.IonSequence; -import software.amazon.ion.IonValue; -import software.amazon.ion.ReadOnlyValueException; -import software.amazon.ion.impl.PrivateIonValue; /** @@ -88,7 +82,7 @@ public void testNullSequenceIndexedAddAll() /** - * TODO amzn/ion-java#47 Implement indexed addAll for datagram + * TODO amzn/ion-java/issues/47 Implement indexed addAll for datagram * Hoist this up to SequencenceTestCase. */ @Test @@ -161,7 +155,7 @@ public void testIndexedAddAll(IonSequence seq) /** - * TODO amzn/ion-java#50 Implement set for datagram + * TODO amzn/ion-java/issues/50 Implement set for datagram * Hoist this up to SequencenceTestCase. */ @Test @@ -207,14 +201,14 @@ private void set(IonSequence s, int index) assertSame(origElement, removed); assertEquals(null, removed.getContainer()); - assertEquals(index, ((PrivateIonValue)newElement).getElementId()); + assertEquals(index, ((_Private_IonValue)newElement).getElementId()); assertEquals(expectedElements, s); } /** - * TODO amzn/ion-java#50 Implement set for datagram + * TODO amzn/ion-java/issues/50 Implement set for datagram * Hoist this up to SequencenceTestCase. */ @Test @@ -226,7 +220,7 @@ public void testSetNullElement() /** - * TODO amzn/ion-java#50 Implement set for datagram + * TODO amzn/ion-java/issues/50 Implement set for datagram * Hoist this up to SequencenceTestCase. */ @Test @@ -240,7 +234,7 @@ public void testSetContainedValue() /** - * TODO amzn/ion-java#50 Implement set for datagram + * TODO amzn/ion-java/issues/50 Implement set for datagram * Hoist this up to SequencenceTestCase. */ @Test @@ -253,7 +247,7 @@ public void testSetDatagram() /** - * TODO amzn/ion-java#50 Implement set for datagram + * TODO amzn/ion-java/issues/50 Implement set for datagram * Hoist this up to SequencenceTestCase. */ @Test @@ -267,7 +261,7 @@ public void testSetReadOnlyChild() /** - * TODO amzn/ion-java#50 Implement set for datagram + * TODO amzn/ion-java/issues/50 Implement set for datagram * Hoist this up to SequencenceTestCase. */ @Test @@ -281,7 +275,7 @@ public void testSetReadOnlyContainer() /** - * TODO amzn/ion-java#50 Implement set for datagram + * TODO amzn/ion-java/issues/50 Implement set for datagram * Hoist this up to SequencenceTestCase. */ @Test diff --git a/test/software/amazon/ion/ValueFactorySequenceTest.java b/test/com/amazon/ion/ValueFactorySequenceTest.java similarity index 77% rename from test/software/amazon/ion/ValueFactorySequenceTest.java rename to test/com/amazon/ion/ValueFactorySequenceTest.java index efa8bc4aba..63d227eab9 100644 --- a/test/software/amazon/ion/ValueFactorySequenceTest.java +++ b/test/com/amazon/ion/ValueFactorySequenceTest.java @@ -1,27 +1,25 @@ + /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion; +package com.amazon.ion; + +import com.amazon.ion.junit.Injected.Inject; +import java.util.Collection; import org.junit.Before; import org.junit.Test; -import software.amazon.ion.ContainedValueException; -import software.amazon.ion.IonList; -import software.amazon.ion.IonSequence; -import software.amazon.ion.IonSexp; -import software.amazon.ion.IonValue; -import software.amazon.ion.ValueFactory; -import software.amazon.ion.junit.Injected.Inject; public class ValueFactorySequenceTest extends IonTestCase @@ -29,6 +27,7 @@ public class ValueFactorySequenceTest private static abstract class SequenceMaker { ValueFactory factory; + abstract S newSeq(Collection values); abstract S newSeq(IonSequence child); abstract S newSeq(IonValue... children); } @@ -37,6 +36,13 @@ private static class ListMaker extends SequenceMaker { @Override public String toString() { return "LIST"; } + @Override + @SuppressWarnings("deprecation") + IonList newSeq(Collection values) + { + return factory.newList(values); + } + @Override IonList newSeq(IonSequence child) { @@ -54,6 +60,13 @@ private static class SexpMaker extends SequenceMaker { @Override public String toString() { return "SEXP"; } + @Override + @SuppressWarnings("deprecation") + IonSexp newSeq(Collection values) + { + return factory.newSexp(values); + } + @Override IonSexp newSeq(IonSequence child) { diff --git a/test/software/amazon/ion/facet/FacetsTest.java b/test/com/amazon/ion/facet/FacetsTest.java similarity index 87% rename from test/software/amazon/ion/facet/FacetsTest.java rename to test/com/amazon/ion/facet/FacetsTest.java index 29b9cd75ea..27d2b0e359 100644 --- a/test/software/amazon/ion/facet/FacetsTest.java +++ b/test/com/amazon/ion/facet/FacetsTest.java @@ -1,27 +1,27 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.facet; +package com.amazon.ion.facet; +import static com.amazon.ion.facet.Facets.asFacet; +import static com.amazon.ion.facet.Facets.assumeFacet; import static org.junit.Assert.assertSame; -import static software.amazon.ion.facet.Facets.asFacet; -import static software.amazon.ion.facet.Facets.assumeFacet; +import com.amazon.ion.Span; import org.junit.Test; -import software.amazon.ion.Span; -import software.amazon.ion.facet.Faceted; -import software.amazon.ion.facet.UnsupportedFacetException; + public class FacetsTest { diff --git a/test/software/amazon/ion/impl/BinaryWriterTest.java b/test/com/amazon/ion/impl/BinaryWriterTest.java similarity index 90% rename from test/software/amazon/ion/impl/BinaryWriterTest.java rename to test/com/amazon/ion/impl/BinaryWriterTest.java index b7f51f8c1b..f2d287859f 100644 --- a/test/software/amazon/ion/impl/BinaryWriterTest.java +++ b/test/com/amazon/ion/impl/BinaryWriterTest.java @@ -1,32 +1,33 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; +import com.amazon.ion.IonException; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.junit.IonAssert; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import org.junit.Assert; import org.junit.Test; -import software.amazon.ion.IonException; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.junit.IonAssert; public class BinaryWriterTest extends OutputStreamWriterTestCase @@ -44,7 +45,7 @@ public void testWriteSymbolWithLockedSymtab() throws Exception { iw = makeWriter(); - iw.writeSymbol("force a local symtab"); // TODO amzn/ion-java#8 + iw.writeSymbol("force a local symtab"); // TODO amzn/ion-java/issues/8 iw.getSymbolTable().makeReadOnly(); thrown.expect(IonException.class); iw.writeSymbol("s"); @@ -55,7 +56,7 @@ public void testAddTypeAnnotationWithLockedSymtab() throws Exception { iw = makeWriter(); - iw.writeSymbol("force a local symtab"); // TODO amzn/ion-java#8 + iw.writeSymbol("force a local symtab"); // TODO amzn/ion-java/issues/8 iw.getSymbolTable().makeReadOnly(); thrown.expect(IonException.class); iw.addTypeAnnotation("a"); @@ -66,7 +67,7 @@ public void testSetFieldNameWithLockedSymtab() throws Exception { iw = makeWriter(); - iw.writeSymbol("force a local symtab"); // TODO amzn/ion-java#8 + iw.writeSymbol("force a local symtab"); // TODO amzn/ion-java/issues/8 iw.getSymbolTable().makeReadOnly(); iw.stepIn(IonType.STRUCT); thrown.expect(IonException.class); @@ -119,7 +120,7 @@ public void testFlushingUnlockedSymtab() throws Exception { iw = makeWriter(); - iw.writeSymbol("force a local symtab"); // TODO amzn/ion-java#8 + iw.writeSymbol("force a local symtab"); // TODO amzn/ion-java/issues/8 SymbolTable symtab = iw.getSymbolTable(); symtab.intern("fred_1"); symtab.intern("fred_2"); diff --git a/test/software/amazon/ion/impl/BinaryWriterWithLocalSymtabsTest.java b/test/com/amazon/ion/impl/BinaryWriterWithLocalSymtabsTest.java similarity index 89% rename from test/software/amazon/ion/impl/BinaryWriterWithLocalSymtabsTest.java rename to test/com/amazon/ion/impl/BinaryWriterWithLocalSymtabsTest.java index 6997522fcb..debc65c925 100644 --- a/test/software/amazon/ion/impl/BinaryWriterWithLocalSymtabsTest.java +++ b/test/com/amazon/ion/impl/BinaryWriterWithLocalSymtabsTest.java @@ -1,33 +1,35 @@ /* - * Copyright 2013-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; -import static software.amazon.ion.impl.PrivateUtils.EMPTY_STRING_ARRAY; -import static software.amazon.ion.impl.Symtabs.FRED_MAX_IDS; -import static software.amazon.ion.impl.Symtabs.LOCAL_SYMBOLS_ABC; -import static software.amazon.ion.impl.Symtabs.makeLocalSymtab; +import static com.amazon.ion.impl.Symtabs.FRED_MAX_IDS; +import static com.amazon.ion.impl.Symtabs.LOCAL_SYMBOLS_ABC; +import static com.amazon.ion.impl.Symtabs.makeLocalSymtab; +import static com.amazon.ion.impl._Private_Utils.EMPTY_STRING_ARRAY; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.system.IonBinaryWriterBuilder; import java.io.ByteArrayOutputStream; import org.junit.After; import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.IonValue; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.system.IonBinaryWriterBuilder; + public class BinaryWriterWithLocalSymtabsTest extends IonTestCase diff --git a/test/com/amazon/ion/impl/ByteBufferTest.java b/test/com/amazon/ion/impl/ByteBufferTest.java new file mode 100644 index 0000000000..aa93de6494 --- /dev/null +++ b/test/com/amazon/ion/impl/ByteBufferTest.java @@ -0,0 +1,399 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import com.amazon.ion.IonException; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.impl.IonBinary.BufferManager; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Random; +import org.junit.After; +import org.junit.Test; + + +public class ByteBufferTest + extends IonTestCase +{ + @Override @After + public void tearDown() + throws Exception + { + super.tearDown(); + BlockedBuffer.resetParameters(); + } + + @Test + public void testSmallInsertion() + { + BufferManager buf = new BufferManager(); + IonBinary.Writer writer = buf.openWriter(); + + // Write enough data to overflow the first block + byte[] initialData = new byte[BlockedBuffer._defaultBlockSizeMin + 5]; + + // Now insert some stuff at the beginning + try { + writer.write(initialData, 0, initialData.length); + writer.setPosition(0); + writer.insert(10); + } + catch (IOException e) { + throw new IonException(e); + } + } + + @Test + public void testUnicodeCodepointOverflow() { + BufferManager buf = new BufferManager (); + IonBinary.Writer writer = buf.openWriter(); + + try { + writer.writeStringData(new String(new char[] { 0xd799 }, 0, 1)); + writer.writeStringData(new String(new char[] { 0xe000 }, 0, 1)); + } catch (Exception e) { + fail("Unexpected exception: " + e.getMessage()); + } + try { + writer.writeStringData(new String(new char[] { 0xd800 }, 0, 1)); + fail("Successfully parsed a partial surrogate"); + } catch (Exception e) { + } + try { + writer.writeStringData(new String(new char[] { 0xdfff}, 0, 1)); + fail("Successfully parsed a partial surrogate"); + } catch (Exception e) { + } + } + + @Test + public void testUnicodeCodepointOverflowStatic() { + OutputStream os = new ByteArrayOutputStream(); + try { + IonBinary.writeString(os, new String(new char[] { 0xd799 }, 0, 1)); + IonBinary.writeString(os, new String(new char[] { 0xe000 }, 0, 1)); + // surrogates + IonBinary.writeString(os, new String(new char[] { 0xd800, 0xdc00 }, 0, 2)); + IonBinary.writeString(os, new String(new char[] { 0xdbff, 0xdfff }, 0, 2)); + } catch (Exception e) { + throw new IonException(e); + } + try { + IonBinary.writeString(os, new String(new int[] { 0xd800 }, 0, 1)); + fail("Successfully parsed a partial surrogate"); + } catch (Exception e) { + } + try { + IonBinary.writeString(os, new String(new int[] { 0xdfff}, 0, 1)); + fail("Successfully parsed a partial surrogate"); + } catch (Exception e) { + } + } + + @Test + public void testLargeInsertion() + { + BufferManager buf = new BufferManager (); + IonBinary.Writer writer = buf.openWriter(); + + // Write enough data to overflow the first block + byte[] initialData = new byte[BlockedBuffer._defaultBlockSizeMin + 5]; + + // Now insert lots of stuff at the beginning + try { + writer.write(initialData, 0, initialData.length); + writer.setPosition(0); + writer.insert(5000); + } + catch (IOException e) { + throw new IonException(e); + } + } + + @Test + public void testRandomUpdatesSmall() throws Exception + { + testRandomUpdates(7, 19, 100); + } + + @Test + public void testRandomUpdatesMedium() throws Exception + { + testRandomUpdates(128, 4096, 1000); + } + + @Test + public void testRandomUpdatesLarge() throws Exception + { + testRandomUpdates(32*1024, 32*1024, 1000); + } + + final static boolean _debug_long_test = false; + + @Test + public void testRandomUpdatesLargeAndLong() throws Exception + { + // TODO: turn on a long test, maybe not this long, when we can know it's + // not going to be oppressive + if (!_debug_long_test) return; + testRandomUpdates(7, 19, 100000); + } + + @Test + public void testPreviousFailuresInRandomUpdates() throws Exception + { + testRandomUpdates(128, 4096, 1000, 1230074384739L); + } + + private void testRandomUpdates(int min, int max, int count) throws Exception + { + long seed = System.currentTimeMillis(); + testRandomUpdates(min, max, count, seed); + } + + private void testRandomUpdates(int min, int max, int count, long seed) throws Exception + { + BlockedBuffer.setBlockSizeParameters(min, max); // make it work hard + + testByteBuffer testBuf = new testByteBuffer(); + + BlockedBuffer blocked = new BlockedBuffer(); + BlockedBuffer.BlockedByteOutputStream blkout = new BlockedBuffer.BlockedByteOutputStream(blocked); + BlockedBuffer.BlockedByteInputStream blkin = new BlockedBuffer.BlockedByteInputStream(blocked); + + + final boolean debug_output = false; + final boolean checkedOften = false; + final boolean checkContentsEverytime = false; + + final int max_update = 1000; + final int rep_count = count; + + final int choiceCheck = 0; + final int choiceAppend = 1; + final int choiceInsert = 2; + final int choiceRemove = 3; + final int choiceWrite = 4; + + Random r = new Random(seed); + if (debug_output) System.out.println("seed: " + seed); + + byte[] data = new byte[max_update]; + int val = -1, len = -1, pos = 0; + boolean justChecked = false; + + try { + for (int ii=0; ii 0) { + testBuf.position(0); + // old: buf.positionForRead(0); + blkin.sync(); + blkin.setPosition(0); + for (int jj = 0; jj 32688 && jj < 32690) { + // old: assert (byte)(bt & 0xff) == (byte)(br & 0xff); + assert (byte)(bt & 0xff) == (byte)(bk & 0xff); + } + else { + // old: assert (byte)(bt & 0xff) == (byte)(br & 0xff); + assert (byte)(bt & 0xff) == (byte)(bk & 0xff); + } + } + // old: buf._validate(); + blkin._validate(); + } + break; + case choiceAppend: // append + if (debug_output) System.out.println("append "+len+" of "+val+" at "+pos); + testBuf.write(data, len); + // old: buf.write(data, 0, len); + blkout.write(data, 0, len); + break; + case choiceInsert: // insert + if (debug_output) System.out.println("insert "+len+" of "+val+" at "+pos); + testBuf.insert(data, len); + // old: buf.insert(len); + // old: buf.write(data, 0, len); + blkout.insert(data, 0, len); + break; + case choiceRemove: // remove + if (pos + len > testBuf.limit()) { + if (r.nextInt(100) < 90) break; + len = testBuf.limit() - pos; + } + if (debug_output) System.out.println("remove "+len+" at "+pos); + testBuf.remove(len); + // old: buf.remove(len); + blkout.remove(len); + break; + case choiceWrite: // write + if (pos + len > testBuf.limit()) { + if (r.nextInt(100) < 90) break; + len = testBuf.limit() - pos; + } + if (debug_output) System.out.println("write "+len+" of "+val+" at "+pos); + testBuf.write(data, len); + // old: buf.write(data, 0, len); + blkout.write(data, 0, len); + break; + default: + assert "" == "this is a bad case in a switch statement"; + throw new RuntimeException("switch case error!"); + } + } + } + catch (Exception e) { + throw new RuntimeException("FAILED with seed: " + seed, e); + } + catch (Error e) { + throw new RuntimeException("FAILED with seed: " + seed, e); + } + } + + static public class testByteBuffer + { + static final int startingBufferSize = 16; + int _position; + int _inUse; + byte[] _buf; + + public testByteBuffer() { + this._buf = new byte[startingBufferSize]; + } + + void expand(int newlen) { + int len = this._buf.length; + while (len < newlen) { + len *= 2; + } + if (len > this._buf.length) { + byte[] newbuf = new byte[len]; + System.arraycopy(this._buf, 0, newbuf, 0, this._inUse); + this._buf = newbuf; + } + } + public int limit() { return this._inUse; } + public int position(int pos) { + assert (pos >= 0 && pos <= _inUse); + this._position = pos; + return this._position; + } + public int insert(byte[] data, int len) { + position(this._position); + int newEnd = this._inUse + len; + expand(newEnd); + System.arraycopy(this._buf, this._position + ,this._buf, this._position + len, this._inUse - this._position); + System.arraycopy(data, 0, this._buf, this._position, len); + this._position += len; + this._inUse += len; + return len; + } + public int remove(int len) { + position(this._position); + assert (this._position + len <= this._inUse); + System.arraycopy(this._buf, this._position + len + ,this._buf, this._position, this._inUse - (this._position + len)); + this._inUse -= len; + return len; + } + public int read() { + position(this._position); + assert(this._position + 1 <= this._inUse); + int ret = this._buf[this._position]; + this._position++; + return ret; + } + public byte[] read(int len) { + position(this._position); + assert(this._position + len <= this._inUse); + byte[] ret = new byte[len]; + System.arraycopy(this._buf, this._position, ret, 0, len); + this._position += len; + return ret; + } + public int write(byte[] data, int len) { + assert(data.length >= len); + position(this._position); + int endWrite = this._position + len; + expand(endWrite); + System.arraycopy(data, 0, this._buf, this._position, len); + this._position = endWrite; + if (this._position > this._inUse) this._inUse = this._position; + return len; + } + } +} diff --git a/test/com/amazon/ion/impl/CharacterReaderTest.java b/test/com/amazon/ion/impl/CharacterReaderTest.java new file mode 100644 index 0000000000..6181a7ac47 --- /dev/null +++ b/test/com/amazon/ion/impl/CharacterReaderTest.java @@ -0,0 +1,248 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import com.amazon.ion.IonTestCase; +import java.io.IOException; +import java.io.StringReader; +import org.junit.Test; + +/** + * Test cases for {@link IonCharacterReader}. + */ +public class CharacterReaderTest extends IonTestCase +{ + public static IonCharacterReader createReader( final String data, final int size ) { + StringReader in = new StringReader( data ); + return new IonCharacterReader( in, size ); + } + + public static IonCharacterReader createReader( final StringBuilder data, final int size ) { + return createReader( data.toString(), size ); + } + + @Test + public void testEmpty() throws Exception { + IonCharacterReader in = createReader( "", 1 ); + assertTrue( in.read() == -1 ); + assertTrue( in.getConsumedAmount() == 0 ); + assertTrue( in.getColumn() == 0 ); + assertTrue( in.getLineNumber() == 1 ); + } + + @Test + public void testEmptyMultipleRead() throws Exception { + IonCharacterReader in = createReader( "", 1 ); + assertTrue( in.read() == -1 ); + assertTrue( in.read() == -1 ); + } + + @Test + public void testEmptyUnreadEOF() throws Exception { + IonCharacterReader in = createReader( "", 1 ); + assertTrue( in.read() == -1 ); + in.unread( -1 ); + assertTrue( in.read() == -1 ); + } + + @Test + public void testSingleRead() throws Exception { + IonCharacterReader in = createReader( "A", 1 ); + + int read = in.read(); + assertTrue( read == 'A' ); + assertTrue( in.getConsumedAmount() == 1 ); + assertTrue( in.getColumn() == 1 ); + assertTrue( in.getLineNumber() == 1 ); + + assertTrue( in.read() == -1 ); + assertTrue( in.getConsumedAmount() == 1 ); + assertTrue( in.getColumn() == 1 ); + assertTrue( in.getLineNumber() == 1 ); + } + + @Test + public void testSingleUnread() throws Exception { + IonCharacterReader in = createReader( "B", 1 ); + + int read = in.read(); + + in.unread( read ); + assertTrue( in.getConsumedAmount() == 0 ); + assertTrue( in.getColumn() == 0 ); + assertTrue( in.getLineNumber() == 1 ); + + assertTrue( in.read() == read ); + assertTrue( in.getConsumedAmount() == 1 ); + assertTrue( in.getColumn() == 1 ); + assertTrue( in.getLineNumber() == 1 ); + } + + @Test + public void testSingleUnreadCarriageReturn() throws Exception { + IonCharacterReader in = createReader( "B", 1 ); + + try { + in.unread( '\r' ); + fail(); + } catch ( final IOException e ) {} + } + + @Test + public void testEmptyUnread() throws Exception { + IonCharacterReader in = createReader( "", 1 ); + + assertTrue( in.read() == -1 ); + in.unread( 'A' ); + assertTrue( in.read() == 'A' ); + assertTrue( in.read() == -1 ); + } + + @Test + public void testReadBatchMultiline() throws Exception { + IonCharacterReader in = createReader( "A\r\nB", 3 ); + + char[] buf = new char[ 3 ]; + int num = in.read( buf ); + + assertTrue( num == 3 ); + assertTrue( buf[ 0 ] == 'A' ); + assertTrue( buf[ 1 ] == '\n' ); + assertTrue( buf[ 2 ] == 'B' ); + assertTrue( in.read() == -1 ); + assertTrue( in.getConsumedAmount() == 3 ); + assertTrue( in.getColumn() == 1 ); + assertTrue( in.getLineNumber() == 2 ); + } + + @Test + public void testUnreadBatch() throws Exception { + IonCharacterReader in = createReader( "ABCD", 3 ); + + char[] buf = new char[ 3 ]; + int num = in.read( buf ); + + assertTrue( num == 3 ); + assertTrue( in.getConsumedAmount() == 3 ); + assertTrue( in.getColumn() == 3 ); + assertTrue( in.getLineNumber() == 1 ); + assertEquals( new String( buf ), "ABC" ); + + in.unread( buf ); + assertTrue( in.getConsumedAmount() == 0 ); + assertTrue( in.getColumn() == 0 ); + assertTrue( in.getLineNumber() == 1 ); + + buf = new char[ 3 ]; + in.read( buf ); + assertTrue( num == 3 ); + assertTrue( in.getConsumedAmount() == 3 ); + assertTrue( in.getColumn() == 3 ); + assertTrue( in.getLineNumber() == 1 ); + assertEquals( new String( buf ), "ABC" ); + } + + @Test + public void testUnreadBatchMultiline() throws Exception { + IonCharacterReader in = createReader( "A\r\n\n\nB\n\nC\n\nD", 3 ); + + in.read(); // Skip the initial 'A' + + char[] buf = new char[ 3 ]; + int num = in.read( buf ); // Consume \r\n\n\n + + assertTrue( num == 3 ); + assertTrue( in.getConsumedAmount() == 4 ); + assertTrue( in.getColumn() == 0 ); + assertTrue( in.getLineNumber() == 4 ); + assertEquals( new String( buf ), "\n\n\n" ); + + in.unread( buf ); + assertTrue( in.getConsumedAmount() == 1 ); + assertTrue( in.getColumn() == 1 ); + assertTrue( in.getLineNumber() == 1 ); + + buf = new char[ 3 ]; + in.read( buf ); + assertTrue( num == 3 ); + assertTrue( in.getConsumedAmount() == 4 ); + assertTrue( in.getColumn() == 0 ); + assertTrue( in.getLineNumber() == 4 ); + assertEquals( new String( buf ), "\n\n\n" ); + } + + @Test + public void testUnreadBatchMultilineAllNewlines1() throws Exception { + implCRUnreadBatchMultilineAllNewlines( "ABC\r\n\r\n\r\n\r\n" ); + } + + @Test + public void testUnreadBatchMultilineAllNewlines2() throws Exception { + implCRUnreadBatchMultilineAllNewlines( "ABC\n\r\n\r\n\r\n\r" ); + } + + @Test + public void testUnreadBatchMultilineAllNewlines3() throws Exception { + implCRUnreadBatchMultilineAllNewlines( "ABC\n\n\n\n\n" ); + } + + @Test + public void testUnreadBatchMultilineAllNewlines4() throws Exception { + implCRUnreadBatchMultilineAllNewlines( "ABC\r\r\r\r\r" ); + } + + protected void implCRUnreadBatchMultilineAllNewlines( final String data ) throws Exception { + IonCharacterReader in = createReader( data, 3 ); + char[] buf = new char[ 3 ]; + in.skip( 3L ); + int num = in.read( buf ); + + assertTrue( num == 3 ); + assertTrue( in.getConsumedAmount() == 6 ); + assertTrue( in.getColumn() == 0 ); + assertTrue( in.getLineNumber() == 4 ); + assertEquals( new String( buf ), "\n\n\n" ); + + in.unread( buf ); + assertTrue( in.getConsumedAmount() == 3 ); + assertTrue( in.getColumn() == 3 ); + assertTrue( in.getLineNumber() == 1 ); + } + + @Test + public void testUnreadUnderflow() throws Exception { + IonCharacterReader in = createReader( "ABCDE", 1 ); + + char[] buf = new char[ 4 ]; + in.read( buf ); + assertEquals( new String( buf ), "ABCD" ); + + try { + in.unread( buf ); + fail(); + } catch ( IOException e ) {} + } + + @Test + public void testSkip() throws Exception { + IonCharacterReader in = createReader( "\r\n\r\nABCDEFG", 4 ); + + in.skip( 4 ); + assertTrue( in.getConsumedAmount() == 4 ); + assertTrue( in.getColumn() == 2 ); + assertTrue( in.getLineNumber() == 3 ); + } +} diff --git a/test/com/amazon/ion/impl/EscapingCallback.java b/test/com/amazon/ion/impl/EscapingCallback.java new file mode 100644 index 0000000000..8f49d57e4f --- /dev/null +++ b/test/com/amazon/ion/impl/EscapingCallback.java @@ -0,0 +1,128 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import com.amazon.ion.IonType; +import com.amazon.ion.util._Private_FastAppendable; +import java.io.IOException; + +class EscapingCallback + extends _Private_MarkupCallback +{ + private _Private_FastAppendable myAppendable; + + public EscapingCallback(_Private_FastAppendable unescaped) + { + super(new EscapingIonAppendable(unescaped)); + myAppendable = unescaped; + } + + @Override + public void beforeValue(IonType iType) + throws IOException + { + myAppendable.append("<><><>&&&<><><>"); + } + + public static class Builder implements _Private_CallbackBuilder { + public _Private_MarkupCallback build(_Private_FastAppendable rawOutput) + { + return new EscapingCallback(rawOutput); + } + } + + static class EscapingIonAppendable + extends _Private_FastAppendableDecorator + { + public EscapingIonAppendable(_Private_FastAppendable output) { + super(output); + } + + @Override + public Appendable append(char c) + throws IOException + { + switch (c) { + case '&': super.appendAscii("&"); break; + case '<': super.appendAscii("<"); break; + case '>': super.appendAscii(">"); break; + // could be non-ASCII + default : super.append(c); break; + } + return this; + } + + @Override + public Appendable append(CharSequence csq) + throws IOException + { + append(csq, 0, csq.length()); + return this; + } + + @Override + public Appendable append(CharSequence csq, int start, int end) + throws IOException + { + int curr_start = start; + for (int i = start; i < end; ++i) { + char c = csq.charAt(i); + if(c == '&' || c == '<' || c == '>') { + super.append(csq, curr_start, i); + append(c); + curr_start = i + 1; + } + } + super.append(csq, curr_start, end); + return this; + } + + @Override + public void appendAscii(char c) + throws IOException + { + switch (c) { + case '&': super.appendAscii("&"); break; + case '<': super.appendAscii("<"); break; + case '>': super.appendAscii(">"); break; + default : super.appendAscii(c); break; + } + } + + @Override + public void appendAscii(CharSequence csq) + throws IOException + { + appendAscii(csq, 0, csq.length()); + } + + @Override + public void appendAscii(CharSequence csq, int start, int end) + throws IOException + { + int curr_start = start; + for (int i = start; i < end; ++i) { + char c = csq.charAt(i); + if(c == '&' || c == '<' || c == '>') { + super.appendAscii(csq, curr_start, i); + appendAscii(c); + curr_start = i + 1; + } + } + super.appendAscii(csq, curr_start, end); + } + } +} diff --git a/test/com/amazon/ion/impl/IonImplUtilsTest.java b/test/com/amazon/ion/impl/IonImplUtilsTest.java new file mode 100644 index 0000000000..3ff9ac9afd --- /dev/null +++ b/test/com/amazon/ion/impl/IonImplUtilsTest.java @@ -0,0 +1,40 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import org.junit.Assert; +import org.junit.Test; + + +public class IonImplUtilsTest +{ + @Test + public void testEmptyUtf8() + { + byte[] bytes = _Private_Utils.utf8(""); + Assert.assertArrayEquals(_Private_Utils.EMPTY_BYTE_ARRAY, bytes); + } + + @Test + public void testEasyUtf8() + throws Exception + { + String input = "abcdefghijklm"; + byte[] bytes = _Private_Utils.utf8(input); + byte[] direct = input.getBytes("UTF-8"); + Assert.assertArrayEquals(direct, bytes); + } +} diff --git a/test/com/amazon/ion/impl/IonMarkupWriterFilesTest.java b/test/com/amazon/ion/impl/IonMarkupWriterFilesTest.java new file mode 100644 index 0000000000..8c86e1e1d7 --- /dev/null +++ b/test/com/amazon/ion/impl/IonMarkupWriterFilesTest.java @@ -0,0 +1,81 @@ + +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import static com.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; +import static com.amazon.ion.TestUtils.GOOD_IONTESTS_FILES; +import static com.amazon.ion.TestUtils.testdataFiles; + +import com.amazon.ion.IonReader; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.IonWriter; +import com.amazon.ion.junit.Injected.Inject; +import com.amazon.ion.system.IonTextWriterBuilder; +import com.amazon.ion.util.NullOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.io.OutputStream; +import org.junit.Test; + +public class IonMarkupWriterFilesTest + extends IonTestCase +{ + @Inject("testFile") + public static final File[] FILES = testdataFiles(GLOBAL_SKIP_LIST, + GOOD_IONTESTS_FILES); + private File myTestFile; + + public void setTestFile(File file) { + myTestFile = file; + } + + /** + * Tests to make sure that the type, the output stream, and the + * annotation/field name aren't null for all the files. This will fail if + * the callback type isn't being set. + * {@link IonMarkupWriterTest#testStandardCallback()} should verify they're + * set correctly. + */ + @Test + public void testPrintingStandard() + throws Exception + { + // Just ignore the output, since we're not checking it + OutputStream out = new NullOutputStream(); + InputStream in = new FileInputStream(myTestFile); + + // Get a Test markup writer + _Private_IonTextWriterBuilder builder = (_Private_IonTextWriterBuilder) + IonTextWriterBuilder.standard(); + IonWriter ionWriter = builder + .withCallbackBuilder(new TestMarkupCallback.Builder()) + .build(out); + + // Load file into a reader + IonReader ionReader = system().newReader(in); + + assertNotNull("ionReader is null", ionReader); + assertNotNull("ionWriter is null", ionWriter); + + // Write data to the writer + ionWriter.writeValues(ionReader); + // Close the writer and reader + ionWriter.close(); + ionReader.close(); + } +} diff --git a/test/com/amazon/ion/impl/IonMarkupWriterTest.java b/test/com/amazon/ion/impl/IonMarkupWriterTest.java new file mode 100644 index 0000000000..b84d7a7da0 --- /dev/null +++ b/test/com/amazon/ion/impl/IonMarkupWriterTest.java @@ -0,0 +1,143 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import com.amazon.ion.IonReader; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.IonWriter; +import com.amazon.ion.system.IonTextWriterBuilder; +import java.io.IOException; +import java.io.StringWriter; +import org.junit.Test; + +public class IonMarkupWriterTest extends IonTestCase { + private String input = + "abcd::{ hello:( sexp 1 2 \"str\" ), 'list':[ 3.2, 32e-1], " + + "blob:annot::" + + "{{ T25lIEJpZyBGYXQgVGVzdCBCbG9iIEZvciBZb3VyIFBsZWFzdXJl }} }"; + private String sExpected = + "abcd" + + "::" + + "{" + + "hello:(" + + "sexp " + + "1" + + " " + + "2 " + + "\"str\"" + + ")," + + "list:" + + "[3.2" + + "," + + "3.2e0]" + + "," + + "blob:annot::" + + "" + + "{{T25lIEJpZyBGYXQgVGVzdCBCbG9iIEZvciBZb3VyIFBsZWFzdXJl}}}"; + // TODO amzn/ion-java/issues/57 determine if platform specific newlines are appropriate for pretty printing + private String pExpected = String.format( + "%nabcd" + + "::{%n hello" + + ":(%n " + + "sexp" + + "%n 1" + + "%n 2" + + "%n " + + "\"str\"%n " + + "),%n " + + "list:[%n 3.2,%n " + + "3.2e0" + + "%n ]" + + ",%n blob" + + ":" + + "annot::" + + "" + + "{{ T25lIEJpZyBGYXQgVGVzdCBCbG9iIEZvciBZb3VyIFBsZWFzdXJl }}" + + "%n}" + ); + + @Test + public void testStandardCallback() + throws IOException + { + // Write to a StringWriter for testing + StringWriter out = new StringWriter(); + IonReader ionReader = system().newReader(input); + + _Private_IonTextWriterBuilder builder = (_Private_IonTextWriterBuilder) + IonTextWriterBuilder.standard(); + IonWriter ionWriter = builder + .withCallbackBuilder(new TestMarkupCallback.Builder()) + .build(out); + + write(ionReader, ionWriter); + assertEquals("Markup callback with standard Ion Writer, " + + "error with data:\n" + input + "\n", + sExpected, out.toString()); + } + + @Test + public void testPrettyCallback() + throws IOException + { + // Write to a StringWriter for testing + StringWriter out = new StringWriter(); + IonReader ionReader = system().newReader(input); + _Private_IonTextWriterBuilder builder = (_Private_IonTextWriterBuilder) + IonTextWriterBuilder.pretty(); + IonWriter ionWriter = builder + .withCallbackBuilder(new TestMarkupCallback.Builder()) + .build(out); + write(ionReader, ionWriter); + assertEquals("Markup callback with pretty printing Ion Writer," + + " error with data:\n" + input + "\n", + pExpected, out.toString()); + } + + @Test + public void testEscaping() throws IOException + { + String input = "These should be escaped <>&"; + String expected = "<><><>&&&<><><>\"These should be escaped <>&\""; + StringWriter out = new StringWriter(); + _Private_IonTextWriterBuilder builder = (_Private_IonTextWriterBuilder) + IonTextWriterBuilder.standard(); + IonWriter ionWriter = builder + .withCallbackBuilder(new EscapingCallback.Builder()) + .build(out); + ionWriter.writeString(input); + ionWriter.finish(); + assertEquals("Escaping failed, with data:\n" + input + "\n", + expected, out.toString()); + } + + public void write(IonReader ionReader, IonWriter ionWriter) + throws IOException + { + assertNotNull("ionReader is null", ionReader); + assertNotNull("ionWriter is null", ionWriter); + + ionWriter.writeValues(ionReader); + + ionWriter.close(); + ionReader.close(); + } +} diff --git a/test/software/amazon/ion/impl/IonUTF8Test.java b/test/com/amazon/ion/impl/IonUTF8Test.java similarity index 92% rename from test/software/amazon/ion/impl/IonUTF8Test.java rename to test/com/amazon/ion/impl/IonUTF8Test.java index 15b09cd9e8..0457d423be 100644 --- a/test/software/amazon/ion/impl/IonUTF8Test.java +++ b/test/com/amazon/ion/impl/IonUTF8Test.java @@ -1,4 +1,19 @@ -package software.amazon.ion.impl; +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; import org.junit.Assert; import org.junit.Test; diff --git a/test/software/amazon/ion/impl/IonWriterTestCase.java b/test/com/amazon/ion/impl/IonWriterTestCase.java similarity index 92% rename from test/software/amazon/ion/impl/IonWriterTestCase.java rename to test/com/amazon/ion/impl/IonWriterTestCase.java index 79690fbfb4..e13ff69dcf 100644 --- a/test/software/amazon/ion/impl/IonWriterTestCase.java +++ b/test/com/amazon/ion/impl/IonWriterTestCase.java @@ -1,29 +1,55 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; - -import static software.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; -import static software.amazon.ion.SystemSymbols.NAME_SID; -import static software.amazon.ion.TestUtils.FERMATA; -import static software.amazon.ion.impl.PrivateIonWriterBase.ERROR_MISSING_FIELD_NAME; -import static software.amazon.ion.impl.PrivateUtils.newSymbolToken; -import static software.amazon.ion.impl.Symtabs.FRED_MAX_IDS; -import static software.amazon.ion.junit.IonAssert.assertIonEquals; -import static software.amazon.ion.junit.IonAssert.expectNextField; - +package com.amazon.ion.impl; + +import static com.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; +import static com.amazon.ion.SystemSymbols.NAME_SID; +import static com.amazon.ion.TestUtils.FERMATA; +import static com.amazon.ion.impl.Symtabs.FRED_MAX_IDS; +import static com.amazon.ion.impl._Private_IonWriterBase.ERROR_MISSING_FIELD_NAME; +import static com.amazon.ion.impl._Private_Utils.newSymbolToken; +import static com.amazon.ion.junit.IonAssert.assertIonEquals; +import static com.amazon.ion.junit.IonAssert.expectNextField; + +import com.amazon.ion.FakeSymbolToken; +import com.amazon.ion.IonBlob; +import com.amazon.ion.IonClob; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonException; +import com.amazon.ion.IonList; +import com.amazon.ion.IonLob; +import com.amazon.ion.IonNull; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonString; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSymbol; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.IonText; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.SystemSymbols; +import com.amazon.ion.TestUtils; +import com.amazon.ion.UnknownSymbolException; +import com.amazon.ion.junit.IonAssert; +import com.amazon.ion.system.IonSystemBuilder; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; @@ -35,30 +61,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import software.amazon.ion.FakeSymbolToken; -import software.amazon.ion.IonBlob; -import software.amazon.ion.IonClob; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonException; -import software.amazon.ion.IonList; -import software.amazon.ion.IonLob; -import software.amazon.ion.IonNull; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonString; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.IonText; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.SystemSymbols; -import software.amazon.ion.TestUtils; -import software.amazon.ion.UnknownSymbolException; -import software.amazon.ion.impl.PrivateIonWriter; -import software.amazon.ion.junit.IonAssert; + @SuppressWarnings("deprecation") public abstract class IonWriterTestCase @@ -448,6 +451,7 @@ public void testWritingDeepNestedList() throws Exception { IonDatagram dg = loader().load("[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]"); iw = makeWriter(); dg.writeTo(iw); + iw.writeValue(dg); } // TODO test failure of getBytes before stepping all the way out @@ -829,7 +833,7 @@ public void testFinishDoesReset() Iterator it = systemIterateOutput(); - if (myOutputForm != OutputForm.TEXT) { // TODO amzn/ion-java#8 + if (myOutputForm != OutputForm.TEXT) { // TODO amzn/ion-java/issues/8 checkSymbol(SystemSymbols.ION_1_0, it.next()); } checkAnnotation(SystemSymbols.ION_SYMBOL_TABLE, it.next()); @@ -916,7 +920,7 @@ public void testWritingEmptySymtab() } - @Test @Ignore // TODO amzn/ion-java#15 + @Test @Ignore // TODO amzn/ion-java/issues/15 public void testWritingSymtabWithExtraAnnotations() throws Exception { @@ -934,6 +938,21 @@ public void testWritingSymtabWithExtraAnnotations() Assert.assertArrayEquals(annotations, v.getTypeAnnotations()); } + /** + * Discovered this old behavior during test builds, some user code relies + * on it. + */ + @Test + public void testWriteValueNull() + throws Exception + { + iw = makeWriter(); + iw.writeValue((IonValue)null); + + IonDatagram dg = reload(); + assertEquals(0, dg.size()); + } + @Test public void testWriteSymbolTokenWithGoodSid() throws Exception @@ -990,7 +1009,7 @@ public void testWriteValuesWithSymtab() } /** - * TODO amzn/ion-java#8 datagram is lazy creating local symtabs. + * TODO amzn/ion-java/issues/8 datagram is lazy creating local symtabs. * Should use a reader to check the results. */ @Test @@ -1032,29 +1051,29 @@ public void testWriteIVMExplicitly() { iw = makeWriter(); iw.writeSymbol("foo"); - ((PrivateIonWriter)iw).writeIonVersionMarker(); + ((_Private_IonWriter)iw).writeIonVersionMarker(); iw.writeInt(1); IonDatagram dg = reload(); assertEquals(2, dg.size()); } - @Test // TODO amzn/ion-java#8 Inconsistencies between writers + @Test // TODO amzn/ion-java/issues/8 Inconsistencies between writers public void testWritingDatagram() throws Exception { IonDatagram dg = loader().load("foo"); iw = makeWriter(); dg.writeTo(iw); + iw.writeValue(dg); Iterator it = systemIterateOutput(); - //if (myOutputIsBinary) - { - checkSymbol(SystemSymbols.ION_1_0, it.next()); - } + checkSymbol(SystemSymbols.ION_1_0, it.next()); + if (myOutputForm != OutputForm.TEXT) { checkAnnotation(SystemSymbols.ION_SYMBOL_TABLE, it.next()); } + // TODO amzn/ion-java#14 if (myOutputForm != OutputForm.TEXT) { @@ -1089,13 +1108,13 @@ public void testWritingNestedSymtab() iw.addTypeAnnotation(SystemSymbols.ION_SYMBOL_TABLE); iw.stepIn(IonType.STRUCT); { - assertEquals(1, ((PrivateIonWriter)iw).getDepth()); + assertEquals(1, ((_Private_IonWriter)iw).getDepth()); iw.setFieldName("open"); iw.addTypeAnnotation(SystemSymbols.ION_SYMBOL_TABLE); iw.stepIn(IonType.STRUCT); { - assertEquals(2, ((PrivateIonWriter)iw).getDepth()); + assertEquals(2, ((_Private_IonWriter)iw).getDepth()); } iw.stepOut(); } diff --git a/test/software/amazon/ion/impl/IonWriterTests.java b/test/com/amazon/ion/impl/IonWriterTests.java similarity index 58% rename from test/software/amazon/ion/impl/IonWriterTests.java rename to test/com/amazon/ion/impl/IonWriterTests.java index 1b51f13f0d..a3a3e6b806 100644 --- a/test/software/amazon/ion/impl/IonWriterTests.java +++ b/test/com/amazon/ion/impl/IonWriterTests.java @@ -1,22 +1,24 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; import org.junit.runner.RunWith; import org.junit.runners.Suite; + @RunWith(Suite.class) @Suite.SuiteClasses({ TextWriterTest.class, @@ -24,6 +26,7 @@ OptimizedBinaryWriterSymbolTableTest.class, OptimizedBinaryWriterLengthPatchingTest.class, BinaryWriterWithLocalSymtabsTest.class, + OldBinaryWriterTest.class, ValueWriterTest.class }) public class IonWriterTests diff --git a/test/software/amazon/ion/impl/IterationTest.java b/test/com/amazon/ion/impl/IterationTest.java similarity index 77% rename from test/software/amazon/ion/impl/IterationTest.java rename to test/com/amazon/ion/impl/IterationTest.java index 346075e541..483b4557ef 100644 --- a/test/software/amazon/ion/impl/IterationTest.java +++ b/test/com/amazon/ion/impl/IterationTest.java @@ -1,24 +1,27 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; + +import com.amazon.ion.IonSymbol; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.IonValue; import java.util.Iterator; import java.util.NoSuchElementException; import org.junit.Test; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.IonValue; + public class IterationTest extends IonTestCase diff --git a/test/software/amazon/ion/impl/LocalSymbolTableTest.java b/test/com/amazon/ion/impl/LocalSymbolTableTest.java similarity index 89% rename from test/software/amazon/ion/impl/LocalSymbolTableTest.java rename to test/com/amazon/ion/impl/LocalSymbolTableTest.java index a680f9055c..d7cddcd594 100644 --- a/test/software/amazon/ion/impl/LocalSymbolTableTest.java +++ b/test/com/amazon/ion/impl/LocalSymbolTableTest.java @@ -1,33 +1,33 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; -import static software.amazon.ion.impl.PrivateUtils.EMPTY_STRING_ARRAY; -import static software.amazon.ion.impl.PrivateUtils.copyLocalSymbolTable; -import static software.amazon.ion.impl.Symtabs.FRED_MAX_IDS; -import static software.amazon.ion.impl.Symtabs.LOCAL_SYMBOLS_ABC; -import static software.amazon.ion.impl.Symtabs.makeLocalSymtab; +import static com.amazon.ion.impl.Symtabs.FRED_MAX_IDS; +import static com.amazon.ion.impl.Symtabs.LOCAL_SYMBOLS_ABC; +import static com.amazon.ion.impl.Symtabs.makeLocalSymtab; +import static com.amazon.ion.impl._Private_Utils.EMPTY_STRING_ARRAY; +import static com.amazon.ion.impl._Private_Utils.copyLocalSymbolTable; +import com.amazon.ion.IonException; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.SubstituteSymbolTableException; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; import org.junit.Test; -import software.amazon.ion.IonException; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.SubstituteSymbolTableException; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.impl.LocalSymbolTable; -import software.amazon.ion.impl.SubstituteSymbolTable; + public class LocalSymbolTableTest extends IonTestCase diff --git a/test/com/amazon/ion/impl/OldBinaryWriterTest.java b/test/com/amazon/ion/impl/OldBinaryWriterTest.java new file mode 100644 index 0000000000..1c7aee2d38 --- /dev/null +++ b/test/com/amazon/ion/impl/OldBinaryWriterTest.java @@ -0,0 +1,57 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import com.amazon.ion.IonBinaryWriter; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; + + +@SuppressWarnings("deprecation") +public class OldBinaryWriterTest + extends IonWriterTestCase +{ + private IonBinaryWriter myWriter; + + + @Override + protected IonWriter makeWriter(SymbolTable... imports) + throws Exception + { + myOutputForm = OutputForm.BINARY; + myWriter = system().newBinaryWriter(imports); + return myWriter; + } + + @Override + protected byte[] outputByteArray() + throws Exception + { + return myWriter.getBytes(); + } + + @Override + protected void checkClosed() + { + // No-op. + } + + @Override + protected void checkFlushed(boolean expectFlushed) + { + // No-op. + } +} diff --git a/test/software/amazon/ion/impl/OptimizedBinaryWriterLengthPatchingTest.java b/test/com/amazon/ion/impl/OptimizedBinaryWriterLengthPatchingTest.java similarity index 90% rename from test/software/amazon/ion/impl/OptimizedBinaryWriterLengthPatchingTest.java rename to test/com/amazon/ion/impl/OptimizedBinaryWriterLengthPatchingTest.java index 905154fe0c..002ac1aaa4 100644 --- a/test/software/amazon/ion/impl/OptimizedBinaryWriterLengthPatchingTest.java +++ b/test/com/amazon/ion/impl/OptimizedBinaryWriterLengthPatchingTest.java @@ -1,31 +1,32 @@ /* - * Copyright 2013-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; -import static software.amazon.ion.junit.IonAssert.assertIonEquals; +import static com.amazon.ion.junit.IonAssert.assertIonEquals; +import com.amazon.ion.IonContainer; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonList; +import com.amazon.ion.IonSequence; +import com.amazon.ion.IonSexp; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SystemSymbols; import org.junit.Test; -import software.amazon.ion.IonContainer; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonList; -import software.amazon.ion.IonSequence; -import software.amazon.ion.IonSexp; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonType; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SystemSymbols; /** * {@link OptimizedBinaryWriterTestCase} tests related to @@ -37,7 +38,7 @@ public class OptimizedBinaryWriterLengthPatchingTest /** * Variants of different container values as Ion text of different type * descriptor lengths (i.e., L in the type desc octet, not VarUInt Length). - * Refer to Ion's binary format spec for more info. + * Refer to Ion's binary format wiki for more info. */ protected enum ContainerTypeDescLengthVariant { diff --git a/test/software/amazon/ion/impl/OptimizedBinaryWriterSymbolTableTest.java b/test/com/amazon/ion/impl/OptimizedBinaryWriterSymbolTableTest.java similarity index 89% rename from test/software/amazon/ion/impl/OptimizedBinaryWriterSymbolTableTest.java rename to test/com/amazon/ion/impl/OptimizedBinaryWriterSymbolTableTest.java index cbd0532431..850fa54ac4 100644 --- a/test/software/amazon/ion/impl/OptimizedBinaryWriterSymbolTableTest.java +++ b/test/com/amazon/ion/impl/OptimizedBinaryWriterSymbolTableTest.java @@ -1,34 +1,34 @@ /* - * Copyright 2013-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; -import static software.amazon.ion.impl.PrivateUtils.isNonSymbolScalar; -import static software.amazon.ion.impl.PrivateUtils.symtabExtends; -import static software.amazon.ion.impl.Symtabs.printLocalSymtab; -import static software.amazon.ion.junit.IonAssert.assertIonEquals; -import static software.amazon.ion.junit.IonAssert.assertIonIteratorEquals; +import static com.amazon.ion.impl.Symtabs.printLocalSymtab; +import static com.amazon.ion.impl._Private_Utils.isNonSymbolScalar; +import static com.amazon.ion.impl._Private_Utils.symtabExtends; +import static com.amazon.ion.junit.IonAssert.assertIonEquals; +import static com.amazon.ion.junit.IonAssert.assertIonIteratorEquals; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonType; +import com.amazon.ion.SymbolTable; import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonType; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.impl.PrivateUtils; /** * {@link OptimizedBinaryWriterTestCase} tests related to - * {@link PrivateUtils#symtabExtends(SymbolTable, SymbolTable)} checks. + * {@link _Private_Utils#symtabExtends(SymbolTable, SymbolTable)} checks. */ public class OptimizedBinaryWriterSymbolTableTest extends OptimizedBinaryWriterTestCase @@ -243,7 +243,7 @@ public void testOptimizedWriteValueDiffImports() /** * Writer's imports superset of Reader's: (could be) fully optimized. - * TODO amzn/ion-java#18 at the moment the compatability code requires + * TODO amzn/ion-java/issues/18 at the moment the compatability code requires * exact-match on imports. */ @Test @@ -289,7 +289,7 @@ public void testOptimizedWriteValueSubsetWriterImport() /** * Reader's source contains interspersed LSTs. - * TODO amzn/ion-java#39 Investigate allowing a config. option to copy reader's LST + * TODO amzn/ion-java/issues/39 Investigate allowing a config. option to copy reader's LST * over to the writer. */ @Test diff --git a/test/software/amazon/ion/impl/OptimizedBinaryWriterTestCase.java b/test/com/amazon/ion/impl/OptimizedBinaryWriterTestCase.java similarity index 72% rename from test/software/amazon/ion/impl/OptimizedBinaryWriterTestCase.java rename to test/com/amazon/ion/impl/OptimizedBinaryWriterTestCase.java index 2941a201c2..684c8a3eaf 100644 --- a/test/software/amazon/ion/impl/OptimizedBinaryWriterTestCase.java +++ b/test/com/amazon/ion/impl/OptimizedBinaryWriterTestCase.java @@ -1,37 +1,35 @@ /* - * Copyright 2013-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; +import static com.amazon.ion.impl.Symtabs.makeLocalSymtab; import static java.lang.reflect.Proxy.newProxyInstance; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; -import static software.amazon.ion.impl.Symtabs.makeLocalSymtab; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.junit.Injected.Inject; +import com.amazon.ion.system.IonBinaryWriterBuilder; +import com.amazon.ion.system.IonSystemBuilder; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.impl.PrivateByteTransferReader; -import software.amazon.ion.impl.PrivateByteTransferSink; -import software.amazon.ion.impl.PrivateIonWriter; -import software.amazon.ion.junit.Injected.Inject; -import software.amazon.ion.system.IonBinaryWriterBuilder; -import software.amazon.ion.system.IonSystemBuilder; /** * Base test case for {@link IonWriter#writeValue(IonReader)}, with and without @@ -48,7 +46,7 @@ public class OptimizedBinaryWriterTestCase /** * Denotes whether the - * {@link PrivateByteTransferReader#transferCurrentValue(IonWriterSystemBinary)} + * {@link _Private_ByteTransferReader#transferCurrentValue(IonWriterSystemBinary)} * has been called after an {@link IonWriter#writeValue(IonReader)}. */ private boolean isTransferCurrentValueInvoked = false; @@ -68,8 +66,8 @@ protected void checkWriterStreamCopyOptimized(IonWriter writer) if (isStreamCopyOptimized()) { assertTrue("IonWriter should be instance of _Private_IonWriter", - writer instanceof PrivateIonWriter); - PrivateIonWriter privateWriter = (PrivateIonWriter) writer; + writer instanceof _Private_IonWriter); + _Private_IonWriter privateWriter = (_Private_IonWriter) writer; assertTrue("IonWriter should be stream copy optimized", privateWriter.isStreamCopyOptimized()); } @@ -110,16 +108,16 @@ protected byte[] outputByteArray() } private class TransferCurrentValueWatchingReader - implements PrivateByteTransferReader + implements _Private_ByteTransferReader { - private final PrivateByteTransferReader myDelegate; + private final _Private_ByteTransferReader myDelegate; - TransferCurrentValueWatchingReader(PrivateByteTransferReader byteTransferReader) + TransferCurrentValueWatchingReader(_Private_ByteTransferReader byteTransferReader) { myDelegate = byteTransferReader; } - public void transferCurrentValue(PrivateByteTransferSink sink) + public void transferCurrentValue(_Private_ByteTransferSink sink) throws IOException { OptimizedBinaryWriterTestCase.this.isTransferCurrentValueInvoked = true; @@ -129,7 +127,7 @@ public void transferCurrentValue(PrivateByteTransferSink sink) /** * Obtains a dynamic proxy of {@link IonReader} over the passed in byte[], - * with an invocation handler hook over {@link PrivateByteTransferReader} facet, + * with an invocation handler hook over {@link _Private_ByteTransferReader} facet, * so as to verify whether the transferCurrentValue() method is actually * being called. * @@ -146,10 +144,10 @@ public Object invoke(Object proxy, Method method, Object[] args) { if (method.getName().equals("asFacet") && args.length == 1 && - args[0] == PrivateByteTransferReader.class) + args[0] == _Private_ByteTransferReader.class) { - PrivateByteTransferReader transferReader = - (PrivateByteTransferReader) method.invoke(reader, args); + _Private_ByteTransferReader transferReader = + (_Private_ByteTransferReader) method.invoke(reader, args); if (transferReader == null) return null; @@ -180,7 +178,7 @@ protected void checkWriteValue(boolean expectedTransferInvoked) // Reset flag before calling IonWriter.writeValue isTransferCurrentValueInvoked = false; - // TODO amzn/ion-java#16 - Currently, doesn't copy annotations or field names, + // TODO amzn/ion-java/issues/16 - Currently, doesn't copy annotations or field names, // so we always expect no transfer of raw bytes if (ir.isInStruct() || ir.getTypeAnnotationSymbols().length > 0) { diff --git a/test/software/amazon/ion/impl/OutputStreamWriterTestCase.java b/test/com/amazon/ion/impl/OutputStreamWriterTestCase.java similarity index 90% rename from test/software/amazon/ion/impl/OutputStreamWriterTestCase.java rename to test/com/amazon/ion/impl/OutputStreamWriterTestCase.java index 7749a64518..acc7c010a8 100644 --- a/test/software/amazon/ion/impl/OutputStreamWriterTestCase.java +++ b/test/com/amazon/ion/impl/OutputStreamWriterTestCase.java @@ -1,33 +1,35 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SystemSymbols; +import com.amazon.ion.Timestamp; import java.io.ByteArrayOutputStream; import java.io.FilterOutputStream; import java.io.IOException; import java.io.OutputStream; import java.math.BigDecimal; import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SystemSymbols; -import software.amazon.ion.Timestamp; + public abstract class OutputStreamWriterTestCase extends IonWriterTestCase @@ -137,12 +139,12 @@ public void testFlushingLockedSymtab() { iw = makeWriter(); - // Force a local symtab. TODO amzn/ion-java#8 Should have an API for this + // Force a local symtab. TODO amzn/ion-java/issues/8 Should have an API for this iw.addTypeAnnotation(SystemSymbols.ION_SYMBOL_TABLE); iw.stepIn(IonType.STRUCT); iw.stepOut(); - SymbolTable symtab = iw.getSymbolTable(); // TODO amzn/ion-java#22 + SymbolTable symtab = iw.getSymbolTable(); // TODO amzn/ion-java/issues/22 symtab.intern("fred_1"); symtab.intern("fred_2"); testFlushing(); @@ -339,6 +341,7 @@ public void testAutoFlushTopLevelValuesForTypedWritesScalars() * * @see #testAutoFlushTopLevelValuesForTypedWritesContainers() */ + @SuppressWarnings("deprecation") @Test public void testAutoFlushTopLevelValuesForOtherWrites() throws Exception @@ -350,6 +353,11 @@ public void testAutoFlushTopLevelValuesForOtherWrites() val.writeTo(iw); checkFlushedAfterTopLevelValueWritten(); + //================== IonWriter.writeValue(IonValue) ==================== + iw = makeWriter(); + iw.writeValue(val); + checkFlushedAfterTopLevelValueWritten(); + //================== IonWriter.writeValues(IonReader) ================== iw = makeWriter(); IonReader reader = system().newReader(val); diff --git a/test/com/amazon/ion/impl/RawValueSpanReaderTest.java b/test/com/amazon/ion/impl/RawValueSpanReaderTest.java new file mode 100644 index 0000000000..e99ee79569 --- /dev/null +++ b/test/com/amazon/ion/impl/RawValueSpanReaderTest.java @@ -0,0 +1,325 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import com.amazon.ion.IonReader; +import com.amazon.ion.IonType; +import com.amazon.ion.OffsetSpan; +import com.amazon.ion.RawValueSpanProvider; +import com.amazon.ion.RawValueSpanReaderBasicTest; +import com.amazon.ion.SeekableReader; +import com.amazon.ion.Span; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.TestUtils; +import static com.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; +import static com.amazon.ion.TestUtils.GOOD_IONTESTS_FILES; +import static com.amazon.ion.TestUtils.testdataFiles; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import com.amazon.ion.junit.Injected; +import com.amazon.ion.junit.Injected.Inject; +import com.amazon.ion.system.IonReaderBuilder; +import java.io.File; +import java.io.FileInputStream; +import java.io.FilenameFilter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Tests the {@link RawValueSpanProvider} reader facet, which provides access + * to the reader's underlying byte buffer and vends OffsetSpans that provide + * positions of the current value in that buffer. + * + * @see RawValueSpanReaderBasicTest + */ +@SuppressWarnings({"deprecation", "javadoc"}) +@RunWith(Injected.class) +public class RawValueSpanReaderTest +{ + + @Inject("testFile") + public static final File[] FILES = + testdataFiles(new TestUtils.And(GLOBAL_SKIP_LIST, new FilenameFilter(){ + + public boolean accept(File dir, String name) + { + // Only accepting binary files because the RawValueSpanProvider + // facet is currently only compatible with the binary reader. + return name.endsWith(".10n"); + } + + }), GOOD_IONTESTS_FILES); + + + private File myTestFile; + + public void setTestFile(File file) + { + myTestFile = file; + } + + private IonReaderBinaryUserX reader; + private RawValueSpanProvider spanProvider; + private SeekableReader seekableReader; + + /* + * Returns a byte array that contains the value ONLY (i.e. no type code + * or length), as encoded by Ion. + */ + private byte[] valueBytes(OffsetSpan valueSpan) + { + // In a real-world use case, this byte buffer may already be allocated. + int valueSize = (int)(valueSpan.getFinishOffset() - valueSpan.getStartOffset()); + byte[] value = new byte[valueSize]; + System.arraycopy(spanProvider.buffer(), (int)valueSpan.getStartOffset(), value, 0, value.length); + return value; + } + + private static class SpanTester + { + final Object expected; + final byte[] expectedBytes; + final IonType expectedType; + final Span span; + + SpanTester(Object expected, IonType expectedType, byte[] expectedBytes, Span span) + { + this.expected = expected; + this.expectedBytes = expectedBytes; + this.expectedType = expectedType; + this.span = span; + } + } + + private SpanTester generateSpan(Object expected, IonType type) + { + // This span includes the TID and length bytes + Span span = seekableReader.currentSpan(); + // This gets just the value bytes + byte[] expectedBytes = valueBytes((OffsetSpan)spanProvider.valueSpan()); + return new SpanTester(expected, type, expectedBytes, span); + } + + private void generateSpans(List output) + { + IonType type = null; + while ((type = reader.next()) != null) + { + if (reader.isNullValue()) + { + output.add(generateSpan(null, type)); + continue; + } + switch (type) + { + case BOOL: + output.add(generateSpan(reader.booleanValue(), type)); + break; + case INT: + output.add(generateSpan(reader.bigIntegerValue(), type)); + break; + case FLOAT: + output.add(generateSpan(reader.doubleValue(), type)); + break; + case DECIMAL: + output.add(generateSpan(reader.bigDecimalValue(), type)); + break; + case TIMESTAMP: + output.add(generateSpan(reader.timestampValue(), type)); + break; + case STRING: + output.add(generateSpan(reader.stringValue(), type)); + break; + case SYMBOL: + output.add(generateSpan(reader.symbolValue(), type)); + break; + case BLOB: + case CLOB: + output.add(generateSpan(reader.newBytes(), type)); + break; + case LIST: + case SEXP: + case STRUCT: + // 'expected' not used for containers because there is no + // corresponding *Value() method on the reader. Instead, + // byte and position comparisons will be used, and the + // containers' values will be recursively checked. + output.add(generateSpan(null, type)); + reader.stepIn(); + generateSpans(output); + reader.stepOut(); + break; + default: + throw new IllegalStateException("unexpected type: " + type); + } + } + } + + private void assertSpan(SpanTester tester) throws Exception + { + + IonType type = tester.expectedType; + seekableReader.hoist(tester.span); // seeks back to the given span + assertEquals(type, reader.next()); + + Object expected = tester.expected; + if (reader.isNullValue()) + { + assertEquals(expected, null); + } + else + { + OffsetSpan valueSpan = (OffsetSpan)spanProvider.valueSpan(); + switch(type) + { + case BOOL: + assertEquals(expected, reader.booleanValue()); + break; + case INT: + assertEquals(expected, reader.bigIntegerValue()); + break; + case FLOAT: + assertEquals(expected, reader.doubleValue()); + break; + case DECIMAL: + assertEquals(expected, reader.bigDecimalValue()); + break; + case TIMESTAMP: + assertEquals(expected, reader.timestampValue()); + break; + case STRING: + // A common use case will be to pass strings along without + // decoding. This tests that case. + assertArrayEquals(((String)expected).getBytes("UTF-8"), valueBytes(valueSpan)); + assertEquals(expected, reader.stringValue()); + break; + case SYMBOL: + // SymbolTokenImpl does not override .equals + SymbolToken expectedToken = (SymbolToken)expected; + SymbolToken actualToken = reader.symbolValue(); + assertEquals(expectedToken.getSid(), actualToken.getSid()); + assertEquals(expectedToken.getText(), actualToken.getText()); + + break; + case BLOB: + case CLOB: + assertArrayEquals((byte[])expected, reader.newBytes()); + break; + case STRUCT: + case LIST: + case SEXP: + reader.stepIn(); + if (reader.next() != null) + { + // The start position of the container's value span should + // be the same as the start position of its first element's + // seekable span. + long expectedValueStart = valueSpan.getStartOffset(); + long expectedValueEnd = ((OffsetSpan) seekableReader.currentSpan()).getStartOffset(); + + // skips any nop pad to get at the actual value start + expectedValueStart += countNopPad((int) expectedValueStart); + + if (reader.isInStruct()) + { + // In structs, however, value spans will start before + // the first value's field name SID (VarUInt - 7 + // bits per byte, hence division by 0x80). + expectedValueStart += (reader.getFieldNameSymbol().getSid() / 0x80) + 1; + } + + assertEquals(expectedValueStart, expectedValueEnd); + } + + reader.stepOut(); + + break; + default: + throw new IllegalStateException("unexpected type: " + type); + } + assertArrayEquals(tester.expectedBytes, valueBytes(valueSpan)); + // All spans over the same value, no matter where they started, should finish at + // the same position. + assertEquals(((OffsetSpan)tester.span).getFinishOffset(), valueSpan.getFinishOffset()); + } + } + + private int countNopPad(int start) throws IOException + { + int index = start; + int len = 0; + + if (reader.isInStruct()) + { + // In structs nop pads have a sid before then + index += (reader.getFieldNameSymbol().getSid() / 0x80) + 1; + len += index - start; + } + + int td = reader.getCurrentBuffer()[index] & 0xFF; + int tid = _Private_IonConstants.getTypeCode(td); + int typeLen = _Private_IonConstants.getLowNibble(td); + + if(tid == _Private_IonConstants.tidNull && typeLen != _Private_IonConstants.lnIsNull){ + if(typeLen == _Private_IonConstants.lnIsVarLen) { + len += reader.readVarUInt(); + } + else { + len += 1; + } + + return len; + } + + return 0; + } + + private static byte[] readFileAsBytes(File file) throws IOException + { + FileInputStream in = new FileInputStream(file); + byte[] data = new byte[(int)file.length()]; + in.read(data); + in.close(); + return data; + } + + /** + * Retrieve a span for each value. Seek back to them in any order and assert + * that the values can be retrieved both through raw inspection of the + * underlying buffer and through the high-level IonReader APIs. + */ + @Test + public void testSpans() throws Exception + { + // Seeking currently only works over byte-backed IonReaders -- not InputStreams + reader = (IonReaderBinaryUserX) IonReaderBuilder.standard().build(readFileAsBytes(myTestFile)); + spanProvider = reader.asFacet(RawValueSpanProvider.class); + seekableReader = reader.asFacet(SeekableReader.class); + + List spans = new ArrayList(); + generateSpans(spans); + Collections.shuffle(spans); // spans can be revisited in any order + for (SpanTester span : spans) + { + assertSpan(span); + } + } +} diff --git a/test/software/amazon/ion/impl/SharedSymbolTableTest.java b/test/com/amazon/ion/impl/SharedSymbolTableTest.java similarity index 90% rename from test/software/amazon/ion/impl/SharedSymbolTableTest.java rename to test/com/amazon/ion/impl/SharedSymbolTableTest.java index 07d9eb5fd0..34bfab7582 100644 --- a/test/software/amazon/ion/impl/SharedSymbolTableTest.java +++ b/test/com/amazon/ion/impl/SharedSymbolTableTest.java @@ -1,37 +1,38 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; - -import static software.amazon.ion.impl.PrivateUtils.EMPTY_STRING_ARRAY; -import static software.amazon.ion.impl.PrivateUtils.stringIterator; -import static software.amazon.ion.impl.SymbolTableTest.checkSharedTable; -import static software.amazon.ion.impl.Symtabs.sharedSymtabStruct; - +package com.amazon.ion.impl; + +import static com.amazon.ion.impl.SymbolTableTest.checkSharedTable; +import static com.amazon.ion.impl.Symtabs.sharedSymtabStruct; +import static com.amazon.ion.impl._Private_Utils.EMPTY_STRING_ARRAY; +import static com.amazon.ion.impl._Private_Utils.stringIterator; + +import com.amazon.ion.IonException; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SharedSymtabMaker; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.SystemSymbols; +import com.amazon.ion.junit.Injected.Inject; import java.io.IOException; import org.junit.Test; -import software.amazon.ion.IonException; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.IonValue; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SharedSymtabMaker; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.SystemSymbols; -import software.amazon.ion.junit.Injected.Inject; /** diff --git a/test/software/amazon/ion/impl/SymbolTableTest.java b/test/com/amazon/ion/impl/SymbolTableTest.java similarity index 94% rename from test/software/amazon/ion/impl/SymbolTableTest.java rename to test/com/amazon/ion/impl/SymbolTableTest.java index d4ea57f8d1..3c62543709 100644 --- a/test/software/amazon/ion/impl/SymbolTableTest.java +++ b/test/com/amazon/ion/impl/SymbolTableTest.java @@ -1,33 +1,57 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; - -import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; -import static software.amazon.ion.SystemSymbols.ION; -import static software.amazon.ion.SystemSymbols.ION_1_0; -import static software.amazon.ion.SystemSymbols.ION_1_0_SID; -import static software.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; -import static software.amazon.ion.SystemSymbols.NAME; -import static software.amazon.ion.SystemSymbols.NAME_SID; -import static software.amazon.ion.SystemSymbols.SYMBOLS; -import static software.amazon.ion.impl.PrivateUtils.EMPTY_STRING_ARRAY; -import static software.amazon.ion.impl.PrivateUtils.stringIterator; -import static software.amazon.ion.impl.PrivateUtils.symtabTree; -import static software.amazon.ion.impl.Symtabs.printLocalSymtab; - +package com.amazon.ion.impl; + +import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; +import com.amazon.ion.SymbolToken; +import static com.amazon.ion.SystemSymbols.ION; +import static com.amazon.ion.SystemSymbols.ION_1_0; +import static com.amazon.ion.SystemSymbols.ION_1_0_SID; +import static com.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; +import static com.amazon.ion.SystemSymbols.NAME; +import static com.amazon.ion.SystemSymbols.NAME_SID; +import static com.amazon.ion.SystemSymbols.SYMBOLS; +import static com.amazon.ion.impl.Symtabs.printLocalSymtab; +import static com.amazon.ion.impl._Private_Utils.EMPTY_STRING_ARRAY; +import static com.amazon.ion.impl._Private_Utils.stringIterator; +import static com.amazon.ion.impl._Private_Utils.symtabTree; + +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonException; +import com.amazon.ion.IonInt; +import com.amazon.ion.IonList; +import com.amazon.ion.IonMutableCatalog; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonSexp; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSymbol; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.ReadOnlyValueException; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SystemSymbols; +import com.amazon.ion.Timestamp; +import com.amazon.ion.system.IonBinaryWriterBuilder; +import com.amazon.ion.system.IonSystemBuilder; +import com.amazon.ion.system.IonTextWriterBuilder; +import com.amazon.ion.system.SimpleCatalog; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.ArrayList; @@ -39,34 +63,11 @@ import java.util.Set; import org.junit.Ignore; import org.junit.Test; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonException; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonList; -import software.amazon.ion.IonMutableCatalog; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonSexp; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.IonWriter; -import software.amazon.ion.ReadOnlyValueException; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.SystemSymbols; -import software.amazon.ion.Timestamp; -import software.amazon.ion.system.IonBinaryWriterBuilder; -import software.amazon.ion.system.IonSystemBuilder; -import software.amazon.ion.system.IonTextWriterBuilder; -import software.amazon.ion.system.SimpleCatalog; /** * @see SharedSymbolTableTest */ +@SuppressWarnings("deprecation") public class SymbolTableTest extends IonTestCase { @@ -256,7 +257,7 @@ public void testLocalSymbolTableAppendImportBoundary() " symbols:[ \"s11\"]" + "}\n" + "1\n" + - LocalSymbolTablePrefix + + LocalSymbolTablePrefix + "{" + " imports:" + ION_SYMBOL_TABLE + "," + " symbols:[ \"s21\"]" + @@ -274,6 +275,7 @@ public void testLocalSymbolTableAppendImportBoundary() // new symbols in `original` don't influence SIDs for new symbols in `appended` after import checkSymbol("s11", systemMaxId() + 1, appended); checkSymbol("o1", systemMaxId() + 2, original); + checkSymbol("s11", systemMaxId() + 1, appended); checkSymbol("s21", systemMaxId() + 2, appended); checkSymbol("a1", systemMaxId() + 3, appended); @@ -439,6 +441,7 @@ public void testImportsFollowSymbols() final int import1id = systemMaxId() + 1; final int local1id = systemMaxId() + IMPORTED_1_MAX_ID + 1; + final int local2id = local1id + 1; String importingText = "$ion_1_0 "+ @@ -750,7 +753,7 @@ public void testDupLocalSymbolOnDatagram() throws Exception { final IonMutableCatalog cat = new SimpleCatalog(); cat.putTable(st); - // amzn/ion-java#46 has the datagram producing something like: + // amzn/ion-java/issues/46 has the datagram producing something like: // $ion_1_0 $ion_symbol_table::{imports:[{name: "foobar", version: 1, max_id: 1}], symbols: ["s1", "l1"]} $11 $12 // local table should not have "s1", user values should be $10 $11 IonDatagram dg = ion1.newDatagram(st); @@ -966,7 +969,7 @@ public void testBasicLocalSymtabCreation() public void testSymtabImageMaintenance() { IonSystem system = system(); - SymbolTable st = ((PrivateValueFactory)system).getLstFactory().newLocalSymtab(system.getSystemSymbolTable()); + SymbolTable st = ((_Private_ValueFactory)system).getLstFactory().newLocalSymtab(system.getSystemSymbolTable()); st.intern("foo"); IonStruct image = symtabTree(st); st.intern("bar"); @@ -1044,7 +1047,7 @@ public void testSharedSymtabCreationWithDuplicates() } - @Test @Ignore // TODO amzn/ion-java#12 + @Test @Ignore // TODO amzn/ion-java/issues/12 public void testSharedSymtabCreationWithEmptyName() { String[] syms = { "a", "b", "", "c" }; @@ -1314,7 +1317,7 @@ public void testWriteWithSymbolTable() throws IOException ByteArrayOutputStream out = new ByteArrayOutputStream(); IonWriter writer = system().newTextWriter(out); - data.writeTo(writer); + writer.writeValue(data); writer.close(); // dataMap.put("value", out.toByteArray()); @@ -1393,7 +1396,7 @@ public void testSymtabBinaryInjection() throws Exception writer.writeNull(); writer.writeInt(10); writer.writeFloat(10.0); - writer.writeTimestamp(Timestamp.forDay(2013, 1, 1)); + writer.writeTimestamp(new Timestamp(2013, 1, 1)); writer.writeSymbol("abc"); // this is where symbol table injection happens writer.writeString("abc"); writer.stepOut(); diff --git a/test/software/amazon/ion/impl/Symtabs.java b/test/com/amazon/ion/impl/Symtabs.java similarity index 81% rename from test/software/amazon/ion/impl/Symtabs.java rename to test/com/amazon/ion/impl/Symtabs.java index c1c0ebf084..f1a78a22df 100644 --- a/test/software/amazon/ion/impl/Symtabs.java +++ b/test/com/amazon/ion/impl/Symtabs.java @@ -1,38 +1,38 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; +import static com.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; +import static com.amazon.ion.util.IonTextUtils.printString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; -import static software.amazon.ion.SystemSymbols.ION_SHARED_SYMBOL_TABLE; -import static software.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; -import static software.amazon.ion.util.IonTextUtils.printString; +import com.amazon.ion.IonList; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSystem; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SystemSymbols; +import com.amazon.ion.ValueFactory; +import com.amazon.ion.system.IonSystemBuilder; +import com.amazon.ion.system.SimpleCatalog; import java.io.IOException; import java.util.Arrays; -import software.amazon.ion.IonList; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSystem; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SystemSymbols; -import software.amazon.ion.ValueFactory; -import software.amazon.ion.impl.PrivateIonSystem; -import software.amazon.ion.impl.PrivateLocalSymbolTableFactory; -import software.amazon.ion.system.IonSystemBuilder; -import software.amazon.ion.system.SimpleCatalog; + public class Symtabs { @@ -110,7 +110,7 @@ private static int[] registerTables(String[] serializedTables) { int[] maxIds = new int[serializedTables.length]; - PrivateIonSystem system = (PrivateIonSystem) + _Private_IonSystem system = (_Private_IonSystem) IonSystemBuilder.standard().withCatalog(CATALOG).build(); for (int i = 1; i < serializedTables.length; i++) @@ -229,12 +229,12 @@ public static SymbolTable makeLocalSymtab(IonSystem system, } /** - * Trampoline to {@link LocalSymbolTable#DEFAULT_LST_FACTORY} - * @return the {@link LocalSymbolTable.Factory} singleton. - */ - public static PrivateLocalSymbolTableFactory localSymbolTableFactory() - { - return LocalSymbolTable.DEFAULT_LST_FACTORY; - } + * Trampoline to {@link LocalSymbolTable#DEFAULT_LST_FACTORY} + * @return the {@link LocalSymbolTable.Factory} singleton. + */ + public static _Private_LocalSymbolTableFactory localSymbolTableFactory() + { + return LocalSymbolTable.DEFAULT_LST_FACTORY; + } } diff --git a/test/com/amazon/ion/impl/TestMarkupCallback.java b/test/com/amazon/ion/impl/TestMarkupCallback.java new file mode 100644 index 0000000000..86187004ff --- /dev/null +++ b/test/com/amazon/ion/impl/TestMarkupCallback.java @@ -0,0 +1,148 @@ + +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import com.amazon.ion.IonType; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.util._Private_FastAppendable; +import java.io.IOException; +import org.junit.Assert; + +public class TestMarkupCallback + extends _Private_MarkupCallback +{ + public static class Builder + implements _Private_CallbackBuilder + { + public _Private_MarkupCallback build(_Private_FastAppendable rawOutput) + { + return new TestMarkupCallback(rawOutput); + } + } + + private final _Private_FastAppendable myAppendable; + + public TestMarkupCallback(_Private_FastAppendable out) { + super(out); + myAppendable = out; + } + + public void checkForNulls(IonType iType) { + Assert.assertNotNull("iType is null", iType); + Assert.assertNotNull("myOut is null", myAppendable); + } + + @Override + public void beforeValue(IonType iType) + throws IOException + { + checkForNulls(iType); + myAppendable.append(""); + } + + @Override + public void afterValue(IonType iType) + throws IOException + { + checkForNulls(iType); + myAppendable.append(""); + } + + @Override + public void afterStepIn(IonType iType) + throws IOException + { + checkForNulls(iType); + myAppendable.append(""); + } + + @Override + public void beforeStepOut(IonType iType) + throws IOException + { + checkForNulls(iType); + myAppendable.append(""); + } + + @Override + public void beforeFieldName(IonType iType, SymbolToken name) + throws IOException + { + checkForNulls(iType); + Assert.assertNotNull("beforeFieldName: name is null", name); + myAppendable.append(""); + } + + @Override + public void afterFieldName(IonType iType, SymbolToken name) + throws IOException + { + checkForNulls(iType); + Assert.assertNotNull("afterFieldName: name is null", name); + myAppendable.append(""); + } + + @Override + public void beforeSeparator(IonType iType) + throws IOException + { + checkForNulls(iType); + myAppendable.append(""); + } + + @Override + public void afterSeparator(IonType iType) + throws IOException + { + checkForNulls(iType); + myAppendable.append(""); + } + + @Override + public void beforeAnnotations(IonType iType) + throws IOException + { + checkForNulls(iType); + myAppendable.append(""); + } + + @Override + public void afterAnnotations(IonType iType) + throws IOException + { + checkForNulls(iType); + myAppendable.append(""); + } + + @Override + public void beforeEachAnnotation(IonType iType, SymbolToken ann) + throws IOException + { + checkForNulls(iType); + Assert.assertNotNull("beforeEachAnnotation: ann is null", ann); + myAppendable.append(""); + } + + @Override + public void afterEachAnnotation(IonType iType, SymbolToken ann) + throws IOException + { + checkForNulls(iType); + Assert.assertNotNull("afterEachAnnotation: ann is null", ann); + myAppendable.append(""); + } +} diff --git a/test/software/amazon/ion/impl/TextWriterTest.java b/test/com/amazon/ion/impl/TextWriterTest.java similarity index 89% rename from test/software/amazon/ion/impl/TextWriterTest.java rename to test/com/amazon/ion/impl/TextWriterTest.java index 536e47e534..ba57d634bf 100644 --- a/test/software/amazon/ion/impl/TextWriterTest.java +++ b/test/com/amazon/ion/impl/TextWriterTest.java @@ -1,39 +1,40 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; +import static com.amazon.ion.SystemSymbols.ION_1_0; +import static com.amazon.ion.system.IonWriterBuilder.InitialIvmHandling.ENSURE; +import static com.amazon.ion.system.IonWriterBuilder.InitialIvmHandling.SUPPRESS; +import static com.amazon.ion.system.IonWriterBuilder.IvmMinimizing.DISTANT; import static java.lang.String.format; -import static software.amazon.ion.SystemSymbols.ION_1_0; -import static software.amazon.ion.system.IonWriterBuilder.InitialIvmHandling.ENSURE; -import static software.amazon.ion.system.IonWriterBuilder.InitialIvmHandling.SUPPRESS; -import static software.amazon.ion.system.IonWriterBuilder.IvmMinimizing.DISTANT; -import java.io.ByteArrayOutputStream; +import com.amazon.ion.IonBinaryWriter; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonSequence; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SystemSymbols; +import com.amazon.ion.system.IonTextWriterBuilder; +import com.amazon.ion.system.IonTextWriterBuilder.LstMinimizing; +import com.amazon.ion.system.IonWriterBuilder.IvmMinimizing; import java.io.OutputStream; import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonSequence; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SystemSymbols; -import software.amazon.ion.impl.PrivateUtils; -import software.amazon.ion.system.IonTextWriterBuilder; -import software.amazon.ion.system.IonTextWriterBuilder.LstMinimizing; -import software.amazon.ion.system.IonWriterBuilder.IvmMinimizing; + public class TextWriterTest extends OutputStreamWriterTestCase @@ -55,7 +56,7 @@ protected String outputString() throws Exception { byte[] utf8Bytes = outputByteArray(); - return PrivateUtils.utf8(utf8Bytes); + return _Private_Utils.utf8(utf8Bytes); } @Test @@ -137,12 +138,11 @@ public void testLstMinimizing() { SymbolTable fred1 = Symtabs.register("fred", 1, catalog()); - ByteArrayOutputStream buf = new ByteArrayOutputStream(); - IonWriter binaryWriter = system().newBinaryWriter(buf, fred1); + IonBinaryWriter binaryWriter = system().newBinaryWriter(fred1); binaryWriter.writeSymbol("fred_1"); binaryWriter.writeSymbol("ginger"); binaryWriter.finish(); - byte[] binaryData = buf.toByteArray(); + byte[] binaryData = binaryWriter.getBytes(); options = IonTextWriterBuilder.standard(); @@ -282,7 +282,7 @@ public void testWritingLongStrings() options.withPrettyPrinting(); expectRendering( - // TODO amzn/ion-java#57 determine if these really should be platform independent newlines + // TODO amzn/ion-java/issues/57 determine if these really should be platform independent newlines format( "%n" + "'''looong'''%n" + diff --git a/test/software/amazon/ion/impl/TreeReaderTest.java b/test/com/amazon/ion/impl/TreeReaderTest.java similarity index 69% rename from test/software/amazon/ion/impl/TreeReaderTest.java rename to test/com/amazon/ion/impl/TreeReaderTest.java index 5ea8ae527d..34af455227 100644 --- a/test/software/amazon/ion/impl/TreeReaderTest.java +++ b/test/com/amazon/ion/impl/TreeReaderTest.java @@ -1,30 +1,32 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; -import static software.amazon.ion.junit.IonAssert.assertTopLevel; +import static com.amazon.ion.junit.IonAssert.assertTopLevel; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonInt; +import com.amazon.ion.IonList; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonType; +import com.amazon.ion.TestUtils; +import com.amazon.ion.streaming.ReaderTestCase; import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonList; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonType; -import software.amazon.ion.TestUtils; -import software.amazon.ion.streaming.ReaderTestCase; + public class TreeReaderTest extends ReaderTestCase diff --git a/test/software/amazon/ion/impl/UnifiedInputStreamXTest.java b/test/com/amazon/ion/impl/UnifiedInputStreamXTest.java similarity index 69% rename from test/software/amazon/ion/impl/UnifiedInputStreamXTest.java rename to test/com/amazon/ion/impl/UnifiedInputStreamXTest.java index f6413bceb0..e353b49e19 100644 --- a/test/software/amazon/ion/impl/UnifiedInputStreamXTest.java +++ b/test/com/amazon/ion/impl/UnifiedInputStreamXTest.java @@ -1,8 +1,19 @@ /* - * Copyright 2017 Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; import org.junit.Assert; import org.junit.Test; diff --git a/test/software/amazon/ion/impl/ValueWriterTest.java b/test/com/amazon/ion/impl/ValueWriterTest.java similarity index 81% rename from test/software/amazon/ion/impl/ValueWriterTest.java rename to test/com/amazon/ion/impl/ValueWriterTest.java index c8141fb9c1..730ce7158e 100644 --- a/test/software/amazon/ion/impl/ValueWriterTest.java +++ b/test/com/amazon/ion/impl/ValueWriterTest.java @@ -1,28 +1,30 @@ /* - * Copyright 2009-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl; +package com.amazon.ion.impl; +import com.amazon.ion.IonContainer; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonList; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonSymbol; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; import org.junit.Ignore; import org.junit.Test; -import software.amazon.ion.IonContainer; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonList; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; + public class ValueWriterTest extends IonWriterTestCase @@ -136,13 +138,13 @@ public void testWriteValuesWithSymtabIntoContainer() } - @Override @Test @Ignore // TODO amzn/ion-java#8 + @Override @Test @Ignore // TODO amzn/ion-java/issues/8 public void testWriteIVMImplicitly() throws Exception { super.testWriteIVMImplicitly(); - // TODO amzn/ion-java#20 + // TODO amzn/ion-java/issues/20 // assertEquals(2, myDatagram.size()); } @@ -152,7 +154,7 @@ public void testWriteIVMExplicitly() throws Exception { super.testWriteIVMExplicitly(); - // TODO amzn/ion-java#20 + // TODO amzn/ion-java/issues/20 //assertEquals(2, myDatagram.size()); } } diff --git a/test/software/amazon/ion/impl/VarIntTest.java b/test/com/amazon/ion/impl/VarIntTest.java similarity index 83% rename from test/software/amazon/ion/impl/VarIntTest.java rename to test/com/amazon/ion/impl/VarIntTest.java index d5ab77d912..1e297b317b 100644 --- a/test/software/amazon/ion/impl/VarIntTest.java +++ b/test/com/amazon/ion/impl/VarIntTest.java @@ -1,11 +1,27 @@ -package software.amazon.ion.impl; +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; import org.junit.Test; -import software.amazon.ion.IonException; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.system.SimpleCatalog; +import com.amazon.ion.IonException; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.system.SimpleCatalog; import java.io.ByteArrayInputStream; +import java.math.BigInteger; public class VarIntTest extends IonTestCase { @@ -86,7 +102,6 @@ public void readVarIntegerNegativeZero() throws Exception { private IonReaderBinaryUserX makeReader(String hex) throws Exception { ByteArrayInputStream input = new ByteArrayInputStream(parseHexBinary("E00100EA" + hex)); - UnifiedInputStreamX uis = UnifiedInputStreamX.makeStream(input); uis.skip(4); @@ -122,4 +137,4 @@ private static int parseHexChar(char c) { } throw new IllegalArgumentException("invalid hex character: " + c); } -} \ No newline at end of file +} diff --git a/test/com/amazon/ion/impl/_Private_ScalarConversionsTest.java b/test/com/amazon/ion/impl/_Private_ScalarConversionsTest.java new file mode 100644 index 0000000000..c607cbb891 --- /dev/null +++ b/test/com/amazon/ion/impl/_Private_ScalarConversionsTest.java @@ -0,0 +1,44 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl; + +import static org.junit.Assert.*; + +import org.junit.Assert; +import org.junit.Test; +import com.amazon.ion.Decimal; +import static org.junit.Assert.*; + +public class _Private_ScalarConversionsTest { + private long decimalToLong(final Decimal d) { + _Private_ScalarConversions.ValueVariant v = new _Private_ScalarConversions.ValueVariant(); + v.setValue(d); + v.cast(_Private_ScalarConversions.FNID_FROM_DECIMAL_TO_LONG); + return v.getLong(); + } + @Test + public void decimalToLong() { + assertEquals(1, decimalToLong(Decimal.valueOf(1L))); + } + @Test + public void decimalToMinLong() { + assertEquals(Long.MAX_VALUE, decimalToLong(Decimal.valueOf(Long.MAX_VALUE))); + } + @Test + public void decimalToMaxLong() { + assertEquals(Long.MIN_VALUE, decimalToLong(Decimal.valueOf(Long.MIN_VALUE))); + } +} diff --git a/test/software/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java b/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java similarity index 83% rename from test/software/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java rename to test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java index fbbe1cfa73..b83bbbd2e1 100644 --- a/test/software/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java +++ b/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java @@ -1,23 +1,37 @@ /* - * Copyright 2016-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.bin; +package com.amazon.ion.impl.bin; import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; import static java.util.Collections.unmodifiableMap; +import com.amazon.ion.IonContainer; +import com.amazon.ion.IonMutableCatalog; +import com.amazon.ion.IonSymbol; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.SystemSymbols; +import com.amazon.ion.impl.bin.IonManagedBinaryWriter.ImportedSymbolResolverMode; +import com.amazon.ion.impl.bin._Private_IonManagedBinaryWriterBuilder.AllocatorMode; +import com.amazon.ion.junit.Injected.Inject; + import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -26,21 +40,8 @@ import java.util.List; import java.util.Map; import org.junit.Test; -import software.amazon.ion.IonContainer; -import software.amazon.ion.IonMutableCatalog; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.SystemSymbols; -import software.amazon.ion.impl.bin.PrivateIonManagedBinaryWriterBuilder; -import software.amazon.ion.impl.bin.Symbols; -import software.amazon.ion.impl.bin.IonManagedBinaryWriter.ImportedSymbolResolverMode; -import software.amazon.ion.impl.bin.PrivateIonManagedBinaryWriterBuilder.AllocatorMode; -import software.amazon.ion.junit.Injected.Inject; +@SuppressWarnings("deprecation") public class IonManagedBinaryWriterTest extends IonRawBinaryWriterTest { @SuppressWarnings("unchecked") @@ -134,7 +135,7 @@ protected IonWriter createWriter(final OutputStream out) throws IOException catalog.putTable(table); } - final IonWriter writer = PrivateIonManagedBinaryWriterBuilder + final IonWriter writer = _Private_IonManagedBinaryWriterBuilder .create(AllocatorMode.POOLED) .withImports(importedSymbolResolverMode, symbolTables) .withPreallocationMode(preallocationMode) diff --git a/test/software/amazon/ion/impl/bin/IonRawBinaryWriterTest.java b/test/com/amazon/ion/impl/bin/IonRawBinaryWriterTest.java similarity index 75% rename from test/software/amazon/ion/impl/bin/IonRawBinaryWriterTest.java rename to test/com/amazon/ion/impl/bin/IonRawBinaryWriterTest.java index 33745b978e..7d23e59afc 100644 --- a/test/software/amazon/ion/impl/bin/IonRawBinaryWriterTest.java +++ b/test/com/amazon/ion/impl/bin/IonRawBinaryWriterTest.java @@ -1,68 +1,69 @@ /* - * Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.bin; - -import static software.amazon.ion.IonType.BLOB; -import static software.amazon.ion.IonType.BOOL; -import static software.amazon.ion.IonType.CLOB; -import static software.amazon.ion.IonType.DECIMAL; -import static software.amazon.ion.IonType.FLOAT; -import static software.amazon.ion.IonType.INT; -import static software.amazon.ion.IonType.LIST; -import static software.amazon.ion.IonType.NULL; -import static software.amazon.ion.IonType.SEXP; -import static software.amazon.ion.IonType.STRING; -import static software.amazon.ion.IonType.STRUCT; -import static software.amazon.ion.IonType.SYMBOL; -import static software.amazon.ion.IonType.TIMESTAMP; -import static software.amazon.ion.SystemSymbols.IMPORTS_SID; -import static software.amazon.ion.SystemSymbols.NAME_SID; -import static software.amazon.ion.SystemSymbols.VERSION_SID; -import static software.amazon.ion.TestUtils.hexDump; -import static software.amazon.ion.impl.bin.Symbols.systemSymbol; - -import java.io.ByteArrayOutputStream; +package com.amazon.ion.impl.bin; + +import static com.amazon.ion.IonType.BLOB; +import static com.amazon.ion.IonType.BOOL; +import static com.amazon.ion.IonType.CLOB; +import static com.amazon.ion.IonType.DECIMAL; +import static com.amazon.ion.IonType.FLOAT; +import static com.amazon.ion.IonType.INT; +import static com.amazon.ion.IonType.LIST; +import static com.amazon.ion.IonType.NULL; +import static com.amazon.ion.IonType.SEXP; +import static com.amazon.ion.IonType.STRING; +import static com.amazon.ion.IonType.STRUCT; +import static com.amazon.ion.IonType.SYMBOL; +import static com.amazon.ion.IonType.TIMESTAMP; +import static com.amazon.ion.SystemSymbols.IMPORTS_SID; +import static com.amazon.ion.SystemSymbols.NAME_SID; +import static com.amazon.ion.SystemSymbols.VERSION_SID; +import static com.amazon.ion.TestUtils.hexDump; +import static com.amazon.ion.impl.bin.Symbols.systemSymbol; + +import com.amazon.ion.IonException; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.TestUtils; +import com.amazon.ion.Timestamp; +import com.amazon.ion.impl.bin.AbstractIonWriter.WriteValueOptimization; +import com.amazon.ion.impl.bin.IonBinaryWriterAdapter.Factory; +import com.amazon.ion.impl.bin.IonRawBinaryWriter.PreallocationMode; +import com.amazon.ion.impl.bin.IonRawBinaryWriter.StreamCloseMode; +import com.amazon.ion.impl.bin.IonRawBinaryWriter.StreamFlushMode; +import com.amazon.ion.junit.Injected; +import com.amazon.ion.junit.Injected.Inject; +import com.amazon.ion.junit.IonAssert; +import com.amazon.ion.system.IonSystemBuilder; import java.io.IOException; import java.io.OutputStream; import java.math.BigDecimal; import java.math.BigInteger; + import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import software.amazon.ion.IonException; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.IonWriter; -import software.amazon.ion.TestUtils; -import software.amazon.ion.Timestamp; -import software.amazon.ion.impl.bin.BlockAllocatorProviders; -import software.amazon.ion.impl.bin.IonRawBinaryWriter; -import software.amazon.ion.impl.bin.AbstractIonWriter.WriteValueOptimization; -import software.amazon.ion.impl.bin.IonRawBinaryWriter.PreallocationMode; -import software.amazon.ion.impl.bin.IonRawBinaryWriter.StreamCloseMode; -import software.amazon.ion.impl.bin.IonRawBinaryWriter.StreamFlushMode; -import software.amazon.ion.junit.Injected; -import software.amazon.ion.junit.IonAssert; -import software.amazon.ion.junit.Injected.Inject; -import software.amazon.ion.system.IonSystemBuilder; // TODO incorporate this into the main reader/writer tests +@SuppressWarnings("deprecation") @RunWith(Injected.class) public class IonRawBinaryWriterTest extends Assert { @@ -81,8 +82,8 @@ protected static void assertEquals(final IonValue v1, final IonValue v2) @Inject("preallocationMode") public static final PreallocationMode[] PREALLOCATION_DIMENSION = PreallocationMode.values(); - protected ByteArrayOutputStream buffer; - protected IonWriter writer; + // XXX use this API to indirectly test it + protected IonBinaryWriterAdapter writer; protected PreallocationMode preallocationMode; public void setPreallocationMode(final PreallocationMode preallocationMode) @@ -93,8 +94,15 @@ public void setPreallocationMode(final PreallocationMode preallocationMode) @Before public final void setup() throws Exception { - buffer = new ByteArrayOutputStream(); - writer = createWriter(buffer); + writer = new IonBinaryWriterAdapter( + new Factory() + { + public IonWriter create(final OutputStream out) throws IOException + { + return createWriter(out); + } + } + ); writeIVMIfPossible(); } @@ -102,7 +110,6 @@ public final void setup() throws Exception public final void teardown() throws Exception { writer.close(); - buffer = null; writer = null; } @@ -122,9 +129,10 @@ protected IonWriter createWriter(final OutputStream out) throws IOException private void writeIVMIfPossible() throws IOException { - if (writer instanceof IonRawBinaryWriter) + final IonWriter delegate = writer.getDelegate(); + if (delegate instanceof IonRawBinaryWriter) { - ((IonRawBinaryWriter) writer).writeIonVersionMarker(); + ((IonRawBinaryWriter) delegate).writeIonVersionMarker(); } } @@ -134,7 +142,7 @@ protected void additionalValueAssertions(final IonValue value) {} protected final void assertValue(final String literal) throws IOException { writer.finish(); - final byte[] data = buffer.toByteArray(); + final byte[] data = writer.getBytes(); final IonValue actual; try { actual = system().singleValue(data); @@ -147,7 +155,7 @@ protected final void assertValue(final String literal) throws IOException additionalValueAssertions(actual); // prepare for next value - buffer.reset(); + writer.reset(); writeIVMIfPossible(); } @@ -245,47 +253,43 @@ public void testFloat() throws Exception } public int ivmLength() { - return 0; + return 4; } @Test - public void testVariableFloat() throws Exception + public void testVariableFloatZero() throws Exception { - // IonRawBinaryWriterTest writes a single BVM, - // but IonManagedBinaryWriterTest writes a BVM - // when a value is written after a #finish() - // call. Normalize before the test. - writer.writeFloat(0.0); - writer.finish(); - buffer.reset(); - // 0e0 - 32-bit writer.writeFloat(0.0); writer.finish(); - assertEquals(ivmLength() + 5, - buffer.size()); - buffer.reset(); + assertEquals(ivmLength() + 5, writer.byteSize()); + } + @Test + public void testVariableFloatInfinity() throws Exception + { // +inf - 32-bit writer.writeFloat(Double.POSITIVE_INFINITY); writer.finish(); - assertEquals(ivmLength() + 5, - buffer.size()); - buffer.reset(); + assertEquals(ivmLength() + 5, writer.byteSize()); + } + @Test + public void testVariableFloatFractionOverflow() throws Exception + { // 64-bit value - fraction overflows 32-bits writer.writeFloat(2.147483647e9); writer.finish(); - assertEquals(ivmLength() + 9, - buffer.size()); - buffer.reset(); + assertEquals(ivmLength() + 9, writer.byteSize()); + } + @Test + public void testVariableFloatExponentOverflow() throws Exception + { // 64-bit value - exponent overflows 32-bits writer.writeFloat(6e128); writer.finish(); - assertEquals(ivmLength() + 9, - buffer.size()); - buffer.reset(); + assertEquals(ivmLength() + 9, writer.byteSize()); } private static final String DECIMAL_10_DIGIT = "1.000000001"; @@ -329,13 +333,13 @@ public void testTimestamp() throws Exception public void testTimestampInvalidFractionalSeconds() throws IOException { Timestamp ts = Timestamp.createFromUtcFields( - Timestamp.Precision.SECOND, 2000, 1, 1, 0, 0, 0, new BigDecimal(new BigInteger("0"), -1), 0); + Timestamp.Precision.FRACTION, 2000, 1, 1, 0, 0, 0, new BigDecimal(new BigInteger("0"), -1), 0); writer.writeTimestamp(ts); writer.finish(); assertEquals( TestUtils.hexDump(new byte[] {(byte)0xe0, 0x01, 0x00, (byte)0xea, 0x68, (byte)0x80, 0x0f, (byte)0xd0, (byte)0x81, (byte)0x81, (byte)0x80, (byte)0x80, (byte)0x80 }), - TestUtils.hexDump(buffer.toByteArray())); + TestUtils.hexDump(writer.getBytes())); } // note that we stick to system symbols for round trip assertions diff --git a/test/software/amazon/ion/impl/bin/PooledBlockAllocatorProviderTest.java b/test/com/amazon/ion/impl/bin/PooledBlockAllocatorProviderTest.java similarity index 81% rename from test/software/amazon/ion/impl/bin/PooledBlockAllocatorProviderTest.java rename to test/com/amazon/ion/impl/bin/PooledBlockAllocatorProviderTest.java index 8f5e7156c8..97ccef5546 100644 --- a/test/software/amazon/ion/impl/bin/PooledBlockAllocatorProviderTest.java +++ b/test/com/amazon/ion/impl/bin/PooledBlockAllocatorProviderTest.java @@ -1,18 +1,19 @@ /* - * Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.bin; +package com.amazon.ion.impl.bin; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotSame; @@ -21,9 +22,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import software.amazon.ion.impl.bin.Block; -import software.amazon.ion.impl.bin.BlockAllocator; -import software.amazon.ion.impl.bin.PooledBlockAllocatorProvider; public class PooledBlockAllocatorProviderTest { diff --git a/test/software/amazon/ion/impl/bin/WriteBufferTest.java b/test/com/amazon/ion/impl/bin/WriteBufferTest.java similarity index 97% rename from test/software/amazon/ion/impl/bin/WriteBufferTest.java rename to test/com/amazon/ion/impl/bin/WriteBufferTest.java index 8c200e9c04..5a5ddb5acf 100644 --- a/test/software/amazon/ion/impl/bin/WriteBufferTest.java +++ b/test/com/amazon/ion/impl/bin/WriteBufferTest.java @@ -1,23 +1,24 @@ /* - * Copyright 2016-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.bin; +package com.amazon.ion.impl.bin; +import static com.amazon.ion.TestUtils.hexDump; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -import static software.amazon.ion.TestUtils.hexDump; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -25,9 +26,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import software.amazon.ion.impl.bin.BlockAllocator; -import software.amazon.ion.impl.bin.BlockAllocatorProviders; -import software.amazon.ion.impl.bin.WriteBuffer; public class WriteBufferTest { diff --git a/test/software/amazon/ion/impl/lite/BaseIonSequenceLiteSublistTestCase.java b/test/com/amazon/ion/impl/lite/BaseIonSequenceLiteSublistTestCase.java similarity index 95% rename from test/software/amazon/ion/impl/lite/BaseIonSequenceLiteSublistTestCase.java rename to test/com/amazon/ion/impl/lite/BaseIonSequenceLiteSublistTestCase.java index 644ece031f..2bc94f7cf4 100644 --- a/test/software/amazon/ion/impl/lite/BaseIonSequenceLiteSublistTestCase.java +++ b/test/com/amazon/ion/impl/lite/BaseIonSequenceLiteSublistTestCase.java @@ -1,4 +1,19 @@ -package software.amazon.ion.impl.lite; +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl.lite; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -11,12 +26,12 @@ import java.util.List; import java.util.ListIterator; import org.junit.Test; -import software.amazon.ion.ContainedValueException; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonSequence; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonValue; -import software.amazon.ion.system.IonSystemBuilder; +import com.amazon.ion.ContainedValueException; +import com.amazon.ion.IonInt; +import com.amazon.ion.IonSequence; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonValue; +import com.amazon.ion.system.IonSystemBuilder; /** * All tests related to {@link IonSequenceLite#subList(int, int)}. Extracted to a separate test due to the amount of diff --git a/test/software/amazon/ion/impl/lite/BaseIonSequenceLiteTestCase.java b/test/com/amazon/ion/impl/lite/BaseIonSequenceLiteTestCase.java similarity index 68% rename from test/software/amazon/ion/impl/lite/BaseIonSequenceLiteTestCase.java rename to test/com/amazon/ion/impl/lite/BaseIonSequenceLiteTestCase.java index 0942a0ae7c..66964cbb59 100644 --- a/test/software/amazon/ion/impl/lite/BaseIonSequenceLiteTestCase.java +++ b/test/com/amazon/ion/impl/lite/BaseIonSequenceLiteTestCase.java @@ -1,4 +1,19 @@ -package software.amazon.ion.impl.lite; +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl.lite; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -6,21 +21,13 @@ import static org.junit.Assert.assertTrue; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; -import java.util.ConcurrentModificationException; -import java.util.Iterator; -import java.util.List; -import java.util.ListIterator; import org.junit.Test; -import software.amazon.ion.ContainedValueException; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonList; -import software.amazon.ion.IonSequence; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonValue; -import software.amazon.ion.ReadOnlyValueException; -import software.amazon.ion.system.IonSystemBuilder; +import com.amazon.ion.IonSequence; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonValue; +import com.amazon.ion.ReadOnlyValueException; +import com.amazon.ion.system.IonSystemBuilder; public abstract class BaseIonSequenceLiteTestCase { diff --git a/test/software/amazon/ion/impl/lite/IonContextTest.java b/test/com/amazon/ion/impl/lite/IonContextTest.java similarity index 91% rename from test/software/amazon/ion/impl/lite/IonContextTest.java rename to test/com/amazon/ion/impl/lite/IonContextTest.java index 823d507706..2b13fb554d 100644 --- a/test/software/amazon/ion/impl/lite/IonContextTest.java +++ b/test/com/amazon/ion/impl/lite/IonContextTest.java @@ -1,38 +1,35 @@ /* - * Copyright 2013-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.impl.lite; - -import static software.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; -import static software.amazon.ion.impl.Symtabs.CATALOG; - +package com.amazon.ion.impl.lite; + +import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; +import static com.amazon.ion.impl.Symtabs.CATALOG; + +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonList; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSymbol; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.IonValue; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.impl.Symtabs; +import com.amazon.ion.junit.Injected.Inject; import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonList; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.IonValue; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.impl.Symtabs; -import software.amazon.ion.impl.lite.ContainerlessContext; -import software.amazon.ion.impl.lite.IonContainerLite; -import software.amazon.ion.impl.lite.IonContext; -import software.amazon.ion.impl.lite.IonValueLite; -import software.amazon.ion.junit.Injected.Inject; public class IonContextTest diff --git a/test/com/amazon/ion/impl/lite/IonDatagramLiteSublistTest.java b/test/com/amazon/ion/impl/lite/IonDatagramLiteSublistTest.java new file mode 100644 index 0000000000..55d9380597 --- /dev/null +++ b/test/com/amazon/ion/impl/lite/IonDatagramLiteSublistTest.java @@ -0,0 +1,31 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl.lite; + +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonSequence; + +public class IonDatagramLiteSublistTest extends BaseIonSequenceLiteSublistTestCase { + + protected IonSequence newSequence() { + final IonDatagram datagram = SYSTEM.newDatagram(); + for(int i : INTS) { + datagram.add(SYSTEM.newInt(INTS[i])); + } + + return datagram; + } +} diff --git a/test/com/amazon/ion/impl/lite/IonDatagramLiteTest.java b/test/com/amazon/ion/impl/lite/IonDatagramLiteTest.java new file mode 100644 index 0000000000..5ab78440ec --- /dev/null +++ b/test/com/amazon/ion/impl/lite/IonDatagramLiteTest.java @@ -0,0 +1,25 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl.lite; + +import com.amazon.ion.IonSequence; + +public class IonDatagramLiteTest extends BaseIonSequenceLiteTestCase { + @Override + protected IonSequence newEmptySequence() { + return SYSTEM.newDatagram(); + } +} \ No newline at end of file diff --git a/test/com/amazon/ion/impl/lite/IonListLiteTest.java b/test/com/amazon/ion/impl/lite/IonListLiteTest.java new file mode 100644 index 0000000000..86db62c90d --- /dev/null +++ b/test/com/amazon/ion/impl/lite/IonListLiteTest.java @@ -0,0 +1,25 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl.lite; + +import com.amazon.ion.IonSequence; + +public class IonListLiteTest extends BaseIonSequenceLiteTestCase { + @Override + protected IonSequence newEmptySequence() { + return SYSTEM.newEmptyList(); + } +} diff --git a/test/software/amazon/ion/impl/lite/IonListSexpLiteSublistTest.java b/test/com/amazon/ion/impl/lite/IonListSexpLiteSublistTest.java similarity index 77% rename from test/software/amazon/ion/impl/lite/IonListSexpLiteSublistTest.java rename to test/com/amazon/ion/impl/lite/IonListSexpLiteSublistTest.java index 6166a5843b..bd607af587 100644 --- a/test/software/amazon/ion/impl/lite/IonListSexpLiteSublistTest.java +++ b/test/com/amazon/ion/impl/lite/IonListSexpLiteSublistTest.java @@ -1,16 +1,33 @@ -package software.amazon.ion.impl.lite; +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl.lite; import static org.junit.Assert.assertEquals; import java.util.List; import java.util.ListIterator; +import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; -import software.amazon.ion.IonInt; -import software.amazon.ion.IonSequence; -import software.amazon.ion.IonValue; -import software.amazon.ion.junit.Injected; -import software.amazon.ion.junit.Injected.Inject; +import com.amazon.ion.IonInt; +import com.amazon.ion.IonSequence; +import com.amazon.ion.IonValue; +import com.amazon.ion.impl.lite.BaseIonSequenceLiteSublistTestCase; +import com.amazon.ion.junit.Injected; +import com.amazon.ion.junit.Injected.Inject; /** * Sublist tests for list and sexp diff --git a/test/com/amazon/ion/impl/lite/IonSexpLiteTest.java b/test/com/amazon/ion/impl/lite/IonSexpLiteTest.java new file mode 100644 index 0000000000..a8575e4705 --- /dev/null +++ b/test/com/amazon/ion/impl/lite/IonSexpLiteTest.java @@ -0,0 +1,25 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl.lite; + +import com.amazon.ion.IonSequence; + +public class IonSexpLiteTest extends BaseIonSequenceLiteTestCase { + @Override + protected IonSequence newEmptySequence() { + return SYSTEM.newEmptySexp(); + } +} diff --git a/test/software/amazon/ion/impl/lite/SIDPresentLifecycleTest.java b/test/com/amazon/ion/impl/lite/SIDPresentLifecycleTest.java similarity index 81% rename from test/software/amazon/ion/impl/lite/SIDPresentLifecycleTest.java rename to test/com/amazon/ion/impl/lite/SIDPresentLifecycleTest.java index d62e8f89fd..b17cd8a889 100644 --- a/test/software/amazon/ion/impl/lite/SIDPresentLifecycleTest.java +++ b/test/com/amazon/ion/impl/lite/SIDPresentLifecycleTest.java @@ -1,10 +1,25 @@ -package software.amazon.ion.impl.lite; +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.impl.lite; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.IonValue; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.UnknownSymbolException; -import software.amazon.ion.impl.PrivateUtils; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.IonValue; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.UnknownSymbolException; +import com.amazon.ion.impl._Private_Utils; import org.junit.Test; /** @@ -31,7 +46,7 @@ public void testFieldsWithSID() { IonStringLite value = (IonStringLite) system().newString("Test Value"); // a new value should create without SID presence as it has no annotations assertFalse(value._isSymbolIdPresent()); - value.setFieldNameSymbol(PrivateUtils.newSymbolToken("Symbol-With-Sid", 12)); + value.setFieldNameSymbol(_Private_Utils.newSymbolToken("Symbol-With-Sid", 12)); // setting a field with a SID means that SID Present now must be true assertTrue(value._isSymbolIdPresent()); @@ -43,11 +58,11 @@ public void testFieldsWithSID() { assertFalse(value._isSymbolIdPresent()); //verify that detaching still retains symbols - as SID is still present - value.setFieldNameSymbol(PrivateUtils.newSymbolToken(13)); + value.setFieldNameSymbol(_Private_Utils.newSymbolToken(13)); assertTrue(value._isSymbolIdPresent()); value.clearSymbolIDValues(); // NOTE: SID is still present as clearSymbolIDValues doesn't clear SID-only tokens - assertEquals(13, value.getFieldNameSymbol().getSid()); + assertEquals(13, value.getFieldId()); assertTrue(value._isSymbolIdPresent()); } @@ -57,8 +72,8 @@ public void testAnnotationsWithSIDs() { // a new value should create without any SID's as it has no annotations assertFalse(value._isSymbolIdPresent()); value.setTypeAnnotationSymbols( - PrivateUtils.newSymbolToken("Symbol-Without-SID", -1), - PrivateUtils.newSymbolToken("Symbol-With-SID", 99)); + _Private_Utils.newSymbolToken("Symbol-Without-SID", -1), + _Private_Utils.newSymbolToken("Symbol-With-SID", 99)); // setting an annotation with a SID means that SID present now must be true assertTrue(value._isSymbolIdPresent()); @@ -67,7 +82,7 @@ public void testAnnotationsWithSIDs() { // ensure that setting annotations that don't have SIDs doesn't change the SID present status (need to wait for // an explicit action such as detatch from container or clone). - value.setTypeAnnotationSymbols(PrivateUtils.newSymbolToken("Symbol-Without-Sid", -1)); + value.setTypeAnnotationSymbols(_Private_Utils.newSymbolToken("Symbol-Without-Sid", -1)); assertTrue(value._isSymbolIdPresent()); // verify detaching from the container clears the SID Present flag @@ -76,7 +91,7 @@ public void testAnnotationsWithSIDs() { // verify that where SID context preserves through the clone due to SID context not being known (believed a bug) // that the SID present flag is *retained* - value.setTypeAnnotationSymbols(PrivateUtils.newSymbolToken(101)); + value.setTypeAnnotationSymbols(_Private_Utils.newSymbolToken(101)); assertTrue(value.clone()._isSymbolIdPresent()); } @@ -87,21 +102,21 @@ public void testPropagationAcrossGraph() { // 1. test addition of child by field WITHOUT SID causes no propagation IonStringLite keyValue1 = (IonStringLite) system().newString("Foo"); - struct.add(PrivateUtils.newSymbolToken("field_1", -1), keyValue1); + struct.add(_Private_Utils.newSymbolToken("field_1", -1), keyValue1); assertFalse(struct._isSymbolIdPresent()); assertFalse(keyValue1._isSymbolIdPresent()); // 2. test addition of child by field WITH SID but also with field text causes no propagation // (struct only takes field name if present and strips SID) IonStringLite keyValue2 = (IonStringLite) system().newString("Bar"); - struct.add(PrivateUtils.newSymbolToken("field_2", 87), keyValue2); + struct.add(_Private_Utils.newSymbolToken("field_2", 87), keyValue2); assertFalse(struct._isSymbolIdPresent()); assertFalse(keyValue1._isSymbolIdPresent()); assertFalse(keyValue2._isSymbolIdPresent()); // 3. test addition of child by field with SID ONLY causes propagation due to retention of field SID IonStringLite keyValue3 = (IonStringLite) system().newString("Car"); - struct.add(PrivateUtils.newSymbolToken(76), keyValue3); + struct.add(_Private_Utils.newSymbolToken(76), keyValue3); assertTrue(struct._isSymbolIdPresent()); assertFalse(keyValue1._isSymbolIdPresent()); assertFalse(keyValue2._isSymbolIdPresent()); @@ -111,7 +126,7 @@ public void testPropagationAcrossGraph() { IonStructLite struct2 = (IonStructLite) system().newEmptyStruct(); IonStringLite keyValue4 = (IonStringLite) system().newString("But"); struct2.add("field_1", keyValue4); - keyValue4.setTypeAnnotationSymbols(PrivateUtils.newSymbolToken("Lah", 13)); + keyValue4.setTypeAnnotationSymbols(_Private_Utils.newSymbolToken("Lah", 13)); assertTrue(struct2._isSymbolIdPresent()); // 5. test clone propagates clearing of status. NOTE - clone with an unresolvable SID for a field ID fails @@ -133,7 +148,7 @@ public void testPropagationAcrossGraph() { // 6. ensure clearing symbols propogates from root to impacted leaves // add back in field ID (SID) only child before clearing to test SID retained - struct.add(PrivateUtils.newSymbolToken(76), keyValue3); + struct.add(_Private_Utils.newSymbolToken(76), keyValue3); struct.clearSymbolIDValues(); assertTrue(struct._isSymbolIdPresent()); assertFalse(keyValue1._isSymbolIdPresent()); @@ -147,7 +162,7 @@ public void testPropagationAcrossGraph() { middle.add(inner); outer.put("foo", middle); // now add a SID only annotation to inner - inner.setTypeAnnotationSymbols(PrivateUtils.newSymbolToken(99)); + inner.setTypeAnnotationSymbols(_Private_Utils.newSymbolToken(99)); // verify that all components are signaling SID present assertTrue(outer._isSymbolIdPresent()); assertTrue(middle._isSymbolIdPresent()); @@ -163,14 +178,14 @@ public void testPropagationAcrossGraph() { @Test public void testSymbolValueLifecyle() { // test creation from SymbolToken - IonSymbolLite symbolLite = (IonSymbolLite) system().newSymbol(PrivateUtils.newSymbolToken("version", 5)); + IonSymbolLite symbolLite = (IonSymbolLite) system().newSymbol(_Private_Utils.newSymbolToken("version", 5)); // SID isn't present as it is stripped if there is text present assertFalse(symbolLite._isSymbolIdPresent()); IonListLite container1 = (IonListLite) system().newEmptyList(); container1.add(symbolLite); // test propagation of context when symbol context is added - assertEquals(5, symbolLite.symbolValue().getSid()); + assertEquals(5, symbolLite.getSymbolId()); // as the getSymbolId memoizes SID into the entity SID present must be set assertTrue(symbolLite._isSymbolIdPresent()); // check memoization propagates to container @@ -179,7 +194,7 @@ public void testSymbolValueLifecyle() { assertFalse(container1._isSymbolIdPresent()); assertFalse(symbolLite._isSymbolIdPresent()); - IonSymbolLite symbolSIDOnly = (IonSymbolLite) system().newSymbol(PrivateUtils.newSymbolToken(321)); + IonSymbolLite symbolSIDOnly = (IonSymbolLite) system().newSymbol(_Private_Utils.newSymbolToken(321)); // SID Present is preserved when only SID is present assertTrue(symbolSIDOnly._isSymbolIdPresent()); IonListLite container2 = (IonListLite) system().newEmptyList(); @@ -198,9 +213,9 @@ public void testSIDRetentionAcrossObjectHierarchy() { IonStructLite outer = (IonStructLite) system().newEmptyStruct(); IonListLite middle = (IonListLite) system().newEmptyList(); IonStringLite inner = (IonStringLite) system().newString("A"); - outer.add(PrivateUtils.newSymbolToken(321), middle); + outer.add(_Private_Utils.newSymbolToken(321), middle); middle.add(inner); - inner.setTypeAnnotationSymbols(PrivateUtils.newSymbolToken("B", 123)); + inner.setTypeAnnotationSymbols(_Private_Utils.newSymbolToken("B", 123)); assertTrue(outer._isSymbolIdPresent()); assertTrue(middle._isSymbolIdPresent()); assertTrue(inner._isSymbolIdPresent()); @@ -211,12 +226,12 @@ public void testSIDRetentionAcrossObjectHierarchy() { // field ID's can be retained on IonSymbolLite - so ensure these are retained as well. IonStructLite container = (IonStructLite) system().newEmptyStruct(); - IonSymbolLite symbol = (IonSymbolLite) system().newSymbol(PrivateUtils.newSymbolToken("Foo", 123)); - container.add(PrivateUtils.newSymbolToken(321), symbol); + IonSymbolLite symbol = (IonSymbolLite) system().newSymbol(_Private_Utils.newSymbolToken("Foo", 123)); + container.add(_Private_Utils.newSymbolToken(321), symbol); container.clearSymbolIDValues(); - assertEquals(SymbolTable.UNKNOWN_SYMBOL_ID, symbol.symbolValue().getSid()); + assertEquals(SymbolTable.UNKNOWN_SYMBOL_ID, symbol.getSymbolId()); assertEquals("Foo", symbol.stringValue()); - assertEquals(321, symbol.getFieldNameSymbol().getSid()); + assertEquals(321, symbol.getFieldId()); assertTrue(symbol._isSymbolIdPresent()); assertTrue(container._isSymbolIdPresent()); diff --git a/test/software/amazon/ion/junit/Injected.java b/test/com/amazon/ion/junit/Injected.java similarity index 88% rename from test/software/amazon/ion/junit/Injected.java rename to test/com/amazon/ion/junit/Injected.java index 3a4ce91120..1272bcc062 100644 --- a/test/software/amazon/ion/junit/Injected.java +++ b/test/com/amazon/ion/junit/Injected.java @@ -1,18 +1,19 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.junit; +package com.amazon.ion.junit; import java.beans.BeanInfo; import java.beans.Introspector; @@ -24,7 +25,6 @@ import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import org.junit.runner.Runner; import org.junit.runner.notification.RunNotifier; @@ -42,12 +42,6 @@ * the test fixture with a set of configured values. This approach is similar * to {@link Parameterized} but utilizes setter injection instead of using * constructors, so its easier to reuse the injection via inheritance. - *

    - * A test fixture may have no {@link Inject} parameters, in which case the - * fixture runs as it would without the {@link Injected} runner. - * This is useful for allowing a hierarchy of test fixtures to be declared - * as {@link Injected} when only some of the sub-classes actually {@link Inject} - * parameters. */ public class Injected extends Suite @@ -176,12 +170,8 @@ public Injected(Class klass) private static List fanout(Class klass) throws Throwable { Dimension[] dimensions = findDimensions(new TestClass(klass)); + assert dimensions.length != 0; - if (dimensions.length == 0) - { - // no dimensions in this test class--run with normal runner - return Collections.singletonList(new BlockJUnit4ClassRunner(klass)); - } return fanout(klass, new ArrayList(), dimensions, new int[dimensions.length], 0); } @@ -214,7 +204,6 @@ private static List fanout(Class klass, return runners; } - private static final Dimension[] EMPTY_DIMENSION_ARRAY = new Dimension[0]; private static Dimension[] findDimensions(TestClass testClass) throws Throwable @@ -223,7 +212,8 @@ private static Dimension[] findDimensions(TestClass testClass) testClass.getAnnotatedFields(Inject.class); if (fields.isEmpty()) { - return EMPTY_DIMENSION_ARRAY; + throw new Exception("No fields of " + testClass.getName() + + " have the @Inject annotation"); } BeanInfo beanInfo = Introspector.getBeanInfo(testClass.getJavaClass()); diff --git a/test/software/amazon/ion/junit/IonAssert.java b/test/com/amazon/ion/junit/IonAssert.java similarity index 82% rename from test/software/amazon/ion/junit/IonAssert.java rename to test/com/amazon/ion/junit/IonAssert.java index c72b577f03..163d5eddc2 100644 --- a/test/software/amazon/ion/junit/IonAssert.java +++ b/test/com/amazon/ion/junit/IonAssert.java @@ -1,27 +1,31 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.junit; +package com.amazon.ion.junit; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.fail; -import static software.amazon.ion.impl.PrivateIonConstants.UNKNOWN_SYMBOL_TEXT_PREFIX; -import static software.amazon.ion.util.IonTextUtils.printSymbol; +import com.amazon.ion.IonLob; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonSequence; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.ReaderChecker; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.UnknownSymbolException; +import com.amazon.ion.util.Equivalence; import java.util.ArrayList; import java.util.HashMap; @@ -29,17 +33,17 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import software.amazon.ion.IonLob; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonSequence; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.ReaderChecker; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.UnknownSymbolException; -import software.amazon.ion.util.Equivalence; + +import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; +import static com.amazon.ion.impl._Private_IonConstants.UNKNOWN_SYMBOL_TEXT_PREFIX; +import static com.amazon.ion.util.IonTextUtils.printSymbol; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public final class IonAssert { @@ -59,6 +63,7 @@ public static void assertTopLevel(IonReader in, boolean inStruct) if (! inStruct) { assertNull("reader field name", in.getFieldName()); + assertEquals("reader fieldId", UNKNOWN_SYMBOL_ID, in.getFieldId()); assertNull("reader field symbol", in.getFieldNameSymbol()); } @@ -78,9 +83,10 @@ public static void assertNoCurrentValue(IonReader in) assertNull(in.getType()); assertNull(in.getFieldName()); + assertTrue(in.getFieldId() < 0); assertNull(in.getFieldNameSymbol()); - // TODO amzn/ion-java#13 Text reader doesn't throw, but others do. + // TODO amzn/ion-java/issues/13 Text reader doesn't throw, but others do. try { String[] ann = in.getTypeAnnotations(); assertEquals(0, ann.length); @@ -103,11 +109,15 @@ public static void assertNoCurrentValue(IonReader in) } + @SuppressWarnings("deprecation") public static void assertEof(IonReader in) { + assertFalse(in.hasNext()); + assertFalse(in.hasNext()); assertNull(in.next()); assertNoCurrentValue(in); assertNull(in.next()); + assertFalse(in.hasNext()); assertNull(in.next()); } @@ -392,8 +402,7 @@ private static void assertSequenceEquals(String path, int actualSize = actual.size(); if (expectedSize != actualSize) { - fail(path + " length, expected:" + expectedSize - + " actual:" + actualSize); + fail(path + " length, expected:" + expectedSize + " actual:" + actualSize); } for (int i = 0; i < expectedSize; i++) @@ -411,33 +420,22 @@ private static void assertStructEquals(String path, { int expectedSize = expected.size(); int actualSize = actual.size(); - if (expectedSize != actualSize) - { - fail(path + " size, expected:" + expectedSize - + " actual:" + actualSize); + if (expectedSize != actualSize) { + fail(String.format("%s size, expected:%s actual:%s", path, expectedSize, actualSize)); } - Map> expectedFields = sortFields(expected); - Map> actualFields = sortFields(actual); + Map> expectedFields = sortFields(expected); + Map> actualFields = sortFields(actual); - for (Entry> expectedEntry - : expectedFields.entrySet()) - { - String fieldName = expectedEntry.getKey(); - String fieldPath = path + '.' + printSymbol(fieldName); + for (Entry> expectedEntry : expectedFields.entrySet()) { + SymbolToken token = expectedEntry.getKey(); + String fieldPath = path + '.' + printSymbol(token); - List actualList = actualFields.get(fieldName); - if (actualList == null) - { - fail("Missing field " + fieldPath - + ", expected: " + expected - + " actual: " + actual); + List actualList = actualFields.get(token); + if (actualList == null) { + fail(String.format("Missing field %s, expected: %s actual: %s",fieldPath, expected, actual)); } - - assertFieldEquals(fieldPath, - expectedEntry.getValue(), - actual, - actualList); + assertFieldEquals(fieldPath, expectedEntry.getValue(), actual, actualList); } } @@ -445,24 +443,15 @@ private static void assertStructEquals(String path, * Problematic with unknown field names. * See {@link Equivalence} for another use of this idiom. */ - private static HashMap> sortFields(IonStruct s) + private static HashMap> sortFields(IonStruct s) { - HashMap> sorted = - new HashMap>(); - for (IonValue v : s) - { + HashMap> sorted = new HashMap>(); + for (IonValue v : s) { SymbolToken tok = v.getFieldNameSymbol(); - String text = tok.getText(); - if (text == null) { - text = UNKNOWN_SYMBOL_TEXT_PREFIX + tok.getSid(); // TODO amzn/ion-java#23 - } - List fields = sorted.get(text); - if (fields == null) - { - fields = new ArrayList(2); - sorted.put(text, fields); + if(!sorted.containsKey(tok)){ + sorted.put(tok, new ArrayList()); } - fields.add(v); + sorted.get(tok).add(v); } return sorted; } diff --git a/test/software/amazon/ion/profile/IteratorTiming.java b/test/com/amazon/ion/profile/IteratorTiming.java similarity index 77% rename from test/software/amazon/ion/profile/IteratorTiming.java rename to test/com/amazon/ion/profile/IteratorTiming.java index 841ad7f3a6..fd5eb3fe7e 100644 --- a/test/software/amazon/ion/profile/IteratorTiming.java +++ b/test/com/amazon/ion/profile/IteratorTiming.java @@ -1,28 +1,30 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.profile; +package com.amazon.ion.profile; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonValue; +import com.amazon.ion.system.IonSystemBuilder; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.Date; import java.util.Iterator; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonValue; -import software.amazon.ion.system.IonSystemBuilder; + public class IteratorTiming { diff --git a/test/com/amazon/ion/streaming/BadIonStreamingTest.java b/test/com/amazon/ion/streaming/BadIonStreamingTest.java new file mode 100644 index 0000000000..b9dcf008c1 --- /dev/null +++ b/test/com/amazon/ion/streaming/BadIonStreamingTest.java @@ -0,0 +1,71 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.streaming; + +import static com.amazon.ion.TestUtils.BAD_IONTESTS_FILES; +import static com.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; +import static com.amazon.ion.TestUtils.testdataFiles; + +import com.amazon.ion.IonException; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.TestUtils; +import com.amazon.ion.impl._Private_Utils; +import com.amazon.ion.junit.Injected.Inject; +import java.io.File; +import java.io.IOException; +import org.junit.Ignore; +import org.junit.Test; + +public class BadIonStreamingTest +extends IonTestCase +{ + private static final boolean _debug_output_errors = false; + + + @Inject("testFile") + public static final File[] FILES = testdataFiles(GLOBAL_SKIP_LIST, + BAD_IONTESTS_FILES); + + private File myTestFile; + + public void setTestFile(File file) + { + myTestFile = file; + } + + @Test(expected = IonException.class) + public void testReadingScalars() + throws Exception + { + try + { + byte[] buf = _Private_Utils.loadFileBytes(myTestFile); + IonReader it = system().newReader(buf); + TestUtils.deepRead(it, true); + } + catch (IonException e) + { + /* good - we're expecting an error, there are testing bad input */ + if (_debug_output_errors) { + System.out.print(this.myTestFile.getName()); + System.out.print(": "); + System.out.println(e.getMessage()); + } + throw e; + } + } +} diff --git a/test/software/amazon/ion/streaming/BinaryStreamingTest.java b/test/com/amazon/ion/streaming/BinaryStreamingTest.java similarity index 88% rename from test/software/amazon/ion/streaming/BinaryStreamingTest.java rename to test/com/amazon/ion/streaming/BinaryStreamingTest.java index f9fa4f060c..05de4a40e3 100644 --- a/test/software/amazon/ion/streaming/BinaryStreamingTest.java +++ b/test/com/amazon/ion/streaming/BinaryStreamingTest.java @@ -1,22 +1,38 @@ + /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.streaming; - -import static software.amazon.ion.impl.PrivateUtils.newSymbolToken; -import static software.amazon.ion.junit.IonAssert.expectField; -import java.io.ByteArrayOutputStream; +package com.amazon.ion.streaming; + +import static com.amazon.ion.impl._Private_Utils.newSymbolToken; +import static com.amazon.ion.junit.IonAssert.expectField; + +import com.amazon.ion.Decimal; +import com.amazon.ion.IonBinaryWriter; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.Timestamp; +import com.amazon.ion.impl._Private_Utils; +import com.amazon.ion.junit.IonAssert; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; @@ -24,18 +40,7 @@ import java.util.Date; import java.util.Iterator; import org.junit.Test; -import software.amazon.ion.Decimal; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.Timestamp; -import software.amazon.ion.junit.IonAssert; + public class BinaryStreamingTest extends IonTestCase @@ -58,7 +63,7 @@ static class TestValue { value = tv_value; } - /** TODO amzn/ion-java#5 why this context? I think it will round-off values */ + /** TODO amzn/ion-java/issues/5 why this context? I think it will round-off values */ static MathContext context = MathContext.DECIMAL128; static Decimal makeBigDecimal(double d) { Decimal bd = Decimal.valueOf(d, context); @@ -347,8 +352,7 @@ else if (value instanceof Integer) { public void testAllValues() throws Exception { - ByteArrayOutputStream buf = new ByteArrayOutputStream(); - IonWriter wr = system().newBinaryWriter(buf); + IonBinaryWriter wr = system().newBinaryWriter(); byte[] buffer = null; byte[] _testbytes1 = new byte[5]; @@ -519,8 +523,7 @@ public void testAllValues() } wr.stepOut(); - wr.close(); - buffer = buf.toByteArray(); + buffer = wr.getBytes(); } catch (IOException e) { throw new Exception(e); @@ -551,7 +554,9 @@ public void testValue1() +",index_suppressed:true," +"offline_store_only:true,version:2,}"; IonReader ir = system().newReader(s); - byte[] buffer = writeBinaryBytes(ir); + IonBinaryWriter wr = system().newBinaryWriter(); + wr.writeValues(ir); + byte[] buffer = wr.getBytes(); dumpBuffer(buffer, buffer.length); } @@ -577,7 +582,10 @@ public void testValue2() Iterator symbols = sym.iterateDeclaredSymbolNames(); SymbolTable u = system().newSharedSymbolTable("items", 1, symbols); - byte[] buffer = writeBinaryBytes(ir, u); + IonBinaryWriter wr = system().newBinaryWriter(u); + + wr.writeValues(ir); + byte[] buffer = wr.getBytes(); dumpBuffer(buffer, buffer.length); } void dumpBuffer(byte[] buffer, int len) @@ -616,8 +624,7 @@ void dumpBuffer(byte[] buffer, int len) public void testBoolValue() throws Exception { - ByteArrayOutputStream buf = new ByteArrayOutputStream(); - IonWriter wr = system().newBinaryWriter(buf); + IonBinaryWriter wr = system().newBinaryWriter(); byte[] buffer = null; try { @@ -625,18 +632,18 @@ public void testBoolValue() wr.setFieldName("Foo"); wr.writeBool(true); wr.stepOut(); - wr.close(); - buffer = buf.toByteArray(); + buffer = wr.getBytes(); } catch (IOException e) { throw new Exception(e); } IonReader ir = system().newReader(buffer); - if (ir.next() != null) { + if (ir.hasNext()) { + ir.next(); ir.stepIn(); - while (ir.next() != null) { - IonType t = ir.getType(); + while (ir.hasNext()) { + IonType t = ir.next(); String name = ir.getFieldName(); boolean value = ir.booleanValue(); assertTrue( value ); @@ -649,18 +656,20 @@ public void testBoolValue() @Test - public void testTwoMagicCookies() throws Exception { - ByteArrayOutputStream buf = new ByteArrayOutputStream(); - IonWriter wr = system().newBinaryWriter(buf); + public void testTwoMagicCookies() { + IonBinaryWriter wr = system().newBinaryWriter(); byte[] buffer = null; - wr.stepIn(IonType.STRUCT); - wr.setFieldName("Foo"); - wr.addTypeAnnotation("boolean"); - wr.writeBool(true); - wr.stepOut(); - wr.close(); - buffer = buf.toByteArray(); + try { + wr.stepIn(IonType.STRUCT); + wr.setFieldName("Foo"); + wr.addTypeAnnotation("boolean"); + wr.writeBool(true); + wr.stepOut(); + buffer = wr.getBytes(); + } catch (IOException e) { + throw new RuntimeException(e); + } byte[] doublebuffer = new byte[buffer.length * 2]; System.arraycopy(buffer, 0, doublebuffer, 0, buffer.length); @@ -699,24 +708,27 @@ public void testTwoMagicCookies() throws Exception { @Test - public void testBoolean() throws Exception { - ByteArrayOutputStream buf = new ByteArrayOutputStream(); - IonWriter wr = system().newBinaryWriter(buf); + public void testBoolean() { + IonBinaryWriter wr = system().newBinaryWriter(); byte[] buffer = null; - wr.stepIn(IonType.STRUCT); - wr.setFieldName("Foo"); - wr.addTypeAnnotation("boolean"); - wr.writeBool(true); - wr.stepOut(); - wr.close(); - buffer = buf.toByteArray(); + try { + wr.stepIn(IonType.STRUCT); + wr.setFieldName("Foo"); + wr.addTypeAnnotation("boolean"); + wr.writeBool(true); + wr.stepOut(); + buffer = wr.getBytes(); + } catch (IOException e) { + throw new RuntimeException(e); + } IonReader ir = system().newReader(buffer); - if (ir.next() != null) { + if (ir.hasNext()) { + ir.next(); ir.stepIn(); - while (ir.next() != null) { - assertEquals(ir.getType(), IonType.BOOL); + while (ir.hasNext()) { + assertEquals(ir.next(), IonType.BOOL); expectField(ir, "Foo"); //assertEquals(ir.getAnnotations(), new String[] { "boolean" }); String[] annotations = ir.getTypeAnnotations(); @@ -732,43 +744,46 @@ public void testBoolean() throws Exception { //Test Sample map. //{hello=true, Almost Done.=true, This is a test String.=true, 12242.124598129=12242.124598129, Something=null, false=false, true=true, long=9326, 12=-12} @Test - public void testSampleMap() throws Exception { - ByteArrayOutputStream buf = new ByteArrayOutputStream(); - IonWriter wr = system().newBinaryWriter(buf); + public void testSampleMap() { + IonBinaryWriter wr = system().newBinaryWriter(); byte[] buffer = null; - wr.stepIn(IonType.STRUCT); + try { + wr.stepIn(IonType.STRUCT); - wr.setFieldName("hello"); - wr.writeBool(true); + wr.setFieldName("hello"); + wr.writeBool(true); - wr.setFieldName("Almost Done."); - wr.writeBool(true); + wr.setFieldName("Almost Done."); + wr.writeBool(true); - wr.setFieldName("This is a test String."); - wr.writeBool(true); + wr.setFieldName("This is a test String."); + wr.writeBool(true); - wr.setFieldName("12242.124598129"); - wr.writeFloat(12242.124598129); + wr.setFieldName("12242.124598129"); + wr.writeFloat(12242.124598129); - wr.setFieldName("Something"); - wr.writeNull(); + wr.setFieldName("Something"); + wr.writeNull(); - wr.setFieldName("false"); - wr.writeBool(false); + wr.setFieldName("false"); + wr.writeBool(false); - wr.setFieldName("true"); - wr.writeBool(true); + wr.setFieldName("true"); + wr.writeBool(true); - wr.setFieldName("long"); - wr.writeInt((long) 9326); + wr.setFieldName("long"); + wr.writeInt((long) 9326); - wr.setFieldName("12"); - wr.writeInt(-12); + wr.setFieldName("12"); + wr.writeInt(-12); - wr.stepOut(); - wr.close(); - buffer = buf.toByteArray(); + wr.stepOut(); + buffer = wr.getBytes(); + } catch (IOException e) { + + throw new RuntimeException(e); + } IonReader ir = system().newReader(buffer); if (ir.next() != null) { @@ -821,8 +836,7 @@ public void testBenchmarkDirect() throws IOException { byte[] bytes ; - ByteArrayOutputStream buf = new ByteArrayOutputStream(); - IonWriter wr = system().newBinaryWriter(buf); + IonBinaryWriter wr = system().newBinaryWriter(); wr.stepIn(IonType.STRUCT); wr.setFieldName("12"); @@ -855,43 +869,51 @@ public void testBenchmarkDirect() throws IOException { wr.stepOut(); - wr.close(); - bytes = buf.toByteArray(); + bytes = wr.getBytes(); IonReader ir = system().newReader(bytes); assertEquals(IonType.STRUCT, ir.next()); ir.stepIn(); + if (! _Private_Utils.READER_HASNEXT_REMOVED) assertTrue(ir.hasNext()); assertEquals(ir.next(), IonType.INT); expectField(ir, "12"); assertEquals(ir.intValue(), -12); + if (! _Private_Utils.READER_HASNEXT_REMOVED) assertTrue(ir.hasNext()); assertEquals(ir.next(), IonType.FLOAT); expectField(ir, "12242.124598129"); assertEquals(ir.doubleValue(), 12242.124598129); + if (! _Private_Utils.READER_HASNEXT_REMOVED) assertTrue(ir.hasNext()); assertEquals(ir.next(), IonType.BOOL); expectField(ir, "Almost Done."); assertEquals(ir.booleanValue(), true); + if (! _Private_Utils.READER_HASNEXT_REMOVED) assertTrue(ir.hasNext()); assertEquals(ir.next(), IonType.BOOL); expectField(ir, "This is a test String."); assertEquals(ir.booleanValue(), true); + if (! _Private_Utils.READER_HASNEXT_REMOVED) assertTrue(ir.hasNext()); assertEquals(ir.next(), IonType.BOOL); expectField(ir, "false"); assertEquals(ir.booleanValue(), false); + if (! _Private_Utils.READER_HASNEXT_REMOVED) assertTrue(ir.hasNext()); assertEquals(ir.next(), IonType.INT); expectField(ir, "long"); assertEquals(ir.longValue(), 9326L); + if (! _Private_Utils.READER_HASNEXT_REMOVED) assertTrue(ir.hasNext()); assertEquals(ir.next(), IonType.BOOL); expectField(ir, "true"); assertEquals(ir.booleanValue(), true); + if (! _Private_Utils.READER_HASNEXT_REMOVED) assertFalse(ir.hasNext()); assertEquals(null, ir.next()); ir.stepOut(); + if (! _Private_Utils.READER_HASNEXT_REMOVED) assertFalse(ir.hasNext()); assertEquals(null, ir.next()); } } diff --git a/test/software/amazon/ion/streaming/GoodIonStreamingTest.java b/test/com/amazon/ion/streaming/GoodIonStreamingTest.java similarity index 53% rename from test/software/amazon/ion/streaming/GoodIonStreamingTest.java rename to test/com/amazon/ion/streaming/GoodIonStreamingTest.java index aca3b69bdd..04336f132b 100644 --- a/test/software/amazon/ion/streaming/GoodIonStreamingTest.java +++ b/test/com/amazon/ion/streaming/GoodIonStreamingTest.java @@ -1,31 +1,32 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.streaming; +package com.amazon.ion.streaming; -import static software.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; -import static software.amazon.ion.TestUtils.GOOD_IONTESTS_FILES; -import static software.amazon.ion.TestUtils.testdataFiles; +import static com.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; +import static com.amazon.ion.TestUtils.GOOD_IONTESTS_FILES; +import static com.amazon.ion.TestUtils.testdataFiles; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.TestUtils; +import com.amazon.ion.impl._Private_Utils; +import com.amazon.ion.junit.Injected.Inject; import java.io.File; import java.io.IOException; import org.junit.Test; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.TestUtils; -import software.amazon.ion.impl.PrivateUtils; -import software.amazon.ion.junit.Injected.Inject; public class GoodIonStreamingTest extends IonTestCase @@ -55,7 +56,7 @@ public void test() void iterateIon(File myTestFile) throws IOException { - byte[] buf = PrivateUtils.loadFileBytes(myTestFile); + byte[] buf = _Private_Utils.loadFileBytes(myTestFile); IonReader reader = system().newReader(buf); TestUtils.deepRead(reader); diff --git a/test/software/amazon/ion/streaming/InputStreamReaderTest.java b/test/com/amazon/ion/streaming/InputStreamReaderTest.java similarity index 72% rename from test/software/amazon/ion/streaming/InputStreamReaderTest.java rename to test/com/amazon/ion/streaming/InputStreamReaderTest.java index 0ee92470bf..d819d09503 100644 --- a/test/software/amazon/ion/streaming/InputStreamReaderTest.java +++ b/test/com/amazon/ion/streaming/InputStreamReaderTest.java @@ -1,30 +1,32 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.streaming; +package com.amazon.ion.streaming; -import static software.amazon.ion.impl.PrivateUtils.EMPTY_BYTE_ARRAY; -import static software.amazon.ion.impl.PrivateUtils.utf8; +import static com.amazon.ion.impl._Private_Utils.EMPTY_BYTE_ARRAY; +import static com.amazon.ion.impl._Private_Utils.utf8; +import com.amazon.ion.InputStreamWrapper; +import com.amazon.ion.IonType; +import com.amazon.ion.ReaderMaker; +import com.amazon.ion.junit.Injected.Inject; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import org.junit.Test; -import software.amazon.ion.InputStreamWrapper; -import software.amazon.ion.IonType; -import software.amazon.ion.ReaderMaker; -import software.amazon.ion.junit.Injected.Inject; + public class InputStreamReaderTest extends ReaderTestCase diff --git a/test/software/amazon/ion/streaming/MiscStreamingTest.java b/test/com/amazon/ion/streaming/MiscStreamingTest.java similarity index 87% rename from test/software/amazon/ion/streaming/MiscStreamingTest.java rename to test/com/amazon/ion/streaming/MiscStreamingTest.java index 8f88d53b5e..0625fa9817 100644 --- a/test/software/amazon/ion/streaming/MiscStreamingTest.java +++ b/test/com/amazon/ion/streaming/MiscStreamingTest.java @@ -1,38 +1,42 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.streaming; - -import static software.amazon.ion.impl.PrivateUtils.utf8; - +package com.amazon.ion.streaming; + +import static com.amazon.ion.impl._Private_Utils.utf8; + +import com.amazon.ion.BinaryTest; +import com.amazon.ion.IonBinaryWriter; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonString; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSymbol; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.TestUtils; +import com.amazon.ion.impl._Private_Utils; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Iterator; import org.junit.Assert; import org.junit.Test; -import software.amazon.ion.BinaryTest; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonString; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSymbol; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.IonWriter; -import software.amazon.ion.TestUtils; + public class MiscStreamingTest extends IonTestCase @@ -50,6 +54,7 @@ public class MiscStreamingTest static final String _QuotingString2_java = "s\'t2"; @Test + @SuppressWarnings("deprecation") public void testQuoting() throws Exception { @@ -68,7 +73,12 @@ public void testQuoting() // value2 symbol 'str2' (2 bytes: 1 typedesc + 1 sid) IonReader ir = system().newReader(s); - byte[] buffer = writeBinaryBytes(ir); + IonSystem system = system(); + IonBinaryWriter wr = system.newBinaryWriter(); + wr = system.newBinaryWriter(); + wr.writeValues(ir); + + byte[] buffer = wr.getBytes(); assertSame("this buffer length is known to be 22", buffer.length, 22); IonReader sir = system().newReader(s); @@ -245,8 +255,12 @@ private void testNullSymbolValue(IonReader reader) testNullTextValue(reader, IonType.SYMBOL); } + @SuppressWarnings("deprecation") private void testNullTextValue(IonReader reader, IonType textType) { + if (! _Private_Utils.READER_HASNEXT_REMOVED) { + assertTrue(reader.hasNext()); + } assertEquals(textType, reader.next()); assertEquals(textType, reader.getType()); assertTrue(reader.isNullValue()); @@ -276,7 +290,6 @@ public void testSkippingOperator() assertEquals(null, reader.next()); } - @Test public void testReaderDataMangling() throws Exception @@ -291,7 +304,6 @@ public void testReaderDataMangling() Assert.assertArrayEquals("UTF-8 text", utf8(dataText), dataBytes); } - @Test public void testIteratorDataMangling() throws Exception @@ -306,7 +318,6 @@ public void testIteratorDataMangling() Assert.assertArrayEquals("UTF-8 text", utf8(dataText), dataBytes); } - @Test public void testTextWriterSymtabs() throws IOException diff --git a/test/software/amazon/ion/streaming/NonOffsetSpanReaderTest.java b/test/com/amazon/ion/streaming/NonOffsetSpanReaderTest.java similarity index 60% rename from test/software/amazon/ion/streaming/NonOffsetSpanReaderTest.java rename to test/com/amazon/ion/streaming/NonOffsetSpanReaderTest.java index f71f045a05..d5d9dfd0ea 100644 --- a/test/software/amazon/ion/streaming/NonOffsetSpanReaderTest.java +++ b/test/com/amazon/ion/streaming/NonOffsetSpanReaderTest.java @@ -1,24 +1,25 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.streaming; +package com.amazon.ion.streaming; +import com.amazon.ion.OffsetSpan; +import com.amazon.ion.ReaderMaker; +import com.amazon.ion.Span; +import com.amazon.ion.junit.Injected.Inject; import org.junit.Test; -import software.amazon.ion.OffsetSpan; -import software.amazon.ion.ReaderMaker; -import software.amazon.ion.Span; -import software.amazon.ion.junit.Injected.Inject; /** * @see OffsetSpanReaderTest diff --git a/test/software/amazon/ion/streaming/NonSeekableReaderTest.java b/test/com/amazon/ion/streaming/NonSeekableReaderTest.java similarity index 58% rename from test/software/amazon/ion/streaming/NonSeekableReaderTest.java rename to test/com/amazon/ion/streaming/NonSeekableReaderTest.java index 376f198cc6..7c22cc9463 100644 --- a/test/software/amazon/ion/streaming/NonSeekableReaderTest.java +++ b/test/com/amazon/ion/streaming/NonSeekableReaderTest.java @@ -1,23 +1,24 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.streaming; +package com.amazon.ion.streaming; +import com.amazon.ion.ReaderMaker; +import com.amazon.ion.SeekableReader; +import com.amazon.ion.junit.Injected.Inject; import org.junit.Test; -import software.amazon.ion.ReaderMaker; -import software.amazon.ion.SeekableReader; -import software.amazon.ion.junit.Injected.Inject; /** * @see SeekableReaderTest diff --git a/test/software/amazon/ion/streaming/NonSpanReaderTest.java b/test/com/amazon/ion/streaming/NonSpanReaderTest.java similarity index 60% rename from test/software/amazon/ion/streaming/NonSpanReaderTest.java rename to test/com/amazon/ion/streaming/NonSpanReaderTest.java index 8770a33945..61389988cb 100644 --- a/test/software/amazon/ion/streaming/NonSpanReaderTest.java +++ b/test/com/amazon/ion/streaming/NonSpanReaderTest.java @@ -1,23 +1,24 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.streaming; +package com.amazon.ion.streaming; +import com.amazon.ion.ReaderMaker; +import com.amazon.ion.SpanProvider; +import com.amazon.ion.junit.Injected.Inject; import org.junit.Test; -import software.amazon.ion.ReaderMaker; -import software.amazon.ion.SpanProvider; -import software.amazon.ion.junit.Injected.Inject; /** * @see SpanReaderTest diff --git a/test/software/amazon/ion/streaming/NonTextSpanTest.java b/test/com/amazon/ion/streaming/NonTextSpanTest.java similarity index 59% rename from test/software/amazon/ion/streaming/NonTextSpanTest.java rename to test/com/amazon/ion/streaming/NonTextSpanTest.java index 5284c30eb0..d54b8188e7 100644 --- a/test/software/amazon/ion/streaming/NonTextSpanTest.java +++ b/test/com/amazon/ion/streaming/NonTextSpanTest.java @@ -1,24 +1,25 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.streaming; +package com.amazon.ion.streaming; +import com.amazon.ion.ReaderMaker; +import com.amazon.ion.Span; +import com.amazon.ion.TextSpan; +import com.amazon.ion.junit.Injected.Inject; import org.junit.Test; -import software.amazon.ion.ReaderMaker; -import software.amazon.ion.Span; -import software.amazon.ion.TextSpan; -import software.amazon.ion.junit.Injected.Inject; /** * @see TextSpanTest diff --git a/test/software/amazon/ion/streaming/OffsetSpanBinaryReaderTest.java b/test/com/amazon/ion/streaming/OffsetSpanBinaryReaderTest.java similarity index 76% rename from test/software/amazon/ion/streaming/OffsetSpanBinaryReaderTest.java rename to test/com/amazon/ion/streaming/OffsetSpanBinaryReaderTest.java index 8dc949c558..665eb853d2 100644 --- a/test/software/amazon/ion/streaming/OffsetSpanBinaryReaderTest.java +++ b/test/com/amazon/ion/streaming/OffsetSpanBinaryReaderTest.java @@ -1,27 +1,29 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.streaming; +package com.amazon.ion.streaming; -import static software.amazon.ion.BinaryTest.hexToBytes; +import static com.amazon.ion.BinaryTest.hexToBytes; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonType; +import com.amazon.ion.ReaderMaker; +import com.amazon.ion.impl._Private_Utils; +import com.amazon.ion.util.RepeatInputStream; import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonType; -import software.amazon.ion.ReaderMaker; -import software.amazon.ion.impl.PrivateUtils; -import software.amazon.ion.util.RepeatInputStream; + public class OffsetSpanBinaryReaderTest extends ReaderFacetTestCase @@ -85,7 +87,7 @@ public void testCurrentSpanBeyondMaxIntForOrderedStruct() + "84 " // name: + "AE 08 80" // blob, len=1024 ); - data = PrivateUtils.copyOf(data, data.length + 1024); + data = _Private_Utils.copyOf(data, data.length + 1024); readBeyondMaxInt(data, IonType.STRUCT); } diff --git a/test/software/amazon/ion/streaming/OffsetSpanReaderTest.java b/test/com/amazon/ion/streaming/OffsetSpanReaderTest.java similarity index 75% rename from test/software/amazon/ion/streaming/OffsetSpanReaderTest.java rename to test/com/amazon/ion/streaming/OffsetSpanReaderTest.java index b892b6551c..9c62e0312f 100644 --- a/test/software/amazon/ion/streaming/OffsetSpanReaderTest.java +++ b/test/com/amazon/ion/streaming/OffsetSpanReaderTest.java @@ -1,26 +1,28 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.streaming; +package com.amazon.ion.streaming; +import com.amazon.ion.BinaryTest; +import com.amazon.ion.IonType; +import com.amazon.ion.ReaderMaker; +import com.amazon.ion.junit.Injected.Inject; import java.io.ByteArrayOutputStream; import java.io.IOException; import org.junit.Test; -import software.amazon.ion.BinaryTest; -import software.amazon.ion.IonType; -import software.amazon.ion.ReaderMaker; -import software.amazon.ion.junit.Injected.Inject; + public class OffsetSpanReaderTest extends ReaderFacetTestCase @@ -78,9 +80,14 @@ public void testCurrentSpanFromStreamMed() throws IOException { buf.write(BinaryTest.hexToBytes("22 03 E8")); } + // Textification is wonky and inconsistent here, it looks like: + // LITE: $ion_1_0 1000 1000 1000 1000 ... + // BACKED: $ion_1_0 1000 $ion_1_0 1000 ... + read(buf.toByteArray()); int binaryStart = 4; int textStart = 9; + for (int i = 0; i < count; i++) { assertSame(IonType.INT, in.next()); checkCurrentSpan(binaryStart, binaryStart+3, textStart); diff --git a/test/software/amazon/ion/streaming/ReaderCompare.java b/test/com/amazon/ion/streaming/ReaderCompare.java similarity index 82% rename from test/software/amazon/ion/streaming/ReaderCompare.java rename to test/com/amazon/ion/streaming/ReaderCompare.java index 6d180cfb17..2a3c68dd94 100644 --- a/test/software/amazon/ion/streaming/ReaderCompare.java +++ b/test/com/amazon/ion/streaming/ReaderCompare.java @@ -1,35 +1,39 @@ /* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.streaming; +package com.amazon.ion.streaming; +import static com.amazon.ion.impl._Private_Utils.READER_HASNEXT_REMOVED; + +import com.amazon.ion.IonReader; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.IonType; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.Timestamp; +import com.amazon.ion.junit.IonAssert; import java.math.BigDecimal; import org.junit.Assert; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.IonType; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.Timestamp; -import software.amazon.ion.junit.IonAssert; + public class ReaderCompare extends Assert { public static void compare(IonReader it1, IonReader it2) { while (hasNext(it1, it2)) { - IonType t1 = it1.getType(); - IonType t2 = it2.getType(); + IonType t1 = (READER_HASNEXT_REMOVED ? it1.getType() : it1.next()); + IonType t2 = (READER_HASNEXT_REMOVED ? it2.getType() : it2.next()); if ((t1 != t2) && (t1 == null || t2 == null || !t1.equals(t2))) { assertEquals("ion type", t1, t2); @@ -81,8 +85,20 @@ public static void compare(IonReader it1, IonReader it2) { public static boolean hasNext(IonReader it1, IonReader it2) { boolean more; - more = (it1.next() != null); - assertEquals("next results don't match", more, it2.next() != null); + if (READER_HASNEXT_REMOVED) + { + more = (it1.next() != null); + assertEquals("next results don't match", more, it2.next() != null); + } + else + { + more = it1.hasNext(); + assertEquals("hasNext results don't match", more, it2.hasNext()); + + // Check that result doesn't change + assertEquals(more, it1.hasNext()); + assertEquals(more, it2.hasNext()); + } if (!more) { assertEquals(null, it1.next()); diff --git a/test/software/amazon/ion/streaming/ReaderDomCopyTest.java b/test/com/amazon/ion/streaming/ReaderDomCopyTest.java similarity index 55% rename from test/software/amazon/ion/streaming/ReaderDomCopyTest.java rename to test/com/amazon/ion/streaming/ReaderDomCopyTest.java index e07018b21d..4afb3ec7a4 100644 --- a/test/software/amazon/ion/streaming/ReaderDomCopyTest.java +++ b/test/com/amazon/ion/streaming/ReaderDomCopyTest.java @@ -1,32 +1,33 @@ /* - * Copyright 2013-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.streaming; +package com.amazon.ion.streaming; -import static software.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; -import static software.amazon.ion.TestUtils.GOOD_IONTESTS_FILES; -import static software.amazon.ion.TestUtils.testdataFiles; -import static software.amazon.ion.junit.IonAssert.assertIonEquals; +import static com.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; +import static com.amazon.ion.TestUtils.GOOD_IONTESTS_FILES; +import static com.amazon.ion.TestUtils.testdataFiles; +import static com.amazon.ion.junit.IonAssert.assertIonEquals; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.IonValue; +import com.amazon.ion.junit.Injected.Inject; import java.io.File; import java.io.IOException; import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.IonValue; -import software.amazon.ion.junit.Injected.Inject; public class ReaderDomCopyTest extends IonTestCase diff --git a/test/software/amazon/ion/streaming/ReaderFacetTestCase.java b/test/com/amazon/ion/streaming/ReaderFacetTestCase.java similarity index 82% rename from test/software/amazon/ion/streaming/ReaderFacetTestCase.java rename to test/com/amazon/ion/streaming/ReaderFacetTestCase.java index 2715d89e46..bfbbeb0d78 100644 --- a/test/software/amazon/ion/streaming/ReaderFacetTestCase.java +++ b/test/com/amazon/ion/streaming/ReaderFacetTestCase.java @@ -1,30 +1,32 @@ /* - * Copyright 2013-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.streaming; +package com.amazon.ion.streaming; -import static software.amazon.ion.facet.Facets.assumeFacet; -import static software.amazon.ion.util.Spans.currentSpan; +import static com.amazon.ion.facet.Facets.assumeFacet; +import static com.amazon.ion.util.Spans.currentSpan; +import com.amazon.ion.OffsetSpan; +import com.amazon.ion.ReaderMaker; +import com.amazon.ion.SeekableReader; +import com.amazon.ion.Span; +import com.amazon.ion.SpanProvider; +import com.amazon.ion.TextSpan; +import com.amazon.ion.facet.Facets; import org.junit.After; -import software.amazon.ion.OffsetSpan; -import software.amazon.ion.ReaderMaker; -import software.amazon.ion.SeekableReader; -import software.amazon.ion.Span; -import software.amazon.ion.SpanProvider; -import software.amazon.ion.TextSpan; -import software.amazon.ion.facet.Facets; + public abstract class ReaderFacetTestCase extends ReaderTestCase @@ -55,7 +57,7 @@ public abstract class ReaderFacetTestCase * * @see NonSeekableReaderTest */ - public static final ReaderMaker[] NON_SEEKABLE_READERS = // TODO amzn/ion-java#17 + public static final ReaderMaker[] NON_SEEKABLE_READERS = // TODO amzn/ion-java/issues/17 { ReaderMaker.FROM_INPUT_STREAM_BINARY, ReaderMaker.FROM_INPUT_STREAM_TEXT, diff --git a/test/software/amazon/ion/streaming/ReaderIntegerSizeTest.java b/test/com/amazon/ion/streaming/ReaderIntegerSizeTest.java similarity index 85% rename from test/software/amazon/ion/streaming/ReaderIntegerSizeTest.java rename to test/com/amazon/ion/streaming/ReaderIntegerSizeTest.java index 07ade48477..983d2b23e5 100644 --- a/test/software/amazon/ion/streaming/ReaderIntegerSizeTest.java +++ b/test/com/amazon/ion/streaming/ReaderIntegerSizeTest.java @@ -1,12 +1,25 @@ -// Copyright (c) 2016 Amazon.com, Inc. All rights reserved. - -package software.amazon.ion.streaming; - +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.streaming; + +import com.amazon.ion.IntegerSize; +import com.amazon.ion.ReaderMaker; +import com.amazon.ion.junit.Injected.Inject; import java.math.BigInteger; import org.junit.Test; -import software.amazon.ion.IntegerSize; -import software.amazon.ion.ReaderMaker; -import software.amazon.ion.junit.Injected.Inject; public class ReaderIntegerSizeTest extends ReaderTestCase diff --git a/test/software/amazon/ion/streaming/ReaderSkippingTest.java b/test/com/amazon/ion/streaming/ReaderSkippingTest.java similarity index 84% rename from test/software/amazon/ion/streaming/ReaderSkippingTest.java rename to test/com/amazon/ion/streaming/ReaderSkippingTest.java index 308c24d287..6a45032cbc 100644 --- a/test/software/amazon/ion/streaming/ReaderSkippingTest.java +++ b/test/com/amazon/ion/streaming/ReaderSkippingTest.java @@ -1,24 +1,32 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.streaming; +package com.amazon.ion.streaming; -import static software.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; -import static software.amazon.ion.TestUtils.GOOD_IONTESTS_FILES; -import static software.amazon.ion.TestUtils.consumeCurrentValue; -import static software.amazon.ion.TestUtils.testdataFiles; +import static com.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; +import static com.amazon.ion.TestUtils.GOOD_IONTESTS_FILES; +import static com.amazon.ion.TestUtils.consumeCurrentValue; +import static com.amazon.ion.TestUtils.testdataFiles; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.junit.Injected.Inject; +import com.amazon.ion.junit.IonAssert; import java.io.File; import java.io.FileInputStream; import java.util.Random; @@ -26,13 +34,6 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.IonType; -import software.amazon.ion.IonValue; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.junit.IonAssert; -import software.amazon.ion.junit.Injected.Inject; public class ReaderSkippingTest extends IonTestCase diff --git a/test/software/amazon/ion/streaming/ReaderTest.java b/test/com/amazon/ion/streaming/ReaderTest.java similarity index 91% rename from test/software/amazon/ion/streaming/ReaderTest.java rename to test/com/amazon/ion/streaming/ReaderTest.java index 72e4a83fa0..50ba1ab6d5 100644 --- a/test/software/amazon/ion/streaming/ReaderTest.java +++ b/test/com/amazon/ion/streaming/ReaderTest.java @@ -1,36 +1,38 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.streaming; +package com.amazon.ion.streaming; -import static software.amazon.ion.impl.Symtabs.printLocalSymtab; -import static software.amazon.ion.junit.IonAssert.checkNullSymbol; +import static com.amazon.ion.impl.Symtabs.printLocalSymtab; +import static com.amazon.ion.junit.IonAssert.checkNullSymbol; +import com.amazon.ion.BinaryTest; +import com.amazon.ion.Decimal; +import com.amazon.ion.IonType; +import com.amazon.ion.ReaderMaker; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.junit.Injected.Inject; +import com.amazon.ion.junit.IonAssert; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.util.Arrays; import org.junit.Ignore; import org.junit.Test; -import software.amazon.ion.BinaryTest; -import software.amazon.ion.Decimal; -import software.amazon.ion.IonType; -import software.amazon.ion.ReaderMaker; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.SymbolToken; -import software.amazon.ion.junit.Injected.Inject; -import software.amazon.ion.junit.IonAssert; + public class ReaderTest extends ReaderTestCase @@ -237,7 +239,7 @@ public void testReadingFloatAsBigInteger() * it's not what one would expect. */ @Test @Ignore - public void testReadingFloatAsBigDecimal() // TODO amzn/ion-java#56 + public void testReadingFloatAsBigDecimal() // TODO amzn/ion-java/issues/56 { // Note that one can't represent Long.MAX_VALUE as a double, there's // not enough bits in the mantissa! diff --git a/test/software/amazon/ion/streaming/ReaderTestCase.java b/test/com/amazon/ion/streaming/ReaderTestCase.java similarity index 74% rename from test/software/amazon/ion/streaming/ReaderTestCase.java rename to test/com/amazon/ion/streaming/ReaderTestCase.java index 54034f9a1d..c39208d8b0 100644 --- a/test/software/amazon/ion/streaming/ReaderTestCase.java +++ b/test/com/amazon/ion/streaming/ReaderTestCase.java @@ -1,28 +1,30 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.streaming; +package com.amazon.ion.streaming; +import com.amazon.ion.InputStreamWrapper; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.IonType; +import com.amazon.ion.ReaderChecker; +import com.amazon.ion.ReaderMaker; +import com.amazon.ion.junit.IonAssert; import java.io.IOException; import org.junit.After; -import software.amazon.ion.InputStreamWrapper; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.IonType; -import software.amazon.ion.ReaderChecker; -import software.amazon.ion.ReaderMaker; -import software.amazon.ion.junit.IonAssert; + public abstract class ReaderTestCase diff --git a/test/software/amazon/ion/streaming/RoundTripStreamingTest.java b/test/com/amazon/ion/streaming/RoundTripStreamingTest.java similarity index 88% rename from test/software/amazon/ion/streaming/RoundTripStreamingTest.java rename to test/com/amazon/ion/streaming/RoundTripStreamingTest.java index 490c5fba4a..f1eb07e002 100644 --- a/test/software/amazon/ion/streaming/RoundTripStreamingTest.java +++ b/test/com/amazon/ion/streaming/RoundTripStreamingTest.java @@ -1,25 +1,40 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.streaming; - -import static software.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; -import static software.amazon.ion.TestUtils.GOOD_IONTESTS_FILES; -import static software.amazon.ion.TestUtils.testdataFiles; -import static software.amazon.ion.impl.PrivateUtils.utf8; -import static software.amazon.ion.system.IonWriterBuilder.InitialIvmHandling.SUPPRESS; - +package com.amazon.ion.streaming; + +import static com.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; +import static com.amazon.ion.TestUtils.GOOD_IONTESTS_FILES; +import static com.amazon.ion.TestUtils.testdataFiles; +import static com.amazon.ion.impl._Private_Utils.utf8; +import static com.amazon.ion.system.IonWriterBuilder.InitialIvmHandling.SUPPRESS; + +import com.amazon.ion.IonBinaryWriter; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonLoader; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.RoundTripTest; +import com.amazon.ion.impl._Private_Utils; +import com.amazon.ion.junit.Injected.Inject; +import com.amazon.ion.junit.IonAssert; +import com.amazon.ion.system.IonTextWriterBuilder; +import com.amazon.ion.util.Equivalence; +import com.amazon.ion.util.Printer; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -27,21 +42,9 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonLoader; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.IonValue; -import software.amazon.ion.IonWriter; -import software.amazon.ion.RoundTripTest; -import software.amazon.ion.impl.PrivateUtils; -import software.amazon.ion.junit.IonAssert; -import software.amazon.ion.junit.Injected.Inject; -import software.amazon.ion.system.IonTextWriterBuilder; -import software.amazon.ion.util.Equivalence; /** - * TODO amzn/ion-java#29 Refactor this test class, possible duplicate test coverage in + * TODO amzn/ion-java/issues/29 Refactor this test class, possible duplicate test coverage in * {@link RoundTripTest}. */ public class RoundTripStreamingTest @@ -51,13 +54,13 @@ public class RoundTripStreamingTest static final boolean _debug_dump_datagrams = false; @Inject("testFile") - public static final File[] FILES = - testdataFiles(GLOBAL_SKIP_LIST, GOOD_IONTESTS_FILES); + public static final File[] FILES = testdataFiles(GLOBAL_SKIP_LIST, GOOD_IONTESTS_FILES); @Inject("copySpeed") public static final StreamCopySpeed[] STREAM_COPY_SPEEDS = StreamCopySpeed.values(); + private Printer myPrinter; private StringBuilder myBuilder; private byte[] myBuffer; private File myTestFile; @@ -73,8 +76,9 @@ public void setUp() throws Exception { super.setUp(); + myPrinter = new Printer(); myBuilder = new StringBuilder(); - myBuffer = PrivateUtils.loadFileBytes(myTestFile); + myBuffer = _Private_Utils.loadFileBytes(myTestFile); } @Override @@ -82,6 +86,7 @@ public void setUp() public void tearDown() throws Exception { + myPrinter = null; myBuilder = null; myBuffer = null; super.tearDown(); @@ -101,13 +106,7 @@ private String makeString(IonDatagram datagram) else { myBuilder.append(' '); } - IonWriter writer = IonTextWriterBuilder.standard().build(myBuilder); - try { - value.writeTo(writer); - } - finally { - writer.close(); - } + myPrinter.print(value, myBuilder); // Why is this here? myBuilder.append('\n'); } @@ -153,7 +152,10 @@ private byte[] makeBinary(byte[] buffer) throws IOException { IonReader in = makeIterator(buffer); - byte[] buf = writeBinaryBytes(in); + IonBinaryWriter bw = system().newBinaryWriter(); + + bw.writeValues(in); + byte[] buf = bw.getBytes(); // this is binary return buf; } diff --git a/test/software/amazon/ion/streaming/SeekableReaderTest.java b/test/com/amazon/ion/streaming/SeekableReaderTest.java similarity index 88% rename from test/software/amazon/ion/streaming/SeekableReaderTest.java rename to test/com/amazon/ion/streaming/SeekableReaderTest.java index 1c73e42b8d..cc6ce61cda 100644 --- a/test/software/amazon/ion/streaming/SeekableReaderTest.java +++ b/test/com/amazon/ion/streaming/SeekableReaderTest.java @@ -1,31 +1,32 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.streaming; +package com.amazon.ion.streaming; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonType; +import com.amazon.ion.ReaderMaker; +import com.amazon.ion.Span; +import com.amazon.ion.TestUtils; +import com.amazon.ion.impl._Private_Utils; +import com.amazon.ion.junit.Injected.Inject; +import com.amazon.ion.junit.IonAssert; import java.io.File; import java.io.IOException; import org.junit.Assert; import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonType; -import software.amazon.ion.ReaderMaker; -import software.amazon.ion.Span; -import software.amazon.ion.TestUtils; -import software.amazon.ion.impl.PrivateUtils; -import software.amazon.ion.junit.IonAssert; -import software.amazon.ion.junit.Injected.Inject; /** * @see NonSeekableReaderTest @@ -205,7 +206,7 @@ public void testHoistingOrderedStruct() throws IOException { File file = getTestdataFile("good/structOrdered.10n"); - byte[] binary = PrivateUtils.loadFileBytes(file); + byte[] binary = _Private_Utils.loadFileBytes(file); read(binary); diff --git a/test/software/amazon/ion/streaming/SpanReaderTest.java b/test/com/amazon/ion/streaming/SpanReaderTest.java similarity index 86% rename from test/software/amazon/ion/streaming/SpanReaderTest.java rename to test/com/amazon/ion/streaming/SpanReaderTest.java index 358b957bc5..646906caca 100644 --- a/test/software/amazon/ion/streaming/SpanReaderTest.java +++ b/test/com/amazon/ion/streaming/SpanReaderTest.java @@ -1,26 +1,27 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.streaming; +package com.amazon.ion.streaming; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonType; +import com.amazon.ion.ReaderMaker; +import com.amazon.ion.Span; +import com.amazon.ion.TestUtils; +import com.amazon.ion.junit.Injected.Inject; import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonType; -import software.amazon.ion.ReaderMaker; -import software.amazon.ion.Span; -import software.amazon.ion.TestUtils; -import software.amazon.ion.junit.Injected.Inject; /** * @see NonSpanReaderTest diff --git a/test/software/amazon/ion/streaming/SpanTests.java b/test/com/amazon/ion/streaming/SpanTests.java similarity index 60% rename from test/software/amazon/ion/streaming/SpanTests.java rename to test/com/amazon/ion/streaming/SpanTests.java index 73f6f57005..54d23fcdde 100644 --- a/test/software/amazon/ion/streaming/SpanTests.java +++ b/test/com/amazon/ion/streaming/SpanTests.java @@ -1,22 +1,24 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.streaming; +package com.amazon.ion.streaming; +import com.amazon.ion.util.SpansTest; import org.junit.runner.RunWith; import org.junit.runners.Suite; -import software.amazon.ion.util.SpansTest; + @RunWith(Suite.class) @Suite.SuiteClasses({ diff --git a/test/software/amazon/ion/streaming/TextSpanTest.java b/test/com/amazon/ion/streaming/TextSpanTest.java similarity index 60% rename from test/software/amazon/ion/streaming/TextSpanTest.java rename to test/com/amazon/ion/streaming/TextSpanTest.java index 4040f4b5b7..cd8c5a340e 100644 --- a/test/software/amazon/ion/streaming/TextSpanTest.java +++ b/test/com/amazon/ion/streaming/TextSpanTest.java @@ -1,28 +1,29 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.streaming; +package com.amazon.ion.streaming; -import static software.amazon.ion.ReaderMaker.valuesExcluding; -import static software.amazon.ion.util.Spans.currentSpan; +import static com.amazon.ion.ReaderMaker.valuesExcluding; +import static com.amazon.ion.util.Spans.currentSpan; +import com.amazon.ion.ReaderMaker; +import com.amazon.ion.Span; +import com.amazon.ion.TextSpan; +import com.amazon.ion.facet.Facets; +import com.amazon.ion.junit.Injected.Inject; import org.junit.Test; -import software.amazon.ion.ReaderMaker; -import software.amazon.ion.Span; -import software.amazon.ion.TextSpan; -import software.amazon.ion.facet.Facets; -import software.amazon.ion.junit.Injected.Inject; /** * @see NonTextSpanTest diff --git a/test/software/amazon/ion/system/IonBinaryWriterBuilderTest.java b/test/com/amazon/ion/system/IonBinaryWriterBuilderTest.java similarity index 86% rename from test/software/amazon/ion/system/IonBinaryWriterBuilderTest.java rename to test/com/amazon/ion/system/IonBinaryWriterBuilderTest.java index bee7917c68..410ac4509e 100644 --- a/test/software/amazon/ion/system/IonBinaryWriterBuilderTest.java +++ b/test/com/amazon/ion/system/IonBinaryWriterBuilderTest.java @@ -1,40 +1,42 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.system; +package com.amazon.ion.system; +import static com.amazon.ion.TestUtils.symbolTableEquals; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static software.amazon.ion.TestUtils.symbolTableEquals; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.impl.Symtabs; +import com.amazon.ion.impl._Private_IonBinaryWriterBuilder; +import com.amazon.ion.impl._Private_IonWriter; +import com.amazon.ion.impl._Private_Utils; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import org.junit.Assert; import org.junit.Test; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.impl.PrivateIonBinaryWriterBuilder; -import software.amazon.ion.impl.PrivateIonWriter; -import software.amazon.ion.impl.PrivateUtils; -import software.amazon.ion.impl.Symtabs; + public class IonBinaryWriterBuilderTest { @@ -76,7 +78,7 @@ public void testCustomCatalog() OutputStream out = new ByteArrayOutputStream(); IonWriter writer = b.build(out); - assertSame(catalog, ((PrivateIonWriter)writer).getCatalog()); + assertSame(catalog, ((_Private_IonWriter)writer).getCatalog()); IonCatalog catalog2 = new SimpleCatalog(); b.setCatalog(catalog2); @@ -123,7 +125,7 @@ public void testStreamCopyOptimized() OutputStream out = new ByteArrayOutputStream(); IonWriter w = b.build(out); - assertTrue(((PrivateIonWriter)w).isStreamCopyOptimized()); + assertTrue(((_Private_IonWriter)w).isStreamCopyOptimized()); } @@ -206,8 +208,8 @@ public void testSymtabValueFactory() { IonSystem system = IonSystemBuilder.standard().build(); - PrivateIonBinaryWriterBuilder b = - PrivateIonBinaryWriterBuilder.standard(); + _Private_IonBinaryWriterBuilder b = + _Private_IonBinaryWriterBuilder.standard(); b.setSymtabValueFactory(system); assertSame(system, b.getSymtabValueFactory()); @@ -219,11 +221,11 @@ public void testSymtabValueFactory() @Test(expected = UnsupportedOperationException.class) public void testSymtabValueFactoryImmutability() { - PrivateIonBinaryWriterBuilder b = - PrivateIonBinaryWriterBuilder.standard(); + _Private_IonBinaryWriterBuilder b = + _Private_IonBinaryWriterBuilder.standard(); b.setSymtabValueFactory(IonSystemBuilder.standard().build()); - PrivateIonBinaryWriterBuilder b2 = b.immutable(); + _Private_IonBinaryWriterBuilder b2 = b.immutable(); b2.setSymtabValueFactory(null); } @@ -235,13 +237,13 @@ public void testSymtabValueFactoryImmutability() public void testInitialSymtab() throws IOException { - SymbolTable sst = PrivateUtils.systemSymtab(1); + SymbolTable sst = _Private_Utils.systemSymtab(1); SymbolTable lst0 = Symtabs.localSymbolTableFactory().newLocalSymtab(sst); lst0.intern("hello"); - PrivateIonBinaryWriterBuilder b = - PrivateIonBinaryWriterBuilder.standard(); + _Private_IonBinaryWriterBuilder b = + _Private_IonBinaryWriterBuilder.standard(); b.setInitialSymbolTable(lst0); assertSame(lst0, b.getInitialSymbolTable()); @@ -277,15 +279,15 @@ public void testInitialSymtab() @Test public void testImmutableInitialSymtab() { - SymbolTable sst = PrivateUtils.systemSymtab(1); + SymbolTable sst = _Private_Utils.systemSymtab(1); // Immutable local symtabs shouldn't get copied. SymbolTable lst = Symtabs.localSymbolTableFactory().newLocalSymtab(sst); lst.intern("hello"); lst.makeReadOnly(); - PrivateIonBinaryWriterBuilder b = - PrivateIonBinaryWriterBuilder.standard(); + _Private_IonBinaryWriterBuilder b = + _Private_IonBinaryWriterBuilder.standard(); b.setInitialSymbolTable(lst); assertSame(lst, b.getInitialSymbolTable()); @@ -300,11 +302,11 @@ public void testImmutableInitialSymtab() @Test(expected = UnsupportedOperationException.class) public void testInitialSymtabImmutability() { - PrivateIonBinaryWriterBuilder b = - PrivateIonBinaryWriterBuilder.standard(); + _Private_IonBinaryWriterBuilder b = + _Private_IonBinaryWriterBuilder.standard(); b.setInitialSymbolTable(null); - PrivateIonBinaryWriterBuilder b2 = b.immutable(); + _Private_IonBinaryWriterBuilder b2 = b.immutable(); b2.setInitialSymbolTable(null); } diff --git a/test/software/amazon/ion/system/IonReaderBuilderTest.java b/test/com/amazon/ion/system/IonReaderBuilderTest.java similarity index 83% rename from test/software/amazon/ion/system/IonReaderBuilderTest.java rename to test/com/amazon/ion/system/IonReaderBuilderTest.java index 06b938396e..6b15bb458b 100644 --- a/test/software/amazon/ion/system/IonReaderBuilderTest.java +++ b/test/com/amazon/ion/system/IonReaderBuilderTest.java @@ -1,33 +1,34 @@ /* - * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.system; +package com.amazon.ion.system; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertSame; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.impl._Private_IonBinaryWriterBuilder; import java.io.ByteArrayOutputStream; import java.io.IOException; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonType; -import software.amazon.ion.IonWriter; -import software.amazon.ion.impl.PrivateIonBinaryWriterBuilder; /** * Note: because the IonReaderBuilder is used by IonSystem.newReader(...), @@ -102,7 +103,7 @@ public void testSystemFreeRoundtrip() throws IOException { // No IonSystem in sight. ByteArrayOutputStream out = new ByteArrayOutputStream(); - IonWriter writer = PrivateIonBinaryWriterBuilder.standard().build(out); + IonWriter writer = _Private_IonBinaryWriterBuilder.standard().build(out); writer.writeInt(42); writer.finish(); IonReader reader = IonReaderBuilder.standard().build(out.toByteArray()); diff --git a/test/software/amazon/ion/system/IonSystemBuilderTest.java b/test/com/amazon/ion/system/IonSystemBuilderTest.java similarity index 80% rename from test/software/amazon/ion/system/IonSystemBuilderTest.java rename to test/com/amazon/ion/system/IonSystemBuilderTest.java index 32914809b9..ad3f40b0fb 100644 --- a/test/software/amazon/ion/system/IonSystemBuilderTest.java +++ b/test/com/amazon/ion/system/IonSystemBuilderTest.java @@ -1,33 +1,34 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.system; +package com.amazon.ion.system; +import static com.amazon.ion.impl.lite._Private_LiteDomTrampoline.isLiteSystem; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; -import static software.amazon.ion.impl.lite.PrivateLiteDomTrampoline.isLiteSystem; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonWriter; +import com.amazon.ion.impl._Private_IonWriter; import java.io.ByteArrayOutputStream; import org.junit.Test; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonWriter; -import software.amazon.ion.impl.PrivateIonWriter; -import software.amazon.ion.system.IonSystemBuilder; -import software.amazon.ion.system.SimpleCatalog; + public class IonSystemBuilderTest { @@ -106,14 +107,6 @@ public void testWithCatalog() assertSame(catalog, b.getCatalog()); } - @Test - public void testLite() - { - IonSystemBuilder b = IonSystemBuilder.standard().copy(); - IonSystem ion = b.build(); - assertTrue(isLiteSystem(ion)); - } - //------------------------------------------------------------------------- @Test @@ -125,7 +118,7 @@ public void testStreamCopyOptimized() assertTrue(isLiteSystem(ion)); ByteArrayOutputStream out = new ByteArrayOutputStream(); IonWriter w = ion.newBinaryWriter(out); - assertTrue(((PrivateIonWriter)w).isStreamCopyOptimized()); + assertTrue(((_Private_IonWriter)w).isStreamCopyOptimized()); } diff --git a/test/software/amazon/ion/system/IonTextWriterBuilderTest.java b/test/com/amazon/ion/system/IonTextWriterBuilderTest.java similarity index 90% rename from test/software/amazon/ion/system/IonTextWriterBuilderTest.java rename to test/com/amazon/ion/system/IonTextWriterBuilderTest.java index 0e10244fe1..5c819debdf 100644 --- a/test/software/amazon/ion/system/IonTextWriterBuilderTest.java +++ b/test/com/amazon/ion/system/IonTextWriterBuilderTest.java @@ -1,45 +1,45 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.system; +package com.amazon.ion.system; +import static com.amazon.ion.SystemSymbols.ION_1_0; +import static com.amazon.ion.system.IonTextWriterBuilder.ASCII; +import static com.amazon.ion.system.IonTextWriterBuilder.UTF8; +import static com.amazon.ion.system.IonTextWriterBuilder.LstMinimizing.EVERYTHING; +import static com.amazon.ion.system.IonTextWriterBuilder.LstMinimizing.LOCALS; +import static com.amazon.ion.system.IonWriterBuilder.InitialIvmHandling.SUPPRESS; +import static com.amazon.ion.system.IonWriterBuilder.IvmMinimizing.ADJACENT; +import static com.amazon.ion.system.IonWriterBuilder.IvmMinimizing.DISTANT; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertSame; import static org.junit.Assert.fail; -import static software.amazon.ion.SystemSymbols.ION_1_0; -import static software.amazon.ion.system.IonTextWriterBuilder.ASCII; -import static software.amazon.ion.system.IonTextWriterBuilder.UTF8; -import static software.amazon.ion.system.IonTextWriterBuilder.LstMinimizing.EVERYTHING; -import static software.amazon.ion.system.IonTextWriterBuilder.LstMinimizing.LOCALS; -import static software.amazon.ion.system.IonWriterBuilder.InitialIvmHandling.SUPPRESS; -import static software.amazon.ion.system.IonWriterBuilder.IvmMinimizing.ADJACENT; -import static software.amazon.ion.system.IonWriterBuilder.IvmMinimizing.DISTANT; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.impl.Symtabs; +import com.amazon.ion.impl._Private_IonWriter; import java.io.IOException; import java.io.OutputStream; import java.nio.charset.Charset; import org.junit.Assert; import org.junit.Test; -import software.amazon.ion.IonCatalog; -import software.amazon.ion.IonWriter; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.impl.PrivateIonWriter; -import software.amazon.ion.impl.Symtabs; -import software.amazon.ion.system.IonTextWriterBuilder; -import software.amazon.ion.system.SimpleCatalog; + public class IonTextWriterBuilderTest { @@ -87,7 +87,7 @@ public void testCustomCatalog() StringBuilder out = new StringBuilder(); IonWriter writer = b.build(out); - assertSame(catalog, ((PrivateIonWriter)writer).getCatalog()); + assertSame(catalog, ((_Private_IonWriter)writer).getCatalog()); IonCatalog catalog2 = new SimpleCatalog(); b.setCatalog(catalog2); diff --git a/test/software/amazon/ion/system/SimpleCatalogTest.java b/test/com/amazon/ion/system/SimpleCatalogTest.java similarity index 87% rename from test/software/amazon/ion/system/SimpleCatalogTest.java rename to test/com/amazon/ion/system/SimpleCatalogTest.java index c5f040ea0f..251dce0001 100644 --- a/test/software/amazon/ion/system/SimpleCatalogTest.java +++ b/test/com/amazon/ion/system/SimpleCatalogTest.java @@ -1,19 +1,27 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.system; +package com.amazon.ion.system; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonString; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.IonValue; +import com.amazon.ion.SymbolTable; import java.util.Arrays; import java.util.Date; import java.util.HashMap; @@ -23,14 +31,7 @@ import java.util.Set; import java.util.TreeMap; import org.junit.Test; -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonString; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonSystem; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.IonValue; -import software.amazon.ion.SymbolTable; -import software.amazon.ion.system.SimpleCatalog; + public class SimpleCatalogTest extends IonTestCase diff --git a/test/software/amazon/ion/util/EquivalenceTest.java b/test/com/amazon/ion/util/EquivalenceTest.java similarity index 93% rename from test/software/amazon/ion/util/EquivalenceTest.java rename to test/com/amazon/ion/util/EquivalenceTest.java index 5cfa7cf979..676469187c 100644 --- a/test/software/amazon/ion/util/EquivalenceTest.java +++ b/test/com/amazon/ion/util/EquivalenceTest.java @@ -1,26 +1,26 @@ /* - * Copyright 2016-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.util; +package com.amazon.ion.util; +import com.amazon.ion.IonFloat; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.IonValue; import org.junit.Ignore; import org.junit.Test; -import software.amazon.ion.IonFloat; -import software.amazon.ion.IonStruct; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.IonValue; -import software.amazon.ion.util.Equivalence; public class EquivalenceTest extends IonTestCase @@ -358,7 +358,7 @@ public void testStringSymbol() { assertNotIonEq(ion("\"hi\""), ion("'hi'")); } - // TODO amzn/ion-java#58 : Remove the ignore annotation from this test after + // TODO amzn/ion-java/issues/58 : Remove the ignore annotation from this test after // making the required changes to Equivalence.Field.hashCode. @Ignore @Test @@ -389,7 +389,7 @@ public void testFieldEquals1() { assertTrue(f1.hashCode() != f3.hashCode()); } - // TODO amzn/ion-java#58 : Remove the ignore annotation from this test after + // TODO amzn/ion-java/issues/58 : Remove the ignore annotation from this test after // making the required changes to Equivalence.Field.hashCode. @Ignore @Test diff --git a/test/software/amazon/ion/util/IonStreamUtilsTest.java b/test/com/amazon/ion/util/IonStreamUtilsTest.java similarity index 78% rename from test/software/amazon/ion/util/IonStreamUtilsTest.java rename to test/com/amazon/ion/util/IonStreamUtilsTest.java index 603f0c24c6..f95f8a6545 100644 --- a/test/software/amazon/ion/util/IonStreamUtilsTest.java +++ b/test/com/amazon/ion/util/IonStreamUtilsTest.java @@ -1,24 +1,25 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.util; +package com.amazon.ion.util; +import static com.amazon.ion.impl._Private_IonConstants.BINARY_VERSION_MARKER_1_0; +import static com.amazon.ion.impl._Private_Utils.EMPTY_BYTE_ARRAY; +import static com.amazon.ion.util.IonStreamUtils.isIonBinary; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static software.amazon.ion.impl.PrivateIonConstants.BINARY_VERSION_MARKER_1_0; -import static software.amazon.ion.impl.PrivateUtils.EMPTY_BYTE_ARRAY; -import static software.amazon.ion.util.IonStreamUtils.isIonBinary; import org.junit.Test; diff --git a/test/software/amazon/ion/util/JarInfoTest.java b/test/com/amazon/ion/util/JarInfoTest.java similarity index 82% rename from test/software/amazon/ion/util/JarInfoTest.java rename to test/com/amazon/ion/util/JarInfoTest.java index 3f7abafa91..3994ed4e15 100644 --- a/test/software/amazon/ion/util/JarInfoTest.java +++ b/test/com/amazon/ion/util/JarInfoTest.java @@ -1,20 +1,25 @@ /* - * Copyright 2013-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.util; +package com.amazon.ion.util; -import static org.junit.Assert.assertEquals; +import com.amazon.ion.IonException; +import com.amazon.ion.Timestamp; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -22,15 +27,11 @@ import java.util.Collections; import java.util.List; import java.util.jar.Manifest; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import software.amazon.ion.IonException; -import software.amazon.ion.Timestamp; + +import static org.junit.Assert.*; public class JarInfoTest { - @Rule public ExpectedException thrown = ExpectedException.none(); @@ -42,8 +43,8 @@ private static String getAttributes(String manifestVersion) private static String getIonJavaAttributes(String buildTime, String version) { return getAttributes("1.0") - + "Ion-Java-Build-Time: " + buildTime + "\n" - + "Ion-Java-Project-Version: " + version + "\n"; + + "Ion-Java-Build-Time: " + buildTime + "\n" + + "Ion-Java-Project-Version: " + version + "\n"; } private static Manifest getManifest(String attributes) throws IOException @@ -105,4 +106,4 @@ public void testConflictingManifests() throws Exception new JarInfo(manifests); } -} +} \ No newline at end of file diff --git a/test/software/amazon/ion/util/NullOutputStream.java b/test/com/amazon/ion/util/NullOutputStream.java similarity index 63% rename from test/software/amazon/ion/util/NullOutputStream.java rename to test/com/amazon/ion/util/NullOutputStream.java index 843f0fc4e5..584531905c 100644 --- a/test/software/amazon/ion/util/NullOutputStream.java +++ b/test/com/amazon/ion/util/NullOutputStream.java @@ -1,18 +1,19 @@ /* - * Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.util; +package com.amazon.ion.util; import java.io.IOException; import java.io.OutputStream; diff --git a/test/com/amazon/ion/util/PrinterTest.java b/test/com/amazon/ion/util/PrinterTest.java new file mode 100644 index 0000000000..fc26658003 --- /dev/null +++ b/test/com/amazon/ion/util/PrinterTest.java @@ -0,0 +1,678 @@ +/* + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.ion.util; + + +import static com.amazon.ion.SystemSymbols.ION_1_0; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; + +import com.amazon.ion.BlobTest; +import com.amazon.ion.ClobTest; +import com.amazon.ion.IntTest; +import com.amazon.ion.IonBlob; +import com.amazon.ion.IonBool; +import com.amazon.ion.IonClob; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonDecimal; +import com.amazon.ion.IonFloat; +import com.amazon.ion.IonInt; +import com.amazon.ion.IonList; +import com.amazon.ion.IonNull; +import com.amazon.ion.IonSexp; +import com.amazon.ion.IonString; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSymbol; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.IonTimestamp; +import com.amazon.ion.IonValue; +import com.amazon.ion.impl._Private_Utils; +import org.junit.Before; +import org.junit.Test; + + +public class PrinterTest + extends IonTestCase +{ + private Printer myPrinter; + + + @Before @Override + public void setUp() + throws Exception + { + super.setUp(); + myPrinter = new Printer(); + } + + public IonInt int123() + { + return system().newInt(123); + } + + + public IonSymbol symbol(String text) + { + return system().newSymbol(text); + } + + public IonSymbol symbolHello() + { + return symbol("hello"); + } + + public IonSymbol symbolNotEquals() + { + return symbol("!="); + } + + public void checkRendering(String expected, IonValue value) + throws Exception + { + StringBuilder w = new StringBuilder(); + myPrinter.print(value, w); + assertEquals("Printer output", expected, w.toString()); + } + + public void checkNullRendering(String image, IonValue value) + throws Exception + { + boolean origUntypedNulls = myPrinter.myOptions.untypedNulls; + + myPrinter.myOptions.untypedNulls = false; + checkRendering(image, value); + + myPrinter.myOptions.untypedNulls = true; + checkRendering("null", value); + + myPrinter.myOptions.untypedNulls = origUntypedNulls; + } + + + //========================================================================= + // Test cases + + @Test + public void testPrintingAnnotations() + throws Exception + { + IonNull value = (IonNull) oneValue("an::$0::null"); + checkRendering("an::$0::null", value); + + value.addTypeAnnotation("+"); + value.addTypeAnnotation("\u0000"); + value.addTypeAnnotation("$0"); + checkRendering("an::$0::'+'::'\\0'::'$0'::null", value); + + myPrinter.setPrintSymbolAsString(true); + checkRendering("an::$0::'+'::'\\0'::'$0'::null", value); + myPrinter.setPrintStringAsJson(true); + checkRendering("an::$0::'+'::'\\0'::'$0'::null", value); + myPrinter.setPrintSymbolAsString(false); + myPrinter.setPrintStringAsJson(false); + + myPrinter.setSkipAnnotations(true); + checkRendering("null", value); + myPrinter.setSkipAnnotations(false); + + IonSexp s = system().newEmptySexp(); + s.add(value); + checkRendering("(an::$0::'+'::'\\0'::'$0'::null)", s); + myPrinter.setPrintSymbolAsString(true); + checkRendering("(an::$0::'+'::'\\0'::'$0'::null)", s); + myPrinter.setPrintStringAsJson(true); + checkRendering("(an::$0::'+'::'\\0'::'$0'::null)", s); + + value.setTypeAnnotations("boo", "boo"); + checkRendering("boo::boo::null", value); + } + + + @Test + public void testPrintingBlob() + throws Exception + { + IonBlob value = system().newNullBlob(); + checkNullRendering("null.blob", value); + + for (int i = 0; i < BlobTest.TEST_DATA.length; i++) + { + BlobTest.TestData td = BlobTest.TEST_DATA[i]; + value.setBytes(td.bytes); + + myPrinter.setPrintBlobAsString(true); + checkRendering("\"" + td.base64 + "\"", value); + + myPrinter.setPrintBlobAsString(false); + checkRendering("{{" + td.base64 + "}}", value); + } + + value = (IonBlob) oneValue("{{}}"); + checkRendering("{{}}", value); + + value.addTypeAnnotation("an"); + checkRendering("an::{{}}", value); + } + + + @Test + public void testPrintingBool() + throws Exception + { + IonBool value = system().newNullBool(); + checkNullRendering("null.bool", value); + + value.setValue(true); + checkRendering("true", value); + + value.setValue(false); + checkRendering("false", value); + + value.addTypeAnnotation("an"); + checkRendering("an::false", value); + } + + + @Test + public void testPrintingClob() + throws Exception + { + IonClob value = system().newNullClob(); + checkNullRendering("null.clob", value); + + value.setBytes(ClobTest.SAMPLE_ASCII_AS_UTF8); + checkRendering("{{\"" + ClobTest.SAMPLE_ASCII + "\"}}", value); + + // TODO test "real" UTF8 and other encodings. + + value = (IonClob) oneValue("{{\"\"}}"); + checkRendering("{{\"\"}}", value); + + value.addTypeAnnotation("an"); + checkRendering("an::{{\"\"}}", value); + + myPrinter.setPrintClobAsString(true); + checkRendering("an::\"\"", value); + + value.clearTypeAnnotations(); + value.setBytes(ClobTest.SAMPLE_ASCII_AS_UTF8); + checkRendering("\"" + ClobTest.SAMPLE_ASCII + "\"", value); + + value = (IonClob) oneValue("{{'''Ab\\0'''}}"); + myPrinter.setPrintClobAsString(false); + checkRendering("{{\"Ab\\0\"}}", value); + myPrinter.setPrintClobAsString(true); + checkRendering("\"Ab\\0\"", value); + myPrinter.setPrintStringAsJson(true); + checkRendering("\"Ab\\u0000\"", value); + } + + + @Test + public void testPrintingDatagram() + throws Exception + { + IonDatagram dg = loader().load("a b c"); + StringBuilder w = new StringBuilder(); + myPrinter.print(dg, w); + String text = w.toString(); + assertTrue("missing version marker", + text.startsWith(ION_1_0 + ' ')); + assertTrue("missing data", + text.endsWith(" a b c")); + + // Just force symtab analysis and make sure output is still okay + dg.getBytes(new byte[dg.byteSize()]); + text = w.toString(); + assertTrue("missing version marker", + text.startsWith(ION_1_0 + ' ')); + assertTrue("missing data", + text.endsWith(" a b c")); + + // We shouldn't inject a local table if its not needed. + + String data = "2 '+' [2,'+']"; + String dataWithIvm = ION_1_0 + ' ' + data; + dg = loader().load(dataWithIvm); + checkRendering(dataWithIvm, dg); + + myPrinter.setSkipSystemValues(true); + checkRendering(data, dg); + + myPrinter.setPrintDatagramAsList(true); + checkRendering("[2,'+',[2,'+']]", dg); + + myPrinter.setPrintDatagramAsList(false); + myPrinter.setSkipSystemValues(false); + myPrinter.setJsonMode(); + checkRendering("[2,\"+\",[2,\"+\"]]", dg); + } + + @Test + public void testDatagramWithoutSymbols() + throws Exception + { + IonDatagram dg = system().newDatagram(); + dg.add().newInt(1); + checkRendering(ION_1_0 + " 1", dg); + } + + @Test + public void testSimplifyingChainedLocalSymtab() + throws Exception + { + myPrinter.myOptions.simplifySystemValues = true; + + String ionText = + ION_SYMBOL_TABLE + "::{}" + + " x" + + " " + ION_SYMBOL_TABLE + "::{}" + + " y"; + IonDatagram dg = loader().load(ionText); + checkRendering(ION_1_0 + " x y", dg); + } + + @Test + public void testPrintingDecimal() + throws Exception + { + IonDecimal value = system().newNullDecimal(); + checkNullRendering("null.decimal", value); + + value.setValue(-123); + checkRendering("-123.", value); + + value.setValue(456); + checkRendering("456.", value); + + value.setValue(0); + checkRendering("0.", value); + + value.addTypeAnnotation("an"); + checkRendering("an::0.", value); + + value = (IonDecimal) oneValue("0d42"); + checkRendering("0d42", value); + + value = (IonDecimal) oneValue("0d+42"); + checkRendering("0d42", value); + + value = (IonDecimal) oneValue("0d-42"); + checkRendering("0d-42", value); + + value = (IonDecimal) oneValue("100d-1"); + checkRendering("10.0", value); + + value = (IonDecimal) oneValue("100d3"); + checkRendering("100d3", value); + + myPrinter.setPrintDecimalAsFloat(true); + checkRendering("100e3", value); + } + + + @Test + public void testPrintingFloat() + throws Exception + { + IonFloat value = system().newNullFloat(); + checkNullRendering("null.float", value); + + value.setValue(-123); + checkRendering("-123e0", value); + + value.setValue(456); + checkRendering("456e0", value); + + value.setValue(0); + checkRendering("0e0", value); + + value.addTypeAnnotation("an"); + checkRendering("an::0e0", value); + + value = (IonFloat) oneValue("1e4"); + checkRendering("10000e0", value); + + value = (IonFloat) oneValue("1e+4"); + checkRendering("10000e0", value); + + value = (IonFloat) oneValue("125e-2"); + checkRendering("1.25e0", value); + } + + + @Test + public void testPrintingInt() + throws Exception + { + IonInt value = system().newNullInt(); + checkNullRendering("null.int", value); + + value.setValue(-123); + checkRendering("-123", value); + + value.setValue(456); + checkRendering("456", value); + + value.setValue(IntTest.A_LONG_INT); + checkRendering(Long.toString(IntTest.A_LONG_INT), value); + + value.setValue(0); + checkRendering("0", value); + + value.addTypeAnnotation("an"); + checkRendering("an::0", value); + } + + + @Test + public void testPrintingList() + throws Exception + { + IonList value = system().newNullList(); + checkNullRendering("null.list", value); + + value.add(system().newNull()); + checkRendering("[null]", value); + + value.add(int123()); + checkRendering("[null,123]", value); + + value.add(symbolNotEquals()); + value.add(symbolHello()); + value.add(symbol("null")); + checkRendering("[null,123,'!=',hello,'null']", value); + + value = (IonList) oneValue("[]"); + checkRendering("[]", value); + + value.addTypeAnnotation("an"); + checkRendering("an::[]", value); + } + + + @Test + public void testPrintingNull() + throws Exception + { + IonNull value = system().newNull(); + checkNullRendering("null", value); + + value.addTypeAnnotation("an"); + checkRendering("an::null", value); + } + + + @Test + public void testPrintingSexp() + throws Exception + { + IonSexp value = system().newNullSexp(); + checkNullRendering("null.sexp", value); + + value.add(system().newNull()); + checkRendering("(null)", value); + + value.add(int123()); + checkRendering("(null 123)", value); + + value.add(symbolNotEquals()); + value.add(symbolHello()); + value.add(symbol("null")); + checkRendering("(null 123 != hello 'null')", value); + + myPrinter.setPrintSexpAsList(true); + checkRendering("[null,123,'!=',hello,'null']", value); + myPrinter.setPrintSymbolAsString(true); + checkRendering("[null,123,\"!=\",\"hello\",\"null\"]", value); + + value = (IonSexp) oneValue("()"); + checkRendering("[]", value); + myPrinter.setPrintSexpAsList(false); + checkRendering("()", value); + + myPrinter.setPrintSymbolAsString(false); + value.addTypeAnnotation("an"); + checkRendering("an::()", value); + } + + + @Test + public void testPrintingString() + throws Exception + { + IonString value = system().newNullString(); + checkNullRendering("null.string", value); + + value.setValue("Adam E"); + checkRendering("\"Adam E\"", value); + + value.setValue("Oh, \"Hello!\""); + checkRendering("\"Oh, \\\"Hello!\\\"\"", value); + + value.addTypeAnnotation("an"); + checkRendering("an::\"Oh, \\\"Hello!\\\"\"", value); + + value = system().newString("Ab\u0000"); + checkRendering("\"Ab\\0\"", value); + myPrinter.setPrintStringAsJson(true); + checkRendering("\"Ab\\u0000\"", value); + + // TODO check escaping + } + + + @Test + public void testPrintingStruct() + throws Exception + { + IonStruct value = system().newNullStruct(); + checkNullRendering("null.struct", value); + + value.put("foo", system().newNull()); + checkRendering("{foo:null}", value); + + // TODO this is too strict, order shouldn't matter. + value.put("123", system().newNull()); + checkRendering("{foo:null,'123':null}", value); + + value.add("foo", int123()); + checkRendering("{foo:null,'123':null,foo:123}", value); + + myPrinter.setPrintSymbolAsString(true); + checkRendering("{\"foo\":null,\"123\":null,\"foo\":123}", value); + + value = (IonStruct) oneValue("{}"); + checkRendering("{}", value); + + value.addTypeAnnotation("an"); + checkRendering("an::{}", value); + + + value = (IonStruct) oneValue("an::{$0:null}"); + value.addTypeAnnotation("\u0007"); + value.put("A\u0000", system().newInt(12)); + checkRendering("an::'\\a'::{\"$0\":null,\"A\\0\":12}", value); + myPrinter.setPrintStringAsJson(true); + checkRendering("an::'\\a'::{\"$0\":null,\"A\\u0000\":12}", value); + } + + + @Test + public void testPrintingSymbol() + throws Exception + { + IonSymbol value = system().newNullSymbol(); + checkNullRendering("null.symbol", value); + + value.setValue("Adam E"); + checkRendering("'Adam E'", value); + + // Symbols that look like keywords. + value.setValue("null"); + checkRendering("'null'", value); + value.setValue("true"); + checkRendering("'true'", value); + value.setValue("false"); + checkRendering("'false'", value); + value.setValue("null.int"); + checkRendering("'null.int'", value); + + // Operators standalone + value.setValue("%"); + checkRendering("'%'", value); + + value.setValue("Oh, \"Hello!\""); + checkRendering("'Oh, \"Hello!\"'", value); + // not: checkRendering("'Oh, \\\"Hello!\\\"'", value); + + myPrinter.setPrintSymbolAsString(true); + checkRendering("\"Oh, \\\"Hello!\\\"\"", value); + myPrinter.setPrintSymbolAsString(false); + + value.setValue("Oh, 'Hello there!'"); + checkRendering("'Oh, \\\'Hello there!\\\''", value); + + myPrinter.setPrintSymbolAsString(true); + checkRendering("\"Oh, 'Hello there!'\"", value); + myPrinter.setPrintSymbolAsString(false); + + value.addTypeAnnotation("an"); + checkRendering("an::'Oh, \\\'Hello there!\\\''", value); + + // TODO check escaping + + value = system().newSymbol("Ab\u0000"); + checkRendering("'Ab\\0'", value); + myPrinter.setPrintSymbolAsString(true); + checkRendering("\"Ab\\0\"", value); + myPrinter.setPrintStringAsJson(true); + checkRendering("\"Ab\\u0000\"", value); + myPrinter.setPrintSymbolAsString(false); + checkRendering("'Ab\\0'", value); + + value = system().newSymbol("$99"); // Known, sidlike text + checkRendering("'$99'", value); + myPrinter.setPrintSymbolAsString(true); + checkRendering("\"$99\"", value); + myPrinter.setPrintStringAsJson(true); + checkRendering("\"$99\"", value); + myPrinter.setPrintSymbolAsString(false); + checkRendering("'$99'", value); + + value = (IonSymbol) system().singleValue("$0"); // Unknown symbol + checkRendering("$0", value); + myPrinter.setPrintSymbolAsString(true); + checkRendering("\"$0\"", value); + myPrinter.setPrintStringAsJson(true); + checkRendering("\"$0\"", value); + myPrinter.setPrintSymbolAsString(false); + checkRendering("$0", value); + } + + @Test + public void testPrintingSidlikeSymbol() + throws Exception + { + IonSymbol value = system().newSymbol("$"); + checkRendering("$", value); + + value = system().newSymbol("$0"); + checkRendering("'$0'", value); + + value = system().newSymbol("$99"); + checkRendering("'$99'", value); + } + + // TODO annotations + // TODO field names + + + @Test + public void testPrintingTimestamp() + throws Exception + { + IonTimestamp value = system().newNullTimestamp(); + checkNullRendering("null.timestamp", value); + + value = (IonTimestamp) oneValue("2007-05-15T18:45-00:00"); + checkRendering("2007-05-15T18:45-00:00", value); + + value = (IonTimestamp) oneValue("2007-05-15T18:45Z"); + checkRendering("2007-05-15T18:45Z", value); + + // offset +0 shortens to Z + value = (IonTimestamp) oneValue("2007-05-15T18:45+00:00"); + checkRendering("2007-05-15T18:45Z", value); + + value = (IonTimestamp) oneValue("2007-05-15T18:45+01:12"); + checkRendering("2007-05-15T18:45+01:12", value); + + value = (IonTimestamp) oneValue("2007-05-15T18:45-10:01"); + checkRendering("2007-05-15T18:45-10:01", value); + + value.addTypeAnnotation("an"); + checkRendering("an::2007-05-15T18:45-10:01", value); + + myPrinter.setPrintTimestampAsString(true); + checkRendering("an::\"2007-05-15T18:45-10:01\"", value); + + myPrinter.setJsonMode(); + checkRendering("" + value.getMillis(), value); + + // TODO test printTimestampAsMillis + } + + private static final String Q = "\""; + + @Test + public void testJsonEscapes() + throws Exception + { + String ionEscapes = + Q + "\\0\\a\\b\\t\\n\\f\\r\\v\\\"\\'\\?\\\\\\/\\\n" + Q; + String jsonEscapes = + Q + "\\u0000\\u0007\\b\\t\\n\\f\\r\\u000b\\\"'?\\\\/" + Q; + + IonString value = (IonString) system().singleValue(ionEscapes); + + myPrinter.setJsonMode(); + checkRendering(jsonEscapes, value); + } + + + @Test + public void testJsonEscapeNonBmp() throws Exception { + final String literal = new StringBuilder() + .append("'''") + .append('\uDAF7') + .append('\uDE56') + .append("'''") + .toString(); + + final byte[] utf8Bytes = _Private_Utils.utf8(literal); + + final IonDatagram dg = loader().load(utf8Bytes); + final StringBuilder out = new StringBuilder(); + final Printer json = new Printer(); + json.setJsonMode(); + json.print(dg.get(0), out); + + assertEquals( + "\"\\uDAF7\\uDE56\"".toLowerCase(), + out.toString().toLowerCase() + ); + } +} diff --git a/test/software/amazon/ion/util/RepeatInputStream.java b/test/com/amazon/ion/util/RepeatInputStream.java similarity index 82% rename from test/software/amazon/ion/util/RepeatInputStream.java rename to test/com/amazon/ion/util/RepeatInputStream.java index 315ce39963..f5bae1c206 100644 --- a/test/software/amazon/ion/util/RepeatInputStream.java +++ b/test/com/amazon/ion/util/RepeatInputStream.java @@ -1,18 +1,19 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.util; +package com.amazon.ion.util; import java.io.IOException; import java.io.InputStream; diff --git a/test/software/amazon/ion/util/SpansTest.java b/test/com/amazon/ion/util/SpansTest.java similarity index 61% rename from test/software/amazon/ion/util/SpansTest.java rename to test/com/amazon/ion/util/SpansTest.java index a097928c30..33355caa4f 100644 --- a/test/software/amazon/ion/util/SpansTest.java +++ b/test/com/amazon/ion/util/SpansTest.java @@ -1,25 +1,26 @@ /* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.util; +package com.amazon.ion.util; +import static com.amazon.ion.util.Spans.currentSpan; import static org.junit.Assert.assertNull; -import static software.amazon.ion.util.Spans.currentSpan; +import com.amazon.ion.Span; +import com.amazon.ion.TextSpan; import org.junit.Test; -import software.amazon.ion.Span; -import software.amazon.ion.TextSpan; public class SpansTest { diff --git a/test/software/amazon/ion/util/TextTest.java b/test/com/amazon/ion/util/TextTest.java similarity index 84% rename from test/software/amazon/ion/util/TextTest.java rename to test/com/amazon/ion/util/TextTest.java index 42d5d02b0d..6011adfd7c 100644 --- a/test/software/amazon/ion/util/TextTest.java +++ b/test/com/amazon/ion/util/TextTest.java @@ -1,27 +1,28 @@ /* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. - * A copy of the License is located at: + * A copy of the License is located at * - * http://aws.amazon.com/apache2.0/ + * http://www.apache.org/licenses/LICENSE-2.0 * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. */ -package software.amazon.ion.util; +package com.amazon.ion.util; +import com.amazon.ion.BlobTest; +import com.amazon.ion.BlobTest.TestData; +import com.amazon.ion.Decimal; +import com.amazon.ion.IonTestCase; +import com.amazon.ion.impl._Private_IonTextAppender; import java.math.BigDecimal; import org.junit.Test; -import software.amazon.ion.BlobTest; -import software.amazon.ion.Decimal; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.BlobTest.TestData; -import software.amazon.ion.impl.PrivateIonTextAppender; -import software.amazon.ion.util.IonTextUtils; + public class TextTest extends IonTestCase @@ -120,9 +121,9 @@ private void unquotedAnywhere(String symbol) IonTextUtils.symbolVariant(symbol)); // unquoted in sexp - assertFalse(PrivateIonTextAppender.symbolNeedsQuoting(symbol, false)); + assertFalse(_Private_IonTextAppender.symbolNeedsQuoting(symbol, false)); // unquoted elsewhere - assertFalse(PrivateIonTextAppender.symbolNeedsQuoting(symbol, true)); + assertFalse(_Private_IonTextAppender.symbolNeedsQuoting(symbol, true)); } private void quotedEverywhere(String symbol) @@ -131,9 +132,9 @@ private void quotedEverywhere(String symbol) IonTextUtils.symbolVariant(symbol)); // Quoted in sexp - assertTrue(PrivateIonTextAppender.symbolNeedsQuoting(symbol, false)); + assertTrue(_Private_IonTextAppender.symbolNeedsQuoting(symbol, false)); // Quoted elsewhere - assertTrue(PrivateIonTextAppender.symbolNeedsQuoting(symbol, true)); + assertTrue(_Private_IonTextAppender.symbolNeedsQuoting(symbol, true)); } private void unquotedInSexp(String symbol) @@ -142,9 +143,9 @@ private void unquotedInSexp(String symbol) IonTextUtils.symbolVariant(symbol)); // unquoted in sexp - assertFalse(PrivateIonTextAppender.symbolNeedsQuoting(symbol, false)); + assertFalse(_Private_IonTextAppender.symbolNeedsQuoting(symbol, false)); // quoted elsewheres - assertTrue(PrivateIonTextAppender.symbolNeedsQuoting(symbol, true)); + assertTrue(_Private_IonTextAppender.symbolNeedsQuoting(symbol, true)); } diff --git a/test/software/amazon/ion/BinaryTestGenerator.py b/test/software/amazon/ion/BinaryTestGenerator.py deleted file mode 100755 index 53717a30b1..0000000000 --- a/test/software/amazon/ion/BinaryTestGenerator.py +++ /dev/null @@ -1,271 +0,0 @@ -#!/apollo/bin/env -e Python python - -import struct - -_COOKIE = struct.pack('>L', 0x10140100L) - -def var_uint(val) : - assert isinstance(val, (int, long)) - assert val >= 0 - bytes = [] - while val > 0x7F : - chunk = (val & 0x7F) - bytes.append(chunk) - val >>= 7 - bytes.append(val) - # terminating high bit - bytes[0] |= 0x80 - return ''.join(chr(x) for x in reversed(bytes)) - -def var_int(val) : - assert isinstance(val, (int, long)) - bytes = [] - if val < 0 : - val = -val - sbit = 0x40 - else : - sbit = 0 - while val > 0x7F : - chunk = (val & 0x7F) - bytes.append(chunk) - val >>= 7 - if val > 0x3F : - # can't quite fit the sign bit - bytes.append(val) - bytes.append(sbit) - else : - bytes.append(val | sbit) - # terminating high bit - bytes[0] |= 0x80 - return ''.join(chr(x) for x in reversed(bytes)) - -def raw_uint(val) : - assert isinstance(val, (int, long)) - assert val >= 0 - bytes = [] - while val > 0xFF : - bytes.append(val & 0xFF) - val >>= 8 - bytes.append(val) - return ''.join(chr(x) for x in reversed(bytes)) - -def raw_int(val) : - assert isinstance(val, (int, long)) - bytes = [] - if val < 1 : - val = -val - sbit = 0x40 - else : - sbit = 0 - while val > 0xFF : - bytes.append(chr(val & 0xFF)) - val >>= 8 - if val > 0x7F : - # can't quite fit the sign bit - bytes.append(val) - bytes.append(sbit) - else : - bytes.append(val | sbit) - return ''.join(chr(x) for x in reversed(bytes)) - -def gen_type(high, low) : - assert high >= 0 and high <= 15 - assert low >= 0 and low <= 15 - return struct.pack('>B', ((high << 4 ) | low)) - -_LN_EXT = 14 -_LN_NULL = 15 - -def gen_type_len(high, len) : - assert high >= 0 and high <= 15 - assert len is None or len >= 0 - if len is None : - return gen_type(high, _LN_NULL) - elif len < _LN_EXT : - return gen_type(high, len) - else : - return gen_type(high, _LN_EXT) + var_uint(len) - -def gen_null(high) : - return gen_type(high, _LN_NULL) - -_HN_NULL = 0 -_HN_BOOL = 1 -_HN_PINT = 2 -_HN_NINT = 3 -_HN_FLOAT = 4 -_HN_DECIMAL = 5 -_HN_TIMESTAMP = 6 -_HN_SYMBOL = 7 -_HN_STRING = 8 -_HN_CLOB = 9 -_HN_BLOB = 10 -_HN_LIST = 11 -_HN_SEXP = 12 -_HN_STRUCT = 13 -_HN_ANNOTATION = 14 - -def gen_int(val) : - assert val is None or isinstance(val, (int, long)) - if val is None : - return gen_null(_HN_PINT) - - if val < 0 : - val = -val - type = _HN_NINT - else : - type = _HN_PINT - - if val != 0 : - raw_bits = raw_uint(val) - else : - raw_bits = '' - return gen_type_len(type, len(raw_bits)) \ - + raw_bits - -def gen_float(val) : - assert val is None or isinstance(val, (float, int)) - if val is None : - return gen_null(_HN_FLOAT) - - if isinstance(val, int) : - val = float(int) - # note this violates the standard - return gen_type(_HN_FLOAT, 8) + struct.pack('>d', val) - -def gen_raw_decimal(val) : - from decimal import Decimal - assert isinstance(val, (str, int, long, Decimal)) - if isinstance(val, (str, int, long)) : - val = Decimal(val) - sign, digits, exponent = val.as_tuple() - mantissa = 0L - for digit in digits : - mantissa *= 10 - mantissa += digit - if sign == 1 : - mantissa = -mantissa - print mantissa, exponent - raw_mantissa = raw_int(mantissa) - raw_exponent = var_int(exponent) - return raw_exponent + raw_mantissa - -def gen_decimal(val) : - from decimal import Decimal - assert val is None or isinstance(val, (str, int, long, Decimal)) - if val is None : - return gen_null(_HN_DECIMAL) - - raw_decimal = gen_raw_decimal(val) - return gen_type_len(_HN_DECIMAL, len(raw_decimal)) \ - + raw_decimal - -def gen_timestamp(milliseconds, offset = 0) : - from decimal import Decimal - assert milliseconds is None or isinstance(milliseconds, (str, int, long, Decimal)) - assert isinstance(offset, (int, long)) - if milliseconds is None : - return gen_null(_HN_TIMESTAMP) - - raw_offset = var_int(offset) - raw_ms = gen_raw_decimal(milliseconds) - return gen_type_len(_HN_TIMESTAMP, len(raw_offset) + len(raw_ms)) \ - + raw_offset \ - + raw_ms - -def gen_symbol(val) : - assert val is None or isinstance(val, (int, long)) - # symbols must be greater than 0 - assert val > 0 - if val is None : - return gen_null(_HN_SYMBOL) - - raw_symbol = raw_uint(val) - return gen_type_len(_HN_SYMBOL, len(raw_symbol)) \ - + raw_symbol - -def gen_byte_type(type, val) : - assert val is None or isinstance(val, str) - if val is None : - return gen_null(type) - - return gen_type_len(type, len(val)) + val - -def gen_string(val) : - assert val is None or isinstance(val, (str, unicode)) - if isinstance(val, unicode): - val = val.encode('UTF-8') - else : - try : - val.decode('UTF-8') - except UnicodeDecodeError : - raise ValueError, 'Byte string must be UTF-8' - return gen_byte_type(_HN_STRING, val) - -def gen_clob(val) : - assert val is None or isinstance(val, str) - return gen_byte_type(_HN_CLOB, val) - -def gen_blob(val) : - assert val is None or isinstance(val, str) - return gen_byte_type(_HN_BLOB, val) - -def gen_sequence_type(type, *blobs) : - assert all(isinstance(blob, str) for blob in blobs) - return gen_type_len(type, sum(len(blob) for blob in blobs)) \ - + ''.join(blobs) - -def gen_list(*blobs) : - return gen_sequence_type(_HN_LIST, *blobs) - -def gen_sexp(*blobs) : - return gen_sequence_type(_HN_SEXP, *blobs) - -def gen_struct(*pairs) : - assert all(isinstance(id, (int, long)) \ - and id > 0 and isinstance(bytes, str) \ - for id, bytes in pairs) - data = ''.join('%s%s' % (var_uint(id), bytes) for id, bytes in pairs) - data_len = len(data) - return gen_type_len(_HN_STRUCT, data_len) + data - -def gen_ann(bytes, *syms) : - assert isinstance(bytes, str) - assert all(isinstance(x, (int, long)) and x > 0 for x in syms) - anns = ''.join(var_uint(x) for x in syms) - anns_len = len(anns) - anns_len_bytes = var_uint(anns_len) - val_len = len(anns_len_bytes) + anns_len + len(bytes) - return gen_type_len(_HN_ANNOTATION, val_len) \ - + anns_len_bytes \ - + anns \ - + bytes - -def lit(*vals): - from itertools import chain - return '"%s"' % ' '.join('%02X' % ord(x) for x in chain(*vals)) - -_SYM_ION = 1 -_SYM_ION_1_0 = 2 -_SYM_ION_SYMBOL_TABLE = 3 -_SYM_NAME = 4 -_SYM_VERSION = 5 -_SYM_IMPORTS = 6 -_SYM_SYMBOLS = 7 -_SYM_MAX_ID = 8 - -_SYS_SYMBOLS = 9; - -if __name__ == '__main__' : - print '$ion_1_0::{symbols : struct.null} 1e0' - print lit( - gen_ann( - gen_struct( - (_SYM_SYMBOLS, gen_null(_HN_STRUCT)) - ), - _SYM_ION_1_0), - gen_float(1.0) - ) - print lit( - gen_decimal(1) - ) diff --git a/test/software/amazon/ion/EquivsTest.java b/test/software/amazon/ion/EquivsTest.java deleted file mode 100644 index 8a6549345d..0000000000 --- a/test/software/amazon/ion/EquivsTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at: - * - * http://aws.amazon.com/apache2.0/ - * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. - */ - -package software.amazon.ion; - -import static software.amazon.ion.TestUtils.EQUIVS_IONTESTS_FILES; -import static software.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; - -import java.io.File; -import software.amazon.ion.junit.Injected.Inject; - -public class EquivsTest - extends EquivsTestCase -{ - @Inject("testFile") - public static final File[] FILES = - TestUtils.testdataFiles(GLOBAL_SKIP_LIST, EQUIVS_IONTESTS_FILES); - - public EquivsTest() - { - super(true); - } -} diff --git a/test/software/amazon/ion/NonEquivsTest.java b/test/software/amazon/ion/NonEquivsTest.java deleted file mode 100644 index ff25ad4ffa..0000000000 --- a/test/software/amazon/ion/NonEquivsTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2013-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at: - * - * http://aws.amazon.com/apache2.0/ - * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. - */ - -package software.amazon.ion; - -import static software.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; -import static software.amazon.ion.TestUtils.NON_EQUIVS_IONTESTS_FILEs; - -import java.io.File; -import software.amazon.ion.junit.Injected.Inject; - -public class NonEquivsTest - extends EquivsTestCase -{ - @Inject("testFile") - public static final File[] FILES = - TestUtils.testdataFiles(GLOBAL_SKIP_LIST, NON_EQUIVS_IONTESTS_FILEs); - - public NonEquivsTest() - { - super(false); - } -} diff --git a/test/software/amazon/ion/impl/IonImplUtilsTest.java b/test/software/amazon/ion/impl/IonImplUtilsTest.java deleted file mode 100644 index b626317f61..0000000000 --- a/test/software/amazon/ion/impl/IonImplUtilsTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2011-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at: - * - * http://aws.amazon.com/apache2.0/ - * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. - */ - -package software.amazon.ion.impl; - -import org.junit.Assert; -import org.junit.Test; -import software.amazon.ion.impl.PrivateUtils; - -public class IonImplUtilsTest -{ - @Test - public void testEmptyUtf8() - { - byte[] bytes = PrivateUtils.utf8(""); - Assert.assertArrayEquals(PrivateUtils.EMPTY_BYTE_ARRAY, bytes); - } - - @Test - public void testEasyUtf8() - throws Exception - { - String input = "abcdefghijklm"; - byte[] bytes = PrivateUtils.utf8(input); - byte[] direct = input.getBytes("UTF-8"); - Assert.assertArrayEquals(direct, bytes); - } -} diff --git a/test/software/amazon/ion/impl/PrivateScalarConversionsTest.java b/test/software/amazon/ion/impl/PrivateScalarConversionsTest.java deleted file mode 100644 index b19b301839..0000000000 --- a/test/software/amazon/ion/impl/PrivateScalarConversionsTest.java +++ /dev/null @@ -1,33 +0,0 @@ -package software.amazon.ion.impl; - -import org.junit.Assert; -import org.junit.Test; -import software.amazon.ion.Decimal; - -import static org.junit.Assert.*; - -public class PrivateScalarConversionsTest { - - private long decimalToLong(final Decimal d) { - PrivateScalarConversions.ValueVariant v = new PrivateScalarConversions.ValueVariant(); - v.setValue(d); - - v.cast(PrivateScalarConversions.FNID_FROM_DECIMAL_TO_LONG); - return v.getLong(); - } - - @Test - public void decimalToLong() { - assertEquals(1, decimalToLong(Decimal.valueOf(1L))); - } - - @Test - public void decimalToMinLong() { - assertEquals(Long.MAX_VALUE, decimalToLong(Decimal.valueOf(Long.MAX_VALUE))); - } - - @Test - public void decimalToMaxLong() { - assertEquals(Long.MIN_VALUE, decimalToLong(Decimal.valueOf(Long.MIN_VALUE))); - } -} \ No newline at end of file diff --git a/test/software/amazon/ion/impl/lite/IonDatagramLiteSublistTest.java b/test/software/amazon/ion/impl/lite/IonDatagramLiteSublistTest.java deleted file mode 100644 index 7bb57e17c9..0000000000 --- a/test/software/amazon/ion/impl/lite/IonDatagramLiteSublistTest.java +++ /dev/null @@ -1,16 +0,0 @@ -package software.amazon.ion.impl.lite; - -import software.amazon.ion.IonDatagram; -import software.amazon.ion.IonSequence; - -public class IonDatagramLiteSublistTest extends BaseIonSequenceLiteSublistTestCase { - - protected IonSequence newSequence() { - final IonDatagram datagram = SYSTEM.newDatagram(); - for(int i : INTS) { - datagram.add(SYSTEM.newInt(INTS[i])); - } - - return datagram; - } -} diff --git a/test/software/amazon/ion/impl/lite/IonDatagramLiteTest.java b/test/software/amazon/ion/impl/lite/IonDatagramLiteTest.java deleted file mode 100644 index 3637df1eba..0000000000 --- a/test/software/amazon/ion/impl/lite/IonDatagramLiteTest.java +++ /dev/null @@ -1,10 +0,0 @@ -package software.amazon.ion.impl.lite; - -import software.amazon.ion.IonSequence; - -public class IonDatagramLiteTest extends BaseIonSequenceLiteTestCase { - @Override - protected IonSequence newEmptySequence() { - return SYSTEM.newDatagram(); - } -} diff --git a/test/software/amazon/ion/impl/lite/IonListLiteTest.java b/test/software/amazon/ion/impl/lite/IonListLiteTest.java deleted file mode 100644 index 2da37f29b7..0000000000 --- a/test/software/amazon/ion/impl/lite/IonListLiteTest.java +++ /dev/null @@ -1,10 +0,0 @@ -package software.amazon.ion.impl.lite; - -import software.amazon.ion.IonSequence; - -public class IonListLiteTest extends BaseIonSequenceLiteTestCase { - @Override - protected IonSequence newEmptySequence() { - return SYSTEM.newEmptyList(); - } -} diff --git a/test/software/amazon/ion/impl/lite/IonSexpLiteTest.java b/test/software/amazon/ion/impl/lite/IonSexpLiteTest.java deleted file mode 100644 index 3b3fbfaa96..0000000000 --- a/test/software/amazon/ion/impl/lite/IonSexpLiteTest.java +++ /dev/null @@ -1,10 +0,0 @@ -package software.amazon.ion.impl.lite; - -import software.amazon.ion.IonSequence; - -public class IonSexpLiteTest extends BaseIonSequenceLiteTestCase { - @Override - protected IonSequence newEmptySequence() { - return SYSTEM.newEmptySexp(); - } -} diff --git a/test/software/amazon/ion/streaming/BadIonStreamingTest.java b/test/software/amazon/ion/streaming/BadIonStreamingTest.java deleted file mode 100644 index ce02918b5a..0000000000 --- a/test/software/amazon/ion/streaming/BadIonStreamingTest.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2008-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at: - * - * http://aws.amazon.com/apache2.0/ - * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific - * language governing permissions and limitations under the License. - */ -package software.amazon.ion.streaming; - -import static software.amazon.ion.TestUtils.BAD_IONTESTS_FILES; -import static software.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; -import static software.amazon.ion.TestUtils.testdataFiles; - -import java.io.File; -import java.io.IOException; -import org.junit.Ignore; -import org.junit.Test; -import software.amazon.ion.IonException; -import software.amazon.ion.IonReader; -import software.amazon.ion.IonTestCase; -import software.amazon.ion.TestUtils; -import software.amazon.ion.impl.PrivateUtils; -import software.amazon.ion.junit.Injected.Inject; - -public class BadIonStreamingTest -extends IonTestCase -{ - private static final boolean _debug_output_errors = false; - - - @Inject("testFile") - public static final File[] FILES = testdataFiles(GLOBAL_SKIP_LIST, - BAD_IONTESTS_FILES); - - - private File myTestFile; - - public void setTestFile(File file) - { - myTestFile = file; - } - - - @Test(expected = IonException.class) - public void testReadingScalars() - throws Exception - { - readFile( true ); - } - - - @Ignore // TODO amzn/ion-java#7 - @Test(expected = IonException.class) - public void testSkippingScalars() - throws Exception - { - // Readers don't validate while skipping scalars - // so we won't throw exceptions for all bad files. - readFile( false ); - } - - private void readFile(boolean materializeScalars) - throws IOException - { - try - { - byte[] buf = PrivateUtils.loadFileBytes(myTestFile); - IonReader it = system().newReader(buf); - TestUtils.deepRead(it, materializeScalars); - } - catch (IonException e) - { - /* good - we're expecting an error, there are testing bad input */ - if (_debug_output_errors) { - System.out.print(this.myTestFile.getName()); - System.out.print(": "); - System.out.println(e.getMessage()); - } - throw e; - } - } -} From 1b0c86b79e887934380173f97f1a2dff937f24fe Mon Sep 17 00:00:00 2001 From: Fernando Raganhan Barbosa Date: Thu, 11 Apr 2019 13:27:23 -0700 Subject: [PATCH 056/490] target JDK 6 Some older users require jdk 1.5 we can't set the target to anything lower than 1.6 starting with JDK 9 so setting it to 1.6 at least Important to note that we are still required to be JDK 1.5 compatible --- pom.xml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 33792c29df..25cb777b6f 100644 --- a/pom.xml +++ b/pom.xml @@ -81,6 +81,15 @@ src test + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.0 + + 1.6 + 1.6 + + - com.amazon.ion.impl.PrivateCommandLine + com.amazon.ion.impl._Private_CommandLine ${build.time} ${project.version} diff --git a/src/com/amazon/ion/impl/_Private_CommandLine.java b/src/com/amazon/ion/impl/_Private_CommandLine.java index 2296d400e6..902b327d67 100644 --- a/src/com/amazon/ion/impl/_Private_CommandLine.java +++ b/src/com/amazon/ion/impl/_Private_CommandLine.java @@ -91,10 +91,10 @@ private static int getArgumentId(String arg) } } if (arg.startsWith("--") && arg.length() > 2) { - if (arg.equals("help")) { + if (arg.equals("--help")) { return argid_HELP; } - if (arg.equals("version")) { + if (arg.equals("--version")) { return argid_VERSION; } } From 938440a2b8a6ec320db8057416f3e9ea413f5eb5 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Wed, 29 May 2019 15:35:49 -0700 Subject: [PATCH 064/490] Adds an Equivalence option to specify an epsilon to use when comparing float values. --- src/com/amazon/ion/util/Equivalence.java | 172 +++++++++++++++--- test/com/amazon/ion/util/EquivalenceTest.java | 62 ++++++- 2 files changed, 202 insertions(+), 32 deletions(-) diff --git a/src/com/amazon/ion/util/Equivalence.java b/src/com/amazon/ion/util/Equivalence.java index dbbd391b2c..70759b884f 100644 --- a/src/com/amazon/ion/util/Equivalence.java +++ b/src/com/amazon/ion/util/Equivalence.java @@ -61,6 +61,20 @@ * * *

    + * Additional options are available by configuring an Equivalence + * instance using {@link Equivalence.Builder}. For example: + * + *

    + *     com.amazon.ion.util.Equivalence equivalence =
    + *         new com.amazon.ion.util.Equivalence.Builder()
    + *             .withEpsilon(1e-6)
    + *             .build();
    + *     IonValue v1 = ...;
    + *     IonValue v2 = ...;
    + *     equivalence.ionValueEquals( v1, v2 );
    + * 
    + * + *

    *

    Ion Equivalence

    * In order to make Ion a useful model to describe data, we must first define * the notion of equivalence for all values representable in Ion. Equivalence @@ -104,7 +118,19 @@ * (A, V), where A is an ordered list of annotations, and V is an Ion Primitive * Data or Ion Complex Data value. The list of annotations, A is an tuple of Ion * Symbols (a specific type of Ion Primitive). - + * + *

    + *

    Terminology

    + * Within this class, strict equivalence refers to Ion data model + * equivalence as defined above and by the + * Ion + * Specification. Structural or non-strict equivalence + * follows the same rules as strict equivalence, except that + *
      + *
    • Annotations are not considered, and
    • + *
    • Timestamps that represent the same instant in time are always + * considered equivalent.
    • + *
    */ public final class Equivalence { @@ -115,7 +141,78 @@ public final class Equivalence { */ private static final boolean PUBLIC_COMPARISON_API = false; - private Equivalence() { + /** + * Configuration that requires strict data equivalence. + * @see #ionEquals(IonValue, IonValue) + */ + private static final Configuration STRICT_CONFIGURATION = new Configuration(new Builder().withStrict(true)); + + /** + * Configuration that requires structural equivalence without considering annotations. + * @see #ionEqualsByContent(IonValue, IonValue) + */ + private static final Configuration NON_STRICT_CONFIGURATION = new Configuration(new Builder().withStrict(false)); + + /** + * Contains the configuration to use when comparing Ion values. + */ + static final class Configuration { + private final boolean isStrict; + private final Double epsilon; + + Configuration(Builder builder) { + this.isStrict = builder.isStrict; + this.epsilon = builder.epsilon; + } + } + + /** + * Constructs {@link Equivalence} instances. + */ + public static final class Builder { + + private boolean isStrict = true; + private Double epsilon = null; + + /** + * When true, checks for strict data equivalence over two Ion Values. + * When false, checks for structural data equivalence over two Ion + * Values. See {@link Equivalence} for discussion of the differences + * between the two. + * Default: true. + * @param isStrict the value. + * @return this builder. + */ + public Builder withStrict(boolean isStrict) { + this.isStrict = isStrict; + return this; + } + + /** + * The maximum absolute difference between two Ion float values for + * which the two values will be considered equivalent. Default: Ion + * float values will only be considered equivalent when + * Double.compare(a, b) == 0. + * @param epsilon the value. + * @return this builder. + */ + public Builder withEpsilon(double epsilon) { + this.epsilon = epsilon; + return this; + } + + /** + * @return a new Equivalence using this builder's configuration. + */ + public Equivalence build() { + return new Equivalence(new Configuration(this)); + } + } + + private final Configuration configuration; + + private Equivalence(Configuration configuration) { + this.configuration = configuration; } @@ -166,13 +263,13 @@ private static int compareSymbolTokens(SymbolToken tok1, * and cannot contain duplicate elements, hence we cannot use it. */ private static final Map - convertToMultiSet(final IonStruct struct, final boolean strict) { + convertToMultiSet(final IonStruct struct, final Configuration configuration) { final Map structMultiSet = new HashMap(); for (final IonValue val : struct) { - final Field item = new Field(val, strict); + final Field item = new Field(val, configuration); Field curr = structMultiSet.put(item, item); // curr will be non-null if the multi-set already contains the // name/value pair @@ -193,7 +290,7 @@ private static int compareSymbolTokens(SymbolToken tok1, private static int compareStructs(final IonStruct s1, final IonStruct s2, - boolean strict) + final Configuration configuration) { int result = s1.size() - s2.size(); if (result == 0) { @@ -201,7 +298,7 @@ private static int compareStructs(final IonStruct s1, // Map). Refer to convertToMultiSet()'s // documentation for more info final Map s1MultiSet - = convertToMultiSet(s1, strict); + = convertToMultiSet(s1, configuration); // Iterates through each name/value pair in IonStruct s2 and // determine if it also occurs in s1MultiSet. @@ -209,7 +306,7 @@ private static int compareStructs(final IonStruct s1, // If it does, remove an occurrence from s1MultiSet // If it doesn't, the two IonStructs aren't equal for (IonValue val : s2) { - Field field = new Field(val, strict); + Field field = new Field(val, configuration); // Find an occurrence of the name/value pair in s1MultiSet Field mappedValue = s1MultiSet.get(field); @@ -231,7 +328,7 @@ private static int compareStructs(final IonStruct s1, private static int compareSequences(final IonSequence s1, final IonSequence s2, - boolean strict) + final Configuration configuration) { int result = s1.size() - s2.size(); if (result == 0) { @@ -240,7 +337,7 @@ private static int compareSequences(final IonSequence s1, while (iter1.hasNext()) { result = ionCompareToImpl(iter1.next(), iter2.next(), - strict); + configuration); if (result != 0) { break; } @@ -315,7 +412,7 @@ private static int compareLobContents(final IonLob lob1, final IonLob lob2) * a single {@code Field} -> {@code Field} with {@code occurrences} of 2. *

    * Refer to - * {@link Equivalence#convertToMultiSet(IonStruct, boolean)} and + * {@link Equivalence#convertToMultiSet(IonStruct, Configuration)} and * {@link Field#equals(Object)} for more info. *

    * NOTE: This class should only be instantiated for the sole purpose of @@ -324,7 +421,7 @@ private static int compareLobContents(final IonLob lob1, final IonLob lob2) static class Field { private final String name; // aka field name private final IonValue value; - private final boolean strict; + private final Configuration configuration; /** * Number of times that this specific field (with the same name @@ -332,7 +429,7 @@ static class Field { */ private int occurrences; - Field(final IonValue value, final boolean strict) + Field(final IonValue value, final Configuration configuration) { SymbolToken tok = value.getFieldNameSymbol(); String name = tok.getText(); @@ -342,7 +439,7 @@ static class Field { } this.name = name; this.value = value; - this.strict = strict; + this.configuration = configuration; // Occurrences of this name/value pair is 0 initially this.occurrences = 0; @@ -371,20 +468,20 @@ public boolean equals(final Object other) { final Field sOther = (Field) other; return name.equals(sOther.name) - && ionEqualsImpl(value, ((Field) other).value, strict); + && ionEqualsImpl(value, ((Field) other).value, configuration); } } private static boolean ionEqualsImpl(final IonValue v1, - final IonValue v2, - final boolean strict) + final IonValue v2, + final Configuration configuration) { - return (ionCompareToImpl(v1, v2, strict) == 0); + return (ionCompareToImpl(v1, v2, configuration) == 0); } private static int ionCompareToImpl(final IonValue v1, final IonValue v2, - final boolean strict) + final Configuration configuration) { int result = 0; @@ -431,8 +528,14 @@ private static int ionCompareToImpl(final IonValue v1, ((IonInt) v2).bigIntegerValue()); break; case FLOAT: - result = Double.compare(((IonFloat) v1).doubleValue(), - ((IonFloat) v2).doubleValue()); + double double1 = ((IonFloat) v1).doubleValue(); + double double2 = ((IonFloat) v2).doubleValue(); + if (configuration.epsilon != null + && (double1 == double2 || Math.abs(double1 - double2) <= configuration.epsilon)) { + result = 0; + } else { + result = Double.compare(double1, double2); + } break; case DECIMAL: assert !PUBLIC_COMPARISON_API; // TODO amzn/ion-java/issues/26 @@ -441,7 +544,7 @@ private static int ionCompareToImpl(final IonValue v1, ? 0 : 1; break; case TIMESTAMP: - if (strict) { + if (configuration.isStrict) { assert !PUBLIC_COMPARISON_API; // TODO amzn/ion-java/issues/26 result = (((IonTimestamp) v1).timestampValue().equals( ((IonTimestamp) v2).timestampValue()) @@ -473,14 +576,14 @@ private static int ionCompareToImpl(final IonValue v1, assert !PUBLIC_COMPARISON_API; // TODO amzn/ion-java/issues/26 result = compareStructs((IonStruct) v1, (IonStruct) v2, - strict); + configuration); break; case LIST: case SEXP: case DATAGRAM: result = compareSequences((IonSequence) v1, (IonSequence) v2, - strict); + configuration); break; } } @@ -488,7 +591,7 @@ private static int ionCompareToImpl(final IonValue v1, // if the values are otherwise equal, but the caller wants strict // comparison, then we check the annotations - if ((result == 0) && strict) { + if ((result == 0) && configuration.isStrict) { // check tuple equality over the annotations // (which are symbol tokens) result = compareAnnotations(v1.getTypeAnnotationSymbols(), @@ -511,7 +614,7 @@ private static int ionCompareToImpl(final IonValue v1, public static boolean ionEquals(final IonValue v1, final IonValue v2) { - return ionEqualsImpl(v1, v2, true); + return ionEqualsImpl(v1, v2, STRICT_CONFIGURATION); } /** @@ -529,7 +632,24 @@ public static boolean ionEquals(final IonValue v1, public static boolean ionEqualsByContent(final IonValue v1, final IonValue v2) { - return ionEqualsImpl(v1, v2, false); + return ionEqualsImpl(v1, v2, NON_STRICT_CONFIGURATION); + } + + /** + * Checks for data equivalence over two Ion values using this Equivalence's + * configuration + * + * @see Builder + * + * @param v1 + * The first Ion value to compare. + * @param v2 + * The second Ion value to compare. + * + * @return true if two Ion Values represent the same data. + */ + public boolean ionValueEquals(final IonValue v1, final IonValue v2) { + return ionEqualsImpl(v1, v2, configuration); } } diff --git a/test/com/amazon/ion/util/EquivalenceTest.java b/test/com/amazon/ion/util/EquivalenceTest.java index 676469187c..8a552ec1dc 100644 --- a/test/com/amazon/ion/util/EquivalenceTest.java +++ b/test/com/amazon/ion/util/EquivalenceTest.java @@ -16,6 +16,7 @@ package com.amazon.ion.util; import com.amazon.ion.IonFloat; +import com.amazon.ion.IonList; import com.amazon.ion.IonStruct; import com.amazon.ion.IonTestCase; import com.amazon.ion.IonValue; @@ -28,6 +29,9 @@ public class EquivalenceTest private static void assertIonEq(final IonValue left, final IonValue right) { assertTrue(Equivalence.ionEquals(left, right)); assertTrue(Equivalence.ionEquals(right, left)); + Equivalence equivalence = new Equivalence.Builder().build(); + assertTrue(equivalence.ionValueEquals(left, right)); + assertTrue(equivalence.ionValueEquals(right, left)); // Redundancy check included here, in the case that IonValue#equals() // doesn't use Equivalence's implementation anymore. @@ -40,6 +44,9 @@ private static void assertIonEq(final IonValue left, final IonValue right) { private static void assertNotIonEq(final IonValue left, final IonValue right) { assertFalse(Equivalence.ionEquals(left, right)); assertFalse(Equivalence.ionEquals(right, left)); + Equivalence equivalence = new Equivalence.Builder().build(); + assertFalse(equivalence.ionValueEquals(left, right)); + assertFalse(equivalence.ionValueEquals(right, left)); // Redundancy check included here, in the case that IonValue#equals() // doesn't use Equivalence's implementation anymore. @@ -52,6 +59,9 @@ private static void assertNotIonEq(final IonValue left, final IonValue right) { private static void assertIonEqForm(final IonValue left, final IonValue right) { assertTrue(Equivalence.ionEqualsByContent(left, right)); assertTrue(Equivalence.ionEqualsByContent(right, left)); + Equivalence equivalence = new Equivalence.Builder().withStrict(false).build(); + assertTrue(equivalence.ionValueEquals(left, right)); + assertTrue(equivalence.ionValueEquals(right, left)); } private IonValue ion(final String raw) { @@ -375,9 +385,10 @@ public void testFieldEquals1() { assertEquals(3, struct.size()); - Equivalence.Field f1 = new Equivalence.Field(v1, true); - Equivalence.Field f2 = new Equivalence.Field(v2, true); - Equivalence.Field f3 = new Equivalence.Field(v3, true); + Equivalence.Configuration configuration = new Equivalence.Configuration(new Equivalence.Builder().withStrict(true)); + Equivalence.Field f1 = new Equivalence.Field(v1, configuration); + Equivalence.Field f2 = new Equivalence.Field(v2, configuration); + Equivalence.Field f3 = new Equivalence.Field(v3, configuration); assertFalse(f1.equals(f2)); assertFalse(f1.equals(f3)); @@ -407,9 +418,10 @@ public void testFieldEquals2() { assertEquals(3, struct.size()); - Equivalence.Field f1 = new Equivalence.Field(v1, true); - Equivalence.Field f2 = new Equivalence.Field(v2, true); - Equivalence.Field f3 = new Equivalence.Field(v3, true); + Equivalence.Configuration configuration = new Equivalence.Configuration(new Equivalence.Builder().withStrict(true)); + Equivalence.Field f1 = new Equivalence.Field(v1, configuration); + Equivalence.Field f2 = new Equivalence.Field(v2, configuration); + Equivalence.Field f3 = new Equivalence.Field(v3, configuration); assertEquals(f1, f2); assertEquals(f2, f1); // symmetric @@ -420,4 +432,42 @@ public void testFieldEquals2() { assertEquals(f2, f3); // transitive assertEquals(f3, f2); // symmetric } + + @Test + public void builderWithoutEpsilon() { + Equivalence equivalence = new Equivalence.Builder().build(); + IonFloat v1 = ionFloat(3.14); + IonFloat v2 = ionFloat(3.14 + 1e-7); + assertFalse(equivalence.ionValueEquals(v1, v2)); + assertFalse(equivalence.ionValueEquals(v2, v1)); + IonStruct struct1 = system().newEmptyStruct(); + struct1.add("foo", v1.clone()); + IonStruct struct2 = system().newEmptyStruct(); + struct2.add("foo", v2.clone()); + assertFalse(equivalence.ionValueEquals(struct1, struct2)); + assertFalse(equivalence.ionValueEquals(struct2, struct1)); + IonList list1 = system().newList(v1.clone()); + IonList list2 = system().newList(v2.clone()); + assertFalse(equivalence.ionValueEquals(list1, list2)); + assertFalse(equivalence.ionValueEquals(list2, list1)); + } + + @Test + public void builderWithEpsilon() { + Equivalence equivalence = new Equivalence.Builder().withEpsilon(1e-6).build(); + IonFloat v1 = ionFloat(3.14); + IonFloat v2 = ionFloat(3.14 + 1e-7); + assertTrue(equivalence.ionValueEquals(v1, v2)); + assertTrue(equivalence.ionValueEquals(v2, v1)); + IonStruct struct1 = system().newEmptyStruct(); + struct1.add("foo", v1.clone()); + IonStruct struct2 = system().newEmptyStruct(); + struct2.add("foo", v2.clone()); + assertTrue(equivalence.ionValueEquals(struct1, struct2)); + assertTrue(equivalence.ionValueEquals(struct2, struct1)); + IonList list1 = system().newList(v1.clone()); + IonList list2 = system().newList(v2.clone()); + assertTrue(equivalence.ionValueEquals(list1, list2)); + assertTrue(equivalence.ionValueEquals(list2, list1)); + } } From 23b412f108279b38a03ada0d76cab4d8d07bfe77 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Fri, 31 May 2019 13:54:35 -0700 Subject: [PATCH 065/490] Makes IonRawBinaryWriter reuse ContainerInfo objects, reducing allocations and garbage. (#244) --- .../ion/impl/bin/IonRawBinaryWriter.java | 51 +++++++++++++------ 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java index 25d4e145a5..603d9e25cb 100644 --- a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java +++ b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java @@ -235,19 +235,20 @@ private ContainerType(final boolean allowedInStepOut) private static class ContainerInfo { /** Whether or not the container is a struct */ - public final ContainerType type; + public ContainerType type; /** The location of the pre-allocated size descriptor in the buffer. */ - public final long position; + public long position; /** The size of the current value. */ public long length; /** The patchlist for this container. */ public PatchList patches; - public ContainerInfo(final ContainerType type, final long offset) + public ContainerInfo() { - this.type = type; - this.position = offset; - this.patches = null; + type = null; + position = -1; + length = -1; + patches = null; } public void appendPatch(final PatchPoint patch) @@ -271,6 +272,13 @@ public void extendPatches(final PatchList newPatches) } } + public void initialize(final ContainerType type, final long offset) { + this.type = type; + this.position = offset; + this.patches = null; + this.length = 0; + } + @Override public String toString() { @@ -466,7 +474,8 @@ public String toString() private final WriteBuffer buffer; private final WriteBuffer patchBuffer; private final PatchList patchPoints; - private final LinkedList containers; + private final List containers; + private int currentContainerIndex; private int depth; private boolean hasWrittenValuesSinceFinished; private boolean hasWrittenValuesSinceConstructed; @@ -501,8 +510,10 @@ public String toString() this.buffer = new WriteBuffer(allocator); this.patchBuffer = new WriteBuffer(allocator); this.patchPoints = new PatchList(); - this.containers = new LinkedList(); - + this.containers = new ArrayList(10); + // Note: this is not the same as depth because ContainerInfo is used for annotations and certain scalars + // in addition to container types. + this.currentContainerIndex = -1; this.depth = 0; this.hasWrittenValuesSinceFinished = false; this.hasWrittenValuesSinceConstructed = false; @@ -651,23 +662,31 @@ public int getDepth() private void updateLength(long length) { - if (containers.isEmpty()) + if (currentContainerIndex < 0) { return; } - containers.getLast().length += length; + containers.get(currentContainerIndex).length += length; } private void pushContainer(final ContainerType type) { // XXX we push before writing the type of container - containers.add(new ContainerInfo(type, buffer.position() + 1)); + currentContainerIndex++; + ContainerInfo info; + if (currentContainerIndex >= containers.size()) { + info = new ContainerInfo(); + containers.add(info); + } else { + info = containers.get(currentContainerIndex); + } + info.initialize(type, buffer.position() + 1); } private ContainerInfo currentContainer() { - return containers.isEmpty() ? null : containers.getLast(); + return currentContainerIndex < 0 ? null : containers.get(currentContainerIndex); } private void addPatchPoint(final long position, final int oldLength, final long value) @@ -712,7 +731,7 @@ private ContainerInfo popContainer() { throw new IllegalStateException("Tried to pop container state without said container"); } - containers.removeLast(); + currentContainerIndex--; // only patch for real containers and annotations -- we use VALUE for tracking only final long length = current.length; @@ -876,7 +895,7 @@ public void stepOut() throws IOException public boolean isInStruct() { - return !containers.isEmpty() && currentContainer().type == ContainerType.STRUCT; + return currentContainerIndex >= 0 && currentContainer().type == ContainerType.STRUCT; } // Write Value Methods @@ -1499,7 +1518,7 @@ public void finish() throws IOException { return; } - if (!containers.isEmpty()) + if (currentContainerIndex >= 0 || depth > 0) { throw new IllegalStateException("Cannot finish within container: " + containers); } From 29176eac409a46f83d64688a07a1fee71a619a0b Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Fri, 31 May 2019 13:42:13 -0700 Subject: [PATCH 066/490] Makes IonRawBinaryWriter's currentFieldSid an int primitive instead of an Integer, cutting down on allocations and garbage caused by autoboxing. --- .../amazon/ion/impl/bin/IonRawBinaryWriter.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java index 603d9e25cb..ffc59f6935 100644 --- a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java +++ b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java @@ -465,6 +465,8 @@ public String toString() FLUSH } + private static final int SID_UNASSIGNED = -1; + private final BlockAllocator allocator; private final OutputStream out; private final StreamCloseMode streamCloseMode; @@ -480,7 +482,7 @@ public String toString() private boolean hasWrittenValuesSinceFinished; private boolean hasWrittenValuesSinceConstructed; - private Integer currentFieldSid; + private int currentFieldSid; private final List currentAnnotationSids; // XXX this is for managed detection of TLV that is a LST--this is easier to track here than at the managed level private boolean hasTopLevelSymbolTableAnnotation; @@ -518,7 +520,7 @@ public String toString() this.hasWrittenValuesSinceFinished = false; this.hasWrittenValuesSinceConstructed = false; - this.currentFieldSid = null; + this.currentFieldSid = SID_UNASSIGNED; this.currentAnnotationSids = new ArrayList(); this.hasTopLevelSymbolTableAnnotation = false; @@ -645,7 +647,7 @@ public IonCatalog getCatalog() public boolean isFieldNameSet() { - return currentFieldSid != null; + return currentFieldSid > SID_UNASSIGNED; } public void writeIonVersionMarker() throws IOException @@ -798,17 +800,17 @@ private static void checkSid(int sid) /** prepare to write values with field name and annotations. */ private void prepareValue() { - if (isInStruct() && currentFieldSid == null) + if (isInStruct() && currentFieldSid <= SID_UNASSIGNED) { throw new IllegalStateException("IonWriter.setFieldName() must be called before writing a value into a struct."); } - if (currentFieldSid != null) + if (currentFieldSid > SID_UNASSIGNED) { checkSid(currentFieldSid); writeVarUInt(currentFieldSid); // clear out field name - currentFieldSid = null; + currentFieldSid = SID_UNASSIGNED; } if (!currentAnnotationSids.isEmpty()) { @@ -873,7 +875,7 @@ public void stepIn(final IonType containerType) throws IOException public void stepOut() throws IOException { - if (currentFieldSid != null) + if (currentFieldSid > SID_UNASSIGNED) { throw new IonException("Cannot step out with field name set"); } From 51a53654138de5c80f2571bf58c2cd7b9008f169 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Fri, 31 May 2019 17:26:48 -0700 Subject: [PATCH 067/490] Refactors the containers stack into the RecyclingStack class, which caches the top element to avoid repetitive ArrayList.get calls in inner-loop code. --- .../ion/impl/bin/IonRawBinaryWriter.java | 143 +++++++++++++----- 1 file changed, 106 insertions(+), 37 deletions(-) diff --git a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java index ffc59f6935..617fa3e5ef 100644 --- a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java +++ b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java @@ -52,7 +52,6 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.Iterator; -import java.util.LinkedList; import java.util.List; import java.util.NoSuchElementException; @@ -465,6 +464,90 @@ public String toString() FLUSH } + /** + * A stack whose elements are recycled. This can be useful when the stack needs to grow and shrink + * frequently and has a predictable maximum depth. + * @param the type of elements stored. + */ + private static final class RecyclingStack { + + /** + * Factory for new stack elements. + * @param the type of element. + */ + public interface ElementFactory { + + /** + * @return a new instance. + */ + T newElement(); + } + + private final List elements; + private final ElementFactory elementFactory; + private int currentIndex; + private T top; + + /** + * @param initialCapacity the initial capacity of the underlying collection. + * @param elementFactory the factory used to create a new element on {@link #push()} when the stack has + * not previously grown to the new depth. + */ + public RecyclingStack(int initialCapacity, ElementFactory elementFactory) { + elements = new ArrayList(initialCapacity); + this.elementFactory = elementFactory; + currentIndex = -1; + top = null; + } + + /** + * Pushes an element onto the top of the stack, instantiating a new element only if the stack has not + * previously grown to the new depth. + * @return the element at the top of the stack after the push. This element must be initialized by the caller. + */ + public T push() { + currentIndex++; + if (currentIndex >= elements.size()) { + top = elementFactory.newElement(); + elements.add(top); + } else { + top = elements.get(currentIndex); + } + return top; + } + + /** + * @return the element at the top of the stack, or null if the stack is empty. + */ + public T peek() { + return top; + } + + /** + * Pops an element from the stack, retaining a reference to the element so that it can be reused the + * next time the stack grows to the element's depth. + * @return the element that was at the top of the stack before the pop, or null if the stack was empty. + */ + public T pop() { + T popped = top; + currentIndex--; + if (currentIndex >= 0) { + top = elements.get(currentIndex); + } else { + top = null; + currentIndex = -1; + } + return popped; + } + + /** + * @return true if the stack is empty; otherwise, false. + */ + public boolean isEmpty() { + return top == null; + } + } + private static final int SID_UNASSIGNED = -1; private final BlockAllocator allocator; @@ -476,8 +559,7 @@ public String toString() private final WriteBuffer buffer; private final WriteBuffer patchBuffer; private final PatchList patchPoints; - private final List containers; - private int currentContainerIndex; + private final RecyclingStack containers; private int depth; private boolean hasWrittenValuesSinceFinished; private boolean hasWrittenValuesSinceConstructed; @@ -512,10 +594,15 @@ public String toString() this.buffer = new WriteBuffer(allocator); this.patchBuffer = new WriteBuffer(allocator); this.patchPoints = new PatchList(); - this.containers = new ArrayList(10); - // Note: this is not the same as depth because ContainerInfo is used for annotations and certain scalars - // in addition to container types. - this.currentContainerIndex = -1; + this.containers = new RecyclingStack( + 10, + new RecyclingStack.ElementFactory() { + @Override + public ContainerInfo newElement() { + return new ContainerInfo(); + } + } + ); this.depth = 0; this.hasWrittenValuesSinceFinished = false; this.hasWrittenValuesSinceConstructed = false; @@ -664,31 +751,18 @@ public int getDepth() private void updateLength(long length) { - if (currentContainerIndex < 0) + if (containers.isEmpty()) { return; } - containers.get(currentContainerIndex).length += length; + containers.peek().length += length; } private void pushContainer(final ContainerType type) { // XXX we push before writing the type of container - currentContainerIndex++; - ContainerInfo info; - if (currentContainerIndex >= containers.size()) { - info = new ContainerInfo(); - containers.add(info); - } else { - info = containers.get(currentContainerIndex); - } - info.initialize(type, buffer.position() + 1); - } - - private ContainerInfo currentContainer() - { - return currentContainerIndex < 0 ? null : containers.get(currentContainerIndex); + containers.push().initialize(type, buffer.position() + 1); } private void addPatchPoint(final long position, final int oldLength, final long value) @@ -697,8 +771,7 @@ private void addPatchPoint(final long position, final int oldLength, final long final long patchPosition = patchBuffer.position(); final int patchLength = patchBuffer.writeVarUInt(value); final PatchPoint patch = new PatchPoint(position, oldLength, patchPosition, patchLength); - final ContainerInfo container = currentContainer(); - if (container == null) + if (containers.isEmpty()) { // not nested, just append to the root list patchPoints.append(patch); @@ -706,15 +779,14 @@ private void addPatchPoint(final long position, final int oldLength, final long else { // nested, apply it to the current container - container.appendPatch(patch); + containers.peek().appendPatch(patch); } updateLength(patchLength - oldLength); } private void extendPatchPoints(final PatchList patches) { - final ContainerInfo container = currentContainer(); - if (container == null) + if (containers.isEmpty()) { // not nested, extend root list patchPoints.extend(patches); @@ -722,18 +794,17 @@ private void extendPatchPoints(final PatchList patches) else { // nested, apply it to the current container - container.extendPatches(patches); + containers.peek().extendPatches(patches); } } private ContainerInfo popContainer() { - final ContainerInfo current = currentContainer(); + final ContainerInfo current = containers.pop(); if (current == null) { throw new IllegalStateException("Tried to pop container state without said container"); } - currentContainerIndex--; // only patch for real containers and annotations -- we use VALUE for tracking only final long length = current.length; @@ -848,8 +919,7 @@ private void prepareValue() /** Closes out annotations. */ private void finishValue() { - final ContainerInfo current = currentContainer(); - if (current != null && current.type == ContainerType.ANNOTATION) + if (!containers.isEmpty() && containers.peek().type == ContainerType.ANNOTATION) { // close out and patch the length popContainer(); @@ -883,8 +953,7 @@ public void stepOut() throws IOException { throw new IonException("Cannot step out with field name set"); } - final ContainerInfo container = currentContainer(); - if (container == null || !container.type.allowedInStepOut) + if (containers.isEmpty() || !containers.peek().type.allowedInStepOut) { throw new IonException("Cannot step out when not in container"); } @@ -897,7 +966,7 @@ public void stepOut() throws IOException public boolean isInStruct() { - return currentContainerIndex >= 0 && currentContainer().type == ContainerType.STRUCT; + return !containers.isEmpty() && containers.peek().type == ContainerType.STRUCT; } // Write Value Methods @@ -1520,7 +1589,7 @@ public void finish() throws IOException { return; } - if (currentContainerIndex >= 0 || depth > 0) + if (!containers.isEmpty() || depth > 0) { throw new IllegalStateException("Cannot finish within container: " + containers); } From 6ef0342d11778649b5f29191293d071d2fa96420 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Tue, 4 Jun 2019 19:29:58 -0700 Subject: [PATCH 068/490] Removes @Override annotation not allowed by old JDK versions. (#249) JDK 5 won't compile @Override annotations on methods that implement an interface. --- src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java index 617fa3e5ef..23ced039c6 100644 --- a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java +++ b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java @@ -597,7 +597,6 @@ public boolean isEmpty() { this.containers = new RecyclingStack( 10, new RecyclingStack.ElementFactory() { - @Override public ContainerInfo newElement() { return new ContainerInfo(); } From 0873d5c1429a3bf6365d56ae2ae8461bd3865289 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Tue, 25 Jun 2019 17:27:51 -0700 Subject: [PATCH 069/490] Release 1.5.0 --- .travis.yml | 1 - README.md | 2 +- pom.xml | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index f1dd7898a3..96dc30e864 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,6 @@ jdk: - openjdk11 - oraclejdk8 - oraclejdk9 - - oraclejdk11 script: mvn test diff --git a/README.md b/README.md index a3f8d729eb..06a886255f 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ dependency into your project's `pom.xml`: com.amazon.ion ion-java - 1.4.0 + 1.5.0 ``` diff --git a/pom.xml b/pom.xml index 59e2f750a7..062a2fa8cc 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.5.0-SNAPSHOT + 1.5.0 bundle ${project.groupId}:${project.artifactId} From 447bd74eb71cdab1432b93f45ee30dc87ec6f4cc Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Wed, 26 Jun 2019 13:24:45 -0700 Subject: [PATCH 070/490] Bumps version to 1.5.1-SNAPSHOT. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 062a2fa8cc..648e241f1c 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.5.0 + 1.5.1-SNAPSHOT bundle ${project.groupId}:${project.artifactId} From 157d47441df57cd11b8fc10526142ff798e44a08 Mon Sep 17 00:00:00 2001 From: David Lurton Date: Mon, 29 Jul 2019 16:09:57 -0700 Subject: [PATCH 071/490] Manually specify the trusty distro --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 96dc30e864..c40e335f83 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,6 @@ sudo: false language: java +dist: trusty jdk: - openjdk8 - openjdk9 @@ -24,4 +25,4 @@ jobs: github-token: "$GITHUB_TOKEN" keep-history: true # keeps commit history of gh-pages branch on: - branch: master \ No newline at end of file + branch: master From 1c6f20e18f79004b6a1b8bc003202af32de6c27f Mon Sep 17 00:00:00 2001 From: Dmitry Zvorygin Date: Tue, 30 Jul 2019 15:44:48 -0700 Subject: [PATCH 072/490] Add IonSystem#singleValue(byte[], int, int) (#257) When ion object resides in the middle of bytearray, this helper method would help to extract it without memory copy overhead. --- src/com/amazon/ion/IonSystem.java | 21 ++++++++ .../amazon/ion/impl/lite/IonSystemLite.java | 7 ++- test/com/amazon/ion/LoaderTest.java | 49 +++++++++++++++++++ 3 files changed, 76 insertions(+), 1 deletion(-) diff --git a/src/com/amazon/ion/IonSystem.java b/src/com/amazon/ion/IonSystem.java index 56871c9ec7..f7bb29c038 100644 --- a/src/com/amazon/ion/IonSystem.java +++ b/src/com/amazon/ion/IonSystem.java @@ -373,6 +373,27 @@ public SymbolTable newSharedSymbolTable(String name, */ public IonValue singleValue(byte[] ionData); + /** + * Extracts a single value from Ion text or binary data. + *

    + * This method will auto-detect and uncompress GZIPped Ion data. + * + * @param ionData is used only within the range of bytes starting at + * {@code offset} for {@code len} bytes. + * The data in that range may be either Ion binary data, or UTF-8 Ion text. + * @param offset must be non-negative and less than {@code ionData.length}. + * @param len must be non-negative and {@code offset+len} must not exceed + * + * @return the first (and only) user value in the data; not null. + * + * @throws NullPointerException if {@code ionData} is null. + * @throws UnexpectedEofException if the data doesn't contain any user + * values. + * @throws IonException if the data does not contain exactly one user + * value. + */ + public IonValue singleValue(byte[] ionData, int offset, int len); + //------------------------------------------------------------------------- // IonReader creation diff --git a/src/com/amazon/ion/impl/lite/IonSystemLite.java b/src/com/amazon/ion/impl/lite/IonSystemLite.java index 1e16f7392b..95bfa4d08a 100644 --- a/src/com/amazon/ion/impl/lite/IonSystemLite.java +++ b/src/com/amazon/ion/impl/lite/IonSystemLite.java @@ -523,7 +523,12 @@ public IonValue singleValue(String ionText) public IonValue singleValue(byte[] ionData) { - IonReader reader = newReader(ionData); + return singleValue(ionData, 0, ionData.length); + } + + @Override + public IonValue singleValue(byte[] ionData, int offset, int len) { + IonReader reader = newReader(ionData, offset, len); try { Iterator it = iterate(reader); return singleValue(it); diff --git a/test/com/amazon/ion/LoaderTest.java b/test/com/amazon/ion/LoaderTest.java index e00685d27f..fc020077a4 100644 --- a/test/com/amazon/ion/LoaderTest.java +++ b/test/com/amazon/ion/LoaderTest.java @@ -32,6 +32,8 @@ import java.io.StringReader; import java.util.Arrays; import java.util.Iterator; + +import org.junit.Assert; import org.junit.Test; @@ -393,6 +395,53 @@ public void testSingleValue() catch (IonException ie) { /* ok */ } } + @Test + public void testSubBufferSingleValue() + { + IonSystem sys = system(); + + String image = "(this is a single sexp)"; + byte[] bytes = sys.newLoader().load(image).getBytes(); + + IonValue singleValue = sys.singleValue(bytes, 0, bytes.length); + assertEquals(singleValue.toString(), image); + + byte[] padded = new byte[bytes.length + 20]; + System.arraycopy(bytes, 0, padded, 10, bytes.length); + + IonValue paddedValue = sys.singleValue(padded, 10, bytes.length); + assertEquals(paddedValue.toString(), image); + + bytes = sys.newLoader().load("(two) (values)").getBytes(); + + try { + sys.singleValue(bytes, 0, bytes.length); + fail("Expected IonException"); + } + catch (IonException ie) { + Assert.assertTrue(ie.getMessage().contains("not a single value")); + } + + padded = new byte[bytes.length + 20]; + System.arraycopy(bytes, 0, padded, 10, bytes.length); + + try { + sys.singleValue(padded, 10, bytes.length); + fail("Expected IonException"); + } + catch (IonException ie) { + Assert.assertTrue(ie.getMessage().contains("not a single value")); + } + + try { + sys.singleValue(padded, 15, bytes.length); + fail("Expected IonException"); + } + catch (IonException ie) { + Assert.assertTrue(ie.getMessage().contains("bad character")); + } + } + @Test public void testCatalogOnLoader() throws Exception { IonSystem sys = newSystem(Symtabs.CATALOG); From 389241fead069414be752574dea8e65adf127deb Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Tue, 24 Sep 2019 14:18:01 -0700 Subject: [PATCH 073/490] Fixes a bug in IonReaderBinaryRawX that caused the reader to prematurely finish reading large streams in certain cases. --- .../amazon/ion/impl/IonReaderBinaryRawX.java | 15 +++- test/AllTests.java | 2 + .../IonReaderBinaryRawLargeStreamTest.java | 73 +++++++++++++++++++ .../amazon/ion/util/RepeatInputStream.java | 4 +- 4 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 test/com/amazon/ion/impl/IonReaderBinaryRawLargeStreamTest.java diff --git a/src/com/amazon/ion/impl/IonReaderBinaryRawX.java b/src/com/amazon/ion/impl/IonReaderBinaryRawX.java index 1ef4ffde21..c5ece958f9 100644 --- a/src/com/amazon/ion/impl/IonReaderBinaryRawX.java +++ b/src/com/amazon/ion/impl/IonReaderBinaryRawX.java @@ -1064,7 +1064,10 @@ protected final Decimal readDecimal(int len) throws IOException } else { // otherwise we to it the hard way .... - int save_limit = _local_remaining - len; + int save_limit = NO_LIMIT; + if (_local_remaining != NO_LIMIT) { + save_limit = _local_remaining - len; + } _local_remaining = len; int exponent = readVarInt(); BigInteger value; @@ -1114,7 +1117,10 @@ protected final Timestamp readTimestamp(int len) throws IOException int year = 0, month = 0, day = 0, hour = 0, minute = 0, second = 0; BigDecimal frac = null; - int save_limit = _local_remaining - len; + int save_limit = NO_LIMIT; + if (_local_remaining != NO_LIMIT) { + save_limit = _local_remaining - len; + } _local_remaining = len; // > 0 // first up is the offset, which requires a special int reader @@ -1182,7 +1188,10 @@ protected final String readString(int len) throws IOException // the char array is way faster than using string buffer char[] chars = new char[len]; int c, ii = 0; - int save_limit = _local_remaining - len; + int save_limit = NO_LIMIT; + if (_local_remaining != NO_LIMIT) { + save_limit = _local_remaining - len; + } _local_remaining = len; while (!isEOF()) { c = readUnicodeScalar(); diff --git a/test/AllTests.java b/test/AllTests.java index 326611b69a..0e433b5b8b 100644 --- a/test/AllTests.java +++ b/test/AllTests.java @@ -48,6 +48,7 @@ import com.amazon.ion.NopPaddingTest; import com.amazon.ion.NullTest; import com.amazon.ion.RawValueSpanReaderBasicTest; +import com.amazon.ion.impl.IonReaderBinaryRawLargeStreamTest; import com.amazon.ion.impl.RawValueSpanReaderTest; import com.amazon.ion.RoundTripTest; import com.amazon.ion.SexpTest; @@ -206,6 +207,7 @@ IonReaderToIonValueTest.class, BinaryReaderWrappedValueLengthTest.class, IonReaderBuilderTest.class, + IonReaderBinaryRawLargeStreamTest.class, // experimental binary writer tests PooledBlockAllocatorProviderTest.class, diff --git a/test/com/amazon/ion/impl/IonReaderBinaryRawLargeStreamTest.java b/test/com/amazon/ion/impl/IonReaderBinaryRawLargeStreamTest.java new file mode 100644 index 0000000000..9ed965105a --- /dev/null +++ b/test/com/amazon/ion/impl/IonReaderBinaryRawLargeStreamTest.java @@ -0,0 +1,73 @@ +package com.amazon.ion.impl; + +import com.amazon.ion.IonReader; +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.Timestamp; +import com.amazon.ion.system.IonBinaryWriterBuilder; +import com.amazon.ion.system.IonReaderBuilder; +import com.amazon.ion.util.RepeatInputStream; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.math.BigDecimal; + +import static com.amazon.ion.impl._Private_IonConstants.BINARY_VERSION_MARKER_1_0; +import static org.junit.Assert.assertEquals; + +public class IonReaderBinaryRawLargeStreamTest { + + // NOTE: this test takes several seconds to complete. + @Test + public void testReadLargeScalarStream() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IonWriter writer = IonBinaryWriterBuilder.standard().build(out); + final Timestamp timestamp = Timestamp.forDay(2000, 1, 1); + writer.writeString("foo"); + writer.writeDecimal(BigDecimal.TEN); + writer.writeTimestamp(timestamp); + writer.close(); + byte[] dataWithIvm = out.toByteArray(); + // Strip the IVM, as this needs to be one continuous stream to avoid resetting the reader's internals. + byte[] data = new byte[dataWithIvm.length - BINARY_VERSION_MARKER_1_0.length]; + System.arraycopy(dataWithIvm, BINARY_VERSION_MARKER_1_0.length, data, 0, data.length); + // The binary reader uses Integer.MIN_VALUE to mean NO_LIMIT for its _local_remaining value, which keeps track + // of the remaining number of bytes in the current value. Between values at the top level, this should always be + // NO_LIMIT. No arithmetic should ever be performed on the value when it is set to NO_LIMIT. If bugs exist that + // violate this, then between top level values _local_remaining will never again be NO_LIMIT, meaning that + // arithmetic will continue to be performed on it. Eventually, due to integer overflow, the value will roll over + // into a small enough positive value that the reader will erroneously determine that there are fewer bytes + // remaining than are needed to complete the current value. The reader will then finish early before reading the + // entire stream. The bug that prompted this test to be written involved an unconditional subtraction of the + // current value's length as declared in its header from the current value of _local_remaining within + // stringValue(), decimalValue(), and timestampValue(). This caused _local_remaining to overflow to a very + // large value immediately. For every top level value subsequently read, the length of that value would be + // subtracted from _local_remaining until eventually _local_remaining prematurely reached 0 around the time + // the stream reached Integer.MAX_VALUE in length. + // Repeat the batch a sufficient number of times to exceed a total stream length of Integer.MAX_VALUE, plus + // a few more to make sure batches continue to be read correctly. + final int totalNumberOfBatches = (Integer.MAX_VALUE / data.length) + 7; + InputStream inputStream = new SequenceInputStream( + new ByteArrayInputStream(BINARY_VERSION_MARKER_1_0), + new RepeatInputStream(data, totalNumberOfBatches - 1) // This will provide the data 'totalNumberOfBatches' times + ); + IonReader reader = IonReaderBuilder.standard().build(inputStream); + reader.next(); + assertEquals("foo", reader.stringValue()); + reader.next(); + assertEquals(BigDecimal.TEN, reader.decimalValue()); + reader.next(); + assertEquals(timestamp, reader.timestampValue()); + int batchesRead = 1; + while (reader.next() != null) { + assertEquals(IonType.STRING, reader.getType()); + assertEquals(IonType.DECIMAL, reader.next()); + assertEquals(IonType.TIMESTAMP, reader.next()); + batchesRead++; + } + assertEquals(totalNumberOfBatches, batchesRead); + } +} diff --git a/test/com/amazon/ion/util/RepeatInputStream.java b/test/com/amazon/ion/util/RepeatInputStream.java index f5bae1c206..c9e254af42 100644 --- a/test/com/amazon/ion/util/RepeatInputStream.java +++ b/test/com/amazon/ion/util/RepeatInputStream.java @@ -33,7 +33,9 @@ public final class RepeatInputStream /** * @param bytes the data to be repeated. - * @param times the number of times to repeat the data. + * @param times the number of times to repeat the data. NOTE: this is the number of times the bytes will be + * repeated after the first time they are read. For example, providing a byte array with a single + * byte 'a' and times=1 will allow 'a' to be read twice before the stream signals EOF. */ public RepeatInputStream(byte[] bytes, long times) { From 00b62fdd7d0210327d2272d205f9d3b959651bec Mon Sep 17 00:00:00 2001 From: Therapon Skoteiniotis Date: Wed, 25 Sep 2019 13:26:52 -0700 Subject: [PATCH 074/490] Release 1.5.1 (#262) --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 06a886255f..17b4327909 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ dependency into your project's `pom.xml`: com.amazon.ion ion-java - 1.5.0 + 1.5.1 ``` diff --git a/pom.xml b/pom.xml index 648e241f1c..051cdd3cb9 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.5.1-SNAPSHOT + 1.5.1 bundle ${project.groupId}:${project.artifactId} From 88ed4c5477a0e84fc4a404df0910a2b173bdd278 Mon Sep 17 00:00:00 2001 From: Therapon Skoteiniotis Date: Fri, 27 Sep 2019 10:03:04 -0700 Subject: [PATCH 075/490] Prepare v1.5.2 SNAPSHOT (#263) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 051cdd3cb9..b212673a52 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.5.1 + 1.5.2-SNAPSHOT bundle ${project.groupId}:${project.artifactId} From 6112773c5f62fb23c2b33f38909add07c5c9072c Mon Sep 17 00:00:00 2001 From: Therapon Skoteiniotis Date: Thu, 31 Oct 2019 12:34:34 -0700 Subject: [PATCH 076/490] Move to new sonatype URL (#265) --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index b212673a52..51e20d3858 100644 --- a/pom.xml +++ b/pom.xml @@ -53,11 +53,11 @@ ossrh - https://oss.sonatype.org/content/repositories/snapshots + https://aws.oss.sonatype.org/content/repositories/snapshots ossrh - https://oss.sonatype.org/service/local/staging/deploy/maven2/ + https://aws.oss.sonatype.org/service/local/staging/deploy/maven2/ From 1673177920347ad9fae69a187ea513f5ffbc1050 Mon Sep 17 00:00:00 2001 From: Zack Slayton Date: Thu, 21 Nov 2019 16:47:30 -0500 Subject: [PATCH 077/490] Optimizations for reading binary strings. (#267) Optimizations for reading binary strings. --- .../amazon/ion/impl/IonReaderBinaryRawX.java | 176 +++++++++++------- .../impl/IonReaderBinaryRawStringTest.java | 97 ++++++++++ 2 files changed, 208 insertions(+), 65 deletions(-) create mode 100644 test/com/amazon/ion/impl/IonReaderBinaryRawStringTest.java diff --git a/src/com/amazon/ion/impl/IonReaderBinaryRawX.java b/src/com/amazon/ion/impl/IonReaderBinaryRawX.java index c5ece958f9..5bbc543503 100644 --- a/src/com/amazon/ion/impl/IonReaderBinaryRawX.java +++ b/src/com/amazon/ion/impl/IonReaderBinaryRawX.java @@ -27,10 +27,16 @@ import com.amazon.ion.impl.UnifiedSavePointManagerX.SavePoint; import com.amazon.ion.impl._Private_ScalarConversions.AS_TYPE; import com.amazon.ion.impl._Private_ScalarConversions.ValueVariant; + import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.math.MathContext; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CoderResult; /** @@ -52,6 +58,8 @@ abstract class IonReaderBinaryRawX static final int DEFAULT_CONTAINER_STACK_SIZE = 12; // a multiple of 3 static final int DEFAULT_ANNOTATION_SIZE = 10; static final int NO_LIMIT = Integer.MIN_VALUE; + static final int UTF8_BUFFER_SIZE_IN_BYTES = 4 * 1024; + protected enum State { S_INVALID, S_BEFORE_FIELD, // only true in structs @@ -96,6 +104,19 @@ protected enum State { int _container_top; long[] _container_stack; // triples of: position, type, local_end + + // `StandardCharsets.UTF_8` wasn't introduced until Java 7, so we have to use Charset#forName(String) instead. + private static final Charset UTF8 = Charset.forName("UTF-8"); + private CharsetDecoder utf8CharsetDecoder = UTF8.newDecoder(); + + // Calling read() to pull in the next byte of a string requires an EOF check to be performed for each byte. + // This reusable buffer allows us to call read(utf8InputBuffer) instead, letting us can pay the cost of an EOF check + // once per buffer rather than once per byte. + private ByteBuffer utf8InputBuffer = ByteBuffer.allocate(UTF8_BUFFER_SIZE_IN_BYTES); + + // A reusable scratch space to hold the decoded bytes as they're read from the utf8InputBuffer. + private CharBuffer utf8DecodingBuffer = CharBuffer.allocate(UTF8_BUFFER_SIZE_IN_BYTES); + protected IonReaderBinaryRawX() { } @@ -1181,81 +1202,106 @@ protected final Timestamp readTimestamp(int len) throws IOException } } - protected final String readString(int len) throws IOException + protected final String readString(int numberOfBytes) throws IOException { - // len is bytes, which is greater than or equal to java - // chars even after utf8 to utf16 decoding nonsense - // the char array is way faster than using string buffer - char[] chars = new char[len]; - int c, ii = 0; - int save_limit = NO_LIMIT; - if (_local_remaining != NO_LIMIT) { - save_limit = _local_remaining - len; + // If the string we're reading is small enough to fit in our reusable buffer, we can avoid the overhead + // of looping and bounds checking. + if (numberOfBytes <= utf8InputBuffer.capacity()) { + return readStringWithReusableBuffer(numberOfBytes); } - _local_remaining = len; - while (!isEOF()) { - c = readUnicodeScalar(); - if (c < 0) throwUnexpectedEOFException(); - if (c < 0x10000) { - chars[ii++] = (char)c; + + // Otherwise, allocate a one-off decoding buffer that's large enough to hold the string + // and prepare to decode the string in chunks. + CharBuffer decodingBuffer = CharBuffer.allocate(numberOfBytes); + utf8CharsetDecoder.reset(); + + int save_limit = NO_LIMIT; + if (_local_remaining != NO_LIMIT) { + save_limit = _local_remaining - numberOfBytes; + } + _local_remaining = numberOfBytes; + + // The following loop will: + // * Fill the input buffer with utf8 bytes + // * Write decoded chars to the decoding buffer + // * Move any remaining partial character bytes to the front of the buffer + // * Repeat until the requested number of bytes have been decoded. + // * Create a new String object from the contents of the decoding buffer. + int totalBytesRead = 0; + int carryoverBytes = 0; + while (totalBytesRead < numberOfBytes) { + + int bytesRemaining = numberOfBytes - totalBytesRead; + // When decoding, it's possible to have 'carryover' bytes left in the buffer which represent + // a partial unicode character. We need to leave these in the buffer so that we can complete the + // partial character in the next call to read(). + int capacityRemaining = utf8InputBuffer.array().length - carryoverBytes; + int bytesToRead = Math.min(bytesRemaining, capacityRemaining); + + int bytesRead = read(utf8InputBuffer.array(), carryoverBytes, bytesToRead); + if (bytesRead <= 0) { + // UnifiedInputStreamX doesn't adhere to InputStream's API. See the comments on + // UnifiedInputStreamX#read() for more information. + throwUnexpectedEOFException(); } - else { // when c is >= 0x10000 we need surrogate encoding - chars[ii++] = (char)_Private_IonConstants.makeHighSurrogate(c); - chars[ii++] = (char)_Private_IonConstants.makeLowSurrogate(c); + + totalBytesRead += bytesRead; + + utf8InputBuffer.position(0); + utf8InputBuffer.limit(carryoverBytes + bytesRead); + + CoderResult coderResult = utf8CharsetDecoder.decode( + utf8InputBuffer, + decodingBuffer, + totalBytesRead >= numberOfBytes + ); + if (coderResult.isError()) { + throw new IonException("Illegal value encountered while validating UTF-8 data in input stream. " + coderResult.toString()); + } + + // Shift leftover partial character bytes (if any) to the beginning of the buffer + carryoverBytes = utf8InputBuffer.remaining(); + if (carryoverBytes > 0) { + System.arraycopy( + utf8InputBuffer.array(), + utf8InputBuffer.position(), + utf8InputBuffer.array(), + 0, + carryoverBytes + ); } } + _local_remaining = save_limit; - return new String(chars, 0, ii); + + decodingBuffer.flip(); + return decodingBuffer.toString(); } - private final int readUnicodeScalar() throws IOException - { - int c = -1, b; - b = read(); - // ascii is all good, even -1 (eof) - if (IonUTF8.isOneByteUTF8(b)) { - return b; + + private String readStringWithReusableBuffer(int numberOfBytes) throws IOException { + int save_limit = NO_LIMIT; + if (_local_remaining != NO_LIMIT) { + save_limit = _local_remaining - numberOfBytes; } - switch(IonUTF8.getUTF8LengthFromFirstByte(b)) { - case 2: - // now we start gluing the multi-byte value together - assert((b & 0xe0) == 0xc0); - // for values from 0x80 to 0x7FF (all legal) - int b2 = read(); - if (!IonUTF8.isContinueByteUTF8(b2)) throwUTF8Exception(); - c = IonUTF8.twoByteScalar(b, b2); - break; - case 3: - assert((b & 0xf0) == 0xe0); - // for values from 0x800 to 0xFFFFF (NOT all legal) - b2 = read(); - if (!IonUTF8.isContinueByteUTF8(b2)) throwUTF8Exception(); - int b3 = read(); - if (!IonUTF8.isContinueByteUTF8(b3)) throwUTF8Exception(); - c = IonUTF8.threeByteScalar(b, b2, b3); - break; - case 4: - assert((b & 0xf8) == 0xf0); - // for values from 0x010000 to 0x1FFFFF (NOT all legal) - b2 = read(); - if (!IonUTF8.isContinueByteUTF8(b2)) throwUTF8Exception(); - b3 = read(); - if (!IonUTF8.isContinueByteUTF8(b3)) throwUTF8Exception(); - int b4 = read(); - if (!IonUTF8.isContinueByteUTF8(b4)) throwUTF8Exception(); - c = IonUTF8.fourByteScalar(b, b2, b3, b4); - if (c > 0x10FFFF) { - throw new IonException("illegal utf value encountered in input utf-8 stream"); - } - break; - default: - throwUTF8Exception(); + _local_remaining = numberOfBytes; + readAll(utf8InputBuffer.array(), 0, numberOfBytes); + _local_remaining = save_limit; + + utf8InputBuffer.position(0); + utf8InputBuffer.limit(numberOfBytes); + + utf8DecodingBuffer.position(0); + utf8DecodingBuffer.limit(utf8DecodingBuffer.capacity()); + + utf8CharsetDecoder.reset(); + CoderResult coderResult = utf8CharsetDecoder.decode(utf8InputBuffer, utf8DecodingBuffer, true); + if (coderResult.isError()) { + throw new IonException("Illegal value encountered while validating UTF-8 data in input stream. " + coderResult.toString()); } - return c; - } - private final void throwUTF8Exception() throws IOException - { - throwErrorAt("Invalid UTF-8 character encounter in a string at position "); + utf8DecodingBuffer.flip(); + return utf8DecodingBuffer.toString(); } + private final void throwUnexpectedEOFException() throws IOException { throwErrorAt("unexpected EOF in value"); } diff --git a/test/com/amazon/ion/impl/IonReaderBinaryRawStringTest.java b/test/com/amazon/ion/impl/IonReaderBinaryRawStringTest.java new file mode 100644 index 0000000000..3ad6e86938 --- /dev/null +++ b/test/com/amazon/ion/impl/IonReaderBinaryRawStringTest.java @@ -0,0 +1,97 @@ +package com.amazon.ion.impl; + +import com.amazon.ion.IonReader; +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.Timestamp; +import com.amazon.ion.system.IonBinaryWriterBuilder; +import com.amazon.ion.system.IonReaderBuilder; +import com.amazon.ion.util.RepeatInputStream; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.math.BigDecimal; + +import static com.amazon.ion.impl._Private_IonConstants.BINARY_VERSION_MARKER_1_0; +import static org.junit.Assert.assertEquals; + +public class IonReaderBinaryRawStringTest { + + @Test + public void testReadShortStrings() throws Exception { + // This test constructs some strings with non-ascii text that are short enough to fit in + // IonBinaryReaderRawX's reusable decoding buffers and then round-trips them. + + String adage = "Brevity is the soul of wit. \uD83D\uDE02"; // Laughing face with tears + String observation = "What's the deal with airline food? \u2708\uFE0F\uD83C\uDF7Dī¸"; // Airplane/fork+knife+plate + String litotes = "Not bad. \uD83D\uDC4D"; // Thumbs up + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IonWriter writer = IonBinaryWriterBuilder.standard().build(out); + + writer.writeString(adage); + writer.writeString(observation); + writer.writeString(litotes); + writer.close(); + + byte[] data = out.toByteArray(); + + IonReader reader = IonReaderBuilder.standard().build(data); + reader.next(); + assertEquals(adage, reader.stringValue()); + reader.next(); + assertEquals(observation, reader.stringValue()); + reader.next(); + assertEquals(litotes, reader.stringValue()); + } + + @Test + public void testReadLargeStrings() throws Exception { + // This test constructs some strings with non-ascii text that are large enough to exceed the size + // of IonBinaryReaderRawX's reusable decoding buffers and then round-trips them. + + String verse = "\uD83C\uDFB6 This is the song that never ends\n" // Musical notes emoji + + "\uD83C\uDFB5 Yes it goes on and on, my friends!\n" + + "\uD83C\uDFB6 Some people started singing it not knowing what it was\n" + + "\uD83C\uDFB5 And they'll continue singing it forever just because...\n"; + + final int numberOfVerses = 1024; + + StringBuilder stringBuilder = new StringBuilder(); + for (int i = 0; i < numberOfVerses; i++) { + stringBuilder.append(verse); + } + String longSong = stringBuilder.toString(); + + for (int i = 0; i < numberOfVerses; i++) { + stringBuilder.append(verse); + } + String longerSong = stringBuilder.toString(); + + for (int i = 0; i < numberOfVerses; i++) { + stringBuilder.append(verse); + } + String longestSong = stringBuilder.toString(); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IonWriter writer = IonBinaryWriterBuilder.standard().build(out); + + writer.writeString(longSong); + writer.writeString(longerSong); + writer.writeString(longestSong); + writer.close(); + + byte[] songData = out.toByteArray(); + + IonReader reader = IonReaderBuilder.standard().build(songData); + reader.next(); + assertEquals(longSong, reader.stringValue()); + reader.next(); + assertEquals(longerSong, reader.stringValue()); + reader.next(); + assertEquals(longestSong, reader.stringValue()); + } +} From 7cb5c0b7f8c0d0503b7c63b0416a2099f8cfcda7 Mon Sep 17 00:00:00 2001 From: Zack Slayton Date: Tue, 26 Nov 2019 07:47:34 -0500 Subject: [PATCH 078/490] Optimizations for writing binary strings. (#268) --- .../ion/impl/bin/IonRawBinaryWriter.java | 162 ++++++++++++------ 1 file changed, 106 insertions(+), 56 deletions(-) diff --git a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java index 23ced039c6..7d192f5bac 100644 --- a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java +++ b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java @@ -50,6 +50,11 @@ import java.io.OutputStream; import java.math.BigDecimal; import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CoderResult; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -120,6 +125,16 @@ private static byte[] bytes(int... vals) { private static final byte VARINT_NEG_ZERO = (byte) 0xC0; + // See IonRawBinaryWriter#writeString(String) for usage information. + static final int SMALL_STRING_SIZE = 64; + static final int MEDIUM_STRING_SIZE = 4 * 1024; + + // Reusable resources for encoding Strings as UTF-8 bytes + final CharsetEncoder utf8Encoder = Charset.forName("UTF-8").newEncoder(); + final ByteBuffer utf8EncodingBuffer = ByteBuffer.allocate((int) (MEDIUM_STRING_SIZE * utf8Encoder.maxBytesPerChar())); + final char[] charArray = new char[MEDIUM_STRING_SIZE]; + final CharBuffer reusableCharBuffer = CharBuffer.wrap(charArray); + private static final byte[] makeTypedPreallocatedBytes(final int typeDesc, final int length) { final byte[] bytes = new byte[length]; @@ -1429,65 +1444,100 @@ public void writeString(final String value) throws IOException } prepareValue(); - // assume the string is ASCII and round up the sizing -- we should revisit this for CJK heavy use cases - int estUtf8Length = value.length(); - int preallocatedLength = 1; - final long lengthPosition = buffer.position() + 1; - if (estUtf8Length <= 0xD) - { - // size fits in low nibble - estUtf8Length = 0xD; - buffer.writeUInt8(STRING_TYPE); - } - else - { - if (estUtf8Length <= 0x7F) - { - estUtf8Length = 0x7F; - preallocatedLength = 2; - buffer.writeBytes(STRING_TYPED_PREALLOCATED_2); - } - else - { - estUtf8Length = 0x3FFF; - preallocatedLength = 3; - buffer.writeBytes(STRING_TYPED_PREALLOCATED_3); - } - // TODO decide if it is worth preallocating for > 16KB strings + /* + This method relies on the standard CharsetEncoder class to encode each String's UTF-16 char[] data into + UTF-8 bytes. Strangely, CharsetEncoders cannot operate directly on instances of a String. The CharsetEncoder + API requires all inputs and outputs to be specified as instances of java.nio.ByteBuffer and + java.nio.CharBuffer, making some number of allocations mandatory. Specifically, for each encoding operation + we need to have: + + 1. An instance of a UTF-8 CharsetEncoder. + 2. A CharBuffer representation of the String's data. + 3. A ByteBuffer into which the CharsetEncoder may write UTF-8 bytes. + + To minimize the overhead involved, the IonRawBinaryWriter will reuse previously initialized resources wherever + possible. However, because CharBuffer and ByteBuffer each have a fixed length, we can only reuse them for + Strings that are small enough to fit. This creates two kinds of input String to encode: those that are small + enough for us to reuse our buffers, and those which are not. + + Further benchmarking shows that there is also a third kind: Strings which are small enough to enable additional + aggressive optimizations. The JIT can eliminate heap allocations for scalar arrays (e.g. byte[], char[]) + that have no more than than a configurable number of elements (default: 64), opting to place them on the stack + instead. (This threshold be can be overridden with the -XX:EliminateAllocationArraySizeLimit=N JVM flag, but + this is neither common nor recommended.) When encoding small strings, requesting new buffers will cause those + buffers to be allocated on the stack instead of on the heap. And, because these buffers only exist for the + duration of the encoding method call (i.e. they are neither returned from the method nor stored in a member + field), the JIT is able to selectively eliminate costly superfluous steps like defensive array copying. For + more details, see the JDK's documentation on Escape Analysis[1]. + + The String#getBytes(Charset) method cannot be used for two reasons: + + 1. It always allocates, so we cannot reuse any resources. + 2. If/when it encounters character data that cannot be encoded as UTF-8, it simply replaces that data + with a substitute character[2]. (Sometimes seen in applications as a '?'.) In order + to surface invalid data to the user, the method must be able to detect these events at encoding time. + + [1] https://docs.oracle.com/javase/8/docs/technotes/guides/vm/performance-enhancements-7.html#escapeAnalysis + [2] https://en.wikipedia.org/wiki/Substitute_character + */ + + CharBuffer stringData; + ByteBuffer encodingBuffer; + + int length = value.length(); + + // While it is possible to encode the Ion string using a fixed-size encodingBuffer, we need to be able to + // write the length of the complete UTF-8 string to the output stream before we write the string itself. + // For simplicity, we reuse or create an encodingBuffer that is large enough to hold the full string. + + // In order to encode the input String, we need to pass it to CharsetEncoder as an implementation of CharBuffer. + // Surprisingly, the intuitive way to achieve this (the CharBuffer#wrap(CharSequence) method) adds a large + // amount of CPU overhead to the encoding process. Benchmarking shows that it's substantially faster + // to use String#getChars(int, int, char[], int) to copy the String's backing array and then call + // CharBuffer#wrap(char[]) on the copy. + + if (length <= SMALL_STRING_SIZE || length > MEDIUM_STRING_SIZE) { + // Small strings and large Strings both request fresh buffers. + encodingBuffer = ByteBuffer.allocate((int) (value.length() * utf8Encoder.maxBytesPerChar())); + char[] chars = new char[value.length()]; + value.getChars(0, value.length(), chars, 0); + stringData = CharBuffer.wrap(chars); + } else { + // Medium-sized strings are encoded using our reusable buffers. + encodingBuffer = utf8EncodingBuffer; + encodingBuffer.position(0); + encodingBuffer.limit(encodingBuffer.capacity()); + stringData = reusableCharBuffer; + value.getChars(0, value.length(), charArray, 0); + reusableCharBuffer.position(0); + reusableCharBuffer.limit(value.length()); + } + + // Because encodingBuffer is guaranteed to be large enough to hold the encoded string, we can + // perform the encoding in a single call to CharsetEncoder#encode(CharBuffer, ByteBuffer, boolean). + CoderResult coderResult = utf8Encoder.encode(stringData, encodingBuffer, true); + + // 'Underflow' is the success state of a CoderResult. + if (!coderResult.isUnderflow()) { + throw new IllegalArgumentException("Could not encode string as UTF8 bytes: " + value); + } + encodingBuffer.flip(); + int utf8Length = encodingBuffer.remaining(); + + // Write the type and length codes to the output stream. + long previousPosition = buffer.position(); + if (utf8Length <= 0xD) { + buffer.writeUInt8(STRING_TYPE | utf8Length); + } else { + buffer.writeUInt8(STRING_TYPE | 0xE); + buffer.writeVarUInt(utf8Length); } - updateLength(preallocatedLength); - // actually encode the string - final int utf8Length = buffer.writeUTF8(value); - if (utf8Length <= estUtf8Length) - { - // we fit! - if (utf8Length <= 0xD) - { - // special case for patching the type byte itself with the length - buffer.writeUInt8At(lengthPosition - 1, STRING_TYPE | utf8Length); - } - else if (utf8Length <= 0x7F) - { - buffer.writeVarUIntDirect1At(lengthPosition, utf8Length); - } - else - { - buffer.writeVarUIntDirect2At(lengthPosition, utf8Length); - } - } - else - { - // side patch - if (estUtf8Length == 0xD) - { - // we need to patch the type with the extended length - buffer.writeUInt8At(lengthPosition - 1, STRING_TYPE_EXTENDED_LENGTH); - } - addPatchPoint(lengthPosition, preallocatedLength - 1, utf8Length); - } + // Write the encoded UTF-8 bytes to the output stream + buffer.writeBytes(encodingBuffer.array(), 0, utf8Length); - updateLength(utf8Length); + long bytesWritten = buffer.position() - previousPosition; + updateLength(bytesWritten); finishValue(); } From a339edb793692ef4408c526d2329dad728a8173a Mon Sep 17 00:00:00 2001 From: Zack Slayton Date: Tue, 26 Nov 2019 18:05:05 -0500 Subject: [PATCH 079/490] Simplified string encoding. (#269) --- .../ion/impl/bin/IonRawBinaryWriter.java | 35 ++++++------------- 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java index 7d192f5bac..887ad9be30 100644 --- a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java +++ b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java @@ -126,13 +126,12 @@ private static byte[] bytes(int... vals) { private static final byte VARINT_NEG_ZERO = (byte) 0xC0; // See IonRawBinaryWriter#writeString(String) for usage information. - static final int SMALL_STRING_SIZE = 64; - static final int MEDIUM_STRING_SIZE = 4 * 1024; + static final int SMALL_STRING_SIZE = 4 * 1024; // Reusable resources for encoding Strings as UTF-8 bytes final CharsetEncoder utf8Encoder = Charset.forName("UTF-8").newEncoder(); - final ByteBuffer utf8EncodingBuffer = ByteBuffer.allocate((int) (MEDIUM_STRING_SIZE * utf8Encoder.maxBytesPerChar())); - final char[] charArray = new char[MEDIUM_STRING_SIZE]; + final ByteBuffer utf8EncodingBuffer = ByteBuffer.allocate((int) (SMALL_STRING_SIZE * utf8Encoder.maxBytesPerChar())); + final char[] charArray = new char[SMALL_STRING_SIZE]; final CharBuffer reusableCharBuffer = CharBuffer.wrap(charArray); private static final byte[] makeTypedPreallocatedBytes(final int typeDesc, final int length) @@ -1458,27 +1457,16 @@ public void writeString(final String value) throws IOException To minimize the overhead involved, the IonRawBinaryWriter will reuse previously initialized resources wherever possible. However, because CharBuffer and ByteBuffer each have a fixed length, we can only reuse them for Strings that are small enough to fit. This creates two kinds of input String to encode: those that are small - enough for us to reuse our buffers, and those which are not. - - Further benchmarking shows that there is also a third kind: Strings which are small enough to enable additional - aggressive optimizations. The JIT can eliminate heap allocations for scalar arrays (e.g. byte[], char[]) - that have no more than than a configurable number of elements (default: 64), opting to place them on the stack - instead. (This threshold be can be overridden with the -XX:EliminateAllocationArraySizeLimit=N JVM flag, but - this is neither common nor recommended.) When encoding small strings, requesting new buffers will cause those - buffers to be allocated on the stack instead of on the heap. And, because these buffers only exist for the - duration of the encoding method call (i.e. they are neither returned from the method nor stored in a member - field), the JIT is able to selectively eliminate costly superfluous steps like defensive array copying. For - more details, see the JDK's documentation on Escape Analysis[1]. + enough for us to reuse our buffers ("small strings"), and those which are not ("large strings"). The String#getBytes(Charset) method cannot be used for two reasons: 1. It always allocates, so we cannot reuse any resources. 2. If/when it encounters character data that cannot be encoded as UTF-8, it simply replaces that data - with a substitute character[2]. (Sometimes seen in applications as a '?'.) In order + with a substitute character[1]. (Sometimes seen in applications as a '?'.) In order to surface invalid data to the user, the method must be able to detect these events at encoding time. - [1] https://docs.oracle.com/javase/8/docs/technotes/guides/vm/performance-enhancements-7.html#escapeAnalysis - [2] https://en.wikipedia.org/wiki/Substitute_character + [1] https://en.wikipedia.org/wiki/Substitute_character */ CharBuffer stringData; @@ -1496,20 +1484,19 @@ duration of the encoding method call (i.e. they are neither returned from the me // to use String#getChars(int, int, char[], int) to copy the String's backing array and then call // CharBuffer#wrap(char[]) on the copy. - if (length <= SMALL_STRING_SIZE || length > MEDIUM_STRING_SIZE) { - // Small strings and large Strings both request fresh buffers. + if (length > SMALL_STRING_SIZE) { + // Allocate a new buffer for large strings encodingBuffer = ByteBuffer.allocate((int) (value.length() * utf8Encoder.maxBytesPerChar())); char[] chars = new char[value.length()]; value.getChars(0, value.length(), chars, 0); stringData = CharBuffer.wrap(chars); } else { - // Medium-sized strings are encoded using our reusable buffers. + // Reuse our existing buffers for small strings encodingBuffer = utf8EncodingBuffer; - encodingBuffer.position(0); - encodingBuffer.limit(encodingBuffer.capacity()); + encodingBuffer.clear(); stringData = reusableCharBuffer; value.getChars(0, value.length(), charArray, 0); - reusableCharBuffer.position(0); + reusableCharBuffer.rewind(); reusableCharBuffer.limit(value.length()); } From 925d50d08b7610e2dc37cbab5cd30d954103eee1 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Tue, 24 Dec 2019 14:00:28 -0800 Subject: [PATCH 080/490] Adds support for writing LST append after IonWriter.flush(). --- src/com/amazon/ion/IonWriter.java | 38 ++++++----- .../ion/impl/bin/IonManagedBinaryWriter.java | 19 ++++-- .../com/amazon/ion/impl/BinaryWriterTest.java | 25 ++++++-- .../amazon/ion/impl/IonWriterTestCase.java | 20 +++++- .../ion/impl/OutputStreamWriterTestCase.java | 1 - .../impl/bin/IonManagedBinaryWriterTest.java | 64 +++++++++++++++++++ 6 files changed, 134 insertions(+), 33 deletions(-) diff --git a/src/com/amazon/ion/IonWriter.java b/src/com/amazon/ion/IonWriter.java index fe1e0e960d..dae9c08e62 100644 --- a/src/com/amazon/ion/IonWriter.java +++ b/src/com/amazon/ion/IonWriter.java @@ -58,12 +58,12 @@ *

    * Once all the top-level values have been written (and stepped-out back to * the starting level), the caller must {@link #close()} the writer - * (or at least {@link #finish()} it) before accessing + * (or at least {@link #flush()} or {@link #finish()} it) before accessing * the data written to the underlying data sink * (for example, via {@link ByteArrayOutputStream#toByteArray()}). - * The writer may have internal buffers and without closing or finishing it, - * it may not have written everything to the underlying data sink. In addition, - * {@link #flush()} isn't guaranteed to be sufficient for all implementations. + * The writer may have internal buffers and without closing, flushing, or + * finishing it, it may not have written everything to the underlying data + * sink. * *

    Exception Handling

    * {@code IonWriter} is a generic interface for generating Ion data, and it's @@ -106,18 +106,19 @@ public interface IonWriter /** * Flushes this writer by writing any buffered output to the underlying - * output target. + * output target without finalizing the stream's local symbol table. *

    * For some implementations this may have no effect even when some data is * buffered, because it's not always possible to fully write partial data. * In particular, when writing binary Ion data, Ion's length-prefixed * encoding requires a complete top-level value to be written at once. *

    - * Furthermore, when writing binary Ion data, nothing can be - * flushed until the writer knows that no more local symbols can be - * encountered. This can be accomplished via {@link #finish()} or by - * making the {@linkplain #getSymbolTable() local symbol table} - * {@linkplain SymbolTable#makeReadOnly() read-only}. + * This feature can be used to flush buffered data before writing more + * values without subsequently having to redeclare the current local + * symbol table. Applications that produce long streams of binary Ion may + * wish to flush occasionally to relieve memory pressure, then continue + * writing data using the same local symbol table. The symbol table will + * be appended with newly-encountered symbols as necessary. * * @throws IOException if thrown by the underlying output target. * @@ -144,13 +145,16 @@ public interface IonWriter * {@link com.amazon.ion.system.IonWriterBuilder.IvmMinimizing * IvmMinimizing}.) *

    - * This feature can be used to flush reliably before writing more values. - * Think about a long-running stream of binary values: without finishing, - * all the values would continue to buffer since the encoder keeps - * expecting more local symbols (which must be written into the local - * symbol table that precedes all top-level values). Such an application - * can finish occasionally to flush the data out, then continue writing - * more data using a fresh local symbol table. + * This feature can be used to flush buffered data and reset the local + * symbol table before writing more values. Applications that produce long + * streams of binary Ion may wish to finish occasionally to relieve memory + * pressure, then continue writing data using a new local symbol table. + * This is particularly useful for streams that contain an ever-growing + * number of unique symbols to avoid unbounded growth of the symbol table, + * which may degrade performance and bloat the encoding. Applications that + * produce long streams with a limited number of unique symbols should + * use {@link #flush()} instead to avoid re-declaring the local symbol + * table unnecessarily. * * @throws IOException if thrown by the underlying output target. * @throws IllegalStateException when not between top-level values. diff --git a/src/com/amazon/ion/impl/bin/IonManagedBinaryWriter.java b/src/com/amazon/ion/impl/bin/IonManagedBinaryWriter.java index e898f2a16a..3c87b72afb 100644 --- a/src/com/amazon/ion/impl/bin/IonManagedBinaryWriter.java +++ b/src/com/amazon/ion/impl/bin/IonManagedBinaryWriter.java @@ -740,16 +740,22 @@ public int getDepth() private void startLocalSymbolTableIfNeeded(final boolean writeIVM) throws IOException { - if (symbolState == SymbolState.SYSTEM_SYMBOLS) + boolean isAppend = symbolState == SymbolState.LOCAL_SYMBOLS_FLUSHED; + if (symbolState == SymbolState.SYSTEM_SYMBOLS || isAppend) { - if (writeIVM) + if (writeIVM && !isAppend) { symbols.writeIonVersionMarker(); } symbols.addTypeAnnotationSymbol(systemSymbol(ION_SYMBOL_TABLE_SID)); symbols.stepIn(STRUCT); { - if (imports.parents.size() > 0) + if (isAppend) + { + symbols.setFieldNameSymbol(systemSymbol(IMPORTS_SID)); + symbols.writeSymbolToken(systemSymbol(ION_SYMBOL_TABLE_SID)); + } + else if (imports.parents.size() > 0) { symbols.setFieldNameSymbol(systemSymbol(IMPORTS_SID)); symbols.stepIn(LIST); @@ -1071,7 +1077,7 @@ public void writeBytes(byte[] data, int off, int len) throws IOException public void flush() throws IOException { - if (getDepth() == 0 && localsLocked) + if (getDepth() == 0 && !user.hasAnnotations()) { unsafeFlush(); } @@ -1083,10 +1089,9 @@ private void unsafeFlush() throws IOException { // this implies that we have a local symbol table of some sort and the user locked it symbolState.closeTable(symbols); + // make sure that until the local symbol state changes we no-op the table closing routine + symbolState = SymbolState.LOCAL_SYMBOLS_FLUSHED; } - - // make sure that until the local symbol state changes we no-op the table closing routine - symbolState = SymbolState.LOCAL_SYMBOLS_FLUSHED; // push the data out symbols.finish(); user.finish(); diff --git a/test/com/amazon/ion/impl/BinaryWriterTest.java b/test/com/amazon/ion/impl/BinaryWriterTest.java index f2d287859f..7110e152cc 100644 --- a/test/com/amazon/ion/impl/BinaryWriterTest.java +++ b/test/com/amazon/ion/impl/BinaryWriterTest.java @@ -15,6 +15,7 @@ package com.amazon.ion.impl; +import com.amazon.ion.IonCatalog; import com.amazon.ion.IonException; import com.amazon.ion.IonReader; import com.amazon.ion.IonType; @@ -26,6 +27,9 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; + +import com.amazon.ion.system.IonReaderBuilder; +import com.amazon.ion.system.SimpleCatalog; import org.junit.Assert; import org.junit.Test; @@ -120,27 +124,36 @@ public void testFlushingUnlockedSymtab() throws Exception { iw = makeWriter(); - iw.writeSymbol("force a local symtab"); // TODO amzn/ion-java/issues/8 + iw.writeSymbol("force a local symtab"); SymbolTable symtab = iw.getSymbolTable(); symtab.intern("fred_1"); symtab.intern("fred_2"); iw.writeSymbol("fred_1"); + // This would cause an appended LST to be written before the next value. iw.flush(); - byte[] bytes = myOutputStream.toByteArray(); - assertEquals(0, bytes.length); + IonReader reader = IonReaderBuilder.standard().build(myOutputStream.toByteArray()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("force a local symtab", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("fred_1", reader.stringValue()); + assertNull(reader.next()); } @Test public void testFlushingUnlockedSymtabWithImports() throws Exception { - SymbolTable fred1 = Symtabs.register("fred", 1, catalog()); + SimpleCatalog catalog = catalog(); + SymbolTable fred1 = Symtabs.register("fred", 1, catalog); iw = makeWriter(fred1); iw.writeSymbol("fred_1"); + // This would cause an appended LST to be written before the next value. iw.flush(); - byte[] bytes = myOutputStream.toByteArray(); - assertEquals(0, bytes.length); + IonReader reader = IonReaderBuilder.standard().withCatalog(catalog).build(myOutputStream.toByteArray()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("fred_1", reader.stringValue()); + assertNull(reader.next()); } @Test diff --git a/test/com/amazon/ion/impl/IonWriterTestCase.java b/test/com/amazon/ion/impl/IonWriterTestCase.java index e13ff69dcf..505a756e9e 100644 --- a/test/com/amazon/ion/impl/IonWriterTestCase.java +++ b/test/com/amazon/ion/impl/IonWriterTestCase.java @@ -47,6 +47,7 @@ import com.amazon.ion.SystemSymbols; import com.amazon.ion.TestUtils; import com.amazon.ion.UnknownSymbolException; +import com.amazon.ion.impl.bin._Private_IonManagedWriter; import com.amazon.ion.junit.IonAssert; import com.amazon.ion.system.IonSystemBuilder; import java.io.ByteArrayInputStream; @@ -805,9 +806,24 @@ public void testFlushDoesNotReset() iw.writeSymbol("now"); iw.close(); - // Should have: IVM SYMTAB hey now IonDatagram dg = reload(); - assertEquals(4, dg.systemSize()); + if (myOutputForm == OutputForm.BINARY && iw instanceof _Private_IonManagedWriter) { + // Note: LST append will only be implemented in the "new" binary writer (which implements + // _Private_IonMangedWriter) + // Should have: IVM SYMTAB hey SYMTAB_APPEND now + assertEquals(5, dg.systemSize()); + assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); + assertEquals("hey", ((IonSymbol) dg.systemGet(2)).stringValue()); + assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(3)).getTypeAnnotations()[0]); + assertEquals("now", ((IonSymbol) dg.systemGet(4)).stringValue()); + } + else { + // Should have: IVM SYMTAB hey SYMTAB_APPEND now + assertEquals(4, dg.systemSize()); + assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); + assertEquals("hey", ((IonSymbol) dg.systemGet(2)).stringValue()); + assertEquals("now", ((IonSymbol) dg.systemGet(3)).stringValue()); + } } Iterator systemIterateOutput() diff --git a/test/com/amazon/ion/impl/OutputStreamWriterTestCase.java b/test/com/amazon/ion/impl/OutputStreamWriterTestCase.java index acc7c010a8..4093ae4b6d 100644 --- a/test/com/amazon/ion/impl/OutputStreamWriterTestCase.java +++ b/test/com/amazon/ion/impl/OutputStreamWriterTestCase.java @@ -179,7 +179,6 @@ private void testFlushing() // Try flushing when there's just a pending annotation. iw.addTypeAnnotation("fred_1"); iw.flush(); - checkFlushed(true); myOutputStreamWrapper.flushed = false; bytes = myOutputStream.toByteArray(); diff --git a/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java b/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java index b83bbbd2e1..8fc3d67b00 100644 --- a/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java +++ b/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java @@ -20,7 +20,11 @@ import static java.util.Collections.unmodifiableMap; import com.amazon.ion.IonContainer; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonInt; import com.amazon.ion.IonMutableCatalog; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonStruct; import com.amazon.ion.IonSymbol; import com.amazon.ion.IonType; import com.amazon.ion.IonValue; @@ -179,6 +183,66 @@ public void testSystemSymbol() throws Exception assertValue("name"); } + @Test + public void testLocalSymbolTableAppend() throws Exception + { + writer.writeSymbol("taco"); + writer.flush(); + writer.writeSymbol("burrito"); + writer.finish(); + IonReader reader = system().newReader(writer.getBytes()); + reader.next(); + assertEquals(reader.getSymbolTable().findSymbol("taco"), 15); + assertEquals(reader.getSymbolTable().findSymbol("burrito"), -1); + reader.next(); + assertEquals(reader.getSymbolTable().findSymbol("taco"), 15); + assertEquals(reader.getSymbolTable().findSymbol("burrito"), 16); + assertNull(reader.next()); + + IonDatagram dg = system().getLoader().load(writer.getBytes()); + // Should be IVM SYMTAB taco SYMTAB burrito + assertEquals(5, dg.systemSize()); + assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); + assertEquals("taco", ((IonSymbol) dg.systemGet(2)).stringValue()); + assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(3)).getTypeAnnotations()[0]); + assertEquals("burrito", ((IonSymbol) dg.systemGet(4)).stringValue()); + } + + @Test + public void testFlushImmediatelyAfterIVM() throws Exception + { + writer.flush(); + writer.writeSymbol("burrito"); + writer.finish(); + IonReader reader = system().newReader(writer.getBytes()); + reader.next(); + assertEquals(reader.getSymbolTable().findSymbol("taco"), -1); + assertEquals(reader.getSymbolTable().findSymbol("burrito"), 15); + assertNull(reader.next()); + + IonDatagram dg = system().getLoader().load(writer.getBytes()); + // Should be IVM SYMTAB burrito + assertEquals(3, dg.systemSize()); + assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); + assertEquals("burrito", ((IonSymbol) dg.systemGet(2)).stringValue()); + } + + @Test + public void testNoNewSymbolsAfterFlush() throws Exception + { + writer.writeSymbol("taco"); + writer.flush(); + writer.writeInt(123); + writer.finish(); + + IonDatagram dg = system().getLoader().load(writer.getBytes()); + // Should be IVM SYMTAB taco 123 + assertEquals(4, dg.systemSize()); + assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); + assertEquals("taco", ((IonSymbol) dg.systemGet(2)).stringValue()); + assertEquals(123, ((IonInt) dg.systemGet(3)).intValue()); + } + @Test public void testUserFieldNames() throws Exception { From b823333c0cea44e3e9d636080b0ed2830c5eaad0 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Tue, 24 Dec 2019 14:50:21 -0800 Subject: [PATCH 081/490] Adds support for intercepting an appended LST when written via the public IonWriter APIs. --- .../ion/impl/bin/IonManagedBinaryWriter.java | 36 ++++++++-- .../impl/bin/IonManagedBinaryWriterTest.java | 68 +++++++++++++++++++ 2 files changed, 99 insertions(+), 5 deletions(-) diff --git a/src/com/amazon/ion/impl/bin/IonManagedBinaryWriter.java b/src/com/amazon/ion/impl/bin/IonManagedBinaryWriter.java index 3c87b72afb..2299baf20e 100644 --- a/src/com/amazon/ion/impl/bin/IonManagedBinaryWriter.java +++ b/src/com/amazon/ion/impl/bin/IonManagedBinaryWriter.java @@ -368,12 +368,20 @@ public void afterStepOut(final IonManagedBinaryWriter self) throws IOException // since we don't know what's coming after (i.e. new local symbols) self.user.truncate(self.userSymbolTablePosition); - // flush out the pre-existing symbol and user content before the user provided symbol table - self.finish(); + if (self.isUserLSTAppend) + { + self.flush(); + } + else + { - // replace the symbol table context with the user provided one - // TODO determine if the resolver mode should be configurable for this use case - self.imports = new ImportedSymbolContext(ImportedSymbolResolverMode.DELEGATE, self.userImports); + // flush out the pre-existing symbol and user content before the user provided symbol table + self.finish(); + + // replace the symbol table context with the user provided one + // TODO determine if the resolver mode should be configurable for this use case + self.imports = new ImportedSymbolContext(ImportedSymbolResolverMode.DELEGATE, self.userImports); + } // explicitly start the local symbol table with no version marker // in case we need the previous symbols @@ -392,10 +400,24 @@ public void afterStepOut(final IonManagedBinaryWriter self) throws IOException self.userCurrentImport.reset(); self.userImports.clear(); self.userSymbols.clear(); + self.isUserLSTAppend = false; self.userState = NORMAL; } } + + @Override + public void writeSymbolToken(final IonManagedBinaryWriter self, final SymbolToken value) + { + if ( + self.user.getDepth() == 1 + && self.user.getFieldId() == IMPORTS_SID + && value.getSid() == ION_SYMBOL_TABLE_SID + ) { + self.isUserLSTAppend = true; + self.userState = LOCALS_AT_TOP; + } + } }, LOCALS_AT_IMPORTS { @@ -515,6 +537,7 @@ public void writeString(final IonManagedBinaryWriter self, String value) public abstract void afterStepOut(final IonManagedBinaryWriter self) throws IOException; public void writeString(final IonManagedBinaryWriter self, final String value) throws IOException {} + public void writeSymbolToken(final IonManagedBinaryWriter self, final SymbolToken value) {} public void writeInt(final IonManagedBinaryWriter self, final long value) throws IOException {} public void writeInt(IonManagedBinaryWriter self, BigInteger value) throws IOException { @@ -636,6 +659,7 @@ public void makeReadOnly() private final List userImports; private final List userSymbols; private final ImportDescriptor userCurrentImport; + private boolean isUserLSTAppend; private boolean closed; @@ -680,6 +704,7 @@ public void makeReadOnly() this.userImports = new ArrayList(); this.userSymbols = new ArrayList(); this.userCurrentImport = new ImportDescriptor(); + this.isUserLSTAppend = false; // TODO decide if initial LST should survive finish() and seed the next LST final SymbolTable lst = builder.initialSymbolTable; @@ -1031,6 +1056,7 @@ public void writeSymbolToken(SymbolToken token) throws IOException return; } token = intern(token); + userState.writeSymbolToken(this, token); user.writeSymbolToken(token); } diff --git a/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java b/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java index 8fc3d67b00..416abc9637 100644 --- a/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java +++ b/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java @@ -243,6 +243,74 @@ public void testNoNewSymbolsAfterFlush() throws Exception assertEquals(123, ((IonInt) dg.systemGet(3)).intValue()); } + @Test + public void testManuallyWriteLSTAppendWithImportsFirst() throws Exception + { + writer.writeSymbol("taco"); + writer.addTypeAnnotation("$ion_symbol_table"); + writer.stepIn(IonType.STRUCT); + writer.setFieldName("imports"); + writer.writeSymbol("$ion_symbol_table"); + writer.setFieldName("symbols"); + writer.stepIn(IonType.LIST); + writer.writeString("burrito"); + writer.stepOut(); + writer.stepOut(); + writer.writeSymbol("burrito"); + writer.finish(); + + IonDatagram dg = system().getLoader().load(writer.getBytes()); + // Should be IVM SYMTAB taco SYMTAB burrito + assertEquals(5, dg.systemSize()); + assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); + assertEquals("taco", ((IonSymbol) dg.systemGet(2)).stringValue()); + assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(3)).getTypeAnnotations()[0]); + assertEquals("burrito", ((IonSymbol) dg.systemGet(4)).stringValue()); + + IonReader reader = system().newReader(writer.getBytes()); + reader.next(); + assertEquals(reader.getSymbolTable().findSymbol("taco"), 15); + assertEquals(reader.getSymbolTable().findSymbol("burrito"), -1); + reader.next(); + assertEquals(reader.getSymbolTable().findSymbol("taco"), 15); + assertEquals(reader.getSymbolTable().findSymbol("burrito"), 16); + assertNull(reader.next()); + } + + @Test + public void testManuallyWriteLSTAppendWithSymbolsFirst() throws Exception + { + writer.writeSymbol("taco"); + writer.addTypeAnnotation("$ion_symbol_table"); + writer.stepIn(IonType.STRUCT); + writer.setFieldName("symbols"); + writer.stepIn(IonType.LIST); + writer.writeString("burrito"); + writer.stepOut(); + writer.setFieldName("imports"); + writer.writeSymbol("$ion_symbol_table"); + writer.stepOut(); + writer.writeSymbol("burrito"); + writer.finish(); + + IonDatagram dg = system().getLoader().load(writer.getBytes()); + // Should be IVM SYMTAB taco SYMTAB burrito + assertEquals(5, dg.systemSize()); + assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); + assertEquals("taco", ((IonSymbol) dg.systemGet(2)).stringValue()); + assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(3)).getTypeAnnotations()[0]); + assertEquals("burrito", ((IonSymbol) dg.systemGet(4)).stringValue()); + + IonReader reader = system().newReader(writer.getBytes()); + reader.next(); + assertEquals(reader.getSymbolTable().findSymbol("taco"), 15); + assertEquals(reader.getSymbolTable().findSymbol("burrito"), -1); + reader.next(); + assertEquals(reader.getSymbolTable().findSymbol("taco"), 15); + assertEquals(reader.getSymbolTable().findSymbol("burrito"), 16); + assertNull(reader.next()); + } + @Test public void testUserFieldNames() throws Exception { From ff3a971781aab36d1da7d632abb2a2cf121846bc Mon Sep 17 00:00:00 2001 From: Fernando Raganhan Barbosa Date: Tue, 28 Jan 2020 10:09:05 -0800 Subject: [PATCH 082/490] Update ion-tests (#275) Adds a filter for md files so we can safely add documentation to any ion-tests subfolder --- ion-tests | 2 +- test/com/amazon/ion/TestUtils.java | 17 ++++++++++++----- .../amazon/ion/streaming/ReaderDomCopyTest.java | 1 - 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/ion-tests b/ion-tests index 48a63157b2..05ecd5de78 160000 --- a/ion-tests +++ b/ion-tests @@ -1 +1 @@ -Subproject commit 48a63157b28b81ed72c71789dac7d7f0ffe87038 +Subproject commit 05ecd5de78ac6244826cd4e77bae932925c966c1 diff --git a/test/com/amazon/ion/TestUtils.java b/test/com/amazon/ion/TestUtils.java index c5fbfb2e76..3f8a2bb241 100644 --- a/test/com/amazon/ion/TestUtils.java +++ b/test/com/amazon/ion/TestUtils.java @@ -63,11 +63,11 @@ public boolean accept(File dir, String name) } }; - public static final FilenameFilter ION_ONLY_FILTER = new FilenameFilter() + public static final FilenameFilter NOT_MARKDOWN_FILTER = new FilenameFilter() { public boolean accept(File dir, String name) { - return name.endsWith(".ion") || name.endsWith(".10n"); + return !name.endsWith(".md"); } }; @@ -131,15 +131,22 @@ public boolean accept(File dir, String name) } public static final FilenameFilter GLOBAL_SKIP_LIST = - new FileIsNot( + new And( + // Skips documentation that accompanies some test vectors + NOT_MARKDOWN_FILTER, + new FileIsNot( "bad/clobWithNullCharacter.ion" // TODO amzn/ion-java/43 ,"bad/emptyAnnotatedInt.10n" // TODO amzn/ion-java/55 ,"good/subfieldVarUInt32bit.ion" // TODO amzn/ion-java/62 ,"good/utf16.ion" // TODO amzn/ion-java/61 ,"good/utf32.ion" // TODO amzn/ion-java/61 ,"good/whitespace.ion" - , "good/item1.10n" // TODO amzn/ion-java#126 (roundtrip symbols with unknown text) - ); + ,"good/item1.10n" // TODO amzn/ion-java#126 (roundtrip symbols with unknown text) + ,"bad/typecodes/type_6_length_0.10n" // TODO amzn/ion-java#272 + ,"good/typecodes/T7-large.10n" // TODO amzn/ion-java#273 + ,"good/equivs/clobNewlines.ion" // TODO amzn/ion-java#274 + ) + ); private static void testdataFiles(FilenameFilter filter, diff --git a/test/com/amazon/ion/streaming/ReaderDomCopyTest.java b/test/com/amazon/ion/streaming/ReaderDomCopyTest.java index 4afb3ec7a4..a9674d4cee 100644 --- a/test/com/amazon/ion/streaming/ReaderDomCopyTest.java +++ b/test/com/amazon/ion/streaming/ReaderDomCopyTest.java @@ -36,7 +36,6 @@ public class ReaderDomCopyTest public static final File[] FILES = testdataFiles(GLOBAL_SKIP_LIST, GOOD_IONTESTS_FILES); - private File myTestFile; public void setTestFile(File file) From 40689a8d59d8299cd5884ad7768a1125c3ce2385 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Fri, 7 Feb 2020 13:39:47 -0800 Subject: [PATCH 083/490] Fixes a bug that could cause an infinite loop when an IonValue was modified while being read with the tree reader. --- src/com/amazon/ion/IonSystem.java | 4 ++++ .../amazon/ion/impl/IonReaderTreeSystem.java | 1 + src/com/amazon/ion/system/IonReaderBuilder.java | 4 ++++ test/com/amazon/ion/impl/TreeReaderTest.java | 17 +++++++++++++++++ 4 files changed, 26 insertions(+) diff --git a/src/com/amazon/ion/IonSystem.java b/src/com/amazon/ion/IonSystem.java index f7bb29c038..8dd489562a 100644 --- a/src/com/amazon/ion/IonSystem.java +++ b/src/com/amazon/ion/IonSystem.java @@ -485,6 +485,10 @@ public SymbolTable newSharedSymbolTable(String name, * model. Typically this is used to iterate over a collection, such as an * {@link IonStruct}. * + * The given value and its children, if any, must not be modified until after + * the IonReader constructed by this method is closed. Violating this + * constraint results in undefined behavior. + * * @param value must not be null. */ public IonReader newReader(IonValue value); diff --git a/src/com/amazon/ion/impl/IonReaderTreeSystem.java b/src/com/amazon/ion/impl/IonReaderTreeSystem.java index f8a12181b0..868e65104b 100644 --- a/src/com/amazon/ion/impl/IonReaderTreeSystem.java +++ b/src/com/amazon/ion/impl/IonReaderTreeSystem.java @@ -497,6 +497,7 @@ public boolean hasNext() _next_idx = ii+1; break; } + ii++; } } // if there anything left? diff --git a/src/com/amazon/ion/system/IonReaderBuilder.java b/src/com/amazon/ion/system/IonReaderBuilder.java index 895933b43d..755cbe1fa8 100644 --- a/src/com/amazon/ion/system/IonReaderBuilder.java +++ b/src/com/amazon/ion/system/IonReaderBuilder.java @@ -253,6 +253,10 @@ public IonReader build(Reader ionText) * {@link IonReader} instance over an {@link IonValue} data model. Typically * this is used to iterate over a collection, such as an {@link IonStruct}. * + * The given value and its children, if any, must not be modified until after + * the IonReader constructed by this method is closed. Violating this + * constraint results in undefined behavior. + * * @param value must not be null. * * @see IonSystem#newReader(IonValue) diff --git a/test/com/amazon/ion/impl/TreeReaderTest.java b/test/com/amazon/ion/impl/TreeReaderTest.java index 34af455227..20108f9e0e 100644 --- a/test/com/amazon/ion/impl/TreeReaderTest.java +++ b/test/com/amazon/ion/impl/TreeReaderTest.java @@ -92,4 +92,21 @@ public void testReadingStructFields() check().next().fieldName((String)null).type(IonType.NULL); assertTopLevel(in, /* inStruct */ false); } + + @Test + public void testModificationDuringRead() + { + IonList list = system().newEmptyList(); + list.add().newString("abc"); + list.add().newString("def"); + in = system().newReader(list); + assertEquals(IonType.LIST, in.next()); + in.stepIn(); + assertEquals(IonType.STRING, in.next()); + // Violate the contract by modifying the value. + list.remove(0); + // Since the value was modified while it was being written, the result of the following is undefined. But + // one thing is for sure: it should not cause an infinite loop. + in.next(); + } } From fa374ac22f18432338b9e88a20c8f32de91cb3cb Mon Sep 17 00:00:00 2001 From: Peter Cornell Date: Mon, 24 Feb 2020 16:46:56 -0800 Subject: [PATCH 084/490] Disables LST append writing by default and adds option to enable it (#278) --- src/com/amazon/ion/IonWriter.java | 8 +- .../impl/_Private_IonBinaryWriterBuilder.java | 30 +++++ .../ion/impl/bin/IonManagedBinaryWriter.java | 6 +- ...Private_IonManagedBinaryWriterBuilder.java | 15 +++ .../ion/system/IonBinaryWriterBuilder.java | 21 ++++ .../com/amazon/ion/impl/BinaryWriterTest.java | 64 ++++++++-- .../amazon/ion/impl/IonWriterTestCase.java | 19 ++- .../impl/bin/IonManagedBinaryWriterTest.java | 113 +++++++++++++----- 8 files changed, 231 insertions(+), 45 deletions(-) diff --git a/src/com/amazon/ion/IonWriter.java b/src/com/amazon/ion/IonWriter.java index dae9c08e62..99270c6eae 100644 --- a/src/com/amazon/ion/IonWriter.java +++ b/src/com/amazon/ion/IonWriter.java @@ -113,7 +113,9 @@ public interface IonWriter * In particular, when writing binary Ion data, Ion's length-prefixed * encoding requires a complete top-level value to be written at once. *

    - * This feature can be used to flush buffered data before writing more + * If localSymbolTableAppend is enabled (see + * {@link com.amazon.ion.system.IonBinaryWriterBuilder#withLocalSymbolTableAppendEnabled}), + * this feature can be used to flush buffered data before writing more * values without subsequently having to redeclare the current local * symbol table. Applications that produce long streams of binary Ion may * wish to flush occasionally to relieve memory pressure, then continue @@ -153,7 +155,9 @@ public interface IonWriter * number of unique symbols to avoid unbounded growth of the symbol table, * which may degrade performance and bloat the encoding. Applications that * produce long streams with a limited number of unique symbols should - * use {@link #flush()} instead to avoid re-declaring the local symbol + * enable localSymbolTableAppend (see + * {@link com.amazon.ion.system.IonBinaryWriterBuilder#withLocalSymbolTableAppendEnabled}) + * and {@link #flush()} instead to avoid re-declaring the local symbol * table unnecessarily. * * @throws IOException if thrown by the underlying output target. diff --git a/src/com/amazon/ion/impl/_Private_IonBinaryWriterBuilder.java b/src/com/amazon/ion/impl/_Private_IonBinaryWriterBuilder.java index c92da72b29..3465bd474d 100644 --- a/src/com/amazon/ion/impl/_Private_IonBinaryWriterBuilder.java +++ b/src/com/amazon/ion/impl/_Private_IonBinaryWriterBuilder.java @@ -193,6 +193,36 @@ _Private_IonBinaryWriterBuilder withInitialSymbolTable(SymbolTable symtab) return b; } + @Override + public void setLocalSymbolTableAppendEnabled(boolean enabled) + { + mutationCheck(); + if (enabled) + { + myBinaryWriterBuilder.withLocalSymbolTableAppendEnabled(); + } + else + { + myBinaryWriterBuilder.withLocalSymbolTableAppendDisabled(); + } + } + + @Override + public _Private_IonBinaryWriterBuilder withLocalSymbolTableAppendEnabled() + { + _Private_IonBinaryWriterBuilder b = mutable(); + b.setLocalSymbolTableAppendEnabled(true); + return b; + } + + @Override + public _Private_IonBinaryWriterBuilder withLocalSymbolTableAppendDisabled() + { + _Private_IonBinaryWriterBuilder b = mutable(); + b.setLocalSymbolTableAppendEnabled(false); + return b; + } + @Override public void setIsFloatBinary32Enabled(boolean enabled) { mutationCheck(); diff --git a/src/com/amazon/ion/impl/bin/IonManagedBinaryWriter.java b/src/com/amazon/ion/impl/bin/IonManagedBinaryWriter.java index 2299baf20e..003dc70abe 100644 --- a/src/com/amazon/ion/impl/bin/IonManagedBinaryWriter.java +++ b/src/com/amazon/ion/impl/bin/IonManagedBinaryWriter.java @@ -659,6 +659,7 @@ public void makeReadOnly() private final List userImports; private final List userSymbols; private final ImportDescriptor userCurrentImport; + private final boolean lstAppendEnabled; private boolean isUserLSTAppend; private boolean closed; @@ -704,6 +705,7 @@ public void makeReadOnly() this.userImports = new ArrayList(); this.userSymbols = new ArrayList(); this.userCurrentImport = new ImportDescriptor(); + this.lstAppendEnabled = builder.isLocalSymbolTableAppendEnabled; this.isUserLSTAppend = false; // TODO decide if initial LST should survive finish() and seed the next LST @@ -765,7 +767,7 @@ public int getDepth() private void startLocalSymbolTableIfNeeded(final boolean writeIVM) throws IOException { - boolean isAppend = symbolState == SymbolState.LOCAL_SYMBOLS_FLUSHED; + boolean isAppend = symbolState == SymbolState.LOCAL_SYMBOLS_FLUSHED && lstAppendEnabled; if (symbolState == SymbolState.SYSTEM_SYMBOLS || isAppend) { if (writeIVM && !isAppend) @@ -1103,7 +1105,7 @@ public void writeBytes(byte[] data, int off, int len) throws IOException public void flush() throws IOException { - if (getDepth() == 0 && !user.hasAnnotations()) + if (getDepth() == 0 && !user.hasAnnotations() && (localsLocked || lstAppendEnabled)) { unsafeFlush(); } diff --git a/src/com/amazon/ion/impl/bin/_Private_IonManagedBinaryWriterBuilder.java b/src/com/amazon/ion/impl/bin/_Private_IonManagedBinaryWriterBuilder.java index 8c1dd6aab9..b439d160a1 100644 --- a/src/com/amazon/ion/impl/bin/_Private_IonManagedBinaryWriterBuilder.java +++ b/src/com/amazon/ion/impl/bin/_Private_IonManagedBinaryWriterBuilder.java @@ -77,6 +77,7 @@ BlockAllocatorProvider createAllocatorProvider() /*package*/ volatile IonCatalog catalog; /*package*/ volatile WriteValueOptimization optimization; /*package*/ volatile SymbolTable initialSymbolTable; + /*package*/ volatile boolean isLocalSymbolTableAppendEnabled; /*package*/ volatile boolean isFloatBinary32Enabled; private _Private_IonManagedBinaryWriterBuilder(final BlockAllocatorProvider provider) @@ -88,6 +89,7 @@ private _Private_IonManagedBinaryWriterBuilder(final BlockAllocatorProvider prov this.preallocationMode = PreallocationMode.PREALLOCATE_2; this.catalog = new SimpleCatalog(); this.optimization = WriteValueOptimization.NONE; + this.isLocalSymbolTableAppendEnabled = false; this.isFloatBinary32Enabled = false; } @@ -101,6 +103,7 @@ private _Private_IonManagedBinaryWriterBuilder(final _Private_IonManagedBinaryWr this.catalog = other.catalog; this.optimization = other.optimization; this.initialSymbolTable = other.initialSymbolTable; + this.isLocalSymbolTableAppendEnabled = other.isLocalSymbolTableAppendEnabled; this.isFloatBinary32Enabled = other.isFloatBinary32Enabled; } @@ -193,6 +196,18 @@ public _Private_IonManagedBinaryWriterBuilder withStreamCopyOptimization(boolean return this; } + public _Private_IonManagedBinaryWriterBuilder withLocalSymbolTableAppendEnabled() + { + isLocalSymbolTableAppendEnabled = true; + return this; + } + + public _Private_IonManagedBinaryWriterBuilder withLocalSymbolTableAppendDisabled() + { + isLocalSymbolTableAppendEnabled = false; + return this; + } + public _Private_IonManagedBinaryWriterBuilder withFloatBinary32Enabled() { isFloatBinary32Enabled = true; return this; diff --git a/src/com/amazon/ion/system/IonBinaryWriterBuilder.java b/src/com/amazon/ion/system/IonBinaryWriterBuilder.java index c6d6a0e545..b6b40257e3 100644 --- a/src/com/amazon/ion/system/IonBinaryWriterBuilder.java +++ b/src/com/amazon/ion/system/IonBinaryWriterBuilder.java @@ -174,6 +174,27 @@ public IvmMinimizing getIvmMinimizing() IonBinaryWriterBuilder withInitialSymbolTable(SymbolTable symtab); + /** + * Enables or disables writing local symbol tables that append symbols + * to the current symbol table. This functionality is disabled by default. + */ + public abstract void setLocalSymbolTableAppendEnabled(boolean enabled); + + + /** + * Enables writing local symbol tables that append symbols to the current + * symbol table. This functionality is disabled by default. + */ + public abstract IonBinaryWriterBuilder withLocalSymbolTableAppendEnabled(); + + + /** + * Disables writing local symbol tables that append symbols to the current + * symbol table. This functionality is disabled by default. + */ + public abstract IonBinaryWriterBuilder withLocalSymbolTableAppendDisabled(); + + /** * Enables or disables writing Binary32 (4-byte, single precision, * IEEE-754) values for floats when there would be no loss in precision. diff --git a/test/com/amazon/ion/impl/BinaryWriterTest.java b/test/com/amazon/ion/impl/BinaryWriterTest.java index 7110e152cc..8ab10fdcef 100644 --- a/test/com/amazon/ion/impl/BinaryWriterTest.java +++ b/test/com/amazon/ion/impl/BinaryWriterTest.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.io.OutputStream; +import com.amazon.ion.system.IonBinaryWriterBuilder; import com.amazon.ion.system.IonReaderBuilder; import com.amazon.ion.system.SimpleCatalog; import org.junit.Assert; @@ -41,7 +42,10 @@ protected IonWriter makeWriter(OutputStream out, SymbolTable... imports) throws Exception { myOutputForm = OutputForm.BINARY; - return system().newBinaryWriter(out, imports); + IonBinaryWriterBuilder builder = IonBinaryWriterBuilder.standard() + .withImports(imports); + builder.setLocalSymbolTableAppendEnabled(myLstAppendEnabled); + return builder.build(out); } @Test @@ -123,6 +127,29 @@ public void testInternUnlockedSymtab() public void testFlushingUnlockedSymtab() throws Exception { + byte[] bytes = flushUnlockedSymtab(false); + assertEquals(0, bytes.length); + } + + @Test + public void testFlushingUnlockedSymtabWithLSTAppend() + throws Exception + { + byte[] bytes = flushUnlockedSymtab(true); + + // iw.flush() should have caused an appended LST to be written before the next value. + IonReader reader = IonReaderBuilder.standard().build(bytes); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("force a local symtab", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("fred_1", reader.stringValue()); + assertNull(reader.next()); + } + + private byte[] flushUnlockedSymtab(boolean lstAppendEnabled) + throws Exception + { + myLstAppendEnabled = lstAppendEnabled; iw = makeWriter(); iw.writeSymbol("force a local symtab"); SymbolTable symtab = iw.getSymbolTable(); @@ -130,14 +157,8 @@ public void testFlushingUnlockedSymtab() symtab.intern("fred_2"); iw.writeSymbol("fred_1"); - // This would cause an appended LST to be written before the next value. iw.flush(); - IonReader reader = IonReaderBuilder.standard().build(myOutputStream.toByteArray()); - assertEquals(IonType.SYMBOL, reader.next()); - assertEquals("force a local symtab", reader.stringValue()); - assertEquals(IonType.SYMBOL, reader.next()); - assertEquals("fred_1", reader.stringValue()); - assertNull(reader.next()); + return myOutputStream.toByteArray(); } @Test @@ -145,15 +166,34 @@ public void testFlushingUnlockedSymtabWithImports() throws Exception { SimpleCatalog catalog = catalog(); + byte[] bytes = flushUnlockedSymtabWithImports(catalog, false); + assertEquals(0, bytes.length); + } + + @Test + public void testFlushingUnlockedSymtabWithImportsWithLSTAppend() + throws Exception + { + SimpleCatalog catalog = catalog(); + byte[] bytes = flushUnlockedSymtabWithImports(catalog, true); + + IonReader reader = IonReaderBuilder.standard().withCatalog(catalog).build(bytes); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("fred_1", reader.stringValue()); + assertNull(reader.next()); + } + + + private byte[] flushUnlockedSymtabWithImports(SimpleCatalog catalog, boolean lstAppendEnabled) + throws Exception + { SymbolTable fred1 = Symtabs.register("fred", 1, catalog); + myLstAppendEnabled = lstAppendEnabled; iw = makeWriter(fred1); iw.writeSymbol("fred_1"); // This would cause an appended LST to be written before the next value. iw.flush(); - IonReader reader = IonReaderBuilder.standard().withCatalog(catalog).build(myOutputStream.toByteArray()); - assertEquals(IonType.SYMBOL, reader.next()); - assertEquals("fred_1", reader.stringValue()); - assertNull(reader.next()); + return myOutputStream.toByteArray(); } @Test diff --git a/test/com/amazon/ion/impl/IonWriterTestCase.java b/test/com/amazon/ion/impl/IonWriterTestCase.java index 505a756e9e..82f8e61914 100644 --- a/test/com/amazon/ion/impl/IonWriterTestCase.java +++ b/test/com/amazon/ion/impl/IonWriterTestCase.java @@ -72,6 +72,8 @@ enum OutputForm { TEXT, BINARY, DOM } OutputForm myOutputForm; + boolean myLstAppendEnabled = false; + protected IonWriter iw; @Rule @@ -799,6 +801,21 @@ public void testFlushMidValue() public void testFlushDoesNotReset() throws Exception { + flushDoesNotReset(false); + } + + @Test + public void testFlushDoesNotResetWithLstAppend() + throws Exception + { + flushDoesNotReset(true); + } + + private void flushDoesNotReset(boolean lstAppendEnabled) + throws Exception + { + myLstAppendEnabled = lstAppendEnabled; + SymbolTable fred1 = Symtabs.register("fred", 1, catalog()); iw = makeWriter(fred1); iw.writeSymbol("hey"); @@ -807,7 +824,7 @@ public void testFlushDoesNotReset() iw.close(); IonDatagram dg = reload(); - if (myOutputForm == OutputForm.BINARY && iw instanceof _Private_IonManagedWriter) { + if (myOutputForm == OutputForm.BINARY && iw instanceof _Private_IonManagedWriter && myLstAppendEnabled) { // Note: LST append will only be implemented in the "new" binary writer (which implements // _Private_IonMangedWriter) // Should have: IVM SYMTAB hey SYMTAB_APPEND now diff --git a/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java b/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java index 416abc9637..033ad91c36 100644 --- a/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java +++ b/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java @@ -43,6 +43,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; + import org.junit.Test; @SuppressWarnings("deprecation") @@ -81,6 +82,21 @@ public class IonManagedBinaryWriterTest extends IonRawBinaryWriterTest SHARED_SYMBOL_LOCAL_SIDS = unmodifiableMap(sidMap); } + private enum LSTAppendMode + { + LST_APPEND_DISABLED, + LST_APPEND_ENABLED; + public boolean isEnabled() { return this == LST_APPEND_ENABLED; } + } + + @Inject("lstAppendMode") + public static final LSTAppendMode[] LST_APPEND_ENABLED_DIMENSIONS = LSTAppendMode.values(); + private LSTAppendMode lstAppendMode; + public void setLstAppendMode(final LSTAppendMode mode) + { + this.lstAppendMode = mode; + } + private void checkSymbolTokenAgainstImport(final SymbolToken token) { final Integer sid = SHARED_SYMBOL_LOCAL_SIDS.get(token.getText()); @@ -139,12 +155,19 @@ protected IonWriter createWriter(final OutputStream out) throws IOException catalog.putTable(table); } - final IonWriter writer = _Private_IonManagedBinaryWriterBuilder + final _Private_IonManagedBinaryWriterBuilder builder = _Private_IonManagedBinaryWriterBuilder .create(AllocatorMode.POOLED) .withImports(importedSymbolResolverMode, symbolTables) .withPreallocationMode(preallocationMode) - .withFloatBinary32Enabled() - .newWriter(out); + .withFloatBinary32Enabled(); + + if (lstAppendMode.isEnabled()) { + builder.withLocalSymbolTableAppendEnabled(); + } else { + builder.withLocalSymbolTableAppendDisabled(); + } + + final IonWriter writer = builder.newWriter(out); final SymbolTable locals = writer.getSymbolTable(); assertEquals(14, locals.getImportedMaxId()); @@ -190,22 +213,34 @@ public void testLocalSymbolTableAppend() throws Exception writer.flush(); writer.writeSymbol("burrito"); writer.finish(); + IonReader reader = system().newReader(writer.getBytes()); reader.next(); assertEquals(reader.getSymbolTable().findSymbol("taco"), 15); - assertEquals(reader.getSymbolTable().findSymbol("burrito"), -1); + assertEquals(reader.getSymbolTable().findSymbol("burrito"), lstAppendMode.isEnabled() ? -1 : 16); reader.next(); assertEquals(reader.getSymbolTable().findSymbol("taco"), 15); assertEquals(reader.getSymbolTable().findSymbol("burrito"), 16); assertNull(reader.next()); IonDatagram dg = system().getLoader().load(writer.getBytes()); - // Should be IVM SYMTAB taco SYMTAB burrito - assertEquals(5, dg.systemSize()); - assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); - assertEquals("taco", ((IonSymbol) dg.systemGet(2)).stringValue()); - assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(3)).getTypeAnnotations()[0]); - assertEquals("burrito", ((IonSymbol) dg.systemGet(4)).stringValue()); + if (lstAppendMode.isEnabled()) + { + // Should be IVM SYMTAB taco SYMTAB burrito + assertEquals(5, dg.systemSize()); + assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); + assertEquals("taco", ((IonSymbol) dg.systemGet(2)).stringValue()); + assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(3)).getTypeAnnotations()[0]); + assertEquals("burrito", ((IonSymbol) dg.systemGet(4)).stringValue()); + } + else + { + // Should be IVM SYMTAB taco burrito + assertEquals(4, dg.systemSize()); + assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); + assertEquals("taco", ((IonSymbol) dg.systemGet(2)).stringValue()); + assertEquals("burrito", ((IonSymbol) dg.systemGet(3)).stringValue()); + } } @Test @@ -259,22 +294,33 @@ public void testManuallyWriteLSTAppendWithImportsFirst() throws Exception writer.writeSymbol("burrito"); writer.finish(); - IonDatagram dg = system().getLoader().load(writer.getBytes()); - // Should be IVM SYMTAB taco SYMTAB burrito - assertEquals(5, dg.systemSize()); - assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); - assertEquals("taco", ((IonSymbol) dg.systemGet(2)).stringValue()); - assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(3)).getTypeAnnotations()[0]); - assertEquals("burrito", ((IonSymbol) dg.systemGet(4)).stringValue()); - IonReader reader = system().newReader(writer.getBytes()); reader.next(); assertEquals(reader.getSymbolTable().findSymbol("taco"), 15); - assertEquals(reader.getSymbolTable().findSymbol("burrito"), -1); + assertEquals(reader.getSymbolTable().findSymbol("burrito"), lstAppendMode.isEnabled() ? -1 : 16); reader.next(); assertEquals(reader.getSymbolTable().findSymbol("taco"), 15); assertEquals(reader.getSymbolTable().findSymbol("burrito"), 16); assertNull(reader.next()); + + IonDatagram dg = system().getLoader().load(writer.getBytes()); + if (lstAppendMode.isEnabled()) + { + // Should be IVM SYMTAB taco SYMTAB burrito + assertEquals(5, dg.systemSize()); + assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); + assertEquals("taco", ((IonSymbol) dg.systemGet(2)).stringValue()); + assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(3)).getTypeAnnotations()[0]); + assertEquals("burrito", ((IonSymbol) dg.systemGet(4)).stringValue()); + } + else + { + // Should be IVM SYMTAB taco burrito + assertEquals(4, dg.systemSize()); + assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); + assertEquals("taco", ((IonSymbol) dg.systemGet(2)).stringValue()); + assertEquals("burrito", ((IonSymbol) dg.systemGet(3)).stringValue()); + } } @Test @@ -293,22 +339,33 @@ public void testManuallyWriteLSTAppendWithSymbolsFirst() throws Exception writer.writeSymbol("burrito"); writer.finish(); - IonDatagram dg = system().getLoader().load(writer.getBytes()); - // Should be IVM SYMTAB taco SYMTAB burrito - assertEquals(5, dg.systemSize()); - assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); - assertEquals("taco", ((IonSymbol) dg.systemGet(2)).stringValue()); - assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(3)).getTypeAnnotations()[0]); - assertEquals("burrito", ((IonSymbol) dg.systemGet(4)).stringValue()); - IonReader reader = system().newReader(writer.getBytes()); reader.next(); assertEquals(reader.getSymbolTable().findSymbol("taco"), 15); - assertEquals(reader.getSymbolTable().findSymbol("burrito"), -1); + assertEquals(reader.getSymbolTable().findSymbol("burrito"), lstAppendMode.isEnabled() ? -1 : 16); reader.next(); assertEquals(reader.getSymbolTable().findSymbol("taco"), 15); assertEquals(reader.getSymbolTable().findSymbol("burrito"), 16); assertNull(reader.next()); + + IonDatagram dg = system().getLoader().load(writer.getBytes()); + if (lstAppendMode.isEnabled()) + { + // Should be IVM SYMTAB taco SYMTAB burrito + assertEquals(5, dg.systemSize()); + assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); + assertEquals("taco", ((IonSymbol) dg.systemGet(2)).stringValue()); + assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(3)).getTypeAnnotations()[0]); + assertEquals("burrito", ((IonSymbol) dg.systemGet(4)).stringValue()); + } + else + { + // Should be IVM SYMTAB taco burrito + assertEquals(4, dg.systemSize()); + assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); + assertEquals("taco", ((IonSymbol) dg.systemGet(2)).stringValue()); + assertEquals("burrito", ((IonSymbol) dg.systemGet(3)).stringValue()); + } } @Test From d96a94a9d7d41f278761a1e45968d024357b98e0 Mon Sep 17 00:00:00 2001 From: Fernando Raganhan Barbosa Date: Tue, 25 Feb 2020 11:57:13 -0800 Subject: [PATCH 085/490] Release 1.6.0 (#280) --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 17b4327909..ccf0d16542 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ dependency into your project's `pom.xml`: com.amazon.ion ion-java - 1.5.1 + 1.6.0 ``` diff --git a/pom.xml b/pom.xml index 51e20d3858..a8392b69f8 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.5.2-SNAPSHOT + 1.6.0 bundle ${project.groupId}:${project.artifactId} From fddacf3316ac5a0901d493287c632ac32adb5c3f Mon Sep 17 00:00:00 2001 From: Fernando Raganhan Barbosa Date: Tue, 25 Feb 2020 16:48:35 -0800 Subject: [PATCH 086/490] Fixing staging nexus URL (#282) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a8392b69f8..82b22f7e07 100644 --- a/pom.xml +++ b/pom.xml @@ -335,7 +335,7 @@ true ossrh - https://oss.sonatype.org/ + https://aws.oss.sonatype.org/ true From 8b4df7d5254c5479aa7b494111f904852f7ab25a Mon Sep 17 00:00:00 2001 From: Fernando Raganhan Barbosa Date: Wed, 26 Feb 2020 11:48:41 -0800 Subject: [PATCH 087/490] Adding SNAPSHOT version suffix (#283) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 82b22f7e07..2ac31b4935 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.6.0 + 1.6.1-SNAPSHOT bundle ${project.groupId}:${project.artifactId} From a563caf436928ac902b1103fd4c7f89294ac142f Mon Sep 17 00:00:00 2001 From: Therapon Skoteiniotis Date: Thu, 19 Mar 2020 14:19:04 -0700 Subject: [PATCH 088/490] Fix(allocator): Allow GC on freed blocks over 64MB (#285) Quick fix to stop memory leaks. Addressing Issue #225 will help decide on what to do with the block allocator implementations. --- .../ion/impl/bin/PooledBlockAllocatorProvider.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/com/amazon/ion/impl/bin/PooledBlockAllocatorProvider.java b/src/com/amazon/ion/impl/bin/PooledBlockAllocatorProvider.java index 831cb4d2ce..cbd30ae76a 100644 --- a/src/com/amazon/ion/impl/bin/PooledBlockAllocatorProvider.java +++ b/src/com/amazon/ion/impl/bin/PooledBlockAllocatorProvider.java @@ -34,13 +34,15 @@ */ private final class PooledBlockAllocator extends BlockAllocator { - private final int blockSize; + private final int blockSize, blockLimit; private final ConcurrentLinkedQueue freeBlocks; + static final int FREE_CAPACITY = 1024 * 1024 * 64; // 64MB public PooledBlockAllocator(final int blockSize) { this.blockSize = blockSize; this.freeBlocks = new ConcurrentLinkedQueue(); + this.blockLimit = FREE_CAPACITY / blockSize; } @Override @@ -54,8 +56,10 @@ public Block allocateBlock() @Override public void close() { - reset(); - freeBlocks.add(this); + if (freeBlocks.size() < blockLimit) { + reset(); + freeBlocks.add(this); + } } }; } From 5e1be319080bca756e8e963a8adef6b2089077eb Mon Sep 17 00:00:00 2001 From: Therapon Skoteiniotis Date: Thu, 19 Mar 2020 14:51:12 -0700 Subject: [PATCH 089/490] Release 1.6.1 (#286) --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ccf0d16542..76c150391f 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ dependency into your project's `pom.xml`: com.amazon.ion ion-java - 1.6.0 + 1.6.1 ``` diff --git a/pom.xml b/pom.xml index 2ac31b4935..d16e9acb32 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.6.1-SNAPSHOT + 1.6.1 bundle ${project.groupId}:${project.artifactId} From 89add66178d8b276fa5b50bf58e23d0894054686 Mon Sep 17 00:00:00 2001 From: Therapon Skoteiniotis Date: Thu, 19 Mar 2020 16:55:10 -0700 Subject: [PATCH 090/490] Create next SNAPSHOT (#287) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d16e9acb32..bab0e92076 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.6.1 + 1.6.2-SNAPSHOT bundle ${project.groupId}:${project.artifactId} From 5d62f9ab6ee912f2812071fcec9454bd17c3d134 Mon Sep 17 00:00:00 2001 From: Peter Cornell Date: Mon, 20 Apr 2020 16:08:43 -0700 Subject: [PATCH 091/490] Adds a space after a fieldname's colon when pretty-printing (#289) --- src/com/amazon/ion/impl/IonWriterSystemText.java | 3 +++ test/com/amazon/ion/impl/IonMarkupWriterTest.java | 6 +++--- test/com/amazon/ion/impl/TextWriterTest.java | 10 +++++----- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/com/amazon/ion/impl/IonWriterSystemText.java b/src/com/amazon/ion/impl/IonWriterSystemText.java index 0e78e95ccd..f577c26ee5 100644 --- a/src/com/amazon/ion/impl/IonWriterSystemText.java +++ b/src/com/amazon/ion/impl/IonWriterSystemText.java @@ -366,6 +366,9 @@ void startValue() throws IOException SymbolToken sym = assumeFieldNameSymbol(); writeFieldNameToken(sym); _output.appendAscii(':'); + if (_options.isPrettyPrintOn()) { + _output.appendAscii(' '); + } clearFieldName(); followingLongString = false; } diff --git a/test/com/amazon/ion/impl/IonMarkupWriterTest.java b/test/com/amazon/ion/impl/IonMarkupWriterTest.java index b84d7a7da0..f7067d057f 100644 --- a/test/com/amazon/ion/impl/IonMarkupWriterTest.java +++ b/test/com/amazon/ion/impl/IonMarkupWriterTest.java @@ -54,7 +54,7 @@ public class IonMarkupWriterTest extends IonTestCase { "%nabcd" + "::{%n hello" + - ":(%n " + + ": (%n " + "sexp" + "%n 1" + "%n 2" + @@ -62,12 +62,12 @@ public class IonMarkupWriterTest extends IonTestCase { "\"str\"%n " + "),%n " + "list:[%n : [%n 3.2,%n " + "3.2e0" + "%n ]" + ",%n blob" + - ":" + + ": " + "annot::" + "" + "{{ T25lIEJpZyBGYXQgVGVzdCBCbG9iIEZvciBZb3VyIFBsZWFzdXJl }}" + diff --git a/test/com/amazon/ion/impl/TextWriterTest.java b/test/com/amazon/ion/impl/TextWriterTest.java index ba57d634bf..8c8b527bed 100644 --- a/test/com/amazon/ion/impl/TextWriterTest.java +++ b/test/com/amazon/ion/impl/TextWriterTest.java @@ -287,14 +287,14 @@ public void testWritingLongStrings() "%n" + "'''looong'''%n" + "{%n" + - " a:'''looong''',%n" + - " b:\"hello\",%n" + - " c:'''hello\n" + + " a: '''looong''',%n" + + " b: \"hello\",%n" + + " c: '''hello\n" + "nurse''',%n" + - " d:'''what\\'s\n" + + " d: '''what\\'s\n" + "up\n" + "doc'''%n" + - "}" + "}" ), dg ); From 363a4a1550c98c2cdc95205eb11a10358169af8d Mon Sep 17 00:00:00 2001 From: David Lurton Date: Wed, 3 Jun 2020 12:26:37 -0700 Subject: [PATCH 092/490] Fix issue causing test failures in IDE only (#293) EquivsTestCase and OptimiziedBinaryWriterTestCase are base classes that were not marked as `abstract` causing IntelliJ's test runner to attempt treat them like regular, non-abstract test classes which fail. Making them both abstract fixes this issue. --- test/com/amazon/ion/EquivsTestCase.java | 21 +++++++------------ .../impl/OptimizedBinaryWriterTestCase.java | 2 +- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/test/com/amazon/ion/EquivsTestCase.java b/test/com/amazon/ion/EquivsTestCase.java index 7bdce6bb9e..3cb88b345e 100644 --- a/test/com/amazon/ion/EquivsTestCase.java +++ b/test/com/amazon/ion/EquivsTestCase.java @@ -15,28 +15,21 @@ package com.amazon.ion; -import static com.amazon.ion.IonType.DATAGRAM; - -import com.amazon.ion.impl._Private_Utils; -import com.amazon.ion.junit.IonAssert; -import java.io.File; -import org.junit.Test; -import com.amazon.ion.IonDatagram; -import com.amazon.ion.IonSequence; -import com.amazon.ion.IonString; -import com.amazon.ion.IonValue; import com.amazon.ion.impl._Private_Utils; import com.amazon.ion.junit.IonAssert; import com.amazon.ion.system.IonBinaryWriterBuilder; import com.amazon.ion.system.IonSystemBuilder; import com.amazon.ion.system.IonTextWriterBuilder; +import org.junit.Test; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; -import java.util.HashSet; -public class EquivsTestCase +import static com.amazon.ion.IonType.DATAGRAM; + +public abstract class EquivsTestCase extends IonTestCase { /** @@ -198,7 +191,7 @@ protected void checkEquivalence(final IonValue left, } } - public static IonDatagram[] roundTripDatagram(IonDatagram input) throws IOException { + protected static IonDatagram[] roundTripDatagram(IonDatagram input) throws IOException { IonSystem system = IonSystemBuilder.standard().build(); IonLoader loader = system.getLoader(); ByteArrayOutputStream textOutputStream = new ByteArrayOutputStream(); @@ -226,7 +219,7 @@ public static IonDatagram[] roundTripDatagram(IonDatagram input) throws IOExcept return data; } - public void roundTripEquivalence(IonDatagram input, boolean myExpectedEquality) throws IOException { + protected void roundTripEquivalence(IonDatagram input, boolean myExpectedEquality) throws IOException { IonDatagram[] data = roundTripDatagram(input); for(int i = 0; i < data.length; i++){ runEquivalenceChecks(data[i], myExpectedEquality); diff --git a/test/com/amazon/ion/impl/OptimizedBinaryWriterTestCase.java b/test/com/amazon/ion/impl/OptimizedBinaryWriterTestCase.java index 684c8a3eaf..f03707ab21 100644 --- a/test/com/amazon/ion/impl/OptimizedBinaryWriterTestCase.java +++ b/test/com/amazon/ion/impl/OptimizedBinaryWriterTestCase.java @@ -37,7 +37,7 @@ * * @see IonSystemBuilder#withStreamCopyOptimized(boolean) */ -public class OptimizedBinaryWriterTestCase +public abstract class OptimizedBinaryWriterTestCase extends IonTestCase { @Inject("copySpeed") From 0e432eddf34449171f89e94113ed69489ae1400b Mon Sep 17 00:00:00 2001 From: David Lurton Date: Thu, 4 Jun 2020 12:31:49 -0700 Subject: [PATCH 093/490] Avoid nesting IonSequenceLite.SubListView instances --- .../amazon/ion/impl/lite/IonSequenceLite.java | 22 ++++--------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/src/com/amazon/ion/impl/lite/IonSequenceLite.java b/src/com/amazon/ion/impl/lite/IonSequenceLite.java index 5214b79ec3..f8e2fdf418 100644 --- a/src/com/amazon/ion/impl/lite/IonSequenceLite.java +++ b/src/com/amazon/ion/impl/lite/IonSequenceLite.java @@ -340,7 +340,7 @@ public int lastIndexOf(Object o) public List subList(int fromIndex, int toIndex) { - return new SubListView(this, fromIndex, toIndex); + return new SubListView(fromIndex, toIndex); } public IonValue[] toArray() @@ -422,10 +422,8 @@ private class SubListView implements List { private int size; private int structuralModificationCount; - private SubListView(final List parent, - final int fromIndex, - final int toIndex) { - this.fromIndex = toTopLevelFromIndex(parent, fromIndex); + private SubListView(final int fromIndex, final int toIndex) { + this.fromIndex = fromIndex; this.size = toIndex - fromIndex; this.structuralModificationCount = IonSequenceLite.this.structuralModificationCount; } @@ -691,7 +689,7 @@ public void add(final IonValue ionValue) { public List subList(final int fromIndex, final int toIndex) { checkForParentModification(); - return new SubListView(this, fromIndex, toIndex); + return new SubListView(toParentIndex(fromIndex), toParentIndex(toIndex)); } private void rangeCheck(int index) { @@ -708,18 +706,6 @@ private int fromParentIndex(int index) { return index - fromIndex; } - /** - * Calculates fromIndex based on the top level parent which must be an {@link IonSequenceLite}. With this - * nested sublists are able to directly call the top level parent instead of delegating up the parent chain - */ - private int toTopLevelFromIndex(final List parent, int fromIndex) { - if (parent instanceof SubListView) { - return fromIndex + ((SubListView) parent).fromIndex; - } - - return fromIndex; - } - private void checkForParentModification() { if (this.structuralModificationCount != IonSequenceLite.this.structuralModificationCount) { throw new ConcurrentModificationException(); From b3115b27ee572e5d14af261dea445f72b8522d06 Mon Sep 17 00:00:00 2001 From: David Lurton Date: Thu, 4 Jun 2020 16:54:01 -0700 Subject: [PATCH 094/490] Throw exception when IonSequenceLite.subList(int, int) arguments invalid Throw exceptions according to the [List.subList(int, in)](https://docs.oracle.com/javase/7/docs/api/java/util/ArrayList.html#subList(int,%20int)) javadoc. Fixes #295 --- .../amazon/ion/impl/lite/IonSequenceLite.java | 18 +++++++++ .../BaseIonSequenceLiteSublistTestCase.java | 40 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/src/com/amazon/ion/impl/lite/IonSequenceLite.java b/src/com/amazon/ion/impl/lite/IonSequenceLite.java index f8e2fdf418..d7e2eeebc3 100644 --- a/src/com/amazon/ion/impl/lite/IonSequenceLite.java +++ b/src/com/amazon/ion/impl/lite/IonSequenceLite.java @@ -338,8 +338,25 @@ public int lastIndexOf(Object o) return indexOf(o); } + private static void checkSublistParameters(int size, int fromIndex, int toIndex) + { + if(fromIndex < 0) + { + throw new IndexOutOfBoundsException("fromIndex is less than zero"); + } + if(toIndex < fromIndex) + { + throw new IllegalArgumentException("toIndex may not be less than fromIndex"); + } + if(toIndex > size) + { + throw new IndexOutOfBoundsException("toIndex exceeds size"); + } + } + public List subList(int fromIndex, int toIndex) { + checkSublistParameters(this.size(), fromIndex, toIndex); return new SubListView(fromIndex, toIndex); } @@ -688,6 +705,7 @@ public void add(final IonValue ionValue) { } public List subList(final int fromIndex, final int toIndex) { + checkSublistParameters(this.size(), fromIndex, toIndex); checkForParentModification(); return new SubListView(toParentIndex(fromIndex), toParentIndex(toIndex)); } diff --git a/test/com/amazon/ion/impl/lite/BaseIonSequenceLiteSublistTestCase.java b/test/com/amazon/ion/impl/lite/BaseIonSequenceLiteSublistTestCase.java index 2bc94f7cf4..13562eefd1 100644 --- a/test/com/amazon/ion/impl/lite/BaseIonSequenceLiteSublistTestCase.java +++ b/test/com/amazon/ion/impl/lite/BaseIonSequenceLiteSublistTestCase.java @@ -44,6 +44,11 @@ public abstract class BaseIonSequenceLiteSublistTestCase { protected abstract IonSequence newSequence(); + List newSubList() { + List seq = newSequence(); + return seq.subList(0, seq.size()); + } + @Test public void sublistSize() { final IonSequence sequence = newSequence(); @@ -52,6 +57,7 @@ public void sublistSize() { assertEquals(3, actual.size()); } + @Test public void sublistIsEmpty() { IonSequence sequence = newSequence(); @@ -60,6 +66,40 @@ public void sublistIsEmpty() { assertFalse(sequence.subList(0, 1).isEmpty()); } + @Test(expected = IndexOutOfBoundsException.class) + public void subListFromIndexLessThanZero() { + newSequence().subList(-1, 1); + } + + @Test(expected = IllegalArgumentException.class) + public void subListToIndexLessThanFromIndex() { + newSequence().subList(2, 1); + } + + @Test(expected = IndexOutOfBoundsException.class) + public void subListToIndexExceedsSize() { + IonSequence seq = newSequence(); + // toIndex is exclusive, hence the + 1 + seq.subList(0, seq.size() + 1); + } + + @Test(expected = IndexOutOfBoundsException.class) + public void subListOfSubListFromIndexLessThanZero() { + newSubList().subList(-1, 1); + } + + @Test(expected = IllegalArgumentException.class) + public void subListOfSubListToIndexLessThanFromIndex() { + newSubList().subList(2, 1); + } + + @Test(expected = IndexOutOfBoundsException.class) + public void subListOfSubListToIndexExceedsSize() { + List seq = newSubList(); + // toIndex is exclusive, hence the + 1 + seq.subList(0, seq.size() + 1); + } + @Test public void sublistGet() { final IonSequence sequence = newSequence(); From 06752a3be4a2cd2f0ef3cd0165457e88b416b07f Mon Sep 17 00:00:00 2001 From: David Lurton Date: Wed, 10 Jun 2020 17:11:50 -0700 Subject: [PATCH 095/490] Make IonSequence.SubListView implement equals and hashCode (#297) * Make IonSequence.SubListView implmenet equals and hashCode This is accomplished by making it extend AbstractList instead of implementing List. Also added tests. --- pom.xml | 6 + src/com/amazon/ion/IonSequence.java | 112 ++++++++++- .../amazon/ion/impl/lite/IonSequenceLite.java | 153 ++++++++------- .../BaseIonSequenceLiteSublistTestCase.java | 7 +- ...nSequenceLiteSubListViewEqualityTests.java | 180 ++++++++++++++++++ 5 files changed, 375 insertions(+), 83 deletions(-) create mode 100644 test/com/amazon/ion/impl/lite/IonSequenceLiteSubListViewEqualityTests.java diff --git a/pom.xml b/pom.xml index bab0e92076..62ac58ae56 100644 --- a/pom.xml +++ b/pom.xml @@ -76,6 +76,12 @@ 4.12 test + + pl.pragmatists + JUnitParams + 1.1.1 + test + diff --git a/src/com/amazon/ion/IonSequence.java b/src/com/amazon/ion/IonSequence.java index 9621859e59..63995fcaea 100644 --- a/src/com/amazon/ion/IonSequence.java +++ b/src/com/amazon/ion/IonSequence.java @@ -44,10 +44,24 @@ * non-Ion {@link Collection}s. * *

  • - * The method {@link #subList(int, int)} is not implemented at all. - * We think it will be quite challenging to get correct, and decided that - * it was still valuable to extend {@link List} even with this contractual - * violation. + * The implementations of {@link List#equals(Object)}} and + * {@link List#hashCode()} does not conform to the specification of those + * methods in order to conform with Ion's definition of equality which + * takes into account the {@link IonSequence}'s {@link IonType} and its + * annotations in addition to the contents of the collection. + *
      + *
    • + * {@link List#equals(Object)} always returns false + * for any non-{@link IonSequence} implementation of {@link List}, + * including the sub-list returned from + * {@link IonSequence#subList(int, int)}. + *
    • + *
    • + * {@link List#hashCode()} returns a different hash code than + * other {@link List#hashCode()} implementations. including the + * sub-list returned from {@link IonSequence#subList(int, int)}. + *
    • + *
    *
  • * */ @@ -400,16 +414,98 @@ public void add(int index, IonValue child) /** *

    - * Returns a view of the portion of this list according to {@link List#subList(int, int)} - * contract. + * Returns a view of the portion of this list according to + * {@link List#subList(int, int)} contract. *

    * *

    - * Sublist methods will throw a {@link java.util.ConcurrentModificationException} if - * its parent list, i.e. this list, had any changes that affect its size the after sublist + * Sublist methods will throw a + * {@link java.util.ConcurrentModificationException} if its parent list, + * i.e. this list, had any changes that affect its size the after sublist * was created. *

    * + * The implementation of {@link List} returned by this method + * implements {@link List#equals(Object)} and + * {@link List#equals(Object)} ()} per the specification of these methods. + * However, the existing implementation of {@link IonSequence} does not + * provide a specification compliant {@link List#equals} and + * {@link List#hashCode()}} which results to the following caveats: + * + * Given: + * + * + * int[] ints = new int[] {1, 2, 3, 4}; + * IonList list = SYSTEM.newList(ints); + * IonSexp sexp = SYSTEM.newSexp(ints) + * IonSexp dgrm = SYSTEM.newDatagram(ints) + * List listSubList = list.subList(0, ints.size()) + * List sexpSubList = sexp.subList(0, ints.size()) + * List dgrmSubList = sexp.subList(0, ints.size()) + * List arrayList = new ArrayList(); + * for(int i : ints) { arrayList.add(SYSTEM.newInt(i)); } + * + * + * {@link IonSequence#equals(Object)} always returns false when presented + * with a non {@link IonSequence} instance of {@link List}. + * Hence, the following invocations of {@link Object#equals(Object)} + * return false even if the contained elements are equivalent. This + * means that {@link Object#equals(Object)} is not symmetric in these + * cases. The reason for the asymmetry is historical: + * {@link IonSequence} has long violated the contract outlined by the + * {@link List} documentation. For the current major version of this + * library we maintain backwards compatibility and support this behaviour + * as-is. + * + * + * list.equals(listSubList) // false + * list.equals(sexpSubList) // false + * list.equals(dgrm) // false + * list.equals(arrayList) // false + * + * sexp.equals(listSubList) // false + * sexp.equals(sexpSubList) // false + * sexp.equals(dgrm) // false + * sexp.equals(arrayList) // false + * + * dgrm.equals(listSubList) // false + * dgrm.equals(sexpSubList) // false + * dgrm.equals(dgrmSubList) // false + * dgrm.equals(arrayList) // false + * + * + * However, {@link IonSequence#subList(int, int)} was implemented much + * later and faithfully implements {@link List#equals(Object)} meaning + * the cases below all work as expected. While {@link IonSequence} + * does not comply with the specification for {@link List#equals(Object)} + * because it has to comply with Ion's rules for equality, the same is + * not true for sub-lists. Unlike {@link IonSequence}, sub-lists have + * no notion of an {@link IonType}, annotations or nullability which + * allows for compliance with the {@link List} specification. + * + * + * listSubList.equals(listSubList); // true + * listSubList.equals(sexpSubList); // true + * listSubList.equals(dgrmSubList); // true + * listSubList.equals(list); // true + * listSubList.equals(sexp); // true + * listSubList.equals(arrayList); // true + * + * sexpSubList.equals(listSubList); // true + * sexpSubList.equals(sexpSubList); // true + * sexpSubList.equals(dgrmSubList); // true + * sexpSubList.equals(list); // true + * sexpSubList.equals(sexp); // true + * sexpSubList.equals(arrayList); // true + * + * dgrmSubList.equals(listSubList); // true + * dgrmSubList.equals(sexpSubList); // true + * dgrmSubList.equals(dgrmSubList); // true + * dgrmSubList.equals(list); // true + * dgrmSubList.equals(sexp); // true + * dgrmSubList.equals(arrayList); // true + * + * * @see List#subList(int, int) */ public List subList(int fromIndex, int toIndex); diff --git a/src/com/amazon/ion/impl/lite/IonSequenceLite.java b/src/com/amazon/ion/impl/lite/IonSequenceLite.java index d7e2eeebc3..d8ae7ac361 100644 --- a/src/com/amazon/ion/impl/lite/IonSequenceLite.java +++ b/src/com/amazon/ion/impl/lite/IonSequenceLite.java @@ -25,6 +25,7 @@ import com.amazon.ion.impl._Private_IonValue; import java.io.IOException; import java.lang.reflect.Array; +import java.util.AbstractList; import java.util.ArrayList; import java.util.Collection; import java.util.ConcurrentModificationException; @@ -429,7 +430,7 @@ void writeBodyTo(IonWriter writer, SymbolTableProvider symbolTableProvider) * * Structural modifications from the sublist itself are allowed. */ - private class SubListView implements List { + private class SubListView extends AbstractList { /** * index in top level IonSequenceLite that marks the start of this sublist view. For nested @@ -437,24 +438,26 @@ private class SubListView implements List { */ private final int fromIndex; private int size; - private int structuralModificationCount; private SubListView(final int fromIndex, final int toIndex) { this.fromIndex = fromIndex; this.size = toIndex - fromIndex; - this.structuralModificationCount = IonSequenceLite.this.structuralModificationCount; + super.modCount = IonSequenceLite.this.structuralModificationCount; } + @Override public int size() { checkForParentModification(); return size; } + @Override public boolean isEmpty() { checkForParentModification(); return size == 0; } + @Override public IonValue get(final int index) { checkForParentModification(); rangeCheck(index); @@ -462,6 +465,7 @@ public IonValue get(final int index) { return IonSequenceLite.this.get(toParentIndex(index)); } + @Override public IonValue set(final int index, final IonValue element) { checkForParentModification(); rangeCheck(index); @@ -469,53 +473,31 @@ public IonValue set(final int index, final IonValue element) { return IonSequenceLite.this.set(toParentIndex(index), element); } + @Override public boolean contains(final Object o) { checkForParentModification(); - return indexOf(o) != -1; + return super.contains(o); } - public boolean containsAll(final Collection c) { - for (Object o : c) { - if (!contains(o)) { - return false; - } - } - - return true; + @Override + public boolean containsAll(final Collection collection) { + checkForParentModification(); + return super.containsAll(collection); } - public IonValue[] toArray() { + @Override + public Object[] toArray() { checkForParentModification(); - - if (size < 1) { - return EMPTY_VALUE_ARRAY; - } - - return toArray(new IonValue[size]); + return super.toArray(); } - @SuppressWarnings("unchecked") - public T[] toArray(T[] array) { + @Override + public T[] toArray(T[] ts) { checkForParentModification(); - - if (array.length < size) { - final Class type = array.getClass().getComponentType(); - // generates unchecked warning - array = (T[]) Array.newInstance(type, size); - } - - if (size > 0) { - System.arraycopy(IonSequenceLite.this._children, fromIndex, array, 0, size); - } - - if (size < array.length) { - // See IonSequence#toArray and ArrayList#toArray - array[size] = null; - } - - return array; + return super.toArray(ts); } + @Override public boolean add(final IonValue ionValue) { checkForParentModification(); @@ -528,42 +510,42 @@ public boolean add(final IonValue ionValue) { IonSequenceLite.this.add(parentIndex, ionValue); } - this.structuralModificationCount = IonSequenceLite.this.structuralModificationCount; + super.modCount = IonSequenceLite.this.structuralModificationCount; size++; return true; } + @Override public void add(final int index, final IonValue ionValue) { checkForParentModification(); rangeCheck(index); IonSequenceLite.this.add(toParentIndex(index), ionValue); - this.structuralModificationCount = IonSequenceLite.this.structuralModificationCount; + super.modCount = IonSequenceLite.this.structuralModificationCount; size++; } - public boolean addAll(final Collection c) { - for (IonValue ionValue : c) { - add(ionValue); - } - return true; + @Override + public boolean addAll(int i, Collection collection) { + checkForParentModification(); + return super.addAll(i, collection); } - public boolean addAll(final int index, final Collection c) { - int i = index; - for (IonValue ionValue : c) { - add(i, ionValue); - i++; - } - - return true; + @Override + public boolean addAll(Collection collection) { + checkForParentModification(); + return super.addAll(collection); } + // retainAll has no functionality that is specific to SubListView but + // needs to be implemented because the AbstractList.retainAll() throws [UnsupportedOperationException]. + @Override public boolean retainAll(final Collection c) { + checkForParentModification(); if (size < 1) { return false; } @@ -585,6 +567,7 @@ public boolean retainAll(final Collection c) { return removeAll(toRemove); } + @Override public void clear() { checkForParentModification(); @@ -595,22 +578,27 @@ public void clear() { } size = 0; - this.structuralModificationCount = IonSequenceLite.this.structuralModificationCount; + super.modCount = IonSequenceLite.this.structuralModificationCount; } + @Override public IonValue remove(final int index) { checkForParentModification(); rangeCheck(index); final IonValue removed = IonSequenceLite.this.remove(toParentIndex(index)); - this.structuralModificationCount = IonSequenceLite.this.structuralModificationCount; + super.modCount = IonSequenceLite.this.structuralModificationCount; size--; return removed; } + // remove(Object) contains no functionality that is specific to SubListView but + // needs to be implemented because the AbstractList.remove(Object) throws [UnsupportedOperationException]. + @Override public boolean remove(final Object o) { + checkForParentModification(); int index = indexOf(o); if (index < 0) { return false; @@ -620,7 +608,11 @@ public boolean remove(final Object o) { return true; } + // removeAll(Collection) contains no functionality that is specific to SubListView but + // needs to be implemented because the AbstractList.remove(Object) throws [UnsupportedOperationException]. + @Override public boolean removeAll(final Collection c) { + checkForParentModification(); boolean changed = false; for (Object o : c) { if (remove(o)) { @@ -631,32 +623,36 @@ public boolean removeAll(final Collection c) { return changed; } + @Override public int indexOf(final Object o) { checkForParentModification(); + return super.indexOf(o); + } - final int parentIndex = IonSequenceLite.this.indexOf(o); - final int index = fromParentIndex(parentIndex); - - // not found - if (parentIndex < 0 || index < 0 || index >= size) { - return -1; - } - - return index; + @Override + public int lastIndexOf(Object o) { + checkForParentModification(); + return super.lastIndexOf(o); } - public int lastIndexOf(final Object o) { - return indexOf(o); + @Override + protected void removeRange(int from, int to) { + checkForParentModification(); + super.removeRange(from, to); } + + @Override public Iterator iterator() { return listIterator(0); } + @Override public ListIterator listIterator() { return listIterator(0); } + @Override public ListIterator listIterator(final int index) { checkForParentModification(); @@ -704,12 +700,31 @@ public void add(final IonValue ionValue) { }; } + @Override public List subList(final int fromIndex, final int toIndex) { checkSublistParameters(this.size(), fromIndex, toIndex); checkForParentModification(); return new SubListView(toParentIndex(fromIndex), toParentIndex(toIndex)); } + @Override + public boolean equals(Object o) { + checkForParentModification(); + return super.equals(o); + } + + @Override + public int hashCode() { + checkForParentModification(); + return super.hashCode(); + } + + @Override + public String toString() { + checkForParentModification(); + return super.toString(); + } + private void rangeCheck(int index) { if (index < 0 || index >= size) { throw new IndexOutOfBoundsException(String.valueOf(index)); @@ -720,12 +735,8 @@ private int toParentIndex(int index) { return index + fromIndex; } - private int fromParentIndex(int index) { - return index - fromIndex; - } - private void checkForParentModification() { - if (this.structuralModificationCount != IonSequenceLite.this.structuralModificationCount) { + if (super.modCount != IonSequenceLite.this.structuralModificationCount) { throw new ConcurrentModificationException(); } } diff --git a/test/com/amazon/ion/impl/lite/BaseIonSequenceLiteSublistTestCase.java b/test/com/amazon/ion/impl/lite/BaseIonSequenceLiteSublistTestCase.java index 13562eefd1..67b60f8fb8 100644 --- a/test/com/amazon/ion/impl/lite/BaseIonSequenceLiteSublistTestCase.java +++ b/test/com/amazon/ion/impl/lite/BaseIonSequenceLiteSublistTestCase.java @@ -15,16 +15,13 @@ package com.amazon.ion.impl.lite; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - import java.util.Arrays; import java.util.Collections; import java.util.ConcurrentModificationException; import java.util.Iterator; import java.util.List; import java.util.ListIterator; + import org.junit.Test; import com.amazon.ion.ContainedValueException; import com.amazon.ion.IonInt; @@ -33,6 +30,8 @@ import com.amazon.ion.IonValue; import com.amazon.ion.system.IonSystemBuilder; +import static org.junit.Assert.*; + /** * All tests related to {@link IonSequenceLite#subList(int, int)}. Extracted to a separate test due to the amount of * tests diff --git a/test/com/amazon/ion/impl/lite/IonSequenceLiteSubListViewEqualityTests.java b/test/com/amazon/ion/impl/lite/IonSequenceLiteSubListViewEqualityTests.java new file mode 100644 index 0000000000..292b4a7bd2 --- /dev/null +++ b/test/com/amazon/ion/impl/lite/IonSequenceLiteSubListViewEqualityTests.java @@ -0,0 +1,180 @@ +package com.amazon.ion.impl.lite; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import com.amazon.ion.IonSequence; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonValue; +import com.amazon.ion.system.IonSystemBuilder; +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +@RunWith(JUnitParamsRunner.class) +public class IonSequenceLiteSubListViewEqualityTests { + static final IonSystem SYSTEM = IonSystemBuilder.standard().build(); + private static List INTS = Arrays.asList(0, 1, 2, 3, 4, 5, 6); + private static List OTHER_INTS = Arrays.asList(0, 1, 2, 33, 4, 5, 6); + private static List ARRAY_LIST = newArrayList(INTS); + private static List OTHER_ARRAY_LIST = newArrayList(OTHER_INTS); + + private interface SublistMaker { + List makeSublist(List seq); + } + + private static void runSubListEquivalenceTests(List seq, SublistMaker maker) { + testSublistEquivalence(seq, maker); + testSublistNonEquivalence(seq, maker); + } + + // makes a sublist of seq and asserts its equivalence to sublists of all sequence types and to a sublist + // derived from ARRAY_LIST + private static void testSublistEquivalence(List seq, SublistMaker maker) { + List subList = maker.makeSublist(seq); + for(IonSequence otherSeq : getTestSequences()) { + List otherSubList = maker.makeSublist(otherSeq); + assertEquals("subLists should be equivalent", subList, otherSubList); + assertEquals("subLists should have the same hashCode", subList.hashCode(), otherSubList.hashCode()); + } + + List ararySubList = maker.makeSublist(ARRAY_LIST); + assertEquals("subList should be equivalent to a sublist of ARRAY_LIST", + ararySubList, subList); + + assertEquals("subList should have the same hash code as a sublist of ARRAY_LIST", + ararySubList.hashCode(), subList.hashCode()); + } + + // makes a sublist of seq and asserts its non-equivalence to the same subList range of the "other" sequence + // and to a sublist derived from OTHER_ARRAY_LIST + private static void testSublistNonEquivalence(List seq, SublistMaker maker) { + // A hash collision is unlikely but possible so we do not assert that the hashCodes are different here. + + List subList = maker.makeSublist(seq); + for(IonSequence otherSeq : getOtherTestSequences()) { + List otherSubList = maker.makeSublist(otherSeq); + assertNotEquals("subLists should *not* be equivalent", subList, otherSubList); + } + + assertNotEquals("subList should *not* be equivalent to a sublist of OTHER_ARRAY_LIST", + maker.makeSublist(OTHER_ARRAY_LIST), subList); + } + + private static IonSequence[] getParameters(List ints) { + return new IonSequence[] { + populateSequence(SYSTEM.newList(), ints), + populateSequence(SYSTEM.newSexp(), ints), + populateSequence(SYSTEM.newDatagram(), ints) + }; + } + + + @Test + @Parameters(method = "getTestSequences") + public void fullSubList(IonSequence seq) { + SublistMaker maker = new SublistMaker() { + @Override + public List makeSublist(List seq) { + return seq.subList(0, seq.size()); + } + }; + runSubListEquivalenceTests(seq, maker); + + // The sublist should also be equivalent to the ARRAY_LIST + List subList = maker.makeSublist(seq); + assertEquals("The full sublist should be equivalent to the ArrayList makeSublist(List seq) { + return seq.subList(0, seq.size()).subList(0, seq.size()); + } + }; + runSubListEquivalenceTests(seq, maker); + } + + @Test + @Parameters(method = "getTestSequences") + public void partialSubList(IonSequence seq) { + SublistMaker maker = new SublistMaker() { + @Override + public List makeSublist(List seq) { + return seq.subList(1, seq.size() - 1); + } + }; + runSubListEquivalenceTests(seq, maker); + } + + @Test + @Parameters(method = "getTestSequences") + public void partialSubListOfPartialSubList(IonSequence seq) { + SublistMaker maker = new SublistMaker() { + @Override + public List makeSublist(List seq) { + return seq.subList(1, seq.size() - 1).subList(1, seq.size() - 2); + } + }; + runSubListEquivalenceTests(seq, maker); + } + + @Test + @Parameters(method = "getTestSequences") + public void sequenceEquivalenceToSubList(IonSequence seq) { + List subList = seq.subList(0, seq.size()); + + // Note the asymmetry. This is expected. See IonSequence.subList javadoc. + assertEquals("the sublist should be equivalent to the sequence", subList, seq); + assertNotEquals("the sequence should *not* be equivalent to the subList", seq, subList); + } + + @Test + @Parameters(method = "getTestSequences") + public void subListNotEquivalentToNull(IonSequence seq) { + List subList = seq.subList(0, seq.size()); + assertNotEquals("subList should not be equivalent to null", subList, null); + } + + @Test + @Parameters(method = "getTestSequences") + public void subListEquivalentToSelf(IonSequence seq) { + List subList = seq.subList(0, seq.size()); + assertEquals("subList should be equivalent to itself", subList, subList); + } + + public static IonSequence[] getTestSequences() { + return getParameters(INTS); + } + + public static IonSequence[] getOtherTestSequences() { + return getParameters(OTHER_INTS); + } + + private static IonSequence populateSequence(IonSequence seq, List ints) { + for(int i : ints) { + seq.add(SYSTEM.newInt(i)); + } + return seq; + } + + private static ArrayList newArrayList(List ints) { + ArrayList values = new ArrayList(); + for(int i : ints) { + values.add(SYSTEM.newInt(i)); + } + return values; + } +} From b6842d54245fdef9366236856130d9cb086b6f37 Mon Sep 17 00:00:00 2001 From: David Lurton Date: Mon, 10 Aug 2020 15:00:31 -0700 Subject: [PATCH 096/490] Remove comment suggesting `IonWriter` isn't stable --- src/com/amazon/ion/IonWriter.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/com/amazon/ion/IonWriter.java b/src/com/amazon/ion/IonWriter.java index 99270c6eae..2f0f100247 100644 --- a/src/com/amazon/ion/IonWriter.java +++ b/src/com/amazon/ion/IonWriter.java @@ -34,8 +34,6 @@ *

    * WARNING: This interface should not be implemented or extended by * code outside of this library. - * We still have some work to do before this interface is stable. - * See issue amzn/ion-java/issues/10 *

    * A value is written via the set of typed {@code write*()} methods such as * {@link #writeBool(boolean)} and {@link #writeInt(long)}. From 56cbf645e6e34dc3970c92443512de59dac7a633 Mon Sep 17 00:00:00 2001 From: David Lurton Date: Fri, 14 Aug 2020 17:14:01 -0700 Subject: [PATCH 097/490] Adds Timestamp.forEpochSecond() (#302) * Adds Timestamp.forEpochSeconds() Implements #301. --- src/com/amazon/ion/Timestamp.java | 151 ++++++++++++++++--------- test/com/amazon/ion/TimestampTest.java | 117 ++++++++++++++++++- 2 files changed, 215 insertions(+), 53 deletions(-) diff --git a/src/com/amazon/ion/Timestamp.java b/src/com/amazon/ion/Timestamp.java index 39df6e0827..591d865222 100644 --- a/src/com/amazon/ion/Timestamp.java +++ b/src/com/amazon/ion/Timestamp.java @@ -84,18 +84,37 @@ public final class Timestamp private static final int NO_SECONDS = 0; private static final BigDecimal NO_FRACTIONAL_SECONDS = null; - // 0001-01-01T00:00:00.0Z in millis + /** + * 0001-01-01T00:00:00.0Z in millis. + */ static final long MINIMUM_TIMESTAMP_IN_MILLIS = -62135769600000L; - // 0001-01-01T00:00:00.0Z in millis + /** + * 0001-01-01T00:00:00.0Z in millis. + */ static final BigDecimal MINIMUM_TIMESTAMP_IN_MILLIS_DECIMAL = new BigDecimal(MINIMUM_TIMESTAMP_IN_MILLIS); - // 10000T in millis, upper bound exclusive + /** + * 10000T in millis, upper bound exclusive. + */ static final long MAXIMUM_TIMESTAMP_IN_MILLIS = 253402300800000L; - // 10000T in millis, upper bound exclusive + /** + * 10000T in millis, upper bound exclusive. + */ static final BigDecimal MAXIMUM_ALLOWED_TIMESTAMP_IN_MILLIS_DECIMAL = new BigDecimal(MAXIMUM_TIMESTAMP_IN_MILLIS); + + /** + * 0001-01-01T00:00:00.0Z epoch seconds. + */ + static final long MINIMUM_TIMESTAMP_IN_EPOCH_SECONDS = MINIMUM_TIMESTAMP_IN_MILLIS / 1000; + + /** + * 10000T in epoch seconds, upper bound exclusive. + */ + static final long MAXIMUM_TIMESTAMP_IN_EPOCH_SECONDS = MAXIMUM_TIMESTAMP_IN_MILLIS / 1000; + /** * Unknown local offset from UTC. */ @@ -1254,16 +1273,19 @@ public static Timestamp forDay(int yearZ, int monthZ, int dayZ) /** * Returns a Timestamp, precise to the minute, with a given local * offset. - *

    - * This is equivalent to the corresponding Ion value + * + *

    This is equivalent to the corresponding Ion value * {@code YYYY-MM-DDThh:mm+-oo:oo}, where {@code oo:oo} represents the - * hour and minutes of the local offset from UTC. + * hour and minutes of the local offset from UTC.

    + * + *

    The values of the {@code year}, {@code month}, {@code day}, + * {@code hour}, and {@code minute} parameters are relative to the + * local {@code offset}.

    * * @param offset * the local offset from UTC, measured in minutes; * may be {@code null} to represent an unknown local offset * - */ public static Timestamp forMinute(int year, int month, int day, int hour, int minute, @@ -1275,10 +1297,14 @@ public static Timestamp forMinute(int year, int month, int day, /** * Returns a Timestamp, precise to the second, with a given local offset. - *

    - * This is equivalent to the corresponding Ion value + * + *

    This is equivalent to the corresponding Ion value * {@code YYYY-MM-DDThh:mm:ss+-oo:oo}, where {@code oo:oo} represents the - * hour and minutes of the local offset from UTC. + * hour and minutes of the local offset from UTC.

    + * + *

    The values of the {@code year}, {@code month}, {@code day}, + * {@code hour}, {@code minute} and {@code second} parameters are relative + * to the local {@code offset}.

    * * @param offset * the local offset from UTC, measured in minutes; @@ -1296,19 +1322,19 @@ public static Timestamp forSecond(int year, int month, int day, /** * Returns a Timestamp, precise to the second, with a given local offset. - *

    - * This is equivalent to the corresponding Ion value + * + *

    This is equivalent to the corresponding Ion value * {@code YYYY-MM-DDThh:mm:ss.sss+-oo:oo}, where {@code oo:oo} represents - * the hour and minutes of the local offset from UTC. + * the hour and minutes of the local offset from UTC.

    * - * @param second must be at least zero and less than 60. - * Must not be null. + *

    The values of the {@code year}, {@code month}, {@code day}, + * {@code hour}, {@code minute} and {@code second} parameters are relative + * to the local {@code offset}.

    * - * @param offset - * the local offset from UTC, measured in minutes; - * may be {@code null} to represent an unknown local offset + * @param second must be at least zero and less than 60. Must not be null. + * @param offset the local offset from UTC, measured in minutes; + * may be {@code null} to represent an unknown local offset * - */ public static Timestamp forSecond(int year, int month, int day, int hour, int minute, BigDecimal second, @@ -1323,46 +1349,33 @@ public static Timestamp forSecond(int year, int month, int day, /** - * Returns a Timestamp that represents the point in time that is - * {@code millis} milliseconds from the epoch, with a given local offset. - *

    - * The resulting Timestamp will be precise to the millisecond. + * Returns a Timestamp that represents the point in time that is {@code millis} milliseconds from the epoch, + * with a given local offset. * - * @param millis - * the number of milliseconds from the epoch (1970-01-01T00:00:00.000Z). - * @param localOffset - * the local offset from UTC, measured in minutes; - * may be {@code null} to represent an unknown local offset. + *

    The resulting Timestamp will be precise to the millisecond.

    * - + *

    {@code millis} is relative to UTC, regardless of the value supplied for {@code localOffset}. This + * varies from the {@link #forMinute} and {@link #forSecond} methods that assume the specified date and time + * values are relative to the local offset. For example, the following two Timestamps represent + * the same point in time:

    + * + * + * Timestamp theEpoch = Timestamp.forMillis(0, 0); + * Timestamp alsoTheEpoch = Timestamp.forSecond(1969, 12, 31, 23, 0, BigDecimal.ZERO, -60); + * + * + * @param millis the number of milliseconds from the epoch (1970-01-01T00:00:00.000Z) in UTC. + * @param localOffset the local offset from UTC, measured in minutes; + * may be {@code null} to represent an unknown local offset. */ public static Timestamp forMillis(long millis, Integer localOffset) { return new Timestamp(millis, localOffset); } - /** - * Returns a Timestamp that represents the point in time that is - * {@code millis} milliseconds (including any fractional - * milliseconds) from the epoch, with a given local offset. - * - *

    - * The resulting Timestamp will be precise to the second if {@code millis} - * doesn't contain information that is more granular than seconds. - * For example, a {@code BigDecimal} of - * value 132541995e4 (132541995 × 104) - * will return a Timestamp of {@code 2012-01-01T12:12:30Z}, - * precise to the second. - * - *

    - * The resulting Timestamp will be precise to the fractional second if - * {@code millis} contains information that is at least granular to - * milliseconds. - * For example, a {@code BigDecimal} of - * value 1325419950555 - * will return a Timestamp of {@code 2012-01-01T12:12:30.555Z}, - * precise to the fractional second. + * The same as {@link #forMillis(long, Integer)} but the millisecond component is specified using a + * {@link BigDecimal} and therefore may include fractional milliseconds. * * @param millis * number of milliseconds (including any fractional @@ -1374,7 +1387,6 @@ public static Timestamp forMillis(long millis, Integer localOffset) * * @throws NullPointerException if {@code millis} is {@code null} * - */ public static Timestamp forMillis(BigDecimal millis, Integer localOffset) { @@ -1382,6 +1394,41 @@ public static Timestamp forMillis(BigDecimal millis, Integer localOffset) } + /** + * Returns a Timestamp that represents the point in time that is {@code seconds} from the unix epoch + * (1970-01-01T00:00:00.000Z), with the {@code nanoOffset} applied and a given local offset. + * + *

    This function is intended to allow easy conversion to Timestamp from Java 8's {@code java.time.Instant} + * without having to port this library to Java 8. The following snippet will yield a Timestamp {@code ts} + * that equivalent to the {@code java.time.Instant} {@code i}:

    + * + * + * Instant i = Instant.now(); + * Timestamp ts = Timestamp.forEpochSecond(i.getEpochSecond(), i.getNano(), 0); + * + * + *

    Like {@link #forMillis}, {@code seconds} is relative to UTC, regardless of the value + * supplied for {@code localOffset}.

    + * + * @param seconds The number of seconds from the unix epoch (1970-01-01T00:00:00.000Z) UTC. + * @param nanoOffset The number of nanoseconds for the fractional component. Must be between 0 and 999,999,999. + * @param localOffset the local offset from UTC, measured in minutes; + * may be {@code null} to represent an unknown local offset. + */ + public static Timestamp forEpochSecond(long seconds, int nanoOffset, Integer localOffset) { + // We do not validate seconds here because that is done within the forMillis static constructor. + long millis = seconds * 1000L; + Timestamp ts = forMillis(millis, localOffset); + if(nanoOffset != 0) { + if(nanoOffset < 0 || nanoOffset > 999999999) { + throw new IllegalArgumentException("nanoOffset must be between 0 and 999,999,999"); + } + ts._fraction = ts._fraction.add(BigDecimal.valueOf(nanoOffset).movePointLeft(9)); + } + return ts; + } + + /** * Converts a {@link Calendar} to a Timestamp, preserving the calendar's * time zone as the equivalent local offset when it has at least minutes diff --git a/test/com/amazon/ion/TimestampTest.java b/test/com/amazon/ion/TimestampTest.java index eef97cb463..fb9129cf2d 100644 --- a/test/com/amazon/ion/TimestampTest.java +++ b/test/com/amazon/ion/TimestampTest.java @@ -42,7 +42,9 @@ import java.util.TimeZone; import org.junit.Ignore; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; /** * Validates Ion date parsing, specified as per W3C but with requiring @@ -53,6 +55,9 @@ public class TimestampTest extends IonTestCase { + @Rule + public ExpectedException thrown = ExpectedException.none(); + /** * Earliest Ion timestamp possible, that is, "0001-01-01". */ @@ -761,9 +766,14 @@ public void testNewTimestamp() ts = new Timestamp(2010, 2, 1, 10, 11, 12, BigDecimal.ZERO, 0); assertEquals(Timestamp.valueOf("2010-02-01T10:11:12Z"), ts); checkFields(2010, 2, 1, 10, 11, 12, null, 0, SECOND, ts); + } - // New static method unifies decimal seconds + @Test + public void testForSecond() { + BigDecimal fraction = new BigDecimal(".34"); BigDecimal second = new BigDecimal("12.34"); + + Timestamp ts; ts = Timestamp.forSecond(2010, 2, 1, 10, 11, second, PST_OFFSET); checkFields(2010, 2, 1, 10, 11, 12, fraction, PST_OFFSET, FRACTION, ts); assertEquals("2010-02-01T10:11:12.34-08:00", ts.toString()); @@ -781,6 +791,111 @@ public void testNewTimestamp() assertEquals("2010-02-01T18:11:12Z", ts.toZString()); } + @Test + public void forEpochSecondTest() { + // Unix epoch + assertEquals( + Timestamp.valueOf("1970-01-01T00:00:00.000Z"), + Timestamp.forEpochSecond(0, 0, 0)); + + // One day before unix epoch + assertEquals( + Timestamp.valueOf("1969-12-31T00:00:00.000Z"), + Timestamp.forEpochSecond(-86400, 0, 0)); + + // One day after unix epoch + assertEquals( + Timestamp.valueOf("1970-01-02T00:00:00.000Z"), + Timestamp.forEpochSecond(86400, 0, 0)); + + // Maximum timestamp value (that can be created with forEpochSecond) + assertEquals( + Timestamp.valueOf("9999-12-31T23:59:59.999999999Z"), + Timestamp.forEpochSecond(Timestamp.MAXIMUM_TIMESTAMP_IN_EPOCH_SECONDS-1, 999999999, 0)); + + // Minimum timestamp value (that can be created with forEpochSecond) + assertEquals( + Timestamp.valueOf("0001-01-01T00:00:00.000Z"), + Timestamp.forEpochSecond(Timestamp.MINIMUM_TIMESTAMP_IN_EPOCH_SECONDS, 0, 0)); + + // Without fractional component + assertEquals( + Timestamp.valueOf("2009-01-20T20:17:00.000Z"), + Timestamp.forEpochSecond(1232482620, 0, 0)); + + // With fractional component (1/2 second) + assertEquals( + Timestamp.valueOf("2009-01-20T20:17:00.500000000Z"), + Timestamp.forEpochSecond(1232482620, 500000000, 0)); + + // With fractional component (one nanosecond) + assertEquals( + Timestamp.valueOf("2009-01-20T20:17:00.000000001Z"), + Timestamp.forEpochSecond(1232482620, 1, 0)); + + // With fractional component (999,999,999 nanoseconds) + assertEquals( + Timestamp.valueOf("2009-01-20T20:17:00.999999999Z"), + Timestamp.forEpochSecond(1232482620, 999999999, 0)); + + // With unknown local offset + assertEquals( + Timestamp.valueOf("1970-01-01T00:00:00.000-00:00"), + Timestamp.forEpochSecond(0, 0, null)); + + // With a local offset + assertEquals( + Timestamp.valueOf("1970-01-01T01:00:00.000+01:00"), + Timestamp.forEpochSecond(0, 0, 60)); + + // With negative local offset + assertEquals( + Timestamp.valueOf("1969-12-31T23:00:00.000-01:00"), + Timestamp.forEpochSecond(0, 0, -60)); + } + + @Test + public void forEpochSecondTestSecondsTooLow() { + thrown.expect(IllegalArgumentException.class); + // MINIMUM_TIMESTAMP_IN_EPOCH_SECONDS is inclusive + Timestamp.forEpochSecond(Timestamp.MINIMUM_TIMESTAMP_IN_EPOCH_SECONDS - 1, 0, 0); + } + + @Test + public void forEpochSecondTestSecondsTooHigh() { + thrown.expect(IllegalArgumentException.class); + // MAXIMUM_TIMESTAMP_IN_EPOCH_SECONDS is exclusive + Timestamp.forEpochSecond(Timestamp.MAXIMUM_TIMESTAMP_IN_EPOCH_SECONDS, 0, 0); + } + + @Test + public void forEpochSecondTestNanosTooLow() { + thrown.expect(IllegalArgumentException.class); + Timestamp.forEpochSecond(0, -1, 0); + } + + @Test + public void forEpochSecondTestNanosTooHigh() { + thrown.expect(IllegalArgumentException.class); + Timestamp.forEpochSecond(0, 1000000000, 0); + } + + @Test + @Ignore("https://github.com/amzn/ion-java/issues/303") + public void forEpochSecondTesOffsetTooLow() { + thrown.expect(IllegalArgumentException.class); + Timestamp.forEpochSecond(0, 0, -24 * 60); + } + + @Test + @Ignore("https://github.com/amzn/ion-java/issues/303") + public void forEpochSecondTestOffsetTooHigh() { + thrown.expect(IllegalArgumentException.class); + Timestamp.forEpochSecond(0, 0, 24 * 60); + } + + + /** Test for {@link Timestamp#Timestamp(BigDecimal, Integer)} */ @Test public void testNewTimestampFromBigDecimal() From 392581921f1bbdbba9dd8eb541680f24faec2220 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Tue, 18 Aug 2020 17:24:30 -0700 Subject: [PATCH 098/490] Sets LocalSymbolTable's initial map size to avoid guaranteed resizing within the constructor. --- src/com/amazon/ion/impl/LocalSymbolTable.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/com/amazon/ion/impl/LocalSymbolTable.java b/src/com/amazon/ion/impl/LocalSymbolTable.java index dde4bb8d67..93480ec5b4 100644 --- a/src/com/amazon/ion/impl/LocalSymbolTable.java +++ b/src/com/amazon/ion/impl/LocalSymbolTable.java @@ -168,7 +168,10 @@ protected LocalSymbolTable(LocalSymbolTableImports imports, List symbols myFirstLocalSid = myImportsList.getMaxId() + 1; // Copy locally declared symbols to mySymbolsMap - mySymbolsMap = new HashMap(); + // The initial size is chosen so that resizing is avoided. The default load factor is 0.75. Resizing + // could also be avoided by setting the initial size to mySymbolsCount and setting the load factor to + // 1.0, but this would lead to more hash collisions. + mySymbolsMap = new HashMap((int) Math.ceil(mySymbolsCount / 0.75)); buildSymbolsMap(); } From c0799556609b904e8c1e2c80c10ee7caf99a7f8c Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Mon, 31 Aug 2020 18:51:51 -0700 Subject: [PATCH 099/490] Improves read performance of streams that use LST append with a large number of symbols. --- src/com/amazon/ion/impl/LocalSymbolTable.java | 50 +++++++++------- .../ion/impl/LocalSymbolTableAsStruct.java | 7 ++- .../amazon/ion/impl/IonWriterTestCase.java | 20 +------ test/com/amazon/ion/impl/SymbolTableTest.java | 34 ++++++----- .../impl/bin/IonManagedBinaryWriterTest.java | 60 +++---------------- 5 files changed, 66 insertions(+), 105 deletions(-) diff --git a/src/com/amazon/ion/impl/LocalSymbolTable.java b/src/com/amazon/ion/impl/LocalSymbolTable.java index 93480ec5b4..b5990d5cce 100644 --- a/src/com/amazon/ion/impl/LocalSymbolTable.java +++ b/src/com/amazon/ion/impl/LocalSymbolTable.java @@ -64,11 +64,16 @@ public SymbolTable newLocalSymtab(IonCatalog catalog, boolean alreadyInStruct) { List symbolsList = new ArrayList(); + SymbolTable currentSymbolTable = reader.getSymbolTable(); LocalSymbolTableImports imports = readLocalSymbolTable(reader, catalog, alreadyInStruct, symbolsList, - reader.getSymbolTable()); + currentSymbolTable); + if (imports == null) { + // This was an LST append, so the existing symbol table was updated. + return currentSymbolTable; + } return new LocalSymbolTable(imports, symbolsList); } @@ -201,11 +206,21 @@ protected LocalSymbolTable(LocalSymbolTable other, int maxId) } } + /** + * Parses the symbol table at the reader's current position. + * @param reader the reader from which to parse the symbol table. + * @param catalog the catalog from which to resolve shared symbol table imports. + * @param isOnStruct true if the reader is already positioned on the symbol table struct; otherwise, false. + * @param symbolsListOut list into which local symbols declared by the parsed symbol table will be deposited. + * @param currentSymbolTable the symbol table currently active in the stream. + * @return a new LocalSymbolTableImports instance, or null if this was an LST append. If null, `currentSymbolTable` + * continues to be the active symbol table. + */ protected static LocalSymbolTableImports readLocalSymbolTable(IonReader reader, IonCatalog catalog, boolean isOnStruct, List symbolsListOut, - SymbolTable symbolTable) + SymbolTable currentSymbolTable) { if (! isOnStruct) { @@ -223,14 +238,11 @@ protected static LocalSymbolTableImports readLocalSymbolTable(IonReader reader, List importsList = new ArrayList(); importsList.add(reader.getSymbolTable().getSystemSymbolTable()); - // Isolate the newly-declared symbols because they must be added after any symbols declared - // by the previous symbol table if this is an appending local symbol table, but the 'imports' and - // 'symbols' declarations can occur in any order. - List newSymbols = new ArrayList(); IonType fieldType; boolean foundImportList = false; boolean foundLocalSymbolList = false; + boolean isAppend = false; while ((fieldType = reader.next()) != null) { if (reader.isNullValue()) continue; @@ -275,7 +287,7 @@ protected static LocalSymbolTableImports readLocalSymbolTable(IonReader reader, text = null; } - newSymbols.add(text); + symbolsListOut.add(text); } reader.stepOut(); } @@ -291,18 +303,9 @@ protected static LocalSymbolTableImports readLocalSymbolTable(IonReader reader, { prepImportsList(importsList, reader, catalog); } - else if (fieldType == IonType.SYMBOL) + else if (fieldType == IonType.SYMBOL && ION_SYMBOL_TABLE.equals(reader.stringValue())) { - // trying to import the current table - if(symbolTable.isLocalTable() && ION_SYMBOL_TABLE.equals(reader.stringValue())) - { - SymbolTable currentSymbolTable = reader.getSymbolTable(); - importsList.addAll(Arrays.asList(currentSymbolTable.getImportedTables())); - Iterator currentSymbols = currentSymbolTable.iterateDeclaredSymbolNames(); - while (currentSymbols.hasNext()) { - symbolsListOut.add(currentSymbols.next()); - } - } + isAppend = true; } break; } @@ -313,9 +316,16 @@ else if (fieldType == IonType.SYMBOL) } } } - reader.stepOut(); - symbolsListOut.addAll(newSymbols); + if (isAppend && currentSymbolTable.isLocalTable()) { + // Because the current symbol table is a local symbol table (i.e. not the system symbol table), it can + // be appended in-place. + LocalSymbolTable currentLocalSymbolTable = (LocalSymbolTable) currentSymbolTable; + for (String newSymbol : symbolsListOut) { + currentLocalSymbolTable.putSymbol(newSymbol); + } + return null; + } return new LocalSymbolTableImports(importsList); } diff --git a/src/com/amazon/ion/impl/LocalSymbolTableAsStruct.java b/src/com/amazon/ion/impl/LocalSymbolTableAsStruct.java index 3ac9e674ae..98d325174b 100644 --- a/src/com/amazon/ion/impl/LocalSymbolTableAsStruct.java +++ b/src/com/amazon/ion/impl/LocalSymbolTableAsStruct.java @@ -62,11 +62,16 @@ public SymbolTable newLocalSymtab(IonCatalog catalog, boolean alreadyInStruct) { List symbolsList = new ArrayList(); + SymbolTable currentSymbolTable = reader.getSymbolTable(); LocalSymbolTableImports imports = readLocalSymbolTable(reader, catalog, alreadyInStruct, symbolsList, - reader.getSymbolTable()); + currentSymbolTable); + if (imports == null) { + // This was an LST append, so the existing symbol table was updated. + return currentSymbolTable; + } return new LocalSymbolTableAsStruct(imageFactory, imports, symbolsList); } diff --git a/test/com/amazon/ion/impl/IonWriterTestCase.java b/test/com/amazon/ion/impl/IonWriterTestCase.java index 82f8e61914..8fa77f299f 100644 --- a/test/com/amazon/ion/impl/IonWriterTestCase.java +++ b/test/com/amazon/ion/impl/IonWriterTestCase.java @@ -824,23 +824,9 @@ private void flushDoesNotReset(boolean lstAppendEnabled) iw.close(); IonDatagram dg = reload(); - if (myOutputForm == OutputForm.BINARY && iw instanceof _Private_IonManagedWriter && myLstAppendEnabled) { - // Note: LST append will only be implemented in the "new" binary writer (which implements - // _Private_IonMangedWriter) - // Should have: IVM SYMTAB hey SYMTAB_APPEND now - assertEquals(5, dg.systemSize()); - assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); - assertEquals("hey", ((IonSymbol) dg.systemGet(2)).stringValue()); - assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(3)).getTypeAnnotations()[0]); - assertEquals("now", ((IonSymbol) dg.systemGet(4)).stringValue()); - } - else { - // Should have: IVM SYMTAB hey SYMTAB_APPEND now - assertEquals(4, dg.systemSize()); - assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); - assertEquals("hey", ((IonSymbol) dg.systemGet(2)).stringValue()); - assertEquals("now", ((IonSymbol) dg.systemGet(3)).stringValue()); - } + assertEquals(2, dg.size()); + assertEquals("hey", ((IonSymbol) dg.get(0)).stringValue()); + assertEquals("now", ((IonSymbol) dg.get(1)).stringValue()); } Iterator systemIterateOutput() diff --git a/test/com/amazon/ion/impl/SymbolTableTest.java b/test/com/amazon/ion/impl/SymbolTableTest.java index 3c62543709..41ec83340f 100644 --- a/test/com/amazon/ion/impl/SymbolTableTest.java +++ b/test/com/amazon/ion/impl/SymbolTableTest.java @@ -61,6 +61,8 @@ import java.util.Iterator; import java.util.List; import java.util.Set; + +import com.amazon.ion.util.Equivalence; import org.junit.Ignore; import org.junit.Test; @@ -249,7 +251,7 @@ public void testLocalSymbolTableAppendEmptyList() } @Test - public void testLocalSymbolTableAppendImportBoundary() + public void testLocalSymbolTableAppendImportBoundary() throws IOException { String text = LocalSymbolTablePrefix + @@ -266,23 +268,23 @@ public void testLocalSymbolTableAppendImportBoundary() IonDatagram datagram = loader().load(text); - SymbolTable original = datagram.get(0).getSymbolTable(); - original.intern("o1"); - - SymbolTable appended = datagram.get(1).getSymbolTable(); - appended.intern("a1"); - - // new symbols in `original` don't influence SIDs for new symbols in `appended` after import - checkSymbol("s11", systemMaxId() + 1, appended); - checkSymbol("o1", systemMaxId() + 2, original); + SymbolTable symbolTable = datagram.get(0).getSymbolTable(); + assertSame(symbolTable, datagram.get(1).getSymbolTable()); + symbolTable.intern("o1"); + symbolTable.intern("a1"); - checkSymbol("s11", systemMaxId() + 1, appended); - checkSymbol("s21", systemMaxId() + 2, appended); - checkSymbol("a1", systemMaxId() + 3, appended); + // new symbols don't influence SIDs for existing symbols; they are appended + checkSymbol("s11", systemMaxId() + 1, symbolTable); + checkSymbol("s21", systemMaxId() + 2, symbolTable); + checkSymbol("o1", systemMaxId() + 3, symbolTable); + checkSymbol("a1", systemMaxId() + 4, symbolTable); - // new symbols in `original` are not accessible from `appended` after import - assertNull(original.find("a1")); - assertNull(appended.find("o1")); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IonWriter writer = IonBinaryWriterBuilder.standard().build(out); + datagram.writeTo(writer); + writer.close(); + IonDatagram roundtripped = loader().load(out.toByteArray()); + assertTrue(Equivalence.ionEquals(datagram, roundtripped)); } @Test diff --git a/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java b/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java index 033ad91c36..853136c59f 100644 --- a/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java +++ b/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java @@ -224,23 +224,9 @@ public void testLocalSymbolTableAppend() throws Exception assertNull(reader.next()); IonDatagram dg = system().getLoader().load(writer.getBytes()); - if (lstAppendMode.isEnabled()) - { - // Should be IVM SYMTAB taco SYMTAB burrito - assertEquals(5, dg.systemSize()); - assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); - assertEquals("taco", ((IonSymbol) dg.systemGet(2)).stringValue()); - assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(3)).getTypeAnnotations()[0]); - assertEquals("burrito", ((IonSymbol) dg.systemGet(4)).stringValue()); - } - else - { - // Should be IVM SYMTAB taco burrito - assertEquals(4, dg.systemSize()); - assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); - assertEquals("taco", ((IonSymbol) dg.systemGet(2)).stringValue()); - assertEquals("burrito", ((IonSymbol) dg.systemGet(3)).stringValue()); - } + assertEquals(2, dg.size()); + assertEquals("taco", ((IonSymbol) dg.get(0)).stringValue()); + assertEquals("burrito", ((IonSymbol) dg.get(1)).stringValue()); } @Test @@ -304,23 +290,9 @@ public void testManuallyWriteLSTAppendWithImportsFirst() throws Exception assertNull(reader.next()); IonDatagram dg = system().getLoader().load(writer.getBytes()); - if (lstAppendMode.isEnabled()) - { - // Should be IVM SYMTAB taco SYMTAB burrito - assertEquals(5, dg.systemSize()); - assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); - assertEquals("taco", ((IonSymbol) dg.systemGet(2)).stringValue()); - assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(3)).getTypeAnnotations()[0]); - assertEquals("burrito", ((IonSymbol) dg.systemGet(4)).stringValue()); - } - else - { - // Should be IVM SYMTAB taco burrito - assertEquals(4, dg.systemSize()); - assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); - assertEquals("taco", ((IonSymbol) dg.systemGet(2)).stringValue()); - assertEquals("burrito", ((IonSymbol) dg.systemGet(3)).stringValue()); - } + assertEquals(2, dg.size()); + assertEquals("taco", ((IonSymbol) dg.get(0)).stringValue()); + assertEquals("burrito", ((IonSymbol) dg.get(1)).stringValue()); } @Test @@ -349,23 +321,9 @@ public void testManuallyWriteLSTAppendWithSymbolsFirst() throws Exception assertNull(reader.next()); IonDatagram dg = system().getLoader().load(writer.getBytes()); - if (lstAppendMode.isEnabled()) - { - // Should be IVM SYMTAB taco SYMTAB burrito - assertEquals(5, dg.systemSize()); - assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); - assertEquals("taco", ((IonSymbol) dg.systemGet(2)).stringValue()); - assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(3)).getTypeAnnotations()[0]); - assertEquals("burrito", ((IonSymbol) dg.systemGet(4)).stringValue()); - } - else - { - // Should be IVM SYMTAB taco burrito - assertEquals(4, dg.systemSize()); - assertEquals("$ion_symbol_table", ((IonStruct) dg.systemGet(1)).getTypeAnnotations()[0]); - assertEquals("taco", ((IonSymbol) dg.systemGet(2)).stringValue()); - assertEquals("burrito", ((IonSymbol) dg.systemGet(3)).stringValue()); - } + assertEquals(2, dg.size()); + assertEquals("taco", ((IonSymbol) dg.get(0)).stringValue()); + assertEquals("burrito", ((IonSymbol) dg.get(1)).stringValue()); } @Test From efd5b3f95131206145a59db1f51a4e49e7080208 Mon Sep 17 00:00:00 2001 From: Therapon Skoteiniotis Date: Wed, 2 Sep 2020 18:46:27 -0700 Subject: [PATCH 100/490] prepare new release 1.7.0 (#309) * prepare new release 1.6.2 * fix version number to be 1.7.0 --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 76c150391f..10bc55c3ac 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ dependency into your project's `pom.xml`: com.amazon.ion ion-java - 1.6.1 + 1.7.0 ``` diff --git a/pom.xml b/pom.xml index 62ac58ae56..3f03843614 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.6.2-SNAPSHOT + 1.7.0 bundle ${project.groupId}:${project.artifactId} From 73e26b1c5e6b7d943b64652d6c6efb69e083a809 Mon Sep 17 00:00:00 2001 From: Therapon Skoteiniotis Date: Wed, 2 Sep 2020 21:45:19 -0700 Subject: [PATCH 101/490] creates v1.7.1-SNAPSHOT (#310) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3f03843614..4ccb147968 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.7.0 + 1.7.1-SNAPSHOT bundle ${project.groupId}:${project.artifactId} From a2f7a4f9e9e6745718a57b4b69662efe47c4f744 Mon Sep 17 00:00:00 2001 From: Therapon Skoteiniotis Date: Fri, 4 Sep 2020 18:30:05 -0700 Subject: [PATCH 102/490] Revert "Adds a space after a fieldname's colon when pretty-printing (#289)" (#311) This reverts commit 5d62f9ab6ee912f2812071fcec9454bd17c3d134. --- src/com/amazon/ion/impl/IonWriterSystemText.java | 3 --- test/com/amazon/ion/impl/IonMarkupWriterTest.java | 6 +++--- test/com/amazon/ion/impl/TextWriterTest.java | 10 +++++----- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/com/amazon/ion/impl/IonWriterSystemText.java b/src/com/amazon/ion/impl/IonWriterSystemText.java index f577c26ee5..0e78e95ccd 100644 --- a/src/com/amazon/ion/impl/IonWriterSystemText.java +++ b/src/com/amazon/ion/impl/IonWriterSystemText.java @@ -366,9 +366,6 @@ void startValue() throws IOException SymbolToken sym = assumeFieldNameSymbol(); writeFieldNameToken(sym); _output.appendAscii(':'); - if (_options.isPrettyPrintOn()) { - _output.appendAscii(' '); - } clearFieldName(); followingLongString = false; } diff --git a/test/com/amazon/ion/impl/IonMarkupWriterTest.java b/test/com/amazon/ion/impl/IonMarkupWriterTest.java index f7067d057f..b84d7a7da0 100644 --- a/test/com/amazon/ion/impl/IonMarkupWriterTest.java +++ b/test/com/amazon/ion/impl/IonMarkupWriterTest.java @@ -54,7 +54,7 @@ public class IonMarkupWriterTest extends IonTestCase { "%nabcd" + "::{%n hello" + - ": (%n " + + ":(%n " + "sexp" + "%n 1" + "%n 2" + @@ -62,12 +62,12 @@ public class IonMarkupWriterTest extends IonTestCase { "\"str\"%n " + "),%n " + "list: [%n :[%n 3.2,%n " + "3.2e0" + "%n ]" + ",%n blob" + - ": " + + ":" + "annot::" + "" + "{{ T25lIEJpZyBGYXQgVGVzdCBCbG9iIEZvciBZb3VyIFBsZWFzdXJl }}" + diff --git a/test/com/amazon/ion/impl/TextWriterTest.java b/test/com/amazon/ion/impl/TextWriterTest.java index 8c8b527bed..ba57d634bf 100644 --- a/test/com/amazon/ion/impl/TextWriterTest.java +++ b/test/com/amazon/ion/impl/TextWriterTest.java @@ -287,14 +287,14 @@ public void testWritingLongStrings() "%n" + "'''looong'''%n" + "{%n" + - " a: '''looong''',%n" + - " b: \"hello\",%n" + - " c: '''hello\n" + + " a:'''looong''',%n" + + " b:\"hello\",%n" + + " c:'''hello\n" + "nurse''',%n" + - " d: '''what\\'s\n" + + " d:'''what\\'s\n" + "up\n" + "doc'''%n" + - "}" + "}" ), dg ); From 4cf824b939c5db3bcf720024ad69814c9f687d4d Mon Sep 17 00:00:00 2001 From: Therapon Skoteiniotis Date: Fri, 4 Sep 2020 20:49:30 -0700 Subject: [PATCH 103/490] preparing release 1.7.1 (#312) --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 10bc55c3ac..b8b6a280df 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ dependency into your project's `pom.xml`: com.amazon.ion ion-java - 1.7.0 + 1.7.1 ``` diff --git a/pom.xml b/pom.xml index 4ccb147968..e7c0280b6f 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.7.1-SNAPSHOT + 1.7.1 bundle ${project.groupId}:${project.artifactId} From 4b99e45b6a55aeb5137bd48ab0b57f68acffe1c4 Mon Sep 17 00:00:00 2001 From: Therapon Skoteiniotis Date: Fri, 4 Sep 2020 21:30:49 -0700 Subject: [PATCH 104/490] prepare 1.7.2-SNAPSHOT (#313) prepare 1.7.2-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e7c0280b6f..e2b56980a5 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.7.1 + 1.7.2-SNAPSHOT bundle ${project.groupId}:${project.artifactId} From d6ea7518153020acbaa29587cba116e5e058ffeb Mon Sep 17 00:00:00 2001 From: Eric Chen <67451029+cheqianh@users.noreply.github.com> Date: Thu, 10 Sep 2020 17:39:11 -0700 Subject: [PATCH 105/490] Adds a CLI that implements the standardized interface required by ion-test-driver. (#314) https://github.com/amzn/ion-test-driver --- ion-java-cli/pom.xml | 121 ++ .../src/com/amazon/tools/cli/CommandType.java | 6 + .../com/amazon/tools/cli/CompareContext.java | 100 ++ .../com/amazon/tools/cli/ComparisonType.java | 8 + .../src/com/amazon/tools/cli/IonJavaCli.java | 1389 +++++++++++++++++ .../amazon/tools/cli/NoCloseOutputStream.java | 17 + .../amazon/tools/cli/NoOpOutputStream.java | 10 + .../com/amazon/tools/cli/OutputFormat.java | 88 ++ .../com/amazon/tools/cli/ProcessContext.java | 95 ++ .../comparisonReport/ComparisonContext.java | 27 + .../comparisonReport/ComparisonResult.java | 55 + .../ComparisonResultType.java | 7 + .../tools/errorReport/ErrorDescription.java | 41 + .../amazon/tools/errorReport/ErrorType.java | 7 + .../src/com/amazon/tools/events/Event.java | 179 +++ .../com/amazon/tools/events/EventType.java | 9 + .../amazon/tools/events/ImportDescriptor.java | 33 + 17 files changed, 2192 insertions(+) create mode 100644 ion-java-cli/pom.xml create mode 100644 ion-java-cli/src/com/amazon/tools/cli/CommandType.java create mode 100644 ion-java-cli/src/com/amazon/tools/cli/CompareContext.java create mode 100644 ion-java-cli/src/com/amazon/tools/cli/ComparisonType.java create mode 100644 ion-java-cli/src/com/amazon/tools/cli/IonJavaCli.java create mode 100644 ion-java-cli/src/com/amazon/tools/cli/NoCloseOutputStream.java create mode 100644 ion-java-cli/src/com/amazon/tools/cli/NoOpOutputStream.java create mode 100644 ion-java-cli/src/com/amazon/tools/cli/OutputFormat.java create mode 100644 ion-java-cli/src/com/amazon/tools/cli/ProcessContext.java create mode 100644 ion-java-cli/src/com/amazon/tools/comparisonReport/ComparisonContext.java create mode 100644 ion-java-cli/src/com/amazon/tools/comparisonReport/ComparisonResult.java create mode 100644 ion-java-cli/src/com/amazon/tools/comparisonReport/ComparisonResultType.java create mode 100644 ion-java-cli/src/com/amazon/tools/errorReport/ErrorDescription.java create mode 100644 ion-java-cli/src/com/amazon/tools/errorReport/ErrorType.java create mode 100644 ion-java-cli/src/com/amazon/tools/events/Event.java create mode 100644 ion-java-cli/src/com/amazon/tools/events/EventType.java create mode 100644 ion-java-cli/src/com/amazon/tools/events/ImportDescriptor.java diff --git a/ion-java-cli/pom.xml b/ion-java-cli/pom.xml new file mode 100644 index 0000000000..17dd85d335 --- /dev/null +++ b/ion-java-cli/pom.xml @@ -0,0 +1,121 @@ + + + + 4.0.0 + com.amazon.ion + ion-java-cli + 1.0 + jar + + ${project.groupId}:${project.artifactId} + + A CLI that implements the standard interface defined by ion-test-driver. + + https://github.com/amzn/ion-java/tree/master/ion-java-cli + + + + The Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + + + + + + Amazon Ion Team + ion-team@amazon.com + Amazon + https://github.com/amzn + + + + + scm:git:git@github.com:amzn/ion-java.git + scm:git:git@github.com:amzn/ion-java.git + git@github.com:amzn/ion-java.git + + + + UTF-8 + yyyy + ${maven.build.timestamp} + 1.8 + + + + + + args4j + args4j + 2.33 + + + com.amazon.ion + ion-java + [1.7.0,) + + + + + src + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + ${project.build.outputDirectory} + + + com.amazon.tools.cli.IonJavaCli + false + true + lib/ + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-dependencies + package + + copy-dependencies + + + jar + jar + ${project.build.directory}/lib + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.0 + + ${jdkVersion} + ${jdkVersion} + + + + + diff --git a/ion-java-cli/src/com/amazon/tools/cli/CommandType.java b/ion-java-cli/src/com/amazon/tools/cli/CommandType.java new file mode 100644 index 0000000000..c2168bcc25 --- /dev/null +++ b/ion-java-cli/src/com/amazon/tools/cli/CommandType.java @@ -0,0 +1,6 @@ +package com.amazon.tools.cli; + +public enum CommandType { + PROCESS, + COMPARE +} diff --git a/ion-java-cli/src/com/amazon/tools/cli/CompareContext.java b/ion-java-cli/src/com/amazon/tools/cli/CompareContext.java new file mode 100644 index 0000000000..ab06a14000 --- /dev/null +++ b/ion-java-cli/src/com/amazon/tools/cli/CompareContext.java @@ -0,0 +1,100 @@ +package com.amazon.tools.cli; + +import com.amazon.tools.events.Event; + +import java.util.List; + +public class CompareContext { + String path; + String compareToPath; + int fileEventIndex; + int compareToFileEventIndex; + String message; + List eventStreamFirst; + List eventStreamSecond; + ComparisonType type; + + public CompareContext(List eventStreamFirst, + List eventStreamSecond) { + this.path = null; + this.compareToPath = null; + this.eventStreamFirst = eventStreamFirst; + this.eventStreamSecond = eventStreamSecond; + this.message = null; + this.type = null; + } + + public void reset(String file, String compareToFile){ + this.setFile(file); + this.setCompareToFile(compareToFile); + this.setFileEventIndex(-1); + this.setCompareToFileEventIndex(-1); + this.message = null; + this.eventStreamFirst = null; + this.eventStreamSecond = null; + } + + public String getFile() { + return path; + } + + public void setFile(String file) { + this.path = file; + } + + public String getCompareToFile() { + return compareToPath; + } + + public void setCompareToFile(String compareToFile) { + this.compareToPath = compareToFile; + } + + public int getFileEventIndex() { + return fileEventIndex; + } + + public void setFileEventIndex(int fileEventIndex) { + this.fileEventIndex = fileEventIndex; + } + + public int getCompareToFileEventIndex() { + return compareToFileEventIndex; + } + + public void setCompareToFileEventIndex(int compareToFileEventIndex) { + this.compareToFileEventIndex = compareToFileEventIndex; + } + + public List getEventStreamFirst() { + return eventStreamFirst; + } + + public void setEventStreamFirst(List eventStreamFirst) { + this.eventStreamFirst = eventStreamFirst; + } + + public List getEventStreamSecond() { + return eventStreamSecond; + } + + public void setEventStreamSecond(List eventStreamSecond) { + this.eventStreamSecond = eventStreamSecond; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public ComparisonType getType() { + return type; + } + + public void setType(ComparisonType type) { + this.type = type; + } +} diff --git a/ion-java-cli/src/com/amazon/tools/cli/ComparisonType.java b/ion-java-cli/src/com/amazon/tools/cli/ComparisonType.java new file mode 100644 index 0000000000..58cb5e76d2 --- /dev/null +++ b/ion-java-cli/src/com/amazon/tools/cli/ComparisonType.java @@ -0,0 +1,8 @@ +package com.amazon.tools.cli; + +public enum ComparisonType { + BASIC, + EQUIVS, + NON_EQUIVS, + EQUIVS_TIMELINE +} diff --git a/ion-java-cli/src/com/amazon/tools/cli/IonJavaCli.java b/ion-java-cli/src/com/amazon/tools/cli/IonJavaCli.java new file mode 100644 index 0000000000..59086f3b54 --- /dev/null +++ b/ion-java-cli/src/com/amazon/tools/cli/IonJavaCli.java @@ -0,0 +1,1389 @@ +package com.amazon.tools.cli; + +import com.amazon.ion.IonException; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonSequence; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonTimestamp; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.impl._Private_Utils; +import com.amazon.ion.system.IonReaderBuilder; +import com.amazon.ion.system.IonSystemBuilder; +import com.amazon.ion.system.IonTextWriterBuilder; +import com.amazon.ion.util.Equivalence; +import org.kohsuke.args4j.Argument; +import org.kohsuke.args4j.CmdLineException; +import org.kohsuke.args4j.CmdLineParser; +import org.kohsuke.args4j.Option; +import com.amazon.tools.comparisonReport.ComparisonContext; +import com.amazon.tools.comparisonReport.ComparisonResult; +import com.amazon.tools.comparisonReport.ComparisonResultType; +import com.amazon.tools.errorReport.ErrorDescription; +import com.amazon.tools.errorReport.ErrorType; +import com.amazon.tools.events.Event; +import com.amazon.tools.events.EventType; +import com.amazon.tools.events.ImportDescriptor; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +public class IonJavaCli { + private static final int CONSOLE_WIDTH = 120; // Only used for formatting the USAGE message + private static final int USAGE_ERROR_EXIT_CODE = 1; + private static final int IO_ERROR_EXIT_CODE = 2; + private static final int BUFFER_SIZE = 128 * 1024; + private static final String SYSTEM_OUT_DEFAULT_VALUE = "out"; + private static final String SYSTEM_ERR_DEFAULT_VALUE = "err"; + private static final IonTextWriterBuilder ION_TEXT_WRITER_BUILDER = IonTextWriterBuilder.standard(); + private static final IonSystem ION_SYSTEM = IonSystemBuilder.standard().build(); + private static final String EMBEDDED_STREAM_ANNOTATION = "embedded_documents"; + private static final String EVENT_STREAM = "$ion_event_stream"; + private static final Pattern ION_VERSION_MARKER_REGEX = Pattern.compile("^\\$ion_[0-9]+_[0-9]+$"); + + public static void main(final String[] args) { + CommandArgs parsedArgs = new CommandArgs(); + CmdLineParser parser = new CmdLineParser(parsedArgs); + parser.getProperties().withUsageWidth(CONSOLE_WIDTH); + ComparisonType comparisonType = null; + OutputFormat outputFormat = null; + CommandType commandType = null; + + try { + //parse + parser.parseArgument(args); + validateArgs(parsedArgs); + //get value + comparisonType = parsedArgs.getComparisonType(); + outputFormat = parsedArgs.getOutputFormat(); + commandType = parsedArgs.getCommand(); + } catch (CmdLineException | IllegalArgumentException e) { + printHelpText(e.getMessage(), parser, parsedArgs); + System.exit(USAGE_ERROR_EXIT_CODE); + } + + ProcessContext processContext = commandType == CommandType.PROCESS ? new ProcessContext(null, -1, + null, ErrorType.READ, null) : null; + try ( + //Initialize output stream, never return null. (default value: STDOUT) + OutputStream outputStream = initOutputStream(parsedArgs, SYSTEM_OUT_DEFAULT_VALUE, processContext); + //Initialize error report, never return null. (default value: STDERR) + OutputStream errorReportOutputStream = + initOutputStream(parsedArgs, SYSTEM_ERR_DEFAULT_VALUE, processContext); + IonWriter ionWriterForErrorReport = outputFormat.createIonWriter(errorReportOutputStream); + IonWriter ionWriterForOutput = outputFormat.createIonWriter(outputStream); + ) { + if (commandType == CommandType.COMPARE) { + compareFiles(ionWriterForOutput, ionWriterForErrorReport, parsedArgs, comparisonType); + } else if (commandType == CommandType.PROCESS) { + processContext.setIonWriter(ionWriterForOutput); + processFiles(ionWriterForErrorReport, parsedArgs, processContext); + } + } catch (IOException e) { + System.err.println("Failed to close OutputStream: " + e.getMessage()); + } finally { + if (processContext != null) { + IonWriter ionWriter = processContext.getIonWriter(); + if (ionWriter != null) { + try { + ionWriter.close(); + } catch (IOException e) { + System.err.println(e.getMessage()); + System.exit(IO_ERROR_EXIT_CODE); + } + } + } + } + } + + private static void validateArgs(CommandArgs parsedArgs) throws CmdLineException { + parsedArgs.setCommand(); + CommandType type = parsedArgs.getCommand(); + switch (type) { + case PROCESS: + break; + case COMPARE: + if (parsedArgs.getOutputFormat() == OutputFormat.EVENTS) { + throw new CmdLineException("COMPARE doesn't support option format \"-f events\""); + } + break; + } + } + + private static void printHelpText(String msg, CmdLineParser parser, CommandArgs parsedArgs) { + CommandType commandType = parsedArgs.getCommand(); + + if (commandType == null) { + System.err.println(msg + "\n"); + System.err.println("Usage : \n" + + " [-h|--help] [-v|--version]\n" + + " -h, --help print the help message and exit\n" + + " -v, --version print the command's version number and exit"); + } else if (commandType == CommandType.PROCESS) { + System.err.println(msg + "\n"); + System.err.println("\"Process\" reads the input file(s) and re-write in the format specified by " + + "--output.\n"); + System.err.println("Usage:\n"); + System.err.println("ion process [--output ] [--error-report ] [--output-format \n" + + "(text | pretty | binary | events | none)] [--catalog ]... [--imports ]... \n" + + "[--perf-report ] [--filter | --traverse ] [-] []...\n"); + System.err.println("Options:\n"); + parser.printUsage(System.err); + } else if (commandType == CommandType.COMPARE) { + System.err.println(msg + "\n"); + System.err.println("\"Compare\" Compare all inputs (which may contain Ion streams and/or EventStreams) \n" + + "against all other inputs using the Ion data model's definition of equality. Write a \n" + + "ComparisonReport to the output.\n"); + System.err.println("Usage:\n"); + System.err.println("ion compare [--output ] [--error-report ] [--output-format (text | \n" + + "pretty | binary | none)] [--catalog ]... [--comparison-type (basic | equivs | \n" + + "non-equivs | equiv-timeline)] [-] []...\n"); + parser.printUsage(System.err); + } + } + + // + // + // functions for processing + // + // + + private static void processFiles(IonWriter ionWriterForErrorReport, + CommandArgs args, + ProcessContext processContext) throws IOException { + boolean finish = false; + for (String path : args.getInputFiles()) { + try ( + InputStream inputStream = new BufferedInputStream(new FileInputStream(path)); + IonReader ionReader = IonReaderBuilder.standard().build(inputStream); + ) { + processContext.setFileName(path); + ReadContext readContext = new ReadContext(new ArrayList<>()); + try { + getEventStream(ionReader, CommandType.PROCESS, readContext); + } catch (IonException | NullPointerException e) { + new ErrorDescription(readContext.getState(), e.getMessage(), processContext.getFileName(), + processContext.getEventIndex()).writeOutput(ionWriterForErrorReport); + finish = true; + } catch (Exception e) { + new ErrorDescription(ErrorType.STATE, e.getMessage(), processContext.getFileName(), + processContext.getEventIndex()).writeOutput(ionWriterForErrorReport); + finish = true; + } + + processContext.setEventStream(readContext.getEventStream()); + processContext.setEventIndex(0); + + if (args.getOutputFormat() == OutputFormat.EVENTS) { + processContext.getIonWriter().writeSymbol(EVENT_STREAM); + processToEventStream(ionWriterForErrorReport, processContext); + } else { + processToIonStream(processContext, args); + } + if (finish) System.exit(IO_ERROR_EXIT_CODE); + processContext.getIonWriter().finish(); + ionWriterForErrorReport.finish(); + } catch (IonException | NullPointerException e) { + new ErrorDescription(processContext.getState(), e.getMessage(), processContext.getFileName(), + processContext.getEventIndex()).writeOutput(ionWriterForErrorReport); + System.exit(IO_ERROR_EXIT_CODE); + } catch (Exception e) { + new ErrorDescription(ErrorType.STATE, e.getMessage(), processContext.getFileName(), + processContext.getEventIndex()).writeOutput(ionWriterForErrorReport); + System.exit(IO_ERROR_EXIT_CODE); + } + } + } + + private static void processToEventStream(IonWriter ionWriterForErrorReport, + ProcessContext processContext) throws IOException { + for (Event event : processContext.getEventStream()) { + event.writeOutput(ionWriterForErrorReport, processContext); + } + } + + private static void processToIonStream(ProcessContext processContext, + CommandArgs args) throws IOException { + List events = processContext.getEventStream(); + int count = 0; + while (count != events.size()) { + //update eventIndex + Event event = events.get(count); + processContext.setEventIndex(processContext.getEventIndex() + 1); + processContext.setLastEventType(event.getEventType()); + if (event.getEventType() == EventType.CONTAINER_START) { + if (isEmbeddedEvent(event)) { + count = embeddedEventToIon(processContext, args, count, event.getIonType()); + } else { + IonType type = event.getIonType(); + setFieldName(event, processContext.getIonWriter()); + setAnnotations(event, processContext.getIonWriter()); + processContext.getIonWriter().stepIn(type); + } + } else if (event.getEventType().equals(EventType.CONTAINER_END)) { + processContext.getIonWriter().stepOut(); + } else if (event.getEventType().equals(EventType.SCALAR)) { + writeIonByType(event, processContext.getIonWriter()); + } else if (event.getEventType().equals(EventType.SYMBOL_TABLE)) { + handleSymbolTableEvent(processContext, event, args, false); + } else if (event.getEventType().equals(EventType.STREAM_END)) { + processContext.getIonWriter().finish(); + } + count++; + } + } + + private static int embeddedEventToIon(ProcessContext processContext, + CommandArgs args, + int count, + IonType ionType) throws IOException { + processContext.getIonWriter().addTypeAnnotation(EMBEDDED_STREAM_ANNOTATION); + processContext.getIonWriter().stepIn(ionType); + List events = processContext.getEventStream(); + int depth = 1; + boolean finish = false; + while (++count < events.size()) { + StringBuilder out = new StringBuilder(); + ProcessContext embeddedContext = new ProcessContext(null,0,null, null, + ION_TEXT_WRITER_BUILDER.withImports(_Private_Utils.systemSymtab(1)).build(out)); + embeddedContext.setEmbeddedOut(out); + try { + do { + Event event = events.get(count); + processContext.setEventIndex(processContext.getEventIndex() + 1); + processContext.setLastEventType(event.getEventType()); + if (event.getEventType() == EventType.STREAM_END) { + break; + } else if (event.getEventType() == EventType.SCALAR) { + writeIonByType(event, embeddedContext.getIonWriter()); + } else if (event.getEventType() == EventType.CONTAINER_START) { + depth++; + setFieldName(event, embeddedContext.getIonWriter()); + setAnnotations(event, embeddedContext.getIonWriter()); + embeddedContext.getIonWriter().stepIn(event.getIonType()); + } else if (event.getEventType() == EventType.CONTAINER_END) { + depth--; + if (depth == 0) { + if (event.getIonType() == IonType.SEXP || event.getIonType() == IonType.LIST) { + finish = true; + break; + } else { + throw new IonException("invalid CONTAINER_END"); + } + } + embeddedContext.getIonWriter().stepOut(); + } else if (event.getEventType() == EventType.SYMBOL_TABLE) { + handleSymbolTableEvent(embeddedContext, event, args, true); + } + } while (++count < events.size()); + + if (!finish) { + embeddedContext.getIonWriter().finish(); + processContext.getIonWriter().writeString(out.toString()); + } + } finally { + IonWriter ionWriter = embeddedContext.getIonWriter(); + if (ionWriter != null) { + try { + ionWriter.close(); + } catch (IOException e) { + System.err.println(e.getMessage()); + System.exit(IO_ERROR_EXIT_CODE); + } + } + } + if (finish) { break; } + } + processContext.getIonWriter().stepOut(); + return count; + } + + // + // + // functions for comparing + // + // + + private static void compareFiles(IonWriter ionWriterForOutput, + IonWriter ionWriterForErrorReport, + CommandArgs args, + ComparisonType comparisonType) throws IOException { + List files = args.getInputFiles(); + CompareContext compareContext = new CompareContext(null, null); + compareContext.setType(args.getComparisonType()); + for (String path : files) { + for (String compareToPath : files) { + compareContext.reset(path, compareToPath); + try (InputStream inputFirst = new BufferedInputStream(new FileInputStream(path)); + IonReader ionReaderFirst = IonReaderBuilder.standard().build(inputFirst); + InputStream inputSecond = new BufferedInputStream(new FileInputStream(compareToPath)); + IonReader ionReaderSecond = IonReaderBuilder.standard().build(inputSecond); + ) { + if (comparisonType == ComparisonType.BASIC) { + if (path.equals(compareToPath)) { continue; } + } + ReadContext readContextFirst = new ReadContext(new ArrayList<>()); + ReadContext readContextSecond = new ReadContext(new ArrayList<>()); + getEventStream(ionReaderFirst, CommandType.COMPARE, readContextFirst); + getEventStream(ionReaderSecond, CommandType.COMPARE, readContextSecond); + + compareContext.setEventStreamFirst(readContextFirst.getEventStream()); + compareContext.setEventStreamSecond(readContextSecond.getEventStream()); + + if (comparisonType != ComparisonType.BASIC) { + if (compareEquivs(compareContext) ^ + (comparisonType == ComparisonType.EQUIVS + || comparisonType == ComparisonType.EQUIVS_TIMELINE)) { + ComparisonResultType type = comparisonType == ComparisonType.NON_EQUIVS ? + ComparisonResultType.EQUAL : ComparisonResultType.NOT_EQUAL; + writeReport(compareContext, ionWriterForOutput, type); + } + } else { + if (!compare(compareContext, 0, readContextFirst.getEventStream().size() - 1, + 0, readContextSecond.getEventStream().size() - 1)) { + writeReport(compareContext, ionWriterForOutput, ComparisonResultType.NOT_EQUAL); + } + } + } catch (Exception e) { + new ErrorDescription(ErrorType.STATE, e.getMessage(), path + ';' + compareToPath, + -1).writeOutput(ionWriterForErrorReport); + System.exit(IO_ERROR_EXIT_CODE); + } + } + } + } + + private static boolean compare(CompareContext compareContext, + int startI, int endI, int startJ, int endJ) throws IOException { + List eventsFirst = compareContext.getEventStreamFirst(); + List eventsSecond = compareContext.getEventStreamSecond(); + int i = startI; + int j = startJ; + while (i <= endI && j <= endJ && i < eventsFirst.size() && j < eventsSecond.size()) { + Event eventFirst = eventsFirst.get(i); + Event eventSecond = eventsSecond.get(j); + SymbolToken fieldNameFirst = eventFirst.getFieldName(); + SymbolToken fieldNameSecond = eventSecond.getFieldName(); + SymbolToken[] annotationFirst = eventFirst.getAnnotations(); + SymbolToken[] annotationSecond = eventSecond.getAnnotations(); + EventType eventTypeFirst = eventFirst.getEventType(); + EventType eventTypeSecond = eventSecond.getEventType(); + if (eventTypeFirst != eventTypeSecond) { + setReportInfo(i, j,"Didn't match event_type", compareContext); + return false; + } else if (eventFirst.getDepth() != eventSecond.getDepth()) { + setReportInfo(i, j,"Didn't match depth", compareContext); + return false; + } else if (eventFirst.getIonType() != eventSecond.getIonType()) { + setReportInfo(i, j,"Didn't match ion_type", compareContext); + return false; + } else if (!isSameSymbolToken(fieldNameFirst, fieldNameSecond)) { + setReportInfo(i, j,"Didn't match field_name", compareContext); + return false; + } else if (!isSameSymbolTokenArray(annotationFirst, annotationSecond)) { + setReportInfo(i, j,"Didn't match annotation", compareContext); + return false; + } + + if (eventTypeFirst == EventType.CONTAINER_START + && eventFirst.getIonType() == IonType.STRUCT) { + int iStart = i; + int jStart = j; + ContainerContext containerContextFirst = new ContainerContext(i); + IonStruct structFirst = parseStruct(containerContextFirst, compareContext, endI, true); + i = containerContextFirst.getIndex(); + ContainerContext containerContextSecond = new ContainerContext(j); + IonStruct structSecond = parseStruct(containerContextSecond, compareContext, endJ, false); + j = containerContextSecond.getIndex(); + if (!Equivalence.ionEquals(structFirst, structSecond)) { + setReportInfo(iStart, jStart, + "Did not find matching field for " + structFirst.toString(), + compareContext); + return false; + } + } else if (eventTypeFirst == EventType.SCALAR) { + boolean compareResult; + if (compareContext.getType() == ComparisonType.EQUIVS_TIMELINE + && eventFirst.getIonType() == IonType.TIMESTAMP) { + IonTimestamp ionTimestampFirst = (IonTimestamp) eventFirst.getValue(); + IonTimestamp ionTimestampSecond = (IonTimestamp) eventSecond.getValue(); + compareResult = ionTimestampFirst.timestampValue() + .compareTo(ionTimestampSecond.timestampValue()) == 0; + } else { + compareResult = Equivalence.ionEquals(eventFirst.getValue(), eventSecond.getValue()); + } + if (!compareResult) { + setReportInfo(i, j, eventFirst.getValue() + " vs. " + + eventSecond.getValue(), compareContext); + return false; + } + } + i++; + j++; + } + if (i <= endI || j <= endJ) { + setReportInfo(i , j, "two event streams have different size", compareContext); + return false; + } + return true; + } + + private static IonStruct parseStruct(ContainerContext containerContext, + CompareContext compareContext, + int end, + boolean isFirst) { + Event initEvent = isFirst ? + compareContext.getEventStreamFirst().get(containerContext.getIndex()) : + compareContext.getEventStreamSecond().get(containerContext.getIndex()); + if (initEvent.getEventType() != EventType.CONTAINER_START || initEvent.getIonType() != IonType.STRUCT) { + return null; + } + IonStruct ionStruct = ION_SYSTEM.newEmptyStruct(); + while (containerContext.increaseIndex() < end) { + Event event = isFirst ? + compareContext.getEventStreamFirst().get(containerContext.getIndex()) : + compareContext.getEventStreamSecond().get(containerContext.getIndex()); + EventType eventType = event.getEventType(); + if (eventType == EventType.CONTAINER_START) { + switch (event.getIonType()) { + case LIST: + ionStruct.add(event.getFieldName(), + parseSequence(containerContext, compareContext, end, isFirst, ION_SYSTEM.newEmptyList())); + break; + case STRUCT: + ionStruct.add(event.getFieldName(), + parseStruct(containerContext, compareContext, end, isFirst)); + break; + case SEXP: + ionStruct.add(event.getFieldName(), + parseSequence(containerContext, compareContext, end, isFirst, ION_SYSTEM.newEmptySexp())); + break; + } + } else if (eventType == EventType.CONTAINER_END) { + break; + } else if (eventType == EventType.STREAM_END) { + throw new IonException("Invalid struct: eventStream ends without CONTAINER_END"); + } else { + IonValue cloneValue = event.getValue().clone(); + cloneValue.setTypeAnnotationSymbols(event.getAnnotations()); + ionStruct.add(event.getFieldName(), cloneValue); + } + } + return ionStruct; + } + + private static IonSequence parseSequence(ContainerContext containerContext, + CompareContext compareContext, + int end, + boolean isFirst, + IonSequence ionSequence) { + while (containerContext.increaseIndex() < end) { + Event event = isFirst ? + compareContext.getEventStreamFirst().get(containerContext.getIndex()) : + compareContext.getEventStreamSecond().get(containerContext.getIndex()); + EventType eventType = event.getEventType(); + if (eventType == EventType.CONTAINER_START) { + switch (event.getIonType()) { + case LIST: + ionSequence.add(parseSequence(containerContext, compareContext, end, isFirst, ION_SYSTEM.newEmptyList())); + break; + case SEXP: + ionSequence.add(parseSequence(containerContext, compareContext, end, isFirst, ION_SYSTEM.newEmptySexp())); + break; + case STRUCT: + ionSequence.add(parseStruct(containerContext, compareContext, end, isFirst)); + break; + } + } else if (eventType == EventType.CONTAINER_END) { + break; + } else if (eventType == EventType.STREAM_END) { + throw new IonException("Invalid ionSequence: eventStream ends without CONTAINER_END"); + } else { + IonValue cloneValue = event.getValue().clone(); + cloneValue.setTypeAnnotationSymbols(event.getAnnotations()); + ionSequence.add(cloneValue); + } + } + return ionSequence; + } + + private static boolean compareEquivs(CompareContext compareContext) throws IOException { + int i = 0; + int j = 0; + List eventStreamFirst = compareContext.getEventStreamFirst(); + List eventStreamSecond = compareContext.getEventStreamSecond(); + ComparisonType type = compareContext.getType(); + + while (i < eventStreamFirst.size() && j < eventStreamSecond.size()) { + Event eventFirst = eventStreamFirst.get(i); + Event eventSecond = eventStreamSecond.get(j); + + if (eventFirst.getEventType() == EventType.STREAM_END + && eventSecond.getEventType() == EventType.STREAM_END) { + break; + } else if (eventFirst.getEventType() == EventType.STREAM_END + || eventSecond.getEventType() == EventType.STREAM_END) { + setReportInfo(i, j, + "The input streams had a different number of comparison sets.", compareContext); + return type == ComparisonType.NON_EQUIVS; + } else if (!(eventFirst.getIonType() == IonType.LIST || eventFirst.getIonType() == IonType.SEXP) + || !(eventSecond.getIonType() == IonType.LIST || eventSecond.getIonType() == IonType.SEXP)) { + throw new IonException("Comparison sets must be lists or s-expressions."); + } else if (isEmbeddedEvent(eventFirst) ^ isEmbeddedEvent(eventSecond)) { + throw new IonException("Embedded streams set expected."); + } + + List pairsFirst; + List pairsSecond; + + if (isEmbeddedEvent(eventFirst) && isEmbeddedEvent(eventSecond)) { + pairsFirst = locateEmbeddedStreamBoundaries(eventStreamFirst, i); + pairsSecond = locateEmbeddedStreamBoundaries(eventStreamSecond, j); + } else { + pairsFirst = locateContainerBoundaries(eventStreamFirst, i); + pairsSecond = locateContainerBoundaries(eventStreamSecond, j); + } + i = pairsFirst.size() == 0 ? i + 1 : pairsFirst.get(pairsFirst.size() - 1).right + 1; + j = pairsSecond.size() == 0 ? j + 1 : pairsSecond.get(pairsSecond.size() - 1).right + 1; + + for (int m = 0; m < pairsFirst.size(); m++) { + for (int n = 0; n < pairsSecond.size(); n++) { + if (compareContext.getType() == ComparisonType.NON_EQUIVS) { + if (m == n) continue; + } + Pair pairFirst = pairsFirst.get(m); + Pair pairSecond = pairsSecond.get(n); + if (compare(compareContext, pairFirst.left, pairFirst.right, pairSecond.left, pairSecond.right) ^ + (type == ComparisonType.EQUIVS || type == ComparisonType.EQUIVS_TIMELINE)) { + if (type == ComparisonType.NON_EQUIVS) { + setReportInfo(pairFirst.left, pairSecond.left, + "Equivalent values in a non-equivs set.", compareContext); + } + return type == ComparisonType.NON_EQUIVS; + } + } + } + i++; + j++; + } + return (type == ComparisonType.EQUIVS || type == ComparisonType.EQUIVS_TIMELINE); + } + + /** + * This function parses an embedded stream into pairs that stores index range for + * each string the embedded stream includes. + * + * Input value 'start' must be index of a CONTAINER_START event. + * + * return List is never null. It will return a list with size zero if the embedded stream is empty. + */ + private static List locateEmbeddedStreamBoundaries(List events, int start) { + List parsedEvents = new ArrayList<>(0); + int left = start + 1; + int right = start; + int depth = 1; + while (++right < events.size()) { + EventType eventType = events.get(right).getEventType(); + + if (eventType == EventType.STREAM_END) { + parsedEvents.add(new Pair(left, right)); + left = right + 1; + } else if (eventType == EventType.CONTAINER_END) { + if (--depth == 0) break; + } else if (eventType == EventType.CONTAINER_START) { + depth++; + } + } + return parsedEvents; + } + + /** + * This function parses a container's values into pairs that stores its index range. + * 1. For SCALAR event with index i, the pair will be (i, i). + * 2. For CONTAINER event, the pair will be (start, end) where start is the index of the CONTAINER_START event + * and end is the index of the CONTAINER_END event. + * + * Input value 'start' must be index of a CONTAINER_START event. + * + * return List is never null. It will return a list with size zero if the container is empty. + */ + private static List locateContainerBoundaries(List events, int start) { + List parsedEvents = new ArrayList<>(0); + while (++start < events.size()) { + EventType eventType = events.get(start).getEventType(); + + if (eventType == EventType.STREAM_END) { + throw new IonException("Invalid container: end without CONTAINER_END event"); + } else if (eventType == EventType.CONTAINER_END) { + break; + } else if (eventType == EventType.SCALAR) { + parsedEvents.add(new Pair(start, start)); + } else if (eventType == EventType.CONTAINER_START) { + int startCount = start; + int endCount = findNextContainer(events, start); + start = endCount; + parsedEvents.add(new Pair(startCount, endCount)); + } + } + return parsedEvents; + } + + /** + * This function will move the count i from the CONTAINER_START event to its corresponding CONTAINER_END event. + * + * return the index of its corresponding CONTAINER_END event. + * + * throw IonException if CONTAINER_END event doesn't exist. + */ + private static int findNextContainer(List eventStream, int i) { + if (!IonType.isContainer(eventStream.get(i).getIonType())) return i; + int depth = 1; + while (i++ < eventStream.size()) { + Event event = eventStream.get(i); + if (event.getEventType() == EventType.CONTAINER_START) { + depth++; + } else if (event.getEventType() == EventType.CONTAINER_END) { + if (--depth == 0) break; + } + } + if (i == eventStream.size()) { + throw new IonException("Invalid container, reach the end of the eventStream without CONTAINER_END event"); + } + return i; + } + + // + // + // helper functions + // + // + + private static void handleSymbolTableEvent(ProcessContext processContext, + Event event, + CommandArgs args, + boolean isEmbedded) throws IOException { + processContext.getIonWriter().close(); + ImportDescriptor[] imports = event.getImports(); + SymbolTable[] symbolTables = new SymbolTable[imports.length]; + for (int i = 0; i < imports.length; i++) { + SymbolTable symbolTable = ION_SYSTEM.newSharedSymbolTable( + imports[i].getImportName(), imports[i].getVersion(), null); + symbolTables[i] = symbolTable; + } + if (!isEmbedded) { + OutputStream out = args.getOutputFile() == null ? + new NoCloseOutputStream(System.out) : new FileOutputStream(processContext.getFile(), true); + processContext.setIonWriter(args.getOutputFormat().createIonWriterWithImports(out, symbolTables)); + } else { + processContext.getEmbeddedOut().append(" "); + processContext.setIonWriter(ION_TEXT_WRITER_BUILDER.withImports(symbolTables) + .build(processContext.getEmbeddedOut())); + } + } + + private static void writeIonByType(Event event, IonWriter ionWriter) { + setFieldName(event, ionWriter); + IonValue value = event.getValue(); + value.setTypeAnnotationSymbols(event.getAnnotations()); + value.writeTo(ionWriter); + } + + private static void setFieldName(Event event, IonWriter ionWriter) { + SymbolToken field = event.getFieldName(); + if (field != null) { + if (!ionWriter.isInStruct()) throw new IonException("invalid field_name inside STRUCT"); + ionWriter.setFieldNameSymbol(field); + } else if (ionWriter.isInStruct()) { + SymbolToken symbolToken = _Private_Utils.newSymbolToken((String) null, 0); + ionWriter.setFieldNameSymbol(symbolToken); + } + } + + private static void setAnnotations(Event event, IonWriter ionWriter) { + SymbolToken[] annotations = event.getAnnotations(); + if (annotations != null && annotations.length != 0) { + ionWriter.setTypeAnnotationSymbols(annotations); + } + } + + private static boolean isSameSymbolToken(SymbolToken symbolTokenX, SymbolToken symbolTokenY) { + if (symbolTokenX == null && symbolTokenY == null) { + return true; + } else if (symbolTokenX != null && symbolTokenY != null) { + if (symbolTokenX.getText() != null && symbolTokenY.getText() != null) { + return symbolTokenX.getText().equals(symbolTokenY.getText()); + } else { + return symbolTokenX.getSid() == symbolTokenY.getSid(); + } + } else return false; + } + + private static boolean isSameSymbolTokenArray(SymbolToken[] symbolTokenArrayX, SymbolToken[] symbolTokenArrayY) { + if ((symbolTokenArrayX == null || symbolTokenArrayX.length == 0) + && (symbolTokenArrayY == null || symbolTokenArrayY.length == 0)) { + return true; + } else if (symbolTokenArrayX != null && symbolTokenArrayY != null) { + if (symbolTokenArrayX.length == symbolTokenArrayY.length) { + for (int i = 0; i < symbolTokenArrayX.length; i++) { + if (!isSameSymbolToken(symbolTokenArrayX[i], symbolTokenArrayY[i])) return false; + } + } else return false; + return true; + } else { + return false; + } + } + + private static boolean isEmbeddedStream(IonReader ionReader) { + IonType ionType = ionReader.getType(); + String[] annotations = ionReader.getTypeAnnotations(); + return (ionType == IonType.SEXP || ionType == IonType.LIST) + && ionReader.getDepth() == 0 + && annotations.length > 0 + && annotations[0].equals(EMBEDDED_STREAM_ANNOTATION); + } + + private static boolean isEmbeddedEvent(Event event) { + IonType ionType = event.getIonType(); + SymbolToken[] annotations = event.getAnnotations(); + return (ionType == IonType.SEXP || ionType == IonType.LIST) + && event.getDepth() == 0 + && annotations != null + && annotations.length > 0 + && annotations[0].getText().equals(EMBEDDED_STREAM_ANNOTATION); + } + + public static boolean isEventStream(IonReader ionReader) { + return ionReader.next() != null + && !ionReader.isNullValue() + && ionReader.getType() == IonType.SYMBOL + && EVENT_STREAM.equals(ionReader.symbolValue().getText()); + } + + private static void setReportInfo(int i, int j, String message, CompareContext compareContext) { + if (i != -1) compareContext.setFileEventIndex(i); + if (j != -1) compareContext.setCompareToFileEventIndex(j); + if (message != null) compareContext.setMessage(message); + } + + private static void writeReport(CompareContext compareContext, + IonWriter ionWriter, + ComparisonResultType type) throws IOException { + new ComparisonResult(type, + new ComparisonContext(compareContext.getFile(), + (compareContext.getFileEventIndex() >= compareContext.getEventStreamFirst().size() + || compareContext.getFileEventIndex() < 0) ? null : + compareContext.getEventStreamFirst().get(compareContext.getFileEventIndex()), + compareContext.getFileEventIndex()), + new ComparisonContext(compareContext.getCompareToFile(), + (compareContext.getCompareToFileEventIndex() >= compareContext.getEventStreamSecond().size() + || compareContext.getCompareToFileEventIndex() < 0) ? null : + compareContext.getEventStreamSecond().get(compareContext.getCompareToFileEventIndex()), + compareContext.getCompareToFileEventIndex()), + compareContext.getMessage()).writeOutput(ionWriter); + } + + private static void getEventStream(IonReader ionReader, + CommandType commandType, + ReadContext readContext) throws IOException { + SymbolTable curTable = ionReader.getSymbolTable(); + boolean isEventStream = isEventStream(ionReader); + + if (isEventStream) { + readContext.setState(ErrorType.WRITE); + while (ionReader.next() != null) { + Event event = eventStreamToEvent(ionReader); + if (event.getEventType() == EventType.SYMBOL_TABLE && commandType == CommandType.COMPARE) { + continue; + } + readContext.getEventStream().add(event); + } + } else { + readContext.setState(ErrorType.READ); + ionStreamToEventStream(ionReader, commandType, curTable, readContext); + readContext.getEventStream().add(new Event(EventType.STREAM_END, null, null, null, + null, null, 0)); + } + + if (commandType == CommandType.PROCESS) { + validateEventStream(readContext.getEventStream()); + } + return; + } + + private static void validateEventStream(List events) { + if (events.size() == 0) return; + + int depth = 0; + int i = -1; + while (++i < events.size()) { + Event event = events.get(i); + EventType eventType = event.getEventType(); + + if (eventType == EventType.CONTAINER_START) { + depth++; + } else if (eventType == EventType.CONTAINER_END) { + if (--depth < 0) { + throw new IonException("Invalid event stream: Invalid CONTAINER_END event"); + } + } + + if (isEmbeddedEvent(event)) { + while (i < events.size()) { + if (events.get(++i).getEventType() == EventType.CONTAINER_END) { + depth--; + break; + } + while (i < events.size()) { + Event aEvent = events.get(i); + EventType aEventType = aEvent.getEventType(); + if (aEventType == EventType.CONTAINER_START) { + depth++; + } else if (aEventType == EventType.CONTAINER_END) { + if (--depth == 0) { + throw new IonException("Invalid EventStream: end without STREAM_END event"); + } + } else if (aEventType == EventType.STREAM_END) { + if (depth != 1) { + throw new IonException("Invalid EventStream: " + + "CONTAINER_START and CONTAINER_END not match"); + } + break; + } + i++; + } + if (i == events.size()) { + throw new IonException("Invalid EventStream: end without CONTAINER_END event"); + } + } + } + } + if (depth != 0) { + throw new IonException("Invalid event stream: CONTAINER_START and CONTAINER_END events doesn't match"); + } + } + + private static void ionStreamToEventStream(IonReader ionReader, + CommandType commandType, + SymbolTable curTable, + ReadContext readContext) throws IOException { + if (ionReader.getType() == null) return; + do { + if (ionReader.isNullValue()) { + IonValue value = ION_SYSTEM.newValue(ionReader); + value.clearTypeAnnotations(); + readContext.getEventStream().add(new Event(EventType.SCALAR, ionReader.getType(), ionReader.getFieldNameSymbol(), + ionReader.getTypeAnnotationSymbols(), value, null, ionReader.getDepth())); + continue; + } + + if (!isSameSymbolTable(ionReader.getSymbolTable(), curTable)) { + curTable = ionReader.getSymbolTable(); + ImportDescriptor[] imports = symbolTableToImports(curTable.getImportedTables()); + + if (commandType != CommandType.COMPARE) { + readContext.getEventStream().add(new Event(EventType.SYMBOL_TABLE, null, null, null, + null, imports, 0)); + } + } + if (isEmbeddedStream(ionReader)) { + //get current Ion type and depth + IonType curType = ionReader.getType(); + int curDepth = ionReader.getDepth(); + + //write a Container_Start event and step in + readContext.getEventStream().add(ionStreamToEvent(ionReader)); + ionReader.stepIn(); + + while (ionReader.next() != null) { + if (ionReader.getType() != IonType.STRING) { + throw new IonException("Elements of embedded streams sets must be strings."); + } + String stream = ionReader.stringValue(); + try (IonReader tempIonReader = IonReaderBuilder.standard().build(stream)) { + SymbolTable symbolTable = tempIonReader.getSymbolTable(); + while (tempIonReader.next() != null) { + ionStreamToEventStream(tempIonReader, commandType, symbolTable, readContext); + } + } + readContext.getEventStream().add (new Event(EventType.STREAM_END, null, null, null, + null, null, 0)); + } + //write a Container_End event and step out + readContext.getEventStream().add(new Event(EventType.CONTAINER_END, curType, null, null, + null, null, curDepth)); + ionReader.stepOut(); + } else if (IonType.isContainer(ionReader.getType())) { + //get current Ion type and depth + IonType curType = ionReader.getType(); + int curDepth = ionReader.getDepth(); + + //write a Container_Start event and step in + readContext.getEventStream().add(ionStreamToEvent(ionReader)); + ionReader.stepIn(); + + //recursive call + ionReader.next(); + ionStreamToEventStream(ionReader, commandType, curTable, readContext); + //write a Container_End event and step out + readContext.getEventStream().add(new Event(EventType.CONTAINER_END, curType, null, null, + null, null, curDepth)); + ionReader.stepOut(); + } else { + readContext.getEventStream().add(ionStreamToEvent(ionReader)); + } + } while (ionReader.next() != null);; + } + + private static ImportDescriptor[] symbolTableToImports(SymbolTable[] tables) { + if (tables == null || tables.length == 0) return null; + int size = tables.length; + + ImportDescriptor[] imports = new ImportDescriptor[size]; + for (int i = 0; i < size; i++) { + ImportDescriptor table = new ImportDescriptor(tables[i]); + imports[i] = table; + } + return imports; + } + + private static boolean isSameSymbolTable(SymbolTable newTable, SymbolTable curTable) { + if (newTable == null && curTable == null) { return true; } + else if (newTable != null && curTable == null) { return false; } + else if (newTable == null) { return false; } + + if (newTable.isLocalTable() && newTable.getImportedTables().length == 0) { + return true; + } else if (newTable.isSystemTable() && curTable.isSystemTable()) { + return newTable.getVersion() == curTable.getVersion(); + } else if (newTable.isSharedTable() && curTable.isSharedTable()) { + return (newTable.getName().equals(curTable.getName())) & (newTable.getVersion() == curTable.getVersion()); + } else if (newTable.isLocalTable() && curTable.isLocalTable()) { + SymbolTable[] xTable = newTable.getImportedTables(); + SymbolTable[] yTable = curTable.getImportedTables(); + //compare imports + if (xTable.length == yTable.length) { + for (int i = 0; i < xTable.length; i++) { + if (!isSameSymbolTable(xTable[i], yTable[i])) return false; + } + } else { + return false; + } + return true; + } else { + return false; + } + } + + private static boolean isIonVersionMarker(String text) { + return text != null && ION_VERSION_MARKER_REGEX.matcher(text).matches(); + } + + private static Event ionStreamToEvent(IonReader ionReader) throws IllegalStateException { + if (ionReader.getType() == null) throw new IllegalStateException("Can't convert ionReader null type to Event"); + IonType ionType = ionReader.getType(); + SymbolToken fieldName = ionReader.getFieldNameSymbol(); + SymbolToken[] annotations = ionReader.getTypeAnnotationSymbols(); + int depth = ionReader.getDepth(); + ImportDescriptor[] imports = null; + + EventType eventType; + IonValue value = null; + + if (IonType.isContainer(ionType)) { + eventType = EventType.CONTAINER_START; + } else { + eventType = EventType.SCALAR; + value = ION_SYSTEM.newValue(ionReader); + value.clearTypeAnnotations(); + if (isIonVersionMarker(value.toString())) { + value.setTypeAnnotationSymbols(_Private_Utils.newSymbolToken("$ion_user_value", 0)); + } + } + + return new Event(eventType, ionType, fieldName, annotations, value, imports, depth); + } + + private static Event eventStreamToEvent(IonReader ionReader) { + if (ionReader.getType() != IonType.STRUCT) throw new IonException("cant convert null"); + String textValue = null; + byte[] binaryValue = null; + IonValue eventValue = null; + EventType eventType = null; + IonType ionType = null; + SymbolToken fieldName = null; + SymbolToken[] annotations = null; + ImportDescriptor[] imports = null; + int depth = -1; + + ionReader.stepIn(); + while (ionReader.next() != null) { + switch (ionReader.getFieldName()) { + case "event_type": + if (eventType != null) throw new IonException("invalid Event: repeat event_type"); + eventType = EventType.valueOf(ionReader.stringValue().toUpperCase()); + break; + case "ion_type": + if (ionType != null) throw new IonException("invalid Event: repeat ion_type"); + ionType = IonType.valueOf(ionReader.stringValue().toUpperCase()); + break; + case "field_name": + if (fieldName != null) throw new IonException("invalid Event: repeat field_name"); + ionReader.stepIn(); + String fieldText = null; + int fieldSid = 0; + while (ionReader.next() != null) { + switch (ionReader.getFieldName()) { + case "text": + fieldText = ionReader.stringValue(); + break; + case "sid": + fieldSid = ionReader.intValue(); + break; + } + } + fieldName = _Private_Utils.newSymbolToken(fieldText, fieldSid); + ionReader.stepOut(); + break; + case "annotations": + if (annotations != null) throw new IonException("invalid Event: repeat annotations"); + ArrayList annotationsList = new ArrayList<>(); + ionReader.stepIn(); + while (ionReader.next() != null) { + ionReader.stepIn(); + String text = null; + int sid = 0; + while (ionReader.next() != null) { + switch (ionReader.getFieldName()) { + case "text": + text = ionReader.isNullValue() ? null : ionReader.stringValue(); + break; + case "sid": + sid = ionReader.intValue(); + break; + } + } + SymbolToken annotation = _Private_Utils.newSymbolToken(text, sid); + annotationsList.add(annotation); + ionReader.stepOut(); + } + annotations = annotationsList.toArray(SymbolToken.EMPTY_ARRAY); + ionReader.stepOut(); + break; + case "value_text": + if (textValue != null) throw new IonException("invalid Event: repeat value_text"); + textValue = ionReader.stringValue(); + break; + case "value_binary": + if (binaryValue != null) throw new IonException("invalid Event: repeat binary_value"); + ArrayList intArray = new ArrayList<>(); + ionReader.stepIn(); + while (ionReader.next() != null) { + intArray.add(ionReader.intValue()); + } + byte[] binary = new byte[intArray.size()]; + for (int i = 0; i < intArray.size(); i++) { + int val = intArray.get(i); + binary[i] = (byte) (val & 0xff); + } + binaryValue = binary; + ionReader.stepOut(); + break; + case "imports": + if (imports != null) throw new IonException("invalid Event: repeat imports"); + imports = ionStreamToImportDescriptors(ionReader); + break; + case "depth": + if (depth != -1) throw new IonException("invalid Event: repeat depth"); + depth = ionReader.intValue(); + break; + } + } + ionReader.stepOut(); + //validate event + validateEvent(textValue, binaryValue, eventType, fieldName, ionType, imports, depth); + if (textValue != null) eventValue = ION_SYSTEM.singleValue(textValue); + + return new Event(eventType, ionType, fieldName, annotations, eventValue, imports, depth); + } + + private static void validateEvent(String textValue, byte[] binaryValue, EventType eventType, SymbolToken fieldName, + IonType ionType, ImportDescriptor[] imports, int depth) { + if (eventType == null) throw new IllegalStateException("only can convert Struct to Event"); + + switch (eventType) { + case CONTAINER_START: + if (ionType == null || depth == -1) { + throw new IonException("Invalid CONTAINER_START: missing field(s)"); + } else if (!IonType.isContainer(ionType)) { + throw new IonException("Invalid CONTAINER_START: not a container"); + } else if (textValue != null || binaryValue != null) { + throw new IonException("Invalid CONTAINER_START: value_binary and value_text are only applicable" + + " for SCALAR events"); + } else if (imports != null) { + throw new IonException("Invalid CONTAINER_START: imports must only be present with SYMBOL_TABLE " + + "events"); + } + break; + case SCALAR: + if (ionType == null || textValue == null || binaryValue == null || depth == -1) { + throw new IonException("Invalid SCALAR: missing field(s)"); + } else if (imports != null) { + throw new IonException("Invalid SCALAR: imports must only be present with SYMBOL_TABLE " + + "events"); + } + //compare text value and binary value + IonValue text = ION_SYSTEM.singleValue(textValue); + IonValue binary = ION_SYSTEM.singleValue(binaryValue); + if (!Equivalence.ionEquals(text, binary)) { + throw new IonException("invalid Event: Text value and Binary value are different"); + } + break; + case SYMBOL_TABLE: + if (depth == -1) { + throw new IonException("Invalid SYMBOL_TABLE: missing depth"); + } else if (imports == null ) { + throw new IonException("Invalid SYMBOL_TABLE: missing imports"); + } else if (textValue != null && binaryValue != null) { + throw new IonException("Invalid SYMBOL_TABLE: text_value and binary_value " + + "are only applicable for SCALAR events"); + } else if (fieldName != null && ionType != null) { + throw new IonException("Invalid SYMBOL_TABLE: unnecessary fields"); + } + break; + case CONTAINER_END: + if (depth == -1 || ionType == null) { + throw new IonException("Invalid CONTAINER_END: missing depth"); + } else if (textValue != null && binaryValue != null) { + throw new IonException("Invalid CONTAINER_END: text_value and binary_value " + + "are only applicable for SCALAR events"); + } else if (fieldName != null && imports != null) { + throw new IonException("Invalid CONTAINER_END: unnecessary fields"); + } + break; + case STREAM_END: + if (depth == -1) { + throw new IonException("Invalid STREAM_END: missing depth"); + } else if (textValue != null && binaryValue != null) { + throw new IonException("Invalid STREAM_END: text_value and binary_value " + + "are only applicable for SCALAR events"); + } else if (fieldName != null && ionType != null && imports != null) { + throw new IonException("Invalid STREAM_END: unnecessary fields"); + } + break; + default: + throw new IonException("Invalid event_type"); + } + } + + private static ImportDescriptor[] ionStreamToImportDescriptors(IonReader ionReader) { + ArrayList imports = new ArrayList<>(); + + ionReader.stepIn(); + while (ionReader.next() != null) { + String importName = null; + int maxId = 0; + int version = 0; + + ionReader.stepIn(); + while (ionReader.next() != null) { + switch (ionReader.getFieldName()) { + case "name": + importName = ionReader.stringValue(); + break; + case "max_id": + maxId = ionReader.intValue(); + break; + case "version": + version = ionReader.intValue(); + break; + } + } + ionReader.stepOut(); + + ImportDescriptor table = new ImportDescriptor(importName, maxId, version); + imports.add(table); + } + ImportDescriptor[] importsArray = imports.toArray(new ImportDescriptor[0]); + + ionReader.stepOut(); + return importsArray; + } + + /** + * This function creates a new file for output stream with the fileName, If file name is empty or undefined, + * it will create a default output which is System.out or System.err. + * + * this function never return null + */ + private static BufferedOutputStream initOutputStream(CommandArgs args, + String defaultValue, + ProcessContext processContext) throws IOException { + BufferedOutputStream outputStream = null; + switch (defaultValue) { + case SYSTEM_OUT_DEFAULT_VALUE: + String outputFile = args.getOutputFile(); + if (outputFile != null && outputFile.length() != 0) { + File myFile = new File(outputFile); + if (args.getCommand() == CommandType.PROCESS && processContext != null) { + processContext.setFile(myFile); + } + FileOutputStream out = new FileOutputStream(myFile); + outputStream = new BufferedOutputStream(out, BUFFER_SIZE); + } else { + outputStream = new BufferedOutputStream(new NoCloseOutputStream(System.out), BUFFER_SIZE); + } + break; + case SYSTEM_ERR_DEFAULT_VALUE: + String errorReport = args.getErrorReport(); + if (errorReport != null && errorReport.length() != 0) { + File myFile = new File(errorReport); + FileOutputStream out = new FileOutputStream(myFile, true); + outputStream = new BufferedOutputStream(out, BUFFER_SIZE); + } else { + outputStream = new BufferedOutputStream(System.err, BUFFER_SIZE); + } + break; + } + return outputStream; + } + + static class Pair { + final int left; + final int right; + + Pair(int left, int right) { + this.left = left; + this.right = right; + } + } + + static class ReadContext { + final private List eventStream; + private ErrorType state; + + public ReadContext(List eventStream) { + this.eventStream = eventStream; + this.state = ErrorType.READ; + } + + public List getEventStream() { + return eventStream; + } + + public ErrorType getState() { + return state; + } + + public void setState(ErrorType state) { + this.state = state; + } + } + + static class ContainerContext { + int index; + + ContainerContext(int index) { + this.index = index; + } + + public int getIndex() { + return index; + } + + public void setIndex(int index) { + this.index = index; + } + + public int increaseIndex() { + this.index = this.index + 1; + return this.index; + } + } + + static class CommandArgs { + private static final String DEFAULT_FORMAT_VALUE = OutputFormat.PRETTY.toString(); + private static final String DEFAULT_COMPARISON_TYPE = ComparisonType.BASIC.toString(); + + @Option(name = "--output", + aliases = {"-o"}, + metaVar = "FILE", + usage = "Output file") + private String outputFile; + + @Option(name = "--error-report", + aliases = {"-e"}, + metaVar = "FILE", + usage = "Error report file") + private String errorReport; + + @Option(name = "--output-format", + aliases = {"-f"}, + metaVar = "TYPE", + usage = "Output format, from the set (text | pretty | binary |\n" + + "events | none). 'events' is only available with the\n" + + "'process' command, and outputs a serialized EventStream\n" + + "representing the input Ion stream(s).") + private String outputFormatName = DEFAULT_FORMAT_VALUE; + + @Option(name = "--comparison-type", + aliases = {"-y"}, + metaVar = "TYPE", + usage = "Comparison semantics to be used with the compare command, from the set (basic | equivs |\n" + + "non-equivs | equiv-timeline). Any embedded streams in the inputs are compared for\n" + + "EventStream equality. 'basic' performs a standard data-model comparison between the\n" + + "corresponding events (or embedded streams) in the inputs. 'equivs' verifies that each \n" + + "value (or embedded stream) in a top-level sequence is equivalent to every other value \n" + + "(or embedded stream) in that sequence. 'non-equivs' does the same, but verifies that the \n" + + "values (or embedded streams) are not equivalent. 'equiv-timeline' is the same as 'equivs',\n" + + " except that when top-level sequences contain timestamp values, they are considered \n" + + "equivalent if they represent the same instant regardless of whether they are considered \n" + + "equivalent by the Ion data model.") + private String comparisonType = DEFAULT_COMPARISON_TYPE; + + @Argument(required = true) + private List inputs; + + private CommandType commandType = null; + + public String getOutputFormatName() { + return outputFormatName; + } + + public ComparisonType getComparisonType() throws IllegalArgumentException { + return ComparisonType.valueOf(comparisonType.replace('-','_').toUpperCase()); + } + + public CommandType getCommand() { + return this.commandType; + } + + public void setCommand() { + this.commandType = CommandType.valueOf(inputs.get(0).toUpperCase()); + } + + public OutputFormat getOutputFormat() throws IllegalArgumentException { + return OutputFormat.valueOf(outputFormatName.toUpperCase()); + } + + public List getInputFiles() { + int length = this.inputs.size(); + return this.inputs.subList(1, length); + } + public String getOutputFile() { return outputFile; } + public String getErrorReport() { return errorReport; } + } +} diff --git a/ion-java-cli/src/com/amazon/tools/cli/NoCloseOutputStream.java b/ion-java-cli/src/com/amazon/tools/cli/NoCloseOutputStream.java new file mode 100644 index 0000000000..b202b7807e --- /dev/null +++ b/ion-java-cli/src/com/amazon/tools/cli/NoCloseOutputStream.java @@ -0,0 +1,17 @@ +package com.amazon.tools.cli; + +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +public class NoCloseOutputStream extends FilterOutputStream { + + public NoCloseOutputStream(OutputStream out) { + super(out); + } + + @Override + public void close() throws IOException{ + out.flush(); + } +} diff --git a/ion-java-cli/src/com/amazon/tools/cli/NoOpOutputStream.java b/ion-java-cli/src/com/amazon/tools/cli/NoOpOutputStream.java new file mode 100644 index 0000000000..2ff022e8e1 --- /dev/null +++ b/ion-java-cli/src/com/amazon/tools/cli/NoOpOutputStream.java @@ -0,0 +1,10 @@ +package com.amazon.tools.cli; + +import java.io.OutputStream; + +public class NoOpOutputStream extends OutputStream { + @Override + public void write(int b) { + return; + } +} diff --git a/ion-java-cli/src/com/amazon/tools/cli/OutputFormat.java b/ion-java-cli/src/com/amazon/tools/cli/OutputFormat.java new file mode 100644 index 0000000000..15a658fcd3 --- /dev/null +++ b/ion-java-cli/src/com/amazon/tools/cli/OutputFormat.java @@ -0,0 +1,88 @@ +package com.amazon.tools.cli; + +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.system.IonBinaryWriterBuilder; +import com.amazon.ion.system.IonTextWriterBuilder; + +import java.io.OutputStream; + +/** + * Represents the different Ion output formats supported by the command line com.amazon.tools in this package. + */ +public enum OutputFormat { + /** + * Nicely spaced, 'prettified' text Ion. + */ + PRETTY { + @Override + public IonWriter createIonWriter(OutputStream outputStream) { + return IonTextWriterBuilder.pretty().build(outputStream); + } + + @Override + public IonWriter createIonWriterWithImports(OutputStream outputStream, SymbolTable[] imports) { + return IonTextWriterBuilder.pretty().withImports(imports).build(outputStream); + } + }, + /** + * Minimally spaced text Ion. + */ + TEXT { + @Override + public IonWriter createIonWriter(OutputStream outputStream) { + return IonTextWriterBuilder.standard().build(outputStream); + } + + @Override + public IonWriter createIonWriterWithImports(OutputStream outputStream, SymbolTable[] imports) { + return IonTextWriterBuilder.standard().withImports(imports).build(outputStream); + } + }, + /** + * Compact, read-optimized binary Ion. + */ + BINARY { + @Override + public IonWriter createIonWriter(OutputStream outputStream) { + return IonBinaryWriterBuilder.standard().build(outputStream); + } + + @Override + public IonWriter createIonWriterWithImports(OutputStream outputStream, SymbolTable[] imports) { + return IonBinaryWriterBuilder.standard().withImports(imports).build(outputStream); + } + }, + /** + * Event Stream + */ + EVENTS { + @Override + public IonWriter createIonWriter(OutputStream outputStream) { + return IonTextWriterBuilder.pretty().build(outputStream); + } + + @Override + public IonWriter createIonWriterWithImports(OutputStream outputStream, SymbolTable[] imports) { + return IonTextWriterBuilder.pretty().withImports(imports).build(outputStream); + } + }, + /** + * None + */ + NONE { + @Override + public IonWriter createIonWriter(OutputStream outputStream) { + NoOpOutputStream out = new NoOpOutputStream(); + return IonTextWriterBuilder.pretty().build(out); + } + + @Override + public IonWriter createIonWriterWithImports(OutputStream outputStream, SymbolTable[] imports) { + return IonTextWriterBuilder.pretty().withImports(imports).build(outputStream); + } + }; + + abstract IonWriter createIonWriter(OutputStream outputStream); + abstract IonWriter createIonWriterWithImports(OutputStream outputStream, SymbolTable[] symbolTable); +} diff --git a/ion-java-cli/src/com/amazon/tools/cli/ProcessContext.java b/ion-java-cli/src/com/amazon/tools/cli/ProcessContext.java new file mode 100644 index 0000000000..82662e8328 --- /dev/null +++ b/ion-java-cli/src/com/amazon/tools/cli/ProcessContext.java @@ -0,0 +1,95 @@ +package com.amazon.tools.cli; + +import com.amazon.ion.IonWriter; +import com.amazon.tools.errorReport.ErrorType; +import com.amazon.tools.events.Event; +import com.amazon.tools.events.EventType; + +import java.io.File; +import java.util.List; + +public class ProcessContext { + private String fileName; + private int eventIndex; + private EventType lastEventType; + private ErrorType state; + private IonWriter ionWriter; + private File file; + private StringBuilder embeddedOut; + private List eventStream; + + public ProcessContext(String file, int index, EventType lastEventType, ErrorType state, IonWriter ionWriter) { + this.fileName = file; + this.eventIndex = index; + this.lastEventType = lastEventType; + this.state = state; + this.ionWriter = ionWriter; + this.file = null; + this.embeddedOut = null; + this.eventStream = null; + } + + public String getFileName() { + return this.fileName; + } + + public int getEventIndex() { + return this.eventIndex; + } + + public EventType getLastEventType() { + return this.lastEventType; + } + + public ErrorType getState() { + return state; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public void setEventIndex(int eventIndex) { + this.eventIndex = eventIndex; + } + + public void setLastEventType(EventType lastEventType) { + this.lastEventType = lastEventType; + } + + public void setState(ErrorType state) { + this.state = state; + } + + public IonWriter getIonWriter() { + return ionWriter; + } + + public void setIonWriter(IonWriter ionWriter) { + this.ionWriter = ionWriter; + } + + public File getFile() { + return file; + } + + public void setFile(File file) { + this.file = file; + } + + public StringBuilder getEmbeddedOut() { + return embeddedOut; + } + + public void setEmbeddedOut(StringBuilder embeddedOut) { + this.embeddedOut = embeddedOut; + } + + public List getEventStream() { + return eventStream; + } + + public void setEventStream(List eventStream) { + this.eventStream = eventStream; + } +} diff --git a/ion-java-cli/src/com/amazon/tools/comparisonReport/ComparisonContext.java b/ion-java-cli/src/com/amazon/tools/comparisonReport/ComparisonContext.java new file mode 100644 index 0000000000..e8104dfcfe --- /dev/null +++ b/ion-java-cli/src/com/amazon/tools/comparisonReport/ComparisonContext.java @@ -0,0 +1,27 @@ +package com.amazon.tools.comparisonReport; + +import com.amazon.tools.events.Event; + +public class ComparisonContext { + private final String location; + private final Event event; + private final int eventIndex; + + public ComparisonContext(String location, Event event, int eventIndex) { + this.location = location; + this.event = event; + this.eventIndex = eventIndex; + } + + public String getLocation() { + return location; + } + + public Event getEvent() { + return event; + } + + public int getEventIndex() { + return eventIndex; + } +} diff --git a/ion-java-cli/src/com/amazon/tools/comparisonReport/ComparisonResult.java b/ion-java-cli/src/com/amazon/tools/comparisonReport/ComparisonResult.java new file mode 100644 index 0000000000..5fb22ecbd5 --- /dev/null +++ b/ion-java-cli/src/com/amazon/tools/comparisonReport/ComparisonResult.java @@ -0,0 +1,55 @@ +package com.amazon.tools.comparisonReport; + +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; + +import java.io.IOException; + +public class ComparisonResult { + private final ComparisonResultType result; + private final ComparisonContext lhs; + private final ComparisonContext rhs; + private final String message; + + public ComparisonResult(ComparisonResultType result, ComparisonContext lhs, + ComparisonContext rhs, String message) { + this.result = result; + this.lhs = lhs; + this.rhs = rhs; + this.message = message; + } + + public void writeOutput(IonWriter ionWriter) throws IOException { + ionWriter.stepIn(IonType.STRUCT); + if (this.result != null) { + ionWriter.setFieldName("result"); + ionWriter.writeString(result.toString()); + } + if (this.message != null) { + ionWriter.setFieldName("message"); + ionWriter.writeString(message); + } + if (this.lhs != null) { + ionWriter.setFieldName("lhs"); + writeComparisonContext(ionWriter, this.lhs); + } + if (this.rhs != null) { + ionWriter.setFieldName("rhs"); + writeComparisonContext(ionWriter, this.rhs); + } + ionWriter.stepOut(); + } + + private void writeComparisonContext(IonWriter ionWriter, ComparisonContext comparisonContext) throws IOException { + ionWriter.stepIn(IonType.STRUCT); + ionWriter.setFieldName("location"); + ionWriter.writeString(comparisonContext.getLocation()); + if (comparisonContext.getEvent() != null) { + ionWriter.setFieldName("event"); + comparisonContext.getEvent().writeOutput(ionWriter, -1); + } + ionWriter.setFieldName("event_index"); + ionWriter.writeInt(comparisonContext.getEventIndex()); + ionWriter.stepOut(); + } +} diff --git a/ion-java-cli/src/com/amazon/tools/comparisonReport/ComparisonResultType.java b/ion-java-cli/src/com/amazon/tools/comparisonReport/ComparisonResultType.java new file mode 100644 index 0000000000..b82864eebe --- /dev/null +++ b/ion-java-cli/src/com/amazon/tools/comparisonReport/ComparisonResultType.java @@ -0,0 +1,7 @@ +package com.amazon.tools.comparisonReport; + +public enum ComparisonResultType { + EQUAL, + NOT_EQUAL, + ERROR +} diff --git a/ion-java-cli/src/com/amazon/tools/errorReport/ErrorDescription.java b/ion-java-cli/src/com/amazon/tools/errorReport/ErrorDescription.java new file mode 100644 index 0000000000..5efccade2c --- /dev/null +++ b/ion-java-cli/src/com/amazon/tools/errorReport/ErrorDescription.java @@ -0,0 +1,41 @@ +package com.amazon.tools.errorReport; + +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; + +import java.io.IOException; + +public class ErrorDescription { + private final ErrorType errorType; + private final String message; + private final String location; + private final int eventIndex; + + public ErrorDescription(ErrorType errorType, String message, String location, int eventIndex) { + this.errorType = errorType; + this.message = message; + this.location = location; + this.eventIndex = eventIndex; + } + + public void writeOutput(IonWriter ionWriterForErrorReport) throws IOException { + ionWriterForErrorReport.stepIn(IonType.STRUCT); + if (this.errorType != null) { + ionWriterForErrorReport.setFieldName("error_type"); + ionWriterForErrorReport.writeSymbol(this.errorType.toString()); + } + if (this.message != null) { + ionWriterForErrorReport.setFieldName("message"); + ionWriterForErrorReport.writeString(this.message); + } + if (this.location != null) { + ionWriterForErrorReport.setFieldName("location"); + ionWriterForErrorReport.writeString(this.location); + } + if (this.eventIndex != -1) { + ionWriterForErrorReport.setFieldName("event_index"); + ionWriterForErrorReport.writeInt(this.eventIndex); + } + ionWriterForErrorReport.stepOut(); + } +} diff --git a/ion-java-cli/src/com/amazon/tools/errorReport/ErrorType.java b/ion-java-cli/src/com/amazon/tools/errorReport/ErrorType.java new file mode 100644 index 0000000000..9bbd678e56 --- /dev/null +++ b/ion-java-cli/src/com/amazon/tools/errorReport/ErrorType.java @@ -0,0 +1,7 @@ +package com.amazon.tools.errorReport; + +public enum ErrorType { + READ, + WRITE, + STATE +} diff --git a/ion-java-cli/src/com/amazon/tools/events/Event.java b/ion-java-cli/src/com/amazon/tools/events/Event.java new file mode 100644 index 0000000000..cb89b328fe --- /dev/null +++ b/ion-java-cli/src/com/amazon/tools/events/Event.java @@ -0,0 +1,179 @@ +package com.amazon.tools.events; + +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.IonWriter; + +import com.amazon.ion.system.IonBinaryWriterBuilder; +import com.amazon.ion.system.IonTextWriterBuilder; +import com.amazon.tools.cli.ProcessContext; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +public class Event { + private static final int IO_ERROR_EXIT_CODE = 2; + + private final EventType eventType; + private final IonType ionType; + private final SymbolToken fieldName; + private final SymbolToken[] annotations; + private final IonValue value; + private final ImportDescriptor[] imports; + private final int depth; + + public Event(EventType eventType, IonType ionType, SymbolToken fieldName, SymbolToken[] annotations, + IonValue value, ImportDescriptor[] imports, int depth) { + this.eventType = eventType; + this.ionType = ionType; + this.fieldName = fieldName; + this.annotations = annotations; + this.value = value; + this.imports = imports; + this.depth = depth; + } + + public void writeOutput(IonWriter ionWriterForErrorReport, + ProcessContext processContext) throws IOException { + int updatedEventIndex = writeOutput(processContext.getIonWriter(),processContext.getEventIndex()); + + processContext.setEventIndex(updatedEventIndex); + } + + /** + * write Event structure to Event Stream and return the updated eventIndex. + */ + public int writeOutput(IonWriter ionWriterForOutput, + int eventIndex) throws IOException { + ionWriterForOutput.stepIn(IonType.STRUCT); + + if (this.eventType != null) { + ionWriterForOutput.setFieldName("event_type"); + ionWriterForOutput.writeSymbol(this.eventType.toString()); + } + + if (this.ionType != null) { + ionWriterForOutput.setFieldName("ion_type"); + ionWriterForOutput.writeSymbol(this.ionType.toString()); + } + + if (this.fieldName != null) { + ionWriterForOutput.setFieldName("field_name"); + ionWriterForOutput.stepIn(IonType.STRUCT); + ionWriterForOutput.setFieldName("text"); + if (this.fieldName.getText() == null) { + ionWriterForOutput.writeNull(); + } else { + ionWriterForOutput.writeString(this.fieldName.getText()); + } + ionWriterForOutput.setFieldName("import_location"); + ionWriterForOutput.writeNull(); + ionWriterForOutput.stepOut(); + } + + if (this.annotations != null && this.annotations.length > 0) { + ionWriterForOutput.setFieldName("annotations"); + ionWriterForOutput.stepIn(IonType.LIST); + for (SymbolToken annotation : this.annotations) { + ionWriterForOutput.stepIn(IonType.STRUCT); + ionWriterForOutput.setFieldName("text"); + String text = annotation.getText(); + if (text == null) { + ionWriterForOutput.writeNull(); + ionWriterForOutput.setFieldName("import_location"); + ionWriterForOutput.writeNull(); + } else { + ionWriterForOutput.writeString(text); + } + ionWriterForOutput.stepOut(); + } + ionWriterForOutput.stepOut(); + } + + if (this.value != null) { + String valueText; + byte[] valueBinary; + try ( + ByteArrayOutputStream textOut = new ByteArrayOutputStream(); + IonWriter textWriter = IonTextWriterBuilder.standard().build(textOut); + ByteArrayOutputStream binaryOut = new ByteArrayOutputStream(); + IonWriter binaryWriter = IonBinaryWriterBuilder.standard().build(binaryOut); + ) { + //write Text + this.value.writeTo(textWriter); + textWriter.finish(); + valueText = textOut.toString("utf-8"); + + ionWriterForOutput.setFieldName("value_text"); + ionWriterForOutput.writeString(valueText); + + //write binary + this.value.writeTo(binaryWriter); + binaryWriter.finish(); + valueBinary = binaryOut.toByteArray(); + + ionWriterForOutput.setFieldName("value_binary"); + ionWriterForOutput.stepIn(IonType.LIST); + for (byte b : valueBinary) { + ionWriterForOutput.writeInt(b & 0xff); + } + ionWriterForOutput.stepOut(); + } + } + + if (this.imports != null && this.imports.length > 0) { + ionWriterForOutput.setFieldName("imports"); + ionWriterForOutput.stepIn(IonType.LIST); + for (ImportDescriptor anImport : this.imports) { + ionWriterForOutput.stepIn(IonType.STRUCT); + ionWriterForOutput.setFieldName("name"); + ionWriterForOutput.writeString(anImport.getImportName()); + ionWriterForOutput.setFieldName("version"); + ionWriterForOutput.writeInt(anImport.getVersion()); + ionWriterForOutput.setFieldName("max_id"); + ionWriterForOutput.writeInt(anImport.getMaxId()); + ionWriterForOutput.stepOut(); + } + ionWriterForOutput.stepOut(); + } + + if (this.depth != -1) { + ionWriterForOutput.setFieldName("depth"); + ionWriterForOutput.writeInt(this.depth); + } + + ionWriterForOutput.stepOut(); + + //event index + 1 if we write OutputStream successfully. + return eventIndex + 1; + } + + public EventType getEventType() { + return eventType; + } + + public IonType getIonType() { + return ionType; + } + + public SymbolToken getFieldName() { + return fieldName; + } + + public SymbolToken[] getAnnotations() { + return annotations; + } + + public ImportDescriptor[] getImports() { + return imports; + } + + public int getDepth() { + return depth; + } + + public IonValue getValue() { + return value; + } +} diff --git a/ion-java-cli/src/com/amazon/tools/events/EventType.java b/ion-java-cli/src/com/amazon/tools/events/EventType.java new file mode 100644 index 0000000000..2fa801152b --- /dev/null +++ b/ion-java-cli/src/com/amazon/tools/events/EventType.java @@ -0,0 +1,9 @@ +package com.amazon.tools.events; + +public enum EventType { + CONTAINER_START, + CONTAINER_END, + SCALAR, + SYMBOL_TABLE, + STREAM_END, +} diff --git a/ion-java-cli/src/com/amazon/tools/events/ImportDescriptor.java b/ion-java-cli/src/com/amazon/tools/events/ImportDescriptor.java new file mode 100644 index 0000000000..f64f74cd1a --- /dev/null +++ b/ion-java-cli/src/com/amazon/tools/events/ImportDescriptor.java @@ -0,0 +1,33 @@ +package com.amazon.tools.events; + +import com.amazon.ion.SymbolTable; + +public class ImportDescriptor { + private final String importName; + private final int maxId; + private final int version; + + public ImportDescriptor(String importName, int maxId, int version) { + this.importName = importName; + this.maxId = maxId; + this.version = version; + } + + public ImportDescriptor(SymbolTable symbolTable) { + this.importName = symbolTable.getName(); + this.maxId = symbolTable.getMaxId(); + this.version = symbolTable.getVersion(); + } + + public int getVersion() { + return version; + } + + public int getMaxId() { + return maxId; + } + + public String getImportName() { + return importName; + } +} From 2f734a4cdcc6d0598fcf209e6e3158049eb1d497 Mon Sep 17 00:00:00 2001 From: Eric Chen Date: Thu, 10 Sep 2020 18:25:16 -0700 Subject: [PATCH 106/490] Implements version command for ion-java-cli. --- ion-java-cli/src/com/amazon/tools/cli/CommandType.java | 3 ++- ion-java-cli/src/com/amazon/tools/cli/IonJavaCli.java | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ion-java-cli/src/com/amazon/tools/cli/CommandType.java b/ion-java-cli/src/com/amazon/tools/cli/CommandType.java index c2168bcc25..4ee74e658b 100644 --- a/ion-java-cli/src/com/amazon/tools/cli/CommandType.java +++ b/ion-java-cli/src/com/amazon/tools/cli/CommandType.java @@ -2,5 +2,6 @@ public enum CommandType { PROCESS, - COMPARE + COMPARE, + VERSION } diff --git a/ion-java-cli/src/com/amazon/tools/cli/IonJavaCli.java b/ion-java-cli/src/com/amazon/tools/cli/IonJavaCli.java index 59086f3b54..e54965af60 100644 --- a/ion-java-cli/src/com/amazon/tools/cli/IonJavaCli.java +++ b/ion-java-cli/src/com/amazon/tools/cli/IonJavaCli.java @@ -42,6 +42,7 @@ import java.util.regex.Pattern; public class IonJavaCli { + private static final String VERSION = "1.0"; private static final int CONSOLE_WIDTH = 120; // Only used for formatting the USAGE message private static final int USAGE_ERROR_EXIT_CODE = 1; private static final int IO_ERROR_EXIT_CODE = 2; @@ -75,6 +76,11 @@ public static void main(final String[] args) { System.exit(USAGE_ERROR_EXIT_CODE); } + if (commandType == CommandType.VERSION) { + System.err.println(VERSION); + System.exit(0); + } + ProcessContext processContext = commandType == CommandType.PROCESS ? new ProcessContext(null, -1, null, ErrorType.READ, null) : null; try ( From 16cb1460e7da04c63b56d87248d0b22f37f450b1 Mon Sep 17 00:00:00 2001 From: Eric Chen Date: Thu, 10 Sep 2020 18:51:38 -0700 Subject: [PATCH 107/490] Adds travis logic to check ion-java-cli version. --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c40e335f83..8be4b80dfb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,10 @@ jdk: - oraclejdk8 - oraclejdk9 -script: mvn test +script: + - mvn test + - mvn package -f ion-java-cli/pom.xml + - java -jar ion-java-cli/target/ion-java-cli-1.0.jar version jobs: include: From 16c1da3e7c6f1dcc3087a1e5a5c7f3ca98e6b9ad Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Fri, 9 Oct 2020 13:44:42 -0700 Subject: [PATCH 108/490] Fixes bug #318 in the binary reader's implementation of IonReader.getBytes. --- .../amazon/ion/impl/IonReaderBinaryRawX.java | 6 +---- test/com/amazon/ion/streaming/ReaderTest.java | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/com/amazon/ion/impl/IonReaderBinaryRawX.java b/src/com/amazon/ion/impl/IonReaderBinaryRawX.java index 5bbc543503..656000c3aa 100644 --- a/src/com/amazon/ion/impl/IonReaderBinaryRawX.java +++ b/src/com/amazon/ion/impl/IonReaderBinaryRawX.java @@ -737,16 +737,12 @@ public int readBytes(byte[] buffer, int offset, int len) if (offset < 0 || len < 0) { throw new IllegalArgumentException(); } - int value_len = byteSize(); // again validation - if (_value_lob_remaining > len) { - len = _value_lob_remaining; - } if (len < 1) { return 0; } int read_len; try { - read_len = read(buffer, offset, value_len); + read_len = read(buffer, offset, len); _value_lob_remaining -= read_len; } catch (IOException e) { diff --git a/test/com/amazon/ion/streaming/ReaderTest.java b/test/com/amazon/ion/streaming/ReaderTest.java index 50ba1ab6d5..04c18a3cfb 100644 --- a/test/com/amazon/ion/streaming/ReaderTest.java +++ b/test/com/amazon/ion/streaming/ReaderTest.java @@ -371,6 +371,28 @@ public void testSkippingLobInStruct() testSkippingLob("{a:1, b:{ c:", "}}"); } + @Test + public void testReadLobUsingGetBytes() throws Exception + { + if (!myReaderMaker.sourceIsBinary()) { + // TODO text implements getBytes() differently: https://github.com/amzn/ion-java/issues/319 + return; + } + String data = "{{\"abcdefghijklmnopqrstuvwxyz\"}}"; + byte[] partialBuffer = new byte[10]; + read(data); + assertEquals(IonType.CLOB, in.next()); + int remaining = in.byteSize(); + assertEquals(26, remaining); + StringBuilder value = new StringBuilder(); + while (remaining > 0) { + int bytesRead = in.getBytes(partialBuffer, 0, partialBuffer.length); + remaining -= bytesRead; + value.append(new String(partialBuffer, 0, bytesRead, "UTF-8")); + } + assertEquals("abcdefghijklmnopqrstuvwxyz", value.toString()); + } + @Test public void testGetSymbolTableBeforeFirstValue() { From 7f590bb4d24d530303c41edee6a5bd51a391b84c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2020 13:49:10 -0700 Subject: [PATCH 109/490] Bump junit from 4.12 to 4.13.1 (#322) Bumps [junit](https://github.com/junit-team/junit4) from 4.12 to 4.13.1. - [Release notes](https://github.com/junit-team/junit4/releases) - [Changelog](https://github.com/junit-team/junit4/blob/main/doc/ReleaseNotes4.12.md) - [Commits](https://github.com/junit-team/junit4/compare/r4.12...r4.13.1) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e2b56980a5..1e9097b005 100644 --- a/pom.xml +++ b/pom.xml @@ -73,7 +73,7 @@ junit junit - 4.12 + 4.13.1 test From 37f48737061d0274c360f39c1dd4af918894b89c Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Tue, 27 Oct 2020 16:41:30 -0700 Subject: [PATCH 110/490] Avoids repetitive reallocation of PooledBlockAllocatorProvider in _PrivateIon_HashTrampoline. --- src/com/amazon/ion/impl/bin/_PrivateIon_HashTrampoline.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/com/amazon/ion/impl/bin/_PrivateIon_HashTrampoline.java b/src/com/amazon/ion/impl/bin/_PrivateIon_HashTrampoline.java index a54da159a1..aea1e31a78 100644 --- a/src/com/amazon/ion/impl/bin/_PrivateIon_HashTrampoline.java +++ b/src/com/amazon/ion/impl/bin/_PrivateIon_HashTrampoline.java @@ -28,10 +28,12 @@ @Deprecated public class _PrivateIon_HashTrampoline { + private static final PooledBlockAllocatorProvider ALLOCATOR_PROVIDER = new PooledBlockAllocatorProvider(); + public static IonWriter newIonWriter(ByteArrayOutputStream baos) throws IOException { return new IonRawBinaryWriter( - new PooledBlockAllocatorProvider(), + ALLOCATOR_PROVIDER, _Private_IonManagedBinaryWriterBuilder.DEFAULT_BLOCK_SIZE, baos, AbstractIonWriter.WriteValueOptimization.NONE, From 69e62f618fbf4f5d4068aca5e7dc6d4ec777b34d Mon Sep 17 00:00:00 2001 From: Eric Chen <67451029+cheqianh@users.noreply.github.com> Date: Thu, 29 Oct 2020 18:43:47 -0700 Subject: [PATCH 111/490] Adds ion-test-driver analysis to Github Actions. (#325) --- .github/ISSUE_TEMPLATE.md | 6 +++ .github/workflows/main.yml | 80 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/workflows/main.yml diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000000..1e2141d9e3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,6 @@ +--- +title: Ion-test-driver issue. +labels: bug +--- +Ion-test-driver complained that behavior changed for the commit {{ env.GITHUB_PR_SHA }} created by `{{ payload.sender.login }}`. +Refer to the [workflow]({{ env.GITHUB_WORKFLOW_URL }}) for more details. diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000000..64ec69e087 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,80 @@ +name: ion-test-driver + +on: [pull_request] + +jobs: + ion-test-driver: + runs-on: macos-10.15 + steps: + - name: Checkout ion-java + uses: actions/checkout@master + with: + repository: amzn/ion-java + ref: master + path: ion-java + + - name: Get main branch HEAD sha + run: cd ion-java && echo "sha=`git rev-parse --short HEAD`" >> $GITHUB_ENV + + - name: Checkout ion-test-driver + uses: actions/checkout@master + with: + repository: amzn/ion-test-driver + ref: master + path: ion-test-driver + + - name: Set up python3 env + run: python3 -m venv ion-test-driver/venv && . ion-test-driver/venv/bin/activate + + - name: Pip install + run: pip3 install -r ion-test-driver/requirements.txt && pip3 install -e ion-test-driver + + - name: Run ion-test-driver + run: python3 ion-test-driver/amazon/iontest/ion_test_driver.py -o output + -i ion-java,${{ github.event.pull_request.head.repo.html_url }},${{ github.event.pull_request.head.sha }} + + - name: Upload result + uses: actions/upload-artifact@v2 + with: + name: ion-test-driver-result.ion + path: output/results/ion-test-driver-results.ion + + - name: showing result + run: cat output/results/ion-test-driver-results.ion + + - name: Analyze two implementations + continue-on-error: true + id: result-diff + run: python3 ion-test-driver/amazon/iontest/ion_test_driver.py -R + ion-java,https://github.com/amzn/ion-java,$sha + ion-java,${{ github.event.pull_request.head.repo.html_url }},${{ github.event.pull_request.head.sha }} + output/results/ion-test-driver-results.ion + + - name: Upload analysis report + uses: actions/upload-artifact@v2 + with: + name: analysis-report.ion + path: result.ion + + - name: showing report + run: cat result.ion + + - name: Check if ion-test-driver fails + if: ${{ steps.result-diff.outcome == 'failure' }} + run: echo 'Implementation behavior changed, Refer to the analysis report in the previous step for the reason.' && exit 1 + + open-issue: + runs-on: ubuntu-latest + needs: ion-test-driver + if: ${{ failure() }} + steps: + - uses: actions/checkout@master + - name: Open an issue + uses: JasonEtco/create-an-issue@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_WORKFLOW_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha }} + with: + assignees: ${{ github.event.sender.login }} + From b967c4f0c11cfecf2aa875de18e6e5cbf4c347c3 Mon Sep 17 00:00:00 2001 From: Eric Chen <67451029+cheqianh@users.noreply.github.com> Date: Wed, 4 Nov 2020 11:51:27 -0800 Subject: [PATCH 112/490] Migrated Travis CI to Github Actions. --- .github/workflows/ion-test-driver.yml | 80 +++++++++++++++++++ .github/workflows/main.yml | 106 ++++++++++---------------- .travis.yml | 31 -------- 3 files changed, 119 insertions(+), 98 deletions(-) create mode 100644 .github/workflows/ion-test-driver.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/ion-test-driver.yml b/.github/workflows/ion-test-driver.yml new file mode 100644 index 0000000000..64ec69e087 --- /dev/null +++ b/.github/workflows/ion-test-driver.yml @@ -0,0 +1,80 @@ +name: ion-test-driver + +on: [pull_request] + +jobs: + ion-test-driver: + runs-on: macos-10.15 + steps: + - name: Checkout ion-java + uses: actions/checkout@master + with: + repository: amzn/ion-java + ref: master + path: ion-java + + - name: Get main branch HEAD sha + run: cd ion-java && echo "sha=`git rev-parse --short HEAD`" >> $GITHUB_ENV + + - name: Checkout ion-test-driver + uses: actions/checkout@master + with: + repository: amzn/ion-test-driver + ref: master + path: ion-test-driver + + - name: Set up python3 env + run: python3 -m venv ion-test-driver/venv && . ion-test-driver/venv/bin/activate + + - name: Pip install + run: pip3 install -r ion-test-driver/requirements.txt && pip3 install -e ion-test-driver + + - name: Run ion-test-driver + run: python3 ion-test-driver/amazon/iontest/ion_test_driver.py -o output + -i ion-java,${{ github.event.pull_request.head.repo.html_url }},${{ github.event.pull_request.head.sha }} + + - name: Upload result + uses: actions/upload-artifact@v2 + with: + name: ion-test-driver-result.ion + path: output/results/ion-test-driver-results.ion + + - name: showing result + run: cat output/results/ion-test-driver-results.ion + + - name: Analyze two implementations + continue-on-error: true + id: result-diff + run: python3 ion-test-driver/amazon/iontest/ion_test_driver.py -R + ion-java,https://github.com/amzn/ion-java,$sha + ion-java,${{ github.event.pull_request.head.repo.html_url }},${{ github.event.pull_request.head.sha }} + output/results/ion-test-driver-results.ion + + - name: Upload analysis report + uses: actions/upload-artifact@v2 + with: + name: analysis-report.ion + path: result.ion + + - name: showing report + run: cat result.ion + + - name: Check if ion-test-driver fails + if: ${{ steps.result-diff.outcome == 'failure' }} + run: echo 'Implementation behavior changed, Refer to the analysis report in the previous step for the reason.' && exit 1 + + open-issue: + runs-on: ubuntu-latest + needs: ion-test-driver + if: ${{ failure() }} + steps: + - uses: actions/checkout@master + - name: Open an issue + uses: JasonEtco/create-an-issue@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_WORKFLOW_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha }} + with: + assignees: ${{ github.event.sender.login }} + diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 64ec69e087..4f4d7d7917 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,80 +1,52 @@ -name: ion-test-driver +name: build + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] -on: [pull_request] jobs: - ion-test-driver: - runs-on: macos-10.15 + test: + runs-on: ubuntu-latest + strategy: + matrix: + java: [8, 9, 10, 11] steps: - - name: Checkout ion-java - uses: actions/checkout@master + - uses: actions/checkout@v2 with: - repository: amzn/ion-java - ref: master - path: ion-java - - - name: Get main branch HEAD sha - run: cd ion-java && echo "sha=`git rev-parse --short HEAD`" >> $GITHUB_ENV - - - name: Checkout ion-test-driver - uses: actions/checkout@master + submodules: recursive + - name: Use java ${{ matrix.java }} + uses: actions/setup-java@v1 with: - repository: amzn/ion-test-driver - ref: master - path: ion-test-driver - - - name: Set up python3 env - run: python3 -m venv ion-test-driver/venv && . ion-test-driver/venv/bin/activate + java-version: ${{ matrix.java }} + - run: mvn test + - run: mvn package -f ion-java-cli/pom.xml + - run: java -jar ion-java-cli/target/ion-java-cli-1.0.jar version - - name: Pip install - run: pip3 install -r ion-test-driver/requirements.txt && pip3 install -e ion-test-driver - - - name: Run ion-test-driver - run: python3 ion-test-driver/amazon/iontest/ion_test_driver.py -o output - -i ion-java,${{ github.event.pull_request.head.repo.html_url }},${{ github.event.pull_request.head.sha }} - - - name: Upload result - uses: actions/upload-artifact@v2 + report-generation: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 with: - name: ion-test-driver-result.ion - path: output/results/ion-test-driver-results.ion - - - name: showing result - run: cat output/results/ion-test-driver-results.ion - - - name: Analyze two implementations - continue-on-error: true - id: result-diff - run: python3 ion-test-driver/amazon/iontest/ion_test_driver.py -R - ion-java,https://github.com/amzn/ion-java,$sha - ion-java,${{ github.event.pull_request.head.repo.html_url }},${{ github.event.pull_request.head.sha }} - output/results/ion-test-driver-results.ion - - - name: Upload analysis report - uses: actions/upload-artifact@v2 + submodules: recursive + - name: Use java 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - run: mvn test site + - uses: codecov/codecov-action@v1 with: - name: analysis-report.ion - path: result.ion + file: target/site/jacoco/jacoco.xml + - name: deploy gh-pages + if: ${{ github.ref == 'refs/heads/master' && github.event_name == 'push' }} + uses: JamesIves/github-pages-deploy-action@3.7.1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH: gh-pages + FOLDER: "./target/site" - - name: showing report - run: cat result.ion - - name: Check if ion-test-driver fails - if: ${{ steps.result-diff.outcome == 'failure' }} - run: echo 'Implementation behavior changed, Refer to the analysis report in the previous step for the reason.' && exit 1 - open-issue: - runs-on: ubuntu-latest - needs: ion-test-driver - if: ${{ failure() }} - steps: - - uses: actions/checkout@master - - name: Open an issue - uses: JasonEtco/create-an-issue@v2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_WORKFLOW_URL: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha }} - with: - assignees: ${{ github.event.sender.login }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 8be4b80dfb..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,31 +0,0 @@ -sudo: false -language: java -dist: trusty -jdk: - - openjdk8 - - openjdk9 - - openjdk10 - - openjdk11 - - oraclejdk8 - - oraclejdk9 - -script: - - mvn test - - mvn package -f ion-java-cli/pom.xml - - java -jar ion-java-cli/target/ion-java-cli-1.0.jar version - -jobs: - include: - stage: report generation - jdk: openjdk11 - script: mvn test site - after_success: - - bash <(curl -s https://codecov.io/bash) - deploy: - provider: pages - local-dir: "./target/site/" - skip-cleanup: true - github-token: "$GITHUB_TOKEN" - keep-history: true # keeps commit history of gh-pages branch - on: - branch: master From b6fd73b73494c44a4d15e1f5eebaf188b1ff29fc Mon Sep 17 00:00:00 2001 From: Eric Chen <67451029+cheqianh@users.noreply.github.com> Date: Thu, 5 Nov 2020 15:19:20 -0800 Subject: [PATCH 113/490] Fixed async main branch change issue for ion-test-driver GitHub Actions. --- .github/workflows/ion-test-driver.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ion-test-driver.yml b/.github/workflows/ion-test-driver.yml index 64ec69e087..e45619c3c4 100644 --- a/.github/workflows/ion-test-driver.yml +++ b/.github/workflows/ion-test-driver.yml @@ -13,9 +13,6 @@ jobs: ref: master path: ion-java - - name: Get main branch HEAD sha - run: cd ion-java && echo "sha=`git rev-parse --short HEAD`" >> $GITHUB_ENV - - name: Checkout ion-test-driver uses: actions/checkout@master with: @@ -29,6 +26,9 @@ jobs: - name: Pip install run: pip3 install -r ion-test-driver/requirements.txt && pip3 install -e ion-test-driver + - name: Get main branch HEAD sha + run: cd ion-java && echo `git rev-parse --short HEAD` && echo "sha=`git rev-parse --short HEAD`" >> $GITHUB_ENV + - name: Run ion-test-driver run: python3 ion-test-driver/amazon/iontest/ion_test_driver.py -o output -i ion-java,${{ github.event.pull_request.head.repo.html_url }},${{ github.event.pull_request.head.sha }} From 44e476fd9eef8b52d899f54011f857907c1ce567 Mon Sep 17 00:00:00 2001 From: Eric Chen <67451029+cheqianh@users.noreply.github.com> Date: Fri, 6 Nov 2020 16:41:05 -0800 Subject: [PATCH 114/490] Prepare new release 1.8.0 --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b8b6a280df..f218f99764 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ dependency into your project's `pom.xml`: com.amazon.ion ion-java - 1.7.1 + 1.8.0 ``` diff --git a/pom.xml b/pom.xml index 1e9097b005..c6731ce47e 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.7.2-SNAPSHOT + 1.8.0 bundle ${project.groupId}:${project.artifactId} From 8742dc8c417ec0911eed8c34870a369b2a4fcc44 Mon Sep 17 00:00:00 2001 From: Eric Chen <67451029+cheqianh@users.noreply.github.com> Date: Mon, 9 Nov 2020 18:09:42 -0800 Subject: [PATCH 115/490] Prepare 1.8.1-SNAPSHOT. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c6731ce47e..00f65d9701 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.8.0 + 1.8.1-SNAPSHOT bundle ${project.groupId}:${project.artifactId} From f218c9534b70edd8956c0004ef9bb290ce78dac0 Mon Sep 17 00:00:00 2001 From: Eric Chen <67451029+cheqianh@users.noreply.github.com> Date: Thu, 19 Nov 2020 11:51:46 -0800 Subject: [PATCH 116/490] Removed issue template. --- .github/{ISSUE_TEMPLATE.md => ion-test-driver-issue.md} | 0 .github/workflows/ion-test-driver.yml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename .github/{ISSUE_TEMPLATE.md => ion-test-driver-issue.md} (100%) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ion-test-driver-issue.md similarity index 100% rename from .github/ISSUE_TEMPLATE.md rename to .github/ion-test-driver-issue.md diff --git a/.github/workflows/ion-test-driver.yml b/.github/workflows/ion-test-driver.yml index e45619c3c4..0690c6db30 100644 --- a/.github/workflows/ion-test-driver.yml +++ b/.github/workflows/ion-test-driver.yml @@ -77,4 +77,4 @@ jobs: GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha }} with: assignees: ${{ github.event.sender.login }} - + filename: .github/ion-test-driver-issue.md From d0656e8856bfb1c0695eecebae03141ae10e3990 Mon Sep 17 00:00:00 2001 From: Eric Chen <67451029+cheqianh@users.noreply.github.com> Date: Mon, 30 Nov 2020 16:11:51 -0800 Subject: [PATCH 117/490] Matched Ion-test-driver spec (Ion-test-driver issue #15). --- .github/workflows/ion-test-driver.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ion-test-driver.yml b/.github/workflows/ion-test-driver.yml index 0690c6db30..206e8a3d7c 100644 --- a/.github/workflows/ion-test-driver.yml +++ b/.github/workflows/ion-test-driver.yml @@ -27,7 +27,11 @@ jobs: run: pip3 install -r ion-test-driver/requirements.txt && pip3 install -e ion-test-driver - name: Get main branch HEAD sha - run: cd ion-java && echo `git rev-parse --short HEAD` && echo "sha=`git rev-parse --short HEAD`" >> $GITHUB_ENV + run: cd ion-java && echo `git rev-parse --short=7 HEAD` && echo "main=`git rev-parse --short=7 HEAD`" >> $GITHUB_ENV + + - name: Get current commit sha + run: cd ion-java && echo `git rev-parse --short=7 ${{ github.event.pull_request.head.sha }}` + && echo "cur=`git rev-parse --short=7 ${{ github.event.pull_request.head.sha }}`" >> $GITHUB_ENV - name: Run ion-test-driver run: python3 ion-test-driver/amazon/iontest/ion_test_driver.py -o output @@ -46,9 +50,7 @@ jobs: continue-on-error: true id: result-diff run: python3 ion-test-driver/amazon/iontest/ion_test_driver.py -R - ion-java,https://github.com/amzn/ion-java,$sha - ion-java,${{ github.event.pull_request.head.repo.html_url }},${{ github.event.pull_request.head.sha }} - output/results/ion-test-driver-results.ion + ion-java,$main ion-java,$cur output/results/ion-test-driver-results.ion - name: Upload analysis report uses: actions/upload-artifact@v2 From 30536fbee0cf93d7b21c1702cf4ba6df6a58965e Mon Sep 17 00:00:00 2001 From: Eric Chen <67451029+cheqianh@users.noreply.github.com> Date: Wed, 2 Dec 2020 17:45:04 -0800 Subject: [PATCH 118/490] Using --replace option in GH Actions to avoid ion-test-driver async issue. --- .github/workflows/ion-test-driver.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ion-test-driver.yml b/.github/workflows/ion-test-driver.yml index 206e8a3d7c..e39846aed3 100644 --- a/.github/workflows/ion-test-driver.yml +++ b/.github/workflows/ion-test-driver.yml @@ -36,6 +36,7 @@ jobs: - name: Run ion-test-driver run: python3 ion-test-driver/amazon/iontest/ion_test_driver.py -o output -i ion-java,${{ github.event.pull_request.head.repo.html_url }},${{ github.event.pull_request.head.sha }} + --replace ion-java,https://github.com/amzn/ion-java.git,$main - name: Upload result uses: actions/upload-artifact@v2 From 34ddd09fc92db6f203b9748fd5fec16e227bb6a9 Mon Sep 17 00:00:00 2001 From: Eric Chen <67451029+cheqianh@users.noreply.github.com> Date: Fri, 26 Mar 2021 14:27:27 -0700 Subject: [PATCH 119/490] Catches exception of unknown SIDs that bigger than INT_MAX. (#349) --- src/com/amazon/ion/impl/IonTokenConstsX.java | 7 ++++++- test/com/amazon/ion/LoaderTest.java | 11 +++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/com/amazon/ion/impl/IonTokenConstsX.java b/src/com/amazon/ion/impl/IonTokenConstsX.java index c361606636..33a8e8e365 100644 --- a/src/com/amazon/ion/impl/IonTokenConstsX.java +++ b/src/com/amazon/ion/impl/IonTokenConstsX.java @@ -513,7 +513,12 @@ public static int decodeSid(CharSequence sidToken) assert length > 1; String digits = sidToken.subSequence(1, length).toString(); - return Integer.parseInt(digits); + + try { + return Integer.parseInt(digits); + } catch (Exception e) { + throw new IonException(String.format("Unable to parse SID %s", digits), e); + } } static public int keyword(CharSequence word, int start_word, int end_word) diff --git a/test/com/amazon/ion/LoaderTest.java b/test/com/amazon/ion/LoaderTest.java index fc020077a4..ff936a6a04 100644 --- a/test/com/amazon/ion/LoaderTest.java +++ b/test/com/amazon/ion/LoaderTest.java @@ -160,6 +160,17 @@ public void testIgnoreHeaderSymbol() checkInt(123, value); } + @Test + public void testLargeSidSymbol() + { + String text = "$11111111111111"; + + // it's supposed to throw an IonException since sid is greater than INT_MAX. + try { + IonSymbol value = (IonSymbol) loadOneValue(text); + Assert.fail("Expected IonException to be thrown."); + } catch (IonException ignore) { /* expected to reach here */ } + } private static class FailingInputStream extends InputStream { From 858a566ac9f2212b99237138e6b8b00e1dc87350 Mon Sep 17 00:00:00 2001 From: Matthew Pope <81593196+popematt@users.noreply.github.com> Date: Fri, 2 Apr 2021 17:05:50 -0700 Subject: [PATCH 120/490] =?UTF-8?q?Add=20options=20to=20configure=20whethe?= =?UTF-8?q?r=20each=20top=20level=20value=20should=20be=20on=20a=20?= =?UTF-8?q?=E2=80=A6=20(#351)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add options to configure whether each top level value should be on a new line and what line separator should be used --- .../amazon/ion/impl/IonWriterSystemText.java | 36 +++-- .../impl/_Private_IonTextWriterBuilder.java | 15 +- .../ion/system/IonTextWriterBuilder.java | 137 ++++++++++++++++++ src/com/amazon/ion/util/IonTextUtils.java | 21 +++ test/com/amazon/ion/impl/TextWriterTest.java | 94 ++++++++++++ 5 files changed, 279 insertions(+), 24 deletions(-) diff --git a/src/com/amazon/ion/impl/IonWriterSystemText.java b/src/com/amazon/ion/impl/IonWriterSystemText.java index 0e78e95ccd..0cbb85942a 100644 --- a/src/com/amazon/ion/impl/IonWriterSystemText.java +++ b/src/com/amazon/ion/impl/IonWriterSystemText.java @@ -66,7 +66,7 @@ class IonWriterSystemText */ private boolean _following_long_string; - int _separator_character; + CharSequence _separator_character; int _top; int [] _stack_parent_type = new int[10]; @@ -89,12 +89,7 @@ protected IonWriterSystemText(SymbolTable defaultSystemSymtab, options.getCharset()); _options = options; - if (_options.isPrettyPrintOn()) { - _separator_character = '\n'; - } - else { - _separator_character = ' '; - } + _separator_character = _options.topLevelSeparator(); int threshold = _options.getLongStringThreshold(); if (threshold < 1) threshold = Integer.MAX_VALUE; @@ -151,14 +146,14 @@ void push(int typeid) _stack_pending_comma[_top] = _pending_separator; switch (typeid) { case _Private_IonConstants.tidSexp: - _separator_character = ' '; + _separator_character = " "; break; case _Private_IonConstants.tidList: case _Private_IonConstants.tidStruct: - _separator_character = ','; + _separator_character = ","; break; default: - _separator_character = _options.isPrettyPrintOn() ? '\n' : ' '; + _separator_character = _options.lineSeparator(); break; } _top++; @@ -182,21 +177,24 @@ int pop() { int parentid = (_top > 0) ? _stack_parent_type[_top - 1] : -1; switch (parentid) { case -1: + _in_struct = false; + _separator_character = _options.topLevelSeparator(); + break; case _Private_IonConstants.tidSexp: _in_struct = false; - _separator_character = ' '; + _separator_character = " "; break; case _Private_IonConstants.tidList: _in_struct = false; - _separator_character = ','; + _separator_character = ","; break; case _Private_IonConstants.tidStruct: _in_struct = true; - _separator_character = ','; + _separator_character = ","; break; default: - _separator_character = _options.isPrettyPrintOn() ? '\n' : ' '; - break; + _separator_character = _options.lineSeparator(); + break; } return typeid; @@ -337,17 +335,17 @@ boolean writeSeparator(boolean followingLongString) throws IOException { if (_options.isPrettyPrintOn()) { - if (_pending_separator && _separator_character > ' ') { + if (_pending_separator && !IonTextUtils.isAllWhitespace(_separator_character)) { // Only bother if the separator is non-whitespace. - _output.appendAscii((char)_separator_character); + _output.appendAscii(_separator_character); followingLongString = false; } _output.appendAscii(_options.lineSeparator()); printLeadingWhiteSpace(); } else if (_pending_separator) { - _output.appendAscii((char)_separator_character); - if (_separator_character > ' ') followingLongString = false; + _output.appendAscii(_separator_character); + if (!IonTextUtils.isAllWhitespace(_separator_character)) followingLongString = false; } return followingLongString; } diff --git a/src/com/amazon/ion/impl/_Private_IonTextWriterBuilder.java b/src/com/amazon/ion/impl/_Private_IonTextWriterBuilder.java index 5c94c62556..971aeb5bc5 100644 --- a/src/com/amazon/ion/impl/_Private_IonTextWriterBuilder.java +++ b/src/com/amazon/ion/impl/_Private_IonTextWriterBuilder.java @@ -34,10 +34,6 @@ public class _Private_IonTextWriterBuilder extends IonTextWriterBuilder { private final static CharSequence SPACE_CHARACTER = " "; - // TODO amzn/ion-java/issues/57 decide if this should be platform-specific - private final static CharSequence LINE_SEPARATOR = - System.getProperty("line.separator"); - public static _Private_IonTextWriterBuilder standard() { @@ -148,13 +144,17 @@ final boolean isPrettyPrintOn() final CharSequence lineSeparator() { if (_pretty_print) { - return LINE_SEPARATOR; + return getNewLineType().getCharSequence(); } else { return SPACE_CHARACTER; } } + final CharSequence topLevelSeparator() + { + return getWriteTopLevelValuesOnNewLines() ? getNewLineType().getCharSequence() : lineSeparator(); + } //========================================================================= @@ -173,6 +173,11 @@ private _Private_IonTextWriterBuilder fillDefaults() b.setCharset(UTF8); } + if (b.getNewLineType() == null) + { + b.setNewLineType(NewLineType.PLATFORM_DEPENDENT); + } + return (_Private_IonTextWriterBuilder) b.immutable(); } diff --git a/src/com/amazon/ion/system/IonTextWriterBuilder.java b/src/com/amazon/ion/system/IonTextWriterBuilder.java index daa90ff689..3f201c6137 100644 --- a/src/com/amazon/ion/system/IonTextWriterBuilder.java +++ b/src/com/amazon/ion/system/IonTextWriterBuilder.java @@ -113,6 +113,38 @@ public enum LstMinimizing */ public static final Charset UTF8 = _Private_Utils.UTF8_CHARSET; + /** + * Represents common new-line separators that are valid in Ion. + * + * Unicode defines several characters that can represent a new line, but only carriage return (CR) and linefeed (LF) are valid whitespace in Ion. + * Using other new-line characters (such as {@code NEL}) will result in printing invalid Ion. + */ + public enum NewLineType { + /** + * A carriage return and linefeed ({@code U+000D} followed by {@code U+000A}). + */ + CRLF("\r\n"), + /** + * A single linefeed ({@code U+000A}). + */ + LF("\n"), + /** + * The new-line separator specified in the "line.separator" system property. + * Using this will result in writing invalid Ion if the platform uses anything other than CR and/or LF as the line separator + * and the IonTextWriter is configured to use pretty printing or to write each top-level value on a separate line. + */ + PLATFORM_DEPENDENT(System.getProperty("line.separator")); + + private final CharSequence charSequence; + + NewLineType(CharSequence cs) { + this.charSequence = cs; + } + + public CharSequence getCharSequence() { + return charSequence; + } + } /** * The standard builder of text {@link IonWriter}s, with all configuration @@ -187,6 +219,8 @@ public static IonTextWriterBuilder json() private IvmMinimizing myIvmMinimizing; private LstMinimizing myLstMinimizing; private int myLongStringThreshold; + private NewLineType myNewLineType; + private boolean myTopLevelValuesOnNewLines; /** NOT FOR APPLICATION USE! */ @@ -204,6 +238,8 @@ protected IonTextWriterBuilder(IonTextWriterBuilder that) this.myIvmMinimizing = that.myIvmMinimizing; this.myLstMinimizing = that.myLstMinimizing; this.myLongStringThreshold = that.myLongStringThreshold; + this.myNewLineType = that.myNewLineType; + this.myTopLevelValuesOnNewLines = that.myTopLevelValuesOnNewLines; } @@ -626,6 +662,107 @@ public final IonTextWriterBuilder withLongStringThreshold(int threshold) //========================================================================= + /** + * Gets the character sequence that will be written as a line separator. + * The default is {@link NewLineType#PLATFORM_DEPENDENT} + * + * @return the character sequence to be written between top-level values; null means the default should be used. + * + * @see #setNewLineType(NewLineType) + * @see #withNewLineType(NewLineType) + */ + public final NewLineType getNewLineType() + { + return myNewLineType; + } + + /** + * Sets the character sequence that will be written as a line separator. + * The default is {@link NewLineType#PLATFORM_DEPENDENT} + * + * @param newLineType the character sequence to be written between top-level values; null means the default should be used. + * + * @see #getNewLineType() + * @see #withNewLineType(NewLineType) + * + * @throws UnsupportedOperationException if this is immutable. + */ + public void setNewLineType(NewLineType newLineType) + { + mutationCheck(); + this.myNewLineType = newLineType; + } + + /** + * Declares the character sequence that will be written as a line separator. + * The default is {@link NewLineType#PLATFORM_DEPENDENT} + * + * @param newLineType the character sequence to be written between top-level values; null means the default should be used. + * + * @see #getNewLineType() + * @see #setNewLineType(NewLineType) + * + * @return this instance, if mutable; + * otherwise a mutable copy of this instance. + */ + public final IonTextWriterBuilder withNewLineType(NewLineType newLineType) + { + IonTextWriterBuilder b = mutable(); + b.setNewLineType(newLineType); + return b; + } + + //========================================================================= + + /** + * Gets whether each top level value for standard printing should start on a new line. The default value is {@code false}. + * When false, the IonTextWriter will insert a single space character (U+0020) between top-level values. + * When pretty-printing, this setting is ignored; the pretty printer will always start top-level values on a new line. + * + * @return value indicating whether standard printing will insert a newline between top-level values + * + * @see #setWriteTopLevelValuesOnNewLines(boolean) + * @see #withWriteTopLevelValuesOnNewLines(boolean) + */ + public final boolean getWriteTopLevelValuesOnNewLines() + { + return myTopLevelValuesOnNewLines; + } + + /** + * Sets whether each top level value for standard printing should start on a new line. The default value is {@code false}. + * When false, the IonTextWriter will insert a single space character (U+0020) between top-level values. + * When pretty-printing, this setting is ignored; the pretty printer will always start top-level values on a new line. + * + * @param writeTopLevelValuesOnNewLines value indicating whether standard printing will insert a newline between top-level values + * + * @see #getWriteTopLevelValuesOnNewLines() + * @see #withWriteTopLevelValuesOnNewLines(boolean) + */ + public void setWriteTopLevelValuesOnNewLines(boolean writeTopLevelValuesOnNewLines) + { + mutationCheck(); + myTopLevelValuesOnNewLines = writeTopLevelValuesOnNewLines; + } + + /** + * Declares whether each top level value for standard printing should start on a new line. The default value is {@code false}. + * When false, the IonTextWriter will insert a single space character (U+0020) between top-level values. + * When pretty-printing, this setting is ignored; the pretty printer will always start top-level values on a new line. + * + * @param writeTopLevelValuesOnNewLines value indicating whether standard printing will insert a newline between top-level values + * + * @see #getWriteTopLevelValuesOnNewLines() + * @see #setWriteTopLevelValuesOnNewLines(boolean) + */ + public final IonTextWriterBuilder withWriteTopLevelValuesOnNewLines(boolean writeTopLevelValuesOnNewLines) + { + IonTextWriterBuilder b = mutable(); + b.setWriteTopLevelValuesOnNewLines(writeTopLevelValuesOnNewLines); + return b; + } + + //========================================================================= /** * Creates a new writer that will write text to the given output diff --git a/src/com/amazon/ion/util/IonTextUtils.java b/src/com/amazon/ion/util/IonTextUtils.java index 47e0d17326..afc997db7e 100644 --- a/src/com/amazon/ion/util/IonTextUtils.java +++ b/src/com/amazon/ion/util/IonTextUtils.java @@ -67,6 +67,27 @@ public static boolean isWhitespace(int codePoint) } } + /** + * Ion whitespace is defined as one of the characters space, tab, newline, + * and carriage-return. This matches the definition of whitespace used by + * JSON. + * + * @param charSequence the CharSequence to test. + * @return {@code true} if {@code charSequence} consists entirely of the four legal + * Ion whitespace characters. + * + * @see #isWhitespace(int) + * @see RFC 4627 + */ + public static boolean isAllWhitespace(CharSequence charSequence) { + for (int i = 0; i < charSequence.length(); i++) { + if (!IonTextUtils.isWhitespace(Character.codePointAt(charSequence, i))) { + return false; + } + } + return true; + } + /** * Determines whether a given code point is one of the valid Ion numeric * terminators. diff --git a/test/com/amazon/ion/impl/TextWriterTest.java b/test/com/amazon/ion/impl/TextWriterTest.java index ba57d634bf..1b3445e423 100644 --- a/test/com/amazon/ion/impl/TextWriterTest.java +++ b/test/com/amazon/ion/impl/TextWriterTest.java @@ -334,6 +334,100 @@ public void testJsonSystemMinimization() assertEquals("\"fred_1\" \"fred_1\"", outputString()); } + @Test + public void testWritingTopLevelValuesOnNewLinesWithoutPrettyPrint() + { + IonTextWriterBuilder writerBuilder = IonTextWriterBuilder.standard() + .withInitialIvmHandling(SUPPRESS) + .withWriteTopLevelValuesOnNewLines(true) + .withNewLineType(IonTextWriterBuilder.NewLineType.LF); + + IonDatagram dg = system().newDatagram(); + dg.add().newString("Foo"); + dg.add().newSymbol("Bar"); + dg.add().newSexp(new int[]{1, 2, 3}); + dg.add().newList(new int[]{4, 5, 6}); + IonStruct struct = dg.add().newEmptyStruct(); + struct.add("def").newInt(42); + struct.addTypeAnnotation("abc"); + + StringBuilder sb = new StringBuilder(); + IonWriter writer = writerBuilder.build(sb); + dg.writeTo(writer); + assertEquals("\"Foo\"\nBar\n(1 2 3)\n[4,5,6]\nabc::{def:42}", sb.toString()); + } + + @Test + public void testWritingTopLevelValuesOnNewLinesShouldHaveNoEffectWithPrettyPrint() + { + // Setting top-level newlines to false should have no effect when pretty printing. + IonTextWriterBuilder writerBuilder = IonTextWriterBuilder.standard() + .withInitialIvmHandling(SUPPRESS) + .withWriteTopLevelValuesOnNewLines(false) + .withPrettyPrinting() + .withNewLineType(IonTextWriterBuilder.NewLineType.LF); + + IonDatagram dg = system().newDatagram(); + dg.add().newString("Foo"); + dg.add().newSymbol("Bar"); + dg.add().newSexp(new int[]{1, 2, 3}); + dg.add().newList(new int[]{4, 5, 6}); + + StringBuilder sb = new StringBuilder(); + IonWriter writer = writerBuilder.build(sb); + dg.writeTo(writer); + assertEquals("\n\"Foo\"\nBar\n(\n 1\n 2\n 3\n)\n[\n 4,\n 5,\n 6\n]", sb.toString()); + } + + @Test + public void testNewLineTypesWithPrettyPrinting() + { + for (IonTextWriterBuilder.NewLineType nlt : IonTextWriterBuilder.NewLineType.values()) { + String expected = String.format("%s\"Foo\"% Date: Mon, 5 Apr 2021 16:09:21 -0700 Subject: [PATCH 121/490] preparing release 1.8.1 --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f218f99764..82eb2b9ad0 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ dependency into your project's `pom.xml`: com.amazon.ion ion-java - 1.8.0 + 1.8.1 ``` diff --git a/pom.xml b/pom.xml index 00f65d9701..89b23c719a 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.8.1-SNAPSHOT + 1.8.1 bundle ${project.groupId}:${project.artifactId} From 0d06a61dd2904bdb0ba1427b0007193d9f0f41cc Mon Sep 17 00:00:00 2001 From: Matthew Pope <81593196+popematt@users.noreply.github.com> Date: Wed, 7 Apr 2021 12:46:56 -0700 Subject: [PATCH 122/490] Bumps version to 1.8.2-SNAPSHOT (#353) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 89b23c719a..9a7eb74b6b 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.8.1 + 1.8.2-SNAPSHOT bundle ${project.groupId}:${project.artifactId} From 0eb62ba92633f03e9f3175bb77f1b3214fcc1cdd Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Wed, 7 Apr 2021 14:45:24 -1000 Subject: [PATCH 123/490] Updates the ion-tests submodule to c125f57. --- ion-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ion-tests b/ion-tests index 05ecd5de78..c125f571d2 160000 --- a/ion-tests +++ b/ion-tests @@ -1 +1 @@ -Subproject commit 05ecd5de78ac6244826cd4e77bae932925c966c1 +Subproject commit c125f571d2b459e0e610eac5ca11900bd3b6d493 From abd3d7f4c0f8ddd940f8095f6290dc767546ef6a Mon Sep 17 00:00:00 2001 From: Eric Chen <67451029+cheqianh@users.noreply.github.com> Date: Fri, 30 Apr 2021 12:58:36 -0700 Subject: [PATCH 124/490] Fixes unexpected assertion error. --- src/com/amazon/ion/impl/IonUTF8.java | 3 +++ test/com/amazon/ion/IonExceptionTest.java | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/com/amazon/ion/impl/IonUTF8.java b/src/com/amazon/ion/impl/IonUTF8.java index 4b1c718bc0..f0980b78c2 100644 --- a/src/com/amazon/ion/impl/IonUTF8.java +++ b/src/com/amazon/ion/impl/IonUTF8.java @@ -378,6 +378,9 @@ public final static int getScalarFromBytes(byte[] bytes, int offset, int maxLeng } public final static boolean needsSurrogateEncoding(int unicodeScalar) { + if (unicodeScalar > Character.MAX_CODE_POINT) { + throw new IonException("Invalid encoding: encountered non-Unicode character."); + } return (unicodeScalar > MAXIMUM_UTF16_1_CHAR_CODE_POINT); } public final static char highSurrogate(int unicodeScalar) { diff --git a/test/com/amazon/ion/IonExceptionTest.java b/test/com/amazon/ion/IonExceptionTest.java index dd7c18b089..52c40caa12 100644 --- a/test/com/amazon/ion/IonExceptionTest.java +++ b/test/com/amazon/ion/IonExceptionTest.java @@ -15,11 +15,15 @@ package com.amazon.ion; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; +import static org.junit.Assert.fail; import java.io.FileNotFoundException; import java.io.IOException; + +import com.amazon.ion.system.IonSystemBuilder; import org.junit.Test; @@ -100,4 +104,20 @@ public void testCauseOfTypeWithCycle() IonException ion = new IonException(ie1); assertNull(ion.causeOfType(IOException.class)); } + + @Test + public void testCauseOfWrongEncoding() { + try { + byte[] bytes_input = new byte[]{ + (byte) 0x27, (byte) 0x31, (byte) -0xB, (byte) 0x31, (byte) 0x31, (byte) 0x31, (byte) 0x27}; + + IonSystemBuilder.standard().build().newLoader().load(bytes_input); + } catch (IonException e) { + // The exception should be caught here + assertEquals(e.getMessage(), "Invalid encoding: encountered non-Unicode character."); + return; + } catch (Exception ignore) {} + + fail(); + } } From 45234885de653eae67db4d8d3ccb6bb85bb82fd9 Mon Sep 17 00:00:00 2001 From: Zack Slayton Date: Wed, 19 May 2021 19:17:19 -0400 Subject: [PATCH 125/490] Adds IonSystem field to IonContainerLite (#361) All IonValueLite subclasses implement `getSystem()` by delegating that call to their internal `_context` field, which represents the parent value. The context will then delegate to its own context, and this process repeats up the tree until the top level is found. In deeply nested data, this linear search becomes especially expensive. The `getSystem()` method is widely used. Among other examples, it is called several times per add() operation on a container. This commit modifies IonContainerLite (the abstract base class for IonStructLite and IonSequenceLite) by adding an IonSystem field that is populated at instantiation time. Storing this reference guarantees that calls to `getSystem()` can always be resolved by either the current value or its immediate parent container. Fixes #360. Co-authored-by: Zack Slayton --- .../amazon/ion/impl/lite/IonContainerLite.java | 15 +++++++++++++++ src/com/amazon/ion/impl/lite/IonStructLite.java | 4 ++-- src/com/amazon/ion/impl/lite/IonValueLite.java | 4 ++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/com/amazon/ion/impl/lite/IonContainerLite.java b/src/com/amazon/ion/impl/lite/IonContainerLite.java index 4efdd23dbb..e0276329a7 100644 --- a/src/com/amazon/ion/impl/lite/IonContainerLite.java +++ b/src/com/amazon/ion/impl/lite/IonContainerLite.java @@ -40,16 +40,25 @@ abstract class IonContainerLite protected int _child_count; protected IonValueLite[] _children; protected int structuralModificationCount; + // IonValueLite delegates calls to getSystem() to the parent context. If the value is deeply nested, the + // the parent will delegate the call to getSystem() to its own parent context. This delegation repeats until + // the top level is reached. This linear walk to the root of the DOM can be very expensive. + // As an optimization, each container holds a reference to its parent's IonSystem and overrides IonValueLite's + // implementation of getSystem(). Scalar IonValueLite implementations will continue to delegate to the parent + // context, but the parent context will always be able to provide the IonSystem without further delegation. + protected IonSystemLite ionSystem; protected IonContainerLite(ContainerlessContext context, boolean isNull) { // we'll let IonValueLite handle this work as we always need to know // our context and if we should start out as a null value or not super(context, isNull); + this.ionSystem = context.getSystem(); } IonContainerLite(IonContainerLite existing, IonContext context, boolean isStruct) { super(existing, context); + this.ionSystem = existing.getSystem(); boolean retainingSIDs = false; int childCount = existing._child_count; this._child_count = childCount; @@ -88,6 +97,12 @@ protected IonContainerLite(ContainerlessContext context, boolean isNull) } } + // See the comment on the `ionSystem` member field for more information. + @Override + public IonSystemLite getSystem() { + return ionSystem; + } + @Override public abstract void accept(ValueVisitor visitor) throws Exception; diff --git a/src/com/amazon/ion/impl/lite/IonStructLite.java b/src/com/amazon/ion/impl/lite/IonStructLite.java index 2f0afd334c..e8864163ba 100644 --- a/src/com/amazon/ion/impl/lite/IonStructLite.java +++ b/src/com/amazon/ion/impl/lite/IonStructLite.java @@ -460,7 +460,7 @@ public boolean add(IonValue child) public ValueFactory add(final String fieldName) { - return new _Private_CurriedValueFactory(_context.getSystem()) + return new _Private_CurriedValueFactory(getSystem()) { @Override protected void handle(IonValue newValue) @@ -531,7 +531,7 @@ public void add(SymbolToken fieldName, IonValue child) public ValueFactory put(final String fieldName) { - return new _Private_CurriedValueFactory(_context.getSystem()) + return new _Private_CurriedValueFactory(getSystem()) { @Override protected void handle(IonValue newValue) diff --git a/src/com/amazon/ion/impl/lite/IonValueLite.java b/src/com/amazon/ion/impl/lite/IonValueLite.java index 8c58a0dd7b..598c4a0a6a 100644 --- a/src/com/amazon/ion/impl/lite/IonValueLite.java +++ b/src/com/amazon/ion/impl/lite/IonValueLite.java @@ -633,7 +633,7 @@ public SymbolTable getSymbolTable() if (symbols != null) { return symbols; } - return _context.getSystem().getSystemSymbolTable(); + return getSystem().getSystemSymbolTable(); } public SymbolTable getAssignedSymbolTable() @@ -651,7 +651,7 @@ public IonSystemLite getSystem() public IonType getType() { - throw new UnsupportedOperationException("this type "+this.getClass().getSimpleName()+" should not be instanciated, there is not IonType associated with it"); + throw new UnsupportedOperationException("this type "+this.getClass().getSimpleName()+" should not be instantiated, there is not IonType associated with it"); } public SymbolToken[] getTypeAnnotationSymbols() From f4e2e6b7b715a2e46bfa82c7ab047cfdc3de3efc Mon Sep 17 00:00:00 2001 From: Zack Slayton Date: Mon, 24 May 2021 17:06:45 -0400 Subject: [PATCH 126/490] Makes the PooledBlockAllocatorProvider a singleton (#363) Makes the PooledBlockAllocatorProvider a singleton Many applications need to initialize IonWriters repeatedly for short-lived tasks. For example, a webserver might create a new IonWriter for each HTTP response that it needs to write. In such cases, the bulk of the compute time needed to serialize the response data is often spent initializing the byte array buffer backing the IonWriter rather than performing the write logic itself. Each time that a (e.g.) binary IonWriter is initialized, a new PooledBlockAllocatorProvider (PBAP) is initialized with it to dole out new byte array buffers. It attempts to reuse these byte arrays wherever possible during its lifespan. Unfortunately, the PBAP only lives as long as the IonWriter itself. If a new IonWriter is created, a new PBAP (and therefore new byte arrays) are created as well. This change modifies the PooledBlockAllocatorProvider (which was already thread-safe) to be a globally shared singleton. This makes repeatedly initializing new IonWriters substantially cheaper. --- .../impl/bin/PooledBlockAllocatorProvider.java | 15 ++++++++++++--- .../ion/impl/bin/_PrivateIon_HashTrampoline.java | 2 +- .../_Private_IonManagedBinaryWriterBuilder.java | 2 +- .../bin/PooledBlockAllocatorProviderTest.java | 2 +- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/com/amazon/ion/impl/bin/PooledBlockAllocatorProvider.java b/src/com/amazon/ion/impl/bin/PooledBlockAllocatorProvider.java index cbd30ae76a..b3d58e2c5f 100644 --- a/src/com/amazon/ion/impl/bin/PooledBlockAllocatorProvider.java +++ b/src/com/amazon/ion/impl/bin/PooledBlockAllocatorProvider.java @@ -20,8 +20,9 @@ import java.util.concurrent.ConcurrentMap; /** - * A simple pooling implementation of {@link BlockAllocatorProvider} with a global thread-safe free block list + * A singleton implementation of {@link BlockAllocatorProvider} offering a thread-safe free block list * for each block size. + * *

    * This implementation is thread-safe. */ @@ -32,7 +33,7 @@ *

    * This implementation is thread-safe. */ - private final class PooledBlockAllocator extends BlockAllocator + private static final class PooledBlockAllocator extends BlockAllocator { private final int blockSize, blockLimit; private final ConcurrentLinkedQueue freeBlocks; @@ -76,13 +77,21 @@ public int getBlockSize() public void close() {} } + // A globally shared instance of the PooledBlockAllocatorProvider. + // This instance allows BlockAllocators to be re-used across instantiations of classes like + // the binary Ion writer, thereby avoiding costly array initializations. + private static final PooledBlockAllocatorProvider INSTANCE = new PooledBlockAllocatorProvider(); private final ConcurrentMap allocators; - public PooledBlockAllocatorProvider() + private PooledBlockAllocatorProvider() { allocators = new ConcurrentHashMap(); } + public static PooledBlockAllocatorProvider getInstance() { + return INSTANCE; + } + @Override public BlockAllocator vendAllocator(final int blockSize) { diff --git a/src/com/amazon/ion/impl/bin/_PrivateIon_HashTrampoline.java b/src/com/amazon/ion/impl/bin/_PrivateIon_HashTrampoline.java index aea1e31a78..6d3537c1f6 100644 --- a/src/com/amazon/ion/impl/bin/_PrivateIon_HashTrampoline.java +++ b/src/com/amazon/ion/impl/bin/_PrivateIon_HashTrampoline.java @@ -28,7 +28,7 @@ @Deprecated public class _PrivateIon_HashTrampoline { - private static final PooledBlockAllocatorProvider ALLOCATOR_PROVIDER = new PooledBlockAllocatorProvider(); + private static final PooledBlockAllocatorProvider ALLOCATOR_PROVIDER = PooledBlockAllocatorProvider.getInstance(); public static IonWriter newIonWriter(ByteArrayOutputStream baos) throws IOException { diff --git a/src/com/amazon/ion/impl/bin/_Private_IonManagedBinaryWriterBuilder.java b/src/com/amazon/ion/impl/bin/_Private_IonManagedBinaryWriterBuilder.java index b439d160a1..35d87e118a 100644 --- a/src/com/amazon/ion/impl/bin/_Private_IonManagedBinaryWriterBuilder.java +++ b/src/com/amazon/ion/impl/bin/_Private_IonManagedBinaryWriterBuilder.java @@ -52,7 +52,7 @@ public enum AllocatorMode @Override BlockAllocatorProvider createAllocatorProvider() { - return new PooledBlockAllocatorProvider(); + return PooledBlockAllocatorProvider.getInstance(); } }, BASIC diff --git a/test/com/amazon/ion/impl/bin/PooledBlockAllocatorProviderTest.java b/test/com/amazon/ion/impl/bin/PooledBlockAllocatorProviderTest.java index 97ccef5546..807f8d142b 100644 --- a/test/com/amazon/ion/impl/bin/PooledBlockAllocatorProviderTest.java +++ b/test/com/amazon/ion/impl/bin/PooledBlockAllocatorProviderTest.java @@ -30,7 +30,7 @@ public class PooledBlockAllocatorProviderTest @Before public void setup() { - provider = new PooledBlockAllocatorProvider(); + provider = PooledBlockAllocatorProvider.getInstance(); } @After From 7e4baa6d9a5c4dd76273a6d677c7d22e4efdd6d6 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Wed, 26 May 2021 14:46:31 -0700 Subject: [PATCH 127/490] Prepares version 1.8.2 for release. --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 82eb2b9ad0..f6ec39d438 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ dependency into your project's `pom.xml`: com.amazon.ion ion-java - 1.8.1 + 1.8.2 ``` diff --git a/pom.xml b/pom.xml index 9a7eb74b6b..302e454473 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.8.2-SNAPSHOT + 1.8.2 bundle ${project.groupId}:${project.artifactId} From 23a7f457c5f5ff6587d097aa5fedb238a3c110c0 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Wed, 26 May 2021 17:41:00 -0700 Subject: [PATCH 128/490] Prepares 1.8.3-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 302e454473..3858356611 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.8.2 + 1.8.3-SNAPSHOT bundle ${project.groupId}:${project.artifactId} From 64a5b8e00423359e87a7db8d9e0b720f7196cb14 Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Tue, 15 Jun 2021 15:07:56 -0700 Subject: [PATCH 129/490] Fixes handling of NaN and +/-Inf when downconverting to json --- .../amazon/ion/impl/IonWriterSystemText.java | 2 +- .../ion/impl/_Private_IonTextAppender.java | 24 ++++++--- .../impl/_Private_IonTextWriterBuilder.java | 6 +++ src/com/amazon/ion/util/IonTextUtils.java | 4 +- test/com/amazon/ion/impl/TextWriterTest.java | 51 +++++++++++++++++++ 5 files changed, 78 insertions(+), 9 deletions(-) diff --git a/src/com/amazon/ion/impl/IonWriterSystemText.java b/src/com/amazon/ion/impl/IonWriterSystemText.java index 0cbb85942a..4c7b2c2192 100644 --- a/src/com/amazon/ion/impl/IonWriterSystemText.java +++ b/src/com/amazon/ion/impl/IonWriterSystemText.java @@ -604,7 +604,7 @@ public void writeFloat(double value) throws IOException { startValue(); - _output.printFloat(value); + _output.printFloat(_options, value); closeValue(); } diff --git a/src/com/amazon/ion/impl/_Private_IonTextAppender.java b/src/com/amazon/ion/impl/_Private_IonTextAppender.java index 4e6f952615..7a15216691 100644 --- a/src/com/amazon/ion/impl/_Private_IonTextAppender.java +++ b/src/com/amazon/ion/impl/_Private_IonTextAppender.java @@ -798,7 +798,7 @@ else if (adjustedExponent >= -6) } - public void printFloat(double value) + public void printFloat(_Private_IonTextWriterBuilder _options, double value) throws IOException { // shortcut zero cases @@ -815,15 +815,27 @@ public void printFloat(double value) } else if (Double.isNaN(value)) { - appendAscii("nan"); + if (_options._float_nan_and_inf_as_null) { + appendAscii("null"); + } else { + appendAscii("nan"); + } } else if (value == Double.POSITIVE_INFINITY) { - appendAscii("+inf"); + if (_options._float_nan_and_inf_as_null) { + appendAscii("null"); + } else { + appendAscii("+inf"); + } } else if (value == Double.NEGATIVE_INFINITY) { - appendAscii("-inf"); + if (_options._float_nan_and_inf_as_null) { + appendAscii("null"); + } else { + appendAscii("-inf"); + } } else { @@ -846,7 +858,7 @@ else if (value == Double.NEGATIVE_INFINITY) } } - public void printFloat(Double value) + public void printFloat(_Private_IonTextWriterBuilder _options, Double value) throws IOException { if (value == null) @@ -855,7 +867,7 @@ public void printFloat(Double value) } else { - printFloat(value.doubleValue()); + printFloat(_options, value.doubleValue()); } } diff --git a/src/com/amazon/ion/impl/_Private_IonTextWriterBuilder.java b/src/com/amazon/ion/impl/_Private_IonTextWriterBuilder.java index 971aeb5bc5..c1fbaed8de 100644 --- a/src/com/amazon/ion/impl/_Private_IonTextWriterBuilder.java +++ b/src/com/amazon/ion/impl/_Private_IonTextWriterBuilder.java @@ -47,9 +47,12 @@ public static _Private_IonTextWriterBuilder standard() //========================================================================= private boolean _pretty_print; + + // These options control whether the IonTextWriter will write standard ion or ion that is down-converted json. public boolean _blob_as_string; public boolean _clob_as_string; public boolean _decimal_as_float; + public boolean _float_nan_and_inf_as_null; public boolean _sexp_as_list; public boolean _skip_annotations; public boolean _string_as_json; @@ -57,6 +60,7 @@ public static _Private_IonTextWriterBuilder standard() public boolean _timestamp_as_millis; public boolean _timestamp_as_string; public boolean _untyped_nulls; + private _Private_CallbackBuilder _callback_builder; @@ -73,6 +77,7 @@ private _Private_IonTextWriterBuilder(_Private_IonTextWriterBuilder that) this._blob_as_string = that._blob_as_string ; this._clob_as_string = that._clob_as_string ; this._decimal_as_float = that._decimal_as_float ; + this._float_nan_and_inf_as_null = that._float_nan_and_inf_as_null; this._sexp_as_list = that._sexp_as_list ; this._skip_annotations = that._skip_annotations ; this._string_as_json = that._string_as_json ; @@ -123,6 +128,7 @@ public final IonTextWriterBuilder withJsonDowngrade() _clob_as_string = true; // datagramAsList = true; // TODO _decimal_as_float = true; + _float_nan_and_inf_as_null = true; _sexp_as_list = true; _skip_annotations = true; // skipSystemValues = true; // TODO diff --git a/src/com/amazon/ion/util/IonTextUtils.java b/src/com/amazon/ion/util/IonTextUtils.java index afc997db7e..089aa4a89a 100644 --- a/src/com/amazon/ion/util/IonTextUtils.java +++ b/src/com/amazon/ion/util/IonTextUtils.java @@ -876,7 +876,7 @@ public static void printFloat(Appendable out, double value) { _Private_IonTextAppender appender = _Private_IonTextAppender.forAppendable(out); - appender.printFloat(value); + appender.printFloat(STANDARD, value); } public static String printFloat(double value) @@ -900,7 +900,7 @@ public static void printFloat(Appendable out, Double value) { _Private_IonTextAppender appender = _Private_IonTextAppender.forAppendable(out); - appender.printFloat(value); + appender.printFloat(STANDARD, value); } public static String printFloat(Double value) diff --git a/test/com/amazon/ion/impl/TextWriterTest.java b/test/com/amazon/ion/impl/TextWriterTest.java index 1b3445e423..1601c3e79e 100644 --- a/test/com/amazon/ion/impl/TextWriterTest.java +++ b/test/com/amazon/ion/impl/TextWriterTest.java @@ -487,6 +487,57 @@ public void testWritingJsonLongClobs() assertEquals("\"a\\\"'\\nc\\u007f\"", actual); } + @Test + public void testWritingFloatNanToJson() + throws Exception + { + options = IonTextWriterBuilder.json(); + options.setInitialIvmHandling(SUPPRESS); + + IonDatagram dg = system().newDatagram(); + dg.add().newFloat(Double.NaN); + + iw = makeWriter(); + dg.writeTo(iw); + + String actual = outputString(); + assertEquals("null", actual); + } + + @Test + public void testWritingFloatPositiveInfinityToJson() + throws Exception + { + options = IonTextWriterBuilder.json(); + options.setInitialIvmHandling(SUPPRESS); + + IonDatagram dg = system().newDatagram(); + dg.add().newFloat(Double.POSITIVE_INFINITY); + + iw = makeWriter(); + dg.writeTo(iw); + + String actual = outputString(); + assertEquals("null", actual); + } + + @Test + public void testWritingFloatNegativeInfinityToJson() + throws Exception + { + options = IonTextWriterBuilder.json(); + options.setInitialIvmHandling(SUPPRESS); + + IonDatagram dg = system().newDatagram(); + dg.add().newFloat(Double.NEGATIVE_INFINITY); + + iw = makeWriter(); + dg.writeTo(iw); + + String actual = outputString(); + assertEquals("null", actual); + } + @Test public void testSuppressInitialIvm() throws Exception From ca85095f497f19846544acc48a54fd2fb6ac428a Mon Sep 17 00:00:00 2001 From: Zack Slayton Date: Wed, 16 Jun 2021 17:42:24 -0400 Subject: [PATCH 130/490] Makes `writeValueRecursively` iterative (#368) * Makes `writeValueRecursively` iterative The method `_Private_IonWriterBase#writeValueRecursively` is recursive in two senses: 1. It visits all of the children in the provided IonReader's current value recursively. 2. The method itself is implemented using recursion. This patch modifies the implementation (#2) to be iterative, eliminating a source of StackOverflowExceptions and offering a modest reduction in CPU cost. * Tweaked javadoc, removed parameter. * Fixed comment. Co-authored-by: Zack Slayton --- .../amazon/ion/impl/IonWriterUserBinary.java | 2 +- .../ion/impl/_Private_IonWriterBase.java | 199 ++++++++++-------- 2 files changed, 110 insertions(+), 91 deletions(-) diff --git a/src/com/amazon/ion/impl/IonWriterUserBinary.java b/src/com/amazon/ion/impl/IonWriterUserBinary.java index 8d9a7823dd..ae1c072f3b 100644 --- a/src/com/amazon/ion/impl/IonWriterUserBinary.java +++ b/src/com/amazon/ion/impl/IonWriterUserBinary.java @@ -105,7 +105,7 @@ public void writeValue(IonReader reader) // From here on, we won't call back into this method, so we won't // bother doing all those checks again. - writeValueRecursively(type, reader); + writeValueRecursively(reader); } diff --git a/src/com/amazon/ion/impl/_Private_IonWriterBase.java b/src/com/amazon/ion/impl/_Private_IonWriterBase.java index 82802d938a..1aa806ec67 100644 --- a/src/com/amazon/ion/impl/_Private_IonWriterBase.java +++ b/src/com/amazon/ion/impl/_Private_IonWriterBase.java @@ -44,10 +44,6 @@ public abstract class _Private_IonWriterBase static final String ERROR_FINISH_NOT_AT_TOP_LEVEL = "IonWriter.finish() can only be called at top-level."; - private static final boolean _debug_on = false; - - - /** * Returns the current depth of containers the writer is at. This is * 0 if the writer is at top-level. @@ -329,7 +325,6 @@ private final void write_value_field_name_helper(IonReader reader) } setFieldNameSymbol(tok); - if (_debug_on) System.out.print(":"); } } @@ -340,7 +335,6 @@ private final void write_value_annotations_helper(IonReader reader) // because local symtab diversion leaves the $ion_symbol_table // dangling on the system writer! TODO fix that, it's broken. setTypeAnnotationSymbols(a); - if (_debug_on) System.out.print(";"); } @@ -355,101 +349,126 @@ public boolean isStreamCopyOptimized() public void writeValue(IonReader reader) throws IOException { // TODO this should do symtab optimization as per writeValues() - IonType type = reader.getType(); - writeValueRecursively(type, reader); + writeValueRecursively(reader); } /** - * Unoptimized copy. This must not recurse back to the public - * {@link #writeValue(IonReader)} method since that will cause the - * optimization test to happen repeatedly. + * Writes the provided IonReader's current value including any annotations. This function will not advance the + * IonReader beyond the end of the current value; users wishing to continue using the IonReader at the current + * depth will need to call {@link IonReader#next()} again. + * + * - If the IonReader is not positioned over a value (for example: because it is at the beginning or end of a + * stream), then this function does nothing. + * - If the current value is a container, this function will visit all of its child values and write those too, + * advancing the IonReader to the end of the container in the process. + * - If both this writer and the IonReader are in a struct, the writer will write the current value's field name. + * - If the writer is not in a struct but the reader is, the writer will ignore the current value's field name. + * - If the writer is in a struct but the IonReader is not, this function throws an IllegalStateException. + * + * @param reader The IonReader that will provide a value to write. + * @throws IOException if either the provided IonReader or this writer's underlying OutputStream throw an + * IOException. + * @throws IllegalStateException if this writer is inside a struct but the IonReader is not. */ - final void writeValueRecursively(IonType type, IonReader reader) - throws IOException + final void writeValueRecursively(IonReader reader) throws IOException { - write_value_field_name_helper(reader); - write_value_annotations_helper(reader); + // The IonReader does not need to be at the top level (getDepth()==0) when the function is called. + // We take note of its initial depth so we can avoid advancing the IonReader beyond the starting value. + int startingDepth = getDepth(); + + // The IonReader will be at `startingDepth` when the function is first called and then again when we + // have finished traversing all of its children. This boolean tracks which of those two states we are + // in when `getDepth() == startingDepth`. + boolean alreadyProcessedTheStartingValue = false; + + // The IonType of the IonReader's current value. + IonType type; + + while (true) { + // Each time we reach the top of the loop we are in one of three states: + // 1. We have not yet begun processing the starting value. + // 2. We are currently traversing the starting value's children. + // 3. We have finished processing the starting value. + if (getDepth() == startingDepth) { + // The IonReader is at the starting depth. We're either beginning our traversal or finishing it. + if (alreadyProcessedTheStartingValue) { + // We're finishing our traversal. + break; + } + // We're beginning our traversal. Don't advance the cursor; instead, use the current + // value's IonType. + type = reader.getType(); + // We've begun processing the starting value. + alreadyProcessedTheStartingValue = true; + } else { + // We're traversing the starting value's children (that is: values at greater depths). We need to + // advance the cursor by calling next(). + type = reader.next(); + } - if (reader.isNullValue()) { - this.writeNull(type); - } - else { - switch (type) { - case NULL: - writeNull(); - if (_debug_on) System.out.print("-"); - break; - case BOOL: - writeBool(reader.booleanValue()); - if (_debug_on) System.out.print("b"); - break; - case INT: - writeInt(reader.bigIntegerValue()); - if (_debug_on) System.out.print("i"); - break; - case FLOAT: - writeFloat(reader.doubleValue()); - if (_debug_on) System.out.print("f"); - break; - case DECIMAL: - writeDecimal(reader.decimalValue()); - if (_debug_on) System.out.print("d"); - break; - case TIMESTAMP: - writeTimestamp(reader.timestampValue()); - if (_debug_on) System.out.print("t"); - break; - case STRING: - writeString(reader.stringValue()); - if (_debug_on) System.out.print("$"); - break; - case SYMBOL: - writeSymbolToken(reader.symbolValue()); - if (_debug_on) System.out.print("y"); - break; - case BLOB: - writeBlob(reader.newBytes()); - if (_debug_on) System.out.print("B"); - break; - case CLOB: - writeClob(reader.newBytes()); - if (_debug_on) System.out.print("L"); - break; - case STRUCT: - if (_debug_on) System.out.print("{"); - writeContainerRecursively(IonType.STRUCT, reader); - if (_debug_on) System.out.print("}"); - break; - case LIST: - if (_debug_on) System.out.print("["); - writeContainerRecursively(IonType.LIST, reader); - if (_debug_on) System.out.print("]"); - break; - case SEXP: - if (_debug_on) System.out.print("("); - writeContainerRecursively(IonType.SEXP, reader); - if (_debug_on) System.out.print(")"); - break; - default: - throw new IllegalStateException("Unknown value type: " + type); + if (type == null) { + // There are no more values at this level. If we're at the starting level, we're done. + if (getDepth() == startingDepth) { + break; + } + // Otherwise, step out once and then try to move forward again. + reader.stepOut(); + stepOut(); + continue; } - } - } - private void writeContainerRecursively(IonType type, IonReader reader) - throws IOException - { - stepIn(type); - reader.stepIn(); - while ((type = reader.next()) != null) - { - writeValueRecursively(type, reader); + // We found a value. Write out its field name and annotations, if any. + write_value_field_name_helper(reader); + write_value_annotations_helper(reader); + + if (reader.isNullValue()) { + this.writeNull(type); + continue; + } + + switch (type) { + case NULL: + // The isNullValue() check above will handle this. + throw new IllegalStateException("isNullValue() was false but IonType was NULL."); + case BOOL: + writeBool(reader.booleanValue()); + break; + case INT: + writeInt(reader.bigIntegerValue()); + break; + case FLOAT: + writeFloat(reader.doubleValue()); + break; + case DECIMAL: + writeDecimal(reader.decimalValue()); + break; + case TIMESTAMP: + writeTimestamp(reader.timestampValue()); + break; + case STRING: + writeString(reader.stringValue()); + break; + case SYMBOL: + writeSymbolToken(reader.symbolValue()); + break; + case BLOB: + writeBlob(reader.newBytes()); + break; + case CLOB: + writeClob(reader.newBytes()); + break; + case STRUCT: // Intentional fallthrough + case LIST: // Intentional fallthrough + case SEXP: + reader.stepIn(); + stepIn(type); + break; + default: + throw new IllegalStateException("Unknown value type: " + type); + } } - reader.stepOut(); - stepOut(); } - // // This code handles the skipped symbol table // support - it is cloned in IonReaderTextUserX, From 4132c761cf1ee0db634af4aa90674fd3b089b13c Mon Sep 17 00:00:00 2001 From: Zack Slayton Date: Wed, 23 Jun 2021 17:21:00 -0400 Subject: [PATCH 131/490] Adds a pool of UTF8 String encoders (#369) * Adds a pool of UTF8 String encoders Most of the expense involved in constructing new binary writers comes from allocating/initializing the buffers needed to encode Java's UTF-16 Strings to UTF-8. This change refactors the UTF-8 encoding logic into its own class (Utf8StringEncoder) and introduces a singleton Utf8StringEncoderPool that allows these encoders to be reused across instantiations of binary writers. --- .../ion/impl/bin/IonRawBinaryWriter.java | 93 ++-------- .../ion/impl/bin/utf8/Utf8StringEncoder.java | 170 ++++++++++++++++++ .../impl/bin/utf8/Utf8StringEncoderPool.java | 60 +++++++ 3 files changed, 242 insertions(+), 81 deletions(-) create mode 100644 src/com/amazon/ion/impl/bin/utf8/Utf8StringEncoder.java create mode 100644 src/com/amazon/ion/impl/bin/utf8/Utf8StringEncoderPool.java diff --git a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java index 887ad9be30..4df80f7993 100644 --- a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java +++ b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java @@ -46,15 +46,13 @@ import com.amazon.ion.SymbolTable; import com.amazon.ion.SymbolToken; import com.amazon.ion.Timestamp; +import com.amazon.ion.impl.bin.utf8.Utf8StringEncoder; +import com.amazon.ion.impl.bin.utf8.Utf8StringEncoderPool; + import java.io.IOException; import java.io.OutputStream; import java.math.BigDecimal; import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.Charset; -import java.nio.charset.CharsetEncoder; -import java.nio.charset.CoderResult; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -125,14 +123,9 @@ private static byte[] bytes(int... vals) { private static final byte VARINT_NEG_ZERO = (byte) 0xC0; - // See IonRawBinaryWriter#writeString(String) for usage information. - static final int SMALL_STRING_SIZE = 4 * 1024; - - // Reusable resources for encoding Strings as UTF-8 bytes - final CharsetEncoder utf8Encoder = Charset.forName("UTF-8").newEncoder(); - final ByteBuffer utf8EncodingBuffer = ByteBuffer.allocate((int) (SMALL_STRING_SIZE * utf8Encoder.maxBytesPerChar())); - final char[] charArray = new char[SMALL_STRING_SIZE]; - final CharBuffer reusableCharBuffer = CharBuffer.wrap(charArray); + final Utf8StringEncoder utf8StringEncoder = Utf8StringEncoderPool + .getInstance() + .getOrCreateUtf8Encoder(); private static final byte[] makeTypedPreallocatedBytes(final int typeDesc, final int length) { @@ -1443,73 +1436,10 @@ public void writeString(final String value) throws IOException } prepareValue(); - /* - This method relies on the standard CharsetEncoder class to encode each String's UTF-16 char[] data into - UTF-8 bytes. Strangely, CharsetEncoders cannot operate directly on instances of a String. The CharsetEncoder - API requires all inputs and outputs to be specified as instances of java.nio.ByteBuffer and - java.nio.CharBuffer, making some number of allocations mandatory. Specifically, for each encoding operation - we need to have: - - 1. An instance of a UTF-8 CharsetEncoder. - 2. A CharBuffer representation of the String's data. - 3. A ByteBuffer into which the CharsetEncoder may write UTF-8 bytes. - - To minimize the overhead involved, the IonRawBinaryWriter will reuse previously initialized resources wherever - possible. However, because CharBuffer and ByteBuffer each have a fixed length, we can only reuse them for - Strings that are small enough to fit. This creates two kinds of input String to encode: those that are small - enough for us to reuse our buffers ("small strings"), and those which are not ("large strings"). - - The String#getBytes(Charset) method cannot be used for two reasons: - - 1. It always allocates, so we cannot reuse any resources. - 2. If/when it encounters character data that cannot be encoded as UTF-8, it simply replaces that data - with a substitute character[1]. (Sometimes seen in applications as a '?'.) In order - to surface invalid data to the user, the method must be able to detect these events at encoding time. - - [1] https://en.wikipedia.org/wiki/Substitute_character - */ - - CharBuffer stringData; - ByteBuffer encodingBuffer; - - int length = value.length(); - - // While it is possible to encode the Ion string using a fixed-size encodingBuffer, we need to be able to - // write the length of the complete UTF-8 string to the output stream before we write the string itself. - // For simplicity, we reuse or create an encodingBuffer that is large enough to hold the full string. - - // In order to encode the input String, we need to pass it to CharsetEncoder as an implementation of CharBuffer. - // Surprisingly, the intuitive way to achieve this (the CharBuffer#wrap(CharSequence) method) adds a large - // amount of CPU overhead to the encoding process. Benchmarking shows that it's substantially faster - // to use String#getChars(int, int, char[], int) to copy the String's backing array and then call - // CharBuffer#wrap(char[]) on the copy. - - if (length > SMALL_STRING_SIZE) { - // Allocate a new buffer for large strings - encodingBuffer = ByteBuffer.allocate((int) (value.length() * utf8Encoder.maxBytesPerChar())); - char[] chars = new char[value.length()]; - value.getChars(0, value.length(), chars, 0); - stringData = CharBuffer.wrap(chars); - } else { - // Reuse our existing buffers for small strings - encodingBuffer = utf8EncodingBuffer; - encodingBuffer.clear(); - stringData = reusableCharBuffer; - value.getChars(0, value.length(), charArray, 0); - reusableCharBuffer.rewind(); - reusableCharBuffer.limit(value.length()); - } - - // Because encodingBuffer is guaranteed to be large enough to hold the encoded string, we can - // perform the encoding in a single call to CharsetEncoder#encode(CharBuffer, ByteBuffer, boolean). - CoderResult coderResult = utf8Encoder.encode(stringData, encodingBuffer, true); - - // 'Underflow' is the success state of a CoderResult. - if (!coderResult.isUnderflow()) { - throw new IllegalArgumentException("Could not encode string as UTF8 bytes: " + value); - } - encodingBuffer.flip(); - int utf8Length = encodingBuffer.remaining(); + // UTF-8 encode the String + Utf8StringEncoder.Result encoderResult = utf8StringEncoder.encode(value); + int utf8Length = encoderResult.getEncodedLength(); + byte[] utf8Buffer = encoderResult.getBuffer(); // Write the type and length codes to the output stream. long previousPosition = buffer.position(); @@ -1521,7 +1451,7 @@ enough for us to reuse our buffers ("small strings"), and those which are not (" } // Write the encoded UTF-8 bytes to the output stream - buffer.writeBytes(encodingBuffer.array(), 0, utf8Length); + buffer.writeBytes(utf8Buffer, 0, utf8Length); long bytesWritten = buffer.position() - previousPosition; updateLength(bytesWritten); @@ -1686,6 +1616,7 @@ public void close() throws IOException buffer.close(); patchBuffer.close(); allocator.close(); + utf8StringEncoder.close(); } finally { diff --git a/src/com/amazon/ion/impl/bin/utf8/Utf8StringEncoder.java b/src/com/amazon/ion/impl/bin/utf8/Utf8StringEncoder.java new file mode 100644 index 0000000000..d890e1beaf --- /dev/null +++ b/src/com/amazon/ion/impl/bin/utf8/Utf8StringEncoder.java @@ -0,0 +1,170 @@ +package com.amazon.ion.impl.bin.utf8; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CoderResult; + +/** + * Encodes {@link String}s to UTF-8. Instances of this class are reusable but are NOT threadsafe. + * + * Users are strongly encouraged to get instances from {@link Utf8StringEncoderPool#getOrCreateUtf8Encoder()}. + * {@link #encode(String)} can be called any number of times. Users are expected to call {@link #close()} when + * the encoder is no longer needed. + */ +public class Utf8StringEncoder implements Closeable { + // The longest String (as measured by {@link java.lang.String#length()}) that this instance can encode without + // requiring additional allocations. + private static final int SMALL_STRING_SIZE = 4 * 1024; + + // Reusable resources for encoding Strings as UTF-8 bytes + final Utf8StringEncoderPool utf8StringEncoderPool; + final CharsetEncoder utf8Encoder; + final ByteBuffer utf8EncodingBuffer; + final char[] charArray; + final CharBuffer charBuffer; + + public Utf8StringEncoder(Utf8StringEncoderPool pool) { + utf8StringEncoderPool = pool; + utf8Encoder = Charset.forName("UTF-8").newEncoder(); + utf8EncodingBuffer = ByteBuffer.allocate((int) (SMALL_STRING_SIZE * utf8Encoder.maxBytesPerChar())); + charArray = new char[SMALL_STRING_SIZE]; + charBuffer = CharBuffer.wrap(charArray); + } + + public Utf8StringEncoder() { + // This instance is not associated with a Utf8StringEncoderPool + this(null); + } + + /** + * Encodes the provided String's text to UTF-8. Unlike {@link String#getBytes(Charset)}, this method will not + * silently replace characters that cannot be encoded with a substitute character. Instead, it will throw + * an {@link IllegalArgumentException}. + * + * Some resources in the returned {@link Result} may be reused across calls to this method. Consequently, + * callers should use the Result and discard it immediately. + * + * @param text A Java String to encode as UTF8 bytes. + * @return A {@link Result} containing a byte array of UTF-8 bytes and encoded length. + * @throws IllegalArgumentException if the String cannot be encoded as UTF-8. + */ + public Result encode(String text) { + /* + This method relies on the standard CharsetEncoder class to encode each String's UTF-16 char[] data into + UTF-8 bytes. Strangely, CharsetEncoders cannot operate directly on instances of a String. The CharsetEncoder + API requires all inputs and outputs to be specified as instances of java.nio.ByteBuffer and + java.nio.CharBuffer, making some number of allocations mandatory. Specifically, for each encoding operation + we need to have: + + 1. An instance of a UTF-8 CharsetEncoder. + 2. A CharBuffer representation of the String's data. + 3. A ByteBuffer into which the CharsetEncoder may write UTF-8 bytes. + + To minimize the overhead involved, the Utf8StringEncoder will reuse previously initialized resources wherever + possible. However, because CharBuffer and ByteBuffer each have a fixed length, we can only reuse them for + Strings that are small enough to fit. This creates two kinds of input String to encode: those that are small + enough for us to reuse our buffers ("small strings"), and those which are not ("large strings"). + + The String#getBytes(Charset) method cannot be used for two reasons: + + 1. It always allocates, so we cannot reuse any resources. + 2. If/when it encounters character data that cannot be encoded as UTF-8, it simply replaces that data + with a substitute character[1]. (Sometimes seen in applications as a '?'.) In order + to surface invalid data to the user, the method must be able to detect these events at encoding time. + + [1] https://en.wikipedia.org/wiki/Substitute_character + */ + + CharBuffer stringData; + ByteBuffer encodingBuffer; + + int length = text.length(); + + // While it is technically possible to encode any String using a fixed-size encodingBuffer, we need + // to be able to write the length of the complete UTF-8 string to the output stream before we write the string + // itself. For simplicity, we reuse or create an encodingBuffer that is large enough to hold the full string. + + // In order to encode the input String, we need to pass it to CharsetEncoder as an implementation of CharBuffer. + // Surprisingly, the intuitive way to achieve this (the CharBuffer#wrap(CharSequence) method) adds a large + // amount of CPU overhead to the encoding process. Benchmarking shows that it's substantially faster + // to use String#getChars(int, int, char[], int) to copy the String's backing array and then call + // CharBuffer#wrap(char[]) on the copy. + + if (length > SMALL_STRING_SIZE) { + // Allocate a new buffer for large strings + encodingBuffer = ByteBuffer.allocate((int) (text.length() * utf8Encoder.maxBytesPerChar())); + char[] chars = new char[text.length()]; + text.getChars(0, text.length(), chars, 0); + stringData = CharBuffer.wrap(chars); + } else { + // Reuse our existing buffers for small strings + encodingBuffer = utf8EncodingBuffer; + encodingBuffer.clear(); + stringData = charBuffer; + text.getChars(0, text.length(), charArray, 0); + charBuffer.rewind(); + charBuffer.limit(text.length()); + } + + // Because encodingBuffer is guaranteed to be large enough to hold the encoded string, we can + // perform the encoding in a single call to CharsetEncoder#encode(CharBuffer, ByteBuffer, boolean). + CoderResult coderResult = utf8Encoder.encode(stringData, encodingBuffer, true); + + // 'Underflow' is the success state of a CoderResult. + if (!coderResult.isUnderflow()) { + throw new IllegalArgumentException("Could not encode string as UTF8 bytes: " + text); + } + encodingBuffer.flip(); + int utf8Length = encodingBuffer.remaining(); + + // In most usages, the JVM should be able to eliminate this allocation via an escape analysis of the caller. + return new Result(utf8Length, encodingBuffer.array()); + } + + /** + * Attempts to return this instance to the Utf8StringEncoderPool with which it is associated, if any. + * + * Do not continue to use this encoder after calling this method. + */ + @Override + public void close() { + if (utf8StringEncoderPool != null) { + utf8StringEncoderPool.returnEncoderToPool(this); + } + } + + /** + * Represents the result of a {@link Utf8StringEncoder#encode(String)} operation. + */ + public static class Result { + final private byte[] buffer; + final private int encodedLength; + + public Result(int encodedLength, byte[] buffer) { + this.encodedLength = encodedLength; + this.buffer = buffer; + } + + /** + * Returns a byte array containing the encoded UTF-8 bytes starting at index 0. This byte array is NOT + * guaranteed to be the same length as the data it contains. Callers must use {@link #getEncodedLength()} + * to determine the number of bytes that should be read from the byte array. + * + * @return the buffer containing UTF-8 bytes. + */ + public byte[] getBuffer() { + return buffer; + } + + /** + * @return the number of encoded bytes in the array returned by {@link #getBuffer()}. + */ + public int getEncodedLength() { + return encodedLength; + } + } +} diff --git a/src/com/amazon/ion/impl/bin/utf8/Utf8StringEncoderPool.java b/src/com/amazon/ion/impl/bin/utf8/Utf8StringEncoderPool.java new file mode 100644 index 0000000000..e926df4a75 --- /dev/null +++ b/src/com/amazon/ion/impl/bin/utf8/Utf8StringEncoderPool.java @@ -0,0 +1,60 @@ +package com.amazon.ion.impl.bin.utf8; + +import java.util.concurrent.ArrayBlockingQueue; + +/** + * A thread-safe shared pool of {@link Utf8StringEncoder}s that can be used for UTF8 encoding and decoding. + */ +public enum Utf8StringEncoderPool { + // The only enum variant; a singleton instance. + INSTANCE; + + // The maximum number of Utf8Encoders that can be waiting in the queue before new ones will be discarded. + private static final int MAX_QUEUE_SIZE = 128; + + // A queue of previously initialized encoders that can be loaned out. + private final ArrayBlockingQueue bufferQueue; + + // Do not allow instantiation; all classes should share the singleton instance. + private Utf8StringEncoderPool() { + bufferQueue = new ArrayBlockingQueue(MAX_QUEUE_SIZE); + } + + /** + * @return a threadsafe shared instance of {@link Utf8StringEncoderPool}. + */ + public static Utf8StringEncoderPool getInstance() { + return INSTANCE; + } + + /** + * If the pool is not empty, removes an instance of {@link Utf8StringEncoder} from the pool and returns it; + * otherwise, constructs a new instance. + * + * @return An instance of {@link Utf8StringEncoder}. + */ + public Utf8StringEncoder getOrCreateUtf8Encoder() { + // The `poll` method does not block. If the queue is empty it returns `null` immediately. + Utf8StringEncoder encoder = bufferQueue.poll(); + if (encoder == null) { + // No buffers were available in the pool. Create a new one. + encoder = new Utf8StringEncoder(this); + } + return encoder; + } + + /** + * Adds the provided instance of {@link Utf8StringEncoder} to the pool. If the pool is full, the instance will + * be discarded. + * + * Callers MUST NOT use an encoder after returning it to the pool. + * + * @param encoder A {@link Utf8StringEncoder} to add to the pool. + */ + public void returnEncoderToPool(Utf8StringEncoder encoder) { + // The `offer` method does not block. If the queue is full, it returns `false` immediately. + // If the provided instance cannot be added to the pool, we discard it silently. + bufferQueue.offer(encoder); + } + +} From edf7809fd579347318e63410c900fb181e332a00 Mon Sep 17 00:00:00 2001 From: Eric Chen <67451029+cheqianh@users.noreply.github.com> Date: Tue, 10 Aug 2021 15:45:51 -0700 Subject: [PATCH 132/490] Prepares version 1.8.3 for release. (#374) --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f6ec39d438..dad1504b2d 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ dependency into your project's `pom.xml`: com.amazon.ion ion-java - 1.8.2 + 1.8.3 ``` diff --git a/pom.xml b/pom.xml index 3858356611..4f7a712c56 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.8.3-SNAPSHOT + 1.8.3 bundle ${project.groupId}:${project.artifactId} From ddaa965e45917881babe4b7277d972a997bf646e Mon Sep 17 00:00:00 2001 From: linlin-s <84819822+linlin-s@users.noreply.github.com> Date: Tue, 10 Aug 2021 16:18:41 -0700 Subject: [PATCH 133/490] Adds GitHub workflow to detect the performance regression of ion-java automatically. (#373) --- ...n-java-performance-regression-detector.yml | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 .github/workflows/ion-java-performance-regression-detector.yml diff --git a/.github/workflows/ion-java-performance-regression-detector.yml b/.github/workflows/ion-java-performance-regression-detector.yml new file mode 100644 index 0000000000..6bf0cd43c5 --- /dev/null +++ b/.github/workflows/ion-java-performance-regression-detector.yml @@ -0,0 +1,123 @@ +# This workflow will build a Java project with Maven +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven + +name: Ion Java performance regression detector + +on: [pull_request] + +jobs: + detect-regression: + name: Detect Regression + + runs-on: ubuntu-latest + + steps: + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + + - name: Checkout ion-java from the new commit. + uses: actions/checkout@v2 + with: + fetch-depth: 2 + submodules: recursive + path: ion-java + + - name: Build ion-java from the new commit + run: cd ion-java && mvn clean install + + - name: Checkout ion-java-benchmark-cli + uses: actions/checkout@v2 + with: + repository: amzn/ion-java-benchmark-cli + ref: master + path: ion-java-benchmark-cli + + - name: Build ion-java-benchmark-cli + run: cd ion-java-benchmark-cli && mvn clean install + + - name: Check the version of ion-java. + run: java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar --version + + - name: Generate test Ion Data + run: | + mkdir -p testData + java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar generate -S 50000 --input-ion-schema ion-java-benchmark-cli/tst/com/amazon/ion/benchmark/testStruct.isl testData/testStruct.10n + java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar generate -S 50000 --input-ion-schema ion-java-benchmark-cli/tst/com/amazon/ion/benchmark/testList.isl testData/testList.10n + java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar generate -S 50000 --input-ion-schema ion-java-benchmark-cli/tst/com/amazon/ion/benchmark/testNestedStruct.isl testData/testNestedStruct.10n + + - name: Upload test Ion Data to artifacts + uses: actions/upload-artifact@v2 + with: + name: test Ion Data + path: testData + + - name: Benchmark ion-java from the new commit + run: | + mkdir -p benchmarkResults + cd ion-java-benchmark-cli && java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar run-suite --test-ion-data /home/runner/work/ion-java/ion-java/testData --benchmark-options-combinations tst/com/amazon/ion/benchmark/optionsCombinations.ion /home/runner/work/ion-java/ion-java/benchmarkResults + + - name: Upload benchmark results to artifacts + uses: actions/upload-artifact@v2 + with: + name: Benchmark result + path: benchmarkResults + + - name: Clean maven dependencies repository + run : rm -r /home/runner/.m2 + + - name: Build ion-java from the previous commit + run: cd ion-java && git checkout HEAD^ && mvn clean install + + - name: Build ion-java-benchmark-cli + run: cd ion-java-benchmark-cli && mvn clean install + + - name: Check the version of ion-java + run: java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar --version + + - name: Create directories for test data and benchmark results + run: | + mkdir -p benchmarkResults + mkdir -p testData + + - name: Download test Ion Data from artifacts + uses: actions/download-artifact@v2 + with: + name: test Ion Data + path: testData + + - name: Download benchmark results of ion-java from the new commit from artifacts + uses: actions/download-artifact@v2 + with: + name: Benchmark result + path: benchmarkResults + + - name: Benchmark ion-java from the previous commit and add the generated benchmark results to the existing directories + run: cd ion-java-benchmark-cli && java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar run-suite --test-ion-data /home/runner/work/ion-java/ion-java/testData --benchmark-options-combinations tst/com/amazon/ion/benchmark/optionsCombinations.ion /home/runner/work/ion-java/ion-java/benchmarkResults + + - name: Upload new benchmark results directory to artifacts + uses: actions/upload-artifact@v2 + with: + name: Benchmark result + path: benchmarkResults + + - name: Detect performance regression + id: regression_result + run: | + result=true + cd benchmarkResults && for FILE in *; do message=$(java -jar /home/runner/work/ion-java/ion-java/ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous $FILE/previous.ion --benchmark-result-new $FILE/new.ion $FILE/report.ion | tee /dev/stderr) && if [ "$message" != "" ]; then result=false; fi; done + echo "::set-output name=regression-result::$result" + echo $result + + - name: Upload comparison reports to the benchmark results directory + uses: actions/upload-artifact@v2 + with: + name: Benchmark result + path: benchmarkResults + + - name: Fail the workflow if regression happened + env: + regression_detect: ${{steps.regression_result.outputs.regression-result}} + if: ${{ env.regression_detect == 'false' }} + run: exit 1 From b183f250919ed7f4196a353a69ec801ca56368c1 Mon Sep 17 00:00:00 2001 From: Eric Chen <67451029+cheqianh@users.noreply.github.com> Date: Tue, 10 Aug 2021 21:50:14 -0700 Subject: [PATCH 134/490] Bumps version to 1.8.4-SNAPSHOT (#377) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4f7a712c56..eb601e57b4 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.8.3 + 1.8.4-SNAPSHOT bundle ${project.groupId}:${project.artifactId} From a398b156bed8e7d110e68a7a00e4b781c3fe8138 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Tue, 7 Sep 2021 17:04:11 -0700 Subject: [PATCH 135/490] Adds a binary IonReader implementation capable of incremental reads. (#355) --- src/com/amazon/ion/BufferConfiguration.java | 265 ++ .../amazon/ion/IonBufferConfiguration.java | 164 + src/com/amazon/ion/IonReader.java | 6 +- .../ion/impl/IonReaderBinaryIncremental.java | 1957 ++++++++++ .../ion/impl/IonReaderLookaheadBuffer.java | 921 +++++ src/com/amazon/ion/impl/IonTypeID.java | 116 + .../ion/impl/ReaderLookaheadBuffer.java | 46 + .../ion/impl/ReaderLookaheadBufferBase.java | 194 + .../ion/impl/ResizingPipedInputStream.java | 557 +++ src/com/amazon/ion/impl/SymbolTokenImpl.java | 14 +- .../ion/impl/_Private_IonReaderFactory.java | 7 + .../ion/impl/_Private_RecyclingStack.java | 95 + src/com/amazon/ion/impl/_Private_Utils.java | 13 +- .../ion/impl/bin/IonRawBinaryWriter.java | 91 +- .../amazon/ion/impl/lite/IonDatagramLite.java | 4 +- .../amazon/ion/system/IonReaderBuilder.java | 213 +- test/AllTests.java | 8 + ...IonReaderIteratorSystemProcessingTest.java | 35 + .../ion/BinaryReaderSystemProcessingTest.java | 2 +- .../BinaryReaderWrappedValueLengthTest.java | 8 +- test/com/amazon/ion/BinaryTest.java | 24 +- test/com/amazon/ion/EquivsTestCase.java | 24 +- test/com/amazon/ion/GoodIonTest.java | 18 +- test/com/amazon/ion/IonTestCase.java | 50 +- ...adBinaryIonReaderSystemProcessingTest.java | 31 + test/com/amazon/ion/NopPaddingTest.java | 2 - .../ion/RawValueSpanReaderBasicTest.java | 9 +- test/com/amazon/ion/ReaderChecker.java | 5 +- test/com/amazon/ion/ReaderMaker.java | 29 + .../ion/ReaderSystemProcessingTestCase.java | 10 - test/com/amazon/ion/RoundTripTest.java | 12 +- .../amazon/ion/SystemProcessingTestCase.java | 26 +- .../com/amazon/ion/SystemProcessingTests.java | 2 + test/com/amazon/ion/TestUtils.java | 29 + .../impl/IonReaderBinaryIncrementalTest.java | 3158 +++++++++++++++++ .../impl/IonReaderLookaheadBufferTest.java | 1197 +++++++ .../IonReaderLookaheadBufferTestBase.java | 679 ++++ .../impl/ResizingPipedInputStreamTest.java | 789 ++++ test/com/amazon/ion/impl/SymbolTableTest.java | 4 +- test/com/amazon/ion/junit/IonAssert.java | 5 +- .../ion/streaming/BadIonStreamingTest.java | 3 +- .../ion/streaming/BinaryStreamingTest.java | 50 +- .../ion/streaming/GoodIonStreamingTest.java | 6 +- .../ion/streaming/ReaderFacetTestCase.java | 13 +- .../ion/streaming/ReaderIntegerSizeTest.java | 11 +- .../ion/streaming/ReaderSkippingTest.java | 4 +- test/com/amazon/ion/streaming/ReaderTest.java | 104 + .../ion/streaming/RoundTripStreamingTest.java | 8 +- .../ion/system/IonReaderBuilderTest.java | 80 + 49 files changed, 10914 insertions(+), 184 deletions(-) create mode 100644 src/com/amazon/ion/BufferConfiguration.java create mode 100644 src/com/amazon/ion/IonBufferConfiguration.java create mode 100644 src/com/amazon/ion/impl/IonReaderBinaryIncremental.java create mode 100644 src/com/amazon/ion/impl/IonReaderLookaheadBuffer.java create mode 100644 src/com/amazon/ion/impl/IonTypeID.java create mode 100644 src/com/amazon/ion/impl/ReaderLookaheadBuffer.java create mode 100644 src/com/amazon/ion/impl/ReaderLookaheadBufferBase.java create mode 100644 src/com/amazon/ion/impl/ResizingPipedInputStream.java create mode 100644 src/com/amazon/ion/impl/_Private_RecyclingStack.java create mode 100644 test/com/amazon/ion/BinaryIonReaderIteratorSystemProcessingTest.java create mode 100644 test/com/amazon/ion/LoadBinaryIonReaderSystemProcessingTest.java create mode 100644 test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java create mode 100644 test/com/amazon/ion/impl/IonReaderLookaheadBufferTest.java create mode 100644 test/com/amazon/ion/impl/IonReaderLookaheadBufferTestBase.java create mode 100644 test/com/amazon/ion/impl/ResizingPipedInputStreamTest.java diff --git a/src/com/amazon/ion/BufferConfiguration.java b/src/com/amazon/ion/BufferConfiguration.java new file mode 100644 index 0000000000..31a13fd41a --- /dev/null +++ b/src/com/amazon/ion/BufferConfiguration.java @@ -0,0 +1,265 @@ +package com.amazon.ion; + +import com.amazon.ion.impl.ReaderLookaheadBuffer; + +/** + * Provides logic common to all BufferConfiguration implementations. + * @param the type of the concrete subclass of this BufferConfiguration that is applicable to the + * ReaderLookaheadBufferBase subclass. + */ +public abstract class BufferConfiguration> { + + /** + * Functional interface for handling oversized values. + */ + public interface OversizedValueHandler { + /** + * Invoked each time a value (and any symbol tables that immediately precede it) exceed the buffer size limit + * specified by the LookaheadReaderWrapper instance, but the symbol tables by themselves do not exceed the + * limit. This is recoverable. If the implementation wishes to recover, it should simply return normally from + * this method. The oversized value will be flushed from the input pipe; normal processing will resume with the + * next value. If the implementation wishes to abort processing immediately, it may throw an exception from this + * method. Such an exception will propagate upward and will be thrown from + * {@link ReaderLookaheadBuffer#fillInput()}. + * @throws Exception if handler logic fails. + */ + void onOversizedValue() throws Exception; + } + + /** + * Functional interface for reporting processed data. + */ + public interface DataHandler { + /** + * Invoked whenever the bytes from a value are processed, regardless of whether the bytes are buffered or + * skipped due to the value being oversized. + * @param numberOfBytes the number of bytes processed. + * @throws Exception if handler logic fails. + */ + void onData(int numberOfBytes) throws Exception; + } + + /** + * Provides logic common to all BufferConfiguration Builder implementations. + * @param the type of BufferConfiguration. + * @param the type of Builder that builds BufferConfiguration subclasses of type `Configuration`. + */ + public static abstract class Builder< + Configuration extends BufferConfiguration, + BuilderType extends BufferConfiguration.Builder + > { + + /** + * Large enough that most streams will never need to grow the buffer. NOTE: this only needs to be large + * enough to exceed the length of the longest top-level value plus any system values that precede it. + */ + static final int DEFAULT_INITIAL_BUFFER_SIZE = 32 * 1024; // bytes + + /** + * The initial size of the lookahead buffer, in bytes. + */ + private int initialBufferSize = DEFAULT_INITIAL_BUFFER_SIZE; + + /** + * The maximum number of bytes that will be buffered. + */ + private int maximumBufferSize = Integer.MAX_VALUE; + + /** + * The handler that will be notified when oversized values are encountered. + */ + private OversizedValueHandler oversizedValueHandler = null; + + /** + * The handler that will be notified when data is processed. + */ + private DataHandler dataHandler = null; + + /** + * Sets the initial size of the buffer that will be used to hold the data between top-level values. Default: + * 32KB. + * + * @param initialBufferSizeInBytes the value. + * @return this Builder. + */ + public final BuilderType withInitialBufferSize(final int initialBufferSizeInBytes) { + initialBufferSize = initialBufferSizeInBytes; + return (BuilderType) this; + } + + /** + * @return the initial size of the lookahead buffer, in bytes. + */ + public final int getInitialBufferSize() { + return initialBufferSize; + } + + /** + * Sets the handler that will be notified when oversized values are encountered. If the maximum buffer size is + * finite (see {@link #withMaximumBufferSize(int)}, this handler is required to be non-null. + * + * @param handler the handler. + * @return this builder. + */ + public final BuilderType onOversizedValue(final OversizedValueHandler handler) { + oversizedValueHandler = handler; + return (BuilderType) this; + } + + /** + * Sets the handler that will be notified when data is processed. The handler may be null, in which case the + * number of bytes processed will not be reported. + * + * @param handler the handler. + * @return this builder. + */ + public final BuilderType onData(final DataHandler handler) { + dataHandler = handler; + return (BuilderType) this; + } + + /** + * @return the handler that will be notified when oversized values are encountered. + */ + public final OversizedValueHandler getOversizedValueHandler() { + return oversizedValueHandler; + } + + /** + * @return the handler that will be notified when data is processed. + */ + public final DataHandler getDataHandler() { + return dataHandler; + } + + /** + * Set the maximum number of bytes between top-level values. This can be used to limit growth of the internal + * buffer. For binary Ion, the minimum value is 5 because all valid binary Ion data begins with a 4-byte Ion + * version marker and the smallest value is 1 byte. For delimited text Ion, the minimum value is 2 because the + * smallest text Ion value is 1 byte and the smallest delimiter is 1 byte. Default: Integer.MAX_VALUE. + * + * @param maximumBufferSizeInBytes the value. + * @return this builder. + */ + public final BuilderType withMaximumBufferSize(final int maximumBufferSizeInBytes) { + maximumBufferSize = maximumBufferSizeInBytes; + return (BuilderType) this; + } + + /** + * @return the maximum number of bytes that will be buffered. + */ + public int getMaximumBufferSize() { + return maximumBufferSize; + } + + /** + * Gets the minimum allowed maximum buffer size. + * @return the value. + */ + public abstract int getMinimumMaximumBufferSize(); + + /** + * @return the no-op {@link OversizedValueHandler} for the type of BufferConfiguration that this Builder builds. + */ + public abstract OversizedValueHandler getNoOpOversizedValueHandler(); + + /** + * @return the no-op {@link DataHandler} for the type of BufferConfiguration that this Builder builds. + */ + public abstract DataHandler getNoOpDataHandler(); + + /** + * Creates a new BufferConfiguration from the Builder's current settings. + * @return a new instance. + */ + public abstract Configuration build(); + } + + /** + * The initial size of the lookahead buffer, in bytes. + */ + private final int initialBufferSize; + + /** + * The maximum number of bytes that will be buffered. + */ + private final int maximumBufferSize; + + /** + * The handler that will be notified when oversized values are encountered. + */ + private final OversizedValueHandler oversizedValueHandler; + + /** + * The handler that will be notified when data is processed. + */ + private final DataHandler dataHandler; + + /** + * Constructs an instance from the given Builder. + * @param builder the builder containing the settings to apply to the new configuration. + */ + protected BufferConfiguration(Builder builder) { + initialBufferSize = builder.getInitialBufferSize(); + maximumBufferSize = builder.getMaximumBufferSize(); + if (initialBufferSize > maximumBufferSize) { + throw new IllegalArgumentException("Initial buffer size may not exceed the maximum buffer size."); + } + if (maximumBufferSize < builder.getMinimumMaximumBufferSize()) { + throw new IllegalArgumentException(String.format( + "Maximum buffer size must be at least %d bytes.", builder.getMinimumMaximumBufferSize() + )); + } + if (builder.getOversizedValueHandler() == null) { + requireUnlimitedBufferSize(); + oversizedValueHandler = builder.getNoOpOversizedValueHandler(); + } else { + oversizedValueHandler = builder.getOversizedValueHandler(); + } + if (builder.getDataHandler() == null) { + dataHandler = builder.getNoOpDataHandler(); + } else { + dataHandler = builder.getDataHandler(); + } + } + + /** + * Requires that the maximum buffer size not be limited. + */ + protected void requireUnlimitedBufferSize() { + if (maximumBufferSize < Integer.MAX_VALUE) { + throw new IllegalArgumentException( + "Must specify an OversizedValueHandler when a maximum buffer size is specified." + ); + } + } + + /** + * @return the initial size of the lookahead buffer, in bytes. + */ + public final int getInitialBufferSize() { + return initialBufferSize; + } + + /** + * @return the maximum number of bytes that will be buffered. + */ + public final int getMaximumBufferSize() { + return maximumBufferSize; + } + + /** + * @return the handler that will be notified when oversized values are encountered. + */ + public final OversizedValueHandler getOversizedValueHandler() { + return oversizedValueHandler; + } + + /** + * @return the handler that will be notified when data is processed. + */ + public final DataHandler getDataHandler() { + return dataHandler; + } +} diff --git a/src/com/amazon/ion/IonBufferConfiguration.java b/src/com/amazon/ion/IonBufferConfiguration.java new file mode 100644 index 0000000000..960ff4a074 --- /dev/null +++ b/src/com/amazon/ion/IonBufferConfiguration.java @@ -0,0 +1,164 @@ +package com.amazon.ion; + +import com.amazon.ion.impl.IonReaderLookaheadBuffer; + +/** + * Configures Ion lookahead buffers. + */ +public final class IonBufferConfiguration extends BufferConfiguration { + + /** + * Functional interface for handling oversized symbol tables. + */ + public interface OversizedSymbolTableHandler { + /** + * Invoked when the user specifies a finite maximum buffer size and that size is exceeded by symbol table(s) + * alone. Because symbol tables cannot be truncated without corrupting values that follow in the stream, + * this condition is not recoverable. After this method is called, + * {@link IonReaderLookaheadBuffer#fillInput()} has no effect. + * @throws Exception if handler logic fails. + */ + void onOversizedSymbolTable() throws Exception; + } + + /** + * Builds IonBufferConfiguration instances. + */ + public static final class Builder extends BufferConfiguration.Builder { + + /** + * 4-byte IVM + 1 byte user value. + */ + private static final int MINIMUM_MAX_VALUE_SIZE = 5; + + /** + * An OversizedValueHandler that does nothing. + */ + private static final OversizedValueHandler NO_OP_OVERSIZED_VALUE_HANDLER = new OversizedValueHandler() { + + @Override + public void onOversizedValue() { + // If no maximum buffer size is configured, values cannot be considered oversized and this + // implementation will never be called. + // If a maximum buffer size is configured, a handler must also be configured. In that case, + // this implementation will only be called if the user provides it to the builder manually. + } + }; + + /** + * A DataHandler that does nothing. + */ + private static final DataHandler NO_OP_DATA_HANDLER = new DataHandler() { + + @Override + public void onData(int bytes) { + // Do nothing. + } + }; + + /** + * An OversizedSymbolTableHandler that does nothing. + */ + private static final OversizedSymbolTableHandler NO_OP_OVERSIZED_SYMBOL_TABLE_HANDLER + = new OversizedSymbolTableHandler() { + + @Override + public void onOversizedSymbolTable() { + // If no maximum buffer size is configured, symbol tables cannot be considered oversized and this + // implementation will never be called. + // If a maximum buffer size is configured, a handler must also be configured. In that case, + // this implementation will only be called if the user provides it to the builder manually. + } + }; + + /** + * The handler that will be notified when oversized symbol tables are encountered. + */ + private OversizedSymbolTableHandler oversizedSymbolTableHandler = null; + + private Builder() { + // Must be publicly instantiated via the factory method. + } + + /** + * Provides the standard builder with the default initial buffer size, an unbounded maximum buffer size, and + * an event handler that does nothing. + * @return a standard Builder. + */ + public static Builder standard() { + return new Builder(); + } + + /** + * Sets the handler that will be notified when oversized symbol tables are encountered. If the maximum buffer + * size is finite (see {@link #withMaximumBufferSize(int)}, this handler is required to be non-null. + * + * @param handler the handler. + * @return this builder. + */ + public Builder onOversizedSymbolTable(OversizedSymbolTableHandler handler) { + oversizedSymbolTableHandler = handler; + return this; + } + + /** + * @return the handler that will be notified when oversized symbol tables are encountered. + */ + public OversizedSymbolTableHandler getOversizedSymbolTableHandler() { + return oversizedSymbolTableHandler; + } + + @Override + public int getMinimumMaximumBufferSize() { + return MINIMUM_MAX_VALUE_SIZE; + } + + @Override + public OversizedValueHandler getNoOpOversizedValueHandler() { + return NO_OP_OVERSIZED_VALUE_HANDLER; + } + + @Override + public DataHandler getNoOpDataHandler() { + return NO_OP_DATA_HANDLER; + } + + /** + * @return an OversizedSymbolTableHandler that does nothing. + */ + public OversizedSymbolTableHandler getNoOpOversizedSymbolTableHandler() { + return NO_OP_OVERSIZED_SYMBOL_TABLE_HANDLER; + } + + @Override + public IonBufferConfiguration build() { + return new IonBufferConfiguration(this); + } + } + + /** + * The handler that will be notified when oversized symbol tables are encountered. + */ + private final OversizedSymbolTableHandler oversizedSymbolTableHandler; + + /** + * Constructs an instance from the given Builder. + * @param builder the builder containing the settings to apply to the new configuration. + */ + private IonBufferConfiguration(Builder builder) { + super(builder); + if (builder.getOversizedSymbolTableHandler() == null) { + requireUnlimitedBufferSize(); + oversizedSymbolTableHandler = builder.getNoOpOversizedSymbolTableHandler(); + } else { + oversizedSymbolTableHandler = builder.getOversizedSymbolTableHandler(); + } + } + + /** + * @return the handler that will be notified when oversized symbol tables are encountered. + */ + public OversizedSymbolTableHandler getOversizedSymbolTableHandler() { + return oversizedSymbolTableHandler; + } +} diff --git a/src/com/amazon/ion/IonReader.java b/src/com/amazon/ion/IonReader.java index 7675b4b436..9066c47326 100644 --- a/src/com/amazon/ion/IonReader.java +++ b/src/com/amazon/ion/IonReader.java @@ -226,7 +226,11 @@ public interface IonReader * iterator is empty (hasNext() returns false on the first call) if * there are no annotations on the current value. * - * @throws UnknownSymbolException if any annotation has unknown text. + * Implementations *may* throw {@link UnknownSymbolException} from + * this method if any annotation contains unknown text. Alternatively, + * implementations may provide an Iterator that throws + * {@link UnknownSymbolException} only when the user navigates the + * iterator to an annotation with unknown text. * * @return not null. */ diff --git a/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java b/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java new file mode 100644 index 0000000000..542a4ea46b --- /dev/null +++ b/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java @@ -0,0 +1,1957 @@ +package com.amazon.ion.impl; + +import com.amazon.ion.Decimal; +import com.amazon.ion.IntegerSize; +import com.amazon.ion.IonBufferConfiguration; +import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonException; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.ReadOnlyValueException; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.Timestamp; +import com.amazon.ion.UnknownSymbolException; +import com.amazon.ion.system.IonReaderBuilder; +import com.amazon.ion.system.SimpleCatalog; + +import java.io.IOException; +import java.io.InputStream; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CoderResult; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; + +/** + *

    + * This implementation differs from the existing non-incremental binary reader implementation in that if + * {@link IonReader#next()} returns {@code null} at the top-level, it indicates that there is not (yet) enough data in + * the stream to complete a top-level value. The user may wait for more data to become available in the stream and + * call {@link IonReader#next()} again to continue reading. Unlike the non-incremental reader, the incremental reader + * will never throw an exception due to unexpected EOF during {@code next()}. If, however, {@link IonReader#close()} is + * called when an incomplete value is buffered, an {@link IonException} will be raised. + *

    + *

    + * Although the incremental binary reader implementation provides performance superior to the non-incremental reader + * implementation for both incremental and non-incremental use cases, there is one caveat: the incremental + * implementation must be able to buffer an entire top-level value and any preceding system values (Ion version + * marker(s) and symbol table(s)) in memory. This means that each value and preceding system values must be no larger + * than any of the following: + *

      + *
    • The configured maximum buffer size of the {@link IonBufferConfiguration}.
    • + *
    • The memory available to the JVM.
    • + *
    • 2GB, because the buffer is held in a Java {@code byte[]}, which is indexed by an {@code int}.
    • + *
    + * This will not be a problem for the vast majority of Ion streams, as it is + * rare for a single top-level value or symbol table to exceed a few megabytes in size. However, if the size of the + * stream's values risk exceeding the available memory, then this implementation must not be used. + *

    + *

    + * To enable this implementation, use {@code IonReaderBuilder.withIncrementalReadingEnabled(true)}. + *

    + */ +class IonReaderBinaryIncremental implements IonReader, _Private_ReaderWriter { + + /* + * Potential future enhancements: + * - Split this implementation into a user-level reader and a system-level reader, like the existing implementation. + * This allows this implementation to be used when the user requests a system reader. + * - Do not require buffering an entire top-level value. This would be a pretty major overhaul. It may be possible + * to implement using different buffers for each depth. Doing this may also make it possible to avoid buffering + * a value (at any depth) until stepIn() or *Value() is called on it, enabling faster skip-scanning. + * - Allow for this implementation to produce the same non-incremental behavior as the old implementation; namely, + * that running out of data during next() would raise an IonException. See the note in the implementation of + * close() below. Implementing this bullet and the previous two bullets would allow us to remove the old binary + * IonReader implementation. + * - Add a builder/constructor option that uses a user-provided byte[] directly. This would allow data to be read + * in-place without the need to copy to a separate buffer. Non-incremental behavior (as described in the previous + * bullet) is likely a requirement of this feature. + * - System symbol table configuration needs to be generalized to support future Ion versions. See the constructor, + * resetSymbolTable(), and resetImports(). + * - When accessed via an iterator, annotations can be parsed incrementally instead of parsing the entire sequence + * up-front. + * - Provide users the option to spawn a thread that pre-buffers the next value. There would be two buffers: one + * for the user thread, and one for the pre-fetching thread. They are swapped every time the user calls next(). + */ + + /** + * Holds the information that the binary reader must keep track of for containers at any depth. + */ + private static class ContainerInfo { + + /** + * The container's type. + */ + private IonType type; + + /** + * The byte position of the end of the container. + */ + private int endPosition; + } + + /** + * The standard {@link IonBufferConfiguration}. This will be used unless the user chooses custom settings. + */ + private static final IonBufferConfiguration STANDARD_BUFFER_CONFIGURATION = + IonBufferConfiguration.Builder.standard().build(); + + // Constructs ContainerInfo instances. + private static final _Private_RecyclingStack.ElementFactory CONTAINER_INFO_FACTORY = + new _Private_RecyclingStack.ElementFactory() { + + @Override + public ContainerInfo newElement() { + return new ContainerInfo(); + } + }; + + // The Ion 1.0 system symbol table. + private static final List SYSTEM_SYMBOLS_1_0 = Collections.unmodifiableList(Arrays.asList( + null, + "$ion", + "$ion_1_0", + "$ion_symbol_table", + "name", + "version", + "imports", + "symbols", + "max_id", + "$ion_shared_symbol_table" + )); + + // The size of the Ion 1.0 system symbol table. + private static final int SYSTEM_SYMBOLS_1_0_SIZE = SYSTEM_SYMBOLS_1_0.size(); + + // Symbol IDs for symbols contained in the system symbol table. + private static class SystemSymbolIDs { + + // The system symbol table SID for the text "$ion_symbol_table". + private static final int ION_SYMBOL_TABLE_ID = 3; + + // The system symbol table SID for the text "name". + private static final int NAME_ID = 4; + + // The system symbol table SID for the text "version". + private static final int VERSION_ID = 5; + + // The system symbol table SID for the text "imports". + private static final int IMPORTS_ID = 6; + + // The system symbol table SID for the text "symbols". + private static final int SYMBOLS_ID = 7; + + // The system symbol table SID for the text "max_id". + private static final int MAX_ID_ID = 8; + } + + // The size of the reusable UTF-8 decoding buffer. + private static final int UTF8_BUFFER_SIZE_IN_BYTES = 4 * 1024; + + // The final byte of the binary IVM. + private static final int IVM_FINAL_BYTE = 0xEA; + + // Isolates the highest bit in a byte. + private static final int HIGHEST_BIT_BITMASK = 0x80; + + // Isolates the lowest seven bits in a byte. + private static final int LOWER_SEVEN_BITS_BITMASK = 0x7F; + + // Isolates the lowest six bits in a byte. + private static final int LOWER_SIX_BITS_BITMASK = 0x3F; + + // The number of significant bits in each UInt byte. + private static final int VALUE_BITS_PER_UINT_BYTE = 8; + + // The number of significant bits in each VarUInt byte. + private static final int VALUE_BITS_PER_VARUINT_BYTE = 7; + + // An IonCatalog containing zero shared symbol tables. + private static final IonCatalog EMPTY_CATALOG = new SimpleCatalog(); + + // Initial capacity of the stack used to hold ContainerInfo. Each additional level of nesting in the data requires + // a new ContainerInfo. Depths greater than 8 will be rare. + private static final int CONTAINER_STACK_INITIAL_CAPACITY = 8; + + // Initial capacity of the ArrayList used to hold the symbol IDs of the annotations on the current value. + private static final int ANNOTATIONS_LIST_INITIAL_CAPACITY = 8; + + // Initial capacity of the ArrayList used to hold the text in the current symbol table. + private static final int SYMBOLS_LIST_INITIAL_CAPACITY = 128; + + // Single byte negative zero, represented as a VarInt. Often used in timestamp encodings to indicate unknown local + // offset. + private static final int VAR_INT_NEGATIVE_ZERO = 0xC0; + + // The number of bytes occupied by a Java int. + private static final int INT_SIZE_IN_BYTES = 4; + + // The number of bytes occupied by a Java long. + private static final int LONG_SIZE_IN_BYTES = 8; + + // The smallest negative 8-byte integer that can fit in a long is -0x80_00_00_00_00_00_00_00. + private static final int MOST_SIGNIFICANT_BYTE_OF_MIN_LONG = 0x80; + + // The largest positive 8-byte integer that can fit in a long is 0x7F_FF_FF_FF_FF_FF_FF_FF. + private static final int MOST_SIGNIFICANT_BYTE_OF_MAX_LONG = 0x7F; + + // The second-most significant bit in the most significant byte of a VarInt is the sign. + private static final int VAR_INT_SIGN_BITMASK = 0x40; + + // 32-bit floats must declare length 4. + private static final int FLOAT_32_BYTE_LENGTH = 4; + + // The InputStream that provides the binary Ion data. + private final InputStream inputStream; + + // Wrapper for the InputStream that ensures an entire top-level value is available. + private final IonReaderLookaheadBuffer lookahead; + + // Buffer that stores top-level values. + private final ResizingPipedInputStream buffer; + + // Converter between scalar types, allowing, for example, for a value encoded as an Ion float to be returned as a + // Java `long` via `IonReader.longValue()`. + private final _Private_ScalarConversions.ValueVariant scalarConverter; + + // Stack to hold container info. Stepping into a container results in a push; stepping out results in a pop. + private final _Private_RecyclingStack containerStack; + + // UTF-8 string decoder. + private final CharsetDecoder utf8CharsetDecoder = Charset.forName("UTF-8").newDecoder(); + + // The symbol IDs for the annotations on the current value. + private final List annotationSids; + + // True if the annotation iterator will be reused across values; otherwise, false. + private final boolean isAnnotationIteratorReuseEnabled; + + // Reusable iterator over the annotations on the current value. + private final AnnotationIterator annotationIterator; + + // The text representations of the symbol table that is currently in scope, indexed by symbol ID. If the element at + // a particular index is null, that symbol has unknown text. + private final List symbols; + + // The catalog used by the reader to resolve shared symbol table imports. + private final IonCatalog catalog; + + // The shared symbol tables imported by the local symbol table that is currently in scope. The key is the highest + // local symbol ID that resolves to a symbol contained in the value's symbol table. + private final TreeMap imports; + + // A map of symbol ID to SymbolToken representation. Because most use cases only require symbol text, this + // is used only if necessary to avoid imposing the extra expense on all symbol lookups. + private List symbolTokensById = null; + + // The highest local symbol ID that resolves to a symbol contained in a shared symbol table imported by the + // current local symbol table. + private int importMaxId; + + // The cached SymbolTable representation of the current local symbol table. Invalidated whenever a local + // symbol table is encountered in the stream. + private SymbolTable cachedReadOnlySymbolTable = null; + + // True if the current symbol table has already been transferred via pop_passed_symbol_table with the reader + // positioned at the current value; otherwise, false. + private boolean hasTransferredSymbolTable = false; + + // The symbol ID of the current value's field name, or -1 if the current value is not in a struct. + private int fieldNameSid = -1; + + // The major version of the Ion encoding currently being read. + private int majorVersion = 1; + + // The minor version of the Ion encoding currently being read. + private int minorVersion = 0; + + // The number of bytes of a lob value that the user has consumed, allowing for piecewise reads. + private int lobBytesRead = 0; + + // A reusable scratch space to hold decoded UTF-8 bytes. + private CharBuffer utf8DecodingBuffer = CharBuffer.allocate(UTF8_BUFFER_SIZE_IN_BYTES); + + // The type of value at which the reader is currently positioned. + private IonType valueType = null; + + // Information about the type ID byte for the value at which the reader is currently positioned. + private IonTypeID valueTypeID = null; + + // Indicates whether there are annotations on the current value. + private boolean hasAnnotations = false; + + // Indicates whether a complete top-level value is currenty buffered. + private boolean completeValueBuffered = false; + + // --- Byte position markers --- + // Note: absolute positions/indexes can be used because the bytes that represent a single top-level value are + // always handled in two sequential phases: first, the bytes are buffered, and then they are read. These operations + // will never be interleaved during the processing of a single value. As a result, the underlying buffer + // will always hold all of the bytes for a single top-level value in a contiguous sequence, even if the buffer + // has to grow to hold all of the value's bytes. + + // The buffer position of the first byte of the value representation (after the type ID and optional length field). + private int valueStartPosition = -1; + + // The buffer position of the byte after the last byte in the value representation. + private int valueEndPosition = -1; + + // The buffer position of the first byte of the annotation wrapper for the current value. + private int annotationStartPosition = -1; + + // The number of bytes occupied by the annotation SIDs in the annotation wrapper for the current value. + private int annotationsLength = -1; + + // The index of the next byte to peek from the underlying buffer. + private int peekIndex = -1; + + // ------ + + /** + * Constructor. + * @param builder the builder containing the configuration for the new reader. + * @param inputStream the InputStream that provides binary Ion data. + */ + IonReaderBinaryIncremental(IonReaderBuilder builder, InputStream inputStream) { + this.inputStream = inputStream; + this.catalog = builder.getCatalog() == null ? EMPTY_CATALOG : builder.getCatalog(); + if (builder.isAnnotationIteratorReuseEnabled()) { + isAnnotationIteratorReuseEnabled = true; + annotationIterator = new AnnotationIterator(); + } else { + isAnnotationIteratorReuseEnabled = false; + annotationIterator = null; + } + if (builder.getBufferConfiguration() == null) { + lookahead = new IonReaderLookaheadBuffer(STANDARD_BUFFER_CONFIGURATION, inputStream); + } else { + lookahead = new IonReaderLookaheadBuffer(builder.getBufferConfiguration(), inputStream); + } + buffer = (ResizingPipedInputStream) lookahead.getPipe(); + containerStack = new _Private_RecyclingStack( + CONTAINER_STACK_INITIAL_CAPACITY, + CONTAINER_INFO_FACTORY + ); + annotationSids = new ArrayList(ANNOTATIONS_LIST_INITIAL_CAPACITY); + symbols = new ArrayList(SYMBOLS_LIST_INITIAL_CAPACITY); + symbols.addAll(SYSTEM_SYMBOLS_1_0); + imports = new TreeMap(); + scalarConverter = new _Private_ScalarConversions.ValueVariant(); + resetImports(); + } + + /** + * Reusable iterator over the annotations on the current value. + */ + private class AnnotationIterator implements Iterator { + + // All of the annotation SIDs on the current value. + protected List annotationSids = Collections.emptyList(); + // The index into `annotationSids` containing the next annotation to be returned. + protected int index = 0; + + @Override + public boolean hasNext() { + return index < annotationSids.size(); + } + + @Override + public String next() { + int sid = annotationSids.get(index); + String annotation = getSymbol(sid); + if (annotation == null) { + throw new UnknownSymbolException(sid); + } + index++; + return annotation; + } + + @Override + public void remove() { + throw new UnsupportedOperationException("This iterator does not support element removal."); + } + + /** + * Reset the iterator so that it may be reused. + */ + public void reset() { + index = 0; + annotationSids = getAnnotationSids(); + } + } + + /** + * Non-reusable iterator over the annotations on the current value. May be iterated even if the reader advances + * past the current value. + */ + private class SingleUseAnnotationIterator extends AnnotationIterator { + + SingleUseAnnotationIterator() { + index = 0; + annotationSids = new ArrayList(getAnnotationSids()); + } + + @Override + public void reset() { + throw new IllegalStateException("Single-use annotation iterators cannot be reset."); + } + } + + /** + * A SymbolToken's import location, allowing for symbols with unknown text to be mapped to a particular slot + * in a shared symbol table. + * NOTE: this is currently not publicly accessible, but it is an important step toward being able to correctly + * round-trip symbols with unknown text from shared symbol tables in different symbol table contexts. See + * https://github.com/amzn/ion-java/issues/126 . Support is added now to avoid risking the appearance of performance + * degradation if ImportLocation support were added after initial release of this IonReader implementation. + */ + static class ImportLocation { + + // The name of the shared symbol table. + final String name; + + // The index into the shared symbol table. + final int sid; + + ImportLocation(String name, int sid) { + this.name = name; + this.sid = sid; + } + + public String getName() { + return name; + } + + public int getSid() { + return sid; + } + + @Override + public String toString() { + return String.format("ImportLocation::{name: %s, sid: %d}", name, sid); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof ImportLocation)) { + return false; + } + ImportLocation that = (ImportLocation) o; + return this.getName().equals(that.getName()) && this.getSid() == that.getSid(); + } + + @Override + public int hashCode() { + int result = 17; + result += 31 * getName().hashCode(); + result += 31 * getSid(); + return result; + } + } + + /** + * SymbolToken implementation that includes ImportLocation. + */ + static class SymbolTokenImpl implements _Private_SymbolToken { + + // The symbol's text, or null if the text is unknown. + private final String text; + + // The local symbol ID of this symbol within a particular local symbol table. + private final int sid; + + // The import location of the symbol (only relevant if the text is unknown). + private final ImportLocation importLocation; + + SymbolTokenImpl(String text, int sid, ImportLocation importLocation) { + this.text = text; + this.sid = sid; + this.importLocation = importLocation; + } + + @Override + public String getText() { + return text; + } + + @Override + public String assumeText() { + if (text == null) { + throw new UnknownSymbolException(sid); + } + return text; + } + + @Override + public int getSid() { + return sid; + } + + // Will be @Override once added to the SymbolToken interface. + public ImportLocation getImportLocation() { + return importLocation; + } + + @Override + public String toString() { + return String.format("SymbolToken::{text: %s, sid: %d, importLocation: %s}", text, sid, importLocation); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof SymbolToken)) return false; + + // NOTE: once ImportLocation is available via the SymbolToken interface, it should be compared here + // when text is null. + SymbolToken other = (SymbolToken) o; + if(getText() == null || other.getText() == null) { + return getText() == other.getText(); + } + return getText().equals(other.getText()); + } + + @Override + public int hashCode() { + if(getText() != null) return getText().hashCode(); + return 0; + } + } + + /** + * Read-only snapshot of the local symbol table at the reader's current position. + */ + private class LocalSymbolTableSnapshot implements SymbolTable { + + // The system symbol table. + private final SymbolTable system = SharedSymbolTable.getSystemSymbolTable(majorVersion); + + // The max ID of this local symbol table. + private final int maxId; + + // The max ID that maps to a shared symbol table imported by this local symbol table. + private final int importedTablesMaxId; + + // The shared symbol tables imported by this local symbol table. + private final SymbolTable[] importedTables; + + // Map representation of this symbol table. Keys are symbol text; values are the lowest symbol ID that maps + // to that text. + final Map mapView; + + // List representation of this symbol table, indexed by symbol ID. + final List listView; + + LocalSymbolTableSnapshot() { + int numberOfSymbols = symbols.size(); + maxId = numberOfSymbols - 1; + importedTablesMaxId = importMaxId; + // Map with initial size the number of symbols and load factor 1, meaning it must be full before growing. + // It is not expected to grow. + listView = new ArrayList(symbols.subList(0, numberOfSymbols)); + mapView = new HashMap((int) Math.ceil(numberOfSymbols / 0.75), 0.75f); + for (int i = 0; i < numberOfSymbols; i++) { + String symbol = listView.get(i); + if (symbol != null) { + mapView.put(symbol, i); + } + } + importedTables = new SymbolTable[imports.size()]; + int i = 0; + for (SymbolTable importedTable : imports.values()) { + importedTables[i] = importedTable; + i++; + } + } + + @Override + public String getName() { + return null; + } + + @Override + public int getVersion() { + return 0; + } + + @Override + public boolean isLocalTable() { + return true; + } + + @Override + public boolean isSharedTable() { + return false; + } + + @Override + public boolean isSubstitute() { + return false; + } + + @Override + public boolean isSystemTable() { + return false; + } + + @Override + public SymbolTable getSystemSymbolTable() { + return system; + } + + @Override + public String getIonVersionId() { + return system.getIonVersionId(); + } + + @Override + public SymbolTable[] getImportedTables() { + return importedTables; + } + + @Override + public int getImportedMaxId() { + return importedTablesMaxId; + } + + @Override + public SymbolToken find(String text) { + Integer sid = mapView.get(text); + if (sid == null) { + return null; + } + // The following per-call allocation is intentional. When weighed against the alternative of making + // 'mapView' a 'Map` instead of a `Map`, the following points should + // be considered: + // 1. A LocalSymbolTableSnapshot is only created when getSymbolTable() is called on the reader. The reader + // does not use the LocalSymbolTableSnapshot internally. There are two cases when getSymbolTable() would be + // called: a) when the user calls it, which will basically never happen, and b) when the user uses + // IonSystem.iterate over the reader, in which case each top-level value holds a reference to the symbol + // table that was in scope when it occurred. In case a), in addition to rarely being called at all, it + // would be even rarer for a user to use find() to retrieve each symbol (especially more than once) from the + // returned symbol table. Case b) may be called more frequently, but it remains equally rare that a user + // would retrieve each symbol at least once. + // 2. If we make mapView a Map, then we are guaranteeing that we will allocate at least + // one SymbolToken per symbol (because mapView is created in the constructor of LocalSymbolTableSnapshot) + // even though it's unlikely most will ever be needed. + return new SymbolTokenImpl(text, sid, null); + } + + @Override + public int findSymbol(String name) { + Integer sid = mapView.get(name); + if (sid == null) { + return UNKNOWN_SYMBOL_ID; + } + return sid; + } + + @Override + public String findKnownSymbol(int id) { + if (id < 0) { + throw new IllegalArgumentException("Symbol IDs must be at least 0."); + } + if (id >= symbols.size()) { + return null; + } + return listView.get(id); + } + + @Override + public Iterator iterateDeclaredSymbolNames() { + return new Iterator() { + + private int index = getImportedMaxId() + 1; + + @Override + public boolean hasNext() { + return index <= getMaxId(); + } + + @Override + public String next() { + String symbol = listView.get(index); + index++; + return symbol; + } + + @Override + public void remove() { + throw new UnsupportedOperationException("This iterator does not support element removal."); + } + }; + } + + @Override + public SymbolToken intern(String text) { + SymbolToken token = find(text); + if (token != null) { + return token; + } + throw new ReadOnlyValueException(); + } + + @Override + public int getMaxId() { + return maxId; + } + + @Override + public boolean isReadOnly() { + return true; + } + + @Override + public void makeReadOnly() { + // The symbol table is already read-only. + } + + @Override + public void writeTo(IonWriter writer) throws IOException { + IonReader reader = new SymbolTableReader(this); + writer.writeValues(reader); + } + + @Override + public String toString() { + return "(LocalSymbolTable max_id:" + getMaxId() + ')'; + } + } + + /** + * Throw if the reader is attempting to process an Ion version that it does not support. + */ + private void requireSupportedIonVersion() { + if (majorVersion != 1 || minorVersion != 0) { + throw new IonException(String.format("Unsupported Ion version: %d.%d", majorVersion, minorVersion)); + } + } + + /** + * Reset the local symbol table to the system symbol table. + */ + private void resetSymbolTable() { + // Note: when there is a new version of Ion, check majorVersion and minorVersion here and set the appropriate + // system symbol table. + symbols.clear(); + symbols.addAll(SYSTEM_SYMBOLS_1_0); + cachedReadOnlySymbolTable = null; + if (symbolTokensById != null) { + symbolTokensById.clear(); + } + } + + /** + * Clear the list of imported shared symbol tables. + */ + private void resetImports() { + imports.clear(); + SymbolTable system = SharedSymbolTable.getSystemSymbolTable(majorVersion); + importMaxId = system.getMaxId(); + } + + /** + * Add a shared symbol table import, resolving it from the catalog if possible. + * @param name the name of the shared symbol table. + * @param version the version of the shared symbol table. + * @param maxId the max_id of the shared symbol table. This value takes precedence over the actual max_id for the + * shared symbol table at the requested version. + */ + private void addImport(String name, int version, int maxId) { + SymbolTable shared = catalog.getTable(name, version); + importMaxId += maxId; + if (shared == null) { + // No match. All symbol IDs that fall within this shared symbol table's range will have unknown text. + imports.put(importMaxId, new SubstituteSymbolTable(name, version, maxId)); + } else if (shared.getMaxId() != maxId || shared.getVersion() != version) { + // Partial match. If the requested max_id exceeds the actual max_id of the resolved shared symbol table, + // symbol IDs that exceed the max_id of the resolved shared symbol table will have unknown text. + imports.put(importMaxId, new SubstituteSymbolTable(shared, version, maxId)); + } else { + // Exact match; the resolved shared symbol table may be used as-is. + imports.put(importMaxId, shared); + } + } + + /** + * Retrieves from the `imports` the next-lowest key to the given key, or `null` if there is no key lower + * than the given key. + * @param key the key. + * @return the lower key, or null. + */ + private Integer lowerKey(int key) { + // Note: with JDK 1.6+, this method is just `return imports.lowerKey(key);` + SortedMap sortedView = imports.headMap(key); + if (sortedView.isEmpty()) { + return null; + } + return sortedView.lastKey(); + } + + /** + * Gets the ImportLocation for the given local SID. The given SID must not point to a system symbol and must be + * less than or equal to `importMaxId`. + * @param sid the local SID. + * @return the ImportLocation for the given local SID. + */ + private ImportLocation getImportLocation(int sid) { + // The system symbol table is never included in the imports map, so the local SID must be adjusted lower + // by the max ID of the system symbol table. + int systemMaxId = SharedSymbolTable.getSystemSymbolTable(majorVersion).getMaxId(); + int systemAdjustedMaxId = sid - systemMaxId; + // Note: with JDK 1.6+, the following line would be + // Map.Entry entry = imports.ceilingEntry(systemAdjustedMaxId); + Map.Entry entry = imports.tailMap(systemAdjustedMaxId).entrySet().iterator().next(); + Integer previousMaxId = lowerKey(systemAdjustedMaxId); + if (previousMaxId == null) { + previousMaxId = systemMaxId; + } + return new ImportLocation(entry.getValue().getName(), sid - previousMaxId); + } + + /** + * Retrieves the String text for the given symbol ID. + * @param sid a symbol ID. + * @return a String. + */ + private String getSymbol(int sid) { + if (sid >= symbols.size()) { + throw new IonException("Symbol ID exceeds the max ID of the symbol table."); + } + return symbols.get(sid); + } + + /** + * Creates a SymbolToken representation of the given symbol ID. + * @param sid a symbol ID. + * @return a SymbolToken. + */ + private SymbolToken getSymbolToken(int sid) { + if (symbolTokensById == null) { + symbolTokensById = new ArrayList(symbols.size()); + } + if (symbolTokensById.size() < symbols.size()) { + for (int i = symbolTokensById.size(); i < symbols.size(); i++) { + symbolTokensById.add(null); + } + } + if (sid >= symbols.size()) { + throw new IonException("Symbol ID exceeds the max ID of the symbol table."); + } + SymbolToken token = symbolTokensById.get(sid); + if (token == null) { + String text = symbols.get(sid); + ImportLocation importLocation = null; + if (text == null) { + // Note: this will never be a system symbol. + if (sid > 0 && sid <= importMaxId) { + importLocation = getImportLocation(sid); + } else { + // All symbols with unknown text in the local symbol range are equivalent to symbol zero. + sid = 0; + } + } + token = new SymbolTokenImpl(text, sid, importLocation); + symbolTokensById.set(sid, token); + } + return token; + } + + /** + * Adds the symbols declared in the given shared symbol table to the reader's current local symbol table, stopping + * once the given max_id has been reached. + * @param shared a shared symbol table to import. + * @param maxId the maximum symbol ID to import from the shared symbol table. + */ + private void addSymbolsFromImport(SymbolTable shared, int maxId) { + // The system symbol table is never included in the imports map, so the local SID must be adjusted lower + // by the max ID of the system symbol table. + int systemMaxId = SharedSymbolTable.getSystemSymbolTable(majorVersion).getMaxId(); + Integer previousMaxId = lowerKey(maxId); + if (previousMaxId == null) { + previousMaxId = systemMaxId; + } + int adjustedMaxId = maxId - previousMaxId; + int id = 1; + Iterator importedSymbols = shared.iterateDeclaredSymbolNames(); + while (importedSymbols.hasNext() && id <= adjustedMaxId) { + symbols.add(importedSymbols.next()); + id++; + } + } + + /** + * Adds the symbols from all of the imported shared symbol tables to the reader's current local symbol table. + */ + private void addSymbolsFromImports() { + for (Map.Entry entry : imports.entrySet()) { + addSymbolsFromImport(entry.getValue(), entry.getKey()); + } + } + + /** + * Reads a local symbol table from the buffer. + * @param marker marker for the start and end positions of the local symbol table in the buffer. + */ + private void readSymbolTable(IonReaderLookaheadBuffer.SymbolTableMarker marker) { + peekIndex = marker.startIndex; + boolean isAppend = false; + boolean hasSeenImports = false; + boolean hasSeenSymbols = false; + int symbolsPosition = -1; + int symbolsEndPosition = -1; + while (peekIndex < marker.endIndex) { + fieldNameSid = readVarUInt(); + IonTypeID typeID = readTypeId(); + calculateEndPosition(typeID); + int currentValueEndPosition = valueEndPosition; + if (fieldNameSid == SystemSymbolIDs.IMPORTS_ID) { + if (hasSeenImports) { + throw new IonException("Symbol table contained multiple imports fields."); + } + if (typeID.type == IonType.SYMBOL) { + isAppend = readUInt(peekIndex, currentValueEndPosition) == SystemSymbolIDs.ION_SYMBOL_TABLE_ID; + peekIndex = currentValueEndPosition; + } else if (typeID.type == IonType.LIST) { + resetImports(); + stepIn(); + IonType type = next(); + while (type != null) { + String name = null; + int version = -1; + int maxId = -1; + if (type == IonType.STRUCT) { + stepIn(); + type = next(); + while (type != null) { + int fieldSid = getFieldId(); + if (fieldSid == SystemSymbolIDs.NAME_ID) { + if (type == IonType.STRING) { + name = stringValue(); + } + } else if (fieldSid == SystemSymbolIDs.VERSION_ID) { + if (type == IonType.INT) { + version = intValue(); + } + } else if (fieldSid == SystemSymbolIDs.MAX_ID_ID) { + if (type == IonType.INT) { + maxId = intValue(); + } + } + type = next(); + } + stepOut(); + } + addImport(name, version, maxId); + type = next(); + } + stepOut(); + } + if (!isAppend) { + // Clear the existing symbols before adding the new imported symbols. + resetSymbolTable(); + addSymbolsFromImports(); + } + hasSeenImports = true; + } else if (fieldNameSid == SystemSymbolIDs.SYMBOLS_ID) { + if (hasSeenSymbols) { + throw new IonException("Symbol table contained multiple symbols fields."); + } + if (typeID.type == IonType.LIST) { + // Just record this position and skip forward. Come back after the imports (if any) are parsed. + symbolsPosition = peekIndex; + symbolsEndPosition = currentValueEndPosition; + } + hasSeenSymbols = true; + } + peekIndex = currentValueEndPosition; + } + if (!hasSeenImports) { + resetSymbolTable(); + resetImports(); + } + if (symbolsPosition > -1) { + peekIndex = symbolsPosition; + valueType = IonType.LIST; + valueEndPosition = symbolsEndPosition; + stepIn(); + while (next() != null) { + if (valueType != IonType.STRING) { + symbols.add(null); + } else { + symbols.add(stringValue()); + } + } + stepOut(); + peekIndex = valueEndPosition; + } + } + + /** + * Advance the reader to the next top-level value. Buffers an entire top-level value, reads any IVMs and/or local + * symbol tables that precede the value, and sets the byte positions of important components of the value. + */ + private void nextAtTopLevel() { + if (completeValueBuffered) { + // There is already data buffered, but the user is choosing to skip it. + buffer.seekTo(valueEndPosition); + completeValueBuffered = false; + } + try { + lookahead.fillInput(); + } catch (Exception e) { + throw new IonException(e); + } + if (lookahead.moreDataRequired()) { + valueType = null; + valueTypeID = null; + return; + } + completeValueBuffered = true; + if (lookahead.getIvmIndex() > -1) { + peekIndex = lookahead.getIvmIndex(); + majorVersion = buffer.peek(peekIndex++); + minorVersion = buffer.peek(peekIndex++); + if (buffer.peek(peekIndex++) != IVM_FINAL_BYTE) { + throw new IonException("Invalid Ion version marker."); + } + requireSupportedIonVersion(); + resetSymbolTable(); + resetImports(); + lookahead.resetIvmIndex(); + } + List symbolTableMarkers = lookahead.getSymbolTableMarkers(); + if (!symbolTableMarkers.isEmpty()) { + // The cached SymbolTable (if any) is a snapshot in time, so it must be cleared whenever a new symbol + // table is read regardless of whether the new LST is an append or a reset. + cachedReadOnlySymbolTable = null; + for (IonReaderLookaheadBuffer.SymbolTableMarker symbolTableMarker : symbolTableMarkers) { + readSymbolTable(symbolTableMarker); + } + lookahead.resetSymbolTableMarkers(); + } + peekIndex = lookahead.getValueStart(); + if (lookahead.getAnnotationSids().isEmpty()) { + valueTypeID = lookahead.getValueTid(); + valueType = valueTypeID.type; + } else { + hasAnnotations = true; + valueTypeID = IonTypeID.TYPE_IDS[buffer.peek(peekIndex++)]; + int wrappedValueLength = valueTypeID.length; + if (valueTypeID.variableLength) { + wrappedValueLength = readVarUInt(); + } + valueType = valueTypeID.type; + if (valueType == IonTypeID.ION_TYPE_ANNOTATION_WRAPPER) { + throw new IonException("Nested annotations are invalid."); + } + if (peekIndex + wrappedValueLength != lookahead.getValueEnd()) { + throw new IonException("Mismatched annotation wrapper length."); + } + } + valueStartPosition = peekIndex; + valueEndPosition = lookahead.getValueEnd(); + lookahead.resetNopPadIndex(); + } + + /** + * Reads the type ID byte. + * @return the TypeAndLength descriptor for the type ID byte. + */ + private IonTypeID readTypeId() { + valueTypeID = IonTypeID.TYPE_IDS[buffer.peek(peekIndex++)]; + valueType = valueTypeID.type; + return valueTypeID; + } + + /** + * Calculates the end position for the given type ID descriptor. + * @param typeID the type ID descriptor. + */ + private void calculateEndPosition(IonTypeID typeID) { + if (typeID.variableLength) { + valueEndPosition = readVarUInt() + peekIndex; + } else { + valueEndPosition = typeID.length + peekIndex; + } + } + + @Override + public boolean hasNext() { + throw new UnsupportedOperationException("Not implemented"); + } + + /** + * Marks the end of the current container by indicating that the reader is no longer positioned on a value. + */ + private void endContainer() { + valueType = null; + valueTypeID = null; + annotationStartPosition = -1; + annotationsLength = -1; + hasAnnotations = false; + } + + /** + * Advance the reader to the next value within a container, which must already be buffered. + */ + private void nextBelowTopLevel() { + // Seek past the previous value. + if (peekIndex < valueEndPosition) { + peekIndex = valueEndPosition; + } + if (peekIndex >= containerStack.peek().endPosition) { + endContainer(); + } else { + if (containerStack.peek().type == IonType.STRUCT) { + fieldNameSid = readVarUInt(); + } + IonTypeID typeID = readTypeId(); + while (typeID.isNopPad) { + calculateEndPosition(typeID); + peekIndex = valueEndPosition; + if (peekIndex >= containerStack.peek().endPosition) { + endContainer(); + return; + } + if (containerStack.peek().type == IonType.STRUCT) { + fieldNameSid = readVarUInt(); + } + typeID = readTypeId(); + } + calculateEndPosition(typeID); + if (valueType == IonTypeID.ION_TYPE_ANNOTATION_WRAPPER) { + hasAnnotations = true; + annotationSids.clear(); + annotationsLength = readVarUInt(); + annotationStartPosition = peekIndex; + peekIndex = annotationStartPosition + annotationsLength; + typeID = readTypeId(); + if (typeID.isNopPad) { + throw new IonException( + "Invalid annotation wrapper: NOP pad may not occur inside an annotation wrapper." + ); + } + if (valueType == IonTypeID.ION_TYPE_ANNOTATION_WRAPPER) { + throw new IonException("Nested annotations are invalid."); + } + long annotationWrapperEndPosition = valueEndPosition; + calculateEndPosition(typeID); + if (annotationWrapperEndPosition != valueEndPosition) { + throw new IonException( + "Invalid annotation wrapper: end of the wrapper did not match end of the value." + ); + } + } else { + annotationStartPosition = -1; + annotationsLength = -1; + hasAnnotations = false; + if (valueEndPosition > containerStack.peek().endPosition) { + throw new IonException("Value overflowed its container."); + } + } + if (!valueTypeID.isValid) { + throw new IonException("Invalid type ID."); + } + valueStartPosition = peekIndex; + } + } + + @Override + public IonType next() { + fieldNameSid = -1; + lobBytesRead = 0; + valueStartPosition = -1; + hasTransferredSymbolTable = false; + hasAnnotations = false; + if (containerStack.isEmpty()) { + nextAtTopLevel(); + } else { + nextBelowTopLevel(); + } + // Note: the following check is necessary to catch empty ordered structs, which are prohibited by the spec. + // Unfortunately, this requires a check on every value for a condition that will probably never happen. + if ( + valueType == IonType.STRUCT && + valueTypeID.lowerNibble == IonTypeID.ORDERED_STRUCT_NIBBLE && + valueStartPosition == valueEndPosition + ) { + throw new IonException("Ordered struct must not be empty."); + } + return valueType; + } + + @Override + public void stepIn() { + if (!IonType.isContainer(valueType)) { + throw new IonException("Must be positioned on a container to step in."); + } + // Note: the IonReader interface dictates that stepping into a null container has the same behavior as + // an empty container. + ContainerInfo containerInfo = containerStack.push(); + containerInfo.type = valueType; + containerInfo.endPosition = valueEndPosition; + valueType = null; + valueTypeID = null; + valueEndPosition = -1; + fieldNameSid = -1; + valueStartPosition = -1; + } + + @Override + public void stepOut() { + if (containerStack.isEmpty()) { + // Note: this is IllegalStateException for consistency with the other binary IonReader implementation. + throw new IllegalStateException("Cannot step out at top level."); + } + ContainerInfo containerInfo = containerStack.pop(); + valueEndPosition = containerInfo.endPosition; + valueType = null; + valueTypeID = null; + fieldNameSid = -1; + valueStartPosition = -1; + } + + @Override + public int getDepth() { + return containerStack.size(); + } + + @Override + public SymbolTable getSymbolTable() { + if (cachedReadOnlySymbolTable == null) { + if (symbols.size() == SYSTEM_SYMBOLS_1_0_SIZE) { + cachedReadOnlySymbolTable = SharedSymbolTable.getSystemSymbolTable(majorVersion); + } else { + cachedReadOnlySymbolTable = new LocalSymbolTableSnapshot(); + } + } + return cachedReadOnlySymbolTable; + } + + @Override + public SymbolTable pop_passed_symbol_table() { + if (hasTransferredSymbolTable) { + // This symbol table has already been returned. Since the contract is that it is a "pop", it should not + // be returned twice. + return null; + } + hasTransferredSymbolTable = true; + return getSymbolTable(); + } + + @Override + public IonType getType() { + return valueType; + } + + @Override + public IntegerSize getIntegerSize() { + if (valueType != IonType.INT || isNullValue()) { + return null; + } + if (valueTypeID.length < INT_SIZE_IN_BYTES) { + // Note: this is conservative. Most integers of size 4 also fit in an int, but since exactly the + // same parsing code is used for ints and longs, there is no point wasting the time to determine the + // smallest possible type. + return IntegerSize.INT; + } else if (valueTypeID.length < LONG_SIZE_IN_BYTES) { + return IntegerSize.LONG; + } else if (valueTypeID.length == LONG_SIZE_IN_BYTES) { + // Because creating BigIntegers is so expensive, it is worth it to look ahead and determine exactly + // which 8-byte integers can fit in a long. + if (valueTypeID.isNegativeInt) { + // The smallest negative 8-byte integer that can fit in a long is -0x80_00_00_00_00_00_00_00. + int firstByte = buffer.peek(valueStartPosition); + if (firstByte < MOST_SIGNIFICANT_BYTE_OF_MIN_LONG) { + return IntegerSize.LONG; + } else if (firstByte > MOST_SIGNIFICANT_BYTE_OF_MIN_LONG) { + return IntegerSize.BIG_INTEGER; + } + for (int i = valueStartPosition + 1; i < valueEndPosition; i++) { + if (0x00 != buffer.peek(i)) { + return IntegerSize.BIG_INTEGER; + } + } + } else { + // The largest positive 8-byte integer that can fit in a long is 0x7F_FF_FF_FF_FF_FF_FF_FF. + if (buffer.peek(valueStartPosition) > MOST_SIGNIFICANT_BYTE_OF_MAX_LONG) { + return IntegerSize.BIG_INTEGER; + } + } + return IntegerSize.LONG; + } + return IntegerSize.BIG_INTEGER; + } + + /** + * Require that the given type matches the type of the current value. + * @param required the required type of current value. + */ + private void requireType(IonType required) { + if (required != valueType) { + // Note: this is IllegalStateException to match the behavior of the other binary IonReader implementation. + throw new IllegalStateException( + String.format("Invalid type. Required %s but found %s.", required, valueType) + ); + } + } + + /** + * Reads a VarUInt. + * @return the value. + */ + private int readVarUInt() { + int currentByte = 0; + int result = 0; + while ((currentByte & HIGHEST_BIT_BITMASK) == 0) { + currentByte = buffer.peek(peekIndex++); + result = (result << VALUE_BITS_PER_VARUINT_BYTE) | (currentByte & LOWER_SEVEN_BITS_BITMASK); + } + return result; + } + + /** + * Reads a UInt. + * @param limit the position of the first byte after the end of the UInt value. + * @return the value. + */ + private long readUInt(int startIndex, int limit) { + long result = 0; + for (int i = startIndex; i < limit; i++) { + result = (result << VALUE_BITS_PER_UINT_BYTE) | buffer.peek(i); + } + return result; + } + + /** + * Reads a UInt starting at `valueStartPosition` and ending at `valueEndPosition`. + * @return the value. + */ + private long readUInt() { + return readUInt(valueStartPosition, valueEndPosition); + } + + /** + * Reads a VarInt. + * @param firstByte the first byte of the VarInt representation, which has already been retrieved from the buffer. + * @return the value. + */ + private int readVarInt(int firstByte) { + int currentByte = firstByte; + int sign = (currentByte & VAR_INT_SIGN_BITMASK) == 0 ? 1 : -1; + int result = currentByte & LOWER_SIX_BITS_BITMASK; + while ((currentByte & HIGHEST_BIT_BITMASK) == 0) { + currentByte = buffer.peek(peekIndex++); + result = (result << VALUE_BITS_PER_VARUINT_BYTE) | (currentByte & LOWER_SEVEN_BITS_BITMASK); + } + return result * sign; + } + + /** + * Reads a VarInt. + * @return the value. + */ + private int readVarInt() { + return readVarInt(buffer.peek(peekIndex++)); + } + + // Scratch space for various byte sizes. Only for use while computing a single value. + private static final byte[][] SCRATCH_FOR_SIZE = new byte[][] { + new byte[0], + new byte[1], + new byte[2], + new byte[3], + new byte[4], + new byte[5], + new byte[6], + new byte[7], + new byte[8], + new byte[9], + new byte[10], + new byte[11], + new byte[12], + }; + + /** + * Copy the requested number of bytes from the buffer into a scratch buffer of exactly the requested length. + * @param startIndex the start index from which to copy. + * @param length the number of bytes to copy. + * @return the scratch byte array. + */ + private byte[] copyBytesToScratch(int startIndex, int length) { + // Note: using reusable scratch buffers makes reading ints and decimals 1-5% faster and causes much less + // GC churn. + byte[] bytes = null; + if (length < SCRATCH_FOR_SIZE.length) { + bytes = SCRATCH_FOR_SIZE[length]; + } + if (bytes == null) { + bytes = new byte[length]; + } + // The correct number of bytes will be requested from the buffer, so the limit is set at the capacity to + // avoid having to calculate a limit. + buffer.copyBytes(startIndex, bytes, 0, bytes.length); + return bytes; + } + + /** + * Reads a UInt value into a BigInteger. + * @param isNegative true if the resulting BigInteger value should be negative; false if it should be positive. + * @return the value. + */ + private BigInteger readUIntAsBigInteger(boolean isNegative) { + int length = valueEndPosition - valueStartPosition; + // NOTE: unfortunately, there is no BigInteger(int signum, byte[] bits, int offset, int length) constructor + // until JDK 9, so copying to scratch space is always required. Migrating to the new constructor will + // lead to a significant performance improvement. + byte[] magnitude = copyBytesToScratch(valueStartPosition, length); + int signum = isNegative ? -1 : 1; + return new BigInteger(signum, magnitude); + } + + /** + * Get and clear the most significant bit in the given byte array. + * @param intBytes bytes representing a signed int. + * @return -1 if the most significant bit was set; otherwise, 1. + */ + private int getAndClearSignBit(byte[] intBytes) { + boolean isNegative = (intBytes[0] & HIGHEST_BIT_BITMASK) != 0; + int signum = isNegative ? -1 : 1; + if (isNegative) { + intBytes[0] &= LOWER_SEVEN_BITS_BITMASK; + } + return signum; + } + + /** + * Reads an Int value into a BigInteger. + * @param limit the position of the first byte after the end of the UInt value. + * @return the value. + */ + private BigInteger readIntAsBigInteger(int limit) { + BigInteger value; + int length = limit - peekIndex; + if (length > 0) { + // NOTE: unfortunately, there is no BigInteger(int signum, byte[] bits, int offset, int length) constructor + // until JDK 9, so copying to scratch space is always required. Migrating to the new constructor will + // lead to a significant performance improvement. + byte[] bytes = copyBytesToScratch(peekIndex, length); + value = new BigInteger(getAndClearSignBit(bytes), bytes); + } + else { + value = BigInteger.ZERO; + } + return value; + } + + @Override + public long longValue() { + long value; + if (valueType == IonType.INT) { + if (valueTypeID.length == 0) { + return 0; + } + value = readUInt(); + if (valueTypeID.isNegativeInt) { + if (value == 0) { + throw new IonException("Int zero may not be negative."); + } + value *= -1; + } + } else if (valueType == IonType.FLOAT) { + scalarConverter.addValue(doubleValue()); + scalarConverter.setAuthoritativeType(_Private_ScalarConversions.AS_TYPE.double_value); + scalarConverter.cast(scalarConverter.get_conversion_fnid(_Private_ScalarConversions.AS_TYPE.long_value)); + value = scalarConverter.getLong(); + scalarConverter.clear(); + } else if (valueType == IonType.DECIMAL) { + scalarConverter.addValue(decimalValue()); + scalarConverter.setAuthoritativeType(_Private_ScalarConversions.AS_TYPE.decimal_value); + scalarConverter.cast(scalarConverter.get_conversion_fnid(_Private_ScalarConversions.AS_TYPE.long_value)); + value = scalarConverter.getLong(); + scalarConverter.clear(); + } else { + throw new IllegalStateException("longValue() may only be called on values of type int, float, or decimal."); + } + return value; + } + + @Override + public BigInteger bigIntegerValue() { + BigInteger value; + if (valueType == IonType.INT) { + if (isNullValue()) { + // NOTE: this mimics existing behavior, but should probably be undefined (as, e.g., longValue() is in this + // case). + return null; + } + if (valueTypeID.length == 0) { + return BigInteger.ZERO; + } + value = readUIntAsBigInteger(valueTypeID.isNegativeInt); + if (valueTypeID.isNegativeInt && value.signum() == 0) { + throw new IonException("Int zero may not be negative."); + } + } else if (valueType == IonType.FLOAT) { + if (isNullValue()) { + value = null; + } else { + scalarConverter.addValue(doubleValue()); + scalarConverter.setAuthoritativeType(_Private_ScalarConversions.AS_TYPE.double_value); + scalarConverter.cast(scalarConverter.get_conversion_fnid(_Private_ScalarConversions.AS_TYPE.bigInteger_value)); + value = scalarConverter.getBigInteger(); + scalarConverter.clear(); + } + } else if (valueType == IonType.DECIMAL) { + if (isNullValue()) { + value = null; + } else { + scalarConverter.addValue(decimalValue()); + scalarConverter.setAuthoritativeType(_Private_ScalarConversions.AS_TYPE.decimal_value); + scalarConverter.cast(scalarConverter.get_conversion_fnid(_Private_ScalarConversions.AS_TYPE.bigInteger_value)); + value = scalarConverter.getBigInteger(); + scalarConverter.clear(); + } + } else { + throw new IllegalStateException("longValue() may only be called on values of type int, float, or decimal."); + } + return value; + } + + @Override + public Date dateValue() { + Timestamp timestamp = timestampValue(); + if (timestamp == null) { + return null; + } + return timestamp.dateValue(); + } + + @Override + public int intValue() { + return (int) longValue(); + } + + @Override + public double doubleValue() { + double value; + if (valueType == IonType.FLOAT) { + int length = valueEndPosition - valueStartPosition; + if (length == 0) { + return 0.0d; + } + ByteBuffer bytes = buffer.getByteBuffer(valueStartPosition, valueEndPosition); + if (length == FLOAT_32_BYTE_LENGTH) { + value = bytes.getFloat(); + } else { + // Note: there is no need to check for other lengths here; the type ID byte is validated during next(). + value = bytes.getDouble(); + } + } else if (valueType == IonType.DECIMAL) { + scalarConverter.addValue(decimalValue()); + scalarConverter.setAuthoritativeType(_Private_ScalarConversions.AS_TYPE.decimal_value); + scalarConverter.cast(scalarConverter.get_conversion_fnid(_Private_ScalarConversions.AS_TYPE.double_value)); + value = scalarConverter.getDouble(); + scalarConverter.clear(); + } else { + throw new IllegalStateException("doubleValue() may only be called on values of type float or decimal."); + } + return value; + } + + /** + * Decodes a string from the buffer into a String value. + * @param valueStart the position in the buffer of the first byte in the string. + * @param valueEnd the position in the buffer of the last byte in the string. + * @return the value. + */ + private String readString(int valueStart, int valueEnd) { + ByteBuffer utf8InputBuffer = buffer.getByteBuffer(valueStart, valueEnd); + + int numberOfBytes = valueEnd - valueStart; + if (numberOfBytes > utf8DecodingBuffer.capacity()) { + utf8DecodingBuffer = CharBuffer.allocate(numberOfBytes); + } + + utf8DecodingBuffer.position(0); + utf8DecodingBuffer.limit(utf8DecodingBuffer.capacity()); + + utf8CharsetDecoder.reset(); + CoderResult coderResult = utf8CharsetDecoder.decode(utf8InputBuffer, utf8DecodingBuffer, true); + if (coderResult.isError()) { + throw new IonException("Illegal value encountered while validating UTF-8 data in input stream. " + coderResult.toString()); + } + utf8DecodingBuffer.flip(); + return utf8DecodingBuffer.toString(); + } + + @Override + public String stringValue() { + String value; + if (valueType == IonType.STRING) { + if (isNullValue()) { + return null; + } + value = readString(valueStartPosition, valueEndPosition); + } else if (valueType == IonType.SYMBOL) { + if (isNullValue()) { + return null; + } + int sid = (int) readUInt(); + value = getSymbol(sid); + if (value == null) { + throw new UnknownSymbolException(sid); + } + } else { + throw new IllegalStateException("Invalid type requested."); + } + return value; + } + + @Override + public SymbolToken symbolValue() { + requireType(IonType.SYMBOL); + if (isNullValue()) { + return null; + } + int sid = (int) readUInt(); + return getSymbolToken(sid); + } + + @Override + public int byteSize() { + if (!IonType.isLob(valueType) && !isNullValue()) { + throw new IonException("Reader must be positioned on a blob or clob."); + } + return valueEndPosition - valueStartPosition; + } + + @Override + public byte[] newBytes() { + byte[] bytes = new byte[byteSize()]; + // The correct number of bytes will be requested from the buffer, so the limit is set at the capacity to + // avoid having to calculate a limit. + buffer.copyBytes(valueStartPosition, bytes, 0, bytes.length); + return bytes; + } + + @Override + public int getBytes(byte[] bytes, int offset, int len) { + int length = Math.min(len, byteSize() - lobBytesRead); + // The correct number of bytes will be requested from the buffer, so the limit is set at the capacity to + // avoid having to calculate a limit. + buffer.copyBytes(valueStartPosition + lobBytesRead, bytes, offset, length); + lobBytesRead += length; + return length; + } + + /** + * Reads a decimal value as a BigDecimal. + * @return the value. + */ + private BigDecimal readBigDecimal() { + int length = valueEndPosition - peekIndex; + if (length == 0) { + return BigDecimal.ZERO; + } + int scale = -readVarInt(); + BigDecimal value; + if (length < LONG_SIZE_IN_BYTES) { + // No need to allocate a BigInteger to hold the coefficient. + long coefficient = 0; + int sign = 1; + if (peekIndex < valueEndPosition) { + int firstByte = buffer.peek(peekIndex++); + sign = (firstByte & HIGHEST_BIT_BITMASK) == 0 ? 1 : -1; + coefficient = firstByte & LOWER_SEVEN_BITS_BITMASK; + } + while (peekIndex < valueEndPosition) { + coefficient = (coefficient << VALUE_BITS_PER_UINT_BYTE) | buffer.peek(peekIndex++); + } + value = BigDecimal.valueOf(coefficient * sign, scale); + } else { + // The coefficient may overflow a long, so a BigInteger is required. + value = new BigDecimal(readIntAsBigInteger(valueEndPosition), scale); + } + return value; + } + + /** + * Reads a decimal value as a Decimal. + * @return the value. + */ + private Decimal readDecimal() { + int length = valueEndPosition - peekIndex; + if (length == 0) { + return Decimal.ZERO; + } + int scale = -readVarInt(); + BigInteger coefficient; + length = valueEndPosition - peekIndex; + if (length > 0) { + // NOTE: unfortunately, there is no BigInteger(int signum, byte[] bits, int offset, int length) constructor, + // so copying to scratch space is always required. + byte[] bits = copyBytesToScratch(peekIndex, length); + int signum = getAndClearSignBit(bits); + // NOTE: there is a BigInteger.valueOf(long unscaledValue, int scale) factory method that avoids allocating + // a BigInteger for coefficients that fit in a long. See its use in readBigDecimal() above. Unfortunately, + // it is not possible to use this for Decimal because the necessary BigDecimal constructor is + // package-private. If a compatible BigDecimal constructor is added in a future JDK revision, a + // corresponding factory method should be added to Decimal to enable this optimization. + coefficient = new BigInteger(signum, bits); + if (coefficient.signum() == 0 && signum < 0) { + return Decimal.negativeZero(scale); + } + } + else { + coefficient = BigInteger.ZERO; + } + return Decimal.valueOf(coefficient, scale); + } + + @Override + public BigDecimal bigDecimalValue() { + requireType(IonType.DECIMAL); + if (isNullValue()) { + return null; + } + peekIndex = valueStartPosition; + return readBigDecimal(); + } + + @Override + public Decimal decimalValue() { + requireType(IonType.DECIMAL); + if (isNullValue()) { + return null; + } + peekIndex = valueStartPosition; + return readDecimal(); + } + + @Override + public Timestamp timestampValue() { + requireType(IonType.TIMESTAMP); + if (isNullValue()) { + return null; + } + peekIndex = valueStartPosition; + int firstByte = buffer.peek(peekIndex++); + Integer offset = null; + if (firstByte != VAR_INT_NEGATIVE_ZERO) { + offset = readVarInt(firstByte); + } + int year = readVarUInt(); + int month = 0; + int day = 0; + int hour = 0; + int minute = 0; + int second = 0; + BigDecimal fractionalSecond = null; + Timestamp.Precision precision = Timestamp.Precision.YEAR; + if (peekIndex < valueEndPosition) { + month = readVarUInt(); + precision = Timestamp.Precision.MONTH; + if (peekIndex < valueEndPosition) { + day = readVarUInt(); + precision = Timestamp.Precision.DAY; + if (peekIndex < valueEndPosition) { + hour = readVarUInt(); + if (peekIndex >= valueEndPosition) { + throw new IonException("Timestamps may not specify hour without specifying minute."); + } + minute = readVarUInt(); + precision = Timestamp.Precision.MINUTE; + if (peekIndex < valueEndPosition) { + second = readVarUInt(); + precision = Timestamp.Precision.SECOND; + if (peekIndex < valueEndPosition) { + fractionalSecond = readBigDecimal(); + if (fractionalSecond.signum() < 0 || fractionalSecond.compareTo(BigDecimal.ONE) >= 0) { + throw new IonException("The fractional seconds value in a timestamp must be greater" + + "than or equal to zero and less than one."); + } + } + } + } + } + } + try { + return Timestamp.createFromUtcFields( + precision, + year, + month, + day, + hour, + minute, + second, + fractionalSecond, + offset + ); + } catch (IllegalArgumentException e) { + throw new IonException("Illegal timestamp encoding. ", e); + } + } + + /** + * Gets the annotation symbol IDs for the current value, reading them from the buffer first if necessary. + * @return the annotation symbol IDs, or an empty list if the current value is not annotated. + */ + private List getAnnotationSids() { + if (containerStack.isEmpty()) { + return lookahead.getAnnotationSids(); + } else { + if (annotationSids.isEmpty()) { + int savedPeekIndex = peekIndex; + peekIndex = annotationStartPosition; + long annotationsEndPosition = peekIndex + annotationsLength; + while (peekIndex < annotationsEndPosition) { + annotationSids.add(readVarUInt()); + } + peekIndex = savedPeekIndex; + } + return annotationSids; + } + } + + @Override + public String[] getTypeAnnotations() { + if (hasAnnotations) { + List annotationSids = getAnnotationSids(); + String[] annotationArray = new String[annotationSids.size()]; + for (int i = 0; i < annotationArray.length; i++) { + String symbol = getSymbol(annotationSids.get(i)); + if (symbol == null) { + throw new UnknownSymbolException(annotationSids.get(i)); + } + annotationArray[i] = symbol; + } + return annotationArray; + } + return _Private_Utils.EMPTY_STRING_ARRAY; + } + + @Override + public SymbolToken[] getTypeAnnotationSymbols() { + if (hasAnnotations) { + List annotationSids = getAnnotationSids(); + SymbolToken[] annotationArray = new SymbolToken[annotationSids.size()]; + for (int i = 0; i < annotationArray.length; i++) { + annotationArray[i] = getSymbolToken(annotationSids.get(i)); + } + return annotationArray; + } + return SymbolToken.EMPTY_ARRAY; + } + + private static final Iterator EMPTY_ITERATOR = new Iterator() { + + @Override + public boolean hasNext() { + return false; + } + + @Override + public String next() { + return null; + } + + @Override + public void remove() { + throw new UnsupportedOperationException("Cannot remove from an empty iterator."); + } + }; + + @Override + public Iterator iterateTypeAnnotations() { + if (hasAnnotations) { + if (isAnnotationIteratorReuseEnabled) { + annotationIterator.reset(); + return annotationIterator; + } else { + return new SingleUseAnnotationIterator(); + } + } + return EMPTY_ITERATOR; + } + + @Override + public int getFieldId() { + return fieldNameSid; + } + + @Override + public String getFieldName() { + if (fieldNameSid < 0) { + return null; + } + String fieldName = getSymbol(fieldNameSid); + if (fieldName == null) { + throw new UnknownSymbolException(fieldNameSid); + } + return fieldName; + } + + @Override + public SymbolToken getFieldNameSymbol() { + if (fieldNameSid < 0) { + return null; + } + return getSymbolToken(fieldNameSid); + } + + @Override + public boolean isNullValue() { + return valueTypeID != null && valueTypeID.isNull; + } + + @Override + public boolean isInStruct() { + return !containerStack.isEmpty() && containerStack.peek().type == IonType.STRUCT; + } + + @Override + public boolean booleanValue() { + requireType(IonType.BOOL); + return valueTypeID.lowerNibble == 1; + } + + @Override + public T asFacet(Class facetType) { + return null; + } + + @Override + public void close() throws IOException { + // NOTE: If we want to replace the other binary IonReader implementation with this one, the following + // validation could be performed in next() if incremental mode is not enabled. That would allow this + // implementation to behave in the same way as the other implementation when an incomplete value is + // encountered. + if (lookahead.isSkippingCurrentValue()) { + throw new IonException("Unexpected EOF."); + } + if (lookahead.available() > 0 && lookahead.moreDataRequired()) { + if (lookahead.getIvmIndex() < 0 + || lookahead.available() != _Private_IonConstants.BINARY_VERSION_MARKER_SIZE) { + throw new IonException("Unexpected EOF."); + } + } + inputStream.close(); + } + +} diff --git a/src/com/amazon/ion/impl/IonReaderLookaheadBuffer.java b/src/com/amazon/ion/impl/IonReaderLookaheadBuffer.java new file mode 100644 index 0000000000..325daa9990 --- /dev/null +++ b/src/com/amazon/ion/impl/IonReaderLookaheadBuffer.java @@ -0,0 +1,921 @@ +package com.amazon.ion.impl; + +import com.amazon.ion.IonBufferConfiguration; +import com.amazon.ion.IonException; +import com.amazon.ion.IonType; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +/** + * Monitors an InputStream over binary Ion data to ensure enough data is available to be navigated successfully by a + * non-incremental IonReader. + *

    + * Error reporting: this wrapper reads the least amount of Ion data possible in order to determine whether a value + * is complete. As such, it will not raise any errors if invalid data exists anywhere outside the header of a + * top-level value. Any such invalid data will be detected as normal by the IonReader. In the few cases where this + * wrapper does detect an error (e.g. upon finding the illegal type 0xF), it will raise {@link IonException}. + */ +public final class IonReaderLookaheadBuffer extends ReaderLookaheadBufferBase { + + private static final int LOWER_SEVEN_BITS_BITMASK = 0x7F; + private static final int HIGHEST_BIT_BITMASK = 0x80; + private static final int VALUE_BITS_PER_VARUINT_BYTE = 7; + // Note: because long is a signed type, Long.MAX_VALUE is represented in Long.SIZE - 1 bits. + private static final int MAXIMUM_SUPPORTED_VAR_UINT_BYTES = (Long.SIZE - 1) / VALUE_BITS_PER_VARUINT_BYTE; + private static final int IVM_START_BYTE = 0xE0; + private static final int IVM_REMAINING_LENGTH = 3; // Length of the IVM after the first byte. + private static final int ION_SYMBOL_TABLE_SID = 3; + + /** + * Represents a VarUInt that may be read in multiple steps. + */ + private static final class VarUInt { + + /** + * The location of the VarUInt in the value header. + */ + private enum Location { + /** + * The length field that is included when the low nibble of a value's type ID is VARIABLE_LENGTH_NIBBLE. + */ + VALUE_LENGTH, + /** + * The length field that is included when the low nibble of an annotation wrapper's type ID is + * VARIABLE_LENGTH_NIBBLE. + */ + ANNOTATION_WRAPPER_LENGTH, + /** + * The annot_length field that always precedes the SIDs in an annotation wrapper. Indicates the number + * of total bytes used to represent the SIDs that follow. + */ + ANNOTATION_WRAPPER_SIDS_LENGTH, + /** + * A symbol ID. An annotation wrapper may contain more than one. + */ + ANNOTATION_WRAPPER_SID + } + + /** + * The location. + */ + private VarUInt.Location location; + + /** + * The value accumulated so far. This will only be the actual value when `isComplete` is true. + */ + private long value; + + /** + * The number of bytes in the VarUInt representation that have been read so far. This is only the total + * number of bytes in the representation when `isComplete` is true. + */ + private int numberOfBytesRead; + + /** + * True when the VarUInt is complete; otherwise, false. + */ + private boolean isComplete; + + /** + * Constructor. + */ + private VarUInt() { + reset(Location.VALUE_LENGTH); + } + + /** + * Resets the value to zero. + * @param nextLocation the location of the next VarUInt to read. + */ + private void reset(final Location nextLocation) { + location = nextLocation; + value = 0; + numberOfBytesRead = 0; + isComplete = false; + } + } + + /** + * The state of the wrapper. + */ + private enum State { + /** + * Positioned before the type ID of a top-level value. + */ + BEFORE_TYPE_ID, + + /** + * Started reading a value's type ID, but did not finish because the byte was not yet available. + */ + READING_TYPE_ID, + + /** + * Reading the value's header, which includes all bytes between the type ID and the first byte of + * the value representation. + */ + READING_HEADER, + + /** + * Skipping over the value representation. + */ + SKIPPING_VALUE, + + /** + * Reading the type ID of a value annotated with $ion_symbol_table to determine whether it is a + * struct. + */ + READING_VALUE_WITH_SYMBOL_TABLE_ANNOTATION, + + /** + * Reading the length of a struct annotated with $ion_symbol_table. + */ + READING_SYMBOL_TABLE_LENGTH, + + /** + * There is nothing left to do. + */ + DONE + } + + /** + * Holds the start and end indices of a buffered symbol table. + */ + static class SymbolTableMarker { + /** + * Index of the first byte of the symbol table struct's contents. + */ + int startIndex; + + /** + * Index of the first byte after the end of the symbol table. + */ + int endIndex; + + /** + * @param startIndex index of the first byte of the symbol table struct's contents. + * @param length the number of bytes that remain in the symbol table struct after 'startIndex'. + */ + private SymbolTableMarker(final int startIndex, final int length) { + this.startIndex = startIndex; + this.endIndex = startIndex + length; + } + } + + /** + * The number of bytes to attempt to buffer each time more bytes are required. + */ + private final int pageSize; + + /** + * The VarUInt currently in progress. + */ + private final VarUInt inProgressVarUInt; + + /** + * Markers for any symbol tables that occurred in the stream between the last value and the current value. + */ + private final List symbolTableMarkers = new ArrayList(2); + + /** + * The symbol IDs of any annotations on the current value. + */ + private final List annotationSids = new ArrayList(3); + + /** + * The handler that will be notified when a symbol table exceeds the maximum buffer size. + */ + private final IonBufferConfiguration.OversizedSymbolTableHandler oversizedSymbolTableHandler; + + /** + * The number of additional bytes that must be read from `input` and stored in `pipe` before + * {@link #moreDataRequired()} can return false. + */ + private long additionalBytesNeeded; + + /** + * True if the current value is a system value (IVM, symbol table, or NOP pad), not a user value. + * `IonReader#next()` consumes any system values before the next user value, so the wrapper + * must be able to identify system values so that their bytes can be included in `pipe` before + * {@link #moreDataRequired()} returns false. + */ + private boolean isSystemValue; + + /** + * True if the current value has an annotation wrapper whose first annotation is `$ion_symbol_table`. + * The value will be deemed a system value if it is later determined to be a struct. + */ + private boolean isSymbolTableAnnotationFirst; + + /** + * The number of bytes of annotation SIDs left to read from the value's annotation wrapper. + */ + private long numberOfAnnotationSidBytesRemaining; + + /** + * The number of annotations in the annotation wrapper that have been processed so far. + */ + private long currentNumberOfAnnotations; + + /** + * The current state of the wrapper. + */ + private State state = State.BEFORE_TYPE_ID; + + /** + * The write index of the start of the current value. + */ + private int valueStartWriteIndex; + + /** + * The number of bytes available in the buffer if truncated to `valueStartWriteIndex`. + */ + private int valueStartAvailable; + + /** + * The read index of the type ID byte of the current value. + */ + private int valuePreHeaderIndex; + + /** + * The read index of the first byte of the value representation of the current value (past the type ID and the + * optional length field). + */ + private int valuePostHeaderIndex; + + /** + * The type ID byte of the current value. + */ + private IonTypeID valueTid; + + /** + * The index of the first byte after the end of the current value. + */ + private int valueEndIndex; + + /** + * The index of the first byte of the first no-op pad that precedes the current value. -1 indicates either that + * the current value was not preceded by no-op padding or that the space occupied by the no-op padding that preceded + * the current value has already been reclaimed. + */ + private int nopPadStartIndex = -1; + + /** + * The index of the second byte of the IVM. + */ + private int ivmSecondByteIndex = -1; + + /** + * The index of the next byte to peek from the buffer. + */ + private int peekIndex = 0; + + /** + * True if the event handler has not yet been notified if the current value is oversized. + */ + private boolean handlerNeedsToBeNotifiedOfOversizedValue = true; + + /** + * Resets the wrapper to the start of a new value. + */ + private void reset() { + additionalBytesNeeded = 0; + isSystemValue = false; + isSymbolTableAnnotationFirst = false; + numberOfAnnotationSidBytesRemaining = 0; + currentNumberOfAnnotations = 0; + valuePreHeaderIndex = -1; + valuePostHeaderIndex = -1; + valueTid = null; + valueEndIndex = -1; + annotationSids.clear(); + valueStartAvailable = pipe.available(); + startNewValue(); + } + + /** + * Constructs a wrapper with the given configuration. + * @param configuration the configuration for the new instance. + * @param inputStream an InputStream over binary Ion data. + */ + public IonReaderLookaheadBuffer(final IonBufferConfiguration configuration, final InputStream inputStream) { + super(configuration, inputStream); + pipe.registerNotificationConsumer( + new ResizingPipedInputStream.NotificationConsumer() { + @Override + public void bytesConsolidatedToStartOfBuffer(int leftShiftAmount) { + // The existing data in the buffer has been shifted to the start. Adjust the saved indexes + // accordingly. -1 indicates that all indices starting at 0 will be shifted. + shiftIndicesLeft(-1, leftShiftAmount); + } + } + ); + pageSize = configuration.getInitialBufferSize(); + oversizedSymbolTableHandler = configuration.getOversizedSymbolTableHandler(); + inProgressVarUInt = new VarUInt(); + reset(); + } + + /** + * Resets the `inProgressVarUInt`. + * @param location the VarUInt's location. + */ + private void initializeVarUInt(final VarUInt.Location location) { + inProgressVarUInt.reset(location); + state = State.READING_HEADER; + } + + /** + * Reads one byte, if possible. + * @return the byte, or -1 if none was available. + * @throws Exception if thrown by a handler method or if an IOException is thrown by the underlying InputStream. + */ + private int readByte() throws Exception { + if (pipe.availableBeyondBoundary() == 0 && fillPage(1) < 1) { + return -1; + } + int b; + if (isSkippingCurrentValue()) { + // If the value is being skipped, the byte will not have been buffered. + b = getInput().read(); + } else { + b = pipe.peek(peekIndex); + pipe.extendBoundary(1); + peekIndex++; + } + return b; + } + + /** + * Reads a VarUInt. NOTE: the VarUInt must fit in a `long`. This is not a true limitation, as IonJava requires + * VarUInts to fit in an `int`. + * @throws Exception if thrown by a handler method or if an IOException is thrown by the underlying InputStream. + */ + private void readVarUInt() throws Exception { + int currentByte; + while (inProgressVarUInt.numberOfBytesRead < MAXIMUM_SUPPORTED_VAR_UINT_BYTES) { + currentByte = readByte(); + if (currentByte < 0) { + return; + } + inProgressVarUInt.numberOfBytesRead++; + inProgressVarUInt.value = + (inProgressVarUInt.value << VALUE_BITS_PER_VARUINT_BYTE) | (currentByte & LOWER_SEVEN_BITS_BITMASK); + if ((currentByte & HIGHEST_BIT_BITMASK) != 0) { + inProgressVarUInt.isComplete = true; + dataHandler.onData(inProgressVarUInt.numberOfBytesRead); + return; + } + } + throw new IonException("Found a VarUInt that was too large to fit in a `long`"); + } + + /** + * Sets `additionalBytesNeeded` if and only if the value is not within an annotation wrapper. When the + * value is contained in an annotation wrapper, `additionalBytesNeeded` was set when reading the annotation + * wrapper's length and already includes the value's length. + * @param value the new value of `additionalBytesNeeded`. + * @param isUnannotated true if this type ID is not on a value within an annotation wrapper; false if it is. + */ + private void setAdditionalBytesNeeded(final long value, final boolean isUnannotated) { + if (isUnannotated) { + additionalBytesNeeded = value; + } + } + + /** + * Conveys the result of {@link #readTypeID(boolean)}. + */ + private enum ReadTypeIdResult { + /** + * The type ID is for a struct value. + */ + STRUCT, + /** + * The type ID is not for a struct value. + */ + NOT_STRUCT, + /** + * The type ID could not be read because there is no data available. `readTypeID` should be called + * again when more data is available. + */ + NO_DATA + } + + /** + * Reads the type ID byte. + * @param isUnannotated true if this type ID is not on a value within an annotation wrapper; false if it is. + * @return the result as a {@link ReadTypeIdResult}. + * @throws Exception if thrown by a handler method or if an IOException is thrown by the underlying InputStream. + */ + private ReadTypeIdResult readTypeID(final boolean isUnannotated) throws Exception { + int header = readByte(); + if (header < 0) { + return ReadTypeIdResult.NO_DATA; + } + valueTid = IonTypeID.TYPE_IDS[header]; + dataHandler.onData(1); + if (header == IVM_START_BYTE) { + if (!isUnannotated) { + throw new IonException("Invalid annotation header."); + } + additionalBytesNeeded = IVM_REMAINING_LENGTH; + isSystemValue = true; + // Encountering an IVM resets the symbol table context; no need to parse any previous symbol tables. + resetSymbolTableMarkers(); + ivmSecondByteIndex = peekIndex; + state = State.SKIPPING_VALUE; + } else if (!valueTid.isValid) { + throw new IonException("Invalid type ID."); + } else if (valueTid.type == IonType.BOOL) { + // bool values are always a single byte. + state = State.BEFORE_TYPE_ID; + } else if (valueTid.type == IonTypeID.ION_TYPE_ANNOTATION_WRAPPER) { + // Annotation. + if (valueTid.variableLength) { + initializeVarUInt(VarUInt.Location.ANNOTATION_WRAPPER_LENGTH); + } else { + setAdditionalBytesNeeded(valueTid.length, isUnannotated); + initializeVarUInt(VarUInt.Location.ANNOTATION_WRAPPER_SIDS_LENGTH); + } + } else { + if (valueTid.isNull) { + // null values are always a single byte. + state = State.BEFORE_TYPE_ID; + } else { + // Not null + if (valueTid.variableLength) { + initializeVarUInt(VarUInt.Location.VALUE_LENGTH); + } else { + setAdditionalBytesNeeded(valueTid.length, isUnannotated); + state = State.SKIPPING_VALUE; + } + } + } + if (valueTid.type == IonType.STRUCT) { + return ReadTypeIdResult.STRUCT; + } + return ReadTypeIdResult.NOT_STRUCT; + } + + /** + * Reads the bytes of the value header that occur after the type ID byte and before the first value byte. + * @throws Exception if thrown by a handler method or if an IOException is thrown by the underlying InputStream. + */ + private void readHeader() throws Exception { + if (inProgressVarUInt.location == VarUInt.Location.VALUE_LENGTH) { + readVarUInt(); + if (inProgressVarUInt.isComplete) { + additionalBytesNeeded = inProgressVarUInt.value; + state = State.SKIPPING_VALUE; + } + return; + } + if (inProgressVarUInt.location == VarUInt.Location.ANNOTATION_WRAPPER_LENGTH) { + readVarUInt(); + if (!inProgressVarUInt.isComplete) { + return; + } + additionalBytesNeeded = inProgressVarUInt.value; + initializeVarUInt(VarUInt.Location.ANNOTATION_WRAPPER_SIDS_LENGTH); + } + if (inProgressVarUInt.location == VarUInt.Location.ANNOTATION_WRAPPER_SIDS_LENGTH) { + readVarUInt(); + if (!inProgressVarUInt.isComplete) { + return; + } + additionalBytesNeeded -= inProgressVarUInt.numberOfBytesRead; + numberOfAnnotationSidBytesRemaining = inProgressVarUInt.value; + initializeVarUInt(VarUInt.Location.ANNOTATION_WRAPPER_SID); + } + if (inProgressVarUInt.location == VarUInt.Location.ANNOTATION_WRAPPER_SID) { + while (true) { + readVarUInt(); + if (inProgressVarUInt.isComplete) { + currentNumberOfAnnotations++; + if (currentNumberOfAnnotations == 1 && inProgressVarUInt.value == ION_SYMBOL_TABLE_SID) { + isSymbolTableAnnotationFirst = true; + } + annotationSids.add((int) inProgressVarUInt.value); + numberOfAnnotationSidBytesRemaining -= inProgressVarUInt.numberOfBytesRead; + additionalBytesNeeded -= inProgressVarUInt.numberOfBytesRead; + if (numberOfAnnotationSidBytesRemaining <= 0) { + state = State.SKIPPING_VALUE; + } else { + initializeVarUInt(VarUInt.Location.ANNOTATION_WRAPPER_SID); + continue; + } + if (isSymbolTableAnnotationFirst) { + state = State.READING_VALUE_WITH_SYMBOL_TABLE_ANNOTATION; + } + } + break; + } + } + } + + /** + * Shift all indices after 'afterIndex' left by the given amount. This is used when data is moved in the underlying + * buffer either due to buffer growth or NOP padding being reclaimed to make room for a value that would otherwise + * exceed the buffer's maximum size. + * @param afterIndex all indices after this index will be shifted (-1 indicates that all indices should be shifted). + * @param shiftAmount the amount to shift left. + */ + private void shiftIndicesLeft(int afterIndex, int shiftAmount) { + peekIndex = Math.max(peekIndex - shiftAmount, 0); + valuePreHeaderIndex -= shiftAmount; + valuePostHeaderIndex -= shiftAmount; + for (SymbolTableMarker symbolTableMarker : symbolTableMarkers) { + if (symbolTableMarker.startIndex > afterIndex) { + symbolTableMarker.startIndex -= shiftAmount; + symbolTableMarker.endIndex -= shiftAmount; + } + } + if (ivmSecondByteIndex > afterIndex) { + ivmSecondByteIndex -= shiftAmount; + } + } + + /** + * Reclaim the NOP padding that occurred before the current value, making space for the value in the buffer. + */ + private void reclaimNopPadding() { + pipe.consolidate(valuePreHeaderIndex, nopPadStartIndex); + shiftIndicesLeft(nopPadStartIndex, valuePreHeaderIndex - nopPadStartIndex); + resetNopPadIndex(); + } + + /** + * Skip bytes from the underlying InputStream without ever buffering them. + * @param numberOfBytesToSkip the number of bytes to attempt to skip. + * @return the number of bytes actually skipped. + * @throws IOException if thrown by the underlying InputStream. + */ + private int skipBytesFromInput(int numberOfBytesToSkip) throws IOException { + try { + return (int) getInput().skip(numberOfBytesToSkip); + } catch (EOFException e) { + // Certain InputStream implementations (e.g. GZIPInputStream) throw EOFException if more bytes are requested + // to skip than are currently available (e.g. if a header or trailer is incomplete). + return 0; + } + } + + /** + * Retrieve and buffer up to {@link #pageSize} bytes from the input. + * @param numberOfBytesRequested the minimum amount of space that must be available before the buffer reaches + * its configured maximum size. + * @return the number of bytes buffered by this operation. + * @throws Exception if thrown by the underlying InputStream. + */ + private int fillPage(int numberOfBytesRequested) throws Exception { + int amountToFill = pipe.capacity() - pipe.size(); + if (amountToFill <= 0) { + // Try to fill the remainder of the existing buffer to avoid growing unnecessarily. If there is no + // space, that indicates that a single value exceeds the size of a page. In that case, fill another page, + // growing the buffer only up to the configured maximum size. + int spaceAvailable = getMaximumBufferSize() - pipe.capacity(); + if (numberOfBytesRequested > spaceAvailable) { + if (nopPadStartIndex > -1 && valuePreHeaderIndex - nopPadStartIndex >= numberOfBytesRequested) { + // Reclaim the NOP pad space if doing so would allow the value to fit. + reclaimNopPadding(); + } else { + startSkippingValue(); + } + amountToFill = numberOfBytesRequested; + } else { + amountToFill = Math.min(pageSize, spaceAvailable); + } + } + int received; + if (isSkippingCurrentValue()) { + if (state == State.SKIPPING_VALUE) { + // This is a seek operation, meaning that the bytes don't need to be interpreted. + received = skipBytesFromInput(amountToFill); + } else { + // The bytes need to be interpreted, so they cannot be skipped. The caller must retrieve them from + // the input. + received = amountToFill; + } + } else { + received = pipe.receive(getInput(), amountToFill); + } + return received; + } + + /** + * Notifies the event handler that the current value is oversized, if the handler has not already been notified. + * @throws Exception if thrown by the handler. + */ + private void notifyHandlerOfOversizedValue() throws Exception { + if (handlerNeedsToBeNotifiedOfOversizedValue) { + if (isSystemValue) { + // Processing cannot continue after system values (symbol tables) are truncated because subsequent + // values may be unreadable. Notify the user. + oversizedSymbolTableHandler.onOversizedSymbolTable(); + } else { + // An oversized user value has been encountered. Notify the user so they can decide whether to continue + // or abort. + oversizedValueHandler.onOversizedValue(); + } + } + handlerNeedsToBeNotifiedOfOversizedValue = false; + } + + /** + * Attempt to retrieve at least `additionalBytesNeeded` bytes from the input and either buffer them (if the value + * is being consumed) or skip them (if the value is being skipped due to being oversize). + * @return the number of bytes filled or skipped. + * @throws Exception if thrown by the event handler. + */ + private int fillOrSkip() throws Exception { + // Clamping at the number of buffered bytes available guarantees that the buffer + // will never grow beyond its initial size. + int bytesRequested = (int) additionalBytesNeeded - pipe.availableBeyondBoundary(); + int bytesFilled; + if (isSkippingCurrentValue()) { + bytesFilled = skipBytesFromInput(bytesRequested); + } else { + bytesFilled = fillPage(bytesRequested); + } + if (bytesFilled < 1) { + return 0; + } + if (isSkippingCurrentValue()) { + // The user cannot be notified of a size violation until it has been determined whether + // the value is a symbol table or user value, which is only true in the SKIPPING_VALUE + // state. + notifyHandlerOfOversizedValue(); + // Skip all of the bytes skipped from the InputStream as well as all bytes previously + // buffered. + bytesFilled = bytesFilled + ((int) additionalBytesNeeded - bytesRequested); + } else { + bytesFilled = (int) Math.min(additionalBytesNeeded, bytesFilled); + pipe.extendBoundary(bytesFilled); + peekIndex += bytesFilled; + } + return bytesFilled; + } + + /* + * The state transitions of the fillInput() method are summarized by the following diagram. + * + * fillInput() + * | + * | +----------------------------------------------------+ + * | | | + * Read first byte of IVM +--------v---v-+ | + * +--------------------------+BEFORE_TYPE_ID<--+ | + * | Or length was inferred | +--+1-byte value(null or bool) read | + * | from the type ID +>+------+-------+ | + * | | | | + * | | |No bytes available | + * | 1-byte value | v | + * | | +----------------+ | + * | +-+READING_TYPE_ID <-+ | + * | | +-+No bytes available | + * | +------+---------+ | + * | |Read type-id of multi-byte value | + * | v | + * | Finished header +----------------+ | + * | +------------------------+READING_HEADER <--+ | + * | | | +--+Not enough bytes to complete header | + * | | +------+---------+ | + * | | |Read annotation wrapper with symbol table annotation | + * | | v | + * | | +---------------------------------------------+ | + * | | |READING_VALUE_WITH_SYMBOL_TABLE_ANNOTATION <--+ | + * | | | +--+No bytes available | + * | | +--+--------------+---------------------------+ | + * | | | |The wrapped value is a struct | + * | | The wrapped | v | + * | | value is not | +-----------------------------+ | + * | | a struct | | READING_SYMBOL_TABLE_LENGTH <--+ | + * | | | | +--+Not enough bytes to complete length | + * | | | +-----------+-----------------+ | + * | | | |Read length | + * | | | v | + * | | | +-----------------+ | + * | +--------------->+----->| SKIPPING_VALUE <--+ | + * | | +--+More bytes needed to complete skip | + * +------------------------>+-------+---------+ | + * | | + * |All bytes skipped | + * | | + * +----------------------------------------------------------+ + */ + @Override + protected void fillInputHelper() throws Exception { + while (true) { + if (state == State.BEFORE_TYPE_ID || state == State.READING_TYPE_ID) { + reset(); + state = State.READING_TYPE_ID; + if (readTypeID(true) != ReadTypeIdResult.NO_DATA) { + // The previous line transfers at most one byte, so the pre-header index is the write index minus + // one. + valuePostHeaderIndex = peekIndex; + valuePreHeaderIndex = valuePostHeaderIndex - 1; + valueStartWriteIndex = valuePreHeaderIndex; + } + } + if (state == State.READING_HEADER) { + readHeader(); + if (!inProgressVarUInt.isComplete) { + return; + } + valuePostHeaderIndex = peekIndex; + } + if (state == State.READING_VALUE_WITH_SYMBOL_TABLE_ANNOTATION) { + ReadTypeIdResult result = readTypeID(false); + if (result == ReadTypeIdResult.NO_DATA) { + return; + } + // When successful, readTypeID reads exactly one byte. + additionalBytesNeeded--; + if (result == ReadTypeIdResult.STRUCT) { + state = State.READING_SYMBOL_TABLE_LENGTH; + } else { + state = State.SKIPPING_VALUE; + } + } + if (state == State.READING_SYMBOL_TABLE_LENGTH) { + isSystemValue = true; + if (inProgressVarUInt.location == VarUInt.Location.VALUE_LENGTH) { + readVarUInt(); + if (!inProgressVarUInt.isComplete) { + return; + } + additionalBytesNeeded = inProgressVarUInt.value; + } + symbolTableMarkers.add(new SymbolTableMarker(peekIndex, (int) additionalBytesNeeded)); + state = State.SKIPPING_VALUE; + } + if (state == State.SKIPPING_VALUE) { + if (valueTid.isNopPad) { + if (pipe.availableBeyondBoundary() <= additionalBytesNeeded) { + // There cannot be any meaningful data beyond the NOP pad, so the buffer can be truncated + // immediately and the rest of the NOP pad skipped. + additionalBytesNeeded -= pipe.availableBeyondBoundary(); + startSkippingValue(); + // NOP padding will not be buffered, so it is never considered oversized. + handlerNeedsToBeNotifiedOfOversizedValue = false; + } + // Else, the rest of the NOP pad is already buffered, and there is a value at least partially + // buffered beyond it. The NOP pad will only be deleted from the buffer if the next value is + // large enough that it doesn't fit within the buffer's configured maximum size. + } + while (additionalBytesNeeded > 0) { + int numberOfBytesToRead; + if (pipe.availableBeyondBoundary() >= additionalBytesNeeded) { + numberOfBytesToRead = (int) additionalBytesNeeded; + pipe.extendBoundary(numberOfBytesToRead); + peekIndex += numberOfBytesToRead; + } else { + numberOfBytesToRead = fillOrSkip(); + if (numberOfBytesToRead < 1) { + return; + } + } + dataHandler.onData(numberOfBytesToRead); + additionalBytesNeeded -= numberOfBytesToRead; + } + state = State.BEFORE_TYPE_ID; + } + if (state == State.BEFORE_TYPE_ID) { + valueEndIndex = peekIndex; + if (isSystemValue || isSkippingCurrentValue() || valueTid.isNopPad) { + if (valueTid.isNopPad && nopPadStartIndex < 0) { + // This is the first NOP before the next value. Mark the start index in case the space needs to + // be reclaimed later. + nopPadStartIndex = valuePreHeaderIndex; + } + if (isSystemValue && isSkippingCurrentValue()) { + // The symbol table(s) currently buffered exceed the maximum buffer size. This is not + // recoverable; future invocations of fillInput() will do nothing. + reset(); + state = State.DONE; + } else { + if (isSystemValue && nopPadStartIndex > -1) { + // Reclaim any NOP pad space that precedes system values. This will usually not be strictly + // necessary, but it simplifies the implementation and will be rare in practice. Without + // this simplification, we would need to keep track of a list of NOP pad start/end indexes + // as we do with the symbol table markers. This way, we know that there can only be one + // uninterrupted run of NOP pad bytes immediately preceding any user value, making it easy + // to reclaim this space if necessary. + reclaimNopPadding(); + } + // Just skipped over system value or an oversized value. Consume the next value too so that a + // call to reader.next() won't return null. + continue; + } + } + } + break; + } + } + + @Override + void truncateToEndOfPreviousValue() { + peekIndex = valueStartWriteIndex; + pipe.truncate(valueStartWriteIndex, valueStartAvailable); + handlerNeedsToBeNotifiedOfOversizedValue = true; + } + + @Override + public boolean moreDataRequired() { + return pipe.available() <= 0 || state != State.BEFORE_TYPE_ID; + } + + /** + * Rewinds to the start of the value currently buffered. Does not include any system values that may precede + * the value. This method is not called in conjunction with {@link #mark()} / {@link #rewind()}, which may be + * used if the caller wants to rewind to the start of any system values that precede the current value. This + * method may be used to re-read the current value and may only be called after {@code IonReader.next()} + * has been called on the current value; otherwise, the data representing any system values that precede the + * current value would be lost. + * + * @throws IllegalStateException if there is no value currently buffered or if system value data would be lost + * as a result of calling this method before {@code IonReader.next()} was called. + */ + public void rewindToValueStart() { + if (valuePreHeaderIndex < 0) { + throw new IllegalStateException("A value must be buffered before calling rewindToValueStart()."); + } + int availableAtValueStart = pipe.getBoundary() - valuePreHeaderIndex; + // If rewinding would reduce the amount of data available, that indicates that system value data would be lost. + if (availableAtValueStart < available()) { + throw new IllegalStateException( + "IonReader.next() must be called on the current value before calling rewindToValueStart()." + ); + } + pipe.rewind(valuePreHeaderIndex, availableAtValueStart); + peekIndex = valuePreHeaderIndex; + } + + /** + * @return the index of the second byte of the IVM. + */ + int getIvmIndex() { + return ivmSecondByteIndex; + } + + /** + * Clears the IVM index. Should be called between user values. + */ + void resetIvmIndex() { + ivmSecondByteIndex = -1; + } + + /** + * Clears the NOP pad index. Should be called between user values. + */ + void resetNopPadIndex() { + nopPadStartIndex = -1; + } + + /** + * @return the index of the first byte of the value representation (past the type ID and the optional length field). + */ + int getValueStart() { + return valuePostHeaderIndex; + } + + /** + * @return the type ID of the current value. + */ + IonTypeID getValueTid() { + return valueTid; + } + + /** + * @return the index of the first byte after the end of the current value. + */ + int getValueEnd() { + return valueEndIndex; + } + + /** + * @return markers for any symbol tables that occurred in the stream between the last value and the current value. + */ + List getSymbolTableMarkers() { + return symbolTableMarkers; + } + + /** + * Clears the symbol table markers. + */ + void resetSymbolTableMarkers() { + symbolTableMarkers.clear(); + } + + /** + * @return the symbol IDs of any annotations on the current value. + */ + List getAnnotationSids() { + return annotationSids; + } + +} diff --git a/src/com/amazon/ion/impl/IonTypeID.java b/src/com/amazon/ion/impl/IonTypeID.java new file mode 100644 index 0000000000..d94cce0d6a --- /dev/null +++ b/src/com/amazon/ion/impl/IonTypeID.java @@ -0,0 +1,116 @@ +package com.amazon.ion.impl; + +import com.amazon.ion.IonType; + +/** + * Holds pre-computed information about a binary Ion type ID byte. + */ +final class IonTypeID { + + private static final int NUMBER_OF_BYTES = 0x100; + private static final int BITS_PER_NIBBLE = 4; + private static final int LOW_NIBBLE_BITMASK = 0x0F; + private static final int NULL_VALUE_NIBBLE = 0xF; + private static final int VARIABLE_LENGTH_NIBBLE = 0xE; + private static final int NEGATIVE_INT_TYPE_CODE = 0x3; + private static final int TYPE_CODE_INVALID = 0xF; + private static final int ANNOTATION_WRAPPER_MIN_LENGTH = 0x3; + private static final int ANNOTATION_WRAPPER_MAX_LENGTH = 0xE; + static final int ORDERED_STRUCT_NIBBLE = 0x1; + + // NOTE: 'annotation wrapper' is not an IonType, but it is simplest to treat it as one for the purposes of this + // implementation in order to have a direct mapping from binary type IDs to IonType enum values. IonType.DATAGRAM + // does not have a type ID, so we will use it to mean 'annotation wrapper' instead. + static final IonType ION_TYPE_ANNOTATION_WRAPPER = IonType.DATAGRAM; + + // Lookup table from type ID to IonType. See https://amzn.github.io/ion-docs/docs/binary.html#typed-value-formats + static final IonType[] ION_TYPES = new IonType[] { + IonType.NULL, + IonType.BOOL, + IonType.INT, + IonType.INT, + IonType.FLOAT, + IonType.DECIMAL, + IonType.TIMESTAMP, + IonType.SYMBOL, + IonType.STRING, + IonType.CLOB, + IonType.BLOB, + IonType.LIST, + IonType.SEXP, + IonType.STRUCT, + ION_TYPE_ANNOTATION_WRAPPER, + null // The 0xF type code is illegal in Ion 1.0. + }; + + // Pre-compute all possible type ID bytes. + static final IonTypeID[] TYPE_IDS; + static { + TYPE_IDS = new IonTypeID[NUMBER_OF_BYTES]; + for (int b = 0x00; b < NUMBER_OF_BYTES; b++) { + TYPE_IDS[b] = new IonTypeID((byte) b); + } + } + + final IonType type; + final byte length; + final boolean variableLength; + final boolean isNull; + final boolean isNopPad; + final byte lowerNibble; + final boolean isValid; + final boolean isNegativeInt; + + /** + * Determines whether the Ion spec allows this particular upperNibble/lowerNibble pair. + */ + private static boolean isValid(byte upperNibble, byte lowerNibble, IonType type) { + if (upperNibble == TYPE_CODE_INVALID) { + // Type code F is unused in Ion 1.0. + return false; + } + if (type == IonType.BOOL) { + // Bool values can only be false (0), true (1), or null (F). + return lowerNibble <= 1 || lowerNibble == NULL_VALUE_NIBBLE; + } + if (type == IonType.INT && upperNibble == NEGATIVE_INT_TYPE_CODE) { + // There is no negative zero int. + return lowerNibble != 0; + } + if (type == IonType.FLOAT) { + // Floats are either 0e0 (0), 32-bit (4), 64-bit (8), or null (F). + return lowerNibble == 0 || lowerNibble == 4 || lowerNibble == 8 || lowerNibble == NULL_VALUE_NIBBLE; + } + if (type == IonType.TIMESTAMP) { + // There is no zero-length timestamp representation. + return lowerNibble > 1; + } + if (type == ION_TYPE_ANNOTATION_WRAPPER) { + return lowerNibble >= ANNOTATION_WRAPPER_MIN_LENGTH && lowerNibble <= ANNOTATION_WRAPPER_MAX_LENGTH; + } + return true; + } + + private IonTypeID(byte id) { + byte upperNibble = (byte) ((id >> BITS_PER_NIBBLE) & LOW_NIBBLE_BITMASK); + this.lowerNibble = (byte) (id & LOW_NIBBLE_BITMASK); + this.type = ION_TYPES[upperNibble]; + this.isValid = isValid(upperNibble, lowerNibble, type); + this.isNull = lowerNibble == NULL_VALUE_NIBBLE; + this.isNopPad = type == IonType.NULL && !isNull; + byte length = lowerNibble; + if ((type == IonType.NULL && !isNopPad) || type == IonType.BOOL || !isValid) { + variableLength = false; + length = 0; + } else if (type == IonType.STRUCT && length == ORDERED_STRUCT_NIBBLE) { + variableLength = true; + } else { + variableLength = length == VARIABLE_LENGTH_NIBBLE; + } + if (isNull) { + length = 0; + } + this.isNegativeInt = type == IonType.INT && upperNibble == NEGATIVE_INT_TYPE_CODE; + this.length = length; + } +} diff --git a/src/com/amazon/ion/impl/ReaderLookaheadBuffer.java b/src/com/amazon/ion/impl/ReaderLookaheadBuffer.java new file mode 100644 index 0000000000..6264b31bd4 --- /dev/null +++ b/src/com/amazon/ion/impl/ReaderLookaheadBuffer.java @@ -0,0 +1,46 @@ +package com.amazon.ion.impl; + +import com.amazon.ion.IonReader; +import com.amazon.ion.system.IonReaderBuilder; + +/** + * Interface for lookahead buffers that enable incremental reading of streaming data. + */ +public interface ReaderLookaheadBuffer { + + /** + * If possible, fills the input pipe with enough bytes to enable one more successful call to + * {@link IonReader#next()} on the non-incremental reader attached to the pipe. If not enough bytes were available + * in the raw input stream to complete the next top-level user value, calling {@link #moreDataRequired()} after this + * method returns will return `true`. In this case, this method must be called again before calling + * `IonReader.next()` to position the reader on this value. Otherwise, `moreDataRequired()` will return `false` and + * a call to `IonReader.next()` may be made to position the reader on this value. Implementations may throw + * `IonException` if invalid Ion data is detected. Implementations may define exceptional cases. + * @throws Exception if an IOException is thrown by the underlying InputStream. + */ + void fillInput() throws Exception; + + /** + * Indicates whether more data must become present in the raw input stream before a successful call to + * {@link IonReader#next()} may be made to position the reader on the most-recently-buffered value. + * @return true if more data is required to complete the next top-level user value; otherwise, false. + */ + boolean moreDataRequired(); + + /** + * Indicates how many bytes are currently stored in the internal buffer. This can be used to detect whether + * calls to fillInput() are successfully retrieving data. + * @return The number of bytes waiting in the input buffer. + */ + int available(); + + /** + * Builds a reader over the input pipe. NOTE: because IonReader construction consumes bytes from the stream + * (to determine whether the stream is binary or text Ion), {@link #fillInput()} must have been called (such that + * {@link #moreDataRequired()} returns `false`) before calling this method. For the same reason, this method must + * only be called once per instance of this class. + * @param builder the builder containing the reader's configuration. + * @return a new IonReader. + */ + IonReader newIonReader(IonReaderBuilder builder); +} diff --git a/src/com/amazon/ion/impl/ReaderLookaheadBufferBase.java b/src/com/amazon/ion/impl/ReaderLookaheadBufferBase.java new file mode 100644 index 0000000000..2535b51750 --- /dev/null +++ b/src/com/amazon/ion/impl/ReaderLookaheadBufferBase.java @@ -0,0 +1,194 @@ +package com.amazon.ion.impl; + +import com.amazon.ion.BufferConfiguration; +import com.amazon.ion.IonReader; +import com.amazon.ion.system.IonReaderBuilder; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Base class for lookahead buffers that enable incremental reading of streaming data. + */ +abstract class ReaderLookaheadBufferBase implements ReaderLookaheadBuffer { + + /** + * An InputStream over binary Ion data. + */ + private final InputStream input; + + /** + * A buffer for the bytes required for a successful call to {@link IonReader#next()}. + */ + protected final ResizingPipedInputStream pipe; + + /** + * The maximum number of bytes that will be buffered. + */ + private final int maximumBufferSize; + + /** + * The handler that will be notified when a value exceeds the maximum buffer size. + */ + protected final BufferConfiguration.OversizedValueHandler oversizedValueHandler; + + /** + * The handler that will be notified when data is processed. + */ + protected final BufferConfiguration.DataHandler dataHandler; + + /** + * The current mark for the pipe's value of 'available'. + */ + private int markedAvailable; + + /** + * The current mark for the pipe's value of 'readIndex'. + */ + private int markedReadIndex; + + /** + * Indicates whether the current value is being skipped due to being oversized. + */ + private boolean isSkippingCurrentValue; + + /** + * Constructs a wrapper over the given stream using the given configuration. + * @param configuration the buffer configuration. + * @param inputStream an InputStream over Ion data. + */ + ReaderLookaheadBufferBase(final BufferConfiguration configuration, final InputStream inputStream) { + input = inputStream; + pipe = new ResizingPipedInputStream( + configuration.getInitialBufferSize(), + configuration.getMaximumBufferSize(), + true + ); + maximumBufferSize = configuration.getMaximumBufferSize(); + oversizedValueHandler = configuration.getOversizedValueHandler(); + dataHandler = configuration.getDataHandler(); + clearMark(); + } + + /** + * @inheritDoc + * @throws Exception if thrown by a handler method or if an IOException is thrown by the underlying InputStream. + */ + @Override + public final void fillInput() throws Exception { + clearMark(); + fillInputHelper(); + } + + /** + * Implements the behavior described in {@link #fillInput()}, except for clearing the mark. + * @throws IOException if thrown by the underlying input stream. + * @throws Exception if thrown by a handler method. + */ + protected abstract void fillInputHelper() throws Exception; + + @Override + public final int available() { + return pipe.available(); + } + + @Override + public final IonReader newIonReader(final IonReaderBuilder builder) { + return builder.build(pipe); + } + + /** + * Marks the current read position in the underlying buffer. This mark remains valid only until the next call + * to {@link #fillInput()}. + * @throws IllegalStateException if more data is required to complete a value. + */ + public final void mark() { + if (moreDataRequired()) { + throw new IllegalStateException("moreDataRequired() must be false before calling mark()."); + } + uncheckedMark(); + } + + /** + * Sets the mark without verifying that a complete value is buffered. + */ + final void uncheckedMark() { + markedAvailable = available(); + markedReadIndex = pipe.getReadIndex(); + } + + /** + * Rewinds the underlying buffer to the mark. + * @see #mark() + * @throws IllegalStateException if there is no valid mark. + */ + public final void rewind() { + if (markedReadIndex < 0 || markedAvailable < 0) { + throw new IllegalStateException("Must call mark() before rewind()."); + } + pipe.rewind(markedReadIndex, markedAvailable); + } + + /** + * Truncates the buffer to the end of the last complete value. + * @throws Exception if thrown by a handler. + */ + abstract void truncateToEndOfPreviousValue() throws Exception; + + /** + * Clears the mark. + * @see #mark() + */ + public final void clearMark() { + markedAvailable = -1; + markedReadIndex = -1; + } + + /** + * @return the underlying pipe. + */ + protected InputStream getPipe() { + return pipe; + } + + /** + * @return the underlying input. + */ + protected InputStream getInput() { + return input; + } + + /** + * Prepares for the start of a new value by clearing the {@link #isSkippingCurrentValue} flag. + */ + protected void startNewValue() { + isSkippingCurrentValue = false; + } + + /** + * Start skipping the current value, if it is not already being skipped. This should be called when the value + * is determined to be oversize. This truncates the buffer to the end of the previous value, reclaiming the space. + * @throws Exception if thrown by the event handler. + */ + protected void startSkippingValue() throws Exception { + if (!isSkippingCurrentValue) { + isSkippingCurrentValue = true; + truncateToEndOfPreviousValue(); + } + } + + /** + * Indicates whether the current value is being skipped due to being oversized. + * @return true if the value is being skipped; otherwise, false. + */ + protected boolean isSkippingCurrentValue() { + return isSkippingCurrentValue; + } + + /** + * @return the maximum size of the buffer. + */ + protected int getMaximumBufferSize() { + return maximumBufferSize; + } +} diff --git a/src/com/amazon/ion/impl/ResizingPipedInputStream.java b/src/com/amazon/ion/impl/ResizingPipedInputStream.java new file mode 100644 index 0000000000..0c7f8ba26d --- /dev/null +++ b/src/com/amazon/ion/impl/ResizingPipedInputStream.java @@ -0,0 +1,557 @@ +package com.amazon.ion.impl; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.BufferOverflowException; +import java.nio.ByteBuffer; + +/** + * Manages a resizing buffer for production and consumption of data within a single thread. + * Buffered bytes may be consumed through the InputStream interface. This provides a few benefits over using + * a PipedOutputStream/PipedInputStream pair in a single thread: + *

      + *
    1. There is no risk of deadlock. Piped streams, which are intended for producing data in one + * thread and consuming it in another, will block on read when no data is available and block on write + * when the buffer is full. In a single-threaded context, avoiding deadlock on read requires checking + * that bytes are available before every read. Avoiding deadlock on write would require checking that + * the buffer is not full before every write, but there is no built-in, publicly-accessible way of doing + * this with a PipedInputStream/PipedOutputStream.
    2. + *
    3. The buffer can grow. Piped streams use a fixed-size buffer that causes blocking when full. If + * used in a single-thread, this serves as a hard limit on the amount of data that can be written without + * a matching read. This can require arbitrary limits on data size to be imposed by the application. The + * ResizingPipedInputStream imposes no such limitation, but optionally allows for a maximum buffer + * size to be configured to protect against unbounded growth.
    4. + *
    + */ +public class ResizingPipedInputStream extends InputStream { + + /** + * Handler of notifications provided by the ResizingPipedInputStream. + */ + interface NotificationConsumer { + + /** + * Bytes have been shifted to the start of the buffer in order to make room for additional bytes + * to be buffered. + * @param leftShiftAmount the amount of the left shift (also: the pre-shift read index of the first shifted + * byte). + */ + void bytesConsolidatedToStartOfBuffer(int leftShiftAmount); + } + + /** + * A NotificationConsumer that does nothing. + */ + private static final NotificationConsumer NO_OP_NOTIFICATION_CONSUMER = new NotificationConsumer() { + @Override + public void bytesConsolidatedToStartOfBuffer(int leftShiftAmount) { + // Do nothing. + } + }; + + /** + * Mask to isolate a single byte. + */ + private static final int SINGLE_BYTE_MASK = 0xFF; + + /** + * The initial size of the buffer and the number of bytes by which the size of the buffer will increase + * each time it grows, unless it must grow by a smaller amount to fit within 'maximumBufferSize'. + */ + private final int initialBufferSize; + + /** + * The maximum size of the buffer. If the user attempts to buffer more bytes than this, an exception will be raised. + */ + private final int maximumBufferSize; + + /** + * Whether to use a boundary to limit the number of available bytes. This can be used to buffer + * arbitrarily-sized chunks of bytes without making them available for consumption. When true, + * the boundary must be manually extended (see {@link #extendBoundary(int)} to make these bytes + * available. When false, all buffered bytes will be available to read. + */ + private final boolean useBoundary; + + /** + * The NotificationConsumer currently registered. + */ + private NotificationConsumer notificationConsumer = NO_OP_NOTIFICATION_CONSUMER; + + /** + * The raw buffer. + */ + private byte[] buffer; + + /** + * View to the raw buffer. + */ + private ByteBuffer byteBuffer; + + /** + * @see #capacity() + */ + private int capacity; + + /** + * The index of the next byte in the buffer that is available to be read. Always less than or equal to `writeIndex`. + */ + private int readIndex = 0; + + /** + * The index at which the next byte received will be written. Always greater than or equal to `readIndex`. + */ + private int writeIndex = 0; + + /** + * @see #available() + */ + private int available = 0; + + /** + * @see #size() + */ + private int size = 0; + + /** + * @see #getBoundary() + */ + private int boundary = 0; + + /** + * Constructor. + * @param initialBufferSize the initial size of the buffer. When full, the buffer will grow by this + * many bytes. The buffer always stores bytes contiguously, so growth requires + * allocation of a new buffer capable of holding the new capacity and copying of the + * existing bytes into the new buffer. As such, a size should be chosen carefully + * such that growth is expected to occur rarely, if ever. + */ + public ResizingPipedInputStream(final int initialBufferSize) { + this(initialBufferSize, Integer.MAX_VALUE, false); + } + + /** + * Constructor. + * @param initialBufferSize the initial size of the buffer. When full, the buffer will grow by this + * many bytes. The buffer always stores bytes contiguously, so growth requires + * allocation of a new buffer capable of holding the new capacity and copying of the + * existing bytes into the new buffer. As such, a size should be chosen carefully + * such that growth is expected to occur rarely, if ever. + * @param maximumBufferSize the maximum size of the buffer. If a call to `receive` attempts to transfer an amount + * of bytes that would cause the buffer to exceed this size, a + * {@link BufferOverflowException} will be thrown. Must be greater than or equal to the + * initial buffer size. + * @param useBoundary whether to use a boundary to limit the number of available bytes. This can be used to buffer + * arbitrarily-sized chunks of bytes without making them available for consumption. When true, + * the boundary must be manually extended (see {@link #extendBoundary(int)} to make these bytes + * available. When false, all buffered bytes will be available to read. + */ + ResizingPipedInputStream(final int initialBufferSize, final int maximumBufferSize, final boolean useBoundary) { + if (initialBufferSize < 1) { + throw new IllegalArgumentException("Initial buffer size must be at least 1."); + } + if (maximumBufferSize < initialBufferSize) { + throw new IllegalArgumentException("Maximum buffer size cannot be less than the initial buffer size."); + } + this.initialBufferSize = initialBufferSize; + this.maximumBufferSize = maximumBufferSize; + this.capacity = initialBufferSize; + buffer = new byte[initialBufferSize]; + byteBuffer = ByteBuffer.wrap(buffer, 0, capacity); + this.useBoundary = useBoundary; + } + + /** + * Moves all buffered (but not yet read) bytes from 'buffer' to the destination buffer. In total, {@link #size()} + * bytes will be moved. + * @param destinationBuffer the destination buffer, which may be 'buffer' itself or a new buffer. + */ + private void moveBytesToStartOfBuffer(byte[] destinationBuffer) { + if (size > 0) { + System.arraycopy(buffer, readIndex, destinationBuffer, 0, size); + } + if (readIndex > 0) { + notificationConsumer.bytesConsolidatedToStartOfBuffer(readIndex); + } + readIndex = 0; + boundary = available; + writeIndex = size; + } + + /** + * @return the number of bytes that can be written at the end of the buffer. + */ + private int freeSpaceAtEndOfBuffer() { + return capacity - writeIndex; + } + + /** + * Ensures that there is at least 'minimumNumberOfBytesRequired' bytes of free space in the buffer, growing the + * buffer if necessary. May consolidate buffered bytes, performing an in-order copy and resetting indices + * such that the `readIndex` points to the same byte and the `writeIndex` is positioned after the last + * byte that is available to read. + * @param minimumNumberOfBytesRequired the minimum amount of free space that needs to be available for writing. + */ + private void ensureSpaceInBuffer(int minimumNumberOfBytesRequired) { + if (size < 1 || freeSpaceAtEndOfBuffer() < minimumNumberOfBytesRequired) { + int shortfall = minimumNumberOfBytesRequired - freeSpaceAtEndOfBuffer() - readIndex; + if (shortfall <= 0) { + // Free up space by moving any unread bytes to the start of the buffer and resetting the indices. + moveBytesToStartOfBuffer(buffer); + } else { + // There is not enough space in the buffer even though all available bytes have already been + // moved to the start of the buffer. Growth is required. + int amountToGrow = Math.max(initialBufferSize, shortfall); + if (capacity + amountToGrow > maximumBufferSize) { + amountToGrow = shortfall; + if (capacity + amountToGrow > maximumBufferSize) { + throw new BufferOverflowException(); + } + } + byte[] newBuffer = new byte[buffer.length + amountToGrow]; + moveBytesToStartOfBuffer(newBuffer); + capacity += amountToGrow; + buffer = newBuffer; + byteBuffer = ByteBuffer.wrap(buffer, readIndex, capacity); + } + } + } + + /** + * Buffers a single additional byte, growing the buffer if it is already full. + * @param b the byte to buffer. + */ + public void receive(final int b) { + ensureSpaceInBuffer(1); + buffer[writeIndex] = (byte) b; + writeIndex++; + size++; + if (!useBoundary) { + extendBoundary(1); + } + } + + /** + * Buffers `len` additional bytes, growing the buffer if it is already full or if it would become full + * by writing `len` bytes. + * @param b the bytes to buffer. + * @param off the offset into `b` that points to the first byte to buffer. + * @param len the number of bytes to buffer. + */ + public void receive(final byte[] b, final int off, final int len) { + ensureSpaceInBuffer(len); + System.arraycopy(b, off, buffer, writeIndex, len); + writeIndex += len; + size += len; + if (!useBoundary) { + extendBoundary(len); + } + } + + /** + * Buffers `b.length` additional bytes. + * @see #receive(byte[], int, int) + * @param b the bytes to buffer. + */ + public void receive(final byte[] b) { + receive(b, 0, b.length); + } + + /** + * Buffers up to `len` additional bytes, growing the buffer if it is already full or if it would become full + * by writing `len` bytes. This method will block if and only if the given `InputStream`'s + * {@link InputStream#read(byte[], int, int)} blocks when trying to read `len` bytes. If this is not desired, + * the caller should ensure that the given `InputStream` has at least `len` bytes available before calling + * this method or provide an InputStream implementation that does not block. + * @param input the source of the bytes. + * @param len the number of bytes to attempt to write. + * @return the number of bytes actually written, which will only be less than `len` if + * {@link InputStream#read(byte[], int, int)} returns less than `len`. + * @throws IOException if thrown by the given `InputStream` during read, except for {@link EOFException}. If an + * EOFException is thrown by the `InputStream`, it will be caught and this method will return the number of bytes + * that were received before the exception was thrown. + */ + public int receive(final InputStream input, final int len) throws IOException { + ensureSpaceInBuffer(len); + int numberOfBytesRead; + try { + numberOfBytesRead = input.read(buffer, writeIndex, len); + } catch (EOFException e) { + // Some InputStream implementations (such as GZIPInputStream) will throw EOFException instead of + // returning -1. + numberOfBytesRead = -1; + } + if (numberOfBytesRead > 0) { + writeIndex += numberOfBytesRead; + size += numberOfBytesRead; + } else { + numberOfBytesRead = 0; + } + if (!useBoundary) { + extendBoundary(numberOfBytesRead); + } + return numberOfBytesRead; + } + + /** + * {@inheritDoc} + *

    + * NOTE: This method adheres to the documented behavior of {@link InputStream#read(byte[], int, int)} + * except that it never blocks. If a read is attempted before the first write, + * this method will return -1. + */ + @Override + public int read(final byte[] b, final int off, final int len) { + if (b.length == 0 || len == 0) { + return 0; + } + if (available < 1) { + return -1; + } + int bytesToRead = Math.min(available, len); + System.arraycopy(buffer, readIndex, b, off, bytesToRead); + readIndex += bytesToRead; + available -= bytesToRead; + size -= bytesToRead; + return bytesToRead; + } + + /** + * Copies all of the available bytes in the buffer without changing the number of bytes available to subsequent + * reads. + * @param outputStream stream to which the bytes will be copied. + * @throws IOException if thrown by {@link OutputStream#write(byte[], int, int)}. + */ + public void copyTo(final OutputStream outputStream) throws IOException { + outputStream.write(buffer, readIndex, available); + } + + /** + * Seeks the read index to the given position. + * @param index the index to which to seek. Must not be negative. + */ + void seekTo(int index) { + int amount = index - readIndex; + available -= amount; + size -= amount; + readIndex = index; + } + + /** + * @return the current read index. + */ + int getReadIndex() { + return readIndex; + } + + /** + * @return the current write index. + */ + int getWriteIndex() { + return writeIndex; + } + + /** + * Rewinds the buffer to the given read index and sets 'available' to the given value. Subsequent + * behavior is undefined unless the values resulted from calling {@link #getReadIndex()} and + * {@link #available()} in immediate sequence, without any calls to 'receive' since. + * @param previousReadIndex the read index value to be set. + * @param previousAvailable the available value to be set. + */ + void rewind(final int previousReadIndex, final int previousAvailable) { + readIndex = previousReadIndex; + available = previousAvailable; + boundary = previousReadIndex + previousAvailable; + size = writeIndex - readIndex; + } + + /** + * Truncates the buffer to the given write index and sets both 'available' and 'size' to the to the given value. + * Subsequent behavior is undefined unless the values resulted from calling {@link #getWriteIndex()} and + * {@link #available()} in immediate sequence, without any calls to 'read' or 'skip' since. It is the caller's + * responsibility to ensure that calling this method will not result in loss of important data beyond the boundary. + * @param previousWriteIndex the write index value to be set. + * @param previousAvailable the available value to be set. + */ + void truncate(final int previousWriteIndex, final int previousAvailable) { + writeIndex = previousWriteIndex; + available = previousAvailable; + boundary = writeIndex; + size = previousAvailable; + } + + /** + * Skips up to `n` buffered bytes. Less than `n` bytes will be skipped if less than `n` bytes are + * available in the buffer. + * @param n the number of bytes to skip. + * @return the number of bytes actually skipped. + */ + @Override + public long skip(final long n) { + if (n < 1 || available < 1) { + return 0; + } + int bytesSkipped = (int) Math.min(available, n); + readIndex += bytesSkipped; + available -= bytesSkipped; + size -= bytesSkipped; + return bytesSkipped; + } + + /** + * {@inheritDoc} + *

    + * NOTE: This method adheres to the documented behavior of {@link InputStream#available()} + * except that it always returns the exact number of bytes that are available in the + * buffer. + * @return the exact number of bytes available in the buffer. + */ + @Override + public int available() { + return available; + } + + /** + * @return the number of bytes actually buffered, which will be greater than or equal to 'available' if a boundary + * has been set, or equal to `available` if no boundary has been set. + */ + int size() { + return size; + } + + /** + * @return the number of bytes buffered beyond the boundary. This is equivalent to subtracting {@link #available()} + * from {@link #size()}. + */ + int availableBeyondBoundary() { + return size - available; + } + + /** + * @return the index of the boundary, which is used to mark the last buffered byte that is available for reading. + * The boundary must always fall within [readIndex, writeIndex]. + */ + int getBoundary() { + return boundary; + } + + /** + * Extends the boundary by the given number of bytes. It is the caller's responsibility to ensure that the + * resulting boundary includes only bytes that have been buffered (i.e. that it does not exceed `writeIndex`). + * @param numberOfBytes the number of bytes by which the boundary should be extended. + */ + void extendBoundary(int numberOfBytes) { + boundary += numberOfBytes; + available += numberOfBytes; + } + + /** + * {@inheritDoc} + *

    + * NOTE: This method adheres to the documented behavior of {@link InputStream#read(byte[], int, int)} + * except that it never blocks. If a read is attempted before the first write, + * this method will return -1. + */ + @Override + public int read() { + if (available < 1) { + return -1; + } + int b = buffer[readIndex]; + readIndex++; + available--; + size--; + return b & SINGLE_BYTE_MASK; + } + + /** + * Peeks the byte at the given index without modifying any internal indexes. It is the caller's responsibility to + * ensure that the given index points to an available byte. + * @param index the index of the byte to peek. + * @return the byte value. + */ + int peek(int index) { + return buffer[index] & SINGLE_BYTE_MASK; + } + + /** + * @return the capacity of the buffer, which is always less than or equal to 'maximumBufferSize'. + */ + public int capacity() { + return capacity; + } + + /** + * @return the initial capacity of the buffer. + */ + int getInitialBufferSize() { + return initialBufferSize; + } + + /** + * Clears the buffer. + */ + void clear() { + readIndex = 0; + writeIndex = 0; + available = 0; + boundary = 0; + size = 0; + } + + /** + * Returns a ByteBuffer view of the underlying buffer. + * @param position the start position of the ByteBuffer. + * @param limit the limit of the ByteBuffer. + * @return a ByteBuffer. + */ + ByteBuffer getByteBuffer(int position, int limit) { + // Setting the limit to the capacity first is required because setting the position will fail if the new + // position is outside the limit. + byteBuffer.limit(capacity); + byteBuffer.position(position); + byteBuffer.limit(limit); + return byteBuffer; + } + + /** + * Copies bytes from the underlying buffer. It is the caller's responsibility to ensure the requested bytes + * are available. + * @param position the start position from which to read. + * @param destination the buffer to copy into. + * @param destinationOffset the offset of the buffer to copy into. + * @param length the number of bytes to copy. + */ + void copyBytes(int position, byte[] destination, int destinationOffset, int length) { + System.arraycopy(buffer, position, destination, destinationOffset, length); + } + + /** + * Moves all bytes starting at 'fromPosition' to 'toPosition', overwriting the bytes in-between. It is the caller's + * responsibility to ensure that the overwritten bytes are not needed. + * @param fromPosition the position to move bytes from. Must be less than or equal to 'writeIndex' and 'boundary'. + * @param toPosition the position to move bytes to. Must be greater than or equal to 'readIndex'. + */ + void consolidate(int fromPosition, int toPosition) { + if (fromPosition > writeIndex || fromPosition > boundary || toPosition < readIndex) { + throw new IllegalArgumentException("Tried to consolidate using an index that violates the constraints."); + } + int indexShift = fromPosition - toPosition; + System.arraycopy(buffer, fromPosition, buffer, toPosition, writeIndex - fromPosition); + size -= indexShift; + available -= indexShift; + writeIndex -= indexShift; + boundary -= indexShift; + // readIndex does not need to change, because none of the consolidated bytes have been read yet. + } + + /** + * Registers the given NotificationConsumer. + * @param consumer the NotificationConsumer to register. + */ + void registerNotificationConsumer(NotificationConsumer consumer) { + notificationConsumer = consumer; + } +} diff --git a/src/com/amazon/ion/impl/SymbolTokenImpl.java b/src/com/amazon/ion/impl/SymbolTokenImpl.java index 91abb3b8b0..b464e1aef0 100644 --- a/src/com/amazon/ion/impl/SymbolTokenImpl.java +++ b/src/com/amazon/ion/impl/SymbolTokenImpl.java @@ -15,8 +15,6 @@ package com.amazon.ion.impl; -import static com.amazon.ion.util.IonTextUtils.printString; - import com.amazon.ion.SymbolToken; import com.amazon.ion.UnknownSymbolException; @@ -76,18 +74,18 @@ public String toString() @Override public boolean equals(Object o) { if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (o == null || !(o instanceof SymbolToken)) return false; - SymbolTokenImpl other = (SymbolTokenImpl) o; - if(myText == null || other.myText == null){ - return myText == other.myText; + SymbolToken other = (SymbolToken) o; + if(getText() == null || other.getText() == null){ + return getText() == other.getText(); } - return myText.equals(other.myText); + return getText().equals(other.getText()); } @Override public int hashCode() { - if(myText != null) return myText.hashCode(); + if(getText() != null) return getText().hashCode(); return 0; } } diff --git a/src/com/amazon/ion/impl/_Private_IonReaderFactory.java b/src/com/amazon/ion/impl/_Private_IonReaderFactory.java index 8239e3ff50..4f4fafd623 100644 --- a/src/com/amazon/ion/impl/_Private_IonReaderFactory.java +++ b/src/com/amazon/ion/impl/_Private_IonReaderFactory.java @@ -19,12 +19,14 @@ import static com.amazon.ion.impl._Private_IonConstants.BINARY_VERSION_MARKER_SIZE; import static com.amazon.ion.util.IonStreamUtils.isIonBinary; +import com.amazon.ion.IonBufferConfiguration; import com.amazon.ion.IonCatalog; import com.amazon.ion.IonException; import com.amazon.ion.IonReader; import com.amazon.ion.IonSystem; import com.amazon.ion.IonTextReader; import com.amazon.ion.IonValue; +import com.amazon.ion.system.IonReaderBuilder; import com.amazon.ion.util.IonStreamUtils; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -253,6 +255,11 @@ public static final IonReader makeSystemReader(IonSystem system, return new IonReaderTreeSystem(value); } + public static final IonReader makeIncrementalReader(IonReaderBuilder builder, InputStream is) + { + return new IonReaderBinaryIncremental(builder, is); + } + //========================================================================= diff --git a/src/com/amazon/ion/impl/_Private_RecyclingStack.java b/src/com/amazon/ion/impl/_Private_RecyclingStack.java new file mode 100644 index 0000000000..a98e09d410 --- /dev/null +++ b/src/com/amazon/ion/impl/_Private_RecyclingStack.java @@ -0,0 +1,95 @@ +package com.amazon.ion.impl; + +import java.util.ArrayList; +import java.util.List; + +/** + * A stack whose elements are recycled. This can be useful when the stack needs to grow and shrink + * frequently and has a predictable maximum depth. + * @param the type of elements stored. + */ +public final class _Private_RecyclingStack { + + /** + * Factory for new stack elements. + * @param the type of element. + */ + public interface ElementFactory { + + /** + * @return a new instance. + */ + T newElement(); + } + + private final List elements; + private final ElementFactory elementFactory; + private int currentIndex; + private T top; + + /** + * @param initialCapacity the initial capacity of the underlying collection. + * @param elementFactory the factory used to create a new element on {@link #push()} when the stack has + * not previously grown to the new depth. + */ + public _Private_RecyclingStack(int initialCapacity, ElementFactory elementFactory) { + elements = new ArrayList(initialCapacity); + this.elementFactory = elementFactory; + currentIndex = -1; + top = null; + } + + /** + * Pushes an element onto the top of the stack, instantiating a new element only if the stack has not + * previously grown to the new depth. + * @return the element at the top of the stack after the push. This element must be initialized by the caller. + */ + public T push() { + currentIndex++; + if (currentIndex >= elements.size()) { + top = elementFactory.newElement(); + elements.add(top); + } else { + top = elements.get(currentIndex); + } + return top; + } + + /** + * @return the element at the top of the stack, or null if the stack is empty. + */ + public T peek() { + return top; + } + + /** + * Pops an element from the stack, retaining a reference to the element so that it can be reused the + * next time the stack grows to the element's depth. + * @return the element that was at the top of the stack before the pop, or null if the stack was empty. + */ + public T pop() { + T popped = top; + currentIndex--; + if (currentIndex >= 0) { + top = elements.get(currentIndex); + } else { + top = null; + currentIndex = -1; + } + return popped; + } + + /** + * @return true if the stack is empty; otherwise, false. + */ + public boolean isEmpty() { + return top == null; + } + + /** + * @return the number of elements on the stack. + */ + public int size() { + return currentIndex + 1; + } +} diff --git a/src/com/amazon/ion/impl/_Private_Utils.java b/src/com/amazon/ion/impl/_Private_Utils.java index 62785fa277..cf0e22f3d7 100644 --- a/src/com/amazon/ion/impl/_Private_Utils.java +++ b/src/com/amazon/ion/impl/_Private_Utils.java @@ -68,7 +68,7 @@ public final class _Private_Utils /** * Marker for code points relevant to removal of IonReader.hasNext(). */ - public static final boolean READER_HASNEXT_REMOVED = false; + public static final boolean READER_HASNEXT_REMOVED = true; /** Just a zero-length byte array, used to avoid allocation. */ @@ -871,9 +871,16 @@ public static SymbolTable initialSymtab(_Private_LocalSymbolTableFactory lstFact * Trampoline to * {@link LocalSymbolTableAsStruct#getIonRepresentation()}; */ - public static IonStruct symtabTree(SymbolTable symtab) + public static IonStruct symtabTree(SymbolTable symtab, ValueFactory valueFactory) { - return ((LocalSymbolTableAsStruct)symtab).getIonRepresentation(); + LocalSymbolTableAsStruct localSymbolTableAsStruct; + if (symtab instanceof LocalSymbolTableAsStruct) { + localSymbolTableAsStruct = (LocalSymbolTableAsStruct) symtab; + } else { + localSymbolTableAsStruct = (LocalSymbolTableAsStruct) new LocalSymbolTableAsStruct.Factory(valueFactory) + .newLocalSymtab(symtab.getSystemSymbolTable(), symtab.getImportedTables()); + } + return localSymbolTableAsStruct.getIonRepresentation(); } /** diff --git a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java index 4df80f7993..7d2e5606d6 100644 --- a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java +++ b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java @@ -46,6 +46,7 @@ import com.amazon.ion.SymbolTable; import com.amazon.ion.SymbolToken; import com.amazon.ion.Timestamp; +import com.amazon.ion.impl._Private_RecyclingStack; import com.amazon.ion.impl.bin.utf8.Utf8StringEncoder; import com.amazon.ion.impl.bin.utf8.Utf8StringEncoderPool; @@ -471,90 +472,6 @@ public String toString() FLUSH } - /** - * A stack whose elements are recycled. This can be useful when the stack needs to grow and shrink - * frequently and has a predictable maximum depth. - * @param the type of elements stored. - */ - private static final class RecyclingStack { - - /** - * Factory for new stack elements. - * @param the type of element. - */ - public interface ElementFactory { - - /** - * @return a new instance. - */ - T newElement(); - } - - private final List elements; - private final ElementFactory elementFactory; - private int currentIndex; - private T top; - - /** - * @param initialCapacity the initial capacity of the underlying collection. - * @param elementFactory the factory used to create a new element on {@link #push()} when the stack has - * not previously grown to the new depth. - */ - public RecyclingStack(int initialCapacity, ElementFactory elementFactory) { - elements = new ArrayList(initialCapacity); - this.elementFactory = elementFactory; - currentIndex = -1; - top = null; - } - - /** - * Pushes an element onto the top of the stack, instantiating a new element only if the stack has not - * previously grown to the new depth. - * @return the element at the top of the stack after the push. This element must be initialized by the caller. - */ - public T push() { - currentIndex++; - if (currentIndex >= elements.size()) { - top = elementFactory.newElement(); - elements.add(top); - } else { - top = elements.get(currentIndex); - } - return top; - } - - /** - * @return the element at the top of the stack, or null if the stack is empty. - */ - public T peek() { - return top; - } - - /** - * Pops an element from the stack, retaining a reference to the element so that it can be reused the - * next time the stack grows to the element's depth. - * @return the element that was at the top of the stack before the pop, or null if the stack was empty. - */ - public T pop() { - T popped = top; - currentIndex--; - if (currentIndex >= 0) { - top = elements.get(currentIndex); - } else { - top = null; - currentIndex = -1; - } - return popped; - } - - /** - * @return true if the stack is empty; otherwise, false. - */ - public boolean isEmpty() { - return top == null; - } - } - private static final int SID_UNASSIGNED = -1; private final BlockAllocator allocator; @@ -566,7 +483,7 @@ public boolean isEmpty() { private final WriteBuffer buffer; private final WriteBuffer patchBuffer; private final PatchList patchPoints; - private final RecyclingStack containers; + private final _Private_RecyclingStack containers; private int depth; private boolean hasWrittenValuesSinceFinished; private boolean hasWrittenValuesSinceConstructed; @@ -601,9 +518,9 @@ public boolean isEmpty() { this.buffer = new WriteBuffer(allocator); this.patchBuffer = new WriteBuffer(allocator); this.patchPoints = new PatchList(); - this.containers = new RecyclingStack( + this.containers = new _Private_RecyclingStack( 10, - new RecyclingStack.ElementFactory() { + new _Private_RecyclingStack.ElementFactory() { public ContainerInfo newElement() { return new ContainerInfo(); } diff --git a/src/com/amazon/ion/impl/lite/IonDatagramLite.java b/src/com/amazon/ion/impl/lite/IonDatagramLite.java index 75645fd862..9bb1a3d037 100644 --- a/src/com/amazon/ion/impl/lite/IonDatagramLite.java +++ b/src/com/amazon/ion/impl/lite/IonDatagramLite.java @@ -1040,7 +1040,7 @@ void load_current_symbol_table(IonValueLite prev_user_value) } else { IonSystem sys = __iterator.get_datagram_system(); - rep = _Private_Utils.symtabTree(new_symbol_table); + rep = _Private_Utils.symtabTree(new_symbol_table, sys); } assert(rep != null && __iterator.get_datagram_system() == rep.getSystem()); @@ -1147,7 +1147,7 @@ private static int count_system_values(IonSystem sys, int count = 0; while (curr.isLocalTable()) { count++; - curr = _Private_Utils.symtabTree(curr).getSymbolTable(); + curr = _Private_Utils.symtabTree(curr, sys).getSymbolTable(); } // we should terminate when the symbol tables symbol table is the system symbol table assert(curr != null); diff --git a/src/com/amazon/ion/system/IonReaderBuilder.java b/src/com/amazon/ion/system/IonReaderBuilder.java index 755cbe1fa8..dd2810639d 100644 --- a/src/com/amazon/ion/system/IonReaderBuilder.java +++ b/src/com/amazon/ion/system/IonReaderBuilder.java @@ -15,8 +15,10 @@ package com.amazon.ion.system; +import static com.amazon.ion.impl._Private_IonReaderFactory.makeIncrementalReader; import static com.amazon.ion.impl._Private_IonReaderFactory.makeReader; +import com.amazon.ion.IonBufferConfiguration; import com.amazon.ion.IonCatalog; import com.amazon.ion.IonException; import com.amazon.ion.IonReader; @@ -24,6 +26,11 @@ import com.amazon.ion.IonSystem; import com.amazon.ion.IonTextReader; import com.amazon.ion.IonValue; +import com.amazon.ion.impl._Private_IonConstants; +import com.amazon.ion.util.IonStreamUtils; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.Reader; @@ -42,6 +49,9 @@ public class IonReaderBuilder { private IonCatalog catalog = null; + private boolean isIncrementalReadingEnabled = false; + private IonBufferConfiguration bufferConfiguration = null; + private boolean isAnnotationIteratorReuseEnabled = true; private IonReaderBuilder() { @@ -50,6 +60,8 @@ private IonReaderBuilder() private IonReaderBuilder(IonReaderBuilder that) { this.catalog = that.catalog; + this.isIncrementalReadingEnabled = that.isIncrementalReadingEnabled; + this.bufferConfiguration = that.bufferConfiguration; } /** @@ -157,6 +169,172 @@ private IonCatalog validateCatalog() return catalog != null ? catalog : new SimpleCatalog(); } + /** + *

    + * Determines whether the IonReader will allow incremental reading of binary Ion data. When enabled, if + * {@link IonReader#next()} returns {@code null} at the top-level, it indicates that there is not enough data + * in the stream to complete a top-level value. The user may wait for more data to become available in the stream + * and call {@link IonReader#next()} again to continue reading. Unlike the non-incremental reader, the incremental + * reader will never throw an exception due to unexpected EOF during {@code next()}. If, however, + * {@link IonReader#close()} is called when an incomplete value is buffered, an {@link IonException} will be raised. + *

    + *

    + * There is currently no incremental text IonReader, so for text data a non-incremental IonReader will be + * returned regardless of the value of this option. If incremental text reading is supported in the future, it + * may be enabled via this option. + *

    + *

    + * When this option is enabled, auto-detection of GZIP data is not supported; the byte array or InputStream + * implementation provided to {@link #build(byte[])} or {@link #build(InputStream)} must return uncompressed bytes. + * This can be achieved by wrapping the data in a GZIPInputStream and passing it to {@link #build(InputStream)}. + *

    + *

    + * Additionally, when this option is enabled, annotation iterators are reused by default, improving performance. + * See {@link #withAnnotationIteratorReuseEnabled(boolean)} for more information and to disable that option. + *

    + *

    + * Although the incremental binary IonReader provides performance superior to the non-incremental binary IonReader + * for both incremental and non-incremental use cases, there is one caveat: the incremental implementation + * must be able to buffer an entire top-level value and any preceding system values (Ion version marker(s) and + * symbol table(s)) in memory. This will not be a problem for the vast majority of Ion streams, as it is rare for a + * single top-level value or symbol table to exceed a few megabytes in size. However, if the size of the stream's + * values risk exceeding the available memory, then this option must not be enabled. + *

    + * @param isEnabled true if the option is enabled; otherwise, false. + * + * @return this builder instance, if mutable; + * otherwise a mutable copy of this builder. + * + * @see #setIncrementalReadingEnabled() + * @see #setIncrementalReadingDisabled() + */ + public IonReaderBuilder withIncrementalReadingEnabled(boolean isEnabled) { + IonReaderBuilder b = mutable(); + if (isEnabled) { + b.setIncrementalReadingEnabled(); + } else { + b.setIncrementalReadingDisabled(); + } + return b; + } + + /** + * @see #withIncrementalReadingEnabled(boolean) + */ + public void setIncrementalReadingEnabled() { + mutationCheck(); + isIncrementalReadingEnabled = true; + } + + /** + * @see #withIncrementalReadingEnabled(boolean) + */ + public void setIncrementalReadingDisabled() { + mutationCheck(); + isIncrementalReadingEnabled = false; + } + + /** + * @see #withIncrementalReadingEnabled(boolean) + * @return true if incremental reading is enabled; otherwise, false. + */ + public boolean isIncrementalReadingEnabled() { + return isIncrementalReadingEnabled; + } + + /** + * Sets the buffer configuration. This can be used, for example, to set a maximum buffer size + * and receive notifications when values would exceed this size. Currently, this is ignored unless incremental + * reading has been enabled via {@link #withIncrementalReadingEnabled(boolean)}) or + * {@link #setIncrementalReadingEnabled()}. This configuration is optional. If not provided, the buffer size will + * be limited only by the available memory. + * + * @param configuration the configuration. + * + * @return this builder instance, if mutable; + * otherwise a mutable copy of this builder. + * + * @see #setBufferConfiguration(IonBufferConfiguration) + */ + public IonReaderBuilder withBufferConfiguration(IonBufferConfiguration configuration) { + IonReaderBuilder b = mutable(); + b.setBufferConfiguration(configuration); + return b; + } + + /** + * @see #withBufferConfiguration(IonBufferConfiguration) + */ + public void setBufferConfiguration(IonBufferConfiguration configuration) { + mutationCheck(); + bufferConfiguration = configuration; + } + + /** + * @see #withBufferConfiguration(IonBufferConfiguration) + * @return the current configuration. + */ + public IonBufferConfiguration getBufferConfiguration() { + return bufferConfiguration; + } + + /** + *

    + * Determines whether readers will reuse the annotation iterator returned by + * {@link IonReader#iterateTypeAnnotations()}. When enabled, the returned iterator remains valid only while the + * reader remains positioned at the current value; storing the iterator and iterating its values after that will + * cause undefined behavior. This provides improved performance and memory efficiency when frequently iterating + * annotations. When disabled, the returned iterator may be stored and used to retrieve the annotations that were + * on the value at the reader's position at the time of the call, regardless of where the reader is currently + * positioned. + *

    + *

    + * Currently, this option only has an effect when incremental reading is enabled (see + * {@link #withIncrementalReadingEnabled(boolean)}). In that case, it is enabled by default. Non-incremental readers + * always act as if this option were disabled. + *

    + * @param isEnabled true if the option is enabled; otherwise, false. + * + * @return this builder instance, if mutable; + * otherwise a mutable copy of this builder. + * + * @see #setAnnotationIteratorReuseEnabled() + * @see #setAnnotationIteratorReuseDisabled() + */ + public IonReaderBuilder withAnnotationIteratorReuseEnabled(boolean isEnabled) { + IonReaderBuilder b = mutable(); + if (isEnabled) { + b.setAnnotationIteratorReuseEnabled(); + } else { + b.setAnnotationIteratorReuseDisabled(); + } + return b; + } + + /** + * @see #withAnnotationIteratorReuseEnabled(boolean) + */ + public void setAnnotationIteratorReuseEnabled() { + mutationCheck(); + isAnnotationIteratorReuseEnabled = true; + } + + /** + * @see #withAnnotationIteratorReuseEnabled(boolean) + */ + public void setAnnotationIteratorReuseDisabled() { + mutationCheck(); + isAnnotationIteratorReuseEnabled = false; + } + + /** + * @see #withAnnotationIteratorReuseEnabled(boolean) + * @return true if annotation iterator reuse is enabled; otherwise, false. + */ + public boolean isAnnotationIteratorReuseEnabled() { + return isAnnotationIteratorReuseEnabled; + } + /** * Based on the builder's configuration properties, creates a new IonReader * instance over the given block of Ion data, detecting whether it's text or @@ -175,7 +353,7 @@ private IonCatalog validateCatalog() */ public IonReader build(byte[] ionData) { - return makeReader(validateCatalog(), ionData); + return build(ionData, 0, ionData.length); } /** @@ -198,6 +376,15 @@ public IonReader build(byte[] ionData) */ public IonReader build(byte[] ionData, int offset, int length) { + if (isIncrementalReadingEnabled) { + if (IonStreamUtils.isGzip(ionData, offset, length)) { + throw new IllegalArgumentException("Automatic GZIP detection is not supported with incremental" + + "support enabled. Wrap the bytes with a GZIPInputStream and call build(InputStream)."); + } + if (IonStreamUtils.isIonBinary(ionData, offset, length)) { + return makeIncrementalReader(this, new ByteArrayInputStream(ionData, offset, length)); + } + } return makeReader(validateCatalog(), ionData, offset, length); } @@ -223,7 +410,29 @@ public IonReader build(byte[] ionData, int offset, int length) */ public IonReader build(InputStream ionData) { - return makeReader(validateCatalog(), ionData); + InputStream wrapper = ionData; + if (isIncrementalReadingEnabled) { + if (!ionData.markSupported()) { + wrapper = new BufferedInputStream(ionData); + } + wrapper.mark(_Private_IonConstants.BINARY_VERSION_MARKER_SIZE); + byte[] possibleIVM = new byte[_Private_IonConstants.BINARY_VERSION_MARKER_SIZE]; + int bytesRead; + try { + bytesRead = wrapper.read(possibleIVM); + wrapper.reset(); + } catch (IOException e) { + throw new IonException(e); + } + if (IonStreamUtils.isGzip(possibleIVM, 0, possibleIVM.length)) { + throw new IllegalArgumentException("Automatic GZIP detection is not supported with incremental" + + "support enabled. Wrap the bytes with a GZIPInputStream and call build(InputStream)."); + } + if (possibleIVM.length == bytesRead && IonStreamUtils.isIonBinary(possibleIVM)) { + return makeIncrementalReader(this, wrapper); + } + } + return makeReader(validateCatalog(), wrapper); } /** diff --git a/test/AllTests.java b/test/AllTests.java index 0e433b5b8b..6817e13d54 100644 --- a/test/AllTests.java +++ b/test/AllTests.java @@ -49,6 +49,9 @@ import com.amazon.ion.NullTest; import com.amazon.ion.RawValueSpanReaderBasicTest; import com.amazon.ion.impl.IonReaderBinaryRawLargeStreamTest; +import com.amazon.ion.impl.IonReaderLookaheadBufferTest; +import com.amazon.ion.impl.ResizingPipedInputStreamTest; +import com.amazon.ion.impl.IonReaderBinaryIncrementalTest; import com.amazon.ion.impl.RawValueSpanReaderTest; import com.amazon.ion.RoundTripTest; import com.amazon.ion.SexpTest; @@ -209,6 +212,11 @@ IonReaderBuilderTest.class, IonReaderBinaryRawLargeStreamTest.class, + // non-blocking binary reader tests + IonReaderBinaryIncrementalTest.class, + ResizingPipedInputStreamTest.class, + IonReaderLookaheadBufferTest.class, + // experimental binary writer tests PooledBlockAllocatorProviderTest.class, WriteBufferTest.class, diff --git a/test/com/amazon/ion/BinaryIonReaderIteratorSystemProcessingTest.java b/test/com/amazon/ion/BinaryIonReaderIteratorSystemProcessingTest.java new file mode 100644 index 0000000000..f363b7c6b0 --- /dev/null +++ b/test/com/amazon/ion/BinaryIonReaderIteratorSystemProcessingTest.java @@ -0,0 +1,35 @@ +package com.amazon.ion; + +import java.util.Iterator; + +public class BinaryIonReaderIteratorSystemProcessingTest + extends IteratorSystemProcessingTestCase +{ + private byte[] myBytes; + + @Override + protected void prepare(String text) + { + myMissingSymbolTokensHaveText = false; + myBytes = encode(text); + } + + @Override + protected Iterator iterate() + { + return system().iterate(getStreamingMode().newIonReader(system().getCatalog(), myBytes)); + } + + @Override + protected Iterator systemIterate() + { + return system().systemIterate(system().newSystemReader(myBytes)); + } + + @Override + protected int expectedLocalNullSlotSymbolId() { + // The spec allows for implementations to treat these malformed symbols as "null slots", and all "null slots" + // in local symbol tables as equivalent to symbol zero. + return getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL ? 0 : 10; + } +} diff --git a/test/com/amazon/ion/BinaryReaderSystemProcessingTest.java b/test/com/amazon/ion/BinaryReaderSystemProcessingTest.java index c1894984bb..3373712a93 100644 --- a/test/com/amazon/ion/BinaryReaderSystemProcessingTest.java +++ b/test/com/amazon/ion/BinaryReaderSystemProcessingTest.java @@ -38,7 +38,7 @@ protected void prepare(String text) @Override public IonReader read() throws Exception { - return system().newReader(myBytes); + return getStreamingMode().newIonReader(system().getCatalog(), myBytes); } @Override diff --git a/test/com/amazon/ion/BinaryReaderWrappedValueLengthTest.java b/test/com/amazon/ion/BinaryReaderWrappedValueLengthTest.java index c1c5324abd..e545486de6 100644 --- a/test/com/amazon/ion/BinaryReaderWrappedValueLengthTest.java +++ b/test/com/amazon/ion/BinaryReaderWrappedValueLengthTest.java @@ -54,9 +54,8 @@ public class BinaryReaderWrappedValueLengthTest public void readInto() throws Exception { thrown.expect(IonException.class); - thrown.expectMessage("Wrapper length mismatch: wrapper 11 wrapped value 9 at position 8"); - final IonReader in = system().newReader(OVERRUN); + final IonReader in = getStreamingMode().newIonReader(system().getCatalog(), OVERRUN); assertEquals(IonType.SYMBOL, in.next()); assertEquals(asList("name"), asList(in.getTypeAnnotations())); @@ -65,17 +64,18 @@ public void readInto() throws Exception assertEquals(IonType.SYMBOL, in.next()); assertEquals(emptyList(), asList(in.getTypeAnnotations())); assertEquals("name", in.stringValue()); + in.close(); } @Test public void readOver() throws Exception { thrown.expect(IonException.class); - thrown.expectMessage("Wrapper length mismatch: wrapper 11 wrapped value 9 at position 8"); - final IonReader in = system().newReader(OVERRUN); + final IonReader in = getStreamingMode().newIonReader(system().getCatalog(), OVERRUN); assertEquals(IonType.SYMBOL, in.next()); assertEquals(IonType.SYMBOL, in.next()); + in.close(); } } diff --git a/test/com/amazon/ion/BinaryTest.java b/test/com/amazon/ion/BinaryTest.java index e7641ae01f..b29d3c998f 100644 --- a/test/com/amazon/ion/BinaryTest.java +++ b/test/com/amazon/ion/BinaryTest.java @@ -17,7 +17,10 @@ import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID; +import java.io.IOException; import java.util.Arrays; +import java.util.Iterator; + import org.junit.Test; public class BinaryTest extends IonTestCase @@ -74,7 +77,26 @@ public static String bytesToHex(final byte[] bytes) */ private IonValue ion(final String hex) { - return system().singleValue(hexToBytes(MAGIC_COOKIE + hex)); + byte[] data = hexToBytes(MAGIC_COOKIE + hex); + // Note: when ion-java#379 is complete, the following branch should be removed. + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + IonReader reader = getStreamingMode().newIonReader(system().getCatalog(), data); + Iterator iterator = system().iterate(reader); + IonValue value = null; + if (iterator.hasNext()) { + value = iterator.next(); + } + if (iterator.hasNext()) { + throw new IonException("not a single value"); + } + try { + reader.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + return value; + } + return system().singleValue(data); } /** Converts a single value to bytes using an empty datagram */ diff --git a/test/com/amazon/ion/EquivsTestCase.java b/test/com/amazon/ion/EquivsTestCase.java index 3cb88b345e..9421a63655 100644 --- a/test/com/amazon/ion/EquivsTestCase.java +++ b/test/com/amazon/ion/EquivsTestCase.java @@ -15,20 +15,23 @@ package com.amazon.ion; +import static com.amazon.ion.IonType.DATAGRAM; + import com.amazon.ion.impl._Private_Utils; import com.amazon.ion.junit.IonAssert; + +import java.io.File; + +import org.junit.Test; import com.amazon.ion.system.IonBinaryWriterBuilder; import com.amazon.ion.system.IonSystemBuilder; import com.amazon.ion.system.IonTextWriterBuilder; -import org.junit.Test; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.File; +import java.io.FileInputStream; import java.io.IOException; -import static com.amazon.ion.IonType.DATAGRAM; - public abstract class EquivsTestCase extends IonTestCase { @@ -191,7 +194,7 @@ protected void checkEquivalence(final IonValue left, } } - protected static IonDatagram[] roundTripDatagram(IonDatagram input) throws IOException { + public IonDatagram[] roundTripDatagram(IonDatagram input) throws IOException { IonSystem system = IonSystemBuilder.standard().build(); IonLoader loader = system.getLoader(); ByteArrayOutputStream textOutputStream = new ByteArrayOutputStream(); @@ -215,7 +218,12 @@ protected static IonDatagram[] roundTripDatagram(IonDatagram input) throws IOExc } data[0] = input; data[1] = loader.load(new ByteArrayInputStream(textOutputStream.toByteArray())); - data[2] = loader.load(new ByteArrayInputStream(binaryOutputStream.toByteArray())); + binaryReader = getStreamingMode().newIonReader( + system().getCatalog(), + new ByteArrayInputStream(binaryOutputStream.toByteArray()) + ); + data[2] = loader.load(binaryReader); + binaryReader.close(); return data; } @@ -231,7 +239,9 @@ protected void roundTripEquivalence(IonDatagram input, boolean myExpectedEqualit public void testEquivsOverFile() throws Exception { - IonDatagram dg = loader().load(myTestFile); + IonReader reader = getStreamingMode().newIonReader(system().getCatalog(), new FileInputStream(myTestFile)); + IonDatagram dg = loader().load(reader); + reader.close(); runEquivalenceChecks(dg, myExpectedEquality); roundTripEquivalence(dg, myExpectedEquality); } diff --git a/test/com/amazon/ion/GoodIonTest.java b/test/com/amazon/ion/GoodIonTest.java index 9ee8505c95..c1bc3fa804 100644 --- a/test/com/amazon/ion/GoodIonTest.java +++ b/test/com/amazon/ion/GoodIonTest.java @@ -62,7 +62,7 @@ public void test() FileInputStream in = new FileInputStream(myTestFile); try { - IonReader fileReader = system().newReader(in); + IonReader fileReader = getStreamingMode().newIonReader(system().getCatalog(), in); ReaderCompare.compare(treeReader, fileReader); } @@ -71,7 +71,7 @@ public void test() } - // Pass 3: Use Iterator + // Pass 3: Use Iterator over InputStream in = new FileInputStream(myTestFile); try { Iterator expected = datagram.iterator(); @@ -83,9 +83,21 @@ public void test() in.close(); } + // Pass 4: Use Iterator over IonReader + in = new FileInputStream(myTestFile); + try { + Iterator expected = datagram.iterator(); + Iterator actual = system().iterate(getStreamingMode().newIonReader(system().getCatalog(), in)); + + assertIonIteratorEquals(expected, actual); + } + finally { + in.close(); + } + treeReader = system().newReader(datagram); byte[] encoded = datagram.getBytes(); - IonReader binaryReader = system().newReader(encoded); + IonReader binaryReader = getStreamingMode().newIonReader(system().getCatalog(), encoded); ReaderCompare.compare(treeReader, binaryReader); } diff --git a/test/com/amazon/ion/IonTestCase.java b/test/com/amazon/ion/IonTestCase.java index 9c96dc2635..170889ce33 100644 --- a/test/com/amazon/ion/IonTestCase.java +++ b/test/com/amazon/ion/IonTestCase.java @@ -22,9 +22,11 @@ import com.amazon.ion.junit.Injected; import com.amazon.ion.junit.Injected.Inject; import com.amazon.ion.junit.IonAssert; +import com.amazon.ion.system.IonReaderBuilder; import com.amazon.ion.system.IonSystemBuilder; import com.amazon.ion.system.SimpleCatalog; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; @@ -58,13 +60,39 @@ public abstract class IonTestCase } } - protected enum StreamingMode { OLD_STREAMING, NEW_STREAMING } + protected enum StreamingMode { + NEW_STREAMING() { + @Override + public IonReader newIonReader(IonCatalog catalog, InputStream inputStream) { + return IonReaderBuilder.standard().withCatalog(catalog).build(inputStream); + } + + @Override + public IonReader newIonReader(IonCatalog catalog, byte[] data) { + return IonReaderBuilder.standard().withCatalog(catalog).build(data); + } + }, + NEW_STREAMING_INCREMENTAL() { + @Override + public IonReader newIonReader(IonCatalog catalog, InputStream inputStream) { + return IonReaderBuilder.standard().withCatalog(catalog).withIncrementalReadingEnabled(true).build(inputStream); + } + + @Override + public IonReader newIonReader(IonCatalog catalog, byte[] data) { + return IonReaderBuilder.standard().withCatalog(catalog).withIncrementalReadingEnabled(true).build(data); + } + }; + + public abstract IonReader newIonReader(IonCatalog catalog, InputStream inputStream); + public abstract IonReader newIonReader(IonCatalog catalog, byte[] data); + } // Using an enum makes the test names more understandable than a boolean. protected enum StreamCopySpeed { COPY_OPTIMIZED, COPY_NON_OPTIMIZED } @Inject("streamingMode") - public static final StreamingMode[] STREAMING_DIMENSION = { StreamingMode.NEW_STREAMING }; + public static final StreamingMode[] STREAMING_DIMENSION = StreamingMode.values(); /** * Flag on whether IonSystems generated is @@ -231,7 +259,16 @@ public IonDatagram load(File ionFile) throws IonException, IOException { IonLoader loader = loader(); - IonDatagram dg = loader.load(ionFile); + IonDatagram dg; + // Note: when ion-java#379 is complete, the following branches should be replaced with + // `dg = loader.load(ionFile)`. + if (ionFile.getName().endsWith(".ion")) { + dg = loader.load(ionFile); + } else { + IonReader reader = myStreamingMode.newIonReader(system().getCatalog(), new FileInputStream(ionFile)); + dg = loader.load(reader); + reader.close(); + } // Flush out any encoding problems in the data. forceDeepMaterialization(dg); @@ -739,14 +776,16 @@ public static void checkString(String text, IonValue value) public static void checkSymbol(String text, int sid, SymbolToken sym) { assertEquals("SymbolToken.text", text, sym.getText()); - assertEquals("SymbolToken.id", sid, sym.getSid()); if (text != null) { + assertEquals("SymbolToken.id", sid, sym.getSid()); assertEquals("SymbolToken.assumeText", text, sym.assumeText()); } else { + // Local symbols with unknown text may be treated equivalently to symbol zero. + assertTrue("SymbolToken.id", sid == sym.getSid() || 0 == sym.getSid()); try { sym.assumeText(); @@ -789,7 +828,7 @@ public static void checkSymbol(String name, int id, IonValue value) assertFalse(value.isNullValue()); - if (name == null) + if (name == null && id != 0) { try { sym.stringValue(); @@ -802,6 +841,7 @@ public static void checkSymbol(String name, int id, IonValue value) } else { + // Note: symbol zero follows this path because it causes stringValue() to return null rather than throw. assertEquals("symbol name", name, sym.stringValue()); } diff --git a/test/com/amazon/ion/LoadBinaryIonReaderSystemProcessingTest.java b/test/com/amazon/ion/LoadBinaryIonReaderSystemProcessingTest.java new file mode 100644 index 0000000000..90ec7b306f --- /dev/null +++ b/test/com/amazon/ion/LoadBinaryIonReaderSystemProcessingTest.java @@ -0,0 +1,31 @@ +package com.amazon.ion; + +public class LoadBinaryIonReaderSystemProcessingTest + extends DatagramIteratorSystemProcessingTest +{ + private byte[] myBytes; + + @Override + protected void prepare(String text) + { + myMissingSymbolTokensHaveText = false; + myBytes = encode(text); + } + + @Override + protected IonDatagram load() throws Exception + { + IonLoader loader = loader(); + IonReader reader = getStreamingMode().newIonReader(system().getCatalog(), myBytes); + IonDatagram datagram = loader.load(reader); + reader.close(); + return datagram; + } + + @Override + protected int expectedLocalNullSlotSymbolId() { + // The spec allows for implementations to treat these malformed symbols as "null slots", and all "null slots" + // in local symbol tables as equivalent to symbol zero. + return getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL ? 0 : 10; + } +} diff --git a/test/com/amazon/ion/NopPaddingTest.java b/test/com/amazon/ion/NopPaddingTest.java index 2fd7fa3555..1ecf5eac65 100644 --- a/test/com/amazon/ion/NopPaddingTest.java +++ b/test/com/amazon/ion/NopPaddingTest.java @@ -126,8 +126,6 @@ public void testNopPaddingWithAnnotation() throws Exception { myExpectedException.expect(IonException.class); - myExpectedException.expectMessage("NOP padding is not allowed within annotation wrappers."); - loadTestFile("bad/nopPadWithAnnotations.10n"); } diff --git a/test/com/amazon/ion/RawValueSpanReaderBasicTest.java b/test/com/amazon/ion/RawValueSpanReaderBasicTest.java index f209214c13..4da4cafe95 100644 --- a/test/com/amazon/ion/RawValueSpanReaderBasicTest.java +++ b/test/com/amazon/ion/RawValueSpanReaderBasicTest.java @@ -30,7 +30,6 @@ * Tests basic behavior of the {@link RawValueSpanProvider} reader facet, which * provides access to the reader's underlying byte buffer and vends OffsetSpans * that provide positions of the current value in that buffer. - * @see RawValueSpanReaderTest */ @SuppressWarnings({"deprecation", "javadoc"}) public class RawValueSpanReaderBasicTest @@ -53,6 +52,14 @@ public void testTextReaderReturnsNullFacet() assertNull(reader.asFacet(RawValueSpanProvider.class)); } + @Test + public void incrementalBinaryReaderReturnsNullFacet() + { + IonReader reader = IonReaderBuilder.standard().withIncrementalReadingEnabled(true).build(dummyData); + assertNull(reader.asFacet(RawValueSpanProvider.class)); + assertNull(reader.asFacet(SeekableReader.class)); + } + @Test public void testNonByteBackedReaderNotSupported() { diff --git a/test/com/amazon/ion/ReaderChecker.java b/test/com/amazon/ion/ReaderChecker.java index 7905173d80..f1fcc96557 100644 --- a/test/com/amazon/ion/ReaderChecker.java +++ b/test/com/amazon/ion/ReaderChecker.java @@ -158,7 +158,10 @@ public ReaderChecker annotation(String expectedText, int expectedSid) } try { - myReader.iterateTypeAnnotations(); + Iterator iterator = myReader.iterateTypeAnnotations(); + while (iterator.hasNext()) { + iterator.next(); + } fail("Expected " + UnknownSymbolException.class); } catch (UnknownSymbolException e) diff --git a/test/com/amazon/ion/ReaderMaker.java b/test/com/amazon/ion/ReaderMaker.java index ff5cc88fb7..b4a3e9a0ca 100644 --- a/test/com/amazon/ion/ReaderMaker.java +++ b/test/com/amazon/ion/ReaderMaker.java @@ -19,6 +19,8 @@ import static com.amazon.ion.TestUtils.ensureText; import com.amazon.ion.impl._Private_Utils; +import com.amazon.ion.system.IonReaderBuilder; + import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -138,6 +140,33 @@ public IonReader newReader(IonSystem system, byte[] ionData) } }, + FROM_INPUT_STREAM_BINARY_INCREMENTAL(Feature.BINARY, Feature.STREAM) + { + @Override + public IonReader newReader(IonSystem system, byte[] ionData, + InputStreamWrapper wrapper) + throws IOException + { + ionData = ensureBinary(system, ionData); + InputStream in = new ByteArrayInputStream(ionData); + InputStream wrapped = wrapper.wrap(in); + return newReader(system, wrapped); + } + + @Override + public IonReader newReader(IonSystem system, byte[] ionData) + { + ionData = ensureBinary(system, ionData); + InputStream in = new ByteArrayInputStream(ionData); + return newReader(system, in); + } + + @Override + public IonReader newReader(IonSystem system, InputStream inputStream) { + IonCatalog catalog = system.getCatalog(); + return IonReaderBuilder.standard().withIncrementalReadingEnabled(true).withCatalog(catalog).build(inputStream); + } + }, /** * Invokes {@link IonSystem#newReader(InputStream)} with Ion text. diff --git a/test/com/amazon/ion/ReaderSystemProcessingTestCase.java b/test/com/amazon/ion/ReaderSystemProcessingTestCase.java index de9b468221..e7c0f39a66 100644 --- a/test/com/amazon/ion/ReaderSystemProcessingTestCase.java +++ b/test/com/amazon/ion/ReaderSystemProcessingTestCase.java @@ -236,10 +236,6 @@ public void testIsInStruct() assertFalse(myReader.isInStruct()); assertEquals(0, myReader.getDepth()); - assertTrue(myReader.hasNext()); - assertFalse(myReader.isInStruct()); - assertEquals(0, myReader.getDepth()); - assertEquals(IonType.STRUCT, myReader.next()); assertFalse(myReader.isInStruct()); assertEquals(0, myReader.getDepth()); @@ -249,10 +245,6 @@ public void testIsInStruct() assertTrue(myReader.isInStruct()); assertEquals(1, myReader.getDepth()); - assertTrue(myReader.hasNext()); // List is coming up - assertTrue(myReader.isInStruct()); // but we're still at struct level - assertEquals(1, myReader.getDepth()); - assertSame(IonType.LIST, myReader.next()); assertTrue(myReader.isInStruct()); // still in struct until we stepIn() assertEquals(1, myReader.getDepth()); @@ -293,10 +285,8 @@ public void testHasNextLeavesCurrentData() String text = "hello 2"; startIteration(text); - assertTrue(myReader.hasNext()); assertEquals(IonType.SYMBOL, myReader.next()); assertEquals(IonType.SYMBOL, myReader.getType()); - assertTrue(myReader.hasNext()); // FIXed ME text reader was broken, now fixed // really the binary readers were returning the diff --git a/test/com/amazon/ion/RoundTripTest.java b/test/com/amazon/ion/RoundTripTest.java index 650bf5cac0..3024a40fd5 100644 --- a/test/com/amazon/ion/RoundTripTest.java +++ b/test/com/amazon/ion/RoundTripTest.java @@ -241,14 +241,18 @@ public void test() // Reload the first-trip binary - IonDatagram dgFromBinary = loader.load(binary1); + IonReader reader = getStreamingMode().newIonReader(system().getCatalog(), binary1); + IonDatagram dgFromBinary = loader.load(reader); + reader.close(); String text2FromBinary = renderUserView(dgFromBinary); byte[] binary2FromBinary = encode(dgFromBinary); checkBinaryHeader(binary2FromBinary); // check strict data equivalence - IonDatagram dgbinary2FromBinary = loader.load(binary2FromBinary); + reader = getStreamingMode().newIonReader(system().getCatalog(), binary2FromBinary); + IonDatagram dgbinary2FromBinary = loader.load(reader); + reader.close(); assertIonEquals(dgFromBinary, dgbinary2FromBinary); if (!compareRenderedTextImages(text2FromText, text2FromBinary)) @@ -259,7 +263,9 @@ public void test() } // check strict data equivalence - IonDatagram dgBinary2FromText = loader.load(binary2FromText); + reader = getStreamingMode().newIonReader(system().getCatalog(), binary2FromText); + IonDatagram dgBinary2FromText = loader.load(reader); + reader.close(); assertIonEquals(dgFromBinary, dgBinary2FromText); } diff --git a/test/com/amazon/ion/SystemProcessingTestCase.java b/test/com/amazon/ion/SystemProcessingTestCase.java index 6fce2cee24..24b9334787 100644 --- a/test/com/amazon/ion/SystemProcessingTestCase.java +++ b/test/com/amazon/ion/SystemProcessingTestCase.java @@ -325,7 +325,7 @@ public void testLocalTableReplacement() nextValue(); checkSymbol("foo"); - assertSame(table2, currentSymtab()); + assertEquals(table2.getMaxId(), currentSymtab().getMaxId()); } @Test @@ -346,14 +346,26 @@ public void testTrivialLocalTableReplacement() checkInt(1); SymbolTable table1 = currentSymtab(); - checkLocalTable(table1); + // Note: there's no need to require implementations to actually create an LST in this case. It's fine to + // treat it as a reset to the system symbol table. + assertFalse("table is substitute", table1.isSubstitute()); + checkUnknownSymbol(" not defined ", UNKNOWN_SYMBOL_ID, table1); + + SymbolTable system = table1.getSystemSymbolTable(); + checkSystemTable(system); + assertEquals(system.getIonVersionId(), table1.getIonVersionId()); + assertEquals(systemMaxId(), table1.getMaxId()); nextValue(); checkInt(2); SymbolTable table2 = currentSymtab(); - checkLocalTable(table2); - assertNotSame(table1, table2); + assertFalse("table is substitute", table2.isSubstitute()); + checkUnknownSymbol(" not defined ", UNKNOWN_SYMBOL_ID, table2); + + system = table2.getSystemSymbolTable(); + checkSystemTable(system); + assertEquals(system.getIonVersionId(), table2.getIonVersionId()); assertEquals(systemMaxId(), table2.getMaxId()); } @@ -1130,6 +1142,10 @@ public void testIvmWithAnnotationSid() checkEof(); } + protected int expectedLocalNullSlotSymbolId() { + return 10; + } + protected void checkLocalSymtabWithMalformedSymbolEntry(String symbolValue) throws Exception { @@ -1141,7 +1157,7 @@ protected void checkLocalSymtabWithMalformedSymbolEntry(String symbolValue) startIteration(text); nextValue(); - checkSymbol(null, 10); + checkSymbol(null, expectedLocalNullSlotSymbolId()); checkEof(); } diff --git a/test/com/amazon/ion/SystemProcessingTests.java b/test/com/amazon/ion/SystemProcessingTests.java index 64e77e7f40..b08c029b82 100644 --- a/test/com/amazon/ion/SystemProcessingTests.java +++ b/test/com/amazon/ion/SystemProcessingTests.java @@ -24,6 +24,7 @@ TextIteratorSystemProcessingTest.class, TextByteArrayIteratorSystemProcessingTest.class, BinaryByteArrayIteratorSystemProcessingTest.class, + BinaryIonReaderIteratorSystemProcessingTest.class, JavaReaderIteratorSystemProcessingTest.class, BinaryStreamIteratorSystemProcessingTest.class, TextStreamIteratorSystemProcessingTest.class, @@ -31,6 +32,7 @@ LoadTextStreamSystemProcessingTest.class, LoadBinaryBytesSystemProcessingTest.class, LoadBinaryStreamSystemProcessingTest.class, + LoadBinaryIonReaderSystemProcessingTest.class, DatagramIteratorSystemProcessingTest.class, BinaryReaderSystemProcessingTest.class, DatagramTreeReaderSystemProcessingTest.class, diff --git a/test/com/amazon/ion/TestUtils.java b/test/com/amazon/ion/TestUtils.java index 3f8a2bb241..f278c46ede 100644 --- a/test/com/amazon/ion/TestUtils.java +++ b/test/com/amazon/ion/TestUtils.java @@ -15,10 +15,14 @@ package com.amazon.ion; +import static com.amazon.ion.BitUtils.bytes; import static com.amazon.ion.impl._Private_Utils.READER_HASNEXT_REMOVED; +import com.amazon.ion.impl._Private_IonConstants; import com.amazon.ion.impl._Private_Utils; import com.amazon.ion.util.IonStreamUtils; + +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FilenameFilter; import java.math.BigDecimal; @@ -138,6 +142,7 @@ public boolean accept(File dir, String name) "bad/clobWithNullCharacter.ion" // TODO amzn/ion-java/43 ,"bad/emptyAnnotatedInt.10n" // TODO amzn/ion-java/55 ,"good/subfieldVarUInt32bit.ion" // TODO amzn/ion-java/62 + ,"good/subfieldVarUInt.ion" // Note: this passes but takes too long. That's fine; it's not a realistic use case. ,"good/utf16.ion" // TODO amzn/ion-java/61 ,"good/utf32.ion" // TODO amzn/ion-java/61 ,"good/whitespace.ion" @@ -551,4 +556,28 @@ public static boolean symbolTableEquals(final SymbolTable first, final SymbolTab throw new AssertionError("Broken encoding"); } } + + /** + * Byte appender for binary Ion streams. + */ + public static class BinaryIonAppender { + private final ByteArrayOutputStream out = new ByteArrayOutputStream(); + + public BinaryIonAppender() throws Exception { + out.write(_Private_IonConstants.BINARY_VERSION_MARKER_1_0); + } + + public BinaryIonAppender append(int... data) throws Exception { + return append(bytes(data)); + } + + public BinaryIonAppender append(byte[] data) throws Exception { + out.write(data); + return this; + } + + public byte[] toByteArray() { + return out.toByteArray(); + } + } } diff --git a/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java b/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java new file mode 100644 index 0000000000..992737f5ed --- /dev/null +++ b/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java @@ -0,0 +1,3158 @@ +package com.amazon.ion.impl; + +import com.amazon.ion.BufferConfiguration; +import com.amazon.ion.Decimal; +import com.amazon.ion.IntegerSize; +import com.amazon.ion.IonBufferConfiguration; +import com.amazon.ion.IonException; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.TestUtils; +import com.amazon.ion.Timestamp; +import com.amazon.ion.impl.bin._Private_IonManagedBinaryWriterBuilder; +import com.amazon.ion.impl.bin._Private_IonManagedWriter; +import com.amazon.ion.impl.bin._Private_IonRawWriter; +import com.amazon.ion.system.IonBinaryWriterBuilder; +import com.amazon.ion.system.IonReaderBuilder; +import com.amazon.ion.system.IonSystemBuilder; +import com.amazon.ion.system.SimpleCatalog; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static com.amazon.ion.BitUtils.bytes; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +public class IonReaderBinaryIncrementalTest { + + private static final IonSystem SYSTEM = IonSystemBuilder.standard().build(); + private static final IonReaderBuilder STANDARD_READER_BUILDER = IonReaderBuilder.standard() + .withIncrementalReadingEnabled(true); + private static final IonBinaryWriterBuilder STANDARD_WRITER_BUILDER = IonBinaryWriterBuilder.standard(); + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + // Builds the incremental reader. May be overwritten by individual tests. + private IonReaderBuilder readerBuilder; + // Builds binary writers for constructing test data. May be overwritten by individual tests. + private IonBinaryWriterBuilder writerBuilder; + + @Before + public void setup() { + readerBuilder = STANDARD_READER_BUILDER; + writerBuilder = STANDARD_WRITER_BUILDER; + } + + /** + * Writes binary Ion streams with a user-level writer. + */ + private interface WriterFunction { + void write(IonWriter writer) throws IOException; + } + + /** + * Writes binary Ion streams with a raw writer. Also allows bytes to be written directly to the stream. + */ + private interface RawWriterFunction { + void write(_Private_IonRawWriter writer, ByteArrayOutputStream out) throws IOException; + } + + /** + * Converts the given text Ion to the equivalent binary Ion. + * @param ion text Ion data. + * @return the equivalent binary Ion data. + * @throws Exception if the given data is invalid. + */ + private static byte[] toBinary(String ion) throws Exception { + return TestUtils.ensureBinary(SYSTEM, ion.getBytes("UTF-8")); + } + + /** + * Creates an incremental reader over the binary equivalent of the given text Ion. + * @param ion text Ion data. + * @return a new reader. + * @throws Exception if an exception is raised while converting the Ion data. + */ + private IonReaderBinaryIncremental readerFor(String ion) throws Exception { + return new IonReaderBinaryIncremental(readerBuilder, new ByteArrayInputStream(toBinary(ion))); + } + + /** + * Creates an incremental reader over the binary Ion data created by invoking the given RawWriterFunction. + * @param writerFunction the function used to generate the data. + * @return a new reader. + * @throws Exception if an exception is raised while writing the Ion data. + */ + private IonReaderBinaryIncremental readerFor(RawWriterFunction writerFunction) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + _Private_IonRawWriter writer = writerBuilder.build(out) + .asFacet(_Private_IonManagedWriter.class) + .getRawWriter(); + out.write(_Private_IonConstants.BINARY_VERSION_MARKER_1_0); + writerFunction.write(writer, out); + writer.close(); + return new IonReaderBinaryIncremental(readerBuilder, new ByteArrayInputStream(out.toByteArray())); + } + + /** + * Creates an incremental reader over the binary Ion data created by invoking the given WriterFunction. + * @param writerFunction the function used to generate the data. + * @return a new reader. + * @throws Exception if an exception is raised while writing the Ion data. + */ + private IonReaderBinaryIncremental readerFor(WriterFunction writerFunction) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IonWriter writer = writerBuilder.build(out); + writerFunction.write(writer); + writer.close(); + return new IonReaderBinaryIncremental(readerBuilder, new ByteArrayInputStream(out.toByteArray())); + } + + /** + * Creates an incremental reader over the given bytes, prepended with the IVM. + * @param ion binary Ion bytes without an IVM. + * @return a new reader. + */ + private IonReaderBinaryIncremental readerFor(int... ion) throws Exception { + return new IonReaderBinaryIncremental( + readerBuilder, + new ByteArrayInputStream(new TestUtils.BinaryIonAppender().append(ion).toByteArray()) + ); + } + + @Test + public void readInts() throws Exception { + final int numberOfValues = 1000000; + IonReaderBinaryIncremental reader = readerFor(new WriterFunction() { + @Override + public void write(IonWriter writer) throws IOException { + for (int i = -(numberOfValues / 2); i < numberOfValues / 2; i++) { + writer.writeInt(i); + } + } + }); + for (int i = -(numberOfValues / 2); i < numberOfValues / 2; i++) { + assertEquals(IonType.INT, reader.next()); + assertEquals(i, reader.intValue()); + } + reader.close(); + } + + @Test + public void emptyContainers() throws Exception { + IonReaderBinaryIncremental reader = readerFor("[{}, [], ()] 123"); + assertEquals(IonType.LIST, reader.next()); + reader.stepIn(); + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertTrue(reader.isInStruct()); + reader.stepOut(); + assertFalse(reader.isInStruct()); + assertEquals(IonType.LIST, reader.next()); + reader.stepIn(); + assertFalse(reader.isInStruct()); + assertNull(reader.next()); + // The following is repeated intentionally to ensure that the reader acts consistently and sanely when + // next() is called multiple times at the end of a container. + assertNull(reader.next()); + reader.stepOut(); + assertEquals(IonType.SEXP, reader.next()); + reader.stepIn(); + assertNull(reader.next()); + reader.stepOut(); + reader.stepOut(); + assertEquals(IonType.INT, reader.next()); + assertEquals(123, reader.intValue()); + assertNull(reader.next()); + reader.close(); + } + + private static void drainAnnotations(IonReader reader, List annotationsSink) { + Iterator iterator = reader.iterateTypeAnnotations(); + while (iterator.hasNext()) { + annotationsSink.add(iterator.next()); + } + } + + @Test + public void annotatedTopLevelIterator() throws Exception { + IonReaderBinaryIncremental reader = readerFor("foo::bar::123 baz::456"); + assertEquals(IonType.INT, reader.next()); + List annotations = new ArrayList(); + drainAnnotations(reader, annotations); + assertEquals(Arrays.asList("foo", "bar"), annotations); + annotations.clear(); + assertEquals(123, reader.intValue()); + assertEquals(IonType.INT, reader.next()); + drainAnnotations(reader, annotations); + assertEquals(Collections.singletonList("baz"), annotations); + assertEquals(456, reader.intValue()); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void annotatedTopLevelAsStrings() throws Exception { + IonReaderBinaryIncremental reader = readerFor("foo::bar::123 baz::456"); + assertEquals(IonType.INT, reader.next()); + assertEquals(Arrays.asList("foo", "bar"), Arrays.asList(reader.getTypeAnnotations())); + assertEquals(123, reader.intValue()); + assertEquals(IonType.INT, reader.next()); + assertEquals(Collections.singletonList("baz"), Arrays.asList(reader.getTypeAnnotations())); + assertEquals(456, reader.intValue()); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void annotatedInContainer() throws Exception { + IonReaderBinaryIncremental reader = readerFor("[foo::bar::123, baz::456]"); + assertEquals(IonType.LIST, reader.next()); + reader.stepIn(); + assertEquals(IonType.INT, reader.next()); + assertEquals(Arrays.asList("foo", "bar"), Arrays.asList(reader.getTypeAnnotations())); + assertEquals(123, reader.intValue()); + assertEquals(IonType.INT, reader.next()); + assertEquals(Collections.singletonList("baz"), Arrays.asList(reader.getTypeAnnotations())); + assertEquals(456, reader.intValue()); + assertNull(reader.next()); + reader.stepOut(); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void annotatedInContainerIterator() throws Exception { + IonReaderBinaryIncremental reader = readerFor("[foo::bar::123, baz::456]"); + assertEquals(IonType.LIST, reader.next()); + reader.stepIn(); + assertEquals(IonType.INT, reader.next()); + List annotations = new ArrayList(); + drainAnnotations(reader, annotations); + assertEquals(Arrays.asList("foo", "bar"), annotations); + annotations.clear(); + assertEquals(123, reader.intValue()); + assertEquals(IonType.INT, reader.next()); + drainAnnotations(reader, annotations); + assertEquals(Collections.singletonList("baz"), annotations); + assertEquals(456, reader.intValue()); + assertNull(reader.next()); + reader.stepOut(); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void nestedContainers() throws Exception { + IonReaderBinaryIncremental reader = readerFor("{abc: foo::bar::(123), def: baz::[456, {}]}"); + assertNull(reader.getType()); + assertEquals(0, reader.getDepth()); + assertEquals(IonType.STRUCT, reader.next()); + assertEquals(IonType.STRUCT, reader.getType()); + reader.stepIn(); + assertTrue(reader.isInStruct()); + assertNull(reader.getType()); + assertEquals(1, reader.getDepth()); + assertEquals(IonType.SEXP, reader.next()); + assertEquals(IonType.SEXP, reader.getType()); + assertEquals(Arrays.asList("foo", "bar"), Arrays.asList(reader.getTypeAnnotations())); + reader.stepIn(); + assertFalse(reader.isInStruct()); + assertEquals(2, reader.getDepth()); + assertNull(reader.getType()); + assertEquals(IonType.INT, reader.next()); + assertEquals(IonType.INT, reader.getType()); + assertEquals(123, reader.intValue()); + assertNull(reader.next()); + assertEquals(2, reader.getDepth()); + assertNull(reader.getType()); + reader.stepOut(); + assertEquals(1, reader.getDepth()); + assertNull(reader.getType()); + assertEquals(IonType.LIST, reader.next()); + assertEquals(IonType.LIST, reader.getType()); + assertEquals(Collections.singletonList("baz"), Arrays.asList(reader.getTypeAnnotations())); + reader.stepIn(); + assertEquals(2, reader.getDepth()); + assertNull(reader.getType()); + assertEquals(IonType.INT, reader.next()); + assertEquals(IonType.INT, reader.getType()); + assertEquals(456, reader.intValue()); + assertEquals(IonType.STRUCT, reader.next()); + assertEquals(IonType.STRUCT, reader.getType()); + assertEquals(2, reader.getDepth()); + assertNull(reader.next()); + assertEquals(2, reader.getDepth()); + assertNull(reader.getType()); + reader.stepOut(); + assertEquals(1, reader.getDepth()); + assertNull(reader.getType()); + assertNull(reader.next()); + assertNull(reader.getType()); + assertEquals(1, reader.getDepth()); + reader.stepOut(); + assertEquals(0, reader.getDepth()); + assertNull(reader.getType()); + assertNull(reader.next()); + assertNull(reader.getType()); + reader.close(); + } + + @Test + public void skipContainers() throws Exception { + IonReaderBinaryIncremental reader = readerFor( + "[123] 456 {abc: foo::bar::123, def: baz::456} [123] 789 [foo::bar::123, baz::456] [123]" + ); + assertEquals(IonType.LIST, reader.next()); + assertEquals(IonType.LIST, reader.getType()); + assertEquals(IonType.INT, reader.next()); + assertEquals(IonType.INT, reader.getType()); + assertEquals(IonType.STRUCT, reader.next()); + assertEquals(IonType.STRUCT, reader.getType()); + assertEquals(IonType.LIST, reader.next()); + assertEquals(IonType.LIST, reader.getType()); + reader.stepIn(); + assertNull(reader.getType()); + assertEquals(IonType.INT, reader.next()); + assertEquals(IonType.INT, reader.getType()); + assertEquals(123, reader.intValue()); + reader.stepOut(); + assertNull(reader.getType()); + assertEquals(IonType.INT, reader.next()); + assertEquals(IonType.INT, reader.getType()); + assertEquals(789, reader.intValue()); + assertEquals(IonType.LIST, reader.next()); + assertEquals(IonType.LIST, reader.getType()); + assertEquals(IonType.LIST, reader.next()); + assertEquals(IonType.LIST, reader.getType()); + assertNull(reader.next()); + assertNull(reader.getType()); + reader.close(); + } + + @Test + public void skipContainerAfterSteppingIn() throws Exception { + IonReaderBinaryIncremental reader = readerFor("{abc: foo::bar::123, def: baz::456} 789"); + assertEquals(IonType.STRUCT, reader.next()); + assertEquals(IonType.STRUCT, reader.getType()); + reader.stepIn(); + assertNull(reader.getType()); + assertEquals(IonType.INT, reader.next()); + assertEquals(IonType.INT, reader.getType()); + assertEquals(Arrays.asList("foo", "bar"), Arrays.asList(reader.getTypeAnnotations())); + assertEquals(123, reader.intValue()); + assertEquals(IonType.INT, reader.getType()); + // Step out before completing the value. + reader.stepOut(); + assertNull(reader.getType()); + assertEquals(IonType.INT, reader.next()); + assertEquals(IonType.INT, reader.getType()); + assertEquals(789, reader.intValue()); + assertNull(reader.next()); + assertNull(reader.getType()); + reader.close(); + } + + @Test + public void skipValueInContainer() throws Exception { + IonReaderBinaryIncremental reader = readerFor("{foo: \"bar\", abc: 123, baz: a}"); + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.STRING, reader.next()); + assertEquals(IonType.INT, reader.next()); + assertEquals("abc", reader.getFieldName()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("baz", reader.getFieldName()); + assertNull(reader.next()); + reader.stepOut(); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void fieldNameLength14() throws Exception { + IonReaderBinaryIncremental reader = readerFor("{multipleValues: [\"first\", \"second\"]}"); + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.LIST, reader.next()); + assertEquals("multipleValues", reader.getFieldName()); + reader.stepIn(); + assertEquals(IonType.STRING, reader.next()); + assertEquals("first", reader.stringValue()); + assertEquals(IonType.STRING, reader.next()); + assertEquals("second", reader.stringValue()); + assertNull(reader.next()); + reader.stepOut(); + assertNull(reader.next()); + reader.stepOut(); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void annotatedStringLength14() throws Exception { + IonReaderBinaryIncremental reader = readerFor("value_type::\"StringValueLong\" 123"); + assertEquals(IonType.STRING, reader.next()); + assertEquals("StringValueLong", reader.stringValue()); + assertEquals(IonType.INT, reader.next()); + assertEquals(123, reader.intValue()); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void annotatedContainerLength14() throws Exception { + IonReaderBinaryIncremental reader = readerFor("value_type::[\"StringValueLong\"] 123"); + assertEquals(IonType.LIST, reader.next()); + reader.stepIn(); + assertEquals(IonType.STRING, reader.next()); + assertEquals("StringValueLong", reader.stringValue()); + assertNull(reader.next()); + reader.stepOut(); + assertEquals(IonType.INT, reader.next()); + assertEquals(123, reader.intValue()); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void symbolsAsStrings() throws Exception { + IonReaderBinaryIncremental reader = readerFor("{foo: uvw::abc, bar: qrs::xyz::def}"); + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("foo", reader.getFieldName()); + assertEquals(Collections.singletonList("uvw"), Arrays.asList(reader.getTypeAnnotations())); + assertEquals("abc", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("bar", reader.getFieldName()); + assertEquals(Arrays.asList("qrs", "xyz"), Arrays.asList(reader.getTypeAnnotations())); + assertEquals("def", reader.stringValue()); + reader.stepOut(); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void lstAppend() throws Exception { + writerBuilder = IonBinaryWriterBuilder.standard().withLocalSymbolTableAppendEnabled(); + IonReaderBinaryIncremental reader = readerFor(new WriterFunction() { + @Override + public void write(IonWriter writer) throws IOException { + writer.stepIn(IonType.STRUCT); + writer.setFieldName("foo"); + writer.addTypeAnnotation("uvw"); + writer.writeSymbol("abc"); + writer.setFieldName("bar"); + writer.setTypeAnnotations("qrs", "xyz"); + writer.writeSymbol("def"); + writer.stepOut(); + writer.flush(); + writer.writeSymbol("orange"); + } + }); + + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("foo", reader.getFieldName()); + assertEquals(Collections.singletonList("uvw"), Arrays.asList(reader.getTypeAnnotations())); + assertEquals("abc", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("bar", reader.getFieldName()); + assertEquals(Arrays.asList("qrs", "xyz"), Arrays.asList(reader.getTypeAnnotations())); + assertEquals("def", reader.stringValue()); + reader.stepOut(); + SymbolTable preAppend = reader.getSymbolTable(); + assertEquals(IonType.SYMBOL, reader.next()); + SymbolTable postAppend = reader.getSymbolTable(); + assertEquals("orange", reader.stringValue()); + assertNull(preAppend.find("orange")); + assertNotNull(postAppend.find("orange")); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void lstNonAppend() throws Exception { + writerBuilder = IonBinaryWriterBuilder.standard().withLocalSymbolTableAppendDisabled(); + IonReaderBinaryIncremental reader = readerFor(new WriterFunction() { + @Override + public void write(IonWriter writer) throws IOException { + writer.stepIn(IonType.STRUCT); + writer.setFieldName("foo"); + writer.addTypeAnnotation("uvw"); + writer.writeSymbol("abc"); + writer.setFieldName("bar"); + writer.setTypeAnnotations("qrs", "xyz"); + writer.writeSymbol("def"); + writer.stepOut(); + writer.setTypeAnnotations("$ion_symbol_table"); + writer.stepIn(IonType.STRUCT); + writer.setFieldName("symbols"); + writer.stepIn(IonType.LIST); + writer.writeString("orange"); + writer.stepOut(); + writer.stepOut(); + writer.writeSymbol("orange"); + } + }); + + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("foo", reader.getFieldName()); + assertEquals(Collections.singletonList("uvw"), Arrays.asList(reader.getTypeAnnotations())); + assertEquals("abc", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("bar", reader.getFieldName()); + assertEquals(Arrays.asList("qrs", "xyz"), Arrays.asList(reader.getTypeAnnotations())); + assertEquals("def", reader.stringValue()); + reader.stepOut(); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("orange", reader.stringValue()); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void ivmBetweenValues() throws Exception { + writerBuilder = IonBinaryWriterBuilder.standard().withLocalSymbolTableAppendDisabled(); + IonReaderBinaryIncremental reader = readerFor(new WriterFunction() { + @Override + public void write(IonWriter writer) throws IOException { + writer.stepIn(IonType.STRUCT); + writer.setFieldName("foo"); + writer.addTypeAnnotation("uvw"); + writer.writeSymbol("abc"); + writer.setFieldName("bar"); + writer.setTypeAnnotations("qrs", "xyz"); + writer.writeSymbol("def"); + writer.stepOut(); + writer.finish(); + writer.writeSymbol("orange"); + } + }); + + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("foo", reader.getFieldName()); + assertEquals(Collections.singletonList("uvw"), Arrays.asList(reader.getTypeAnnotations())); + assertEquals("abc", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("bar", reader.getFieldName()); + assertEquals(Arrays.asList("qrs", "xyz"), Arrays.asList(reader.getTypeAnnotations())); + assertEquals("def", reader.stringValue()); + reader.stepOut(); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("orange", reader.stringValue()); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void multipleSymbolTablesBetweenValues() throws Exception { + IonReaderBinaryIncremental reader = readerFor(new WriterFunction() { + @Override + public void write(IonWriter writer) throws IOException { + writer.setTypeAnnotations("$ion_symbol_table"); + writer.stepIn(IonType.STRUCT); + writer.setFieldName("symbols"); + writer.stepIn(IonType.LIST); + writer.writeString("abc"); + writer.stepOut(); + writer.stepOut(); + writer.setTypeAnnotations("$ion_symbol_table"); + writer.stepIn(IonType.STRUCT); + writer.setFieldName("symbols"); + writer.stepIn(IonType.LIST); + writer.writeString("def"); + writer.stepOut(); + writer.setFieldName("imports"); + writer.writeSymbol("$ion_symbol_table"); + writer.stepOut(); + writer.writeSymbol("abc"); + writer.writeSymbol("def"); + writer.setTypeAnnotations("$ion_symbol_table"); + writer.stepIn(IonType.STRUCT); + writer.setFieldName("symbols"); + writer.stepIn(IonType.LIST); + writer.writeString("orange"); + writer.stepOut(); + writer.stepOut(); + writer.setTypeAnnotations("$ion_symbol_table"); + writer.stepIn(IonType.STRUCT); + writer.setFieldName("symbols"); + writer.stepIn(IonType.LIST); + writer.writeString("purple"); + writer.stepOut(); + writer.stepOut(); + writer.writeSymbol("purple"); + } + }); + + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("abc", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("def", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("purple", reader.stringValue()); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void multipleIvmsBetweenValues() throws Exception { + IonReaderBinaryIncremental reader = readerFor(new RawWriterFunction() { + @Override + public void write(_Private_IonRawWriter writer, ByteArrayOutputStream out) throws IOException { + writer.setTypeAnnotationSymbols(3); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(7); + writer.stepIn(IonType.LIST); + writer.writeString("abc"); + writer.stepOut(); + writer.stepOut(); + writer.finish(); + out.write(_Private_IonConstants.BINARY_VERSION_MARKER_1_0); + writer.setTypeAnnotationSymbols(3); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(7); + writer.stepIn(IonType.LIST); + writer.writeString("def"); + writer.stepOut(); + writer.stepOut(); + writer.writeSymbolToken(10); + writer.finish(); + out.write(_Private_IonConstants.BINARY_VERSION_MARKER_1_0); + out.write(_Private_IonConstants.BINARY_VERSION_MARKER_1_0); + writer.writeSymbolToken(4); + } + }); + + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("def", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("name", reader.stringValue()); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void invalidVersion() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write(bytes(0xE0, 0x01, 0x74, 0xEA, 0x20)); + InputStream input = new ByteArrayInputStream(out.toByteArray()); + IonReaderBinaryIncremental reader = new IonReaderBinaryIncremental(STANDARD_READER_BUILDER, input); + thrown.expect(IonException.class); + reader.next(); + } + + @Test + public void invalidVersionMarker() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write(bytes(0xE0, 0x01, 0x00, 0xEB, 0x20)); + InputStream input = new ByteArrayInputStream(out.toByteArray()); + IonReaderBinaryIncremental reader = new IonReaderBinaryIncremental(STANDARD_READER_BUILDER, input); + thrown.expect(IonException.class); + reader.next(); + } + + @Test + public void incrementalRead() throws Exception { + ResizingPipedInputStream pipe = new ResizingPipedInputStream(128); + ByteArrayOutputStream firstValue = new ByteArrayOutputStream(); + _Private_IonRawWriter firstValueWriter = IonBinaryWriterBuilder.standard().build(firstValue) + .asFacet(_Private_IonManagedWriter.class) + .getRawWriter(); + firstValue.write(_Private_IonConstants.BINARY_VERSION_MARKER_1_0); + firstValueWriter.setTypeAnnotationSymbols(3); + firstValueWriter.stepIn(IonType.STRUCT); + firstValueWriter.setFieldNameSymbol(7); + firstValueWriter.stepIn(IonType.LIST); + firstValueWriter.writeString("abc"); + firstValueWriter.writeString("def"); + firstValueWriter.stepOut(); + firstValueWriter.stepOut(); + firstValueWriter.stepIn(IonType.STRUCT); + firstValueWriter.setTypeAnnotationSymbols(11); + firstValueWriter.setFieldNameSymbol(10); + firstValueWriter.writeString("foo"); + firstValueWriter.stepOut(); + firstValueWriter.close(); + IonReaderBinaryIncremental reader = new IonReaderBinaryIncremental(STANDARD_READER_BUILDER, pipe); + assertNull(reader.getType()); + assertNull(reader.next()); + assertNull(reader.getType()); + byte[] firstValueBytes = firstValue.toByteArray(); + pipe.receive(firstValueBytes, 0, firstValueBytes.length / 2); + assertNull(reader.next()); + assertNull(reader.getType()); + pipe.receive( + firstValueBytes, + firstValueBytes.length / 2, + firstValueBytes.length - firstValueBytes.length / 2 + ); + assertEquals(IonType.STRUCT, reader.next()); + assertEquals(IonType.STRUCT, reader.getType()); + reader.stepIn(); + assertNull(reader.getType()); + assertEquals(IonType.STRING, reader.next()); + assertEquals("abc", reader.getFieldName()); + assertEquals(Collections.singletonList("def"), Arrays.asList(reader.getTypeAnnotations())); + assertEquals("foo", reader.stringValue()); + assertEquals(IonType.STRING, reader.getType()); + assertNull(reader.next()); + assertNull(reader.getType()); + reader.stepOut(); + assertNull(reader.getType()); + assertNull(reader.next()); + assertNull(reader.getType()); + pipe.receive(0x71); + assertNull(reader.next()); + assertNull(reader.getType()); + pipe.receive(0x0B); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals(IonType.SYMBOL, reader.getType()); + assertEquals("def", reader.stringValue()); + assertNull(reader.next()); + assertNull(reader.getType()); + reader.close(); + } + + @Test + public void incrementalValue() throws Exception { + ResizingPipedInputStream pipe = new ResizingPipedInputStream(128); + IonReaderBinaryIncremental reader = new IonReaderBinaryIncremental(STANDARD_READER_BUILDER, pipe); + byte[] bytes = toBinary("\"StringValueLong\""); + for (byte b : bytes) { + assertNull(reader.next()); + pipe.receive(b); + } + assertEquals(IonType.STRING, reader.next()); + assertEquals("StringValueLong", reader.stringValue()); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void incrementalMultipleValues() throws Exception { + ResizingPipedInputStream pipe = new ResizingPipedInputStream(128); + IonReaderBinaryIncremental reader = new IonReaderBinaryIncremental(STANDARD_READER_BUILDER, pipe); + byte[] bytes = toBinary("value_type::\"StringValueLong\""); + for (byte b : bytes) { + assertNull(reader.next()); + pipe.receive(b); + } + assertEquals(IonType.STRING, reader.next()); + assertEquals("StringValueLong", reader.stringValue()); + assertEquals(Collections.singletonList("value_type"), Arrays.asList(reader.getTypeAnnotations())); + bytes = toBinary("{foobar: \"StringValueLong\"}"); + for (byte b : bytes) { + assertNull(reader.next()); + pipe.receive(b); + } + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.STRING, reader.next()); + assertEquals("foobar", reader.getFieldName()); + assertEquals("StringValueLong", reader.stringValue()); + assertNull(reader.next()); + reader.stepOut(); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void incrementalSymbolTables() throws Exception { + ResizingPipedInputStream pipe = new ResizingPipedInputStream(128); + ByteArrayOutputStream firstValue = new ByteArrayOutputStream(); + _Private_IonRawWriter writer = IonBinaryWriterBuilder.standard().build(firstValue) + .asFacet(_Private_IonManagedWriter.class) + .getRawWriter(); + firstValue.write(_Private_IonConstants.BINARY_VERSION_MARKER_1_0); + writer.setTypeAnnotationSymbols(3); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(7); + writer.stepIn(IonType.LIST); + writer.writeString("abcdefghijklmnopqrstuvwxyz"); + writer.writeString("def"); + writer.stepOut(); + writer.stepOut(); + writer.stepIn(IonType.STRUCT); + writer.setTypeAnnotationSymbols(11); + writer.setFieldNameSymbol(10); + writer.writeString("foo"); + writer.stepOut(); + writer.close(); + + ByteArrayOutputStream secondValue = new ByteArrayOutputStream(); + writer = IonBinaryWriterBuilder.standard().build(secondValue) + .asFacet(_Private_IonManagedWriter.class) + .getRawWriter(); + writer.setTypeAnnotationSymbols(3); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(6); + writer.writeSymbolToken(3); + writer.setFieldNameSymbol(7); + writer.stepIn(IonType.LIST); + writer.writeString("foo"); + writer.writeString("bar"); + writer.stepOut(); + writer.stepOut(); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(10); + writer.setTypeAnnotationSymbols(12, 13); + writer.writeString("fairlyLongString"); + writer.stepOut(); + writer.close(); + + IonReaderBinaryIncremental reader = new IonReaderBinaryIncremental(STANDARD_READER_BUILDER, pipe); + byte[] bytes = firstValue.toByteArray(); + for (byte b : bytes) { + assertNull(reader.next()); + pipe.receive(b); + } + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.STRING, reader.next()); + assertEquals(Collections.singletonList("def"), Arrays.asList(reader.getTypeAnnotations())); + assertEquals("abcdefghijklmnopqrstuvwxyz", reader.getFieldName()); + assertEquals("foo", reader.stringValue()); + assertNull(reader.next()); + reader.stepOut(); + bytes = secondValue.toByteArray(); + for (byte b : bytes) { + assertNull(reader.next()); + pipe.receive(b); + } + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.STRING, reader.next()); + assertEquals("fairlyLongString", reader.stringValue()); + assertEquals("abcdefghijklmnopqrstuvwxyz", reader.getFieldName()); + assertEquals(Arrays.asList("foo", "bar"), Arrays.asList(reader.getTypeAnnotations())); + assertNull(reader.next()); + reader.stepOut(); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void floats() throws Exception { + double acceptableDelta = 1e-9; + writerBuilder = IonBinaryWriterBuilder.standard().withFloatBinary32Enabled(); + IonReaderBinaryIncremental reader = readerFor(new WriterFunction() { + @Override + public void write(IonWriter writer) throws IOException { + writer.writeFloat(0.); + writer.writeFloat(Double.MAX_VALUE); + writer.writeFloat(Double.MIN_VALUE); + writer.writeFloat(Float.MAX_VALUE); + writer.writeFloat(Float.MIN_VALUE); + writer.writeFloat(1.23e4); + writer.writeFloat(1.23e-4); + writer.writeFloat(-1.23e4); + writer.writeFloat(-1.23e-4); + } + }); + + assertNull(reader.getType()); + assertEquals(IonType.FLOAT, reader.next()); + assertEquals(0., reader.doubleValue(), acceptableDelta); + assertEquals(IonType.FLOAT, reader.next()); + assertEquals(Double.MAX_VALUE, reader.doubleValue(), acceptableDelta); + assertEquals(IonType.FLOAT, reader.next()); + assertEquals(Double.MIN_VALUE, reader.doubleValue(), acceptableDelta); + assertEquals(IonType.FLOAT, reader.next()); + assertEquals(Float.MAX_VALUE, reader.doubleValue(), acceptableDelta); + assertEquals(IonType.FLOAT, reader.next()); + assertEquals(IonType.FLOAT, reader.getType()); + assertEquals(Float.MIN_VALUE, reader.doubleValue(), acceptableDelta); + assertEquals(IonType.FLOAT, reader.next()); + assertEquals(1.23e4, reader.doubleValue(), acceptableDelta); + assertEquals(IonType.FLOAT, reader.next()); + assertEquals(1.23e-4, reader.doubleValue(), acceptableDelta); + assertEquals(IonType.FLOAT, reader.next()); + assertEquals(-1.23e4, reader.doubleValue(), acceptableDelta); + assertEquals(IonType.FLOAT, reader.next()); + assertEquals(-1.23e-4, reader.doubleValue(), acceptableDelta); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void timestamps() throws Exception { + final List timestamps = new ArrayList(); + timestamps.add(Timestamp.valueOf("2000T")); + timestamps.add(Timestamp.valueOf("2000-01T")); + timestamps.add(Timestamp.valueOf("2000-01-02T")); + timestamps.add(Timestamp.valueOf("2000-01-02T03:04Z")); + timestamps.add(Timestamp.valueOf("2000-01-02T03:04:05Z")); + timestamps.add(Timestamp.valueOf("2000-01-02T03:04:05.6Z")); + timestamps.add(Timestamp.valueOf("2000-01-02T03:04:05.06Z")); + timestamps.add(Timestamp.valueOf("2000-01-02T03:04:05.006Z")); + timestamps.add(Timestamp.valueOf("2000-01-02T03:04:05.600Z")); + timestamps.add(Timestamp.valueOf("2000-01-02T03:04:05.060Z")); + timestamps.add(Timestamp.valueOf("2000-01-02T03:04:05.060-07:00")); + timestamps.add(Timestamp.valueOf("2000-01-02T03:04:05.060+07:00")); + timestamps.add(Timestamp.valueOf("2000-01-02T03:04:05+07:00")); + timestamps.add(Timestamp.valueOf("2000-01-02T03:04+07:00")); + timestamps.add(Timestamp.valueOf("2000-01-02T03:04:05.9999999Z")); + + IonReaderBinaryIncremental reader = readerFor(new WriterFunction() { + @Override + public void write(IonWriter writer) throws IOException { + for (Timestamp timestamp : timestamps) { + writer.writeTimestamp(timestamp); + } + } + }); + + for (Timestamp timestamp : timestamps) { + assertEquals(IonType.TIMESTAMP, reader.next()); + assertEquals(timestamp, reader.timestampValue()); + } + assertNull(reader.next()); + assertNull(reader.getType()); + reader.close(); + } + + @Test + public void nullValues() throws Exception { + IonReaderBinaryIncremental reader = readerFor( + "null " + + "null.bool " + + "null.int " + + "null.float " + + "null.decimal " + + "null.timestamp " + + "null.symbol " + + "null.string " + + "null.blob " + + "null.clob " + + "null.list " + + "null.sexp " + + "foo::null.struct " + + "[null.null, null.bool, null.int, {zar: bar::baz::null.float, abc: null.decimal}]" + ); + + assertEquals(IonType.NULL, reader.next()); + assertTrue(reader.isNullValue()); + assertEquals(IonType.BOOL, reader.next()); + assertTrue(reader.isNullValue()); + assertEquals(IonType.INT, reader.next()); + assertTrue(reader.isNullValue()); + assertEquals(IonType.FLOAT, reader.next()); + assertTrue(reader.isNullValue()); + assertEquals(IonType.DECIMAL, reader.next()); + assertTrue(reader.isNullValue()); + assertEquals(IonType.TIMESTAMP, reader.next()); + assertTrue(reader.isNullValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertTrue(reader.isNullValue()); + assertEquals(IonType.STRING, reader.next()); + assertTrue(reader.isNullValue()); + assertEquals(IonType.BLOB, reader.next()); + assertTrue(reader.isNullValue()); + assertEquals(IonType.CLOB, reader.next()); + assertTrue(reader.isNullValue()); + assertEquals(IonType.LIST, reader.next()); + assertTrue(reader.isNullValue()); + assertEquals(IonType.SEXP, reader.next()); + assertTrue(reader.isNullValue()); + assertEquals(IonType.STRUCT, reader.next()); + assertTrue(reader.isNullValue()); + assertEquals(Collections.singletonList("foo"), Arrays.asList(reader.getTypeAnnotations())); + assertEquals(IonType.LIST, reader.next()); + assertFalse(reader.isNullValue()); + reader.stepIn(); + assertFalse(reader.isNullValue()); + assertEquals(IonType.NULL, reader.next()); + assertTrue(reader.isNullValue()); + assertEquals(IonType.BOOL, reader.next()); + assertTrue(reader.isNullValue()); + assertEquals(IonType.INT, reader.next()); + assertTrue(reader.isNullValue()); + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertFalse(reader.isNullValue()); + assertEquals(IonType.FLOAT, reader.next()); + assertEquals(Arrays.asList("bar", "baz"), Arrays.asList(reader.getTypeAnnotations())); + assertEquals("zar", reader.getFieldName()); + assertTrue(reader.isNullValue()); + assertEquals(IonType.DECIMAL, reader.next()); + assertEquals("abc", reader.getFieldName()); + assertTrue(reader.isNullValue()); + assertNull(reader.next()); + assertFalse(reader.isNullValue()); + reader.stepOut(); + assertNull(reader.next()); + assertFalse(reader.isNullValue()); + reader.stepOut(); + assertNull(reader.next()); + assertFalse(reader.isNullValue()); + reader.close(); + } + + @Test + public void booleans() throws Exception { + IonReaderBinaryIncremental reader = readerFor("true false foo::true {bar: true, baz: zar::false}"); + assertEquals(IonType.BOOL, reader.next()); + assertTrue(reader.booleanValue()); + assertEquals(IonType.BOOL, reader.next()); + assertFalse(reader.booleanValue()); + assertEquals(IonType.BOOL, reader.next()); + assertEquals(Collections.singletonList("foo"), Arrays.asList(reader.getTypeAnnotations())); + assertTrue(reader.booleanValue()); + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.BOOL, reader.next()); + assertEquals("bar", reader.getFieldName()); + assertTrue(reader.booleanValue()); + assertEquals(IonType.BOOL, reader.next()); + assertEquals("baz", reader.getFieldName()); + assertEquals(Collections.singletonList("zar"), Arrays.asList(reader.getTypeAnnotations())); + assertFalse(reader.booleanValue()); + reader.stepOut(); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void lobsNewBytes() throws Exception { + final byte[] blobBytes = "abcdef".getBytes("UTF-8"); + final byte[] clobBytes = "ghijklmnopqrstuv".getBytes("UTF-8"); + IonReaderBinaryIncremental reader = readerFor(new WriterFunction() { + @Override + public void write(IonWriter writer) throws IOException { + writer.writeBlob(blobBytes); + writer.writeClob(clobBytes); + writer.setTypeAnnotations("foo"); + writer.writeBlob(blobBytes); + writer.stepIn(IonType.STRUCT); + writer.setFieldName("bar"); + writer.writeClob(clobBytes); + writer.stepOut(); + } + }); + + assertEquals(IonType.BLOB, reader.next()); + assertArrayEquals(blobBytes, reader.newBytes()); + assertEquals(IonType.CLOB, reader.next()); + assertArrayEquals(clobBytes, reader.newBytes()); + assertEquals(IonType.BLOB, reader.next()); + assertEquals(Collections.singletonList("foo"), Arrays.asList(reader.getTypeAnnotations())); + assertArrayEquals(blobBytes, reader.newBytes()); + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.CLOB, reader.next()); + assertEquals("bar", reader.getFieldName()); + assertArrayEquals(clobBytes, reader.newBytes()); + assertNull(reader.next()); + reader.stepOut(); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void lobsGetBytes() throws Exception { + final byte[] blobBytes = "abcdef".getBytes("UTF-8"); + final byte[] clobBytes = "ghijklmnopqrstuv".getBytes("UTF-8"); + IonReaderBinaryIncremental reader = readerFor(new WriterFunction() { + @Override + public void write(IonWriter writer) throws IOException { + writer.writeBlob(blobBytes); + writer.writeClob(clobBytes); + writer.setTypeAnnotations("foo"); + writer.writeBlob(blobBytes); + writer.stepIn(IonType.STRUCT); + writer.setFieldName("bar"); + writer.writeClob(clobBytes); + writer.stepOut(); + } + }); + + assertEquals(IonType.BLOB, reader.next()); + byte[] fullBlob = new byte[blobBytes.length]; + assertEquals(fullBlob.length, reader.getBytes(fullBlob, 0, fullBlob.length)); + assertArrayEquals(blobBytes, fullBlob); + assertEquals(IonType.CLOB, reader.next()); + byte[] partialClob = new byte[clobBytes.length]; + assertEquals(3, reader.getBytes(partialClob, 0, 3)); + assertEquals(clobBytes.length - 3, reader.getBytes(partialClob, 3, clobBytes.length - 3)); + assertArrayEquals(clobBytes, partialClob); + Arrays.fill(fullBlob, (byte) 0); + assertEquals(IonType.BLOB, reader.next()); + assertEquals(fullBlob.length, reader.getBytes(fullBlob, 0, 100000)); + assertEquals(Collections.singletonList("foo"), Arrays.asList(reader.getTypeAnnotations())); + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + Arrays.fill(partialClob, (byte) 0); + assertEquals(IonType.CLOB, reader.next()); + assertEquals(5, reader.getBytes(partialClob, 0, 5)); + assertEquals(clobBytes.length - 5, reader.getBytes(partialClob, 5, 100000)); + assertArrayEquals(clobBytes, partialClob); + assertNull(reader.next()); + reader.stepOut(); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void nopPad() throws Exception { + IonReaderBinaryIncremental reader = readerFor( + // One byte no-op pad. + 0x00, + // Two byte no-op pad. + 0x01, 0xFF, + // Int 0. + 0x20, + // Three byte no-op pad. + 0x02, 0x99, 0x42, + // 16-byte no-op pad. + 0x0E, 0x8E, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // Int 1. + 0x21, 0x01, + // Struct with no-op pad at the start. + 0xD9, + // Field SID 0. + 0x80, + // Five byte no-op pad. + 0x04, 0x00, 0x00, 0x00, 0x00, + // Field SID 4 ("name"). + 0x84, + // Int -1. + 0x31, 0x01, + // Struct (empty) with no-op pad at the end. + 0xD8, + // Field SID 0. + 0x80, + // Seven byte no-op pad. + 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // List (empty) with long no-op pad. + 0xBE, + // Length 16. + 0x90, + // 16-byte no-op pad. + 0x0E, 0x8E, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + ); + + assertEquals(IonType.INT, reader.next()); + assertEquals(0, reader.intValue()); + assertEquals(IonType.INT, reader.next()); + assertEquals(1, reader.intValue()); + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.INT, reader.next()); + assertEquals(-1, reader.intValue()); + assertNull(reader.next()); + reader.stepOut(); + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertNull(reader.next()); + reader.stepOut(); + assertEquals(IonType.LIST, reader.next()); + reader.stepIn(); + assertNull(reader.next()); + reader.stepOut(); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void decimals() throws Exception { + final List decimals = Arrays.asList( + Decimal.ZERO, + Decimal.NEGATIVE_ZERO, + Decimal.valueOf("1.23e4"), + Decimal.valueOf("1.23e-4"), + Decimal.valueOf("-1.23e4"), + Decimal.valueOf("-1.23e-4"), + Decimal.valueOf(Long.MAX_VALUE).add(BigDecimal.ONE), + Decimal.valueOf(Long.MIN_VALUE).subtract(BigDecimal.ONE) + ); + IonReaderBinaryIncremental reader = readerFor(new WriterFunction() { + @Override + public void write(IonWriter writer) throws IOException { + for (BigDecimal decimal : decimals) { + writer.writeDecimal(decimal); + } + writer.stepIn(IonType.STRUCT); + writer.setFieldName("foo"); + writer.setTypeAnnotations("bar"); + writer.writeDecimal(Decimal.valueOf(Long.MAX_VALUE).add(BigDecimal.ONE).scaleByPowerOfTen(-7)); + writer.stepOut(); + } + }); + + for (BigDecimal decimal : decimals) { + assertEquals(IonType.DECIMAL, reader.next()); + assertEquals(decimal, reader.decimalValue()); + } + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.DECIMAL, reader.next()); + assertEquals( + BigDecimal.valueOf(Long.MAX_VALUE).add(BigDecimal.ONE).scaleByPowerOfTen(-7), + reader.bigDecimalValue() + ); + assertNull(reader.next()); + reader.stepOut(); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void bigInts() throws Exception { + IonReaderBinaryIncremental reader = readerFor(new RawWriterFunction() { + @Override + public void write(_Private_IonRawWriter writer, ByteArrayOutputStream out) throws IOException { + writer.writeInt(0); + writer.writeInt(-1); + writer.writeInt(1); + writer.writeInt((long) Integer.MAX_VALUE + 1); + writer.writeInt((long) Integer.MIN_VALUE - 1); + writer.writeInt(Long.MIN_VALUE + 1); // Just shy of the boundary. + writer.writeInt(BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE)); + writer.writeInt(BigInteger.valueOf(Long.MIN_VALUE).subtract(BigInteger.ONE)); + writer.close(); + out.write(bytes(0x38, 0x81, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00)); + } + }); + + assertEquals(IonType.INT, reader.next()); + assertEquals(IntegerSize.INT, reader.getIntegerSize()); + assertEquals(BigInteger.ZERO, reader.bigIntegerValue()); + assertEquals(IonType.INT, reader.next()); + assertEquals(IntegerSize.INT, reader.getIntegerSize()); + assertEquals(BigInteger.ONE.negate(), reader.bigIntegerValue()); + assertEquals(IonType.INT, reader.next()); + assertEquals(IntegerSize.INT, reader.getIntegerSize()); + assertEquals(BigInteger.ONE, reader.bigIntegerValue()); + assertEquals(IonType.INT, reader.next()); + assertEquals(IntegerSize.LONG, reader.getIntegerSize()); + assertEquals(BigInteger.valueOf(Integer.MAX_VALUE).add(BigInteger.ONE), reader.bigIntegerValue()); + assertEquals(IonType.INT, reader.next()); + assertEquals(IntegerSize.LONG, reader.getIntegerSize()); + assertEquals(BigInteger.valueOf(Integer.MIN_VALUE).subtract(BigInteger.ONE), reader.bigIntegerValue()); + assertEquals(IonType.INT, reader.next()); + assertEquals(IntegerSize.LONG, reader.getIntegerSize()); + assertEquals(Long.MIN_VALUE + 1, reader.longValue()); + assertEquals(IonType.INT, reader.next()); + assertEquals(IntegerSize.BIG_INTEGER, reader.getIntegerSize()); + assertEquals(BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE), reader.bigIntegerValue()); + assertEquals(IonType.INT, reader.next()); + assertEquals(IntegerSize.BIG_INTEGER, reader.getIntegerSize()); + assertEquals(BigInteger.valueOf(Long.MIN_VALUE).subtract(BigInteger.ONE), reader.bigIntegerValue()); + assertEquals(IonType.INT, reader.next()); + assertEquals(IntegerSize.BIG_INTEGER, reader.getIntegerSize()); + assertEquals( + new BigInteger(-1, bytes(0x81, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00)), + reader.bigIntegerValue() + ); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void symbolTableWithImportsThenSymbols() throws Exception { + SimpleCatalog catalog = new SimpleCatalog(); + catalog.putTable(SYSTEM.newSharedSymbolTable("foo", 1, Arrays.asList("abc", "def").iterator())); + readerBuilder = IonReaderBuilder.standard().withCatalog(catalog); + writerBuilder = IonBinaryWriterBuilder.standard().withCatalog(catalog); + + IonReaderBinaryIncremental reader = readerFor(new WriterFunction() { + @Override + public void write(IonWriter writer) throws IOException { + writer.setTypeAnnotations("$ion_symbol_table"); + writer.stepIn(IonType.STRUCT); + writer.setFieldName("imports"); + writer.stepIn(IonType.LIST); + writer.stepIn(IonType.STRUCT); + writer.setFieldName("name"); + writer.writeString("foo"); + writer.setFieldName("version"); + writer.writeInt(1); + writer.setFieldName("max_id"); + writer.writeInt(2); + writer.stepOut(); + writer.stepOut(); + writer.setFieldName("symbols"); + writer.stepIn(IonType.LIST); + writer.writeString("ghi"); + writer.stepOut(); + writer.stepOut(); + writer.writeSymbol("abc"); + writer.writeSymbol("def"); + writer.writeSymbol("ghi"); + } + }); + + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("abc", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("def", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("ghi", reader.stringValue()); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void symbolTableWithSymbolsThenImports() throws Exception { + SimpleCatalog catalog = new SimpleCatalog(); + catalog.putTable(SYSTEM.newSharedSymbolTable("foo", 1, Arrays.asList("abc", "def").iterator())); + readerBuilder = IonReaderBuilder.standard().withCatalog(catalog); + + IonReaderBinaryIncremental reader = readerFor(new RawWriterFunction() { + @Override + public void write(_Private_IonRawWriter writer, ByteArrayOutputStream out) throws IOException { + SymbolTable systemTable = SharedSymbolTable.getSystemSymbolTable(1); + writer.addTypeAnnotationSymbol(systemTable.findSymbol("$ion_symbol_table")); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(systemTable.findSymbol("symbols")); + writer.stepIn(IonType.LIST); + writer.writeString("ghi"); + writer.stepOut(); + writer.setFieldNameSymbol(systemTable.findSymbol("imports")); + writer.stepIn(IonType.LIST); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(systemTable.findSymbol("name")); + writer.writeString("foo"); + writer.setFieldNameSymbol(systemTable.findSymbol("version")); + writer.writeInt(1); + writer.setFieldNameSymbol(systemTable.findSymbol("max_id")); + writer.writeInt(2); + writer.stepOut(); + writer.stepOut(); + writer.stepOut(); + writer.writeSymbolToken(10); + writer.writeSymbolToken(11); + writer.writeSymbolToken(12); + } + }); + + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("abc", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("def", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("ghi", reader.stringValue()); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void symbolTableWithManySymbolsThenImports() throws Exception { + SimpleCatalog catalog = new SimpleCatalog(); + catalog.putTable(SYSTEM.newSharedSymbolTable("foo", 1, Arrays.asList("abc", "def").iterator())); + readerBuilder = IonReaderBuilder.standard().withCatalog(catalog); + + IonReaderBinaryIncremental reader = readerFor(new RawWriterFunction() { + @Override + public void write(_Private_IonRawWriter writer, ByteArrayOutputStream out) throws IOException { + SymbolTable systemTable = SharedSymbolTable.getSystemSymbolTable(1); + writer.addTypeAnnotationSymbol(systemTable.findSymbol("$ion_symbol_table")); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(systemTable.findSymbol("symbols")); + writer.stepIn(IonType.LIST); + writer.writeString("ghi"); + writer.writeString("jkl"); + writer.writeString("mno"); + writer.writeString("pqr"); + writer.stepOut(); + writer.setFieldNameSymbol(systemTable.findSymbol("imports")); + writer.stepIn(IonType.LIST); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(systemTable.findSymbol("name")); + writer.writeString("foo"); + writer.setFieldNameSymbol(systemTable.findSymbol("version")); + writer.writeInt(1); + writer.setFieldNameSymbol(systemTable.findSymbol("max_id")); + writer.writeInt(2); + writer.stepOut(); + writer.stepOut(); + writer.stepOut(); + writer.writeSymbolToken(10); + writer.writeSymbolToken(11); + writer.writeSymbolToken(12); + writer.writeSymbolToken(13); + writer.writeSymbolToken(14); + writer.writeSymbolToken(15); + } + }); + + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("abc", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("def", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("ghi", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("jkl", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("mno", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("pqr", reader.stringValue()); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void multipleSymbolTablesWithSymbolsThenImports() throws Exception { + SimpleCatalog catalog = new SimpleCatalog(); + catalog.putTable(SYSTEM.newSharedSymbolTable("foo", 1, Arrays.asList("abc", "def").iterator())); + catalog.putTable(SYSTEM.newSharedSymbolTable("bar", 1, Collections.singletonList("baz").iterator())); + readerBuilder = IonReaderBuilder.standard().withCatalog(catalog); + + IonReaderBinaryIncremental reader = readerFor(new RawWriterFunction() { + @Override + public void write(_Private_IonRawWriter writer, ByteArrayOutputStream out) throws IOException { + SymbolTable systemTable = SharedSymbolTable.getSystemSymbolTable(1); + writer.addTypeAnnotationSymbol(systemTable.findSymbol("$ion_symbol_table")); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(systemTable.findSymbol("symbols")); + writer.stepIn(IonType.LIST); + writer.writeString("ghi"); + writer.stepOut(); + writer.setFieldNameSymbol(systemTable.findSymbol("imports")); + writer.stepIn(IonType.LIST); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(systemTable.findSymbol("name")); + writer.writeString("foo"); + writer.setFieldNameSymbol(systemTable.findSymbol("version")); + writer.writeInt(1); + writer.setFieldNameSymbol(systemTable.findSymbol("max_id")); + writer.writeInt(2); + writer.stepOut(); + writer.stepOut(); + writer.stepOut(); + writer.writeSymbolToken(10); + writer.writeSymbolToken(11); + writer.writeSymbolToken(12); + writer.addTypeAnnotationSymbol(systemTable.findSymbol("$ion_symbol_table")); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(systemTable.findSymbol("symbols")); + writer.stepIn(IonType.LIST); + writer.writeString("xyz"); + writer.writeString("uvw"); + writer.writeString("rst"); + writer.stepOut(); + writer.setFieldNameSymbol(systemTable.findSymbol("imports")); + writer.stepIn(IonType.LIST); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(systemTable.findSymbol("name")); + writer.writeString("bar"); + writer.setFieldNameSymbol(systemTable.findSymbol("version")); + writer.writeInt(1); + writer.setFieldNameSymbol(systemTable.findSymbol("max_id")); + writer.writeInt(1); + writer.stepOut(); + writer.stepOut(); + writer.stepOut(); + writer.writeSymbolToken(10); + writer.writeSymbolToken(11); + writer.writeSymbolToken(12); + writer.writeSymbolToken(13); + } + }); + + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("abc", reader.stringValue()); + SymbolTable[] imports = reader.getSymbolTable().getImportedTables(); + assertEquals(1, imports.length); + assertEquals("foo", imports[0].getName()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("def", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("ghi", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("baz", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + imports = reader.getSymbolTable().getImportedTables(); + assertEquals(1, imports.length); + assertEquals("bar", imports[0].getName()); + assertEquals("xyz", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("uvw", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("rst", reader.stringValue()); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void ivmResetsImports() throws Exception { + SimpleCatalog catalog = new SimpleCatalog(); + catalog.putTable(SYSTEM.newSharedSymbolTable("foo", 1, Arrays.asList("abc", "def").iterator())); + readerBuilder = IonReaderBuilder.standard().withCatalog(catalog); + + IonReaderBinaryIncremental reader = readerFor(new RawWriterFunction() { + @Override + public void write(_Private_IonRawWriter writer, ByteArrayOutputStream out) throws IOException { + SymbolTable systemTable = SharedSymbolTable.getSystemSymbolTable(1); + writer.addTypeAnnotationSymbol(systemTable.findSymbol("$ion_symbol_table")); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(systemTable.findSymbol("symbols")); + writer.stepIn(IonType.LIST); + writer.writeString("ghi"); + writer.stepOut(); + writer.setFieldNameSymbol(systemTable.findSymbol("imports")); + writer.stepIn(IonType.LIST); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(systemTable.findSymbol("name")); + writer.writeString("foo"); + writer.setFieldNameSymbol(systemTable.findSymbol("version")); + writer.writeInt(1); + writer.setFieldNameSymbol(systemTable.findSymbol("max_id")); + writer.writeInt(2); + writer.stepOut(); + writer.stepOut(); + writer.stepOut(); + writer.writeSymbolToken(10); + writer.writeSymbolToken(11); + writer.writeSymbolToken(12); + writer.close(); + out.write(_Private_IonConstants.BINARY_VERSION_MARKER_1_0); + out.write(0x20); + } + }); + + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("abc", reader.stringValue()); + SymbolTable[] imports = reader.getSymbolTable().getImportedTables(); + assertEquals(1, imports.length); + assertEquals("foo", imports[0].getName()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("def", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("ghi", reader.stringValue()); + assertEquals(IonType.INT, reader.next()); + assertTrue(reader.getSymbolTable().isSystemTable()); + assertEquals(0, reader.intValue()); + assertNull(reader.next()); + reader.close(); + } + + private static void assertSymbolEquals( + String expectedText, + IonReaderBinaryIncremental.ImportLocation expectedImportLocation, + SymbolToken actual + ) { + assertEquals(expectedText, actual.getText()); + IonReaderBinaryIncremental.SymbolTokenImpl impl = (IonReaderBinaryIncremental.SymbolTokenImpl) actual; + assertEquals(expectedImportLocation, impl.getImportLocation()); + } + + @Test + public void symbolsAsTokens() throws Exception { + IonReaderBinaryIncremental reader = readerFor("{foo: uvw::abc, bar: qrs::xyz::def}"); + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.SYMBOL, reader.next()); + assertSymbolEquals("foo", null, reader.getFieldNameSymbol()); + SymbolToken[] annotations = reader.getTypeAnnotationSymbols(); + assertEquals(1, annotations.length); + assertSymbolEquals("uvw", null, annotations[0]); + assertSymbolEquals("abc", null, reader.symbolValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertSymbolEquals("bar", null, reader.getFieldNameSymbol()); + annotations = reader.getTypeAnnotationSymbols(); + assertEquals(2, annotations.length); + assertSymbolEquals("qrs", null, annotations[0]); + assertSymbolEquals("xyz", null, annotations[1]); + assertSymbolEquals("def", null, reader.symbolValue()); + reader.stepOut(); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void nullContainers() throws Exception { + IonReader reader = readerFor("null.struct null.list null.sexp"); + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertNull(reader.next()); + reader.stepOut(); + assertEquals(IonType.LIST, reader.next()); + reader.stepIn(); + assertNull(reader.next()); + reader.stepOut(); + assertEquals(IonType.SEXP, reader.next()); + reader.stepIn(); + assertNull(reader.next()); + reader.stepOut(); + reader.close(); + } + + @Test + public void intNegativeZeroFails() throws Exception { + IonReaderBinaryIncremental reader = readerFor(0x31, 0x00); + reader.next(); + thrown.expect(IonException.class); + reader.longValue(); + } + + @Test + public void bigIntNegativeZeroFails() throws Exception { + IonReaderBinaryIncremental reader = readerFor(0x31, 0x00); + reader.next(); + thrown.expect(IonException.class); + reader.bigIntegerValue(); + } + + @Test + public void listWithLengthTooShortFails() throws Exception { + IonReaderBinaryIncremental reader = readerFor(0xB1, 0x21, 0x01); + assertEquals(IonType.LIST, reader.next()); + reader.stepIn(); + thrown.expect(IonException.class); + reader.next(); + } + + @Test + public void noOpPadTooShort1() throws Exception { + IonReaderBinaryIncremental reader = readerFor(0x37, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01); + assertEquals(IonType.INT, reader.next()); + assertNull(reader.next()); + thrown.expect(IonException.class); + reader.close(); + } + + @Test + public void noOpPadTooShort2() throws Exception { + IonReaderBinaryIncremental reader = readerFor( + 0x0e, 0x90, 0x00, 0xde, 0xad, 0xbe, 0xef, 0xca, 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + ); + assertNull(reader.next()); + thrown.expect(IonException.class); + reader.close(); + } + + @Test + public void nopPadOneByte() throws Exception { + IonReaderBinaryIncremental reader = readerFor(0); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void localSidOutOfRangeStringValue() throws Exception { + IonReaderBinaryIncremental reader = readerFor(0x71, 0x0A); // SID 10 + assertEquals(IonType.SYMBOL, reader.next()); + thrown.expect(IonException.class); + reader.stringValue(); + } + + @Test + public void localSidOutOfRangeSymbolValue() throws Exception { + IonReaderBinaryIncremental reader = readerFor(0x71, 0x0A); // SID 10 + assertEquals(IonType.SYMBOL, reader.next()); + thrown.expect(IonException.class); + reader.symbolValue(); + } + + @Test + public void localSidOutOfRangeFieldName() throws Exception { + IonReaderBinaryIncremental reader = readerFor( + 0xD2, // Struct, length 2 + 0x8A, // SID 10 + 0x20 // int 0 + ); + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.INT, reader.next()); + thrown.expect(IonException.class); + reader.getFieldName(); + } + + @Test + public void localSidOutOfRangeFieldNameSymbol() throws Exception { + IonReaderBinaryIncremental reader = readerFor( + 0xD2, // Struct, length 2 + 0x8A, // SID 10 + 0x20 // int 0 + ); + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.INT, reader.next()); + thrown.expect(IonException.class); + reader.getFieldNameSymbol(); + } + + @Test + public void localSidOutOfRangeAnnotation() throws Exception { + IonReaderBinaryIncremental reader = readerFor( + 0xE3, // Annotation wrapper, length 3 + 0x81, // annotation SID length 1 + 0x8A, // SID 10 + 0x20 // int 0 + ); + assertEquals(IonType.INT, reader.next()); + thrown.expect(IonException.class); + reader.getTypeAnnotations(); + } + + @Test + public void localSidOutOfRangeAnnotationSymbol() throws Exception { + IonReaderBinaryIncremental reader = readerFor( + 0xE3, // Annotation wrapper, length 3 + 0x81, // annotation SID length 1 + 0x8A, // SID 10 + 0x20 // int 0 + ); + assertEquals(IonType.INT, reader.next()); + thrown.expect(IonException.class); + reader.getTypeAnnotationSymbols(); + } + + @Test + public void localSidOutOfRangeIterateAnnotations() throws Exception { + IonReaderBinaryIncremental reader = readerFor( + 0xE3, // Annotation wrapper, length 3 + 0x81, // annotation SID length 1 + 0x8A, // SID 10 + 0x20 // int 0 + ); + assertEquals(IonType.INT, reader.next()); + Iterator annotationIterator = reader.iterateTypeAnnotations(); + thrown.expect(IonException.class); + annotationIterator.next(); + } + + @Test + public void stepInOnScalarFails() throws Exception { + IonReaderBinaryIncremental reader = readerFor(0x20); + assertEquals(IonType.INT, reader.next()); + thrown.expect(IonException.class); + reader.stepIn(); + } + + @Test + public void stepInBeforeNextFails() throws Exception { + IonReaderBinaryIncremental reader = readerFor(0xD2, 0x84, 0xD0); + reader.next(); + reader.stepIn(); + thrown.expect(IonException.class); + reader.stepIn(); + } + + @Test + public void stepOutAtDepthZeroFails() throws Exception { + IonReaderBinaryIncremental reader = readerFor(0x20); + reader.next(); + thrown.expect(IllegalStateException.class); + reader.stepOut(); + } + + @Test + public void byteSizeNotOnLobFails() throws Exception { + IonReaderBinaryIncremental reader = readerFor(0x20); + reader.next(); + thrown.expect(IonException.class); + reader.byteSize(); + } + + @Test + public void doubleValueOnIntFails() throws Exception { + IonReaderBinaryIncremental reader = readerFor(0x20); + reader.next(); + thrown.expect(IllegalStateException.class); + reader.doubleValue(); + } + + @Test + public void floatWithInvalidLengthFails() throws Exception { + IonReaderBinaryIncremental reader = readerFor(0x43, 0x01, 0x02, 0x03); + thrown.expect(IonException.class); + reader.next(); + } + + @Test + public void invalidTypeIdFFailsAtTopLevel() throws Exception { + IonReaderBinaryIncremental reader = readerFor(0xF0); + thrown.expect(IonException.class); + reader.next(); + } + + @Test + public void invalidTypeIdFFailsBelowTopLevel() throws Exception { + IonReaderBinaryIncremental reader = readerFor(0xB1, 0xF0); + reader.next(); + reader.stepIn(); + thrown.expect(IonException.class); + reader.next(); + } + + @Test + public void reallyLargeString() throws Exception { + StringBuilder sb = new StringBuilder(); + // 8192 is a arbitrarily large; it requires a couple bytes of length, and it doesn't fit in the preallocated + // string decoding buffer of size 4096. + for (int i = 0; i < 8192; i++) { + sb.append('a'); + } + String string = sb.toString(); + IonReaderBinaryIncremental reader = readerFor("\"" + string + "\""); + assertEquals(IonType.STRING, reader.next()); + assertEquals(string, reader.stringValue()); + reader.close(); + } + + @Test + public void nopPadInAnnotationWrapperFails() throws Exception { + IonReaderBinaryIncremental reader = readerFor( + 0xB5, // list + 0xE4, // annotation wrapper + 0x81, // 1 byte of annotations + 0x84, // annotation: "name" + 0x01, // 2-byte no-op pad + 0x00 + ); + reader.next(); + reader.stepIn(); + thrown.expect(IonException.class); + reader.next(); + } + + @Test + public void nestedAnnotationWrapperFails() throws Exception { + IonReaderBinaryIncremental reader = readerFor( + 0xB5, // list + 0xE4, // annotation wrapper + 0x81, // 1 byte of annotations + 0x84, // annotation: "name" + 0xE3, // annotation wrapper + 0x81, // 1 byte of annotations + 0x84, // annotation: "name" + 0x20 // int 0 + ); + reader.next(); + reader.stepIn(); + thrown.expect(IonException.class); + reader.next(); + } + + @Test + public void annotationWrapperLengthMismatchFails() throws Exception { + IonReaderBinaryIncremental reader = readerFor( + 0xB5, // list + 0xE4, // annotation wrapper + 0x81, // 1 byte of annotations + 0x84, // annotation: "name" + 0x20, // int 0 + 0x20 // next value + ); + reader.next(); + reader.stepIn(); + thrown.expect(IonException.class); + reader.next(); + } + + @Test + public void multipleSymbolTableImportsFieldsFails() throws Exception { + IonReaderBinaryIncremental reader = readerFor(new RawWriterFunction() { + @Override + public void write(_Private_IonRawWriter writer, ByteArrayOutputStream out) throws IOException { + SymbolTable systemTable = SharedSymbolTable.getSystemSymbolTable(1); + writer.addTypeAnnotationSymbol(systemTable.findSymbol("$ion_symbol_table")); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(systemTable.findSymbol("imports")); + writer.stepIn(IonType.LIST); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(systemTable.findSymbol("name")); + writer.writeString("bar"); + writer.setFieldNameSymbol(systemTable.findSymbol("version")); + writer.writeInt(1); + writer.setFieldNameSymbol(systemTable.findSymbol("max_id")); + writer.writeInt(1); + writer.stepOut(); + writer.stepOut(); + writer.setFieldNameSymbol(systemTable.findSymbol("symbols")); + writer.stepIn(IonType.LIST); + writer.writeString("ghi"); + writer.stepOut(); + writer.setFieldNameSymbol(systemTable.findSymbol("imports")); + writer.stepIn(IonType.LIST); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(systemTable.findSymbol("name")); + writer.writeString("foo"); + writer.setFieldNameSymbol(systemTable.findSymbol("version")); + writer.writeInt(1); + writer.setFieldNameSymbol(systemTable.findSymbol("max_id")); + writer.writeInt(2); + writer.stepOut(); + writer.stepOut(); + writer.stepOut(); + writer.writeSymbolToken(10); + } + }); + + thrown.expect(IonException.class); + reader.next(); + } + + @Test + public void multipleSymbolTableSymbolsFieldsFails() throws Exception { + IonReaderBinaryIncremental reader = readerFor(new RawWriterFunction() { + @Override + public void write(_Private_IonRawWriter writer, ByteArrayOutputStream out) throws IOException { + SymbolTable systemTable = SharedSymbolTable.getSystemSymbolTable(1); + writer.addTypeAnnotationSymbol(systemTable.findSymbol("$ion_symbol_table")); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(systemTable.findSymbol("imports")); + writer.stepIn(IonType.LIST); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(systemTable.findSymbol("name")); + writer.writeString("bar"); + writer.setFieldNameSymbol(systemTable.findSymbol("version")); + writer.writeInt(1); + writer.setFieldNameSymbol(systemTable.findSymbol("max_id")); + writer.writeInt(1); + writer.stepOut(); + writer.stepOut(); + writer.setFieldNameSymbol(systemTable.findSymbol("symbols")); + writer.stepIn(IonType.LIST); + writer.writeString("ghi"); + writer.stepOut(); + writer.setFieldNameSymbol(systemTable.findSymbol("symbols")); + writer.stepIn(IonType.LIST); + writer.writeString("abc"); + writer.stepOut(); + writer.stepOut(); + writer.writeSymbolToken(10); + } + }); + + thrown.expect(IonException.class); + reader.next(); + } + + @Test + public void nonStringInSymbolsListCreatesNullSlot() throws Exception { + IonReaderBinaryIncremental reader = readerFor(new RawWriterFunction() { + @Override + public void write(_Private_IonRawWriter writer, ByteArrayOutputStream out) throws IOException { + SymbolTable systemTable = SharedSymbolTable.getSystemSymbolTable(1); + writer.addTypeAnnotationSymbol(systemTable.findSymbol("$ion_symbol_table")); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(systemTable.findSymbol("symbols")); + writer.stepIn(IonType.LIST); + writer.writeString(null); + writer.writeString("abc"); + writer.writeInt(123); + writer.stepOut(); + writer.stepOut(); + writer.writeSymbolToken(10); + writer.writeSymbolToken(11); + writer.writeSymbolToken(12); + } + }); + + assertEquals(IonType.SYMBOL, reader.next()); + SymbolToken symbolValue = reader.symbolValue(); + assertNull(symbolValue.getText()); + assertEquals(0, symbolValue.getSid()); + assertEquals(IonType.SYMBOL, reader.next()); + symbolValue = reader.symbolValue(); + assertEquals("abc", symbolValue.getText()); + assertEquals(IonType.SYMBOL, reader.next()); + symbolValue = reader.symbolValue(); + assertNull(symbolValue.getText()); + assertEquals(0, symbolValue.getSid()); + reader.close(); + } + + @Test + public void symbolTableWithMultipleImportsCorrectlyAssignsImportLocations() throws Exception { + SimpleCatalog catalog = new SimpleCatalog(); + catalog.putTable(SYSTEM.newSharedSymbolTable("foo", 1, Arrays.asList("abc", "def").iterator())); + catalog.putTable(SYSTEM.newSharedSymbolTable("bar", 1, Arrays.asList("123", "456").iterator())); + readerBuilder = IonReaderBuilder.standard().withCatalog(catalog); + + IonReaderBinaryIncremental reader = readerFor(new RawWriterFunction() { + @Override + public void write(_Private_IonRawWriter writer, ByteArrayOutputStream out) throws IOException { + SymbolTable systemTable = SharedSymbolTable.getSystemSymbolTable(1); + writer.addTypeAnnotationSymbol(systemTable.findSymbol("$ion_symbol_table")); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(systemTable.findSymbol("imports")); + writer.stepIn(IonType.LIST); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(systemTable.findSymbol("name")); + writer.writeString("foo"); + writer.setFieldNameSymbol(systemTable.findSymbol("version")); + writer.writeInt(1); + writer.setFieldNameSymbol(systemTable.findSymbol("max_id")); + writer.writeInt(4); // The matching shared symbol table in the catalog only declares two symbols. + writer.stepOut(); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(systemTable.findSymbol("name")); + writer.writeString("bar"); + writer.setFieldNameSymbol(systemTable.findSymbol("version")); + writer.writeInt(1); + writer.setFieldNameSymbol(systemTable.findSymbol("max_id")); + // The matching shared symbol table in the catalog declares two symbols, but only one is used. + writer.writeInt(1); + writer.stepOut(); + writer.stepIn(IonType.STRUCT); + writer.setFieldNameSymbol(systemTable.findSymbol("name")); + writer.writeString("baz"); // There is no match in the catalog; all symbols have unknown text. + writer.setFieldNameSymbol(systemTable.findSymbol("version")); + writer.writeInt(1); + writer.setFieldNameSymbol(systemTable.findSymbol("max_id")); + writer.writeInt(2); + writer.stepOut(); + writer.stepOut(); + writer.stepOut(); + writer.writeSymbolToken(10); // abc + writer.writeSymbolToken(11); // def + writer.writeSymbolToken(12); // unknown text, import SID 3 (from foo) + writer.writeSymbolToken(13); // unknown text, import SID 4 (from foo) + writer.writeSymbolToken(14); // 123 + writer.writeSymbolToken(15); // unknown text, import SID 1 (from baz) + writer.writeSymbolToken(16); // unknown text, import SID 2 (from baz) + } + }); + + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("abc", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("def", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + IonReaderBinaryIncremental.SymbolTokenImpl symbolValue = + (IonReaderBinaryIncremental.SymbolTokenImpl) reader.symbolValue(); + assertNull(symbolValue.getText()); + assertEquals(new IonReaderBinaryIncremental.ImportLocation("foo", 3), symbolValue.getImportLocation()); + assertEquals(IonType.SYMBOL, reader.next()); + symbolValue = (IonReaderBinaryIncremental.SymbolTokenImpl) reader.symbolValue(); + assertNull(symbolValue.getText()); + assertEquals(new IonReaderBinaryIncremental.ImportLocation("foo", 4), symbolValue.getImportLocation()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("123", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + symbolValue = (IonReaderBinaryIncremental.SymbolTokenImpl) reader.symbolValue(); + assertEquals( + new IonReaderBinaryIncremental.SymbolTokenImpl( + null, + 15, + new IonReaderBinaryIncremental.ImportLocation("baz", 1) + ), + symbolValue + ); + assertEquals(IonType.SYMBOL, reader.next()); + symbolValue = (IonReaderBinaryIncremental.SymbolTokenImpl) reader.symbolValue(); + assertEquals( + new IonReaderBinaryIncremental.SymbolTokenImpl( + null, + 16, + new IonReaderBinaryIncremental.ImportLocation("baz", 2) + ), + symbolValue + ); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void symbolTableSnapshotImplementsBasicMethods() throws Exception { + IonReaderBinaryIncremental reader = readerFor("'abc'"); + reader.next(); + SymbolTable symbolTable = reader.getSymbolTable(); + assertNull(symbolTable.getName()); + assertEquals(0, symbolTable.getVersion()); + assertTrue(symbolTable.isLocalTable()); + assertTrue(symbolTable.isReadOnly()); + assertFalse(symbolTable.isSharedTable()); + assertFalse(symbolTable.isSystemTable()); + assertFalse(symbolTable.isSubstitute()); + symbolTable.makeReadOnly(); + assertEquals(10, symbolTable.getMaxId()); + assertEquals("abc", symbolTable.findKnownSymbol(10)); + assertNull(symbolTable.findKnownSymbol(symbolTable.getMaxId() + 1)); + thrown.expect(IllegalArgumentException.class); + symbolTable.findKnownSymbol(-1); + } + + @Test + public void singleValueExceedsBufferSize() throws Exception { + readerBuilder = IonReaderBuilder.standard().withBufferConfiguration( + IonBufferConfiguration.Builder.standard().withInitialBufferSize(8).build() + ); + IonReaderBinaryIncremental reader = readerFor("\"abcdefghijklmnopqrstuvwxyz\""); + assertEquals(IonType.STRING, reader.next()); + assertEquals("abcdefghijklmnopqrstuvwxyz", reader.stringValue()); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void maximumBufferSizeTooSmallFails() { + IonBufferConfiguration.Builder builder = IonBufferConfiguration.Builder.standard(); + builder + .withMaximumBufferSize(builder.getMinimumMaximumBufferSize() - 1) + .withInitialBufferSize(builder.getMinimumMaximumBufferSize() - 1) + .onOversizedValue(builder.getNoOpOversizedValueHandler()) + .onOversizedSymbolTable(builder.getNoOpOversizedSymbolTableHandler()) + .onData(builder.getNoOpDataHandler()); + thrown.expect(IllegalArgumentException.class); + builder.build(); + } + + @Test + public void maximumBufferSizeWithoutHandlerFails() { + IonBufferConfiguration.Builder builder = IonBufferConfiguration.Builder.standard(); + builder + .withMaximumBufferSize(9) + .withInitialBufferSize(9); + thrown.expect(IllegalArgumentException.class); + builder.build(); + } + + /** + * Unified handler interface to reduce boilerplate when defining test handlers. + */ + private interface UnifiedTestHandler extends + BufferConfiguration.OversizedValueHandler, + IonBufferConfiguration.OversizedSymbolTableHandler, + BufferConfiguration.DataHandler { + // Empty. + } + + @Test + public void oversizeValueDetectedDuringMultiByteRead() throws Exception { + byte[] bytes = toBinary( + "\"abcdefghijklmnopqrstuvwxyz\" " + // Requires 32 bytes (4 IVM, 1 TID, 1 length, 26 chars) + "\"abc\" " + + "\"abcdefghijklmnopqrstuvwxyz\" " + + "\"def\"" + ); + final AtomicInteger oversizedCounter = new AtomicInteger(); + final AtomicInteger byteCounter = new AtomicInteger(); + UnifiedTestHandler handler = new UnifiedTestHandler() { + @Override + public void onOversizedSymbolTable() { + throw new IllegalStateException("not expected"); + } + + @Override + public void onOversizedValue() { + oversizedCounter.incrementAndGet(); + } + + @Override + public void onData(int numberOfBytes) { + byteCounter.addAndGet(numberOfBytes); + } + }; + readerBuilder = IonReaderBuilder.standard().withBufferConfiguration( + IonBufferConfiguration.Builder.standard() + .withInitialBufferSize(8) + .withMaximumBufferSize(16) + .onOversizedValue(handler) + .onOversizedSymbolTable(handler) + .onData(handler) + .build() + ); + IonReaderBinaryIncremental reader = new IonReaderBinaryIncremental(readerBuilder, new ByteArrayInputStream(bytes)); + + assertEquals(IonType.STRING, reader.next()); + assertEquals("abc", reader.stringValue()); + assertEquals(1, oversizedCounter.get()); + assertEquals(IonType.STRING, reader.next()); + assertEquals("def", reader.stringValue()); + assertNull(reader.next()); + reader.close(); + assertEquals(2, oversizedCounter.get()); + assertEquals(bytes.length, byteCounter.get()); + } + + @Test + public void oversizeValueDetectedDuringMultiByteReadIncremental() throws Exception { + byte[] bytes = toBinary( + "\"abcdefghijklmnopqrstuvwxyz\" " + // Requires 32 bytes (4 IVM, 1 TID, 1 length, 26 chars) + "\"abc\" " + + "\"abcdefghijklmnopqrstuvwxyz\" " + + "\"def\"" + ); + final AtomicInteger oversizedCounter = new AtomicInteger(); + final AtomicInteger byteCounter = new AtomicInteger(); + UnifiedTestHandler handler = new UnifiedTestHandler() { + @Override + public void onOversizedSymbolTable() { + throw new IllegalStateException("not expected"); + } + + @Override + public void onOversizedValue() { + oversizedCounter.incrementAndGet(); + } + + @Override + public void onData(int numberOfBytes) { + byteCounter.addAndGet(numberOfBytes); + } + }; + ResizingPipedInputStream pipe = new ResizingPipedInputStream(bytes.length); + IonReaderBuilder builder = IonReaderBuilder.standard().withBufferConfiguration( + IonBufferConfiguration.Builder.standard() + .withInitialBufferSize(8) + .withMaximumBufferSize(16) + .onOversizedValue(handler) + .onOversizedSymbolTable(handler) + .onData(handler) + .build() + ); + IonReaderBinaryIncremental reader = new IonReaderBinaryIncremental(builder, pipe); + int valueCounter = 0; + for (byte b : bytes) { + pipe.receive(b); + IonType type = reader.next(); + if (type != null) { + valueCounter++; + assertTrue(valueCounter < 3); + if (valueCounter == 1) { + assertEquals(IonType.STRING, type); + assertEquals("abc", reader.stringValue()); + assertEquals(1, oversizedCounter.get()); + } else { + assertEquals(2, valueCounter); + assertEquals(IonType.STRING, type); + assertEquals("def", reader.stringValue()); + assertEquals(2, oversizedCounter.get()); + } + } + } + assertEquals(2, valueCounter); + assertNull(reader.next()); + reader.close(); + assertEquals(2, oversizedCounter.get()); + assertEquals(bytes.length, byteCounter.get()); + } + + @Test + public void oversizeValueDetectedDuringSingleByteRead() throws Exception { + // Unlike the previous test, where excessive size is detected when trying to skip past the value portion, + // this test verifies that excessive size can be detected while reading a value header, which happens + // byte-by-byte. + byte[] bytes = toBinary("\"abcdefghijklmnopqrstuvwxyz\""); // Requires a 2-byte header. + final AtomicInteger oversizedCounter = new AtomicInteger(); + final AtomicInteger byteCounter = new AtomicInteger(); + UnifiedTestHandler handler = new UnifiedTestHandler() { + @Override + public void onOversizedSymbolTable() { + throw new IllegalStateException("not expected"); + } + + @Override + public void onOversizedValue() { + oversizedCounter.incrementAndGet(); + } + + @Override + public void onData(int numberOfBytes) { + byteCounter.addAndGet(numberOfBytes); + } + }; + readerBuilder = IonReaderBuilder.standard().withBufferConfiguration( + IonBufferConfiguration.Builder.standard() + .withInitialBufferSize(5) + .withMaximumBufferSize(5) + .onOversizedValue(handler) + .onOversizedSymbolTable(handler) + .onData(handler) + .build() + ); + IonReaderBinaryIncremental reader = new IonReaderBinaryIncremental(readerBuilder, new ByteArrayInputStream(bytes)); + + // The maximum buffer size is 5, which will be exceeded after the IVM (4 bytes), the type ID (1 byte), and + // the length byte (1 byte). + assertNull(reader.next()); + reader.close(); + assertEquals(1, oversizedCounter.get()); + assertEquals(bytes.length, byteCounter.get()); + } + + @Test + public void oversizeValueDetectedDuringSingleByteReadIncremental() throws Exception { + byte[] bytes = toBinary("\"abcdefghijklmnopqrstuvwxyz\""); // Requires a 2-byte header. + final AtomicInteger oversizedCounter = new AtomicInteger(); + final AtomicInteger byteCounter = new AtomicInteger(); + UnifiedTestHandler handler = new UnifiedTestHandler() { + @Override + public void onOversizedSymbolTable() { + throw new IllegalStateException("not expected"); + } + + @Override + public void onOversizedValue() { + oversizedCounter.incrementAndGet(); + } + + @Override + public void onData(int numberOfBytes) { + byteCounter.addAndGet(numberOfBytes); + } + }; + ResizingPipedInputStream pipe = new ResizingPipedInputStream(bytes.length); + IonReaderBuilder builder = IonReaderBuilder.standard().withBufferConfiguration( + IonBufferConfiguration.Builder.standard() + .withInitialBufferSize(5) + .withMaximumBufferSize(5) + .onOversizedValue(handler) + .onOversizedSymbolTable(handler) + .onData(handler) + .build() + ); + IonReaderBinaryIncremental reader = new IonReaderBinaryIncremental(builder, pipe); + for (byte b : bytes) { + assertNull(reader.next()); + pipe.receive(b); + } + // The maximum buffer size is 5, which will be exceeded after the IVM (4 bytes), the type ID (1 byte), and + // the length byte (1 byte). + assertNull(reader.next()); + reader.close(); + assertEquals(1, oversizedCounter.get()); + assertEquals(bytes.length, byteCounter.get()); + } + + @Test + public void oversizeValueDetectedDuringReadOfTypeIdOfSymbolTableAnnotatedValue() throws Exception { + // This value is not a symbol table, but follows most of the code path that symbol tables follow. Ensure that + // `onOversizedValue` (NOT `onOversizedSymbolTable`) is called, and that the stream continues to be read. + byte[] bytes = toBinary("$ion_symbol_table::123 \"a\""); + final AtomicInteger oversizedCounter = new AtomicInteger(); + final AtomicInteger byteCounter = new AtomicInteger(); + UnifiedTestHandler handler = new UnifiedTestHandler() { + @Override + public void onOversizedSymbolTable() { + throw new IllegalStateException("not expected"); + } + + @Override + public void onOversizedValue() { + oversizedCounter.incrementAndGet(); + } + + @Override + public void onData(int numberOfBytes) { + byteCounter.addAndGet(numberOfBytes); + } + }; + readerBuilder = IonReaderBuilder.standard().withBufferConfiguration( + IonBufferConfiguration.Builder.standard() + .withInitialBufferSize(7) + .withMaximumBufferSize(7) + .onOversizedValue(handler) + .onOversizedSymbolTable(handler) + .onData(handler) + .build() + ); + IonReaderBinaryIncremental reader = new IonReaderBinaryIncremental(readerBuilder, new ByteArrayInputStream(bytes)); + + // The maximum buffer size is 7, which will be exceeded after the IVM (4 bytes), the annotation wrapper type ID + // (1 byte), the annotations length (1 byte), and the annotation SID 3 (1 byte). The next byte is the wrapped + // value type ID byte. + assertEquals(IonType.STRING, reader.next()); + assertEquals("a", reader.stringValue()); + assertEquals(1, oversizedCounter.get()); + assertNull(reader.next()); + reader.close(); + assertEquals(bytes.length, byteCounter.get()); + } + + @Test + public void oversizeValueDetectedDuringReadOfTypeIdOfSymbolTableAnnotatedValueIncremental() throws Exception { + // This value is not a symbol table, but follows most of the code path that symbol tables follow. Ensure that + // `onOversizedValue` (NOT `onOversizedSymbolTable`) is called, and that the stream continues to be read. + byte[] bytes = toBinary("$ion_symbol_table::123 \"a\""); + final AtomicInteger oversizedCounter = new AtomicInteger(); + final AtomicInteger byteCounter = new AtomicInteger(); + UnifiedTestHandler handler = new UnifiedTestHandler() { + @Override + public void onOversizedSymbolTable() { + throw new IllegalStateException("not expected"); + } + + @Override + public void onOversizedValue() { + oversizedCounter.incrementAndGet(); + } + + @Override + public void onData(int numberOfBytes) { + byteCounter.addAndGet(numberOfBytes); + } + }; + ResizingPipedInputStream pipe = new ResizingPipedInputStream(bytes.length); + IonReaderBuilder builder = IonReaderBuilder.standard().withBufferConfiguration( + IonBufferConfiguration.Builder.standard() + .withInitialBufferSize(7) + .withMaximumBufferSize(7) + .onOversizedValue(handler) + .onOversizedSymbolTable(handler) + .onData(handler) + .build() + ); + IonReaderBinaryIncremental reader = new IonReaderBinaryIncremental(builder, pipe); + + // The maximum buffer size is 7, which will be exceeded after the IVM (4 bytes), the annotation wrapper type ID + // (1 byte), the annotations length (1 byte), and the annotation SID 3 (1 byte). The next byte is the wrapped + // value type ID byte. + boolean foundValue = false; + for (byte b : bytes) { + pipe.receive(b); + IonType type = reader.next(); + if (type != null) { + assertFalse(foundValue); + assertEquals(IonType.STRING, type); + assertEquals("a", reader.stringValue()); + assertEquals(1, oversizedCounter.get()); + foundValue = true; + } + } + assertTrue(foundValue); + assertEquals(1, oversizedCounter.get()); + assertNull(reader.next()); + reader.close(); + assertEquals(bytes.length, byteCounter.get()); + } + + @Test + public void oversizeValueWithSymbolTable() throws Exception { + // The first value is oversized because it cannot be buffered at the same time as the preceding symbol table + // without exceeding the maximum buffer size. The next value fits. + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IonWriter writer = IonBinaryWriterBuilder.standard().build(out); + writer.writeString("12345678"); + writer.writeSymbol("abcdefghijklmnopqrstuvwxyz"); // Requires 32 bytes (4 IVM, 1 TID, 1 length, 26 chars) + writer.close(); + // The system values require ~40 bytes (4 IVM, 5 symtab struct header, 1 'symbols' sid, 2 list header, 2 + 26 + // for symbol 10. + // The string "12345678" requires 9 bytes, bringing the total to ~49, above the max of 48. + final AtomicInteger oversizedCounter = new AtomicInteger(); + final AtomicInteger byteCounter = new AtomicInteger(); + UnifiedTestHandler handler = new UnifiedTestHandler() { + @Override + public void onOversizedSymbolTable() { + throw new IllegalStateException("not expected"); + } + + @Override + public void onOversizedValue() { + oversizedCounter.incrementAndGet(); + } + + @Override + public void onData(int numberOfBytes) { + byteCounter.addAndGet(numberOfBytes); + } + }; + IonReaderBuilder builder = IonReaderBuilder.standard().withBufferConfiguration( + IonBufferConfiguration.Builder.standard() + .withInitialBufferSize(8) + .withMaximumBufferSize(48) + .onOversizedValue(handler) + .onOversizedSymbolTable(handler) + .onData(handler) + .build() + ); + IonReaderBinaryIncremental reader = new IonReaderBinaryIncremental( + builder, + new ByteArrayInputStream(out.toByteArray()) + ); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals(1, oversizedCounter.get()); + assertEquals("abcdefghijklmnopqrstuvwxyz", reader.stringValue()); + assertNull(reader.next()); + reader.close(); + assertEquals(1, oversizedCounter.get()); + assertEquals(out.size(), byteCounter.get()); + } + + @Test + public void oversizeValueWithSymbolTableIncremental() throws Exception { + // The first value is oversized because it cannot be buffered at the same time as the preceding symbol table + // without exceeding the maximum buffer size. The next value fits. + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IonWriter writer = IonBinaryWriterBuilder.standard().build(out); + writer.writeString("12345678"); + writer.writeSymbol("abcdefghijklmnopqrstuvwxyz"); // Requires 32 bytes (4 IVM, 1 TID, 1 length, 26 chars) + writer.close(); + // The system values require ~40 bytes (4 IVM, 5 symtab struct header, 1 'symbols' sid, 2 list header, 2 + 26 + // for symbol 10. + // The string "12345678" requires 9 bytes, bringing the total to ~49, above the max of 48. + final AtomicInteger oversizedCounter = new AtomicInteger(); + final AtomicInteger byteCounter = new AtomicInteger(); + UnifiedTestHandler handler = new UnifiedTestHandler() { + @Override + public void onOversizedSymbolTable() { + throw new IllegalStateException("not expected"); + } + + @Override + public void onOversizedValue() { + oversizedCounter.incrementAndGet(); + } + + @Override + public void onData(int numberOfBytes) { + byteCounter.addAndGet(numberOfBytes); + } + }; + ResizingPipedInputStream pipe = new ResizingPipedInputStream(out.size()); + IonReaderBuilder builder = IonReaderBuilder.standard().withBufferConfiguration( + IonBufferConfiguration.Builder.standard() + .withInitialBufferSize(8) + .withMaximumBufferSize(48) + .onOversizedValue(handler) + .onOversizedSymbolTable(handler) + .onData(handler) + .build() + ); + IonReaderBinaryIncremental reader = new IonReaderBinaryIncremental(builder, pipe); + byte[] bytes = out.toByteArray(); + boolean foundValue = false; + for (byte b : bytes) { + pipe.receive(b); + IonType type = reader.next(); + if (type != null) { + assertFalse(foundValue); + assertEquals(IonType.SYMBOL, type); + assertEquals("abcdefghijklmnopqrstuvwxyz", reader.stringValue()); + assertEquals(1, oversizedCounter.get()); + foundValue = true; + } + } + assertTrue(foundValue); + assertNull(reader.next()); + reader.close(); + assertEquals(1, oversizedCounter.get()); + assertEquals(out.size(), byteCounter.get()); + } + + private void oversizeSymbolTableDetectedInHeader(int maximumBufferSize) throws Exception { + // The system values require ~40 bytes (4 IVM, 5 symtab struct header, 1 'symbols' sid, 2 list header, 2 + 26 + // for symbol 10. + byte[] bytes = toBinary("abcdefghijklmnopqrstuvwxyz"); + final AtomicInteger oversizedCounter = new AtomicInteger(); + UnifiedTestHandler handler = new UnifiedTestHandler() { + @Override + public void onOversizedSymbolTable() { + oversizedCounter.incrementAndGet(); + } + + @Override + public void onOversizedValue() { + throw new IllegalStateException("not expected"); + } + + @Override + public void onData(int numberOfBytes) { + + } + }; + IonReaderBuilder builder = IonReaderBuilder.standard().withBufferConfiguration( + IonBufferConfiguration.Builder.standard() + .withInitialBufferSize(maximumBufferSize) + .withMaximumBufferSize(maximumBufferSize) + .onOversizedValue(handler) + .onOversizedSymbolTable(handler) + .onData(handler) + .build() + ); + IonReaderBinaryIncremental reader = new IonReaderBinaryIncremental(builder, new ByteArrayInputStream(bytes)); + assertNull(reader.next()); + assertEquals(1, oversizedCounter.get()); + reader.close(); + } + + @Test + public void oversizeSymbolTableDetectedInHeader() throws Exception { + // The symbol table is determined to be oversized when reading the length of the annotation wrapper. + oversizeSymbolTableDetectedInHeader(5); + // The symbol table is determined to be oversized when reading the annotations length. + oversizeSymbolTableDetectedInHeader(6); + // The symbol table is determined to be oversized when reading SID 3. + oversizeSymbolTableDetectedInHeader(7); + // The symbol table is determined to be oversized when reading the type ID of the wrapped struct. + oversizeSymbolTableDetectedInHeader(8); + // The symbol table is determined to be oversized when reading the length of the wrapped struct. + oversizeSymbolTableDetectedInHeader(9); + } + + private void oversizeSymbolTableDetectedInHeaderIncremental(int maximumBufferSize) throws Exception { + // The system values require ~40 bytes (4 IVM, 5 symtab struct header, 1 'symbols' sid, 2 list header, 2 + 26 + // for symbol 10. + byte[] bytes = toBinary("abcdefghijklmnopqrstuvwxyz"); + final AtomicInteger oversizedCounter = new AtomicInteger(); + UnifiedTestHandler handler = new UnifiedTestHandler() { + @Override + public void onOversizedSymbolTable() { + oversizedCounter.incrementAndGet(); + } + + @Override + public void onOversizedValue() { + throw new IllegalStateException("not expected"); + } + + @Override + public void onData(int numberOfBytes) { + + } + }; + ResizingPipedInputStream pipe = new ResizingPipedInputStream(bytes.length); + IonReaderBuilder builder = IonReaderBuilder.standard().withBufferConfiguration( + IonBufferConfiguration.Builder.standard() + .withInitialBufferSize(maximumBufferSize) + .withMaximumBufferSize(maximumBufferSize) + .onOversizedValue(handler) + .onOversizedSymbolTable(handler) + .onData(handler) + .build() + ); + IonReaderBinaryIncremental reader = new IonReaderBinaryIncremental(builder, pipe); + for (byte b : bytes) { + assertNull(reader.next()); + pipe.receive(b); + } + assertNull(reader.next()); + assertEquals(1, oversizedCounter.get()); + reader.close(); + } + + @Test + public void oversizeSymbolTableDetectedInHeaderIncremental() throws Exception { + // The symbol table is determined to be oversized when reading the length of the annotation wrapper. + oversizeSymbolTableDetectedInHeaderIncremental(5); + // The symbol table is determined to be oversized when reading the annotations length. + oversizeSymbolTableDetectedInHeaderIncremental(6); + // The symbol table is determined to be oversized when reading SID 3. + oversizeSymbolTableDetectedInHeaderIncremental(7); + // The symbol table is determined to be oversized when reading the type ID of the wrapped struct. + oversizeSymbolTableDetectedInHeaderIncremental(8); + // The symbol table is determined to be oversized when reading the length of the wrapped struct. + oversizeSymbolTableDetectedInHeaderIncremental(9); + } + + @Test + public void oversizeSymbolTableDetectedInTheMiddle() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IonWriter writer = IonBinaryWriterBuilder.standard().build(out); + writer.writeString("12345678"); + writer.finish(); + writer.writeSymbol("abcdefghijklmnopqrstuvwxyz"); // Requires 32 bytes (4 IVM, 1 TID, 1 length, 26 chars) + writer.close(); + // The system values require ~40 bytes (4 IVM, 5 symtab struct header, 1 'symbols' sid, 2 list header, 2 + 26 + // for symbol 10. + final AtomicInteger oversizedCounter = new AtomicInteger(); + UnifiedTestHandler handler = new UnifiedTestHandler() { + @Override + public void onOversizedSymbolTable() { + oversizedCounter.incrementAndGet(); + } + + @Override + public void onOversizedValue() { + throw new IllegalStateException("not expected"); + } + + @Override + public void onData(int numberOfBytes) { + + } + }; + IonReaderBuilder builder = IonReaderBuilder.standard().withBufferConfiguration( + IonBufferConfiguration.Builder.standard() + .withInitialBufferSize(8) + .withMaximumBufferSize(32) + .onOversizedValue(handler) + .onOversizedSymbolTable(handler) + .onData(handler) + .build() + ); + IonReaderBinaryIncremental reader = new IonReaderBinaryIncremental( + builder, + new ByteArrayInputStream(out.toByteArray()) + ); + assertEquals(IonType.STRING, reader.next()); + assertEquals("12345678", reader.stringValue()); + assertNull(reader.next()); + assertEquals(1, oversizedCounter.get()); + reader.close(); + } + + @Test + public void oversizeSymbolTableDetectedInTheMiddleIncremental() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IonWriter writer = IonBinaryWriterBuilder.standard().build(out); + writer.writeString("12345678"); + writer.finish(); + writer.writeSymbol("abcdefghijklmnopqrstuvwxyz"); // Requires 32 bytes (4 IVM, 1 TID, 1 length, 26 chars) + writer.close(); + // The system values require ~40 bytes (4 IVM, 5 symtab struct header, 1 'symbols' sid, 2 list header, 2 + 26 + // for symbol 10. + final AtomicInteger oversizedCounter = new AtomicInteger(); + UnifiedTestHandler handler = new UnifiedTestHandler() { + @Override + public void onOversizedSymbolTable() { + oversizedCounter.incrementAndGet(); + } + + @Override + public void onOversizedValue() { + throw new IllegalStateException("not expected"); + } + + @Override + public void onData(int numberOfBytes) { + + } + }; + ResizingPipedInputStream pipe = new ResizingPipedInputStream(out.size()); + byte[] bytes = out.toByteArray(); + IonReaderBuilder builder = IonReaderBuilder.standard().withBufferConfiguration( + IonBufferConfiguration.Builder.standard() + .withInitialBufferSize(8) + .withMaximumBufferSize(32) + .onOversizedValue(handler) + .onOversizedSymbolTable(handler) + .onData(handler) + .build() + ); + IonReaderBinaryIncremental reader = new IonReaderBinaryIncremental(builder, pipe); + boolean foundValue = false; + for (byte b : bytes) { + IonType type = reader.next(); + if (type != null) { + assertFalse(foundValue); + assertEquals(IonType.STRING, type); + assertEquals("12345678", reader.stringValue()); + foundValue = true; + } + pipe.receive(b); + } + assertTrue(foundValue); + assertNull(reader.next()); + assertEquals(1, oversizedCounter.get()); + reader.close(); + } + + private static void writeFirstStruct(IonWriter writer) throws IOException { + //{ + // foo: bar, + // abc: [123, 456] + //} + writer.stepIn(IonType.STRUCT); + writer.setFieldName("foo"); + writer.writeSymbol("bar"); + writer.setFieldName("abc"); + writer.stepIn(IonType.LIST); + writer.writeInt(123); + writer.writeInt(456); + writer.stepOut(); + writer.stepOut(); + } + + private static void assertFirstStruct(IonReader reader) { + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("foo", reader.getFieldName()); + assertEquals("bar", reader.stringValue()); + assertEquals(IonType.LIST, reader.next()); + assertEquals("abc", reader.getFieldName()); + reader.stepIn(); + assertEquals(IonType.INT, reader.next()); + assertEquals(123, reader.intValue()); + assertEquals(IonType.INT, reader.next()); + assertEquals(456, reader.intValue()); + reader.stepOut(); + reader.stepOut(); + } + + private static void writeSecondStruct(IonWriter writer) throws IOException { + //{ + // foo: baz, + // abc: [42.0, 43e0] + //} + writer.stepIn(IonType.STRUCT); + writer.setFieldName("foo"); + writer.writeSymbol("baz"); + writer.setFieldName("abc"); + writer.stepIn(IonType.LIST); + writer.writeDecimal(new BigDecimal("42.0")); + writer.writeFloat(43.); + writer.stepOut(); + writer.stepOut(); + } + + private static void assertSecondStruct(IonReader reader) { + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("foo", reader.getFieldName()); + assertEquals("baz", reader.stringValue()); + assertEquals(IonType.LIST, reader.next()); + assertEquals("abc", reader.getFieldName()); + reader.stepIn(); + assertEquals(IonType.DECIMAL, reader.next()); + assertEquals(new BigDecimal("42.0"), reader.decimalValue()); + assertEquals(IonType.FLOAT, reader.next()); + assertEquals(43., reader.doubleValue(), 1e-9); + reader.stepOut(); + reader.stepOut(); + assertNull(reader.next()); + } + + IonReader newBoundedIncrementalReader(byte[] bytes, int maximumBufferSize) { + UnifiedTestHandler handler = new UnifiedTestHandler() { + @Override + public void onOversizedSymbolTable() { + Assert.fail("Oversized symbol table not expected."); + } + + @Override + public void onOversizedValue() { + Assert.fail("Oversized value not expected."); + } + + @Override + public void onData(int numberOfBytes) { + // Do nothing. + } + }; + IonReaderBuilder builder = IonReaderBuilder.standard().withBufferConfiguration( + IonBufferConfiguration.Builder.standard() + .withInitialBufferSize(maximumBufferSize) + .withMaximumBufferSize(maximumBufferSize) + .onOversizedValue(handler) + .onOversizedSymbolTable(handler) + .onData(handler) + .build() + ); + return new IonReaderBinaryIncremental( + builder, + new ByteArrayInputStream(bytes) + ); + } + + @Test + public void flushBetweenStructs() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IonWriter writer = _Private_IonManagedBinaryWriterBuilder + .create(_Private_IonManagedBinaryWriterBuilder.AllocatorMode.BASIC) + .withLocalSymbolTableAppendEnabled() + .newWriter(out); + writeFirstStruct(writer); + writer.flush(); + writeSecondStruct(writer); + writer.close(); + + IonReader reader = newBoundedIncrementalReader(out.toByteArray(), 64); + assertFirstStruct(reader); + assertSecondStruct(reader); + reader.close(); + } + + @Test + public void structsWithFloat32AndPreallocatedLength() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IonWriter writer = _Private_IonManagedBinaryWriterBuilder + .create(_Private_IonManagedBinaryWriterBuilder.AllocatorMode.BASIC) + .withPaddedLengthPreallocation(2) + .withFloatBinary32Enabled() + .newWriter(out); + writeFirstStruct(writer); + writeSecondStruct(writer); + writer.close(); + + IonReader reader = newBoundedIncrementalReader(out.toByteArray(), 64); + assertFirstStruct(reader); + assertSecondStruct(reader); + } + + @Test + public void nopPadThatFillsBufferFollowedByValueNotOversized() throws Exception { + TestUtils.BinaryIonAppender out = new TestUtils.BinaryIonAppender().append( + 0x03, 0x00, 0x00, 0x00, // 4 byte NOP pad. + 0x20 // Int 0. + ); + // The IVM is 4 bytes and the NOP pad is 4 bytes. The first value is the 9th byte and should not be considered + // oversize because the NOP pad can be discarded. + IonReader reader = newBoundedIncrementalReader(out.toByteArray(), 8); + assertEquals(IonType.INT, reader.next()); + assertEquals(0, reader.intValue()); + reader.close(); + } + + @Test + public void nopPadFollowedByValueThatOverflowsBufferNotOversized() throws Exception { + TestUtils.BinaryIonAppender out = new TestUtils.BinaryIonAppender().append( + 0x03, 0x00, 0x00, 0x00, // 4 byte NOP pad. + 0x21, 0x01 // Int 1. + ); + // The IVM is 4 bytes and the NOP pad is 4 bytes. The first byte of the value is the 9th byte and fits in the + // buffer. Even though there is a 10th byte, the value should not be considered oversize because the NOP pad + // can be discarded. + IonReader reader = newBoundedIncrementalReader(out.toByteArray(), 9); + assertEquals(IonType.INT, reader.next()); + assertEquals(1, reader.intValue()); + reader.close(); + } + + @Test + public void symbolTableFollowedByNopPadFollowedByValueThatOverflowsBufferNotOversized() throws Exception { + TestUtils.BinaryIonAppender out = new TestUtils.BinaryIonAppender().append( + // Symbol table with the symbol 'hello'. + 0xEB, 0x81, 0x83, 0xD8, 0x87, 0xB6, 0x85, 'h', 'e', 'l', 'l', 'o', + 0x00, // 1-byte NOP pad. + 0x71, 0x0A // SID 10 (hello). + ); + // The IVM is 4 bytes, the symbol table is 12 bytes, and the symbol value is 2 bytes (total 18). The 1-byte NOP + // pad needs to be reclaimed to make space for the value. Once that is done, the value will fit. + IonReader reader = newBoundedIncrementalReader(out.toByteArray(), 18); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("hello", reader.stringValue()); + reader.close(); + } + + @Test + public void multipleNopPadsFollowedByValueThatOverflowsBufferNotOversized() throws Exception { + TestUtils.BinaryIonAppender out = new TestUtils.BinaryIonAppender().append( + // One byte no-op pad. + 0x00, + // Two byte no-op pad. + 0x01, 0xFF, + // Three byte no-op pad. + 0x02, 0x99, 0x42, + // Int 1 + 0x21, 0x01, + // IVM 1.0 + 0xE0, 0x01, 0x00, 0xEA, + // The following no-op pads exceed the maximum buffer size, but should not cause an error to be raised. + // 16-byte no-op pad. + 0x0E, 0x8E, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // Three byte no-op pad. + 0x02, 0x99, 0x42, + // Int 2. + 0x21, 0x02, + // 16-byte no-op pad. + 0x0E, 0x8E, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + ); + + IonReader reader = newBoundedIncrementalReader(out.toByteArray(), 11); + assertEquals(IonType.INT, reader.next()); + assertEquals(1, reader.intValue()); + assertEquals(IonType.INT, reader.next()); + assertEquals(2, reader.intValue()); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void nopPadsInterspersedWithSystemValuesDoNotCauseOversizedErrors() throws Exception { + TestUtils.BinaryIonAppender out = new TestUtils.BinaryIonAppender().append( + // One byte no-op pad. + 0x00, + // Two byte no-op pad. + 0x01, 0xFF, + // Three byte no-op pad. + 0x02, 0x99, 0x42, + // IVM 1.0 + 0xE0, 0x01, 0x00, 0xEA, + // 15-byte no-op pad. + 0x0E, 0x8D, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // Symbol table with the symbol 'hello'. + 0xEB, 0x81, 0x83, 0xD8, 0x87, 0xB6, 0x85, 'h', 'e', 'l', 'l', 'o', + // Three byte no-op pad. + 0x02, 0x99, 0x42, + // Symbol 10 (hello) + 0x71, 0x0A + ); + + // Set the maximum size at 2 IVMs (8 bytes) + the symbol table (12 bytes) + the value (2 bytes). + IonReader reader = newBoundedIncrementalReader(out.toByteArray(), 22); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("hello", reader.stringValue()); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void nopPadsInterspersedWithSystemValuesDoNotCauseOversizedErrors2() throws Exception { + TestUtils.BinaryIonAppender out = new TestUtils.BinaryIonAppender().append( + // One byte no-op pad. + 0x00, + // Two byte no-op pad. + 0x01, 0xFF, + // Three byte no-op pad. + 0x02, 0x99, 0x42, + // IVM 1.0 + 0xE0, 0x01, 0x00, 0xEA, + // 16-byte no-op pad. + 0x0E, 0x8E, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // Symbol table with the symbol 'hello'. + 0xEB, 0x81, 0x83, 0xD8, 0x87, 0xB6, 0x85, 'h', 'e', 'l', 'l', 'o', + // Three byte no-op pad. + 0x02, 0x99, 0x42, + // Symbol 10 (hello) + 0x71, 0x0A + ); + + // Set the maximum size at 2 IVMs (8 bytes) + the symbol table (12 bytes) + the value (2 bytes). + IonReader reader = newBoundedIncrementalReader(out.toByteArray(), 22); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("hello", reader.stringValue()); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void nopPadsInterspersedWithSystemValuesDoNotCauseOversizedErrors3() throws Exception { + TestUtils.BinaryIonAppender out = new TestUtils.BinaryIonAppender().append( + // One byte no-op pad. + 0x00, + // Two byte no-op pad. + 0x01, 0xFF, + // Three byte no-op pad. + 0x02, 0x99, 0x42, + // IVM 1.0 + 0xE0, 0x01, 0x00, 0xEA, + // 14-byte no-op pad. + 0x0E, 0x8C, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // Symbol table with the symbol 'hello'. + 0xEB, 0x81, 0x83, 0xD8, 0x87, 0xB6, 0x85, 'h', 'e', 'l', 'l', 'o', + // Three byte no-op pad. + 0x02, 0x99, 0x42, + // Symbol 10 (hello) + 0x71, 0x0A + ); + + // Set the maximum size at 2 IVMs (8 bytes) + the symbol table (12 bytes) + the value (2 bytes). + IonReader reader = newBoundedIncrementalReader(out.toByteArray(), 22); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("hello", reader.stringValue()); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void nopPadSurroundingSymbolTableThatFitsInBuffer() throws Exception { + TestUtils.BinaryIonAppender out = new TestUtils.BinaryIonAppender().append( + // 14-byte no-op pad. + 0x0E, 0x8C, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // Symbol table with the symbol 'hello'. + 0xEB, 0x81, 0x83, 0xD8, 0x87, 0xB6, 0x85, 'h', 'e', 'l', 'l', 'o', + // 14-byte no-op pad. + 0x0E, 0x8C, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // String abcdefg + 0x87, 'a', 'b', 'c', 'd', 'e', 'f', 'g', + // Symbol 10 (hello) + 0x71, 0x0A + ); + + // Set the maximum size at IVM (4 bytes) + 14-byte NOP pad + the symbol table (12 bytes) + 2 value bytes. + IonReader reader = newBoundedIncrementalReader(out.toByteArray(), 32); + assertEquals(IonType.STRING, reader.next()); + assertEquals("abcdefg", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("hello", reader.stringValue()); + assertNull(reader.next()); + reader.close(); + } + + /** + * Compares an iterator to the given list. + * @param expected the list containing the elements to compare to. + * @param actual the iterator to be compared. + * @param isEqualityExpected true if the iterator is expected to contain the same elements as the list; otherwise, + * false. + */ + private static void compareIterator(List expected, Iterator actual, boolean isEqualityExpected) { + int numberOfElements = 0; + while (actual.hasNext()) { + String actualValue = actual.next(); + if (numberOfElements >= expected.size()) { + assertEquals(isEqualityExpected, !actual.hasNext()); + break; + } + String expectedValue = expected.get(numberOfElements++); + if (isEqualityExpected) { + assertEquals(expectedValue, actualValue); + } else { + assertNotEquals(expectedValue, actualValue); + } + } + assertEquals(isEqualityExpected, numberOfElements == expected.size()); + } + + /** + * Tests the annotation iterator reuse option. + * @param isEnabled true if the option is enabled, false if it is disabled. + */ + private void annotationIteratorReuse(boolean isEnabled) throws Exception { + readerBuilder = IonReaderBuilder.standard().withAnnotationIteratorReuseEnabled(isEnabled); + IonReaderBinaryIncremental reader = readerFor("foo::bar::123 baz::456"); + + assertEquals(IonType.INT, reader.next()); + Iterator firstValueAnnotationIterator = reader.iterateTypeAnnotations(); + assertEquals(123, reader.intValue()); + assertEquals(IonType.INT, reader.next()); + compareIterator(Arrays.asList("foo", "bar"), firstValueAnnotationIterator, !isEnabled); + Iterator secondValueAnnotationIterator = reader.iterateTypeAnnotations(); + assertEquals(456, reader.intValue()); + assertNull(reader.next()); + compareIterator(Collections.singletonList("baz"), secondValueAnnotationIterator, !isEnabled); + reader.close(); + } + + @Test + public void annotationIteratorReuseEnabled() throws Exception { + annotationIteratorReuse(true); + } + + @Test + public void annotationIteratorReuseDisabled() throws Exception { + annotationIteratorReuse(false); + } +} diff --git a/test/com/amazon/ion/impl/IonReaderLookaheadBufferTest.java b/test/com/amazon/ion/impl/IonReaderLookaheadBufferTest.java new file mode 100644 index 0000000000..d560275075 --- /dev/null +++ b/test/com/amazon/ion/impl/IonReaderLookaheadBufferTest.java @@ -0,0 +1,1197 @@ +package com.amazon.ion.impl; + +import com.amazon.ion.BufferConfiguration; +import com.amazon.ion.IonBufferConfiguration; +import com.amazon.ion.IonDatagram; +import com.amazon.ion.IonException; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonType; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.TestUtils; +import com.amazon.ion.system.IonBinaryWriterBuilder; +import com.amazon.ion.system.IonReaderBuilder; +import com.amazon.ion.system.IonSystemBuilder; +import com.amazon.ion.util.Equivalence; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +import static com.amazon.ion.BitUtils.bytes; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@RunWith(Parameterized.class) +public class IonReaderLookaheadBufferTest + extends IonReaderLookaheadBufferTestBase +{ + + private static final IonSystem SYSTEM = IonSystemBuilder.standard().build(); + + // The source of data that the standard lookahead buffer will draw from. + private ResizingPipedInputStream input; + // The lookahead buffer to test. + private IonReaderLookaheadBuffer lookahead; + // Builds IonBufferConfigurations used by 'lookahead'. May be overwritten by individual tests. + private IonBufferConfiguration.Builder builder; + + @Before + public void reset() { + input = null; + lookahead = null; + builder = IonBufferConfiguration.Builder.standard(); + } + + /** + * Creates a standard buffer (`lookahead`) that draws from a ResizingPipedInputStream (`input`) that can be + * incrementally filled with data. + */ + private void initializeStandardPipedBuffer() { + input = new ResizingPipedInputStream(1); + if (initialBufferSize != null) { + builder.withInitialBufferSize(initialBufferSize); + } + lookahead = new IonReaderLookaheadBuffer(builder.build(), input); + } + + /** + * Creates a lookahead buffer over the given bytes, prepended with the IVM. + * @param ion binary Ion bytes without an IVM. + * @return a new buffer. + */ + private IonReaderLookaheadBuffer bufferFor(int... ion) throws Exception { + return new IonReaderLookaheadBuffer( + builder.build(), + new ByteArrayInputStream(new TestUtils.BinaryIonAppender().append(ion).toByteArray()) + ); + } + + @Override + BuilderSupplier defaultBuilderSupplier() { + return new BuilderSupplier() { + + @Override + public IonBufferConfiguration.Builder get() { + return IonBufferConfiguration.Builder.standard(); + } + }; + } + + @Override + ReaderLookaheadBufferBase build( + IonBufferConfiguration.Builder configuration, + InputStream inputStream + ) { + return new IonReaderLookaheadBuffer(configuration.build(), inputStream); + } + + @Override + void createThrowingOversizedEventHandlers(IonBufferConfiguration.Builder builder) { + builder.onOversizedValue(new BufferConfiguration.OversizedValueHandler() { + @Override + public void onOversizedValue() { + throw new IllegalStateException(); + } + }) + .onOversizedSymbolTable(new IonBufferConfiguration.OversizedSymbolTableHandler() { + @Override + public void onOversizedSymbolTable() { + throw new IllegalStateException(); + } + }); + } + + @Override + void createCountingEventHandler(IonBufferConfiguration.Builder builder, final AtomicLong byteCount) { + createThrowingOversizedEventHandlers(builder); + builder.onData(new BufferConfiguration.DataHandler() { + @Override + public void onData(int numberOfBytes) { + byteCount.addAndGet(numberOfBytes); + } + }); + } + + @Override + byte[] toBytes(String textIon) throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + _Private_IonSystem system = (_Private_IonSystem) IonSystemBuilder.standard().build(); + IonReader reader = system.newSystemReader(textIon); + IonWriter writer = IonBinaryWriterBuilder.standard().build(output); + writer.writeValues(reader); + reader.close(); + writer.close(); + return output.toByteArray(); + } + + @Override + byte[] intZeroWithoutIvm() { + return bytes(0x20); + } + + @Test + public void nopPadding() throws Exception { + byte[] threeByteNopPadBeforeIntegerZero = bytes(0xE0, 0x01, 0x00, 0xEA, 0x0E, 0x81, 0x00, 0x20); + readSingleValueOneByteAtATime( + threeByteNopPadBeforeIntegerZero, + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.INT, reader.next()); + assertEquals(0, reader.intValue()); + } + } + ); + } + + @Test + public void annotationWithMultiByteSID() throws Exception { + // The SID 1010 takes multiple bytes to express, but the total length of the annotation wrapper is still small + // enough that the length is encoded in the lower nibble of the type ID. This test verifies that the + // annot_length VarUInt is correctly treated as a length of the SIDs, not the number of SIDs. + readSingleValueOneByteAtATime( + toBytes("$ion_symbol_table::{imports:[{name:\"foo\", version:1, max_id:1000}], symbols:[\"abc\"]} $1010::123"), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.INT, reader.next()); + String[] annotations = reader.getTypeAnnotations(); + assertEquals(1, annotations.length); + assertEquals("abc", annotations[0]); + assertEquals(123, reader.intValue()); + } + } + ); + // This test is similar, except that the annotation wrapper's length requires the 'length' VarUInt. + readSingleValueOneByteAtATime( + toBytes("$ion_symbol_table::{imports:[{name:\"foo\", version:1, max_id:1000}], symbols:[\"abc\"]} $1010::\"abcdefghijklmnopqrstuvwxyz0123456789\""), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.STRING, reader.next()); + String[] annotations = reader.getTypeAnnotations(); + assertEquals(1, annotations.length); + assertEquals("abc", annotations[0]); + assertEquals("abcdefghijklmnopqrstuvwxyz0123456789", reader.stringValue()); + } + } + ); + // Two multibyte SIDs. + readSingleValueOneByteAtATime( + toBytes("$ion_symbol_table::{imports:[{name:\"foo\", version:1, max_id:1000}], symbols:[\"abc\"]} $1010::$1010::123"), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.INT, reader.next()); + String[] annotations = reader.getTypeAnnotations(); + assertEquals(2, annotations.length); + assertEquals("abc", annotations[0]); + assertEquals("abc", annotations[1]); + assertEquals(123, reader.intValue()); + } + } + ); + // One multibyte SID and one one-byte SID. + readSingleValueOneByteAtATime( + toBytes("$ion_symbol_table::{imports:[{name:\"foo\", version:1, max_id:1000}], symbols:[\"abc\"]} $4::$1010::\"abcdefghijklmnopqrstuvwxyz0123456789\""), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.STRING, reader.next()); + String[] annotations = reader.getTypeAnnotations(); + assertEquals(2, annotations.length); + assertEquals("name", annotations[0]); // SID 4 is "name" + assertEquals("abc", annotations[1]); + assertEquals("abcdefghijklmnopqrstuvwxyz0123456789", reader.stringValue()); + } + } + ); + } + + @Test + public void symbolTableAnnotationOnStructNotFirst() throws Exception { + readSingleValueOneByteAtATime( + // Note: the following more readable line will be possible once ion-java#222 is fixed. + // toBytes("name::$ion_symbol_table::{}"), + bytes(0xE0, 0x01, 0x00, 0xEA, 0xE4, 0x82, 0x84, 0x83, 0xD0), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.STRUCT, reader.next()); + String[] annotations = reader.getTypeAnnotations(); + assertEquals(2, annotations.length); + assertEquals("name", annotations[0]); + assertEquals("$ion_symbol_table", annotations[1]); + reader.stepIn(); + assertNull(reader.next()); + reader.stepOut(); + } + } + ); + } + + @Test + public void errorOnInvalidTypeF() throws Exception { + // Type F is illegal in Ion 1.0. + thrown.expect(IonException.class); + readSingleValueOneByteAtATime( + bytes(0xE0, 0x01, 0x00, 0xEA, 0xF0), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + fail(); + } + } + ); + } + + @Test + public void errorOnInvalidAnnotationTooShort() throws Exception { + // Annotation wrappers must have length at least three. + thrown.expect(IonException.class); + readSingleValueOneByteAtATime( + bytes(0xE0, 0x01, 0x00, 0xEA, 0xE2, 0x81, 0x80), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + fail(); + } + } + ); + } + + @Test + public void errorOnInvalidAnnotationNull() throws Exception { + // Annotation wrappers cannot be null. + thrown.expect(IonException.class); + readSingleValueOneByteAtATime( + bytes(0xE0, 0x01, 0x00, 0xEA, 0xE2), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + fail(); + } + } + ); + } + + @Test + public void errorOnIVMInAnnotation() throws Exception { + // The IVM must only occur at the top level. + thrown.expect(IonException.class); + readSingleValueOneByteAtATime( + bytes(0xE0, 0x01, 0x00, 0xEA, 0xE6, 0x81, 0x83, 0xE0, 0x01, 0x00, 0xEA), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + fail(); + } + } + ); + } + + @Test + public void errorOnInvalidAnnotationWithinAnnotation() throws Exception { + // An annotation wrapper cannot wrap another annotation wrapper. This is an example of invalid Ion + // that is not detected by the lookahead wrapper. The error will be conveyed by the reader as normal. + readSingleValueOneByteAtATime( + bytes(0xE0, 0x01, 0x00, 0xEA, 0xE6, 0x81, 0x84, 0xE3, 0x81, 0x84, 0x20), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + thrown.expect(IonException.class); + reader.next(); + } + } + ); + } + + @Test + public void errorOnSpecifiedMaxSizeAndNullSymbolTableHandler() { + IonBufferConfiguration.Builder builder = builderSupplier.get(); + thrown.expect(IllegalArgumentException.class); + build( + builder + .withMaximumBufferSize(10) + .onOversizedValue(builder.getNoOpOversizedValueHandler()) + .onOversizedSymbolTable(null), + new ByteArrayInputStream(new byte[]{}) + ); + } + + private static final int VALUE_BITS_PER_VARUINT_BYTE = 7; + private static final int LOWER_SEVEN_BITS_BITMASK = 0x7F; + + @Test + public void succeedsOnVarUIntMaxLong() throws Exception { + initializeStandardPipedBuffer(); + // The IVM and the start of a variable-length NOP pad. + input.receive(bytes(0xE0, 0x01, 0x00, 0xEA, 0x0E)); + lookahead.fillInput(); + assertTrue(lookahead.moreDataRequired()); + // The following writes the VarUInt representation of Long.MAX_VALUE one byte at a time. + // Note: Long.MAX_VALUE is represented in Long.SIZE - 1 bits because long is signed. + for (int i = Long.SIZE - 1; i > 0; i -= VALUE_BITS_PER_VARUINT_BYTE) { + int bitsToShift = i - VALUE_BITS_PER_VARUINT_BYTE; + int endMask = 0; + if (i <= VALUE_BITS_PER_VARUINT_BYTE) { + endMask = 0x80; + bitsToShift = 0; + } + input.receive((byte) (((Long.MAX_VALUE >>> bitsToShift) & LOWER_SEVEN_BITS_BITMASK) | endMask)); + lookahead.fillInput(); + assertTrue(lookahead.moreDataRequired()); + } + // This is the first of the Long.MAX_VALUE bytes of NOP pad that the Ion reader would skip. Writing + // all of them is out of scope for this test, but testing that the first one succeeds verifies that + // the lookahead wrapper has exited its VarUInt parsing state. + input.receive(0); + lookahead.fillInput(); + assertTrue(lookahead.moreDataRequired()); + } + + @Test + public void errorOnVarUIntTooLarge() throws Exception { + BigInteger longPlusOne = BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE); + initializeStandardPipedBuffer(); + // The IVM and the start of a variable-length NOP pad. + input.receive(bytes(0xE0, 0x01, 0x00, 0xEA, 0x0E)); + lookahead.fillInput(); + // The following writes the VarUInt representation of `longPlusOne` one byte at a time. + for (int i = longPlusOne.bitLength(); i > 0; i -= VALUE_BITS_PER_VARUINT_BYTE) { + assertTrue(lookahead.moreDataRequired()); + if (i >= VALUE_BITS_PER_VARUINT_BYTE) { + int bitsToShift = i - VALUE_BITS_PER_VARUINT_BYTE; + input.receive( + longPlusOne + .shiftRight(bitsToShift) + .and(BigInteger.valueOf(LOWER_SEVEN_BITS_BITMASK)) + .byteValue() + ); + if (bitsToShift < VALUE_BITS_PER_VARUINT_BYTE) { + // This is the second-to-last byte. A failure is expected because this is not the end + // byte, so no matter what the next byte is, the value can't fit in a long. + thrown.expect(IonException.class); + } + lookahead.fillInput(); + } else { + fail("Reached the last byte, which should not be possible."); + } + } + } + + @Test + public void errorOnRewindAtBeginning() { + initializeStandardPipedBuffer(); + thrown.expect(IllegalStateException.class); + lookahead.rewind(); + } + + @Test + public void errorOnRewindToValueStartAtBeginning() { + initializeStandardPipedBuffer(); + thrown.expect(IllegalStateException.class); + lookahead.rewindToValueStart(); + } + + @Test + public void errorOnMarkWhenMoreDataIsRequired() throws Exception { + initializeStandardPipedBuffer(); + // The IVM and the start of a variable-length NOP pad. + input.receive(bytes(0xE0, 0x01, 0x00, 0xEA, 0xE0)); + lookahead.fillInput(); + assertTrue(lookahead.moreDataRequired()); + thrown.expect(IllegalStateException.class); + lookahead.mark(); + } + + @Test + public void errorOnRewindToValueStartWhenDataWouldBeLost() throws Exception { + initializeStandardPipedBuffer(); + // The IVM and the start of a variable-length NOP pad. + input.receive(bytes(0xE0, 0x01, 0x00, 0xEA, 0xE0)); + lookahead.fillInput(); + assertTrue(lookahead.moreDataRequired()); + thrown.expect(IllegalStateException.class); + lookahead.rewindToValueStart(); + } + + @Test + public void errorOnRewindAfterClear() throws Exception { + initializeStandardPipedBuffer(); + // The IVM followed by int 0. + input.receive(bytes(0xE0, 0x01, 0x00, 0xEA, 0x20)); + lookahead.fillInput(); + lookahead.mark(); + lookahead.clearMark(); + thrown.expect(IllegalStateException.class); + lookahead.rewind(); + } + + @Test + public void errorOnRewindToValueStartWhenNoValueIsBuffered() throws Exception { + initializeStandardPipedBuffer(); + // The IVM followed by int 0. + input.receive(bytes(0xE0, 0x01, 0x00, 0xEA, 0x20)); + lookahead.fillInput(); + assertFalse(lookahead.moreDataRequired()); + IonReader reader = lookahead.newIonReader(IonReaderBuilder.standard()); + assertEquals(IonType.INT, reader.next()); + assertEquals(0, reader.intValue()); + lookahead.fillInput(); + assertTrue(lookahead.moreDataRequired()); + thrown.expect(IllegalStateException.class); + lookahead.rewindToValueStart(); + } + + @Test + public void errorOnRewindAfterFillInput() throws Exception { + initializeStandardPipedBuffer(); + // The IVM followed by int 0. + input.receive(bytes(0xE0, 0x01, 0x00, 0xEA, 0x20)); + lookahead.fillInput(); + lookahead.mark(); + input.receive(bytes(0x20)); + lookahead.fillInput(); + thrown.expect(IllegalStateException.class); + lookahead.rewind(); + } + + @Test + public void rewind() throws Exception { + initializeStandardPipedBuffer(); + input.receive(bytes(0xE0, 0x01, 0x00, 0xEA, 0x20)); + lookahead.fillInput(); + assertFalse(lookahead.moreDataRequired()); + lookahead.mark(); + IonReader reader = lookahead.newIonReader(IonReaderBuilder.standard()); + assertEquals(IonType.INT, reader.next()); + assertEquals(0, reader.intValue()); + lookahead.rewind(); + // 4-byte IVM + 1-byte value + assertEquals(5, lookahead.available()); + assertEquals(IonType.INT, reader.next()); + assertEquals(0, reader.intValue()); + input.receive(bytes(0x21, 0x01)); + lookahead.fillInput(); + assertFalse(lookahead.moreDataRequired()); + lookahead.mark(); + assertEquals(IonType.INT, reader.next()); + assertEquals(1, reader.intValue()); + lookahead.rewind(); + assertEquals(2, lookahead.available()); + assertEquals(IonType.INT, reader.next()); + assertEquals(1, reader.intValue()); + lookahead.rewind(); + assertEquals(2, lookahead.available()); + assertEquals(IonType.INT, reader.next()); + assertEquals(1, reader.intValue()); + assertTrue(lookahead.moreDataRequired()); + assertEquals(0, lookahead.available()); + assertNull(reader.next()); + reader.close(); + } + + @Test + public void rewindToValueStart() throws Exception { + initializeStandardPipedBuffer(); + input.receive(bytes(0xE0, 0x01, 0x00, 0xEA, 0x20)); + lookahead.fillInput(); + assertFalse(lookahead.moreDataRequired()); + IonReader reader = lookahead.newIonReader(IonReaderBuilder.standard()); + assertEquals(IonType.INT, reader.next()); + assertEquals(0, reader.intValue()); + lookahead.rewindToValueStart(); + // No IVM, just 1-byte value. + assertEquals(1, lookahead.available()); + assertEquals(IonType.INT, reader.next()); + assertEquals(0, reader.intValue()); + input.receive(bytes(0x21, 0x01)); + lookahead.fillInput(); + assertFalse(lookahead.moreDataRequired()); + assertEquals(IonType.INT, reader.next()); + assertEquals(1, reader.intValue()); + lookahead.rewindToValueStart(); + assertEquals(2, lookahead.available()); + assertEquals(IonType.INT, reader.next()); + assertEquals(1, reader.intValue()); + lookahead.rewindToValueStart(); + assertEquals(2, lookahead.available()); + assertEquals(IonType.INT, reader.next()); + assertEquals(1, reader.intValue()); + assertTrue(lookahead.moreDataRequired()); + assertEquals(0, lookahead.available()); + assertNull(reader.next()); + reader.close(); + } + + private static class RecordingEventHandler implements + BufferConfiguration.OversizedValueHandler, + IonBufferConfiguration.OversizedSymbolTableHandler, + BufferConfiguration.DataHandler { + + int oversizedSymbolTableCount = 0; + int oversizedValueCount = 0; + long numberOfBytesProcessed = 0; + + @Override + public void onOversizedSymbolTable() { + oversizedSymbolTableCount++; + } + + @Override + public void onOversizedValue() { + oversizedValueCount++; + } + + @Override + public void onData(int numberOfBytes) { + numberOfBytesProcessed += numberOfBytes; + } + } + + @Test + public void skipOversizedValues() throws Exception { + if (initialBufferSize == null) { + // This test tests buffers of limited size, while an initialBufferSize of null indicates that the size + // of the buffer is unlimited. Skip. + return; + } + // 14 values ranging in size from 1-14 bytes. There are 14 - x values larger than x bytes. + TestUtils.BinaryIonAppender appender = new TestUtils.BinaryIonAppender().append( + 0x11, // boolean true + 0x21, 0x00, // int 0 (overpadded) + 0x22, 0x00, 0x01, // int 1 (overpadded) + 0x33, 0x00, 0x00, 0x02, // int -2 (overpadded) + 0x44, 0x00, 0x00, 0x00, 0x00, // float 0e0 + 0x55, 0x00, 0x00, 0x80, 0x00, 0x00, // decimal 0d0 (overpadded) + 0x66, 0x80, 0x81, 0x81, 0x81, 0x80, 0x80, // timestamp 0001-01-01T00:00Z + 0x7E, 0x00, 0x00, 0x00, 0x00, 0x82, 0x00, 0x00, // symbol 0 (overpadded length and value) + 0x88, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', // string abcdefgh + 0x99, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', // clob abcdefghi + 0xAA, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, // blob + // list with ten int 0 + 0xBB, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + // sexp with ten decimal 0d0 + 0xCC, 0x50, 0x50, 0x50, 0x50, 0x50, 0x50, 0x50, 0x50, 0x50, 0x50, 0x50, 0x50, + // struct with system symbol field names and int 0 values (some overpadded) + 0xDD, 0x84, 0x20, 0x85, 0x20, 0x86, 0x20, 0x87, 0x21, 0x00, 0x88, 0x22, 0x00, 0x00 + ); + byte[] data = appender.toByteArray(); + IonDatagram expectedValues = SYSTEM.getLoader().load(data);; + for (int maxValueSize : Arrays.asList(5, 7, 8, 10, 13)) { + if (maxValueSize < initialBufferSize) { + // This would violate the IonBufferConfiguration contract; no need to test here. + continue; + } + ByteArrayInputStream input = new ByteArrayInputStream(data); + RecordingEventHandler eventHandler = new RecordingEventHandler(); + IonBufferConfiguration.Builder builder = IonBufferConfiguration.Builder.standard() + .withMaximumBufferSize(maxValueSize) + .onOversizedValue(eventHandler) + .onOversizedSymbolTable(eventHandler) + .onData(eventHandler) + .withInitialBufferSize(initialBufferSize); + IonReaderLookaheadBuffer lookahead = new IonReaderLookaheadBuffer(builder.build(), input); + IonReader reader = null; + try { + int valueIndex = 0; + while (true) { + lookahead.fillInput(); + if (lookahead.moreDataRequired()) { + break; + } + if (reader == null) { + reader = lookahead.newIonReader(IonReaderBuilder.standard()); + } + assertNotNull(reader.next()); + assertTrue(Equivalence.ionEquals(expectedValues.get(valueIndex), SYSTEM.newValue(reader))); + valueIndex++; + } + // There is one value per size. For a max size of x, `moreDataRequired` should be false x times. + assertEquals(valueIndex, maxValueSize); + } finally { + if (reader != null) { + reader.close(); + } + } + int expectedOversizedValues = expectedValues.size() - maxValueSize; + assertEquals( + String.format("Expect %d oversized values for max size %d", expectedOversizedValues, maxValueSize), + expectedOversizedValues, + eventHandler.oversizedValueCount + ); + assertEquals(0, eventHandler.oversizedSymbolTableCount); + ResizingPipedInputStream buffer = (ResizingPipedInputStream) lookahead.getPipe(); + // The buffer grows in increments of its initial buffer size. Therefore, the final capacity will be at + // most `initialBufferSize` larger than the lookahead wrapper's configured max value size. + assertTrue(buffer.capacity() <= maxValueSize + buffer.getInitialBufferSize()); + } + } + + @Test + public void skipOversizedValuesButRetainSymbolTablesWhenReadingOneByOne() throws Exception { + if (initialBufferSize == null) { + // This test tests buffers of limited size, while an initialBufferSize of null indicates that the size + // of the buffer is unlimited. Skip. + return; + } + // 4 byte IVM + 30 bytes for the symbol tables + 2 bytes for a symbol value. + final int maximumSize = 36; + RecordingEventHandler eventHandler = new RecordingEventHandler(); + builder = IonBufferConfiguration.Builder.standard() + .withMaximumBufferSize(maximumSize) + .onOversizedValue(eventHandler) + .onOversizedSymbolTable(eventHandler) + .onData(eventHandler) + .withInitialBufferSize(initialBufferSize); + lookahead = bufferFor( + // Total buffered data: 4 bytes. The following value is 33 bytes in order to exceed the max of 36. + 0xBE, 0x9F, // list of size 31 + 0x7E, 0x00, 0x00, 0x00, 0x00, 0x82, 0x00, 0x00, // symbol 0 (overpadded length and value) + 0xAA, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, // blob + 0xBB, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, // list with ten int 0 + 0xE7, 0x81, 0x83, 0xD4, 0x87, 0xB2, 0x81, 'A', // symbol table declaring the symbol 'A'. + // Total buffered data: 4 + 8 = 12 bytes. The following value is 25 bytes in order to exceed the max of 36. + 0xBE, 0x97, // list of size 23 + 0xAA, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, // blob + 0xBB, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, // list with ten int 0 + 0xEA, 0x81, 0x83, 0xD7, 0x86, 0x71, 0x03, 0x87, 0xB2, 0x81, 'B', // symbol table appending the symbol 'B'. + // Total buffered data: 4 + 8 + 11 = 23 bytes. The following value is 14 bytes in order to exceed the max of 36. + // struct with system symbol field names and int 0 values (some overpadded) + 0xDD, 0x84, 0x20, 0x85, 0x20, 0x86, 0x20, 0x87, 0x21, 0x00, 0x88, 0x22, 0x00, 0x00, + 0xEA, 0x81, 0x83, 0xD7, 0x86, 0x71, 0x03, 0x87, 0xB2, 0x81, 'C', // symbol table appending the symbol 'C'. + // Total buffered data: 4 + 8 + 11 + 11 = 34 bytes. The following value is 3 bytes in order to exceed the max of 36. + 0x22, 0x00, 0x01, // int 1 (overpadded) + 0x71, 0x0A, // symbol 'A' + // Since the previous value was successfully read, the buffer is now empty. A 3-byte value can be read. + 0x72, 0x00, 0x0B, // symbol 'B' + 0x71, 0x0C // symbol 'C' + ); + + lookahead.fillInput(); + assertFalse(lookahead.moreDataRequired()); + IonReader reader = lookahead.newIonReader(IonReaderBuilder.standard()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("A", reader.stringValue()); + + lookahead.fillInput(); + assertFalse(lookahead.moreDataRequired()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("B", reader.stringValue()); + + lookahead.fillInput(); + assertFalse(lookahead.moreDataRequired()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("C", reader.stringValue()); + + lookahead.fillInput(); + assertTrue(lookahead.moreDataRequired()); + reader.close(); + + assertEquals(4, eventHandler.oversizedValueCount); + assertEquals(0, eventHandler.oversizedSymbolTableCount); + ResizingPipedInputStream buffer = (ResizingPipedInputStream) lookahead.getPipe(); + // The buffer grows in increments of its initial buffer size. Therefore, the final capacity will be at + // most `initialBufferSize` larger than the lookahead wrapper's configured max value size. + assertTrue(buffer.capacity() <= maximumSize + buffer.getInitialBufferSize()); + } + + @Test + public void skipOversizedValuesButRetainSymbolTablesWhenReadingAllAtOnce() throws Exception { + if (initialBufferSize == null) { + // This test tests buffers of limited size, while an initialBufferSize of null indicates that the size + // of the buffer is unlimited. Skip. + return; + } + // 4 byte IVM + 8 bytes for the symbol tables + 6 bytes for the integer and symbol values. + final int maximumSize = 18; + RecordingEventHandler eventHandler = new RecordingEventHandler(); + builder = IonBufferConfiguration.Builder.standard() + .withMaximumBufferSize(maximumSize) + .onOversizedValue(eventHandler) + .onOversizedSymbolTable(eventHandler) + .onData(eventHandler) + .withInitialBufferSize(initialBufferSize); + lookahead = bufferFor( + 0xE7, 0x81, 0x83, 0xD4, 0x87, 0xB2, 0x81, 'A', // symbol table declaring the symbol 'A'. + // Total buffered data: 12 bytes. The following value is 7 bytes in order to exceed the max of 18. + 0x66, 0x80, 0x81, 0x81, 0x81, 0x80, 0x80, // timestamp 0001-01-01T00:00Z + 0x21, 0x01, // int 1 + 0x71, 0x0A, // symbol 'A' + 0x31, 0x01 // int -1 + ); + + // Skips the oversized values and buffers the int 1. + lookahead.fillInput(); + assertFalse(lookahead.moreDataRequired()); + + // Buffers the symbol value 'A'. + lookahead.fillInput(); + assertFalse(lookahead.moreDataRequired()); + + // Buffers the int -1. + lookahead.fillInput(); + assertFalse(lookahead.moreDataRequired()); + + // End of the input. + lookahead.fillInput(); + assertTrue(lookahead.moreDataRequired()); + + IonReader reader = lookahead.newIonReader(IonReaderBuilder.standard()); + assertEquals(IonType.INT, reader.next()); + assertEquals(1, reader.intValue()); + + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("A", reader.stringValue()); + + assertEquals(IonType.INT, reader.next()); + assertEquals(-1, reader.intValue()); + + assertNull(reader.next()); + reader.close(); + + assertEquals(1, eventHandler.oversizedValueCount); + assertEquals(0, eventHandler.oversizedSymbolTableCount); + ResizingPipedInputStream buffer = (ResizingPipedInputStream) lookahead.getPipe(); + // The buffer grows in increments of its initial buffer size. Therefore, the final capacity will be at + // most `initialBufferSize` larger than the lookahead wrapper's configured max value size. + assertTrue(buffer.capacity() <= maximumSize + buffer.getInitialBufferSize()); + } + + @Test + public void oversizedSymbolTable() throws Exception { + // 4 byte IVM + 3 byte value. + final int maximumSize = 7; + if (initialBufferSize == null || initialBufferSize > maximumSize) { + // This test tests buffers of limited size, while an initialBufferSize of null indicates that the size + // of the buffer is unlimited. Skip. + return; + } + RecordingEventHandler eventHandler = new RecordingEventHandler(); + builder = IonBufferConfiguration.Builder.standard() + .withMaximumBufferSize(maximumSize) + .onOversizedValue(eventHandler) + .onOversizedSymbolTable(eventHandler) + .onData(eventHandler) + .withInitialBufferSize(initialBufferSize); + lookahead = bufferFor( + 0x22, 0x00, 0x01, // int 1 (overpadded) + 0xE7, 0x81, 0x83, 0xD4, 0x87, 0xB2, 0x81, 'A', // symbol table declaring the symbol 'A'. + 0x71, 0x0A // symbol 'A' + ); + + lookahead.fillInput(); + assertFalse(lookahead.moreDataRequired()); + IonReader reader = lookahead.newIonReader(IonReaderBuilder.standard()); + assertEquals(IonType.INT, reader.next()); + assertEquals(1, reader.intValue()); + + // This causes the oversized symbol table to be skipped. + lookahead.fillInput(); + assertEquals(0, lookahead.available()); + assertTrue(lookahead.moreDataRequired()); + assertEquals(1, eventHandler.oversizedSymbolTableCount); + + // Subsequent invocations of fillInput have no effect. + lookahead.fillInput(); + assertEquals(0, lookahead.available()); + assertTrue(lookahead.moreDataRequired()); + assertEquals(1, eventHandler.oversizedSymbolTableCount); + reader.close(); + + assertEquals(0, eventHandler.oversizedValueCount); + ResizingPipedInputStream buffer = (ResizingPipedInputStream) lookahead.getPipe(); + // The buffer grows in increments of its initial buffer size. Therefore, the final capacity will be at + // most `initialBufferSize` larger than the lookahead wrapper's configured max value size. + assertTrue(buffer.capacity() <= maximumSize + buffer.getInitialBufferSize()); + } + + @Test + public void moreDataRequiredInTheMiddleOfOversizedValue() throws Exception { + // 4 byte IVM + 3 byte value. + final int maximumSize = 7; + if (initialBufferSize == null || initialBufferSize > maximumSize) { + // This test tests buffers of limited size, while an initialBufferSize of null indicates that the size + // of the buffer is unlimited. Skip. + return; + } + TestUtils.BinaryIonAppender appender = new TestUtils.BinaryIonAppender().append( + // Once the fourth byte in the following value is reached, the value is considered oversized. It will remain + // that way even though fillInput yields back to the user for every remaining byte in the value. + 0x66, 0x80, 0x81, 0x81, 0x81, 0x80, 0x80, // timestamp 0001-01-01T00:00Z + 0x20 // int 0 + ); + final RecordingEventHandler eventHandler = new RecordingEventHandler(); + builderSupplier = new BuilderSupplier() { + @Override + public IonBufferConfiguration.Builder get() { + return IonBufferConfiguration.Builder.standard() + .withMaximumBufferSize(maximumSize) + .onOversizedValue(eventHandler) + .onOversizedSymbolTable(eventHandler) + .onData(eventHandler); + } + } ; + + readSingleValueOneByteAtATime( + appender.toByteArray(), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.INT, reader.next()); + assertEquals(0, reader.intValue()); + assertNull(reader.next()); + } + } + ); + + assertEquals(1, eventHandler.oversizedValueCount); + assertEquals(0, eventHandler.oversizedSymbolTableCount); + } + + @Test + public void oversizedIncompleteGzipValueDoesNotThrow() throws Exception { + // Tests that InputStream implementations that throw EOFException when too many bytes are requested to be + // skipped do not cause the lookahead wrapper to throw. + ByteArrayOutputStream out = new ByteArrayOutputStream(); + OutputStream gzip = new GZIPOutputStream(out); + gzip.write(bytes(0xE0, 0x01, 0x00, 0xEA)); + gzip.write(bytes(0x66, 0x80, 0x81, 0x81, 0x81, 0x80, 0x80)); // timestamp 0001-01-01T00:00Z + gzip.close(); + byte[] bytes = out.toByteArray(); + final int maximumSize = 5; // Less than the length of the timestamp value. + // Cutting off 15 bytes removes the entire GZIP trailer and part of the value. + byte[] truncatedBytes = new byte[bytes.length - 15]; + System.arraycopy(bytes, 0, truncatedBytes, 0, truncatedBytes.length); + final RecordingEventHandler eventHandler = new RecordingEventHandler(); + builderSupplier = new BuilderSupplier() { + @Override + public IonBufferConfiguration.Builder get() { + return IonBufferConfiguration.Builder.standard() + .withMaximumBufferSize(maximumSize) + .withInitialBufferSize(maximumSize) + .onOversizedValue(eventHandler) + .onOversizedSymbolTable(eventHandler) + .onData(eventHandler); + } + }; + + InputStream input = new GZIPInputStream(new ByteArrayInputStream(truncatedBytes)); + IonReaderLookaheadBuffer lookahead = new IonReaderLookaheadBuffer(builder.build(), input); + lookahead.fillInput(); + assertTrue(lookahead.moreDataRequired()); + lookahead.fillInput(); + assertTrue(lookahead.moreDataRequired()); + input.close(); + // Note: even though the value would be considered oversize, the event handler may not have been notified + // before the input is closed, so nothing is asserted here. The purpose of this test is to verify that + // an EOFException is not thrown. + } + + @Test + public void rewindToValueStartWithLstAppend() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IonWriter writer = IonBinaryWriterBuilder.standard().withLocalSymbolTableAppendEnabled().build(out); + writer.writeSymbol("abc"); + writer.flush(); + writer.writeSymbol("def"); + writer.close(); + InputStream input = new ByteArrayInputStream(out.toByteArray()); + IonReaderLookaheadBuffer lookahead = new IonReaderLookaheadBuffer(builder.build(), input); + lookahead.fillInput(); + IonReader reader = lookahead.newIonReader(IonReaderBuilder.standard()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("abc", reader.stringValue()); + assertTrue(lookahead.moreDataRequired()); + lookahead.fillInput(); + assertFalse(lookahead.moreDataRequired()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("def", reader.stringValue()); + lookahead.rewindToValueStart(); + // 2-byte value (0x71 0x0B). No IVM or symbol table. + assertEquals(2, lookahead.available()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("def", reader.stringValue()); + // Note: if mark() / rewind() is used instead of rewindToValueStart(), the following lines will fail; there + // will be 12 local symbols and "def" will occur twice in the symbol table. That's because mark() includes + // the symbol table (in this case an LST append), and rewinding past the symbol table causes the append + // to be processed a second time by IonReader.next(). + SymbolTable symbolTable = reader.getSymbolTable(); + assertEquals(symbolTable.getSystemSymbolTable().getMaxId() + 2, reader.getSymbolTable().getMaxId()); + List symbols = new ArrayList(2); + Iterator iterator = symbolTable.iterateDeclaredSymbolNames(); + while (iterator.hasNext()) { + symbols.add(iterator.next()); + } + assertEquals(Arrays.asList("abc", "def"), symbols); + input.close(); + } + + private static void assertIonTypeId(IonType expectedType, int expectedLowerNibble, IonTypeID actualTypeID) { + assertEquals(expectedType, actualTypeID.type); + assertEquals(expectedLowerNibble, actualTypeID.lowerNibble); + } + + /** + * Returns one of the provided indices, depending on the value of the 'initialBufferSize' test parameter. Bytes may + * occur at different indices in the buffer depending on the buffer's initial size because space will be reclaimed + * when possible to avoid unnecessary growth. So a buffer that starts out larger than the total size of the data, + * for example, will have ever-increasing indices, while smaller buffers may have indices that roll back to the + * beginning when space is reclaimed. + * @param indexIfSizeIs1 the expected index when the initial buffer size is 1. + * @param indexIfSizeIs10 the expected index when the initial buffer size is 10. + * @param indexIfSizeIsLarge the expected index when the initial buffer size is larger than the test data. + * @return the index for the current value of initialBufferSize. + */ + private int indexForInitialBufferSize(int indexIfSizeIs1, int indexIfSizeIs10, int indexIfSizeIsLarge) { + if (initialBufferSize == null) { + return indexIfSizeIsLarge; + } + if (initialBufferSize == 1) { + return indexIfSizeIs1; + } + if (initialBufferSize == 10) { + return indexIfSizeIs10; + } + throw new IllegalStateException("Add a branch for initialBufferSize " + initialBufferSize); + } + + @Test + public void valueMarkersAreSet() throws Exception { + if (initialBufferSize != null) { + builder.withInitialBufferSize(initialBufferSize); + } + lookahead = bufferFor( + // Value start is at byte index 5. Type ID is 0x21. Value end is at byte index 6. + 0x21, 0x0D, + // The value will be consumed, so the indices reset if the initial buffer size is smaller than the data size. + 0xE0, 0x01, 0x00, 0xEA, + // Type ID is 0xD1. + 0xD1, 0x82, 0x84, 0x20, + // The value will not be consumed, so the indices continue. + // Annotations are SIDs 4 (name) and 5 (version). Type ID is 0xE5. + 0xE5, 0x82, 0x84, 0x85, 0x31, 0x01, + // The value will not be consumed, so the indices continue. + // Type ID is 0x20. + 0x20 + ); + + lookahead.fillInput(); + assertEquals(1, lookahead.getIvmIndex()); + assertTrue(lookahead.getAnnotationSids().isEmpty()); + assertTrue(lookahead.getSymbolTableMarkers().isEmpty()); + assertEquals(5, lookahead.getValueStart()); + assertIonTypeId(IonType.INT, 0x1, lookahead.getValueTid()); + assertEquals(6, lookahead.getValueEnd()); + IonReader reader = lookahead.newIonReader(IonReaderBuilder.standard()); + assertEquals(IonType.INT, reader.next()); + assertEquals(0x0D, reader.intValue()); + assertTrue(lookahead.moreDataRequired()); + + lookahead.fillInput(); + assertEquals(indexForInitialBufferSize(1, 1, 7), lookahead.getIvmIndex()); + lookahead.resetIvmIndex(); + assertEquals(-1, lookahead.getIvmIndex()); + assertTrue(lookahead.getAnnotationSids().isEmpty()); + assertTrue(lookahead.getSymbolTableMarkers().isEmpty()); + assertEquals(indexForInitialBufferSize(6, 6, 12), lookahead.getValueStart()); + assertIonTypeId(IonType.STRUCT, 0x1, lookahead.getValueTid()); + assertEquals(indexForInitialBufferSize(8, 8, 14), lookahead.getValueEnd()); + + lookahead.fillInput(); + assertEquals(-1, lookahead.getIvmIndex()); + assertTrue(lookahead.getSymbolTableMarkers().isEmpty()); + assertEquals(Arrays.asList(4, 5), lookahead.getAnnotationSids()); + assertEquals(indexForInitialBufferSize(12, 12, 18), lookahead.getValueStart()); + assertIonTypeId(IonTypeID.ION_TYPE_ANNOTATION_WRAPPER, 0x5, lookahead.getValueTid()); + assertEquals(indexForInitialBufferSize(14, 14, 20), lookahead.getValueEnd()); + + lookahead.fillInput(); + assertEquals(-1, lookahead.getIvmIndex()); + assertTrue(lookahead.getSymbolTableMarkers().isEmpty()); + assertTrue(lookahead.getAnnotationSids().isEmpty()); + assertEquals(indexForInitialBufferSize(15, 15, 21), lookahead.getValueStart()); + assertIonTypeId(IonType.INT, 0x0, lookahead.getValueTid()); + assertEquals(indexForInitialBufferSize(15, 15, 21), lookahead.getValueEnd()); + + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.INT, reader.next()); + assertEquals("name", reader.getFieldName()); + assertEquals(0, reader.intValue()); + reader.stepOut(); + + assertEquals(IonType.INT, reader.next()); + assertArrayEquals(new String[]{"name", "version"}, reader.getTypeAnnotations()); + assertEquals(-1, reader.intValue()); + + assertEquals(IonType.INT, reader.next()); + assertEquals(0, reader.intValue()); + + assertTrue(lookahead.moreDataRequired()); + reader.close(); + } + + @Test + public void symbolTableMarkersAreSet() throws Exception { + if (initialBufferSize != null) { + builder.withInitialBufferSize(initialBufferSize); + } + lookahead = bufferFor( + // Symbol table declaring symbol 'x'. + 0xE7, 0x81, 0x83, 0xD4, 0x87, 0xB2, 0x81, 'x', + // Symbol value with SID 10 ('x'). + 0x71, 0x0A, + // The value will not be consumed, so the indices continue. + // Symbol table appending symbol 'y'. + 0xEA, 0x81, 0x83, 0xD7, 0x86, 0x71, 0x03, 0x87, 0xB2, 0x81, 'y', + // Symbol value with SID 11 ('y'). + 0x71, 0x0B, + // The value will be consumed, so the indices reset if the initial buffer size is smaller than the data size. + // Second byte of IVM is at byte index 1. The IVM resets the symbol table markers. + 0xE0, 0x01, 0x00, 0xEA, + // Symbol table declaring symbol 'a'. + 0xE7, 0x81, 0x83, 0xD4, 0x87, 0xB2, 0x81, 'a', + // Symbol table declaring symbol 'b'. + 0xE7, 0x81, 0x83, 0xD4, 0x87, 0xB2, 0x81, 'b', + // The IVM resets the symbol table markers. + 0xE0, 0x01, 0x00, 0xEA, + // Symbol table declaring symbol 'c'. + 0xE7, 0x81, 0x83, 0xD4, 0x87, 0xB2, 0x81, 'c', + // Symbol table declaring symbol 'd'. + 0xE7, 0x81, 0x83, 0xD4, 0x87, 0xB2, 0x81, 'd', + // Symbol value with SID 10 ('d'). + 0x71, 0x0A + ); + lookahead.fillInput(); + assertEquals(1, lookahead.getIvmIndex()); + assertTrue(lookahead.getAnnotationSids().isEmpty()); + List symbolTableMarkers = lookahead.getSymbolTableMarkers(); + assertEquals(1, symbolTableMarkers.size()); + assertEquals(8, symbolTableMarkers.get(0).startIndex); + assertEquals(12, symbolTableMarkers.get(0).endIndex); + assertEquals(13, lookahead.getValueStart()); + assertIonTypeId(IonType.SYMBOL, 0x1, lookahead.getValueTid()); + assertEquals(14, lookahead.getValueEnd()); + lookahead.resetSymbolTableMarkers(); + assertTrue(lookahead.getSymbolTableMarkers().isEmpty()); + + lookahead.fillInput(); + symbolTableMarkers = lookahead.getSymbolTableMarkers(); + assertEquals(1, symbolTableMarkers.size()); + assertEquals(18, symbolTableMarkers.get(0).startIndex); + assertEquals(25, symbolTableMarkers.get(0).endIndex); + assertEquals(26, lookahead.getValueStart()); + assertIonTypeId(IonType.SYMBOL, 0x1, lookahead.getValueTid()); + assertEquals(27, lookahead.getValueEnd()); + + IonReader reader = lookahead.newIonReader(IonReaderBuilder.standard()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("x", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("y", reader.stringValue()); + + lookahead.fillInput(); + symbolTableMarkers = lookahead.getSymbolTableMarkers(); + assertEquals(2, symbolTableMarkers.size()); + assertEquals(indexForInitialBufferSize(28, 28, 55), symbolTableMarkers.get(0).startIndex); + assertEquals(indexForInitialBufferSize(32, 32, 59), symbolTableMarkers.get(0).endIndex); + assertEquals(indexForInitialBufferSize(36, 36, 63), symbolTableMarkers.get(1).startIndex); + assertEquals(indexForInitialBufferSize(40, 40, 67), symbolTableMarkers.get(1).endIndex); + assertEquals(indexForInitialBufferSize(41, 41, 68), lookahead.getValueStart()); + assertIonTypeId(IonType.SYMBOL, 0x1, lookahead.getValueTid()); + assertEquals(indexForInitialBufferSize(42, 42, 69), lookahead.getValueEnd()); + + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("d", reader.stringValue()); + + assertTrue(lookahead.moreDataRequired()); + reader.close(); + } + + @Test + public void nopPadFollowedByValueThatOverflowsBufferNotOversized() throws Exception { + if (initialBufferSize == null || initialBufferSize != 1) { + return; + } + builder = IonBufferConfiguration.Builder.standard() + .withInitialBufferSize(9) + .withMaximumBufferSize(9); + createCountingEventHandler(builder, new AtomicLong()); + lookahead = bufferFor( + 0x03, 0x00, 0x00, 0x00, // 4 byte NOP pad. + 0x21, 0x01 // Int 1. + ); + // The IVM is 4 bytes and the NOP pad is 4 bytes. The first byte of the value is the 9th byte and fits in the + // buffer. Even though there is a 10th byte, the value should not be considered oversize because the NOP pad + // can be discarded. + lookahead.fillInput(); + assertFalse(lookahead.moreDataRequired()); + IonReader reader = lookahead.newIonReader(IonReaderBuilder.standard()); + lookahead.resetNopPadIndex(); + assertEquals(IonType.INT, reader.next()); + assertEquals(1, reader.intValue()); + reader.close(); + } + + @Test + public void nopPadsInterspersedWithSystemValuesDoNotCauseOversizedErrors() throws Exception { + if (initialBufferSize == null || initialBufferSize != 1) { + return; + } + // Set the maximum size at 2 IVMs (8 bytes) + the symbol table (12 bytes) + the value (2 bytes). + builder = IonBufferConfiguration.Builder.standard() + .withInitialBufferSize(22) + .withMaximumBufferSize(22); + createCountingEventHandler(builder, new AtomicLong()); + lookahead = bufferFor( + // One byte no-op pad. + 0x00, + // Two byte no-op pad. + 0x01, 0xFF, + // Three byte no-op pad. + 0x02, 0x99, 0x42, + // IVM 1.0 + 0xE0, 0x01, 0x00, 0xEA, + // 15-byte no-op pad. + 0x0E, 0x8D, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // Symbol table with the symbol 'hello'. + 0xEB, 0x81, 0x83, 0xD8, 0x87, 0xB6, 0x85, 'h', 'e', 'l', 'l', 'o', + // Three byte no-op pad. + 0x02, 0x99, 0x42, + // Symbol 10 (hello) + 0x71, 0x0A + ); + + lookahead.fillInput(); + assertFalse(lookahead.moreDataRequired()); + IonReader reader = lookahead.newIonReader(IonReaderBuilder.standard()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("hello", reader.stringValue()); + assertNull(reader.next()); + reader.close(); + } +} diff --git a/test/com/amazon/ion/impl/IonReaderLookaheadBufferTestBase.java b/test/com/amazon/ion/impl/IonReaderLookaheadBufferTestBase.java new file mode 100644 index 0000000000..98135f7e7c --- /dev/null +++ b/test/com/amazon/ion/impl/IonReaderLookaheadBufferTestBase.java @@ -0,0 +1,679 @@ +package com.amazon.ion.impl; + +import com.amazon.ion.BufferConfiguration; +import com.amazon.ion.Decimal; +import com.amazon.ion.IonLob; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonSystem; +import com.amazon.ion.IonText; +import com.amazon.ion.IonType; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.Timestamp; +import com.amazon.ion.system.IonReaderBuilder; +import com.amazon.ion.system.IonSystemBuilder; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigDecimal; +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.runners.Parameterized.Parameter; +import static org.junit.runners.Parameterized.Parameters; + +public abstract class IonReaderLookaheadBufferTestBase< + Configuration extends BufferConfiguration, + Builder extends BufferConfiguration.Builder +> { + + private static final IonSystem ION_SYSTEM = IonSystemBuilder.standard().build(); + private static final Charset UTF_8 = Charset.forName("UTF-8"); + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Parameters(name = "initialBufferSize={0}") + public static Iterable initialBufferSizes() { + return Arrays.asList(1, 10, null); + } + + @Parameter + public Integer initialBufferSize; + + interface IonReaderAssertion { + void evaluate(IonReader reader); + } + + static class Value { + byte[] stream; + IonReaderAssertion assertion; + + Value(byte[] stream, IonReaderAssertion assertion) { + this.stream = stream; + this.assertion = assertion; + } + } + + interface BuilderSupplier< + Configuration extends BufferConfiguration, + Builder extends BufferConfiguration.Builder + > { + Builder get(); + } + + BuilderSupplier builderSupplier = null; + + abstract BuilderSupplier defaultBuilderSupplier(); + + @Before + public void setup() { + builderSupplier = defaultBuilderSupplier(); + } + + abstract ReaderLookaheadBufferBase build(Builder builder, InputStream inputStream); + + abstract void createThrowingOversizedEventHandlers(Builder builder); + abstract void createCountingEventHandler(Builder builder, AtomicLong byteCount); + + void readValuesOneByteAtATime(Value... values) throws Exception { + IonReader reader = null; + ResizingPipedInputStream input = new ResizingPipedInputStream(1); + Builder builder = builderSupplier.get(); + if (initialBufferSize != null) { + builder.withInitialBufferSize(initialBufferSize); + } + ReaderLookaheadBufferBase lookahead = build(builder, input); + try { + for (Value value : values) { + for (int i = -1; i < value.stream.length; i++) { + if (i >= 0) { + input.receive(value.stream[i]); + lookahead.fillInput(); + } + assertEquals(i < value.stream.length - 1, lookahead.moreDataRequired()); + } + if (reader == null) { + reader = lookahead.newIonReader(IonReaderBuilder.standard()); + } + value.assertion.evaluate(reader); + } + } finally { + if (reader != null) { + reader.close(); + } + } + } + + void readSingleValueOneByteAtATime(byte[] stream, IonReaderAssertion assertion) throws Exception { + readValuesOneByteAtATime(new Value(stream, assertion)); + } + + abstract byte[] toBytes(String textIon) throws IOException; + abstract byte[] intZeroWithoutIvm(); + + private void typedNull(final IonType type) throws Exception { + readSingleValueOneByteAtATime( + toBytes("null." + type.name().toLowerCase()), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(type, reader.next()); + assertTrue(reader.isNullValue()); + } + } + ); + } + + @Test + public void nulls() throws Exception { + for (IonType type : IonType.values()) { + if (type == IonType.DATAGRAM) continue; + typedNull(type); + } + } + + private void booleanValue(final boolean value) throws Exception { + readSingleValueOneByteAtATime( + toBytes("" + value), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.BOOL, reader.next()); + assertEquals(value, reader.booleanValue()); + } + } + ); + } + + @Test + public void booleans() throws Exception { + booleanValue(true); + booleanValue(false); + } + + private void integerValue(final int value) throws Exception { + readSingleValueOneByteAtATime( + toBytes("" + value), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.INT, reader.next()); + assertEquals(value, reader.intValue()); + } + } + ); + } + + @Test + public void integers() throws Exception { + integerValue(0); + integerValue(-1); + integerValue(Integer.MAX_VALUE); + integerValue(Integer.MIN_VALUE); + } + + private void floatValue(final String ion, final double value) throws Exception { + readSingleValueOneByteAtATime( + toBytes(ion), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.FLOAT, reader.next()); + assertEquals(value, reader.doubleValue(), 1e-9); + } + } + ); + } + + @Test + public void floats() throws Exception { + floatValue("0e0", 0.); + floatValue("nan", Double.NaN); + floatValue("+inf", Double.POSITIVE_INFINITY); + floatValue("-inf", Double.NEGATIVE_INFINITY); + } + + private void decimalValue(final String ion, final Decimal value) throws Exception { + readSingleValueOneByteAtATime( + toBytes(ion), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.DECIMAL, reader.next()); + assertEquals(value, reader.decimalValue()); + } + } + ); + } + + @Test + public void decimals() throws Exception { + decimalValue("0.", Decimal.ZERO); + decimalValue("-0.", Decimal.NEGATIVE_ZERO); + decimalValue("1000000000000.0", Decimal.valueOf(new BigDecimal("1000000000000.0"))); + } + + private void timestampValue(final Timestamp value) throws Exception { + readSingleValueOneByteAtATime( + toBytes(value.toString()), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.TIMESTAMP, reader.next()); + assertEquals(value, reader.timestampValue()); + } + } + ); + } + + @Test + public void timestamps() throws Exception { + timestampValue(Timestamp.valueOf("2000T")); + timestampValue(Timestamp.valueOf("2000-01T")); + timestampValue(Timestamp.valueOf("2000-01-01")); + timestampValue(Timestamp.valueOf("2000-01-01T00:00Z")); + timestampValue(Timestamp.valueOf("2000-01-01T00:00:00Z")); + timestampValue(Timestamp.valueOf("2000-01-01T00:00:00.000Z")); + timestampValue(Timestamp.valueOf("2000-01-01T00:00:00.000-07:00")); + } + + private void textValue(final IonText value) throws Exception { + readSingleValueOneByteAtATime( + toBytes(value.toString()), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(value.getType(), reader.next()); + assertEquals(value.stringValue(), reader.stringValue()); + } + } + ); + } + + @Test + public void symbols() throws Exception { + // NOTE: these exercise skipping over a symbol table. + textValue(ION_SYSTEM.newSymbol("")); + textValue(ION_SYSTEM.newString("abc")); + textValue(ION_SYSTEM.newString("abcdefghijklmnopqrstuvwxyz0123456789")); + } + + @Test + public void symbolWithUnknownText() throws Exception { + readSingleValueOneByteAtATime( + toBytes("$0"), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.SYMBOL, reader.next()); + SymbolToken symbolToken = reader.symbolValue(); + assertNull(symbolToken.getText()); + assertEquals(0, symbolToken.getSid()); + } + } + ); + } + + @Test + public void strings() throws Exception { + textValue(ION_SYSTEM.newString("")); + textValue(ION_SYSTEM.newString("abc")); + textValue(ION_SYSTEM.newString("abcdefghijklmnopqrstuvwxyz0123456789")); + } + + private void lobValue(final IonLob lob) throws Exception { + readSingleValueOneByteAtATime( + toBytes(lob.toString()), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(lob.getType(), reader.next()); + assertArrayEquals(lob.getBytes(), reader.newBytes()); + } + } + ); + } + + @Test + public void clobs() throws Exception { + lobValue(ION_SYSTEM.newClob("a".getBytes(UTF_8))); + lobValue(ION_SYSTEM.newClob("abcdefghijklmnopqrstuvwxyz0123456789".getBytes(UTF_8))); + } + + @Test + public void blobs() throws Exception { + lobValue(ION_SYSTEM.newBlob(new byte[]{})); + lobValue(ION_SYSTEM.newBlob(new byte[]{ 0x01 })); + lobValue(ION_SYSTEM.newBlob("abcdefghijklmnopqrstuvwxyz0123456789".getBytes(UTF_8))); + } + + private void emptyContainer(final String ion, final IonType type) throws Exception { + readSingleValueOneByteAtATime( + toBytes(ion), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(type, reader.next()); + reader.stepIn(); + assertNull(reader.next()); + reader.stepOut(); + } + } + ); + } + + @Test + public void emptyContainers() throws Exception { + emptyContainer("[]", IonType.LIST); + emptyContainer("()", IonType.SEXP); + emptyContainer("{}", IonType.STRUCT); + } + + @Test + public void annotations() throws Exception { + readSingleValueOneByteAtATime( + toBytes("abc::0"), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.INT, reader.next()); + String[] annotations = reader.getTypeAnnotations(); + assertEquals(1, annotations.length); + assertEquals("abc", annotations[0]); + assertEquals(0, reader.intValue()); + } + } + ); + readSingleValueOneByteAtATime( + toBytes("abc::abcdefghijklmnopqrstuvwxyz0123456789"), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.SYMBOL, reader.next()); + String[] annotations = reader.getTypeAnnotations(); + assertEquals(1, annotations.length); + assertEquals("abc", annotations[0]); + assertEquals("abcdefghijklmnopqrstuvwxyz0123456789", reader.stringValue()); + } + } + ); + readSingleValueOneByteAtATime( + toBytes("abc::def::abcdefghijklmnopqrstuvwxyz0123456789::\"abcdefghijklmnopqrstuvwxyz0123456789\""), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.STRING, reader.next()); + String[] annotations = reader.getTypeAnnotations(); + assertEquals(3, annotations.length); + assertEquals("abc", annotations[0]); + assertEquals("def", annotations[1]); + assertEquals("abcdefghijklmnopqrstuvwxyz0123456789", annotations[2]); + assertEquals("abcdefghijklmnopqrstuvwxyz0123456789", reader.stringValue()); + } + } + ); + } + + @Test + public void list() throws Exception { + readSingleValueOneByteAtATime( + toBytes("[-0., 2000T, (abc)]"), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.LIST, reader.next()); + reader.stepIn(); + assertEquals(IonType.DECIMAL, reader.next()); + assertEquals(Decimal.NEGATIVE_ZERO, reader.decimalValue()); + assertEquals(IonType.TIMESTAMP, reader.next()); + assertEquals(Timestamp.valueOf("2000T"), reader.timestampValue()); + assertEquals(IonType.SEXP, reader.next()); + assertNull(reader.next()); + reader.stepOut(); + } + } + ); + } + + @Test + public void sexp() throws Exception { + readSingleValueOneByteAtATime( + toBytes("'123'::($ion_symbol_table::{} abcdefghijklmnopqrstuvwxyz0123456789)"), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.SEXP, reader.next()); + String[] annotations = reader.getTypeAnnotations(); + assertEquals(1, annotations.length); + assertEquals("123", annotations[0]); + reader.stepIn(); + assertEquals(IonType.STRUCT, reader.next()); + annotations = reader.getTypeAnnotations(); + assertEquals(1, annotations.length); + assertEquals("$ion_symbol_table", annotations[0]); + reader.stepIn(); + assertNull(reader.next()); + reader.stepOut(); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("abcdefghijklmnopqrstuvwxyz0123456789", reader.stringValue()); + assertNull(reader.next()); + reader.stepOut(); + } + } + ); + } + + @Test + public void struct() throws Exception { + readSingleValueOneByteAtATime( + toBytes("{'':[-1], a:b::nan, xyz:null.clob, bool:true}"), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.LIST, reader.next()); + assertEquals("", reader.getFieldName()); + reader.stepIn(); + assertEquals(IonType.INT, reader.next()); + assertEquals(-1, reader.intValue()); + assertNull(reader.next()); + reader.stepOut(); + assertEquals(IonType.FLOAT, reader.next()); + assertEquals("a", reader.getFieldName()); + String[] annotations = reader.getTypeAnnotations(); + assertEquals(1, annotations.length); + assertEquals("b", annotations[0]); + assertEquals(Double.NaN, reader.doubleValue(), 1e-9); + assertEquals(IonType.CLOB, reader.next()); + assertTrue(reader.isNullValue()); + assertEquals(IonType.BOOL, reader.next()); + assertEquals("bool", reader.getFieldName()); + assertTrue(reader.booleanValue()); + assertNull(reader.next()); + reader.stepOut(); + } + } + ); + } + + @Test + public void multipleTopLevelScalars() throws Exception { + readValuesOneByteAtATime( + new Value( + toBytes("abc"), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("abc", reader.stringValue()); + } + } + ), + new Value( + intZeroWithoutIvm(), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.INT, reader.next()); + assertEquals(0, reader.intValue()); + } + } + ) + ); + } + + @Test + public void multipleTopLevelValuesWithSystemValuesInBetween() throws Exception { + readValuesOneByteAtATime( + new Value( + toBytes("0e0"), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.FLOAT, reader.next()); + assertEquals(0., reader.doubleValue(), 0e-9); + } + } + ), + new Value( + toBytes("abcdefghijklmnopqrstuvwxyz0123456789"), // Includes an IVM and symbol table. + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("abcdefghijklmnopqrstuvwxyz0123456789", reader.stringValue()); + } + } + ), + new Value( + toBytes("{foo:abc::2001-01T}"), // Includes an IVM and symbol table. + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.TIMESTAMP, reader.next()); + assertEquals("foo", reader.getFieldName()); + String[] annotations = reader.getTypeAnnotations(); + assertEquals(1, annotations.length); + assertEquals("abc", annotations[0]); + assertEquals(Timestamp.valueOf("2001-01T"), reader.timestampValue()); + assertNull(reader.next()); + reader.stepOut(); + } + } + ) + ); + } + + @Test + public void multipleTopLevelContainers() throws Exception { + readValuesOneByteAtATime( + new Value( + toBytes("{foo:abc::2001-01T}"), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.TIMESTAMP, reader.next()); + assertEquals("foo", reader.getFieldName()); + String[] annotations = reader.getTypeAnnotations(); + assertEquals(1, annotations.length); + assertEquals("abc", annotations[0]); + assertEquals(Timestamp.valueOf("2001-01T"), reader.timestampValue()); + assertNull(reader.next()); + reader.stepOut(); + } + } + ), + new Value( + toBytes("[{foo:bar}, (baz zar), 123]"), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.LIST, reader.next()); + reader.stepIn(); + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("foo", reader.getFieldName()); + assertEquals("bar", reader.stringValue()); + reader.stepOut(); + assertEquals(IonType.SEXP, reader.next()); + assertEquals(IonType.INT, reader.next()); + assertEquals(123, reader.intValue()); + assertNull(reader.next()); + reader.stepOut(); + } + } + ), + new Value( + toBytes("()"), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.SEXP, reader.next()); + reader.stepIn(); + assertNull(reader.next()); + reader.stepOut(); + } + } + ) + ); + } + + @Test + public void symbolTableAnnotationOnNonSymbolTable() throws Exception { + readSingleValueOneByteAtATime( + toBytes("$ion_symbol_table::[123]"), + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.LIST, reader.next()); + String[] annotations = reader.getTypeAnnotations(); + assertEquals(1, annotations.length); + assertEquals("$ion_symbol_table", annotations[0]); + reader.stepIn(); + assertEquals(IonType.INT, reader.next()); + assertEquals(123, reader.intValue()); + reader.stepOut(); + } + } + ); + } + + @Test + public void handlerWithUnlimitedMaxSizeCountsBytesProcessed() throws Exception { + final AtomicLong totalBytesProcessed = new AtomicLong(); + builderSupplier = new BuilderSupplier() { + @Override + public Builder get() { + Builder builder = defaultBuilderSupplier().get(); + builder.withMaximumBufferSize(Integer.MAX_VALUE); + createCountingEventHandler(builder, totalBytesProcessed); + return builder; + } + }; + byte[] valueBytes = toBytes("[{foo:bar}, (baz zar), 123]"); + readValuesOneByteAtATime( + new Value( + valueBytes, + new IonReaderAssertion() { + @Override + public void evaluate(IonReader reader) { + assertEquals(IonType.LIST, reader.next()); + reader.stepIn(); + assertEquals(IonType.STRUCT, reader.next()); + reader.stepIn(); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("foo", reader.getFieldName()); + assertEquals("bar", reader.stringValue()); + reader.stepOut(); + assertEquals(IonType.SEXP, reader.next()); + assertEquals(IonType.INT, reader.next()); + assertEquals(123, reader.intValue()); + assertNull(reader.next()); + reader.stepOut(); + } + } + ) + ); + assertEquals(valueBytes.length, totalBytesProcessed.longValue()); + } + + @Test + public void errorOnSpecifiedMaxSizeAndNullHandler() { + thrown.expect(IllegalArgumentException.class); + build( + builderSupplier.get() + .withMaximumBufferSize(10) + .onOversizedValue(null), + new ByteArrayInputStream(new byte[]{}) + ); + } + + @Test + public void errorOnMaximumSizeLessThanFive() { + Builder builder = builderSupplier.get(); + int minimumMaximumBufferSize = builder.getMinimumMaximumBufferSize(); + builder.withMaximumBufferSize(minimumMaximumBufferSize - 1); + createThrowingOversizedEventHandlers(builder); + thrown.expect(IllegalArgumentException.class); + build( + builder, + new ByteArrayInputStream(new byte[]{}) + ); + } +} diff --git a/test/com/amazon/ion/impl/ResizingPipedInputStreamTest.java b/test/com/amazon/ion/impl/ResizingPipedInputStreamTest.java new file mode 100644 index 0000000000..126476ac8b --- /dev/null +++ b/test/com/amazon/ion/impl/ResizingPipedInputStreamTest.java @@ -0,0 +1,789 @@ +package com.amazon.ion.impl; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.EOFException; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.BufferOverflowException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +import static com.amazon.ion.BitUtils.bytes; +import static org.junit.Assert.assertTrue; +import static org.junit.runners.Parameterized.Parameter; +import static org.junit.runners.Parameterized.Parameters; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +@RunWith(Parameterized.class) +public class ResizingPipedInputStreamTest { + + private enum ReceiveMethod { + BYTE_ARRAY { + @Override + void receive(ResizingPipedInputStream input, byte[] bytes, int off, int len) { + if (off == 0 && len == bytes.length) { + // Not necessary, but provides coverage of this method. + input.receive(bytes); + } else { + input.receive(bytes, off, len); + } + } + }, + INDIVIDUAL_BYTES { + @Override + void receive(ResizingPipedInputStream input, byte[] bytes, int off, int len) { + for (int i = off; i < off + len; i++) { + input.receive(bytes[i]); + } + } + }, + INPUT_STREAM { + @Override + void receive(ResizingPipedInputStream input, byte[] bytes, int off, int len) throws IOException { + byte[] subset = bytes; + if (off > 0) { + int subsetLen = bytes.length - off; + subset = new byte[subsetLen]; + System.arraycopy(bytes, off, subset, 0, subsetLen); + } + ByteArrayInputStream inputStream = new ByteArrayInputStream(subset); + assertEquals(len, input.receive(inputStream, len)); + } + } + ; + + abstract void receive(ResizingPipedInputStream input, byte[] bytes, int off, int len) throws IOException; + } + + @Parameters(name = "initialSize={0}, receive={1}, boundary={2}") + public static Iterable bufferSizes() { + List parameters = new ArrayList(); + for (int initialSize : Arrays.asList(1, 2, 3, 4, 5, 6, 7)) { + for (ReceiveMethod receiveMethod : ReceiveMethod.values()) { + parameters.add(new Object[]{initialSize, receiveMethod, true}); + parameters.add(new Object[]{initialSize, receiveMethod, false}); + } + } + return parameters; + } + + @Parameter + public int bufferSize; + + @Parameter(1) + public ReceiveMethod receiveMethod; + + @Parameter(2) + public boolean useBoundary; + + public ResizingPipedInputStream input; + public int knownCapacity; + + @Before + public void setup() { + input = new ResizingPipedInputStream(bufferSize, Integer.MAX_VALUE, useBoundary); + knownCapacity = input.capacity(); + } + + private void handleBoundary(int len) { + if (useBoundary) { + assertTrue(input.available() <= input.size()); + assertEquals(len, input.availableBeyondBoundary()); + assertTrue(input.getBoundary() <= input.getWriteIndex()); + input.extendBoundary(len); + } + assertEquals(input.available(), input.size()); + assertEquals(input.getBoundary(), input.getWriteIndex()); + assertEquals(0, input.availableBeyondBoundary()); + } + + private void receive(int b) { + input.receive(b); + handleBoundary(1); + } + + private void receive(byte[] bytes, int off, int len) throws IOException { + receiveMethod.receive(input, bytes, off, len); + handleBoundary(len); + } + + private void receive(byte[] bytes) throws IOException { + receive(bytes, 0, bytes.length); + } + + private int receive(InputStream stream, int len) throws IOException { + int received = input.receive(stream, len); + handleBoundary(received); + return received; + } + + // Asserts that growth (since the start of the test or the last time this method was called) only occurs + // if the initial buffer size less than the given threshold. + private void assertGrowthThreshold(int growthThreshold) { + if (bufferSize < growthThreshold) { + assertTrue("Growth was expected but did not occur.", knownCapacity < input.capacity()); + } else { + assertEquals("Growth occurred but was not expected.", knownCapacity, input.capacity()); + } + knownCapacity = input.capacity(); + } + + // Asserts that growth (since the start of the test or the last time this method was called) did not occur. + private void assertNoGrowth() { + assertGrowthThreshold(0); + } + + @Test + public void basicWriteRead() throws Exception { + assertEquals(0, input.available()); + receive(1); + assertNoGrowth(); + assertEquals(1, input.available()); + assertEquals(1, input.read()); + assertEquals(0, input.available()); + receive(bytes(2, 3), 0, 2); + assertGrowthThreshold(2); + assertEquals(2, input.available()); + byte[] readTwo = new byte[2]; + assertEquals(2, input.read(readTwo)); + assertArrayEquals(bytes(2, 3), readTwo); + assertEquals(0, input.available()); + } + + @Test + public void alternatingWriteRead() throws Exception { + receive((byte) 1); + assertNoGrowth(); + assertEquals(1, input.read()); + receive(bytes(2, 3)); + assertGrowthThreshold(2); + assertEquals(2, input.read(new byte[2])); + receive(bytes(4, 5, 6)); + assertGrowthThreshold(3); + assertEquals(3, input.read(new byte[3])); + receive(bytes(14, 15)); + assertNoGrowth(); + assertEquals(2, input.read(new byte[2])); + receive(bytes(16)); + assertNoGrowth(); + assertEquals(16, input.read()); + } + + @Test + public void consecutiveReads() throws Exception { + assertEquals(0, input.available()); + receive(bytes(1, 2, 3), 0, 3); + assertGrowthThreshold(3); + assertEquals(3, input.available()); + assertEquals(1, input.read()); + assertEquals(2, input.available()); + assertEquals(2, input.read()); + assertEquals(1, input.available()); + assertEquals(3, input.read()); + assertEquals(0, input.available()); + } + + @Test + public void consecutiveSkips() throws Exception { + assertEquals(0, input.available()); + receive(bytes(1, 2, 3), 0, 3); + assertGrowthThreshold(3); + assertEquals(3, input.available()); + assertEquals(1, input.skip(1)); + assertEquals(2, input.available()); + assertEquals(1, input.skip(1)); + assertEquals(1, input.available()); + assertEquals(1, input.skip(1)); + assertEquals(0, input.available()); + } + + @Test + public void growsMultipleTimesOnOneWrite() throws Exception { + // Grows multiple times when the buffer size is less than 3. + receive(bytes(1, 2, 3, 4, 5), 0, 5); + assertGrowthThreshold(5); + assertEquals(5, input.available()); + byte[] readFive = new byte[5]; + assertEquals(5, input.read(readFive)); + assertArrayEquals(bytes(1, 2, 3, 4, 5), readFive); + assertEquals(0, input.available()); + } + + @Test + public void returnsNegativeOneFromRead() throws Exception { + assertEquals(0, input.available()); + assertEquals(-1, input.read()); + assertEquals(0, input.available()); + assertEquals(-1, input.read(new byte[5])); + assertEquals(0, input.available()); + assertNoGrowth(); + } + + @Test + public void readsLessBytesThanRequested() throws Exception { + receive(bytes(1, 2, 3, 4, 5)); + assertGrowthThreshold(5); + assertEquals(5, input.available()); + byte[] tryReadSix = bytes(0, 0, 0, 0, 0, 42); + assertEquals(5, input.read(tryReadSix)); + assertArrayEquals(bytes(1, 2, 3, 4, 5, 42), tryReadSix); + assertEquals(0, input.available()); + } + + @Test + public void skipsLessBytesThanRequested() throws Exception { + receive(bytes(1)); + assertEquals(1, input.skip(10)); + assertEquals(-1, input.read()); + assertEquals(0, input.available()); + assertEquals(0, input.skip(1)); + assertNoGrowth(); + } + + @Test + public void skipsLessBytesThanRequestedLongMaxValue() throws Exception { + receive(bytes(1, 2, 3, 4, 5)); + assertGrowthThreshold(5); + assertEquals(5, input.skip(Long.MAX_VALUE)); + assertEquals(-1, input.read()); + assertEquals(0, input.available()); + assertEquals(0, input.skip(1)); + assertNoGrowth(); + } + + @Test + public void skipsLessBytesThanRequestedWhenEmpty() throws Exception { + assertEquals(0, input.skip(10)); + receive( bytes(1)); + assertEquals(1, input.read()); + assertNoGrowth(); + } + + @Test + public void receiveSubsetOfBufferAfterStart() throws Exception { + receive(bytes(1, 2, 3, 4), 1, 3); + assertGrowthThreshold(3); + assertEquals(3, input.available()); + byte[] readThree = new byte[3]; + assertEquals(3, input.read(readThree)); + assertArrayEquals(bytes(2, 3, 4), readThree); + assertEquals(0, input.available()); + } + + @Test + public void receiveSubsetOfBufferBeforeEnd() throws Exception { + receive(bytes(1, 2, 3, 4), 1, 2); + assertGrowthThreshold(2); + assertEquals(2, input.available()); + byte[] readTwo = new byte[2]; + assertEquals(2, input.read(readTwo)); + assertArrayEquals(bytes(2, 3), readTwo); + assertEquals(0, input.available()); + } + + @Test + public void receiveLessBytesThanRequestedFromInputStream() throws Exception { + receive(bytes(1, 2)); + assertGrowthThreshold(2); + ByteArrayInputStream source = new ByteArrayInputStream(new byte[]{3, 4}); + assertEquals(2, receive(source, 3)); + assertEquals(4, input.available()); + byte[] readFour = new byte[4]; + assertEquals(4, input.read(readFour)); + assertArrayEquals(bytes(1, 2, 3, 4), readFour); + assertEquals(0, input.available()); + } + + @Test + public void readZeroReturnsZero() throws Exception { + receive( bytes(42)); + assertEquals(1, input.available()); + assertEquals(0, input.read(new byte[0])); + assertEquals(0, input.read(new byte[5], 0, 0)); + assertEquals(1, input.available()); + assertNoGrowth(); + } + + @Test + public void skipZeroReturnsZero() throws Exception { + receive(bytes(42)); + assertEquals(1, input.available()); + assertEquals(0, input.skip(0)); + assertEquals(1, input.available()); + assertNoGrowth(); + } + + @Test + public void readReturnsToStart() throws Exception { + receive(bytes(1, 2)); + assertGrowthThreshold(2); + assertEquals(1, input.read()); + assertEquals(2, input.read()); + assertEquals(0, input.available()); + assertEquals(-1, input.read()); + // Because there are no un-read bytes, the following should write at the beginning of the buffer and + // set the write and read indexes appropriately. + receive(bytes(3)); + assertNoGrowth(); + assertEquals(1, input.available()); + assertEquals(3, input.read()); + } + + @Test + public void skipReturnsToStart() throws Exception { + receive(bytes(1, 2)); + assertGrowthThreshold(2); + assertEquals(1, input.skip(1)); + assertEquals(1, input.skip(1)); + assertEquals(0, input.available()); + assertEquals(-1, input.read()); + // Because there are no un-read bytes, the following should write at the beginning of the buffer and + // set the write and read indexes appropriately. + receive(bytes(3)); + assertNoGrowth(); + assertEquals(1, input.available()); + assertEquals(1, input.skip(1)); + } + + @Test + public void skipsAndReads() throws Exception { + receive(bytes(1, 42, 2, 42, 42, 3, 4, 42, 42, 42, 5, 6, 7)); + assertEquals(1, input.read()); + assertEquals(1, input.skip(1)); + assertEquals(2, input.read()); + assertEquals(2, input.skip(2)); + byte[] readTwo = new byte[2]; + assertEquals(2, input.read(readTwo)); + assertArrayEquals(bytes(3, 4), readTwo); + assertEquals(3, input.skip(3)); + byte[] readThree = new byte[3]; + assertEquals(3, input.read(readThree)); + assertArrayEquals(bytes(5, 6, 7), readThree); + assertEquals(0, input.skip(1)); + assertEquals(-1, input.read()); + assertEquals(0, input.available()); + } + + @Test + public void rewind() throws Exception { + receive(bytes(1, 2, 3)); + assertEquals(1, input.read()); + int available = input.available(); + int readIndex = input.getReadIndex(); + int size = input.size(); + assertEquals(2, input.read()); + assertEquals(3, input.read()); + input.rewind(readIndex, available); + assertEquals(available, input.available()); + assertEquals(readIndex, input.getReadIndex()); + assertEquals(size, input.size()); + assertEquals(2, input.read()); + assertEquals(3, input.read()); + assertEquals(-1, input.read()); + assertEquals(0, input.available()); + input.rewind(0, 3); + assertEquals(3, input.available()); + assertEquals(3, input.size()); + assertEquals(0, input.availableBeyondBoundary()); + assertEquals(0, input.getReadIndex()); + byte[] readThree = new byte[3]; + assertEquals(3, input.read(readThree)); + assertArrayEquals(bytes(1, 2, 3), readThree); + + } + + private void assertCopyEquals(byte[] expected) throws Exception { + final int startAvailable = input.available(); + final int startReadIndex = input.getReadIndex(); + final int startCapacity = input.capacity(); + ByteArrayOutputStream copiedBytes = new ByteArrayOutputStream(); + input.copyTo(copiedBytes); + assertArrayEquals(expected, copiedBytes.toByteArray()); + // copyTo never mutates internal state. + assertEquals(startAvailable, input.available()); + assertEquals(startReadIndex, input.getReadIndex()); + assertEquals(startCapacity, input.capacity()); + } + + @Test + public void copyTo() throws Exception { + byte[] data = bytes(1, 2, 3, 4, 5); + receive(data); + assertCopyEquals(data); + assertEquals(1, input.read()); + assertCopyEquals(bytes(2, 3, 4, 5)); + assertEquals(2, input.read()); + assertCopyEquals(bytes(3, 4, 5)); + assertEquals(3, input.read()); + assertCopyEquals(bytes(4, 5)); + assertEquals(4, input.read()); + assertCopyEquals(bytes(5)); + assertEquals(5, input.read()); + assertCopyEquals(bytes()); + receive(bytes(6)); + assertCopyEquals(bytes(6)); + receive(bytes(7, 8)); + assertEquals(6, input.read()); + assertCopyEquals(bytes(7, 8)); + assertCopyEquals(bytes(7, 8)); + assertEquals(7, input.read()); + assertCopyEquals(bytes(8)); + assertEquals(8, input.read()); + assertCopyEquals(bytes()); + } + + @Test + public void truncate() throws Exception { + int markedAvailable = input.available(); + int markedWriteIndex = input.getWriteIndex(); + receive(bytes(9, 9)); + input.truncate(markedWriteIndex, markedAvailable); + assertEquals(0, input.available()); + receive(bytes(1, 2)); + markedAvailable = input.available(); + markedWriteIndex = input.getWriteIndex(); + receive(bytes(9, 9, 9, 9, 9)); + input.truncate(markedWriteIndex, markedAvailable); + assertEquals(2, input.available()); + receive(bytes(3, 4, 5)); + markedAvailable = input.available(); + markedWriteIndex = input.getWriteIndex(); + input.truncate(markedWriteIndex, markedAvailable); + assertEquals(5, input.available()); + byte[] readFive = new byte[5]; + assertEquals(5, input.read(readFive)); + assertArrayEquals(bytes(1, 2, 3, 4, 5), readFive); + assertEquals(0, input.available()); + } + + @Test(expected = IllegalArgumentException.class) + public void errorsOnInvalidInitialBufferSize() { + new ResizingPipedInputStream(0); + } + + @Test(expected = IllegalArgumentException.class) + public void errorsOnInvalidMaximumBufferSize() { + new ResizingPipedInputStream(10, 9, useBoundary); + } + + @Test + public void initialBufferSizeAndCapacity() { + assertEquals(bufferSize, input.getInitialBufferSize()); + assertEquals(bufferSize, input.capacity()); + } + + private static class ThrowingInputStream extends FilterInputStream { + + ThrowingInputStream(InputStream in) { + super(in); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + throw new EOFException(); + } + } + + @Test + public void receiveOfThrowingInputStreamDoesNotThrow() throws Exception { + // Tests that InputStream implementations that throw EOFException when too many bytes are requested do not + // cause ResizingPipedInputStream.receive to throw. + ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write("abc".getBytes("UTF-8")); + InputStream in = new ThrowingInputStream(new ByteArrayInputStream(out.toByteArray())); + assertEquals(0, receive(in, 100)); + in.close(); + } + + @Test + public void receiveOfPartialGzipStreamDoesNotThrow() throws Exception { + // Similar to the previous test, but specifically tests with GZIPInputStream since it is probably the most + // commonly-used InputStream implementation that has been known to exhibit this behavior. + ByteArrayOutputStream out = new ByteArrayOutputStream(); + OutputStream gzipOut = new GZIPOutputStream(out); + gzipOut.write("abc".getBytes("UTF-8")); + gzipOut.close(); + byte[] completeGzip = out.toByteArray(); + // Cutting off 7 bytes removes part of the GZIP trailer. + byte[] truncatedGzip = new byte[completeGzip.length - 7]; + System.arraycopy(completeGzip, 0, truncatedGzip, 0, truncatedGzip.length); + InputStream gzipIn = new GZIPInputStream(new ByteArrayInputStream(truncatedGzip)); + // Ask for more bytes than are available. The GZIPInputStream will throw EOFException because the trailer + // is incomplete. The ResizingPipedInputStream should handle this exception in the same way as a + // graceful EOF conveyed by any other InputStream implementation. Since only the trailer was missing, all + // three data bytes will be read. + assertEquals(3, receive(gzipIn, 100)); + assertEquals(0, receive(gzipIn, 100)); + byte[] bytesRead = new byte[3]; + assertEquals(3, input.read(bytesRead)); + assertEquals("abc", new String(bytesRead, "UTF-8")); + gzipIn.close(); + } + + @Test + public void seekTo() throws Exception { + receive(bytes(1, 2, 3)); + assertEquals(3, input.available()); + assertEquals(1, input.read()); + assertEquals(2, input.available()); + input.seekTo(0); + assertEquals(3, input.available()); + input.seekTo(2); + assertEquals(1, input.available()); + assertEquals(3, input.read()); + assertEquals(0, input.available()); + input.seekTo(1); + assertEquals(2, input.available()); + assertEquals(2, input.read()); + input.seekTo(3); + assertEquals(0, input.available()); + assertEquals(-1, input.read()); + input.seekTo(0); + assertEquals(3, input.available()); + byte[] readThree = new byte[3]; + assertEquals(3, input.read(readThree)); + assertArrayEquals(bytes(1, 2, 3), readThree); + assertEquals(0, input.available()); + input.seekTo(3); + assertEquals(0, input.available()); + } + + @Test + public void clear() throws Exception { + receive(bytes(1, 2, 3)); + assertEquals(1, input.read()); + input.clear(); + assertEquals(-1, input.read()); + assertEquals(0, input.available()); + receive(bytes(4, 5, 6)); + assertEquals(3, input.available()); + byte[] readThree = new byte[3]; + assertEquals(3, input.read(readThree)); + assertArrayEquals(bytes(4, 5, 6), readThree); + assertEquals(0, input.available()); + input.clear(); + assertEquals(-1, input.read()); + assertEquals(0, input.available()); + } + + @Test + public void getByteBuffer() throws Exception { + receive((byte) 1); + java.nio.ByteBuffer byteBuffer = input.getByteBuffer(0, 1); + assertEquals(1, byteBuffer.get()); + receive(bytes(2, 3)); + byteBuffer = input.getByteBuffer(0, 3); + byte[] bytes = new byte[3]; + byteBuffer.get(bytes); + assertArrayEquals(new byte[]{1, 2, 3}, bytes); + receive(bytes(4, 5, 6)); + bytes = new byte[3]; + byteBuffer = input.getByteBuffer(3, 6); + byteBuffer.get(bytes); + assertArrayEquals(new byte[]{4, 5, 6}, bytes); + } + + @Test + public void boundary() throws Exception { + if (!useBoundary) { + return; + } + input.receive(bytes(1, 2, 3)); + assertEquals(1, input.peek(0)); + assertEquals(2, input.peek(1)); + assertEquals(3, input.peek(2)); + byte[] twoThree = new byte[2]; + input.copyBytes(1, twoThree, 0, 2); + assertArrayEquals(bytes(2, 3), twoThree); + // The boundary has not been extended, so bytes are available through the InputStream interface. + assertEquals(0, input.available()); + assertEquals(0, input.getBoundary()); + assertEquals(-1, input.read()); + // size() indicates the total number of bytes. + assertEquals(3, input.size()); + assertEquals(3, input.availableBeyondBoundary()); + // Now extend the boundary and verify bytes become available. + input.extendBoundary(1); + assertEquals(1, input.getBoundary()); + assertEquals(1, input.available()); + assertEquals(2, input.availableBeyondBoundary()); + assertEquals(3, input.size()); + assertEquals(1, input.read()); + input.extendBoundary(2); + assertEquals(3, input.getBoundary()); + assertEquals(2, input.available()); + assertEquals(0, input.availableBeyondBoundary()); + assertEquals(2, input.size()); + twoThree = new byte[2]; + assertEquals(2, input.read(twoThree)); + assertArrayEquals(bytes(2, 3), twoThree); + assertEquals(0, input.available()); + assertEquals(0, input.availableBeyondBoundary()); + assertEquals(0, input.size()); + input.receive(4); + assertEquals(0, input.available()); + assertEquals(1, input.availableBeyondBoundary()); + assertEquals(1, input.size()); + byte[] fourFive = new byte[3]; + assertEquals(-1, input.read(fourFive)); + assertArrayEquals(bytes(0, 0, 0), fourFive); + input.receive(5); + assertEquals(0, input.available()); + assertEquals(2, input.availableBeyondBoundary()); + assertEquals(2, input.size()); + input.extendBoundary(1); + assertEquals(1, input.available()); + assertEquals(1, input.availableBeyondBoundary()); + assertEquals(2, input.size()); + assertEquals(1, input.read(fourFive)); + assertArrayEquals(bytes(4, 0, 0), fourFive); + assertEquals(0, input.available()); + assertEquals(1, input.availableBeyondBoundary()); + assertEquals(1, input.size()); + input.extendBoundary(1); + assertEquals(1, input.available()); + assertEquals(0, input.availableBeyondBoundary()); + assertEquals(1, input.size()); + assertEquals(1, input.read(fourFive, 1, 2)); + assertArrayEquals(bytes(4, 5, 0), fourFive); + assertEquals(0, input.available()); + assertEquals(0, input.availableBeyondBoundary()); + assertEquals(0, input.size()); + } + + @Test + public void rewindWithBoundary() throws Exception { + if (!useBoundary) { + return; + } + input.receive(bytes(1, 2, 3)); + input.extendBoundary(2); + assertEquals(1, input.read()); + int available = input.available(); + int readIndex = input.getReadIndex(); + int size = input.size(); + int availableBeyondBoundary = input.availableBeyondBoundary(); + assertEquals(2, input.read()); + assertEquals(1, input.availableBeyondBoundary()); + input.rewind(readIndex, available); + assertEquals(available, input.available()); + assertEquals(readIndex, input.getReadIndex()); + assertEquals(size, input.size()); + assertEquals(availableBeyondBoundary, input.availableBeyondBoundary()); + assertEquals(2, input.read()); + assertEquals(0, input.available()); + assertEquals(1, input.availableBeyondBoundary()); + input.extendBoundary(1); + assertEquals(3, input.read()); + assertEquals(0, input.available()); + assertEquals(0, input.availableBeyondBoundary()); + assertEquals(-1, input.read()); + input.rewind(0, 3); + assertEquals(3, input.available()); + assertEquals(0, input.availableBeyondBoundary()); + assertEquals(0, input.getReadIndex()); + assertEquals(3, input.getBoundary()); + byte[] readThree = new byte[3]; + assertEquals(3, input.read(readThree)); + assertArrayEquals(bytes(1, 2, 3), readThree); + } + + @Test + public void consolidate() throws Exception { + receive(bytes(1, 2, 98, 99, 3)); + assertEquals(5, input.available()); + assertEquals(5, input.size()); + input.consolidate(4, 2); + assertEquals(3, input.available()); + assertEquals(3, input.size()); + byte[] readThree = new byte[3]; + assertEquals(3, input.read(readThree)); + assertArrayEquals(bytes(1, 2, 3), readThree); + receive(bytes(4, 5)); + assertEquals(2, input.available()); + assertEquals(2, input.size()); + input.consolidate(input.getWriteIndex() - 1, input.getReadIndex()); + assertEquals(1, input.available()); + assertEquals(1, input.size()); + assertEquals(5, input.read()); + receive(bytes(6)); + assertEquals(1, input.available()); + assertEquals(1, input.size()); + input.consolidate(input.getWriteIndex(), input.getReadIndex()); + assertEquals(0, input.available()); + assertEquals(0, input.size()); + assertEquals(-1, input.read()); + } + + @Test + public void consolidateWithBadIndicesFails() throws Exception { + receive(bytes(1, 2, 3)); + try { + input.consolidate(4, 1); + Assert.fail("Expected exception because fromIndex 4 is beyond the writeIndex."); + } catch (IllegalArgumentException e) { + // Expected. + } + assertEquals(1, input.read()); + try { + input.consolidate(2, 0); + Assert.fail("Expected exception because toIndex 0 is before the readIndex."); + } catch (IllegalArgumentException e) { + // Expected. + } + input.receive(4); + if (useBoundary) { + // The boundary has not been extended beyond the last received byte. + try { + input.consolidate(input.getBoundary() + 1, input.getReadIndex()); + Assert.fail("Expected exception because fromIndex 4 is beyond the boundary."); + } catch (IllegalArgumentException e) { + // Expected. + } + } + } + + @Test(expected = BufferOverflowException.class) + public void errorsWhenMaximumSizeExceeded() { + input = new ResizingPipedInputStream(bufferSize, bufferSize, useBoundary); + input.receive(new byte[bufferSize + 1]); + } + + private static class RecordingNotificationConsumer implements ResizingPipedInputStream.NotificationConsumer { + + int leftShiftAmount = 0; + + @Override + public void bytesConsolidatedToStartOfBuffer(int leftShiftAmount) { + this.leftShiftAmount = leftShiftAmount; + } + } + + @Test + public void registerNotificationConsumer() throws Exception { + RecordingNotificationConsumer notificationConsumer = new RecordingNotificationConsumer(); + input.registerNotificationConsumer(notificationConsumer); + receive(bytes(1, 2)); + assertEquals(1, input.read()); + assertEquals(2, input.read()); + assertEquals(2, input.getReadIndex()); + // Receive one more byte than is available at the end of the buffer, requiring the bytes currently + // buffered to be moved to the start of the buffer. + receive(new byte[input.capacity() - input.getWriteIndex() + 1]); + // The bytes will have been shifted left by 2 since the readIndex was 2 before the shift. + assertEquals(2, notificationConsumer.leftShiftAmount); + } +} diff --git a/test/com/amazon/ion/impl/SymbolTableTest.java b/test/com/amazon/ion/impl/SymbolTableTest.java index 41ec83340f..a1a49073c7 100644 --- a/test/com/amazon/ion/impl/SymbolTableTest.java +++ b/test/com/amazon/ion/impl/SymbolTableTest.java @@ -973,9 +973,9 @@ public void testSymtabImageMaintenance() IonSystem system = system(); SymbolTable st = ((_Private_ValueFactory)system).getLstFactory().newLocalSymtab(system.getSystemSymbolTable()); st.intern("foo"); - IonStruct image = symtabTree(st); + IonStruct image = symtabTree(st, system); st.intern("bar"); - image = symtabTree(st); + image = symtabTree(st, system); IonList symbols = (IonList) image.get(SYMBOLS); assertEquals("[\"foo\",\"bar\"]", symbols.toString()); } diff --git a/test/com/amazon/ion/junit/IonAssert.java b/test/com/amazon/ion/junit/IonAssert.java index 163d5eddc2..c50a2e6427 100644 --- a/test/com/amazon/ion/junit/IonAssert.java +++ b/test/com/amazon/ion/junit/IonAssert.java @@ -112,12 +112,11 @@ public static void assertNoCurrentValue(IonReader in) @SuppressWarnings("deprecation") public static void assertEof(IonReader in) { - assertFalse(in.hasNext()); - assertFalse(in.hasNext()); assertNull(in.next()); assertNoCurrentValue(in); assertNull(in.next()); - assertFalse(in.hasNext()); + // The following is repeated intentionally to ensure that the reader acts consistently and sanely when + // next() is called multiple times at EOF. assertNull(in.next()); } diff --git a/test/com/amazon/ion/streaming/BadIonStreamingTest.java b/test/com/amazon/ion/streaming/BadIonStreamingTest.java index b9dcf008c1..b847fd79e0 100644 --- a/test/com/amazon/ion/streaming/BadIonStreamingTest.java +++ b/test/com/amazon/ion/streaming/BadIonStreamingTest.java @@ -54,8 +54,9 @@ public void testReadingScalars() try { byte[] buf = _Private_Utils.loadFileBytes(myTestFile); - IonReader it = system().newReader(buf); + IonReader it = getStreamingMode().newIonReader(system().getCatalog(), buf); TestUtils.deepRead(it, true); + it.close(); } catch (IonException e) { diff --git a/test/com/amazon/ion/streaming/BinaryStreamingTest.java b/test/com/amazon/ion/streaming/BinaryStreamingTest.java index 05de4a40e3..46b843f646 100644 --- a/test/com/amazon/ion/streaming/BinaryStreamingTest.java +++ b/test/com/amazon/ion/streaming/BinaryStreamingTest.java @@ -529,7 +529,7 @@ public void testAllValues() throw new Exception(e); } - IonReader r = system().newReader(buffer); + IonReader r = getStreamingMode().newIonReader(system().getCatalog(), buffer); IonType t; t = r.next(); @@ -539,6 +539,7 @@ public void testAllValues() for (TestValue tv : testvalues) { tv.readAndTestValue(r); } + r.close(); } @Test @@ -587,6 +588,12 @@ public void testValue2() wr.writeValues(ir); byte[] buffer = wr.getBytes(); dumpBuffer(buffer, buffer.length); + + ir = getStreamingMode().newIonReader(system().getCatalog(), buffer); + wr = system().newBinaryWriter(u); + wr.writeValues(ir); + buffer = wr.getBytes(); + dumpBuffer(buffer, buffer.length); } void dumpBuffer(byte[] buffer, int len) { @@ -638,25 +645,27 @@ public void testBoolValue() throw new Exception(e); } - IonReader ir = system().newReader(buffer); - if (ir.hasNext()) { - ir.next(); + IonReader ir = getStreamingMode().newIonReader(system().getCatalog(), buffer); + IonType t = ir.next(); + if (t != null) { ir.stepIn(); - while (ir.hasNext()) { - IonType t = ir.next(); + t = ir.next(); + while (t != null) { String name = ir.getFieldName(); boolean value = ir.booleanValue(); assertTrue( value ); if (BinaryStreamingTest._debug_flag) { System.out.println(t + " " + name +": " + value); } + t = ir.next(); } } + ir.close(); } @Test - public void testTwoMagicCookies() { + public void testTwoMagicCookies() throws Exception { IonBinaryWriter wr = system().newBinaryWriter(); byte[] buffer = null; @@ -675,7 +684,7 @@ public void testTwoMagicCookies() { System.arraycopy(buffer, 0, doublebuffer, 0, buffer.length); System.arraycopy(buffer, 0, doublebuffer, buffer.length, buffer.length); - IonReader ir = system().newReader(doublebuffer); + IonReader ir = getStreamingMode().newIonReader(system().getCatalog(), doublebuffer); // first copy assertEquals(IonType.STRUCT, ir.next()); @@ -704,11 +713,12 @@ public void testTwoMagicCookies() { ir.stepOut(); assertEquals(null, ir.next()); + ir.close(); } @Test - public void testBoolean() { + public void testBoolean() throws Exception { IonBinaryWriter wr = system().newBinaryWriter(); byte[] buffer = null; @@ -723,12 +733,13 @@ public void testBoolean() { throw new RuntimeException(e); } - IonReader ir = system().newReader(buffer); - if (ir.hasNext()) { - ir.next(); + IonReader ir = getStreamingMode().newIonReader(system().getCatalog(), buffer); + IonType t = ir.next(); + if (t != null) { ir.stepIn(); - while (ir.hasNext()) { - assertEquals(ir.next(), IonType.BOOL); + t = ir.next(); + while (t != null) { + assertEquals(t, IonType.BOOL); expectField(ir, "Foo"); //assertEquals(ir.getAnnotations(), new String[] { "boolean" }); String[] annotations = ir.getTypeAnnotations(); @@ -737,14 +748,16 @@ public void testBoolean() { assertTrue("boolean".equals(annotations[0])); } assertEquals(ir.booleanValue(), true); + t = ir.next(); } } + ir.close(); } //Test Sample map. //{hello=true, Almost Done.=true, This is a test String.=true, 12242.124598129=12242.124598129, Something=null, false=false, true=true, long=9326, 12=-12} @Test - public void testSampleMap() { + public void testSampleMap() throws Exception { IonBinaryWriter wr = system().newBinaryWriter(); byte[] buffer = null; @@ -785,7 +798,7 @@ public void testSampleMap() { throw new RuntimeException(e); } - IonReader ir = system().newReader(buffer); + IonReader ir = getStreamingMode().newIonReader(system().getCatalog(), buffer); if (ir.next() != null) { ir.stepIn(); while (ir.next() != null) { @@ -829,6 +842,7 @@ public void testSampleMap() { assertEquals(ir.intValue(), -12); } } + ir.close(); } @Test @@ -871,7 +885,7 @@ public void testBenchmarkDirect() throws IOException { bytes = wr.getBytes(); - IonReader ir = system().newReader(bytes); + IonReader ir = getStreamingMode().newIonReader(system().getCatalog(), bytes); assertEquals(IonType.STRUCT, ir.next()); ir.stepIn(); @@ -915,5 +929,7 @@ public void testBenchmarkDirect() throws IOException { ir.stepOut(); if (! _Private_Utils.READER_HASNEXT_REMOVED) assertFalse(ir.hasNext()); assertEquals(null, ir.next()); + + ir.close(); } } diff --git a/test/com/amazon/ion/streaming/GoodIonStreamingTest.java b/test/com/amazon/ion/streaming/GoodIonStreamingTest.java index 04336f132b..164bffb20e 100644 --- a/test/com/amazon/ion/streaming/GoodIonStreamingTest.java +++ b/test/com/amazon/ion/streaming/GoodIonStreamingTest.java @@ -58,9 +58,11 @@ void iterateIon(File myTestFile) { byte[] buf = _Private_Utils.loadFileBytes(myTestFile); - IonReader reader = system().newReader(buf); + IonReader reader = getStreamingMode().newIonReader(system().getCatalog(), buf); TestUtils.deepRead(reader); - IonReader reader2 = system().newReader(buf); + reader.close(); + IonReader reader2 = getStreamingMode().newIonReader(system().getCatalog(), buf); TestUtils.deepRead(reader2, false); + reader2.close(); } } diff --git a/test/com/amazon/ion/streaming/ReaderFacetTestCase.java b/test/com/amazon/ion/streaming/ReaderFacetTestCase.java index bfbbeb0d78..3627a839eb 100644 --- a/test/com/amazon/ion/streaming/ReaderFacetTestCase.java +++ b/test/com/amazon/ion/streaming/ReaderFacetTestCase.java @@ -38,13 +38,19 @@ public abstract class ReaderFacetTestCase */ public static final ReaderMaker[] NON_SPAN_READERS = { + ReaderMaker.FROM_INPUT_STREAM_BINARY_INCREMENTAL }; /** * These are the readers that don't support {@link OffsetSpan}s. */ - public static final ReaderMaker[] NON_OFFSET_SPAN_READERS = - ReaderMaker.valuesWith(ReaderMaker.Feature.DOM); + public static final ReaderMaker[] NON_OFFSET_SPAN_READERS; + static { + ReaderMaker[] domReaders = ReaderMaker.valuesWith(ReaderMaker.Feature.DOM); + NON_OFFSET_SPAN_READERS = new ReaderMaker[domReaders.length + 1]; + System.arraycopy(domReaders, 0, NON_OFFSET_SPAN_READERS, 0, domReaders.length); + NON_OFFSET_SPAN_READERS[domReaders.length] = ReaderMaker.FROM_INPUT_STREAM_BINARY_INCREMENTAL; + } /** * These are the readers that don't support {@link TextSpan}s. @@ -61,7 +67,8 @@ public abstract class ReaderFacetTestCase { ReaderMaker.FROM_INPUT_STREAM_BINARY, ReaderMaker.FROM_INPUT_STREAM_TEXT, - ReaderMaker.FROM_READER + ReaderMaker.FROM_READER, + ReaderMaker.FROM_INPUT_STREAM_BINARY_INCREMENTAL }; diff --git a/test/com/amazon/ion/streaming/ReaderIntegerSizeTest.java b/test/com/amazon/ion/streaming/ReaderIntegerSizeTest.java index 983d2b23e5..73fcb7193b 100644 --- a/test/com/amazon/ion/streaming/ReaderIntegerSizeTest.java +++ b/test/com/amazon/ion/streaming/ReaderIntegerSizeTest.java @@ -142,9 +142,11 @@ public void testGetIntegerSizeNegativeIntBoundary() private void testGetIntegerSizeIntBoundary(int boundaryValue, long pastBoundary) { in.next(); - assertEquals(IntegerSize.INT, in.getIntegerSize()); + // It's fine if IntegerSize recommends a larger-than-necessary type, just not a smaller one. + assertTrue(IntegerSize.INT == in.getIntegerSize() || IntegerSize.LONG == in.getIntegerSize()); assertEquals(boundaryValue, in.intValue()); - assertEquals(IntegerSize.INT, in.getIntegerSize()); // assert nothing changes until next() + // assert nothing changes until next() + assertTrue(IntegerSize.INT == in.getIntegerSize() || IntegerSize.LONG == in.getIntegerSize()); in.next(); assertEquals(IntegerSize.LONG, in.getIntegerSize()); assertEquals(pastBoundary, in.longValue()); @@ -154,9 +156,10 @@ private void testGetIntegerSizeIntBoundary(int boundaryValue, long pastBoundary) private void testGetIntegerSizeLongBoundary(long boundaryValue, BigInteger pastBoundary) { in.next(); - assertEquals(IntegerSize.LONG, in.getIntegerSize()); + // It's fine if IntegerSize recommends a larger-than-necessary type, just not a smaller one. + assertTrue(IntegerSize.LONG == in.getIntegerSize() || IntegerSize.BIG_INTEGER == in.getIntegerSize()); assertEquals(boundaryValue, in.longValue()); - assertEquals(IntegerSize.LONG, in.getIntegerSize()); + assertTrue(IntegerSize.LONG == in.getIntegerSize() || IntegerSize.BIG_INTEGER == in.getIntegerSize()); in.next(); assertEquals(IntegerSize.BIG_INTEGER, in.getIntegerSize()); assertEquals(pastBoundary, in.bigIntegerValue()); diff --git a/test/com/amazon/ion/streaming/ReaderSkippingTest.java b/test/com/amazon/ion/streaming/ReaderSkippingTest.java index 6a45032cbc..887210bbaa 100644 --- a/test/com/amazon/ion/streaming/ReaderSkippingTest.java +++ b/test/com/amazon/ion/streaming/ReaderSkippingTest.java @@ -72,8 +72,8 @@ public void setUp() throws Exception { super.setUp(); - myFullReader = system().newReader(new FileInputStream(myTestFile)); - mySkipReader = system().newReader(new FileInputStream(myTestFile)); + myFullReader = getStreamingMode().newIonReader(system().getCatalog(), new FileInputStream(myTestFile)); + mySkipReader = getStreamingMode().newIonReader(system().getCatalog(), new FileInputStream(myTestFile)); myRandom = new Random(SEED); } diff --git a/test/com/amazon/ion/streaming/ReaderTest.java b/test/com/amazon/ion/streaming/ReaderTest.java index 04c18a3cfb..4ca247ce29 100644 --- a/test/com/amazon/ion/streaming/ReaderTest.java +++ b/test/com/amazon/ion/streaming/ReaderTest.java @@ -178,6 +178,45 @@ public void testSymbolValueOnNonSymbol() } } + @Test + public void testReadingDecimalAsInteger() + { + // decimal value int conversion + read( + "0. 0 " + + "-2147483648. -2147483648 " + // Min int + "2147483647. 2147483647 " // Max int + ); + while (in.next() != null) + { + int actual = in.intValue(); + + in.next(); + int expected = in.intValue(); + + assertEquals(expected, actual); + } + } + + @Test + public void testReadingDecimalAsLong() + { + // decimal value int conversion + read( + "0. 0 " + + "2147483647. 2147483647 " + // Max int + "9223372036854775807. 9223372036854775807 " // Max long + ); + while (in.next() != null) + { + long actual = in.longValue(); + + in.next(); + long expected = in.longValue(); + + assertEquals(expected, actual); + } + } @Test public void testReadingDecimalAsBigInteger() @@ -201,6 +240,71 @@ public void testReadingDecimalAsBigInteger() } } + @Test + public void testReadingDecimalAsDouble() + { + // decimal value float conversion + read( + "0. 0e0 " + + "1.23 123e-2 " + + "-1.23 -1.23e0 " + + "2d24 2e24 " + ); + + while (in.next() != null) + { + double actual = in.doubleValue(); + + in.next(); + double expected = in.doubleValue(); + + assertEquals(expected, actual, 1e-9); + } + } + + @Test + public void testReadingFloatAsInteger() + { + // float value int conversion + read( + "-0e0 0 " + + " 0e0 0 " + + "-2147483648e0 -2147483648 " + // Min int + "2147483647e0 2147483647 " // Max int + ); + + while (in.next() != null) + { + int actual = in.intValue(); + + in.next(); + int expected = in.intValue(); + + assertEquals(expected, actual); + } + } + + @Test + public void testReadingFloatAsLong() + { + // float value int conversion + read( + "-0e0 0 " + + " 0e0 0 " + + "-2147483648e0 -2147483648 " + // Min int + "2147483647e0 2147483647 " // Max int + ); + + while (in.next() != null) + { + long actual = in.longValue(); + + in.next(); + long expected = in.longValue(); + + assertEquals(expected, actual); + } + } @Test public void testReadingFloatAsBigInteger() diff --git a/test/com/amazon/ion/streaming/RoundTripStreamingTest.java b/test/com/amazon/ion/streaming/RoundTripStreamingTest.java index f1eb07e002..98b27e1dcf 100644 --- a/test/com/amazon/ion/streaming/RoundTripStreamingTest.java +++ b/test/com/amazon/ion/streaming/RoundTripStreamingTest.java @@ -134,6 +134,7 @@ private byte[] makeText(byte[] buffer, boolean prettyPrint) tw.writeValues(in); tw.close(); + in.close(); byte[] buf = out.toByteArray(); // this is utf-8 return buf; @@ -156,7 +157,7 @@ private byte[] makeBinary(byte[] buffer) bw.writeValues(in); byte[] buf = bw.getBytes(); // this is binary - + in.close(); return buf; } @@ -174,7 +175,7 @@ private IonDatagram makeTree(byte[] buffer) tw.writeValues(in); //IonValue v = tw.getContentAsIonValue(); - + in.close(); return dg; } @@ -381,8 +382,7 @@ roundTripBufferResults roundTripBuffer(String pass, byte[] testBuffer) } IonReader makeIterator(byte [] testBuffer) { - IonReader inputIterator = system().newReader(testBuffer); - return inputIterator; + return getStreamingMode().newIonReader(system().getCatalog(), testBuffer); } diff --git a/test/com/amazon/ion/system/IonReaderBuilderTest.java b/test/com/amazon/ion/system/IonReaderBuilderTest.java index 6b15bb458b..f4433f2136 100644 --- a/test/com/amazon/ion/system/IonReaderBuilderTest.java +++ b/test/com/amazon/ion/system/IonReaderBuilderTest.java @@ -16,16 +16,27 @@ package com.amazon.ion.system; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import com.amazon.ion.IonBufferConfiguration; import com.amazon.ion.IonCatalog; +import com.amazon.ion.IonException; import com.amazon.ion.IonReader; import com.amazon.ion.IonType; import com.amazon.ion.IonWriter; import com.amazon.ion.impl._Private_IonBinaryWriterBuilder; + +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.zip.GZIPOutputStream; + +import com.amazon.ion.impl._Private_IonConstants; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -111,4 +122,73 @@ public void testSystemFreeRoundtrip() throws IOException assertEquals(42, reader.intValue()); } + @Test + public void testEnableIncrementalReading() throws IOException + { + IonReaderBuilder builder = IonReaderBuilder.standard(); + assertFalse(builder.isIncrementalReadingEnabled()); + builder.withIncrementalReadingEnabled(true); + assertTrue(builder.isIncrementalReadingEnabled()); + builder.setIncrementalReadingDisabled(); + assertFalse(builder.isIncrementalReadingEnabled()); + builder.setIncrementalReadingEnabled(); + assertTrue(builder.isIncrementalReadingEnabled()); + builder.withIncrementalReadingEnabled(false); + assertFalse(builder.isIncrementalReadingEnabled()); + ByteArrayOutputStream data = new ByteArrayOutputStream(); + data.write(_Private_IonConstants.BINARY_VERSION_MARKER_1_0); + data.write(0xE5); // 5-byte annotation wrapper (incomplete). + IonReader reader1 = builder.build(data.toByteArray()); + try { + reader1.next(); + fail(); + } catch (IonException e) { + // Expected; this is a non-incremental reader, but a complete value was not available. + } + builder.withIncrementalReadingEnabled(true); + IonReader reader2 = builder.build(data.toByteArray()); + assertNull(reader2.next()); + IonReader reader3 = builder.build(new ByteArrayInputStream(data.toByteArray())); + assertNull(reader3.next()); + } + + @Test + public void testBufferConfiguration() + { + IonBufferConfiguration configuration1 = IonBufferConfiguration.Builder.standard().build(); + IonBufferConfiguration configuration2 = IonBufferConfiguration.Builder.standard().build(); + IonReaderBuilder builder = IonReaderBuilder.standard(); + assertNull(builder.getBufferConfiguration()); + builder.withBufferConfiguration(configuration1); + assertSame(configuration1, builder.getBufferConfiguration()); + builder.setBufferConfiguration(configuration2); + assertSame(configuration2, builder.getBufferConfiguration()); + builder.withBufferConfiguration(null); + assertNull(builder.getBufferConfiguration()); + } + + @Test + public void testIncrementalReadingDoesNotSupportAutoGzip() throws IOException + { + IonReaderBuilder builder = IonReaderBuilder.standard(); + builder.withIncrementalReadingEnabled(true); + ByteArrayOutputStream data = new ByteArrayOutputStream(); + GZIPOutputStream gzip = new GZIPOutputStream(data); + gzip.write(_Private_IonConstants.BINARY_VERSION_MARKER_1_0); + gzip.write(0x20); // int 0. + gzip.close(); + try { + builder.build(data.toByteArray()); + fail(); + } catch (IllegalArgumentException e) { + // Expected; non-incremental readers do not support auto-deflate of GZIP. + } + try { + builder.build(new ByteArrayInputStream(data.toByteArray())); + fail(); + } catch (IllegalArgumentException e) { + // Expected; non-incremental readers do not support auto-deflate of GZIP. + } + } + } From 9e03a92303cd43aa7064126a031473930f85d76c Mon Sep 17 00:00:00 2001 From: jobarr-amzn <70981087+jobarr-amzn@users.noreply.github.com> Date: Tue, 21 Sep 2021 16:39:01 -0700 Subject: [PATCH 136/490] Clean up assertions in ReaderTest (#384) Fixes #380 --- test/com/amazon/ion/streaming/ReaderTest.java | 233 +++++++----------- 1 file changed, 91 insertions(+), 142 deletions(-) diff --git a/test/com/amazon/ion/streaming/ReaderTest.java b/test/com/amazon/ion/streaming/ReaderTest.java index 4ca247ce29..ea822acdd8 100644 --- a/test/com/amazon/ion/streaming/ReaderTest.java +++ b/test/com/amazon/ion/streaming/ReaderTest.java @@ -20,6 +20,7 @@ import com.amazon.ion.BinaryTest; import com.amazon.ion.Decimal; +import com.amazon.ion.IonReader; import com.amazon.ion.IonType; import com.amazon.ion.ReaderMaker; import com.amazon.ion.SymbolTable; @@ -178,132 +179,104 @@ public void testSymbolValueOnNonSymbol() } } - @Test - public void testReadingDecimalAsInteger() - { - // decimal value int conversion - read( - "0. 0 " + - "-2147483648. -2147483648 " + // Min int - "2147483647. 2147483647 " // Max int - ); - while (in.next() != null) - { - int actual = in.intValue(); + int readInt(String text) { + IonReader r = myReaderMaker.newReader(system(), text); + if (r.next() == null) { + fail("Expected to read an integer"); + } + return r.intValue(); + } - in.next(); - int expected = in.intValue(); + long readLong(String text) { + IonReader r = myReaderMaker.newReader(system(), text); + if (r.next() == null) { + fail("Expected to read a long"); + } + return r.longValue(); + } - assertEquals(expected, actual); + double readDouble(String text) { + IonReader r = myReaderMaker.newReader(system(), text); + if (r.next() == null) { + fail("Expected to read a double"); } + return r.doubleValue(); } - @Test - public void testReadingDecimalAsLong() - { - // decimal value int conversion - read( - "0. 0 " + - "2147483647. 2147483647 " + // Max int - "9223372036854775807. 9223372036854775807 " // Max long - ); - while (in.next() != null) - { - long actual = in.longValue(); + BigInteger readBigInteger(String text) { + IonReader r = myReaderMaker.newReader(system(), text); + if (r.next() == null) { + fail("Expected to read a BigInteger"); + } + return r.bigIntegerValue(); + } - in.next(); - long expected = in.longValue(); + Decimal readDecimal(String text) { + IonReader r = myReaderMaker.newReader(system(), text); + if (r.next() == null) { + fail("Expected to read a Decimal"); + } + return r.decimalValue(); + } - assertEquals(expected, actual); + BigDecimal readBigDecimal(String text) { + IonReader r = myReaderMaker.newReader(system(), text); + if (r.next() == null) { + fail("Expected to read an BigDecimal"); } + return r.bigDecimalValue(); } @Test - public void testReadingDecimalAsBigInteger() + public void testReadingDecimalAsInteger() { - // decimal value int conversion - read("null.decimal null.int " + - "0. 0 " + - "9223372036854775807. 9223372036854775807 " + // Max long - "2d24 2000000000000000000000000 " - ); - - while (in.next() != null) - { - BigInteger actual = in.bigIntegerValue(); - assertEquals(in.isNullValue(), actual == null); + assertEquals(readInt("0."), readInt("0")); + assertEquals(readInt("-2147483648."), readInt("-2147483648")); // Min int + assertEquals(readInt("2147483647."), readInt("2147483647")); // Max int + } - in.next(); - BigInteger expected = in.bigIntegerValue(); + @Test + public void testReadingDecimalAsLong() + { + assertEquals(readLong("0."), readLong("0")); + assertEquals(readLong("2147483647."), readLong("2147483647")); // Max int + assertEquals(readLong("9223372036854775807."), readLong("9223372036854775807")); // Max long + } - assertEquals(expected, actual); - } + @Test + public void testReadingDecimalAsBigInteger() + { + assertEquals(readBigInteger("0."), readBigInteger("0")); + assertEquals(readBigInteger("null.decimal"), readBigInteger("null.int")); + assertEquals(readBigInteger("9223372036854775807."), readBigInteger("9223372036854775807")); // Max long + assertEquals(readBigInteger("2d24"), readBigInteger("2000000000000000000000000")); } @Test public void testReadingDecimalAsDouble() { - // decimal value float conversion - read( - "0. 0e0 " + - "1.23 123e-2 " + - "-1.23 -1.23e0 " + - "2d24 2e24 " - ); - - while (in.next() != null) - { - double actual = in.doubleValue(); - - in.next(); - double expected = in.doubleValue(); - - assertEquals(expected, actual, 1e-9); - } + assertEquals(readDouble("0."), readDouble("0e0")); + assertEquals(readDouble("1.23"), readDouble("123e-2")); + assertEquals(readDouble("-1.23"), readDouble("-1.23e0")); + assertEquals(readDouble("2d24"), readDouble("2e24")); } @Test public void testReadingFloatAsInteger() { - // float value int conversion - read( - "-0e0 0 " + - " 0e0 0 " + - "-2147483648e0 -2147483648 " + // Min int - "2147483647e0 2147483647 " // Max int - ); - - while (in.next() != null) - { - int actual = in.intValue(); - - in.next(); - int expected = in.intValue(); - - assertEquals(expected, actual); - } + assertEquals(readInt("-0e0"), readInt("0")); + assertEquals(readInt("-0e0"), readInt("0")); + assertEquals(readInt("-2147483648e0"), readInt("-2147483648")); // Min int + assertEquals(readInt("2147483647e0"), readInt("2147483647")); // Max int } @Test public void testReadingFloatAsLong() { - // float value int conversion - read( - "-0e0 0 " + - " 0e0 0 " + - "-2147483648e0 -2147483648 " + // Min int - "2147483647e0 2147483647 " // Max int - ); - - while (in.next() != null) - { - long actual = in.longValue(); - - in.next(); - long expected = in.longValue(); - - assertEquals(expected, actual); - } + assertEquals(readLong("0e0"), readLong("0")); + assertEquals(readLong("0e0"), readLong("0")); + assertEquals(readLong("-2147483648e0"), readLong("-2147483648")); // Min int + assertEquals(readLong("2147483647e0"), readLong("2147483647")); // Max int } @Test @@ -311,26 +284,16 @@ public void testReadingFloatAsBigInteger() { // Note that one can't represent Long.MAX_VALUE as a double, there's // not enough bits in the mantissa! - - // float value int conversion - read("null.float null.int " + - "-0e0 0 " + - " 0e0 0 " + - "9223372036854776000e0 9223372036854776000 " + - "2e24 2000000000000000000000000 " + - "" - ); - - while (in.next() != null) - { - BigInteger actual = in.bigIntegerValue(); - assertEquals(in.isNullValue(), actual == null); - - in.next(); - BigInteger expected = in.bigIntegerValue(); - - assertEquals(expected, actual); - } + BigInteger actual = readBigInteger("null.float"); + assertNull(actual); + BigInteger expected = readBigInteger("null.int"); + assertNull(expected); + assertEquals(actual, expected); + + assertEquals(readBigInteger("-0e0"), readBigInteger("0")); + assertEquals(readBigInteger("0e0"), readBigInteger("0")); + assertEquals(readBigInteger("9223372036854776000e0"), readBigInteger("9223372036854776000")); + assertEquals(readBigInteger("2e24"), readBigInteger("2000000000000000000000000")); } @@ -345,33 +308,19 @@ public void testReadingFloatAsBigInteger() @Test @Ignore public void testReadingFloatAsBigDecimal() // TODO amzn/ion-java/issues/56 { + BigDecimal actualBd = readBigDecimal("null.float"); + assertEquals(in.isNullValue(), actualBd == null); + Decimal actualDec = readDecimal("null.decimal"); + assertEquals(in.isNullValue(), actualDec == null); + assertEquals(actualBd, actualDec); + // Note that one can't represent Long.MAX_VALUE as a double, there's // not enough bits in the mantissa! - - // float value decimal conversion - read("null.float null.decimal " + - "-0e0 -0. " + - " 0e0 0. " + - "9223372036854776000e0 9223372036854776000. " + - "9223372036854776000e12 9223372036854776000d12 " + - "2e24 2000000000000000000000000. " + - "" - ); - - while (in.next() != null) - { - BigDecimal actualBd = in.bigDecimalValue(); - Decimal actualDec = in.decimalValue(); - assertEquals(in.isNullValue(), actualBd == null); - assertEquals(in.isNullValue(), actualDec == null); - - in.next(); - BigDecimal expectedBd = in.bigDecimalValue(); - Decimal expectedDec = in.decimalValue(); - - assertEquals(expectedBd, actualBd); - assertEquals(expectedDec, actualDec); - } + assertEquals(readBigDecimal("-0e0"), readDecimal("-0.")); + assertEquals(readBigDecimal("0e0"), readDecimal("0.")); + assertEquals(readBigDecimal("9223372036854776000e0"), readDecimal("9223372036854776000.")); + assertEquals(readBigDecimal("9223372036854776000e12"), readDecimal("9223372036854776000d12")); + assertEquals(readBigDecimal("2e24"), readDecimal("2000000000000000000000000.")); } From de7fefe285a6e59f125c274da55efa047361e4af Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Fri, 24 Sep 2021 17:16:05 -0700 Subject: [PATCH 137/490] Adds an option to IonSystemBuilder to allow the incremental reader to be used by the DOM. (#385) --- .../ion/impl/IonReaderBinaryIncremental.java | 28 ++- .../ion/impl/_Private_IncrementalReader.java | 14 ++ .../ion/impl/_Private_IonReaderBuilder.java | 172 ++++++++++++++++++ .../ion/impl/_Private_IonReaderFactory.java | 5 +- src/com/amazon/ion/impl/_Private_Utils.java | 7 + .../amazon/ion/impl/lite/IonLoaderLite.java | 31 +++- .../amazon/ion/impl/lite/IonSymbolLite.java | 4 +- .../amazon/ion/impl/lite/IonSystemLite.java | 15 +- .../amazon/ion/system/IonReaderBuilder.java | 108 ++--------- .../amazon/ion/system/IonSystemBuilder.java | 62 ++++++- ...ByteArrayIteratorSystemProcessingTest.java | 5 + ...aryStreamIteratorSystemProcessingTest.java | 5 + test/com/amazon/ion/BinaryTest.java | 18 -- ...atagramTreeReaderSystemProcessingTest.java | 8 + test/com/amazon/ion/IonSystemTest.java | 17 ++ test/com/amazon/ion/IonTestCase.java | 44 ++--- .../LoadBinaryBytesSystemProcessingTest.java | 5 + .../LoadBinaryStreamSystemProcessingTest.java | 5 + .../impl/IonReaderBinaryIncrementalTest.java | 109 +++++++++++ .../impl/OptimizedBinaryWriterTestCase.java | 5 +- .../streaming/OffsetSpanBinaryReaderTest.java | 10 + .../ion/streaming/OffsetSpanReaderTest.java | 10 + .../ion/streaming/SeekableReaderTest.java | 35 ++++ .../amazon/ion/streaming/SpanReaderTest.java | 60 ++++++ 24 files changed, 608 insertions(+), 174 deletions(-) create mode 100644 src/com/amazon/ion/impl/_Private_IncrementalReader.java create mode 100644 src/com/amazon/ion/impl/_Private_IonReaderBuilder.java diff --git a/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java b/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java index 542a4ea46b..b5fe5c74d8 100644 --- a/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java +++ b/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java @@ -64,7 +64,7 @@ * To enable this implementation, use {@code IonReaderBuilder.withIncrementalReadingEnabled(true)}. *

    */ -class IonReaderBinaryIncremental implements IonReader, _Private_ReaderWriter { +class IonReaderBinaryIncremental implements IonReader, _Private_ReaderWriter, _Private_IncrementalReader { /* * Potential future enhancements: @@ -266,9 +266,8 @@ private static class SystemSymbolIDs { // symbol table is encountered in the stream. private SymbolTable cachedReadOnlySymbolTable = null; - // True if the current symbol table has already been transferred via pop_passed_symbol_table with the reader - // positioned at the current value; otherwise, false. - private boolean hasTransferredSymbolTable = false; + // The SymbolTable that was transferred via the last call to pop_passed_symbol_table. + private SymbolTable symbolTableLastTransferred = null; // The symbol ID of the current value's field name, or -1 if the current value is not in a struct. private int fieldNameSid = -1; @@ -1033,6 +1032,12 @@ private void nextAtTopLevel() { resetSymbolTable(); resetImports(); lookahead.resetIvmIndex(); + } else if (peekIndex < 0) { + // peekIndex is initialized to -1 and only increases. This branch is reached if the IVM does not occur + // first in the stream. This is necessary because currently a binary incremental reader will be created if + // an empty stream is provided to the IonReaderBuilder. If, once bytes appear in the stream, those bytes do + // not represent valid binary Ion, a quick failure is necessary. + throw new IonException("Binary Ion must start with an Ion version marker."); } List symbolTableMarkers = lookahead.getSymbolTableMarkers(); if (!symbolTableMarkers.isEmpty()) { @@ -1176,7 +1181,6 @@ public IonType next() { fieldNameSid = -1; lobBytesRead = 0; valueStartPosition = -1; - hasTransferredSymbolTable = false; hasAnnotations = false; if (containerStack.isEmpty()) { nextAtTopLevel(); @@ -1245,13 +1249,14 @@ public SymbolTable getSymbolTable() { @Override public SymbolTable pop_passed_symbol_table() { - if (hasTransferredSymbolTable) { + SymbolTable currentSymbolTable = getSymbolTable(); + if (currentSymbolTable == symbolTableLastTransferred) { // This symbol table has already been returned. Since the contract is that it is a "pop", it should not // be returned twice. return null; } - hasTransferredSymbolTable = true; - return getSymbolTable(); + symbolTableLastTransferred = currentSymbolTable; + return symbolTableLastTransferred; } @Override @@ -1937,7 +1942,7 @@ public T asFacet(Class facetType) { } @Override - public void close() throws IOException { + public void requireCompleteValue() { // NOTE: If we want to replace the other binary IonReader implementation with this one, the following // validation could be performed in next() if incremental mode is not enabled. That would allow this // implementation to behave in the same way as the other implementation when an incomplete value is @@ -1951,6 +1956,11 @@ public void close() throws IOException { throw new IonException("Unexpected EOF."); } } + } + + @Override + public void close() throws IOException { + requireCompleteValue(); inputStream.close(); } diff --git a/src/com/amazon/ion/impl/_Private_IncrementalReader.java b/src/com/amazon/ion/impl/_Private_IncrementalReader.java new file mode 100644 index 0000000000..aebb1db0ce --- /dev/null +++ b/src/com/amazon/ion/impl/_Private_IncrementalReader.java @@ -0,0 +1,14 @@ +package com.amazon.ion.impl; + +/** + * Interface to be implemented by all incremental IonReaders. See + * {@link com.amazon.ion.system.IonReaderBuilder#withIncrementalReadingEnabled(boolean)}. + */ +public interface _Private_IncrementalReader { + + /** + * Requires that the reader not currently be buffering an incomplete value. + * @throws com.amazon.ion.IonException if the reader is buffering an incomplete value. + */ + void requireCompleteValue(); +} diff --git a/src/com/amazon/ion/impl/_Private_IonReaderBuilder.java b/src/com/amazon/ion/impl/_Private_IonReaderBuilder.java new file mode 100644 index 0000000000..6e4f29263c --- /dev/null +++ b/src/com/amazon/ion/impl/_Private_IonReaderBuilder.java @@ -0,0 +1,172 @@ +package com.amazon.ion.impl; + +import com.amazon.ion.IonException; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonTextReader; +import com.amazon.ion.IonValue; +import com.amazon.ion.system.IonReaderBuilder; +import com.amazon.ion.util.IonStreamUtils; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; + +import static com.amazon.ion.impl.LocalSymbolTable.DEFAULT_LST_FACTORY; +import static com.amazon.ion.impl._Private_IonReaderFactory.makeIncrementalReader; +import static com.amazon.ion.impl._Private_IonReaderFactory.makeReader; + +/** + * {@link IonReaderBuilder} extension for internal use only. + */ +public class _Private_IonReaderBuilder extends IonReaderBuilder { + + private _Private_LocalSymbolTableFactory lstFactory; + + private _Private_IonReaderBuilder() { + super(); + lstFactory = DEFAULT_LST_FACTORY; + } + + private _Private_IonReaderBuilder(_Private_IonReaderBuilder that) { + super(that); + this.lstFactory = that.lstFactory; + } + + /** + * Declares the {@link _Private_LocalSymbolTableFactory} to use when constructing applicable readers. + * + * @param factory the factory to use, or {@link LocalSymbolTable#DEFAULT_LST_FACTORY} if null. + * + * @return this builder instance, if mutable; + * otherwise a mutable copy of this builder. + * + * @see #setLstFactory(_Private_LocalSymbolTableFactory) + */ + public IonReaderBuilder withLstFactory(_Private_LocalSymbolTableFactory factory) { + _Private_IonReaderBuilder b = (_Private_IonReaderBuilder) mutable(); + b.setLstFactory(factory); + return b; + } + + /** + * @see #withLstFactory(_Private_LocalSymbolTableFactory) + */ + public void setLstFactory(_Private_LocalSymbolTableFactory factory) { + mutationCheck(); + if (factory == null) { + lstFactory = DEFAULT_LST_FACTORY; + } else { + lstFactory = factory; + } + } + + public static class Mutable extends _Private_IonReaderBuilder { + + public Mutable() { + } + + public Mutable(IonReaderBuilder that) { + super((_Private_IonReaderBuilder) that); + } + + @Override + public IonReaderBuilder immutable() { + return new _Private_IonReaderBuilder(this); + } + + @Override + public IonReaderBuilder mutable() { + return this; + } + + @Override + protected void mutationCheck() { + } + + } + + @Override + public IonReader build(byte[] ionData, int offset, int length) + { + if (isIncrementalReadingEnabled()) { + if (IonStreamUtils.isGzip(ionData, offset, length)) { + throw new IllegalArgumentException("Automatic GZIP detection is not supported with incremental" + + "support enabled. Wrap the bytes with a GZIPInputStream and call build(InputStream)."); + } + if (IonStreamUtils.isIonBinary(ionData, offset, length)) { + return makeIncrementalReader(this, new ByteArrayInputStream(ionData, offset, length)); + } + } + return makeReader(validateCatalog(), ionData, offset, length, lstFactory); + } + + /** + * Determines whether a stream that begins with the bytes in the provided buffer could be binary Ion. + * @param buffer up to the first four bytes in a stream. + * @param length the actual number of bytes in the buffer. + * @return true if the first 'length' bytes in 'buffer' match the first 'length' bytes in the binary IVM. + */ + private static boolean startsWithIvm(byte[] buffer, int length) { + for (int i = 0; i < length; i++) { + if (_Private_IonConstants.BINARY_VERSION_MARKER_1_0[i] != buffer[i]) { + return false; + } + } + return true; + } + + @Override + public IonReader build(InputStream ionData) + { + InputStream wrapper = ionData; + if (isIncrementalReadingEnabled()) { + if (!ionData.markSupported()) { + wrapper = new BufferedInputStream(ionData); + } + wrapper.mark(_Private_IonConstants.BINARY_VERSION_MARKER_SIZE); + byte[] possibleIVM = new byte[_Private_IonConstants.BINARY_VERSION_MARKER_SIZE]; + int bytesRead; + try { + bytesRead = wrapper.read(possibleIVM); + wrapper.reset(); + } catch (IOException e) { + throw new IonException(e); + } + if (IonStreamUtils.isGzip(possibleIVM, 0, possibleIVM.length)) { + throw new IllegalArgumentException("Automatic GZIP detection is not supported with incremental" + + "support enabled. Wrap the bytes with a GZIPInputStream and call build(InputStream)."); + } + // If the input stream is growing, it is possible that fewer than BINARY_VERSION_MARKER_SIZE bytes are + // available yet. Simply check whether the stream *could* contain binary Ion based on the available bytes. + // If it can't, fall back to text. + // NOTE: if incremental text reading is added, there will need to be logic that handles the case where + // the reader is created with 0 bytes available, as it is impossible to determine text vs. binary without + // reading at least one byte. Currently, in that case, just create a binary incremental reader. Either the + // stream will always be empty (in which case it doesn't matter whether a text or binary reader is used) + // or it's a binary stream (in which case the correct reader was created) or it's a growing text stream + // (which has always been unsupported). + if (startsWithIvm(possibleIVM, bytesRead)) { + return makeIncrementalReader(this, wrapper); + } + } + return makeReader(validateCatalog(), wrapper, lstFactory); + } + + @Override + public IonReader build(Reader ionText) { + return makeReader(validateCatalog(), ionText, lstFactory); + } + + @Override + public IonReader build(IonValue value) { + return makeReader(validateCatalog(), value, lstFactory); + } + + @Override + public IonTextReader build(String ionText) { + return makeReader(validateCatalog(), ionText, lstFactory); + } + +} diff --git a/src/com/amazon/ion/impl/_Private_IonReaderFactory.java b/src/com/amazon/ion/impl/_Private_IonReaderFactory.java index 4f4fafd623..be09960dc4 100644 --- a/src/com/amazon/ion/impl/_Private_IonReaderFactory.java +++ b/src/com/amazon/ion/impl/_Private_IonReaderFactory.java @@ -241,9 +241,10 @@ public static final IonReader makeSystemReader(Reader chars) } public static final IonReader makeReader(IonCatalog catalog, - IonValue value) + IonValue value, + _Private_LocalSymbolTableFactory lstFactory) { - return new IonReaderTreeUserX(value, catalog, LocalSymbolTable.DEFAULT_LST_FACTORY); + return new IonReaderTreeUserX(value, catalog, lstFactory); } public static final IonReader makeSystemReader(IonSystem system, diff --git a/src/com/amazon/ion/impl/_Private_Utils.java b/src/com/amazon/ion/impl/_Private_Utils.java index cf0e22f3d7..e957377b32 100644 --- a/src/com/amazon/ion/impl/_Private_Utils.java +++ b/src/com/amazon/ion/impl/_Private_Utils.java @@ -879,6 +879,13 @@ public static IonStruct symtabTree(SymbolTable symtab, ValueFactory valueFactory } else { localSymbolTableAsStruct = (LocalSymbolTableAsStruct) new LocalSymbolTableAsStruct.Factory(valueFactory) .newLocalSymtab(symtab.getSystemSymbolTable(), symtab.getImportedTables()); + Iterator localSymbolsIterator = symtab.iterateDeclaredSymbolNames(); + while (localSymbolsIterator.hasNext()) { + String localSymbol = localSymbolsIterator.next(); + if (localSymbol != null) { + localSymbolTableAsStruct.intern(localSymbol); + } + } } return localSymbolTableAsStruct.getIonRepresentation(); } diff --git a/src/com/amazon/ion/impl/lite/IonLoaderLite.java b/src/com/amazon/ion/impl/lite/IonLoaderLite.java index 62e963de4c..e0a2aff94e 100644 --- a/src/com/amazon/ion/impl/lite/IonLoaderLite.java +++ b/src/com/amazon/ion/impl/lite/IonLoaderLite.java @@ -15,8 +15,6 @@ package com.amazon.ion.impl.lite; -import static com.amazon.ion.impl._Private_IonReaderFactory.makeReader; - import com.amazon.ion.IonCatalog; import com.amazon.ion.IonDatagram; import com.amazon.ion.IonException; @@ -24,8 +22,10 @@ import com.amazon.ion.IonReader; import com.amazon.ion.IonSystem; import com.amazon.ion.IonWriter; +import com.amazon.ion.impl._Private_IncrementalReader; import com.amazon.ion.impl._Private_IonWriterFactory; -import com.amazon.ion.impl._Private_LocalSymbolTableFactory; +import com.amazon.ion.system.IonReaderBuilder; + import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -41,7 +41,7 @@ final class IonLoaderLite /** Not null. */ private final IonCatalog _catalog; - private final _Private_LocalSymbolTableFactory _lstFactory; + private final IonReaderBuilder _readerBuilder; /** * @param system must not be null. @@ -54,7 +54,13 @@ public IonLoaderLite(IonSystemLite system, IonCatalog catalog) _system = system; _catalog = catalog; - _lstFactory = _system.getLstFactory(); + if (catalog == system.getCatalog()) { + _readerBuilder = system.getReaderBuilder(); + } else { + // The IonCatalog provided in the constructor always takes precedence over the one already configured on the + // system's IonReaderBuilder. + _readerBuilder = system.getReaderBuilder().withCatalog(catalog).immutable(); + } } public IonSystem getSystem() @@ -99,7 +105,7 @@ public IonDatagram load(File ionFile) throws IonException, IOException public IonDatagram load(String ionText) throws IonException { try { - IonReader reader = makeReader(_catalog, ionText, _lstFactory); + IonReader reader = _readerBuilder.build(ionText); IonDatagramLite datagram = load_helper(reader); return datagram; } @@ -111,7 +117,7 @@ public IonDatagram load(String ionText) throws IonException public IonDatagram load(Reader ionText) throws IonException, IOException { try { - IonReader reader = makeReader(_catalog, ionText, _lstFactory); + IonReader reader = _readerBuilder.build(ionText); IonDatagramLite datagram = load_helper(reader); return datagram; } @@ -124,7 +130,7 @@ public IonDatagram load(Reader ionText) throws IonException, IOException public IonDatagram load(byte[] ionData) throws IonException { - IonReader reader = makeReader(_catalog, ionData, 0, ionData.length, _lstFactory); + IonReader reader = _readerBuilder.build(ionData, 0, ionData.length); try { return load(reader); } @@ -142,14 +148,21 @@ public IonDatagram load(byte[] ionData) throws IonException public IonDatagram load(InputStream ionData) throws IonException, IOException { + IonReader reader = null; try { - IonReader reader = makeReader(_catalog, ionData, _lstFactory); + reader = _readerBuilder.build(ionData); return load(reader); } catch (IonException e) { IOException io = e.causeOfType(IOException.class); if (io != null) throw io; throw e; + } finally { + // If a value was incomplete, incremental readers will not yet have raised an error. Force an error + // to be raised in this case. + if (_readerBuilder.isIncrementalReadingEnabled() && reader instanceof _Private_IncrementalReader) { + ((_Private_IncrementalReader) reader).requireCompleteValue(); + } } } diff --git a/src/com/amazon/ion/impl/lite/IonSymbolLite.java b/src/com/amazon/ion/impl/lite/IonSymbolLite.java index 88ba206756..01bf587afb 100644 --- a/src/com/amazon/ion/impl/lite/IonSymbolLite.java +++ b/src/com/amazon/ion/impl/lite/IonSymbolLite.java @@ -308,8 +308,8 @@ private String stringValue(SymbolTableProvider symbolTableProvider) return null; } String name = _stringValue(symbolTableProvider); - if (name == null && _sid != 0) { - assert(_sid > 0); + if (name == null) { + assert(_sid >= 0); throw new UnknownSymbolException(_sid); } return name; diff --git a/src/com/amazon/ion/impl/lite/IonSystemLite.java b/src/com/amazon/ion/impl/lite/IonSystemLite.java index 95bfa4d08a..415bb3933a 100644 --- a/src/com/amazon/ion/impl/lite/IonSystemLite.java +++ b/src/com/amazon/ion/impl/lite/IonSystemLite.java @@ -18,7 +18,6 @@ import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID; import static com.amazon.ion.SystemSymbols.ION_1_0; import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; -import static com.amazon.ion.impl._Private_IonReaderFactory.makeReader; import static com.amazon.ion.impl._Private_IonReaderFactory.makeSystemReader; import static com.amazon.ion.impl._Private_Utils.addAllNonNull; import static com.amazon.ion.impl._Private_Utils.initialSymtab; @@ -43,6 +42,7 @@ import com.amazon.ion.UnexpectedEofException; import com.amazon.ion.UnsupportedIonVersionException; import com.amazon.ion.impl._Private_IonBinaryWriterBuilder; +import com.amazon.ion.impl._Private_IonReaderBuilder; import com.amazon.ion.impl._Private_IonSystem; import com.amazon.ion.impl._Private_IonWriterFactory; import com.amazon.ion.impl._Private_ScalarConversions.CantConvertException; @@ -89,6 +89,7 @@ public IonSystemLite(IonTextWriterBuilder twb, assert catalog == rb.getCatalog(); _catalog = catalog; + myReaderBuilder = ((_Private_IonReaderBuilder) rb).withLstFactory(_lstFactory).immutable(); _loader = new IonLoaderLite(this, catalog); _system_symbol_table = bwb.getInitialSymbolTable(); assert _system_symbol_table.isSystemTable(); @@ -99,8 +100,10 @@ public IonSystemLite(IonTextWriterBuilder twb, bwb.setSymtabValueFactory(this); myBinaryWriterBuilder = bwb.immutable(); + } - myReaderBuilder = rb.immutable(); + IonReaderBuilder getReaderBuilder() { + return myReaderBuilder; } //========================================================================== @@ -179,7 +182,7 @@ public SymbolTable getSystemSymbolTable(String ionVersionId) public Iterator iterate(Reader ionText) { - IonReader reader = makeReader(_catalog, ionText, _lstFactory); + IonReader reader = myReaderBuilder.build(ionText); ReaderIterator iterator = new ReaderIterator(this, reader); return iterator; } @@ -187,13 +190,13 @@ public Iterator iterate(Reader ionText) public Iterator iterate(InputStream ionData) { // This method causes a memory leak when reading a gzipped stream, see deprecation notice. - IonReader reader = makeReader(_catalog, ionData, _lstFactory); + IonReader reader = myReaderBuilder.build(ionData); return iterate(reader); } public Iterator iterate(String ionText) { - IonReader reader = makeReader(_catalog, ionText, _lstFactory); + IonReader reader = myReaderBuilder.build(ionText); ReaderIterator iterator = new ReaderIterator(this, reader); return iterator; } @@ -201,7 +204,7 @@ public Iterator iterate(String ionText) public Iterator iterate(byte[] ionData) { // This method causes a memory leak when reading a gzipped stream, see deprecation notice. - IonReader reader = makeReader(_catalog, ionData, _lstFactory); + IonReader reader = myReaderBuilder.build(ionData); return iterate(reader); } diff --git a/src/com/amazon/ion/system/IonReaderBuilder.java b/src/com/amazon/ion/system/IonReaderBuilder.java index dd2810639d..17a99c1110 100644 --- a/src/com/amazon/ion/system/IonReaderBuilder.java +++ b/src/com/amazon/ion/system/IonReaderBuilder.java @@ -15,9 +15,6 @@ package com.amazon.ion.system; -import static com.amazon.ion.impl._Private_IonReaderFactory.makeIncrementalReader; -import static com.amazon.ion.impl._Private_IonReaderFactory.makeReader; - import com.amazon.ion.IonBufferConfiguration; import com.amazon.ion.IonCatalog; import com.amazon.ion.IonException; @@ -26,11 +23,8 @@ import com.amazon.ion.IonSystem; import com.amazon.ion.IonTextReader; import com.amazon.ion.IonValue; -import com.amazon.ion.impl._Private_IonConstants; -import com.amazon.ion.util.IonStreamUtils; +import com.amazon.ion.impl._Private_IonReaderBuilder; -import java.io.BufferedInputStream; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.Reader; @@ -45,7 +39,7 @@ * in this class. */ @SuppressWarnings("deprecation") -public class IonReaderBuilder +public abstract class IonReaderBuilder { private IonCatalog catalog = null; @@ -53,15 +47,16 @@ public class IonReaderBuilder private IonBufferConfiguration bufferConfiguration = null; private boolean isAnnotationIteratorReuseEnabled = true; - private IonReaderBuilder() + protected IonReaderBuilder() { } - private IonReaderBuilder(IonReaderBuilder that) + protected IonReaderBuilder(IonReaderBuilder that) { this.catalog = that.catalog; this.isIncrementalReadingEnabled = that.isIncrementalReadingEnabled; this.bufferConfiguration = that.bufferConfiguration; + this.isAnnotationIteratorReuseEnabled = that.isAnnotationIteratorReuseEnabled; } /** @@ -72,7 +67,7 @@ private IonReaderBuilder(IonReaderBuilder that) */ public static IonReaderBuilder standard() { - return new Mutable(); + return new _Private_IonReaderBuilder.Mutable(); } /** @@ -82,7 +77,7 @@ public static IonReaderBuilder standard() */ public IonReaderBuilder copy() { - return new Mutable(this); + return new _Private_IonReaderBuilder.Mutable(this); } /** @@ -163,7 +158,7 @@ public IonCatalog getCatalog() return catalog; } - private IonCatalog validateCatalog() + protected IonCatalog validateCatalog() { // matches behavior in IonSystemBuilder when no catalog provided return catalog != null ? catalog : new SimpleCatalog(); @@ -374,19 +369,7 @@ public IonReader build(byte[] ionData) * * @see IonSystem#newReader(byte[], int, int) */ - public IonReader build(byte[] ionData, int offset, int length) - { - if (isIncrementalReadingEnabled) { - if (IonStreamUtils.isGzip(ionData, offset, length)) { - throw new IllegalArgumentException("Automatic GZIP detection is not supported with incremental" + - "support enabled. Wrap the bytes with a GZIPInputStream and call build(InputStream)."); - } - if (IonStreamUtils.isIonBinary(ionData, offset, length)) { - return makeIncrementalReader(this, new ByteArrayInputStream(ionData, offset, length)); - } - } - return makeReader(validateCatalog(), ionData, offset, length); - } + public abstract IonReader build(byte[] ionData, int offset, int length); /** * Based on the builder's configuration properties, creates a new IonReader @@ -408,32 +391,7 @@ public IonReader build(byte[] ionData, int offset, int length) * * @see IonSystem#newReader(InputStream) */ - public IonReader build(InputStream ionData) - { - InputStream wrapper = ionData; - if (isIncrementalReadingEnabled) { - if (!ionData.markSupported()) { - wrapper = new BufferedInputStream(ionData); - } - wrapper.mark(_Private_IonConstants.BINARY_VERSION_MARKER_SIZE); - byte[] possibleIVM = new byte[_Private_IonConstants.BINARY_VERSION_MARKER_SIZE]; - int bytesRead; - try { - bytesRead = wrapper.read(possibleIVM); - wrapper.reset(); - } catch (IOException e) { - throw new IonException(e); - } - if (IonStreamUtils.isGzip(possibleIVM, 0, possibleIVM.length)) { - throw new IllegalArgumentException("Automatic GZIP detection is not supported with incremental" + - "support enabled. Wrap the bytes with a GZIPInputStream and call build(InputStream)."); - } - if (possibleIVM.length == bytesRead && IonStreamUtils.isIonBinary(possibleIVM)) { - return makeIncrementalReader(this, wrapper); - } - } - return makeReader(validateCatalog(), wrapper); - } + public abstract IonReader build(InputStream ionData); /** * Based on the builder's configuration properties, creates a new @@ -452,10 +410,7 @@ public IonReader build(InputStream ionData) * * @see IonSystem#newReader(Reader) */ - public IonReader build(Reader ionText) - { - return makeReader(validateCatalog(), ionText); - } + public abstract IonReader build(Reader ionText); /** * Based on the builder's configuration properties, creates a new @@ -470,10 +425,7 @@ public IonReader build(Reader ionText) * * @see IonSystem#newReader(IonValue) */ - public IonReader build(IonValue value) - { - return makeReader(validateCatalog(), value); - } + public abstract IonReader build(IonValue value); /** * Based on the builder's configuration properties, creates an new @@ -483,40 +435,6 @@ public IonReader build(IonValue value) * * @see IonSystem#newReader(String) */ - public IonTextReader build(String ionText) - { - return makeReader(validateCatalog(), ionText); - } - - private static class Mutable extends IonReaderBuilder - { - - private Mutable() - { - } - - private Mutable(IonReaderBuilder that) - { - super(that); - } - - @Override - public IonReaderBuilder immutable() - { - return new IonReaderBuilder(this); - } - - @Override - public IonReaderBuilder mutable() - { - return this; - } - - @Override - protected void mutationCheck() - { - } - - } + public abstract IonTextReader build(String ionText); } diff --git a/src/com/amazon/ion/system/IonSystemBuilder.java b/src/com/amazon/ion/system/IonSystemBuilder.java index 22141e83b7..c6c69c74f8 100644 --- a/src/com/amazon/ion/system/IonSystemBuilder.java +++ b/src/com/amazon/ion/system/IonSystemBuilder.java @@ -101,6 +101,7 @@ public static IonSystemBuilder standard() IonCatalog myCatalog; boolean myStreamCopyOptimized = false; + IonReaderBuilder readerBuilder; /** You no touchy. */ @@ -113,6 +114,7 @@ private IonSystemBuilder(IonSystemBuilder that) { this.myCatalog = that.myCatalog; this.myStreamCopyOptimized = that.myStreamCopyOptimized; + this.readerBuilder = that.readerBuilder; } //========================================================================= @@ -271,6 +273,59 @@ public final IonSystemBuilder withStreamCopyOptimized(boolean optimized) + //========================================================================= + + /** + * Gets the reader builder whose options will be used when building an + * {@link IonSystem}. By default, {@link IonReaderBuilder#standard()} will + * be used. + * + * @see #setReaderBuilder(IonReaderBuilder) + * @see #withReaderBuilder(IonReaderBuilder) + */ + public final IonReaderBuilder getReaderBuilder() { + return readerBuilder; + } + + /** + * Sets the reader builder whose options will be used to use when building + * an {@link IonSystem}. The reader builder's catalog will never be used; the + * catalog provided to {@link #setCatalog(IonCatalog)} or + * {@link #withCatalog(IonCatalog)} will always be used instead. + * + * @param builder the reader builder to use in built systems. + * If null, each system will be built with {@link IonReaderBuilder#standard()}. + * + * @see #getReaderBuilder() + * @see #withReaderBuilder(IonReaderBuilder) + * + * @throws UnsupportedOperationException if this is immutable. + */ + public final void setReaderBuilder(IonReaderBuilder builder) { + mutationCheck(); + readerBuilder = builder; + } + + /** + * Declares the reader builder whose options will be used to use when building + * an {@link IonSystem}, returning a new mutable builder if this is immutable. + * The reader builder's catalog will never be used; the catalog provided to + * {@link #setCatalog(IonCatalog)} or {@link #withCatalog(IonCatalog)} will + * always be used instead. + * + * @param builder the reader builder to use in built systems. + * If null, each system will be built with {@link IonReaderBuilder#standard()}. + * + * @see #getReaderBuilder() + * @see #setReaderBuilder(IonReaderBuilder) + */ + public final IonSystemBuilder withReaderBuilder(IonReaderBuilder builder) + { + IonSystemBuilder b = mutable(); + b.setReaderBuilder(builder); + return b; + } + //========================================================================= /** @@ -298,10 +353,9 @@ public final IonSystem build() bwb.setInitialSymbolTable(systemSymtab); // This is what we need, more or less. // bwb = bwb.fillDefaults(); - IonReaderBuilder rb = IonReaderBuilder.standard().withCatalog(catalog); - IonSystem sys = newLiteSystem(twb, bwb, rb); - - return sys; + IonReaderBuilder rb = readerBuilder == null ? IonReaderBuilder.standard() : readerBuilder; + rb = rb.withCatalog(catalog); + return newLiteSystem(twb, bwb, rb); } //========================================================================= diff --git a/test/com/amazon/ion/BinaryByteArrayIteratorSystemProcessingTest.java b/test/com/amazon/ion/BinaryByteArrayIteratorSystemProcessingTest.java index 1bc2fac1a7..cfa28c8f3c 100644 --- a/test/com/amazon/ion/BinaryByteArrayIteratorSystemProcessingTest.java +++ b/test/com/amazon/ion/BinaryByteArrayIteratorSystemProcessingTest.java @@ -23,6 +23,11 @@ public class BinaryByteArrayIteratorSystemProcessingTest { private byte[] myBytes; + @Override + protected int expectedLocalNullSlotSymbolId() + { + return getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL ? 0 : 10; + } @Override protected void prepare(String text) diff --git a/test/com/amazon/ion/BinaryStreamIteratorSystemProcessingTest.java b/test/com/amazon/ion/BinaryStreamIteratorSystemProcessingTest.java index a0986d1201..e814e2982b 100644 --- a/test/com/amazon/ion/BinaryStreamIteratorSystemProcessingTest.java +++ b/test/com/amazon/ion/BinaryStreamIteratorSystemProcessingTest.java @@ -26,6 +26,11 @@ public class BinaryStreamIteratorSystemProcessingTest private byte[] myBytes; private InputStream myStream; + @Override + protected int expectedLocalNullSlotSymbolId() + { + return getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL ? 0 : 10; + } @Override protected void prepare(String text) diff --git a/test/com/amazon/ion/BinaryTest.java b/test/com/amazon/ion/BinaryTest.java index b29d3c998f..5cfe3d6995 100644 --- a/test/com/amazon/ion/BinaryTest.java +++ b/test/com/amazon/ion/BinaryTest.java @@ -78,24 +78,6 @@ public static String bytesToHex(final byte[] bytes) private IonValue ion(final String hex) { byte[] data = hexToBytes(MAGIC_COOKIE + hex); - // Note: when ion-java#379 is complete, the following branch should be removed. - if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { - IonReader reader = getStreamingMode().newIonReader(system().getCatalog(), data); - Iterator iterator = system().iterate(reader); - IonValue value = null; - if (iterator.hasNext()) { - value = iterator.next(); - } - if (iterator.hasNext()) { - throw new IonException("not a single value"); - } - try { - reader.close(); - } catch (IOException e) { - throw new RuntimeException(e); - } - return value; - } return system().singleValue(data); } diff --git a/test/com/amazon/ion/DatagramTreeReaderSystemProcessingTest.java b/test/com/amazon/ion/DatagramTreeReaderSystemProcessingTest.java index f8077abb08..9c328e8043 100644 --- a/test/com/amazon/ion/DatagramTreeReaderSystemProcessingTest.java +++ b/test/com/amazon/ion/DatagramTreeReaderSystemProcessingTest.java @@ -40,6 +40,14 @@ public void setDatagramMaker(DatagramMaker maker) myDatagramMaker = maker; } + @Override + protected int expectedLocalNullSlotSymbolId() + { + return getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL && myDatagramMaker.sourceIsBinary() + ? 0 + : 10; + } + /** * Load the datagram. The datagram is only loaded during * {@link #read()} and {@link #systemRead()}. diff --git a/test/com/amazon/ion/IonSystemTest.java b/test/com/amazon/ion/IonSystemTest.java index 9aa5df6ce5..ff1f20d486 100644 --- a/test/com/amazon/ion/IonSystemTest.java +++ b/test/com/amazon/ion/IonSystemTest.java @@ -29,12 +29,19 @@ import java.util.Arrays; import java.util.Iterator; import java.util.zip.GZIPOutputStream; + +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; public class IonSystemTest extends IonTestCase { + + @Rule + public ExpectedException thrown = ExpectedException.none(); + //======================================================================== // iterate(Reader) @@ -312,6 +319,12 @@ private void checkGzipDetection(byte[] bytes) checkInt(1234, dg.get(0)); } + private void expectIncrementalReaderToFail() { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + thrown.expect(IllegalArgumentException.class); + } + } + @Test public void testGzipDetection() throws Exception @@ -321,12 +334,14 @@ public void testGzipDetection() byte[] gzipTextBytes = gzip(textBytes); checkGzipDetection(textBytes); + expectIncrementalReaderToFail(); checkGzipDetection(gzipTextBytes); byte[] binaryBytes = loader().load(ionText).getBytes(); byte[] gzipBinaryBytes = gzip(binaryBytes); checkGzipDetection(binaryBytes); + expectIncrementalReaderToFail(); checkGzipDetection(gzipBinaryBytes); } @@ -347,6 +362,7 @@ public void singleValueTextGzip() throws IOException String ionText = "1234"; byte[] textBytes = _Private_Utils.utf8(ionText); byte[] gzipTextBytes = gzip(textBytes); + expectIncrementalReaderToFail(); IonInt ionValue = (IonInt) system().singleValue(gzipTextBytes); assertEquals(1234, ionValue.intValue()); @@ -358,6 +374,7 @@ public void singleValueBinaryGzip() throws IOException String ionText = "1234"; byte[] binaryIon = toBinaryIon(ionText); byte[] gzipBytes = gzip(binaryIon); + expectIncrementalReaderToFail(); IonInt ionValue = (IonInt) system().singleValue(gzipBytes); assertEquals(1234, ionValue.intValue()); diff --git a/test/com/amazon/ion/IonTestCase.java b/test/com/amazon/ion/IonTestCase.java index 170889ce33..92af228d3c 100644 --- a/test/com/amazon/ion/IonTestCase.java +++ b/test/com/amazon/ion/IonTestCase.java @@ -63,29 +63,26 @@ public abstract class IonTestCase protected enum StreamingMode { NEW_STREAMING() { @Override - public IonReader newIonReader(IonCatalog catalog, InputStream inputStream) { - return IonReaderBuilder.standard().withCatalog(catalog).build(inputStream); - } - - @Override - public IonReader newIonReader(IonCatalog catalog, byte[] data) { - return IonReaderBuilder.standard().withCatalog(catalog).build(data); + public IonReaderBuilder getReaderBuilder() { + return IonReaderBuilder.standard(); } }, NEW_STREAMING_INCREMENTAL() { @Override - public IonReader newIonReader(IonCatalog catalog, InputStream inputStream) { - return IonReaderBuilder.standard().withCatalog(catalog).withIncrementalReadingEnabled(true).build(inputStream); - } - - @Override - public IonReader newIonReader(IonCatalog catalog, byte[] data) { - return IonReaderBuilder.standard().withCatalog(catalog).withIncrementalReadingEnabled(true).build(data); + public IonReaderBuilder getReaderBuilder() { + return IonReaderBuilder.standard().withIncrementalReadingEnabled(true); } }; - public abstract IonReader newIonReader(IonCatalog catalog, InputStream inputStream); - public abstract IonReader newIonReader(IonCatalog catalog, byte[] data); + public abstract IonReaderBuilder getReaderBuilder(); + + public final IonReader newIonReader(IonCatalog catalog, InputStream inputStream) { + return getReaderBuilder().withCatalog(catalog).build(inputStream); + } + + public final IonReader newIonReader(IonCatalog catalog, byte[] data) { + return getReaderBuilder().withCatalog(catalog).build(data); + } } // Using an enum makes the test names more understandable than a boolean. @@ -259,16 +256,7 @@ public IonDatagram load(File ionFile) throws IonException, IOException { IonLoader loader = loader(); - IonDatagram dg; - // Note: when ion-java#379 is complete, the following branches should be replaced with - // `dg = loader.load(ionFile)`. - if (ionFile.getName().endsWith(".ion")) { - dg = loader.load(ionFile); - } else { - IonReader reader = myStreamingMode.newIonReader(system().getCatalog(), new FileInputStream(ionFile)); - dg = loader.load(reader); - reader.close(); - } + IonDatagram dg = loader.load(ionFile); // Flush out any encoding problems in the data. forceDeepMaterialization(dg); @@ -317,6 +305,7 @@ protected _Private_IonSystem newSystem(IonCatalog catalog) IonSystemBuilder b = IonSystemBuilder.standard().withCatalog(catalog); b.withStreamCopyOptimized(myStreamCopyOptimized); + b.withReaderBuilder(getStreamingMode().getReaderBuilder()); IonSystem system = b.build(); return (_Private_IonSystem) system; @@ -828,7 +817,7 @@ public static void checkSymbol(String name, int id, IonValue value) assertFalse(value.isNullValue()); - if (name == null && id != 0) + if (name == null) { try { sym.stringValue(); @@ -841,7 +830,6 @@ public static void checkSymbol(String name, int id, IonValue value) } else { - // Note: symbol zero follows this path because it causes stringValue() to return null rather than throw. assertEquals("symbol name", name, sym.stringValue()); } diff --git a/test/com/amazon/ion/LoadBinaryBytesSystemProcessingTest.java b/test/com/amazon/ion/LoadBinaryBytesSystemProcessingTest.java index d90dffb0e2..5e46a99545 100644 --- a/test/com/amazon/ion/LoadBinaryBytesSystemProcessingTest.java +++ b/test/com/amazon/ion/LoadBinaryBytesSystemProcessingTest.java @@ -23,6 +23,11 @@ public class LoadBinaryBytesSystemProcessingTest { private byte[] myBytes; + @Override + protected int expectedLocalNullSlotSymbolId() + { + return getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL ? 0 : 10; + } @Override protected void prepare(String text) diff --git a/test/com/amazon/ion/LoadBinaryStreamSystemProcessingTest.java b/test/com/amazon/ion/LoadBinaryStreamSystemProcessingTest.java index d938fd9a3b..487c0203bd 100644 --- a/test/com/amazon/ion/LoadBinaryStreamSystemProcessingTest.java +++ b/test/com/amazon/ion/LoadBinaryStreamSystemProcessingTest.java @@ -24,6 +24,11 @@ public class LoadBinaryStreamSystemProcessingTest { private byte[] myBytes; + @Override + protected int expectedLocalNullSlotSymbolId() + { + return getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL ? 0 : 10; + } @Override protected void prepare(String text) diff --git a/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java b/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java index 992737f5ed..5d114b19a4 100644 --- a/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java +++ b/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java @@ -4,10 +4,15 @@ import com.amazon.ion.Decimal; import com.amazon.ion.IntegerSize; import com.amazon.ion.IonBufferConfiguration; +import com.amazon.ion.IonDatagram; import com.amazon.ion.IonException; +import com.amazon.ion.IonLoader; import com.amazon.ion.IonReader; +import com.amazon.ion.IonString; +import com.amazon.ion.IonStruct; import com.amazon.ion.IonSystem; import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; import com.amazon.ion.IonWriter; import com.amazon.ion.SymbolTable; import com.amazon.ion.SymbolToken; @@ -790,6 +795,110 @@ public void incrementalMultipleValues() throws Exception { reader.close(); } + @Test + public void incrementalMultipleValuesLoadFromReader() throws Exception { + ResizingPipedInputStream pipe = new ResizingPipedInputStream(128); + final IonReaderBinaryIncremental reader = new IonReaderBinaryIncremental(STANDARD_READER_BUILDER, pipe); + final IonLoader loader = SYSTEM.getLoader(); + byte[] bytes = toBinary("value_type::\"StringValueLong\""); + for (byte b : bytes) { + IonDatagram empty = loader.load(reader); + assertTrue(empty.isEmpty()); + pipe.receive(b); + } + IonDatagram firstValue = loader.load(reader); + assertEquals(1, firstValue.size()); + IonString string = (IonString) firstValue.get(0); + assertEquals("StringValueLong", string.stringValue()); + assertEquals(Collections.singletonList("value_type"), Arrays.asList(string.getTypeAnnotations())); + bytes = toBinary("{foobar: \"StringValueLong\"}"); + for (byte b : bytes) { + IonDatagram empty = loader.load(reader); + assertTrue(empty.isEmpty()); + pipe.receive(b); + } + IonDatagram secondValue = loader.load(reader); + assertEquals(1, secondValue.size()); + IonStruct struct = (IonStruct) secondValue.get(0); + string = (IonString) struct.get("foobar"); + assertEquals("StringValueLong", string.stringValue()); + IonDatagram empty = loader.load(reader); + assertTrue(empty.isEmpty()); + reader.close(); + } + + @Test + public void incrementalMultipleValuesLoadFromInputStreamFails() throws Exception { + final ResizingPipedInputStream pipe = new ResizingPipedInputStream(1); + final IonLoader loader = IonSystemBuilder.standard() + .withReaderBuilder(STANDARD_READER_BUILDER) + .build() + .getLoader(); + IonDatagram empty = loader.load(pipe); + assertTrue(empty.isEmpty()); + pipe.receive(_Private_IonConstants.BINARY_VERSION_MARKER_1_0[0]); + // Because reader does not persist across load invocations, the loader must throw an exception if the reader + // had an incomplete value buffered. + thrown.expect(IonException.class); + loader.load(pipe); + } + + private static void incrementalMultipleValuesIterate( + Iterator iterator, + ResizingPipedInputStream pipe + ) throws Exception { + byte[] bytes = toBinary("value_type::\"StringValueLong\""); + for (byte b : bytes) { + assertFalse(iterator.hasNext()); + pipe.receive(b); + } + assertTrue(iterator.hasNext()); + IonString string = (IonString) iterator.next(); + assertEquals("StringValueLong", string.stringValue()); + assertEquals(Collections.singletonList("value_type"), Arrays.asList(string.getTypeAnnotations())); + bytes = toBinary("{foobar: \"StringValueLong\"}"); + for (byte b : bytes) { + assertFalse(iterator.hasNext()); + pipe.receive(b); + } + assertTrue(iterator.hasNext()); + IonStruct struct = (IonStruct) iterator.next(); + string = (IonString) struct.get("foobar"); + assertEquals("StringValueLong", string.stringValue()); + assertFalse(iterator.hasNext()); + } + + @Test + public void incrementalMultipleValuesIterateFromReader() throws Exception { + ResizingPipedInputStream pipe = new ResizingPipedInputStream(128); + IonReader reader = STANDARD_READER_BUILDER.build(pipe); + Iterator iterator = SYSTEM.iterate(reader); + incrementalMultipleValuesIterate(iterator, pipe); + reader.close(); + } + + @Test + public void incrementalMultipleValuesIterateFromInputStream() throws Exception { + ResizingPipedInputStream pipe = new ResizingPipedInputStream(128); + IonSystem system = IonSystemBuilder.standard().withReaderBuilder(STANDARD_READER_BUILDER).build(); + Iterator iterator = system.iterate(pipe); + incrementalMultipleValuesIterate(iterator, pipe); + } + + @Test + public void incrementalReadInitiallyEmptyStreamThatTurnsOutToBeText() { + // Note: if incremental text read support is added, this test will start failing, which is expected. For now, + // we ensure that this fails quickly. + ResizingPipedInputStream pipe = new ResizingPipedInputStream(1); + IonReader reader = STANDARD_READER_BUILDER.build(pipe); + assertNull(reader.next()); + // Valid text Ion. Also hex 0x20, which is binary int 0. However, it is not preceded by the IVM, so it must be + // interpreted as text. The binary reader must fail. + pipe.receive(' '); + thrown.expect(IonException.class); + reader.next(); + } + @Test public void incrementalSymbolTables() throws Exception { ResizingPipedInputStream pipe = new ResizingPipedInputStream(128); diff --git a/test/com/amazon/ion/impl/OptimizedBinaryWriterTestCase.java b/test/com/amazon/ion/impl/OptimizedBinaryWriterTestCase.java index f03707ab21..cb44b93335 100644 --- a/test/com/amazon/ion/impl/OptimizedBinaryWriterTestCase.java +++ b/test/com/amazon/ion/impl/OptimizedBinaryWriterTestCase.java @@ -180,7 +180,10 @@ protected void checkWriteValue(boolean expectedTransferInvoked) // TODO amzn/ion-java/issues/16 - Currently, doesn't copy annotations or field names, // so we always expect no transfer of raw bytes - if (ir.isInStruct() || ir.getTypeAnnotationSymbols().length > 0) + // TODO amzn/ion-java/issues/381 - IonReaderBinaryIncremental currently doesn't support the byte transfer + // facet. Once this is fixed, remove the `asFacet` check below. + if (ir.isInStruct() || ir.getTypeAnnotationSymbols().length > 0 + || ir.asFacet(_Private_ByteTransferReader.class) == null) { expectedTransferInvoked = false; } diff --git a/test/com/amazon/ion/streaming/OffsetSpanBinaryReaderTest.java b/test/com/amazon/ion/streaming/OffsetSpanBinaryReaderTest.java index 665eb853d2..5720b15701 100644 --- a/test/com/amazon/ion/streaming/OffsetSpanBinaryReaderTest.java +++ b/test/com/amazon/ion/streaming/OffsetSpanBinaryReaderTest.java @@ -65,6 +65,11 @@ private void readBeyondMaxInt(byte[] data, IonType valueType) @Test public void testCurrentSpanBeyondMaxInt() { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + return; + } IonDatagram dg = system().newDatagram(); dg.add().newBlob(new byte[2000]); byte[] binary = dg.getBytes(); @@ -80,6 +85,11 @@ public void testCurrentSpanBeyondMaxInt() @Test public void testCurrentSpanBeyondMaxIntForOrderedStruct() { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + return; + } // Value is ordered-struct { name:{{ /* 1024 bytes */}} } byte[] data = hexToBytes("E0 01 00 EA " diff --git a/test/com/amazon/ion/streaming/OffsetSpanReaderTest.java b/test/com/amazon/ion/streaming/OffsetSpanReaderTest.java index 9c62e0312f..5c6fa61776 100644 --- a/test/com/amazon/ion/streaming/OffsetSpanReaderTest.java +++ b/test/com/amazon/ion/streaming/OffsetSpanReaderTest.java @@ -55,6 +55,11 @@ public void checkCurrentSpan(int binaryStart, int binaryFinish, int textStart) @Test public void testCurrentSpan() { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + return; + } read("'''hello''' 1 2 3 4 5 6 7 8 9 10 '''Kumo the fluffy dog! He is so fluffy and yet so happy!'''"); assertSame(IonType.STRING, in.next()); checkCurrentSpan(4, 10, 0); @@ -72,6 +77,11 @@ public void testCurrentSpan() @Test public void testCurrentSpanFromStreamMed() throws IOException { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + return; + } final ByteArrayOutputStream buf = new ByteArrayOutputStream(); final int count = 8000; for (int i = 0; i < count; i++) { diff --git a/test/com/amazon/ion/streaming/SeekableReaderTest.java b/test/com/amazon/ion/streaming/SeekableReaderTest.java index cc6ce61cda..b31c2ad9aa 100644 --- a/test/com/amazon/ion/streaming/SeekableReaderTest.java +++ b/test/com/amazon/ion/streaming/SeekableReaderTest.java @@ -58,6 +58,11 @@ private void checkSpans(IonDatagram dg, Span[] positions) @Test public void testTrivialSpan() { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + return; + } String text = "null"; read(text); in.next(); @@ -73,6 +78,11 @@ public void testTrivialSpan() @Test public void testWalkingBackwards() { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + return; + } String text = "null true 3 4e0 5.0 6666-06-06T '7' \"8\" {{\"\"}} {{}} [] () {}"; @@ -113,6 +123,11 @@ public void testWalkingBackwards() @Test public void testHoistingWithinContainers() { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + return; + } read("{f:v,g:[c, (d), e], /* h */ $0:null} s"); in.next(); @@ -188,6 +203,11 @@ public void testHoistingWithinContainers() @Test public void testHoistingLongValue() { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + return; + } // This value is "long" in that it has a length subfield in the prefix. String text = " \"123456789012345\" "; read(text); @@ -205,6 +225,11 @@ public void testHoistingLongValue() public void testHoistingOrderedStruct() throws IOException { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + return; + } File file = getTestdataFile("good/structOrdered.10n"); byte[] binary = _Private_Utils.loadFileBytes(file); @@ -224,6 +249,11 @@ public void testHoistingOrderedStruct() public void testHoistingAnnotatedTopLevelValue() throws IOException { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + return; + } read("a::v"); in.next(); Span span = sr.currentSpan(); @@ -241,6 +271,11 @@ public void testHoistingAnnotatedTopLevelValue() public void testHoistingAnnotatedContainedValue() throws IOException { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + return; + } read("[a::v]"); in.next(); in.stepIn(); diff --git a/test/com/amazon/ion/streaming/SpanReaderTest.java b/test/com/amazon/ion/streaming/SpanReaderTest.java index 646906caca..d582049b36 100644 --- a/test/com/amazon/ion/streaming/SpanReaderTest.java +++ b/test/com/amazon/ion/streaming/SpanReaderTest.java @@ -43,6 +43,11 @@ public SpanReaderTest() @Test public void testCallingCurrentSpan() { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + return; + } String text = "null true 3 4e0 5.0 6666-06-06T '7' \"8\" {{\"\"}} {{}} [] () {}"; @@ -80,6 +85,11 @@ public void testCallingCurrentSpan() @Test public void testCurrentSpanWithinContainers() { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + return; + } read("{f:v,g:[c]} s"); in.next(); @@ -110,6 +120,11 @@ public void testCurrentSpanWithinContainers() @Test(expected=IllegalStateException.class) public void testCurrentSpanBeforeFirstTopLevel() { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + throw new IllegalStateException(); + } read("foo"); sp.currentSpan(); } @@ -117,6 +132,11 @@ public void testCurrentSpanBeforeFirstTopLevel() private void callCurrentSpanBeforeFirstChild(String ionText) { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + return; + } read(ionText); in.next(); in.stepIn(); @@ -126,24 +146,44 @@ private void callCurrentSpanBeforeFirstChild(String ionText) @Test(expected=IllegalStateException.class) public void testCurrentSpanBeforeFirstListChild() { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + throw new IllegalStateException(); + } callCurrentSpanBeforeFirstChild("[v]"); } @Test(expected=IllegalStateException.class) public void testCurrentSpanBeforeFirstSexpChild() { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + throw new IllegalStateException(); + } callCurrentSpanBeforeFirstChild("(v)"); } @Test(expected=IllegalStateException.class) public void testCurrentSpanBeforeFirstStructChild() { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + throw new IllegalStateException(); + } callCurrentSpanBeforeFirstChild("{f:v}"); } private void callCurrentSpanAfterLastChild(String ionText) { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + return; + } read(ionText); in.next(); in.stepIn(); @@ -155,18 +195,33 @@ private void callCurrentSpanAfterLastChild(String ionText) @Test(expected=IllegalStateException.class) public void testCurrentSpanAfterLastListChild() { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + throw new IllegalStateException(); + } callCurrentSpanAfterLastChild("[v]"); } @Test(expected=IllegalStateException.class) public void testCurrentSpanAfterLastSexpChild() { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + throw new IllegalStateException(); + } callCurrentSpanAfterLastChild("(v)"); } @Test(expected=IllegalStateException.class) public void testCurrentSpanAfterLastStructChild() { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + throw new IllegalStateException(); + } callCurrentSpanAfterLastChild("{f:v}"); } @@ -174,6 +229,11 @@ public void testCurrentSpanAfterLastStructChild() @Test(expected=IllegalStateException.class) public void testCurrentSpanAtEndOfStream() { + if (getStreamingMode() == StreamingMode.NEW_STREAMING_INCREMENTAL) { + // TODO the incremental reader does not currently support the SpanProvider or SeekableReader facets. + // See ion-java/issues/382 and ion-java/issues/383. + throw new IllegalStateException(); + } read("foo"); in.next(); assertEquals(null, in.next()); From f6fa517d9368deabab14e8ea2c1220faa3635fc6 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Fri, 24 Sep 2021 17:28:48 -0700 Subject: [PATCH 138/490] Adds caching of the IonStruct representations of local symbol tables created by the incremental reader, improving performance for repetitive system iterates/gets. (#386) --- .../ion/impl/IonReaderBinaryIncremental.java | 14 +- .../ion/impl/LocalSymbolTableAsStruct.java | 148 ++---------------- .../amazon/ion/impl/SymbolTableAsStruct.java | 18 +++ .../ion/impl/SymbolTableStructCache.java | 137 ++++++++++++++++ src/com/amazon/ion/impl/_Private_Utils.java | 14 +- 5 files changed, 190 insertions(+), 141 deletions(-) create mode 100644 src/com/amazon/ion/impl/SymbolTableAsStruct.java create mode 100644 src/com/amazon/ion/impl/SymbolTableStructCache.java diff --git a/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java b/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java index b5fe5c74d8..94c5d053d7 100644 --- a/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java +++ b/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java @@ -6,6 +6,7 @@ import com.amazon.ion.IonCatalog; import com.amazon.ion.IonException; import com.amazon.ion.IonReader; +import com.amazon.ion.IonStruct; import com.amazon.ion.IonType; import com.amazon.ion.IonWriter; import com.amazon.ion.ReadOnlyValueException; @@ -13,6 +14,7 @@ import com.amazon.ion.SymbolToken; import com.amazon.ion.Timestamp; import com.amazon.ion.UnknownSymbolException; +import com.amazon.ion.ValueFactory; import com.amazon.ion.system.IonReaderBuilder; import com.amazon.ion.system.SimpleCatalog; @@ -534,7 +536,7 @@ public int hashCode() { /** * Read-only snapshot of the local symbol table at the reader's current position. */ - private class LocalSymbolTableSnapshot implements SymbolTable { + private class LocalSymbolTableSnapshot implements SymbolTable, SymbolTableAsStruct { // The system symbol table. private final SymbolTable system = SharedSymbolTable.getSystemSymbolTable(majorVersion); @@ -555,6 +557,8 @@ private class LocalSymbolTableSnapshot implements SymbolTable { // List representation of this symbol table, indexed by symbol ID. final List listView; + private SymbolTableStructCache structCache = null; + LocalSymbolTableSnapshot() { int numberOfSymbols = symbols.size(); maxId = numberOfSymbols - 1; @@ -729,6 +733,14 @@ public void writeTo(IonWriter writer) throws IOException { public String toString() { return "(LocalSymbolTable max_id:" + getMaxId() + ')'; } + + @Override + public IonStruct getIonRepresentation(ValueFactory valueFactory) { + if (structCache == null) { + structCache = new SymbolTableStructCache(this, getImportedTables(), null); + } + return structCache.getIonRepresentation(valueFactory); + } } /** diff --git a/src/com/amazon/ion/impl/LocalSymbolTableAsStruct.java b/src/com/amazon/ion/impl/LocalSymbolTableAsStruct.java index 98d325174b..eba71e282e 100644 --- a/src/com/amazon/ion/impl/LocalSymbolTableAsStruct.java +++ b/src/com/amazon/ion/impl/LocalSymbolTableAsStruct.java @@ -15,19 +15,9 @@ package com.amazon.ion.impl; -import static com.amazon.ion.SystemSymbols.IMPORTS; -import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; -import static com.amazon.ion.SystemSymbols.MAX_ID; -import static com.amazon.ion.SystemSymbols.NAME; -import static com.amazon.ion.SystemSymbols.SYMBOLS; -import static com.amazon.ion.SystemSymbols.VERSION; - import com.amazon.ion.IonCatalog; -import com.amazon.ion.IonList; import com.amazon.ion.IonReader; import com.amazon.ion.IonStruct; -import com.amazon.ion.IonType; -import com.amazon.ion.IonValue; import com.amazon.ion.SymbolTable; import com.amazon.ion.ValueFactory; import java.util.ArrayList; @@ -41,6 +31,7 @@ @Deprecated class LocalSymbolTableAsStruct extends LocalSymbolTable + implements SymbolTableAsStruct { static class Factory implements _Private_LocalSymbolTableFactory @@ -72,7 +63,7 @@ public SymbolTable newLocalSymtab(IonCatalog catalog, // This was an LST append, so the existing symbol table was updated. return currentSymbolTable; } - return new LocalSymbolTableAsStruct(imageFactory, imports, symbolsList); + return new LocalSymbolTableAsStruct(imports, symbolsList, null); } public SymbolTable newLocalSymtab(SymbolTable defaultSystemSymtab, @@ -81,9 +72,7 @@ public SymbolTable newLocalSymtab(SymbolTable defaultSystemSymtab, LocalSymbolTableImports unifiedSymtabImports = new LocalSymbolTableImports(defaultSystemSymtab, imports); - return new LocalSymbolTableAsStruct(imageFactory, - unifiedSymtabImports, - null /* local symbols */); + return new LocalSymbolTableAsStruct(unifiedSymtabImports, null /* local symbols */, null); } /** @@ -109,49 +98,32 @@ public SymbolTable newLocalSymtab(IonCatalog catalog, symbolsList, ionRep.getSymbolTable()); - LocalSymbolTableAsStruct table = new LocalSymbolTableAsStruct(imageFactory, - imports, - symbolsList); - table.myImage = ionRep; - - return table; + return new LocalSymbolTableAsStruct(imports, symbolsList, ionRep); } } - /** - * The factory used to build the {@link #myImage} of a local symtab. - * It's used by the datagram level to maintain the tree representation. - * It cannot be changed since local symtabs can't be moved between trees. - */ - private final ValueFactory myImageFactory; - - /** - * Memoized result of {@link #getIonRepresentation()}; - * Once this is created, we maintain it as symbols are added. - */ - private IonStruct myImage; + private final SymbolTableStructCache structCache; /** - * @param imageFactory never null * @param imports never null * @param symbolsList may be null or empty */ - private LocalSymbolTableAsStruct(ValueFactory imageFactory, - LocalSymbolTableImports imports, - List symbolsList) + private LocalSymbolTableAsStruct(LocalSymbolTableImports imports, + List symbolsList, + IonStruct image) { super(imports, symbolsList); - myImageFactory = imageFactory; + structCache = new SymbolTableStructCache(this, imports.getImportedTablesNoCopy(), image); } @Override int putSymbol(String symbolName) { int sid = super.putSymbol(symbolName); - if (myImage != null) + if (structCache.hasStruct()) { - recordLocalSymbolInIonRep(myImage, symbolName, sid); + structCache.addSymbol(symbolName, sid); } return sid; } @@ -170,102 +142,10 @@ int putSymbol(String symbolName) // then extra content doesn't matter. // - /** - * Only valid on local symtabs that already have an _image_factory set. - * - * @return Not null. - */ - IonStruct getIonRepresentation() - { - synchronized (this) - { - IonStruct image = myImage; - - if (image == null) - { - // Start a new image from scratch - myImage = image = makeIonRepresentation(myImageFactory); - } - - return image; - } - } - - /** - * NOT SYNCHRONIZED! Call only from a synch'd method. - * - * @return a new struct, not null. - */ - private IonStruct makeIonRepresentation(ValueFactory factory) - { - IonStruct ionRep = factory.newEmptyStruct(); - - ionRep.addTypeAnnotation(ION_SYMBOL_TABLE); - - SymbolTable[] importedTables = getImportedTablesNoCopy(); - - if (importedTables.length > 1) - { - IonList importsList = factory.newEmptyList(); - for (int i = 1; i < importedTables.length; i++) - { - SymbolTable importedTable = importedTables[i]; - IonStruct importStruct = factory.newEmptyStruct(); - - importStruct.add(NAME, - factory.newString(importedTable.getName())); - importStruct.add(VERSION, - factory.newInt(importedTable.getVersion())); - importStruct.add(MAX_ID, - factory.newInt(importedTable.getMaxId())); - - importsList.add(importStruct); - } - ionRep.add(IMPORTS, importsList); - } - - if (mySymbolsCount > 0) - { - int sid = myFirstLocalSid; - for (int offset = 0; offset < mySymbolsCount; offset++, sid++) - { - String symbolName = mySymbolNames[offset]; - recordLocalSymbolInIonRep(ionRep, symbolName, sid); - } - } - - return ionRep; - } - - /** - * NOT SYNCHRONIZED! Call within constructor or from synched method. - * @param symbolName can be null when there's a gap in the local symbols list. - */ - private void recordLocalSymbolInIonRep(IonStruct ionRep, - String symbolName, - int sid) + @Override + public IonStruct getIonRepresentation(ValueFactory factory) { - assert sid >= myFirstLocalSid; - - ValueFactory sys = ionRep.getSystem(); - - // TODO this is crazy inefficient and not as reliable as it looks - // since it doesn't handle the case where's theres more than one list - IonValue syms = ionRep.get(SYMBOLS); - while (syms != null && syms.getType() != IonType.LIST) - { - ionRep.remove(syms); - syms = ionRep.get(SYMBOLS); - } - if (syms == null) - { - syms = sys.newEmptyList(); - ionRep.put(SYMBOLS, syms); - } - - int this_offset = sid - myFirstLocalSid; - IonValue name = sys.newString(symbolName); - ((IonList)syms).add(this_offset, name); + return structCache.getIonRepresentation(factory); } } diff --git a/src/com/amazon/ion/impl/SymbolTableAsStruct.java b/src/com/amazon/ion/impl/SymbolTableAsStruct.java new file mode 100644 index 0000000000..6987a0e6b9 --- /dev/null +++ b/src/com/amazon/ion/impl/SymbolTableAsStruct.java @@ -0,0 +1,18 @@ +package com.amazon.ion.impl; + +import com.amazon.ion.IonStruct; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.ValueFactory; + +/** + * Identifies {@link SymbolTable} implementations capable of producing IonStruct representations of themselves. + */ +interface SymbolTableAsStruct { + + /** + * Provides an IonStruct representation of the SymbolTable. + * @param valueFactory the {@link ValueFactory} from which to construct the IonStruct. + * @return an IonStruct representing the SymbolTable. + */ + IonStruct getIonRepresentation(ValueFactory valueFactory); +} diff --git a/src/com/amazon/ion/impl/SymbolTableStructCache.java b/src/com/amazon/ion/impl/SymbolTableStructCache.java new file mode 100644 index 0000000000..df88e600ea --- /dev/null +++ b/src/com/amazon/ion/impl/SymbolTableStructCache.java @@ -0,0 +1,137 @@ +package com.amazon.ion.impl; + +import com.amazon.ion.IonList; +import com.amazon.ion.IonStruct; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.ValueFactory; + +import java.util.Iterator; + +import static com.amazon.ion.SystemSymbols.IMPORTS; +import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE; +import static com.amazon.ion.SystemSymbols.MAX_ID; +import static com.amazon.ion.SystemSymbols.NAME; +import static com.amazon.ion.SystemSymbols.SYMBOLS; +import static com.amazon.ion.SystemSymbols.VERSION; + +/** + * Caches the IonStruct representation of a {@link SymbolTable}. + */ +class SymbolTableStructCache { + + private final SymbolTable symbolTable; + private final SymbolTable[] importedTables; + private final int firstLocalSid; + + /** + * Memoized result of {@link #getIonRepresentation(ValueFactory)}; + * Once this is created, it may be mutated as symbols are added using {@link #addSymbol(String, int)}. + */ + private IonStruct image; + + /** + * @param symbolTable the SymbolTable to represent as an IonStruct. + * @param importedTables the symbol table's imported shared symbol tables. + * @param image the IonStruct representation, if available. If null, an IonStruct will be created lazily. + */ + SymbolTableStructCache(SymbolTable symbolTable, SymbolTable[] importedTables, IonStruct image) { + this.symbolTable = symbolTable; + this.importedTables = importedTables; + this.firstLocalSid = symbolTable.getImportedMaxId() + 1; + this.image = image; + } + + /** + * Returns the IonStruct representation of the symbol table. Creates and stores a new IonStruct on the first + * invocation. + * @param factory the {@link ValueFactory} from which to construct the IonStruct. + * @return an IonStruct representing the symbol table. + */ + public IonStruct getIonRepresentation(ValueFactory factory) { + synchronized (this) { + if (image == null) { + makeIonRepresentation(factory); + } + return image; + } + } + + /** + * @return true if an IonStruct has already been created for the symbol table; otherwise, false. + */ + public boolean hasStruct() { + return image != null; + } + + /** + * Create a new IonStruct representation of the symbol table. + * @param factory the {@link ValueFactory} from which to construct the IonStruct. + */ + private void makeIonRepresentation(ValueFactory factory) { + image = factory.newEmptyStruct(); + + image.addTypeAnnotation(ION_SYMBOL_TABLE); + + if (importedTables.length > 0) { + // The system symbol table may be the first import. If it is, skip it. + int i = importedTables[0].isSystemTable() ? 1 : 0; + if (i < importedTables.length) { + IonList importsList = factory.newEmptyList(); + while (i < importedTables.length) { + SymbolTable importedTable = importedTables[i]; + IonStruct importStruct = factory.newEmptyStruct(); + + importStruct.add(NAME, + factory.newString(importedTable.getName())); + importStruct.add(VERSION, + factory.newInt(importedTable.getVersion())); + importStruct.add(MAX_ID, + factory.newInt(importedTable.getMaxId())); + + importsList.add(importStruct); + i++; + } + image.add(IMPORTS, importsList); + } + } + + if (symbolTable.getMaxId() > symbolTable.getImportedMaxId()) { + Iterator localSymbolIterator = symbolTable.iterateDeclaredSymbolNames(); + int sid = symbolTable.getImportedMaxId() + 1; + while (localSymbolIterator.hasNext()) { + addSymbol(localSymbolIterator.next(), sid); + sid++; + } + } + } + + /** + * Adds a new symbol table with the given symbol ID to the IonStruct's 'symbols' list. + * @param symbolName can be null when there's a gap in the local symbols list. + * @param sid the symbol ID to assign to the new symbol. + */ + void addSymbol(String symbolName, int sid) { + assert sid >= firstLocalSid; + + ValueFactory sys = image.getSystem(); + + IonValue syms = image.get(SYMBOLS); + // If the user has manually created a symbol table via an IonStruct, it may be malformed. + // The following removes any "symbols" fields that aren't lists, then creates a list for + // the symbols field if necessary. + while (syms != null && syms.getType() != IonType.LIST) { + image.remove(syms); + syms = image.get(SYMBOLS); + } + if (syms == null) { + syms = sys.newEmptyList(); + image.put(SYMBOLS, syms); + } + + int thisOffset = sid - firstLocalSid; + IonValue name = sys.newString(symbolName); + ((IonList) syms).add(thisOffset, name); + } +} diff --git a/src/com/amazon/ion/impl/_Private_Utils.java b/src/com/amazon/ion/impl/_Private_Utils.java index e957377b32..09fff660d7 100644 --- a/src/com/amazon/ion/impl/_Private_Utils.java +++ b/src/com/amazon/ion/impl/_Private_Utils.java @@ -873,21 +873,23 @@ public static SymbolTable initialSymtab(_Private_LocalSymbolTableFactory lstFact */ public static IonStruct symtabTree(SymbolTable symtab, ValueFactory valueFactory) { - LocalSymbolTableAsStruct localSymbolTableAsStruct; - if (symtab instanceof LocalSymbolTableAsStruct) { - localSymbolTableAsStruct = (LocalSymbolTableAsStruct) symtab; + SymbolTableAsStruct localSymbolTableAsStruct; + if (symtab instanceof SymbolTableAsStruct) { + localSymbolTableAsStruct = (SymbolTableAsStruct) symtab; } else { - localSymbolTableAsStruct = (LocalSymbolTableAsStruct) new LocalSymbolTableAsStruct.Factory(valueFactory) + LocalSymbolTableAsStruct table = + (LocalSymbolTableAsStruct) new LocalSymbolTableAsStruct.Factory(valueFactory) .newLocalSymtab(symtab.getSystemSymbolTable(), symtab.getImportedTables()); Iterator localSymbolsIterator = symtab.iterateDeclaredSymbolNames(); while (localSymbolsIterator.hasNext()) { String localSymbol = localSymbolsIterator.next(); if (localSymbol != null) { - localSymbolTableAsStruct.intern(localSymbol); + table.intern(localSymbol); } } + localSymbolTableAsStruct = table; } - return localSymbolTableAsStruct.getIonRepresentation(); + return localSymbolTableAsStruct.getIonRepresentation(valueFactory); } /** From e1a24e862ec9fabd04afb183e8263b15dcb8002d Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Tue, 5 Oct 2021 16:51:38 -0700 Subject: [PATCH 139/490] Updates actions/setup-java to v2 in Github Actions main workflow --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4f4d7d7917..9d242e2d16 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,8 +18,9 @@ jobs: with: submodules: recursive - name: Use java ${{ matrix.java }} - uses: actions/setup-java@v1 + uses: actions/setup-java@v2 with: + distribution: 'zulu' java-version: ${{ matrix.java }} - run: mvn test - run: mvn package -f ion-java-cli/pom.xml From 2518f3107555f4f3e4c2b9a4274658257afea308 Mon Sep 17 00:00:00 2001 From: Gregg Date: Fri, 6 Aug 2021 16:28:09 -0700 Subject: [PATCH 140/490] Adds a pool of UTF-8 decoders, making reader instantiation less expensive. --- .../ion/impl/IonReaderBinaryIncremental.java | 33 ++----- .../amazon/ion/impl/IonReaderBinaryRawX.java | 55 +++-------- .../ion/impl/bin/IonRawBinaryWriter.java | 2 +- .../ion/impl/bin/utf8/ByteBufferPool.java | 26 +++++ src/com/amazon/ion/impl/bin/utf8/Pool.java | 64 ++++++++++++ .../amazon/ion/impl/bin/utf8/Poolable.java | 30 ++++++ .../ion/impl/bin/utf8/PoolableByteBuffer.java | 33 +++++++ .../ion/impl/bin/utf8/Utf8StringDecoder.java | 97 +++++++++++++++++++ .../impl/bin/utf8/Utf8StringDecoderPool.java | 26 +++++ .../ion/impl/bin/utf8/Utf8StringEncoder.java | 29 +----- .../impl/bin/utf8/Utf8StringEncoderPool.java | 51 ++-------- 11 files changed, 313 insertions(+), 133 deletions(-) create mode 100644 src/com/amazon/ion/impl/bin/utf8/ByteBufferPool.java create mode 100644 src/com/amazon/ion/impl/bin/utf8/Pool.java create mode 100644 src/com/amazon/ion/impl/bin/utf8/Poolable.java create mode 100644 src/com/amazon/ion/impl/bin/utf8/PoolableByteBuffer.java create mode 100644 src/com/amazon/ion/impl/bin/utf8/Utf8StringDecoder.java create mode 100644 src/com/amazon/ion/impl/bin/utf8/Utf8StringDecoderPool.java diff --git a/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java b/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java index 94c5d053d7..054a7f6902 100644 --- a/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java +++ b/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java @@ -16,6 +16,8 @@ import com.amazon.ion.UnknownSymbolException; import com.amazon.ion.ValueFactory; import com.amazon.ion.system.IonReaderBuilder; +import com.amazon.ion.impl.bin.utf8.Utf8StringDecoder; +import com.amazon.ion.impl.bin.utf8.Utf8StringDecoderPool; import com.amazon.ion.system.SimpleCatalog; import java.io.IOException; @@ -23,11 +25,8 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.Charset; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.CoderResult; import java.util.Arrays; + import java.util.ArrayList; import java.util.Collections; import java.util.Date; @@ -161,9 +160,6 @@ private static class SystemSymbolIDs { private static final int MAX_ID_ID = 8; } - // The size of the reusable UTF-8 decoding buffer. - private static final int UTF8_BUFFER_SIZE_IN_BYTES = 4 * 1024; - // The final byte of the binary IVM. private static final int IVM_FINAL_BYTE = 0xEA; @@ -233,8 +229,7 @@ private static class SystemSymbolIDs { // Stack to hold container info. Stepping into a container results in a push; stepping out results in a pop. private final _Private_RecyclingStack containerStack; - // UTF-8 string decoder. - private final CharsetDecoder utf8CharsetDecoder = Charset.forName("UTF-8").newDecoder(); + private final Utf8StringDecoder utf8Decoder = Utf8StringDecoderPool.getInstance().getOrCreate(); // The symbol IDs for the annotations on the current value. private final List annotationSids; @@ -283,9 +278,6 @@ private static class SystemSymbolIDs { // The number of bytes of a lob value that the user has consumed, allowing for piecewise reads. private int lobBytesRead = 0; - // A reusable scratch space to hold decoded UTF-8 bytes. - private CharBuffer utf8DecodingBuffer = CharBuffer.allocate(UTF8_BUFFER_SIZE_IN_BYTES); - // The type of value at which the reader is currently positioned. private IonType valueType = null; @@ -1599,22 +1591,8 @@ public double doubleValue() { */ private String readString(int valueStart, int valueEnd) { ByteBuffer utf8InputBuffer = buffer.getByteBuffer(valueStart, valueEnd); - int numberOfBytes = valueEnd - valueStart; - if (numberOfBytes > utf8DecodingBuffer.capacity()) { - utf8DecodingBuffer = CharBuffer.allocate(numberOfBytes); - } - - utf8DecodingBuffer.position(0); - utf8DecodingBuffer.limit(utf8DecodingBuffer.capacity()); - - utf8CharsetDecoder.reset(); - CoderResult coderResult = utf8CharsetDecoder.decode(utf8InputBuffer, utf8DecodingBuffer, true); - if (coderResult.isError()) { - throw new IonException("Illegal value encountered while validating UTF-8 data in input stream. " + coderResult.toString()); - } - utf8DecodingBuffer.flip(); - return utf8DecodingBuffer.toString(); + return utf8Decoder.decode(utf8InputBuffer, numberOfBytes); } @Override @@ -1974,6 +1952,7 @@ public void requireCompleteValue() { public void close() throws IOException { requireCompleteValue(); inputStream.close(); + utf8Decoder.close(); } } diff --git a/src/com/amazon/ion/impl/IonReaderBinaryRawX.java b/src/com/amazon/ion/impl/IonReaderBinaryRawX.java index 656000c3aa..8d23590b47 100644 --- a/src/com/amazon/ion/impl/IonReaderBinaryRawX.java +++ b/src/com/amazon/ion/impl/IonReaderBinaryRawX.java @@ -27,16 +27,16 @@ import com.amazon.ion.impl.UnifiedSavePointManagerX.SavePoint; import com.amazon.ion.impl._Private_ScalarConversions.AS_TYPE; import com.amazon.ion.impl._Private_ScalarConversions.ValueVariant; +import com.amazon.ion.impl.bin.utf8.ByteBufferPool; +import com.amazon.ion.impl.bin.utf8.PoolableByteBuffer; +import com.amazon.ion.impl.bin.utf8.Utf8StringDecoder; +import com.amazon.ion.impl.bin.utf8.Utf8StringDecoderPool; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.math.MathContext; import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.Charset; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.CoderResult; /** @@ -58,7 +58,6 @@ abstract class IonReaderBinaryRawX static final int DEFAULT_CONTAINER_STACK_SIZE = 12; // a multiple of 3 static final int DEFAULT_ANNOTATION_SIZE = 10; static final int NO_LIMIT = Integer.MIN_VALUE; - static final int UTF8_BUFFER_SIZE_IN_BYTES = 4 * 1024; protected enum State { S_INVALID, @@ -104,18 +103,13 @@ protected enum State { int _container_top; long[] _container_stack; // triples of: position, type, local_end - - // `StandardCharsets.UTF_8` wasn't introduced until Java 7, so we have to use Charset#forName(String) instead. - private static final Charset UTF8 = Charset.forName("UTF-8"); - private CharsetDecoder utf8CharsetDecoder = UTF8.newDecoder(); + // Pooled decoder for UTF-8 strings. + private final Utf8StringDecoder utf8Decoder = Utf8StringDecoderPool.getInstance().getOrCreate(); // Calling read() to pull in the next byte of a string requires an EOF check to be performed for each byte. // This reusable buffer allows us to call read(utf8InputBuffer) instead, letting us can pay the cost of an EOF check // once per buffer rather than once per byte. - private ByteBuffer utf8InputBuffer = ByteBuffer.allocate(UTF8_BUFFER_SIZE_IN_BYTES); - - // A reusable scratch space to hold the decoded bytes as they're read from the utf8InputBuffer. - private CharBuffer utf8DecodingBuffer = CharBuffer.allocate(UTF8_BUFFER_SIZE_IN_BYTES); + private final PoolableByteBuffer pooledUtf8InputBuffer = ByteBufferPool.getInstance().getOrCreate(); protected IonReaderBinaryRawX() { } @@ -169,6 +163,8 @@ public void close() throws IOException { _input.close(); + utf8Decoder.close(); + pooledUtf8InputBuffer.close(); } static private final int POS_OFFSET = 0; @@ -1200,16 +1196,14 @@ protected final Timestamp readTimestamp(int len) throws IOException protected final String readString(int numberOfBytes) throws IOException { + ByteBuffer utf8InputBuffer = pooledUtf8InputBuffer.getBuffer(); // If the string we're reading is small enough to fit in our reusable buffer, we can avoid the overhead // of looping and bounds checking. if (numberOfBytes <= utf8InputBuffer.capacity()) { - return readStringWithReusableBuffer(numberOfBytes); + return readStringWithReusableBuffer(numberOfBytes, utf8InputBuffer); } - // Otherwise, allocate a one-off decoding buffer that's large enough to hold the string - // and prepare to decode the string in chunks. - CharBuffer decodingBuffer = CharBuffer.allocate(numberOfBytes); - utf8CharsetDecoder.reset(); + utf8Decoder.prepareDecode(numberOfBytes); int save_limit = NO_LIMIT; if (_local_remaining != NO_LIMIT) { @@ -1246,14 +1240,7 @@ protected final String readString(int numberOfBytes) throws IOException utf8InputBuffer.position(0); utf8InputBuffer.limit(carryoverBytes + bytesRead); - CoderResult coderResult = utf8CharsetDecoder.decode( - utf8InputBuffer, - decodingBuffer, - totalBytesRead >= numberOfBytes - ); - if (coderResult.isError()) { - throw new IonException("Illegal value encountered while validating UTF-8 data in input stream. " + coderResult.toString()); - } + utf8Decoder.partialDecode(utf8InputBuffer, totalBytesRead >= numberOfBytes); // Shift leftover partial character bytes (if any) to the beginning of the buffer carryoverBytes = utf8InputBuffer.remaining(); @@ -1270,11 +1257,10 @@ protected final String readString(int numberOfBytes) throws IOException _local_remaining = save_limit; - decodingBuffer.flip(); - return decodingBuffer.toString(); + return utf8Decoder.finishDecode(); } - private String readStringWithReusableBuffer(int numberOfBytes) throws IOException { + private String readStringWithReusableBuffer(int numberOfBytes, ByteBuffer utf8InputBuffer) throws IOException { int save_limit = NO_LIMIT; if (_local_remaining != NO_LIMIT) { save_limit = _local_remaining - numberOfBytes; @@ -1286,16 +1272,7 @@ private String readStringWithReusableBuffer(int numberOfBytes) throws IOExceptio utf8InputBuffer.position(0); utf8InputBuffer.limit(numberOfBytes); - utf8DecodingBuffer.position(0); - utf8DecodingBuffer.limit(utf8DecodingBuffer.capacity()); - - utf8CharsetDecoder.reset(); - CoderResult coderResult = utf8CharsetDecoder.decode(utf8InputBuffer, utf8DecodingBuffer, true); - if (coderResult.isError()) { - throw new IonException("Illegal value encountered while validating UTF-8 data in input stream. " + coderResult.toString()); - } - utf8DecodingBuffer.flip(); - return utf8DecodingBuffer.toString(); + return utf8Decoder.decode(utf8InputBuffer, numberOfBytes); } private final void throwUnexpectedEOFException() throws IOException { diff --git a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java index 7d2e5606d6..bc7d63b5ee 100644 --- a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java +++ b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java @@ -126,7 +126,7 @@ private static byte[] bytes(int... vals) { final Utf8StringEncoder utf8StringEncoder = Utf8StringEncoderPool .getInstance() - .getOrCreateUtf8Encoder(); + .getOrCreate(); private static final byte[] makeTypedPreallocatedBytes(final int typeDesc, final int length) { diff --git a/src/com/amazon/ion/impl/bin/utf8/ByteBufferPool.java b/src/com/amazon/ion/impl/bin/utf8/ByteBufferPool.java new file mode 100644 index 0000000000..fe6a7608a2 --- /dev/null +++ b/src/com/amazon/ion/impl/bin/utf8/ByteBufferPool.java @@ -0,0 +1,26 @@ +package com.amazon.ion.impl.bin.utf8; + +/** + * A thread-safe shared pool of {@link PoolableByteBuffer}s. + */ +public class ByteBufferPool extends Pool { + + private static final ByteBufferPool INSTANCE = new ByteBufferPool(); + + // Do not allow instantiation; all classes should share the singleton instance. + private ByteBufferPool() { + super(new Allocator() { + @Override + public PoolableByteBuffer newInstance(Pool pool) { + return new PoolableByteBuffer(pool); + } + }); + } + + /** + * @return a threadsafe shared instance of {@link ByteBufferPool}. + */ + public static ByteBufferPool getInstance() { + return INSTANCE; + } +} diff --git a/src/com/amazon/ion/impl/bin/utf8/Pool.java b/src/com/amazon/ion/impl/bin/utf8/Pool.java new file mode 100644 index 0000000000..647055bcc4 --- /dev/null +++ b/src/com/amazon/ion/impl/bin/utf8/Pool.java @@ -0,0 +1,64 @@ +package com.amazon.ion.impl.bin.utf8; + +import java.util.concurrent.ArrayBlockingQueue; + +abstract class Pool> { + + /** + * Allocates objects to be pooled. + * @param the type of object. + */ + interface Allocator> { + + /** + * Allocate a new object and link it to the given pool. + * @param pool the pool to which the new object will be linked. + * @return a new instance. + */ + T newInstance(Pool pool); + } + + // The maximum number of objects that can be waiting in the queue before new ones will be discarded. + private static final int MAX_QUEUE_SIZE = 128; + + // A queue of previously initialized objects that can be loaned out. + private final ArrayBlockingQueue bufferQueue; + + // Allocator of objects to be pooled. + private final Allocator allocator; + + Pool(Allocator allocator) { + this.allocator = allocator; + bufferQueue = new ArrayBlockingQueue(MAX_QUEUE_SIZE); + } + + /** + * If the pool is not empty, removes an object from the pool and returns it; + * otherwise, constructs a new object. + * + * @return An object. + */ + public T getOrCreate() { + // The `poll` method does not block. If the queue is empty it returns `null` immediately. + T object = bufferQueue.poll(); + if (object == null) { + // No buffers were available in the pool. Create a new one. + object = allocator.newInstance(this); + } + return object; + } + + /** + * Adds the provided instance to the pool. If the pool is full, the instance will + * be discarded. + * + * Callers MUST NOT use an object after returning it to the pool. + * + * @param object An object to add to the pool. + */ + public void returnToPool(T object) { + // The `offer` method does not block. If the queue is full, it returns `false` immediately. + // If the provided instance cannot be added to the pool, we discard it silently. + bufferQueue.offer(object); + } +} diff --git a/src/com/amazon/ion/impl/bin/utf8/Poolable.java b/src/com/amazon/ion/impl/bin/utf8/Poolable.java new file mode 100644 index 0000000000..52a86ca367 --- /dev/null +++ b/src/com/amazon/ion/impl/bin/utf8/Poolable.java @@ -0,0 +1,30 @@ +package com.amazon.ion.impl.bin.utf8; + +import java.io.Closeable; + +/** + * Base class for types that may be pooled. + * @param the concrete type. + */ +abstract class Poolable> implements Closeable { + + // The pool to which this object is linked. + private final Pool pool; + + /** + * @param pool the pool to which the object will be returned upon {@link #close()}. + */ + Poolable(Pool pool) { + this.pool = pool; + } + + /** + * Attempts to return this instance to the pool with which it is associated, if any. + * + * Do not continue to use this instance after calling this method. + */ + @Override + public void close() { + pool.returnToPool((T) this); + } +} diff --git a/src/com/amazon/ion/impl/bin/utf8/PoolableByteBuffer.java b/src/com/amazon/ion/impl/bin/utf8/PoolableByteBuffer.java new file mode 100644 index 0000000000..df43d079da --- /dev/null +++ b/src/com/amazon/ion/impl/bin/utf8/PoolableByteBuffer.java @@ -0,0 +1,33 @@ +package com.amazon.ion.impl.bin.utf8; + +import java.nio.ByteBuffer; + +/** + * Holds a reusable {@link ByteBuffer}. Instances of this class are reusable but are NOT threadsafe. + * + * Instances are vended by {@link ByteBufferPool#getOrCreate()}. + * + * Users are expected to call {@link #close()} when the decoder is no longer needed. + */ +public class PoolableByteBuffer extends Poolable { + + static final int BUFFER_SIZE_IN_BYTES = 4 * 1024; + + // The reusable buffer. + private final ByteBuffer buffer; + + /** + * @param pool the pool to which the object will be returned upon {@link #close()}. + */ + PoolableByteBuffer(Pool pool) { + super(pool); + buffer = ByteBuffer.allocate(BUFFER_SIZE_IN_BYTES); + } + + /** + * @return the buffer. + */ + public ByteBuffer getBuffer() { + return buffer; + } +} diff --git a/src/com/amazon/ion/impl/bin/utf8/Utf8StringDecoder.java b/src/com/amazon/ion/impl/bin/utf8/Utf8StringDecoder.java new file mode 100644 index 0000000000..6302d77194 --- /dev/null +++ b/src/com/amazon/ion/impl/bin/utf8/Utf8StringDecoder.java @@ -0,0 +1,97 @@ +package com.amazon.ion.impl.bin.utf8; + +import com.amazon.ion.IonException; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CoderResult; + +/** + * Decodes {@link String}s from UTF-8. Instances of this class are reusable but are NOT threadsafe. + * + * Instances are vended by {@link Utf8StringDecoderPool#getOrCreate()}. + * + * Users are expected to call {@link #close()} when the decoder is no longer needed. + * + * There are two ways of using this class: + *
      + *
    1. Use {@link #decode(ByteBuffer, int)} to decode the requested number of bytes from the given ByteBuffer in + * a single step. Or,
    2. + *
    3. Use the following sequence of method calls: + *
        + *
      1. {@link #prepareDecode(int)} to prepare the decoder to decode the requested number of bytes.
      2. + *
      3. {@link #partialDecode(ByteBuffer, boolean)} to decode the available bytes from the byte buffer. This may + * be repeated as more bytes are made available in the ByteBuffer, which is the caller's responsibility.
      4. + *
      5. {@link #finishDecode()} to finish decoding and return the resulting String.
      6. + *
      + * Note: {@link #decode(ByteBuffer, int)} must not be called between calls to {@link #prepareDecode(int)} and + * {@link #finishDecode()}. + *
    4. + *
    + */ +public class Utf8StringDecoder extends Poolable { + + // The size of the UTF-8 decoding buffer. + private static final int UTF8_BUFFER_SIZE_IN_BYTES = 4 * 1024; + + private final CharBuffer reusableUtf8DecodingBuffer; + private final CharsetDecoder utf8CharsetDecoder; + private CharBuffer utf8DecodingBuffer; + + Utf8StringDecoder(Pool pool) { + super(pool); + reusableUtf8DecodingBuffer = CharBuffer.allocate(UTF8_BUFFER_SIZE_IN_BYTES); + utf8CharsetDecoder = Charset.forName("UTF-8").newDecoder(); + } + + /** + * Prepares the decoder to decode the given number of UTF-8 bytes. + * @param numberOfBytes the number of bytes to decode. + */ + public void prepareDecode(int numberOfBytes) { + utf8CharsetDecoder.reset(); + utf8DecodingBuffer = reusableUtf8DecodingBuffer; + if (numberOfBytes > reusableUtf8DecodingBuffer.capacity()) { + utf8DecodingBuffer = CharBuffer.allocate(numberOfBytes); + } + } + + /** + * Decodes the available bytes from the given ByteBuffer. + * @param utf8InputBuffer a ByteBuffer containing UTF-8 bytes. + * @param endOfInput true if the end of the UTF-8 sequence is expected to occur in the buffer; otherwise, false. + */ + public void partialDecode(ByteBuffer utf8InputBuffer, boolean endOfInput) { + CoderResult coderResult = utf8CharsetDecoder.decode(utf8InputBuffer, utf8DecodingBuffer, endOfInput); + if (coderResult.isError()) { + throw new IonException("Illegal value encountered while validating UTF-8 data in input stream. " + coderResult.toString()); + } + } + + /** + * Finishes decoding and returns the resulting String. + * @return the decoded Java String. + */ + public String finishDecode() { + utf8DecodingBuffer.flip(); + return utf8DecodingBuffer.toString(); + } + + /** + * Decodes the given number of UTF-8 bytes from the given ByteBuffer into a Java String. + * @param utf8InputBuffer a ByteBuffer containing UTF-8 bytes. + * @param numberOfBytes the number of bytes from the utf8InputBuffer to decode. + * @return the decoded Java String. + */ + public String decode(ByteBuffer utf8InputBuffer, int numberOfBytes) { + prepareDecode(numberOfBytes); + + utf8DecodingBuffer.position(0); + utf8DecodingBuffer.limit(utf8DecodingBuffer.capacity()); + + partialDecode(utf8InputBuffer, true); + return finishDecode(); + } +} diff --git a/src/com/amazon/ion/impl/bin/utf8/Utf8StringDecoderPool.java b/src/com/amazon/ion/impl/bin/utf8/Utf8StringDecoderPool.java new file mode 100644 index 0000000000..97a801bb63 --- /dev/null +++ b/src/com/amazon/ion/impl/bin/utf8/Utf8StringDecoderPool.java @@ -0,0 +1,26 @@ +package com.amazon.ion.impl.bin.utf8; + +/** + * A thread-safe shared pool of {@link Utf8StringDecoder}s that can be used for UTF8 decoding. + */ +public class Utf8StringDecoderPool extends Pool { + + private static final Utf8StringDecoderPool INSTANCE = new Utf8StringDecoderPool(); + + // Do not allow instantiation; all classes should share the singleton instance. + private Utf8StringDecoderPool() { + super(new Allocator() { + @Override + public Utf8StringDecoder newInstance(Pool pool) { + return new Utf8StringDecoder(pool); + } + }); + } + + /** + * @return a threadsafe shared instance of {@link Utf8StringDecoderPool}. + */ + public static Utf8StringDecoderPool getInstance() { + return INSTANCE; + } +} diff --git a/src/com/amazon/ion/impl/bin/utf8/Utf8StringEncoder.java b/src/com/amazon/ion/impl/bin/utf8/Utf8StringEncoder.java index d890e1beaf..8a4f780c1b 100644 --- a/src/com/amazon/ion/impl/bin/utf8/Utf8StringEncoder.java +++ b/src/com/amazon/ion/impl/bin/utf8/Utf8StringEncoder.java @@ -1,7 +1,5 @@ package com.amazon.ion.impl.bin.utf8; -import java.io.Closeable; -import java.io.IOException; import java.nio.ByteBuffer; import java.nio.CharBuffer; import java.nio.charset.Charset; @@ -11,35 +9,30 @@ /** * Encodes {@link String}s to UTF-8. Instances of this class are reusable but are NOT threadsafe. * - * Users are strongly encouraged to get instances from {@link Utf8StringEncoderPool#getOrCreateUtf8Encoder()}. + * Instances are vended by {@link Utf8StringEncoderPool#getOrCreate()}. + * * {@link #encode(String)} can be called any number of times. Users are expected to call {@link #close()} when * the encoder is no longer needed. */ -public class Utf8StringEncoder implements Closeable { +public class Utf8StringEncoder extends Poolable { // The longest String (as measured by {@link java.lang.String#length()}) that this instance can encode without // requiring additional allocations. private static final int SMALL_STRING_SIZE = 4 * 1024; // Reusable resources for encoding Strings as UTF-8 bytes - final Utf8StringEncoderPool utf8StringEncoderPool; final CharsetEncoder utf8Encoder; final ByteBuffer utf8EncodingBuffer; final char[] charArray; final CharBuffer charBuffer; - public Utf8StringEncoder(Utf8StringEncoderPool pool) { - utf8StringEncoderPool = pool; + Utf8StringEncoder(Pool pool) { + super(pool); utf8Encoder = Charset.forName("UTF-8").newEncoder(); utf8EncodingBuffer = ByteBuffer.allocate((int) (SMALL_STRING_SIZE * utf8Encoder.maxBytesPerChar())); charArray = new char[SMALL_STRING_SIZE]; charBuffer = CharBuffer.wrap(charArray); } - public Utf8StringEncoder() { - // This instance is not associated with a Utf8StringEncoderPool - this(null); - } - /** * Encodes the provided String's text to UTF-8. Unlike {@link String#getBytes(Charset)}, this method will not * silently replace characters that cannot be encoded with a substitute character. Instead, it will throw @@ -125,18 +118,6 @@ enough for us to reuse our buffers ("small strings"), and those which are not (" return new Result(utf8Length, encodingBuffer.array()); } - /** - * Attempts to return this instance to the Utf8StringEncoderPool with which it is associated, if any. - * - * Do not continue to use this encoder after calling this method. - */ - @Override - public void close() { - if (utf8StringEncoderPool != null) { - utf8StringEncoderPool.returnEncoderToPool(this); - } - } - /** * Represents the result of a {@link Utf8StringEncoder#encode(String)} operation. */ diff --git a/src/com/amazon/ion/impl/bin/utf8/Utf8StringEncoderPool.java b/src/com/amazon/ion/impl/bin/utf8/Utf8StringEncoderPool.java index e926df4a75..fda772a198 100644 --- a/src/com/amazon/ion/impl/bin/utf8/Utf8StringEncoderPool.java +++ b/src/com/amazon/ion/impl/bin/utf8/Utf8StringEncoderPool.java @@ -1,23 +1,20 @@ package com.amazon.ion.impl.bin.utf8; -import java.util.concurrent.ArrayBlockingQueue; - /** - * A thread-safe shared pool of {@link Utf8StringEncoder}s that can be used for UTF8 encoding and decoding. + * A thread-safe shared pool of {@link Utf8StringEncoder}s that can be used for UTF8 encoding. */ -public enum Utf8StringEncoderPool { - // The only enum variant; a singleton instance. - INSTANCE; - - // The maximum number of Utf8Encoders that can be waiting in the queue before new ones will be discarded. - private static final int MAX_QUEUE_SIZE = 128; +public class Utf8StringEncoderPool extends Pool { - // A queue of previously initialized encoders that can be loaned out. - private final ArrayBlockingQueue bufferQueue; + private static final Utf8StringEncoderPool INSTANCE = new Utf8StringEncoderPool(); // Do not allow instantiation; all classes should share the singleton instance. private Utf8StringEncoderPool() { - bufferQueue = new ArrayBlockingQueue(MAX_QUEUE_SIZE); + super(new Allocator() { + @Override + public Utf8StringEncoder newInstance(Pool pool) { + return new Utf8StringEncoder(pool); + } + }); } /** @@ -27,34 +24,4 @@ public static Utf8StringEncoderPool getInstance() { return INSTANCE; } - /** - * If the pool is not empty, removes an instance of {@link Utf8StringEncoder} from the pool and returns it; - * otherwise, constructs a new instance. - * - * @return An instance of {@link Utf8StringEncoder}. - */ - public Utf8StringEncoder getOrCreateUtf8Encoder() { - // The `poll` method does not block. If the queue is empty it returns `null` immediately. - Utf8StringEncoder encoder = bufferQueue.poll(); - if (encoder == null) { - // No buffers were available in the pool. Create a new one. - encoder = new Utf8StringEncoder(this); - } - return encoder; - } - - /** - * Adds the provided instance of {@link Utf8StringEncoder} to the pool. If the pool is full, the instance will - * be discarded. - * - * Callers MUST NOT use an encoder after returning it to the pool. - * - * @param encoder A {@link Utf8StringEncoder} to add to the pool. - */ - public void returnEncoderToPool(Utf8StringEncoder encoder) { - // The `offer` method does not block. If the queue is full, it returns `false` immediately. - // If the provided instance cannot be added to the pool, we discard it silently. - bufferQueue.offer(encoder); - } - } From 82478a85ad7a378e59467e8b4a12b5062363760e Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Mon, 11 Oct 2021 12:57:24 -0700 Subject: [PATCH 141/490] Eliminates the use of ConcurrentLinkedQueue.size() is PooledBlockAllocator, improving performance when the queue gets large. --- .../bin/PooledBlockAllocatorProvider.java | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/com/amazon/ion/impl/bin/PooledBlockAllocatorProvider.java b/src/com/amazon/ion/impl/bin/PooledBlockAllocatorProvider.java index b3d58e2c5f..3918e00b55 100644 --- a/src/com/amazon/ion/impl/bin/PooledBlockAllocatorProvider.java +++ b/src/com/amazon/ion/impl/bin/PooledBlockAllocatorProvider.java @@ -18,6 +18,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; /** * A singleton implementation of {@link BlockAllocatorProvider} offering a thread-safe free block list @@ -37,6 +38,7 @@ private static final class PooledBlockAllocator extends BlockAllocator { private final int blockSize, blockLimit; private final ConcurrentLinkedQueue freeBlocks; + private final AtomicInteger size = new AtomicInteger(0); static final int FREE_CAPACITY = 1024 * 1024 * 64; // 64MB public PooledBlockAllocator(final int blockSize) @@ -57,13 +59,31 @@ public Block allocateBlock() @Override public void close() { - if (freeBlocks.size() < blockLimit) { + // In the common case, the pool is not full. Optimistically increment the size. + if (size.getAndIncrement() < blockLimit) + { reset(); freeBlocks.add(this); } + else + { + // The pool was full. Since the size was optimistically incremented, decrement it now. + // Note: there is a race condition here that is deliberately allowed as an optimization. + // Under high contention, multiple threads could end up here before the first one + // decrements the size, causing blocks to be dropped wastefully. This is not harmful + // because blocks will be re-allocated when necessary; the pool is kept as close as + // possible to capacity on a best-effort basis. This race condition should not be "fixed" + // without a thorough study of the performance implications. + size.decrementAndGet(); + } } }; } + else + { + // A block was retrieved from the pool; decrement the pool size. + size.decrementAndGet(); + } return block; } From 2d1eddc969fb38348e57a9262729a68964830cdb Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Wed, 13 Oct 2021 16:22:53 -0700 Subject: [PATCH 142/490] Prepares version 1.9.0 for release. --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dad1504b2d..c5a1c7a6b9 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ dependency into your project's `pom.xml`: com.amazon.ion ion-java - 1.8.3 + 1.9.0 ``` diff --git a/pom.xml b/pom.xml index eb601e57b4..3ca0f05e5c 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.8.4-SNAPSHOT + 1.9.0 bundle ${project.groupId}:${project.artifactId} From 44aabbc8a47afe12b7f03ef5e3c91a62674ff096 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Wed, 13 Oct 2021 16:55:17 -0700 Subject: [PATCH 143/490] Bumps version to 1.9.1-SNAPSHOT. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3ca0f05e5c..978df8b635 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.9.0 + 1.9.1-SNAPSHOT bundle ${project.groupId}:${project.artifactId} From 8870d73fac8881677a4fbd4ab58775337b32772b Mon Sep 17 00:00:00 2001 From: Eric Chen <67451029+cheqianh@users.noreply.github.com> Date: Fri, 19 Nov 2021 14:56:45 -0800 Subject: [PATCH 144/490] Adds README for ion-java-cli (#394) * Adds README for ion-java-cli This PR set up description of Ion-java-cli including setup instruction and a link to its design doc. * Adds a Getting Started section. --- ion-java-cli/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 ion-java-cli/README.md diff --git a/ion-java-cli/README.md b/ion-java-cli/README.md new file mode 100644 index 0000000000..6bc6d9b84e --- /dev/null +++ b/ion-java-cli/README.md @@ -0,0 +1,18 @@ +# Amazon Ion Java CLI +A Java implementation of CLI where its design document is located in [here](https://github.com/amzn/ion-test-driver#design). + +The package is stored under `ion-java/ion-java-cli`. + +## Setup +Build ion-java-cli. Note that using -f option for ion-java-cli's `pom.xml`. +``` +$ mvn -f ion-java-cli/pom.xml install +``` + +## Getting Started +Invoking `ion-java-cli-x.y.jar` under `ion-java-cli/target/` directory.
    + +For example: +``` +java -jar ion-java-cli/target/ion-java-cli-1.0.jar process test_file.ion -f pretty -o output.ion +``` From ac594aca46bccad84b736348efb5bd8c780deded Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Mon, 22 Nov 2021 15:36:33 -0800 Subject: [PATCH 145/490] Fixes performance regression detector so that it refreshes the git submodules when checking out the prior commit --- .github/workflows/ion-java-performance-regression-detector.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ion-java-performance-regression-detector.yml b/.github/workflows/ion-java-performance-regression-detector.yml index 6bf0cd43c5..abf209448c 100644 --- a/.github/workflows/ion-java-performance-regression-detector.yml +++ b/.github/workflows/ion-java-performance-regression-detector.yml @@ -68,7 +68,7 @@ jobs: run : rm -r /home/runner/.m2 - name: Build ion-java from the previous commit - run: cd ion-java && git checkout HEAD^ && mvn clean install + run: cd ion-java && git checkout HEAD^ && git submodule update --recursive && mvn clean install - name: Build ion-java-benchmark-cli run: cd ion-java-benchmark-cli && mvn clean install From 676455a0309ac3141bdd68d37bb435dbcfdd16ad Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Tue, 23 Nov 2021 14:16:16 -0800 Subject: [PATCH 146/490] Fixes Ion Binary Reader to fail fast when an IVM type code is encountered in an illegal position --- ion-tests | 2 +- src/com/amazon/ion/impl/IonReaderBinaryRawX.java | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ion-tests b/ion-tests index c125f571d2..ef0451a72a 160000 --- a/ion-tests +++ b/ion-tests @@ -1 +1 @@ -Subproject commit c125f571d2b459e0e610eac5ca11900bd3b6d493 +Subproject commit ef0451a72a39f572175ad8e90b1a77110e9aec4c diff --git a/src/com/amazon/ion/impl/IonReaderBinaryRawX.java b/src/com/amazon/ion/impl/IonReaderBinaryRawX.java index 8d23590b47..3c3494c678 100644 --- a/src/com/amazon/ion/impl/IonReaderBinaryRawX.java +++ b/src/com/amazon/ion/impl/IonReaderBinaryRawX.java @@ -283,6 +283,12 @@ private final void has_next_helper_raw() throws IOException else if (_value_tid == _Private_IonConstants.tidTypedecl) { assert (_value_tid == (BINARY_VERSION_MARKER_TID & 0xff)); // the bvm tid happens to be type decl if (_value_len == BINARY_VERSION_MARKER_LEN ) { + if (getDepth() != 0) { + // In Ion text, we can interpret an IVM in the wrong position as an ordinary Symbol, + // but in Ion binary, the BVM is unambiguously an IVM rather than a Symbol, and it + // is not allowed in any container type. + throw newErrorAt("Encountered IVM type code E0 below the top level"); + } // this isn't valid for any type descriptor except the first byte // of a 4 byte version marker - so lets read the rest load_version_marker(); From ba4cda5c2ce49712130daf831f9884d212e07c8b Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Wed, 19 Jan 2022 14:57:58 -0800 Subject: [PATCH 147/490] Adds members to the IonNumber interface and makes IonDecimal, IonFloat, and IonInt implement IonNumber --- src/com/amazon/ion/IonFloat.java | 5 ++- src/com/amazon/ion/IonInt.java | 12 +++++- src/com/amazon/ion/IonNumber.java | 39 ++++++++++++++++--- .../amazon/ion/impl/lite/IonDecimalLite.java | 6 +++ .../amazon/ion/impl/lite/IonFloatLite.java | 2 + src/com/amazon/ion/impl/lite/IonIntLite.java | 16 ++++++++ test/com/amazon/ion/DecimalTest.java | 10 +++++ test/com/amazon/ion/IntTest.java | 26 +++++++++++++ 8 files changed, 109 insertions(+), 7 deletions(-) diff --git a/src/com/amazon/ion/IonFloat.java b/src/com/amazon/ion/IonFloat.java index e4fa2be69d..e6890da3e0 100644 --- a/src/com/amazon/ion/IonFloat.java +++ b/src/com/amazon/ion/IonFloat.java @@ -37,7 +37,7 @@ * @see IonDecimal */ public interface IonFloat - extends IonValue + extends IonNumber { /** * Gets the value of this Ion float as a Java @@ -66,6 +66,8 @@ public double doubleValue() * recommended to call {@link IonFloat#isNumericValue()} before calling * this method. * + * If you need negative zeros, use {@link #doubleValue()}. + * * @return the {@link BigDecimal} value, or {@code null} if * {@code this.isNullValue()}. * @@ -73,6 +75,7 @@ public double doubleValue() * {@code +inf}, or {@code -inf}, because {@link BigDecimal} cannot * represent those values. */ + @Override public BigDecimal bigDecimalValue() throws NullValueException; diff --git a/src/com/amazon/ion/IonInt.java b/src/com/amazon/ion/IonInt.java index 2270b0a918..bf8d44a3dd 100644 --- a/src/com/amazon/ion/IonInt.java +++ b/src/com/amazon/ion/IonInt.java @@ -15,6 +15,7 @@ package com.amazon.ion; +import java.math.BigDecimal; import java.math.BigInteger; /** @@ -24,7 +25,7 @@ * code outside of this library. */ public interface IonInt - extends IonValue + extends IonNumber { /** * Gets the content of this Ion int as a Java @@ -56,6 +57,15 @@ public long longValue() */ public BigInteger bigIntegerValue(); + /** + * Gets the value of this Ion {@code int} as a {@link BigDecimal}. + * The scale of the {@code BigDecimal} is zero. + * + * @return the {@code BigDecimal} value, + * or {@code null} if {@code this.isNullValue()}. + */ + public BigDecimal bigDecimalValue(); + /** * Gets an {@link IntegerSize} representing the smallest-possible * Java type of the underlying content, or {@code null} if this is diff --git a/src/com/amazon/ion/IonNumber.java b/src/com/amazon/ion/IonNumber.java index dd17c55975..88002c6842 100644 --- a/src/com/amazon/ion/IonNumber.java +++ b/src/com/amazon/ion/IonNumber.java @@ -15,16 +15,45 @@ package com.amazon.ion; +import java.math.BigDecimal; + /** - * The IonNumber interface is a fore runner of a common base for the - * ion numeric value types. Currently only IonDecimal extends this - * interface. In due course IonFloat and IonInt will be added to the - * the family. + * Common functionality of Ion {@code int}, {@code decimal}, and {@code float} + * types. *

    * WARNING: This interface should not be implemented or extended by * code outside of this library. */ -public interface IonNumber // TODO amzn/ion-java/issues/53 Complete this interface +public interface IonNumber extends IonValue { + /** + * Gets the value of this Ion number as a {@link BigDecimal}. + * + *

    This method will throw an exception for non-null, non-numeric values. + * It's recommended to call {@link #isNumericValue()} before calling + * this method.

    + * + *

    Negative zero is supported by {@link IonDecimal} and {@link IonFloat}, + * but not by {@link BigDecimal}. If you need to distinguish positive and + * negative zero, you should call {@link IonDecimal#decimalValue()} or + * {@link IonFloat#doubleValue()} after casting to the appropriate type.

    + * + * @return the {@link BigDecimal} value, or {@code null} if + * {@code this.isNullValue()}. + * + * @throws NumberFormatException if this value is {@code nan}, + * {@code +inf}, or {@code -inf}, because {@link BigDecimal} cannot + * represent those values. + */ + public BigDecimal bigDecimalValue(); + + /** + * Determines whether this value is numeric. Returns true if this value + * is none of {@code null}, {@code nan}, {@code +inf}, and {@code -inf}, + * and false if it is any of them. + * + * @return a checked condition whether this value is numeric. + */ + public boolean isNumericValue(); } diff --git a/src/com/amazon/ion/impl/lite/IonDecimalLite.java b/src/com/amazon/ion/impl/lite/IonDecimalLite.java index 4a8d71e7a1..63ecea4fdd 100644 --- a/src/com/amazon/ion/impl/lite/IonDecimalLite.java +++ b/src/com/amazon/ion/impl/lite/IonDecimalLite.java @@ -124,6 +124,7 @@ public double doubleValue() return d; } + @Override public BigDecimal bigDecimalValue() throws NullValueException { @@ -136,6 +137,11 @@ public Decimal decimalValue() return Decimal.valueOf(_decimal_value); // Works for null. } + @Override + public boolean isNumericValue() { + return !isNullValue(); + } + public void setValue(long value) { // base setValue will check for the lock diff --git a/src/com/amazon/ion/impl/lite/IonFloatLite.java b/src/com/amazon/ion/impl/lite/IonFloatLite.java index 66118c291d..551ceb6d84 100644 --- a/src/com/amazon/ion/impl/lite/IonFloatLite.java +++ b/src/com/amazon/ion/impl/lite/IonFloatLite.java @@ -95,6 +95,7 @@ public double doubleValue() return _float_value.doubleValue(); } + @Override public BigDecimal bigDecimalValue() throws NullValueException { @@ -151,6 +152,7 @@ final void writeBodyTo(IonWriter writer, SymbolTableProvider symbolTableProvider } } + @Override public boolean isNumericValue() { return !(isNullValue() || _float_value.isNaN() || _float_value.isInfinite()); diff --git a/src/com/amazon/ion/impl/lite/IonIntLite.java b/src/com/amazon/ion/impl/lite/IonIntLite.java index 3fce4810aa..89747dd600 100644 --- a/src/com/amazon/ion/impl/lite/IonIntLite.java +++ b/src/com/amazon/ion/impl/lite/IonIntLite.java @@ -146,6 +146,22 @@ public BigInteger bigIntegerValue() return _big_int_value; } + @Override + public BigDecimal bigDecimalValue() { + if (isNullValue()) { + return null; + } else if (_big_int_value == null) { + return BigDecimal.valueOf(_long_value); + } else { + return new BigDecimal(_big_int_value); + } + } + + @Override + public boolean isNumericValue() { + return !isNullValue(); + } + public void setValue(int value) { setValue((long)value); diff --git a/test/com/amazon/ion/DecimalTest.java b/test/com/amazon/ion/DecimalTest.java index 6d79dacfe7..eb009d0a85 100644 --- a/test/com/amazon/ion/DecimalTest.java +++ b/test/com/amazon/ion/DecimalTest.java @@ -31,6 +31,7 @@ public static void checkNullDecimal(IonDecimal value) { assertSame(IonType.DECIMAL, value.getType()); assertTrue("isNullValue is false", value.isNullValue()); + assertFalse(value.isNumericValue()); try { @@ -346,4 +347,13 @@ public void testSetValue() checkDecimal(123, 0, value.bigDecimalValue()); } + @Test + public void testIsNumeric() + { + IonNumber value = (IonNumber) oneValue("1.23"); + assertTrue(value.isNumericValue()); + + IonNumber nullValue = (IonNumber) oneValue("null.decimal"); + assertFalse(nullValue.isNumericValue()); + } } diff --git a/test/com/amazon/ion/IntTest.java b/test/com/amazon/ion/IntTest.java index d21e29042c..969098362d 100644 --- a/test/com/amazon/ion/IntTest.java +++ b/test/com/amazon/ion/IntTest.java @@ -32,6 +32,7 @@ public static void checkNullInt(IonInt value) { assertSame(IonType.INT, value.getType()); assertTrue("isNullValue() is false", value.isNullValue()); + assertFalse(value.isNumericValue()); try { @@ -48,6 +49,7 @@ public static void checkNullInt(IonInt value) catch (NullValueException e) { } assertNull("toBigInteger() isn't null", value.bigIntegerValue()); + assertNull("bigDecimalValue() isn't null", value.bigDecimalValue()); } @@ -63,6 +65,7 @@ public void modifyInt(IonInt value) value.setValue(A_LONG_INT); assertEquals(A_LONG_INT, value.longValue()); assertEquals(BigInteger.valueOf(A_LONG_INT), value.bigIntegerValue()); + assertEquals(BigDecimal.valueOf(A_LONG_INT), value.bigDecimalValue()); value.setValue(null); checkNullInt(value); @@ -224,6 +227,9 @@ public void testBigDecimals() { assertEquals(SUPER_BIG_TRUNC_32, val.intValue()); assertEquals(SUPER_BIG.hashCode(), val.hashCode()); + + IonNumber result = (IonNumber) reload(val); + assertEquals(0, big.compareTo(result.bigDecimalValue())); } @Test @@ -481,6 +487,26 @@ public void testGetIntegerSizeNegativeIntBoundary() { testGetIntegerSizeIntBoundary(Integer.MIN_VALUE); } + @Test + public void testIsNumeric() + { + IonNumber value = (IonNumber) oneValue("1"); + assertTrue(value.isNumericValue()); + + IonNumber nullValue = (IonNumber) oneValue("null.int"); + assertFalse(nullValue.isNumericValue()); + } + + @Test + public void testBigDecimalValue() + { + IonNumber value = (IonNumber) oneValue("1"); + assertEquals(BigDecimal.valueOf(1), value.bigDecimalValue()); + + IonNumber nullValue = (IonNumber) oneValue("null.int"); + assertNull(nullValue.bigDecimalValue()); + } + private void testGetIntegerSizeLongBoundary(long boundaryValue) { BigInteger boundary = BigInteger.valueOf(boundaryValue); IonInt boundaryIon = (IonInt)oneValue(boundary.toString()); From cf97f97c04cda01a9e2806b1a27e8d04394bda3f Mon Sep 17 00:00:00 2001 From: Zack Slayton Date: Fri, 21 Jan 2022 12:32:37 -0500 Subject: [PATCH 148/490] Compacts preallocated headers when possible (#401) `IonRawBinaryWriter` can be configured to preallocate buffer bytes to hold the length prefix of a new container or annotations wrapper. This option trades processing speed for data size. Preallocating more bytes makes the encoded data larger, but increases the chances that the container's length (once calculated) can be written directly into the existing buffer. This patch modifies the writer such that, when finalizing the encoding of a container or annotations wrapper, the writer will check to see if the encoded length is small enough to fit in the type descriptor byte. If it is, the encoded bytes are shifted backwards in the buffer to eliminate the unnecessary preallocation. This means that the encoding of small containers isn't penalized for having preallocation enabled. Benchmarks showed no substantial difference in processing speed from the previous release, so this data size improvement does not come at the cost of throughput. --- .../ion/impl/bin/IonRawBinaryWriter.java | 75 +++++++---- src/com/amazon/ion/impl/bin/WriteBuffer.java | 116 ++++++++++++++++++ .../amazon/ion/impl/bin/WriteBufferTest.java | 38 ++++++ 3 files changed, 207 insertions(+), 22 deletions(-) diff --git a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java index bc7d63b5ee..de6a719b1c 100644 --- a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java +++ b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java @@ -202,6 +202,15 @@ private PreallocationMode(final int contentMaxLength, final int typedLength) /*package*/ abstract void patchLength(final WriteBuffer buffer, final long position, final long length); + /** + * Returns the number of header bytes that this mode would preallocate to hold the VarUInt-encoded length of + * the current value. This number is equal to the total header length (i.e. `typedLength`) minus one, as it does + * not include the type descriptor byte. (Examples: PREALLOCATE_0 returns `0`, PREALLOCATE_1 returns `1`, etc.) + */ + int numberOfLengthBytes() { + return typedLength - 1; + } + /*package*/ static PreallocationMode withPadSize(final int pad) { switch (pad) @@ -723,48 +732,70 @@ private void extendPatchPoints(final PatchList patches) private ContainerInfo popContainer() { - final ContainerInfo current = containers.pop(); - if (current == null) + final ContainerInfo currentContainer = containers.pop(); + if (currentContainer == null) { throw new IllegalStateException("Tried to pop container state without said container"); } // only patch for real containers and annotations -- we use VALUE for tracking only - final long length = current.length; - if (current.type != ContainerType.VALUE) + long length = currentContainer.length; + if (currentContainer.type != ContainerType.VALUE) { // patch in the length - final long position = current.position; - if (current.length <= preallocationMode.contentMaxLength && preallocationMode != PreallocationMode.PREALLOCATE_0) + final long positionOfFirstLengthByte = currentContainer.position; + if (length <= 0xD) { + // The body of this container/wrapper is small enough that its length can fit in the lower nibble of + // the type descriptor byte; we don't need the extra length bytes that were preallocated (if any). + // We'll shift the encoded body of the container/wrapper backwards in the buffer to overwrite them. + + // The number of bytes we need to shift by is determined by the writer's preallocation mode. + final int numberOfBytesToShiftBy = preallocationMode.numberOfLengthBytes(); + + // `length` is the encoded length of the container/wrapper we're stepping out of. It does not + // include any header bytes. In this `if` branch, we've confirmed that `length` is <= 0xD, + // so this downcast from `long` to `int` is safe. + final int lengthOfSliceToShift = (int) length; + + // Shift the container/wrapper body backwards in the buffer. Because this only happens when + // `lengthOfSliceToShift` is 13 or fewer bytes, this will usually be a very fast memcpy. + // It's slightly more work if the slice we're shifting happens to straddle two memory blocks + // inside the buffer. + buffer.shiftBytesLeft(lengthOfSliceToShift, numberOfBytesToShiftBy); + + // Overwrite the lower nibble of the original type descriptor byte with the body's encoded length. + final long typeDescriptorPosition = positionOfFirstLengthByte - 1; + final long type = (buffer.getUInt8At(typeDescriptorPosition) & 0xF0) | length; + buffer.writeUInt8At(typeDescriptorPosition, type); + + // We've reclaimed some number of bytes; adjust the container length as appropriate. + length -= numberOfBytesToShiftBy; + } + else if (currentContainer.length <= preallocationMode.contentMaxLength) { - preallocationMode.patchLength(buffer, position, length); + // The container's encoded body is too long to fit the length in the type descriptor byte, but it will + // fit in the preallocated length bytes that were added to the buffer when the container was started. + // Update those bytes with the VarUInt encoding of the length value. + preallocationMode.patchLength(buffer, positionOfFirstLengthByte, length); } else { - // side patch - if (current.length <= 0xD && preallocationMode == PreallocationMode.PREALLOCATE_0) - { - // XXX if we're not using padding we can get here and optimize the length a little without side patching! - final long typePosition = position - 1; - final long type = (buffer.getUInt8At(typePosition) & 0xF0) | current.length; - buffer.writeUInt8At(typePosition, type); - } - else - { - addPatchPoint(position, preallocationMode.typedLength - 1, length); - } + // The container's encoded body is too long to fit in the length bytes that were preallocated. + // Write the VarUInt encoding of the length in a secondary buffer and make a note to include that + // when we go to flush the primary buffer to the output stream. + addPatchPoint(positionOfFirstLengthByte, preallocationMode.numberOfLengthBytes(), length); } } - if (current.patches != null) + if (currentContainer.patches != null) { // at this point, we've appended our patch points upward, lets make sure we get // our child patch points in - extendPatchPoints(current.patches); + extendPatchPoints(currentContainer.patches); } // make sure to record length upward updateLength(length); - return current; + return currentContainer; } private void writeVarUInt(final long value) diff --git a/src/com/amazon/ion/impl/bin/WriteBuffer.java b/src/com/amazon/ion/impl/bin/WriteBuffer.java index e3973c56e4..0df5ba6895 100644 --- a/src/com/amazon/ion/impl/bin/WriteBuffer.java +++ b/src/com/amazon/ion/impl/bin/WriteBuffer.java @@ -170,6 +170,122 @@ public void writeBytes(final byte[] bytes, final int off, final int len) block.limit += len; } + /** + * Shifts the last `length` bytes in the buffer to the left. This can be used when a value's header was + * preallocated but the value's encoded size proved to be much smaller than anticipated. + * + * The caller must guarantee that the buffer contains enough bytes to perform the requested shift. + * + * @param length The number of bytes at the end of the buffer that we'll be shifting to the left. + * @param shiftBy The number of bytes to the left that we'll be shifting. + */ + public void shiftBytesLeft(int length, int shiftBy) { + if (shiftBy == 0) { + // Nothing to do. + return; + } + + // If all of the bytes that we need to shift are in the current block, do a simple memcpy. + if (current.limit >= length + shiftBy) { + shiftBytesLeftWithinASingleBlock(length, shiftBy); + return; + } + + // Otherwise, the slice we're shifting straddles multiple blocks. We'll need to iterate across those blocks + // applying shifting logic to each one. + shiftBytesLeftAcrossBlocks(length, shiftBy); + } + + /** + * Shifts the last `length` bytes in the buffer to the left. The caller must guarantee that the `current` Block + * contains at least `length + shiftBy` bytes. This ensures that we're shifting a contiguous slice of bytes within + * a single block. + * + * @param length The number of bytes at the end of the buffer that we'll be shifting to the left. + * @param shiftBy The number of bytes to the left that we'll be shifting. + */ + private void shiftBytesLeftWithinASingleBlock(int length, int shiftBy) { + int startOfSliceToShift = current.limit - length; + System.arraycopy( + current.data, + startOfSliceToShift, + current.data, + startOfSliceToShift - shiftBy, + length + ); + // Update the `limit` (cursor position) within the current block to reflect that + // we have reclaimed `length` bytes of space in the buffer. + current.limit -= shiftBy; + } + + /** + * Shifts the last `length` bytes in the buffer to the left. Unlike + * {@link #shiftBytesLeftWithinASingleBlock(int, int)}, this method supports shifting bytes across multiple blocks + * in the buffer. + * + * @param length The number of bytes at the end of the buffer that we'll be shifting to the left. + * @param shiftBy The number of bytes to the left that we'll be shifting. + */ + private void shiftBytesLeftAcrossBlocks(int length, int shiftBy) { + // Our starting position is the first byte that we plan to shift backwards. The `bufferOffset` is the cursor's + // current position in the stream; we can use this to derive which block it's in as well as its position within + // that block. + long bufferOffset = position() - length; + + Block block = null; + while (length > 0) + { + // Using the `bufferOffset`, determine which block we're in... + int blockIndex = index(bufferOffset); + block = blocks.get(blockIndex); + // ...and our offset within that block. + int blockOffset = offset(bufferOffset); + + // If the block offset is within `shiftBy` bytes of beginning of the block, some bytes we're shifting + // will end up in the previous block. Here's an illustrated example: + // + // shiftBy = 2, blockIndex = 1, blockOffset = 1 + // v---- Cursor is here + // Before: [A B C D E] [F G H I J] + // After : [A B C D G] [F G H I J] + // Now this is G --^ ^-- And the cursor is here, ready to continue copy the rest. + if (blockOffset < shiftBy) { + Block previousBlock = blocks.get(blockIndex - 1); + int numberOfBytesToShift = Math.min(length, shiftBy) - blockOffset; + System.arraycopy(block.data, blockOffset, previousBlock.data, previousBlock.data.length - numberOfBytesToShift, numberOfBytesToShift); + + // Now that we've shifted some bytes, update our position within the buffer. + bufferOffset += numberOfBytesToShift; + length -= numberOfBytesToShift; + + // If there are no more bytes to shift... + if (length == 0) { + // ...lower the `limit` because we've reclaimed some bytes in this block... + block.limit -= numberOfBytesToShift; + // ...and early return. + return; + } + // Otherwise, use our new buffer offset to recalculate our block-specific position. + blockIndex = index(bufferOffset); + block = blocks.get(blockIndex); + blockOffset = offset(bufferOffset); + } + + // At this point, the block offset is at least `shiftBy` bytes away from the beginning of the block, so + // we can do a memcpy to shift the rest of the bytes in the block. + int numberOfBytesToShift = Math.min(length, block.data.length - blockOffset); + System.arraycopy(block.data, blockOffset, block.data, blockOffset - shiftBy, numberOfBytesToShift); + + // Update our counters and see if there are any more bytes to shift. + bufferOffset += numberOfBytesToShift; + length -= numberOfBytesToShift; + } + if (block != null) { + // We've reclaimed some space in this block; lower the `limit` accordingly. + block.limit -= shiftBy; + } + } + /** Writes an array of bytes to the buffer expanding if necessary, defaulting to the entire array. */ public void writeBytes(byte[] bytes) { diff --git a/test/com/amazon/ion/impl/bin/WriteBufferTest.java b/test/com/amazon/ion/impl/bin/WriteBufferTest.java index 5a5ddb5acf..24822820a2 100644 --- a/test/com/amazon/ion/impl/bin/WriteBufferTest.java +++ b/test/com/amazon/ion/impl/bin/WriteBufferTest.java @@ -921,4 +921,42 @@ public void testTruncate() throws IOException buf.truncate(3); assertBuffer("ARG".getBytes("UTF-8")); } + + @Test + public void shiftBytesLeftWithinFirstBufferBlock() throws IOException { + assertEquals(11, ALLOCATOR.getBlockSize()); + // All bytes are being shifted within the first buffer block. + buf.writeBytes("01234567".getBytes()); + buf.shiftBytesLeft(4, 1); + assertBuffer("0124567".getBytes()); + } + + @Test + public void shiftBytesLeftWithinLastBufferBlock() throws IOException { + assertEquals(11, ALLOCATOR.getBlockSize()); + // All bytes are being shifted within the last buffer block. + buf.writeBytes("0123456789ABCDEF".getBytes()); + buf.shiftBytesLeft(3, 2); + assertBuffer("0123456789ADEF".getBytes()); + } + + @Test + public void shiftBytesLeftAcrossBufferBlocks() throws IOException { + assertEquals(11, ALLOCATOR.getBlockSize()); + // Some bytes are shifted to the previous block, some are shifted within + // the last block. + buf.writeBytes("0123456789ABCDEF".getBytes()); + buf.shiftBytesLeft(8, 3); + assertBuffer("0123489ABCDEF".getBytes()); + } + + @Test + public void shiftBytesLeftAcrossBufferBlocksExclusively() throws IOException { + assertEquals(11, ALLOCATOR.getBlockSize()); + // Unlike `shiftBytesLeftAcrossBufferBlocks`, EVERY byte that is shifted + // in the buffer ends up in the previous block. + buf.writeBytes("0123456789ABCDEF".getBytes()); + buf.shiftBytesLeft(5, 5); + assertBuffer("012345BCDEF".getBytes()); + } } From eb3766e975d23ae54455064b8440f49e6a6ff3a5 Mon Sep 17 00:00:00 2001 From: Zack Slayton Date: Mon, 24 Jan 2022 17:44:05 -0500 Subject: [PATCH 149/490] Eliminates allocations in the binary writer (#404) This patch changes some very hot code paths in the binary writer to eliminate unnecessary allocations. In particular: * Setting the annotations list no longer constructs a temporary array of `SymbolToken`s. * `prepareValue` (which is called for every value in the stream) now uses a plain `for` loop instead of an iterator to encode each annotation. * `IonRawBinaryWriter` now uses a custom class called `IntList` to store its annotation symbol ID ints instead of a `List`, which required each int to be boxed. --- src/com/amazon/ion/impl/bin/IntList.java | 105 ++++++++++++++++++ .../ion/impl/bin/IonManagedBinaryWriter.java | 20 ++-- .../ion/impl/bin/IonRawBinaryWriter.java | 8 +- test/com/amazon/ion/impl/bin/IntListTest.java | 59 ++++++++++ 4 files changed, 178 insertions(+), 14 deletions(-) create mode 100644 src/com/amazon/ion/impl/bin/IntList.java create mode 100644 test/com/amazon/ion/impl/bin/IntListTest.java diff --git a/src/com/amazon/ion/impl/bin/IntList.java b/src/com/amazon/ion/impl/bin/IntList.java new file mode 100644 index 0000000000..8c0ba5c214 --- /dev/null +++ b/src/com/amazon/ion/impl/bin/IntList.java @@ -0,0 +1,105 @@ +package com.amazon.ion.impl.bin; + +/** + * A list of integer values that grows as necessary. + * + * Unlike {@link java.util.List}, IntList does not require each int to be boxed. This makes it helpful in use cases + * where storing {@link Integer} leads to excessive time spent in garbage collection. + */ +public class IntList { + public static final int DEFAULT_INITIAL_CAPACITY = 8; + private static final int GROWTH_MULTIPLIER = 2; + private int[] data; + private int numberOfValues; + + /** + * Constructs a new IntList with a capacity of {@link IntList#DEFAULT_INITIAL_CAPACITY}. + */ + public IntList() { + this(DEFAULT_INITIAL_CAPACITY); + } + + /** + * Constructs a new IntList with the specified capacity. + * @param initialCapacity The number of ints that can be stored in this IntList before it will need to be + * reallocated. + */ + public IntList(final int initialCapacity) { + data = new int[initialCapacity]; + numberOfValues = 0; + } + + /** + * Accessor. + * @return The number of ints currently stored in the list. + */ + public int size() { + return numberOfValues; + } + + /** + * @return {@code true} if there are ints stored in the list. + */ + public boolean isEmpty() { + return numberOfValues == 0; + } + + /** + * Empties the list. + * + * Note that this method does not shrink the size of the backing data store. + */ + public void clear() { + numberOfValues = 0; + } + + /** + * Returns the {@code index}th int in the list. + * @param index The list index of the desired int. + * @return The int at index {@code index} in the list. + * @throws IndexOutOfBoundsException if the index is negative or greater than the number of ints stored in the + * list. + */ + public int get(int index) { + if (index < 0 || index >= numberOfValues) { + throw new IndexOutOfBoundsException( + "Invalid index " + index + " requested from IntList with " + numberOfValues + " values." + ); + } + return data[index]; + } + + /** + * Appends an int to the end of the list, growing the list if necessary. + * @param value The int to add to the end of the list. + */ + public void add(int value) { + if (numberOfValues == data.length) { + grow(); + } + data[numberOfValues] = value; + numberOfValues += 1; + } + + /** + * Reallocates the backing array to accommodate storing more ints. + */ + private void grow() { + int[] newData = new int[data.length * GROWTH_MULTIPLIER]; + System.arraycopy(data, 0, newData, 0, data.length); + data = newData; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("IntList{data=["); + if (numberOfValues > 0) { + for (int m = 0; m < numberOfValues; m++) { + builder.append(data[m]).append(","); + } + } + builder.append("]}"); + return builder.toString(); + } +} diff --git a/src/com/amazon/ion/impl/bin/IonManagedBinaryWriter.java b/src/com/amazon/ion/impl/bin/IonManagedBinaryWriter.java index 003dc70abe..75ca54c574 100644 --- a/src/com/amazon/ion/impl/bin/IonManagedBinaryWriter.java +++ b/src/com/amazon/ion/impl/bin/IonManagedBinaryWriter.java @@ -927,18 +927,16 @@ public void requireLocalSymbolTable() throws IOException public void setTypeAnnotations(final String... annotations) { - if (annotations == null) - { - user.setTypeAnnotationSymbols((SymbolToken[]) null); + // Clear the current list of annotations + user.setTypeAnnotationSymbols((SymbolToken[]) null); + if (annotations == null) { + return; } - else - { - final SymbolToken[] tokens = new SymbolToken[annotations.length]; - for (int i = 0; i < tokens.length; i++) - { - tokens[i] = intern(annotations[i]); - } - user.setTypeAnnotationSymbols(tokens); + + // Add each string to the annotations list. + // XXX: This is a very hot path. This code avoids allocating temporary iterators/arrays. + for (int i = 0; i < annotations.length; i++) { + addTypeAnnotation(annotations[i]); } } diff --git a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java index de6a719b1c..8bc415829a 100644 --- a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java +++ b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java @@ -498,7 +498,7 @@ public String toString() private boolean hasWrittenValuesSinceConstructed; private int currentFieldSid; - private final List currentAnnotationSids; + private final IntList currentAnnotationSids; // XXX this is for managed detection of TLV that is a LST--this is easier to track here than at the managed level private boolean hasTopLevelSymbolTableAnnotation; @@ -540,7 +540,7 @@ public ContainerInfo newElement() { this.hasWrittenValuesSinceConstructed = false; this.currentFieldSid = SID_UNASSIGNED; - this.currentAnnotationSids = new ArrayList(); + this.currentAnnotationSids = new IntList(); this.hasTopLevelSymbolTableAnnotation = false; this.closed = false; @@ -847,8 +847,10 @@ private void prepareValue() final long annotationsLengthPosition = buffer.position(); buffer.writeVarUInt(0L); int annotationsLength = 0; - for (final int symbol : currentAnnotationSids) + // XXX: This is a very hot path. This code intentionally avoids creating iterators. + for (int m = 0; m < currentAnnotationSids.size(); m++) { + final int symbol = currentAnnotationSids.get(m); checkSid(symbol); final int symbolLength = buffer.writeVarUInt(symbol); annotationsLength += symbolLength; diff --git a/test/com/amazon/ion/impl/bin/IntListTest.java b/test/com/amazon/ion/impl/bin/IntListTest.java new file mode 100644 index 0000000000..4adcf57ec1 --- /dev/null +++ b/test/com/amazon/ion/impl/bin/IntListTest.java @@ -0,0 +1,59 @@ +package com.amazon.ion.impl.bin; + +import junit.framework.TestCase; + +public class IntListTest extends TestCase { + + public void testSize() { + IntList intList = new IntList(); + assertEquals(intList.size(), 0); + + intList.add(0); + assertEquals(intList.size(), 1); + + intList.add(0); + assertEquals(intList.size(), 2); + + intList.clear(); + assertEquals(intList.size(), 0); + } + + public void testIsEmpty() { + IntList intList = new IntList(); + assertTrue(intList.isEmpty()); + intList.add(1); + assertFalse(intList.isEmpty()); + } + + public void testClear() { + IntList intList = new IntList(); + assertTrue(intList.isEmpty()); + intList.add(1); + intList.add(2); + intList.add(3); + assertFalse(intList.isEmpty()); + intList.clear(); + assertTrue(intList.isEmpty()); + } + + public void testAddAndGet() { + IntList intList = new IntList(); + intList.add(1); + intList.add(2); + intList.add(3); + assertEquals(intList.get(0), 1); + assertEquals(intList.get(1), 2); + assertEquals(intList.get(2), 3); + } + + public void testGrow() { + // Create the list with insufficient capacity + IntList intList = new IntList(1); + intList.add(1); + intList.add(2); + intList.add(3); + assertEquals(intList.get(0), 1); + assertEquals(intList.get(1), 2); + assertEquals(intList.get(2), 3); + } +} \ No newline at end of file From 0747831d0d8a8fe05a65f94e612f669e209c2f52 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Mon, 24 Jan 2022 16:47:24 -0800 Subject: [PATCH 150/490] Backs Pool with ConcurrentLinkedQueue instead of ArrayBlockingQueue. --- src/com/amazon/ion/impl/bin/utf8/Pool.java | 33 ++++++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/com/amazon/ion/impl/bin/utf8/Pool.java b/src/com/amazon/ion/impl/bin/utf8/Pool.java index 647055bcc4..3647541872 100644 --- a/src/com/amazon/ion/impl/bin/utf8/Pool.java +++ b/src/com/amazon/ion/impl/bin/utf8/Pool.java @@ -1,6 +1,8 @@ package com.amazon.ion.impl.bin.utf8; -import java.util.concurrent.ArrayBlockingQueue; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; abstract class Pool> { @@ -22,14 +24,19 @@ interface Allocator> { private static final int MAX_QUEUE_SIZE = 128; // A queue of previously initialized objects that can be loaned out. - private final ArrayBlockingQueue bufferQueue; + private final Queue objectQueue; + + // The current size of the queue. Note: some implementations of Queue.size() (including ConcurrentLinkedQueue's) + // are not constant-time operations. Tracking the size externally is a performance optimization. + private final AtomicInteger size; // Allocator of objects to be pooled. private final Allocator allocator; Pool(Allocator allocator) { this.allocator = allocator; - bufferQueue = new ArrayBlockingQueue(MAX_QUEUE_SIZE); + objectQueue = new ConcurrentLinkedQueue(); + size = new AtomicInteger(0); } /** @@ -40,10 +47,13 @@ interface Allocator> { */ public T getOrCreate() { // The `poll` method does not block. If the queue is empty it returns `null` immediately. - T object = bufferQueue.poll(); + T object = objectQueue.poll(); if (object == null) { - // No buffers were available in the pool. Create a new one. + // No objects were available in the pool. Create a new one. object = allocator.newInstance(this); + } else { + // An object was retrieved from the pool; decrement the pool size. + size.decrementAndGet(); } return object; } @@ -59,6 +69,17 @@ public T getOrCreate() { public void returnToPool(T object) { // The `offer` method does not block. If the queue is full, it returns `false` immediately. // If the provided instance cannot be added to the pool, we discard it silently. - bufferQueue.offer(object); + if (size.getAndIncrement() < MAX_QUEUE_SIZE) { + objectQueue.offer(object); + } else { + // The pool was full. Since the size was optimistically incremented, decrement it now. + // Note: there is a race condition here that is deliberately allowed as an optimization. + // Under high contention, multiple threads could end up here before the first one + // decrements the size, causing objects to be dropped wastefully. This is not harmful + // because objects will be re-allocated when necessary; the pool is kept as close as + // possible to capacity on a best-effort basis. This race condition should not be "fixed" + // without a thorough study of the performance implications. + size.decrementAndGet(); + } } } From 1baad70fa6d6ce6b0f20b3f2f92297465b00adf6 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Mon, 24 Jan 2022 18:49:01 -0800 Subject: [PATCH 151/490] Optimizes the incremental reader's symbol table reading logic for tables with imports. --- src/com/amazon/ion/impl/ImportLocation.java | 53 ++++ .../ion/impl/IonReaderBinaryIncremental.java | 270 ++++++------------ .../ion/impl/LocalSymbolTableImports.java | 15 + .../impl/IonReaderBinaryIncrementalTest.java | 10 +- 4 files changed, 155 insertions(+), 193 deletions(-) create mode 100644 src/com/amazon/ion/impl/ImportLocation.java diff --git a/src/com/amazon/ion/impl/ImportLocation.java b/src/com/amazon/ion/impl/ImportLocation.java new file mode 100644 index 0000000000..acd5873887 --- /dev/null +++ b/src/com/amazon/ion/impl/ImportLocation.java @@ -0,0 +1,53 @@ +package com.amazon.ion.impl; + +/** + * A SymbolToken's import location, allowing for symbols with unknown text to be mapped to a particular slot + * in a shared symbol table. + * NOTE: this is currently not publicly accessible, but it is an important step toward being able to correctly + * round-trip symbols with unknown text from shared symbol tables in different symbol table contexts. See + * https://github.com/amzn/ion-java/issues/126 . Support is added now to avoid risking the appearance of performance + * degradation if ImportLocation support were added after initial release of the incremental IonReader. + */ +class ImportLocation { + + // The name of the shared symbol table. + final String name; + + // The index into the shared symbol table. + final int sid; + + ImportLocation(String name, int sid) { + this.name = name; + this.sid = sid; + } + + public String getName() { + return name; + } + + public int getSid() { + return sid; + } + + @Override + public String toString() { + return String.format("ImportLocation::{name: %s, sid: %d}", name, sid); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof ImportLocation)) { + return false; + } + ImportLocation that = (ImportLocation) o; + return this.getName().equals(that.getName()) && this.getSid() == that.getSid(); + } + + @Override + public int hashCode() { + int result = 17; + result += 31 * getName().hashCode(); + result += 31 * getSid(); + return result; + } +} diff --git a/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java b/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java index 054a7f6902..7ff310702d 100644 --- a/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java +++ b/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java @@ -25,7 +25,6 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; -import java.util.Arrays; import java.util.ArrayList; import java.util.Collections; @@ -34,8 +33,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; /** *

    @@ -121,23 +118,6 @@ public ContainerInfo newElement() { } }; - // The Ion 1.0 system symbol table. - private static final List SYSTEM_SYMBOLS_1_0 = Collections.unmodifiableList(Arrays.asList( - null, - "$ion", - "$ion_1_0", - "$ion_symbol_table", - "name", - "version", - "imports", - "symbols", - "max_id", - "$ion_shared_symbol_table" - )); - - // The size of the Ion 1.0 system symbol table. - private static final int SYSTEM_SYMBOLS_1_0_SIZE = SYSTEM_SYMBOLS_1_0.size(); - // Symbol IDs for symbols contained in the system symbol table. private static class SystemSymbolIDs { @@ -213,6 +193,10 @@ private static class SystemSymbolIDs { // 32-bit floats must declare length 4. private static final int FLOAT_32_BYTE_LENGTH = 4; + // The imports for Ion 1.0 data with no shared user imports. + private static final LocalSymbolTableImports ION_1_0_IMPORTS + = new LocalSymbolTableImports(SharedSymbolTable.getSystemSymbolTable(1)); + // The InputStream that provides the binary Ion data. private final InputStream inputStream; @@ -247,18 +231,13 @@ private static class SystemSymbolIDs { // The catalog used by the reader to resolve shared symbol table imports. private final IonCatalog catalog; - // The shared symbol tables imported by the local symbol table that is currently in scope. The key is the highest - // local symbol ID that resolves to a symbol contained in the value's symbol table. - private final TreeMap imports; + // The shared symbol tables imported by the local symbol table that is currently in scope. + private LocalSymbolTableImports imports = ION_1_0_IMPORTS; // A map of symbol ID to SymbolToken representation. Because most use cases only require symbol text, this // is used only if necessary to avoid imposing the extra expense on all symbol lookups. private List symbolTokensById = null; - // The highest local symbol ID that resolves to a symbol contained in a shared symbol table imported by the - // current local symbol table. - private int importMaxId; - // The cached SymbolTable representation of the current local symbol table. Invalidated whenever a local // symbol table is encountered in the stream. private SymbolTable cachedReadOnlySymbolTable = null; @@ -341,8 +320,6 @@ private static class SystemSymbolIDs { ); annotationSids = new ArrayList(ANNOTATIONS_LIST_INITIAL_CAPACITY); symbols = new ArrayList(SYMBOLS_LIST_INITIAL_CAPACITY); - symbols.addAll(SYSTEM_SYMBOLS_1_0); - imports = new TreeMap(); scalarConverter = new _Private_ScalarConversions.ValueVariant(); resetImports(); } @@ -404,58 +381,6 @@ public void reset() { } } - /** - * A SymbolToken's import location, allowing for symbols with unknown text to be mapped to a particular slot - * in a shared symbol table. - * NOTE: this is currently not publicly accessible, but it is an important step toward being able to correctly - * round-trip symbols with unknown text from shared symbol tables in different symbol table contexts. See - * https://github.com/amzn/ion-java/issues/126 . Support is added now to avoid risking the appearance of performance - * degradation if ImportLocation support were added after initial release of this IonReader implementation. - */ - static class ImportLocation { - - // The name of the shared symbol table. - final String name; - - // The index into the shared symbol table. - final int sid; - - ImportLocation(String name, int sid) { - this.name = name; - this.sid = sid; - } - - public String getName() { - return name; - } - - public int getSid() { - return sid; - } - - @Override - public String toString() { - return String.format("ImportLocation::{name: %s, sid: %d}", name, sid); - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof ImportLocation)) { - return false; - } - ImportLocation that = (ImportLocation) o; - return this.getName().equals(that.getName()) && this.getSid() == that.getSid(); - } - - @Override - public int hashCode() { - int result = 17; - result += 31 * getName().hashCode(); - result += 31 * getSid(); - return result; - } - } - /** * SymbolToken implementation that includes ImportLocation. */ @@ -525,22 +450,30 @@ public int hashCode() { } } + /** + * Gets the system symbol table for the Ion version currently active. + * @return a system SymbolTable. + */ + private SymbolTable getSystemSymbolTable() { + // Note: Ion 1.1 currently proposes changes to the system symbol table. If this is finalized, then + // 'majorVersion' cannot be used to look up the system symbol table; both 'majorVersion' and 'minorVersion' + // will need to be used. + return SharedSymbolTable.getSystemSymbolTable(majorVersion); + } + /** * Read-only snapshot of the local symbol table at the reader's current position. */ private class LocalSymbolTableSnapshot implements SymbolTable, SymbolTableAsStruct { // The system symbol table. - private final SymbolTable system = SharedSymbolTable.getSystemSymbolTable(majorVersion); + private final SymbolTable system = IonReaderBinaryIncremental.this.getSystemSymbolTable(); // The max ID of this local symbol table. private final int maxId; - // The max ID that maps to a shared symbol table imported by this local symbol table. - private final int importedTablesMaxId; - // The shared symbol tables imported by this local symbol table. - private final SymbolTable[] importedTables; + private final LocalSymbolTableImports importedTables; // Map representation of this symbol table. Keys are symbol text; values are the lowest symbol ID that maps // to that text. @@ -552,25 +485,21 @@ private class LocalSymbolTableSnapshot implements SymbolTable, SymbolTableAsStru private SymbolTableStructCache structCache = null; LocalSymbolTableSnapshot() { - int numberOfSymbols = symbols.size(); - maxId = numberOfSymbols - 1; - importedTablesMaxId = importMaxId; + int importsMaxId = imports.getMaxId(); + int numberOfLocalSymbols = symbols.size(); + // Note: 'imports' is immutable, so a clone is not needed. + importedTables = imports; + maxId = importsMaxId + numberOfLocalSymbols; // Map with initial size the number of symbols and load factor 1, meaning it must be full before growing. // It is not expected to grow. - listView = new ArrayList(symbols.subList(0, numberOfSymbols)); - mapView = new HashMap((int) Math.ceil(numberOfSymbols / 0.75), 0.75f); - for (int i = 0; i < numberOfSymbols; i++) { + listView = new ArrayList(symbols.subList(0, numberOfLocalSymbols)); + mapView = new HashMap((int) Math.ceil(numberOfLocalSymbols / 0.75), 0.75f); + for (int i = 0; i < numberOfLocalSymbols; i++) { String symbol = listView.get(i); if (symbol != null) { - mapView.put(symbol, i); + mapView.put(symbol, i + importsMaxId + 1); } } - importedTables = new SymbolTable[imports.size()]; - int i = 0; - for (SymbolTable importedTable : imports.values()) { - importedTables[i] = importedTable; - i++; - } } @Override @@ -615,16 +544,20 @@ public String getIonVersionId() { @Override public SymbolTable[] getImportedTables() { - return importedTables; + return importedTables.getImportedTables(); } @Override public int getImportedMaxId() { - return importedTablesMaxId; + return importedTables.getMaxId(); } @Override public SymbolToken find(String text) { + SymbolToken token = importedTables.find(text); + if (token != null) { + return token; + } Integer sid = mapView.get(text); if (sid == null) { return null; @@ -648,7 +581,11 @@ public SymbolToken find(String text) { @Override public int findSymbol(String name) { - Integer sid = mapView.get(name); + Integer sid = importedTables.findSymbol(name); + if (sid > UNKNOWN_SYMBOL_ID) { + return sid; + } + sid = mapView.get(name); if (sid == null) { return UNKNOWN_SYMBOL_ID; } @@ -660,21 +597,21 @@ public String findKnownSymbol(int id) { if (id < 0) { throw new IllegalArgumentException("Symbol IDs must be at least 0."); } - if (id >= symbols.size()) { + if (id > getMaxId()) { return null; } - return listView.get(id); + return IonReaderBinaryIncremental.this.getSymbolString(id, importedTables, listView); } @Override public Iterator iterateDeclaredSymbolNames() { return new Iterator() { - private int index = getImportedMaxId() + 1; + private int index = 0; @Override public boolean hasNext() { - return index <= getMaxId(); + return index < listView.size(); } @Override @@ -751,7 +688,6 @@ private void resetSymbolTable() { // Note: when there is a new version of Ion, check majorVersion and minorVersion here and set the appropriate // system symbol table. symbols.clear(); - symbols.addAll(SYSTEM_SYMBOLS_1_0); cachedReadOnlySymbolTable = null; if (symbolTokensById != null) { symbolTokensById.clear(); @@ -762,68 +698,54 @@ private void resetSymbolTable() { * Clear the list of imported shared symbol tables. */ private void resetImports() { - imports.clear(); - SymbolTable system = SharedSymbolTable.getSystemSymbolTable(majorVersion); - importMaxId = system.getMaxId(); + // Note: when support for the next version of Ion is added, conditionals on 'majorVersion' and 'minorVersion' + // must be added here. + imports = ION_1_0_IMPORTS; } /** - * Add a shared symbol table import, resolving it from the catalog if possible. + * Creates a shared symbol table import, resolving it from the catalog if possible. * @param name the name of the shared symbol table. * @param version the version of the shared symbol table. * @param maxId the max_id of the shared symbol table. This value takes precedence over the actual max_id for the * shared symbol table at the requested version. */ - private void addImport(String name, int version, int maxId) { + private SymbolTable createImport(String name, int version, int maxId) { SymbolTable shared = catalog.getTable(name, version); - importMaxId += maxId; if (shared == null) { // No match. All symbol IDs that fall within this shared symbol table's range will have unknown text. - imports.put(importMaxId, new SubstituteSymbolTable(name, version, maxId)); + return new SubstituteSymbolTable(name, version, maxId); } else if (shared.getMaxId() != maxId || shared.getVersion() != version) { // Partial match. If the requested max_id exceeds the actual max_id of the resolved shared symbol table, // symbol IDs that exceed the max_id of the resolved shared symbol table will have unknown text. - imports.put(importMaxId, new SubstituteSymbolTable(shared, version, maxId)); + return new SubstituteSymbolTable(shared, version, maxId); } else { // Exact match; the resolved shared symbol table may be used as-is. - imports.put(importMaxId, shared); + return shared; } } /** - * Retrieves from the `imports` the next-lowest key to the given key, or `null` if there is no key lower - * than the given key. - * @param key the key. - * @return the lower key, or null. + * Gets the String representation of the given symbol ID. It is the caller's responsibility to ensure that the + * given symbol ID is within the max ID of the symbol table. + * @param sid the symbol ID. + * @param importedSymbols the symbol table's shared symbol table imports. + * @param localSymbols the symbol table's local symbols. + * @return a String, which will be null if the requested symbol ID has undefined text. */ - private Integer lowerKey(int key) { - // Note: with JDK 1.6+, this method is just `return imports.lowerKey(key);` - SortedMap sortedView = imports.headMap(key); - if (sortedView.isEmpty()) { - return null; + private String getSymbolString(int sid, LocalSymbolTableImports importedSymbols, List localSymbols) { + if (sid <= importedSymbols.getMaxId()) { + return importedSymbols.findKnownSymbol(sid); } - return sortedView.lastKey(); + return localSymbols.get(sid - (importedSymbols.getMaxId() + 1)); } /** - * Gets the ImportLocation for the given local SID. The given SID must not point to a system symbol and must be - * less than or equal to `importMaxId`. - * @param sid the local SID. - * @return the ImportLocation for the given local SID. + * Calculates the symbol table's max ID. + * @return the max ID. */ - private ImportLocation getImportLocation(int sid) { - // The system symbol table is never included in the imports map, so the local SID must be adjusted lower - // by the max ID of the system symbol table. - int systemMaxId = SharedSymbolTable.getSystemSymbolTable(majorVersion).getMaxId(); - int systemAdjustedMaxId = sid - systemMaxId; - // Note: with JDK 1.6+, the following line would be - // Map.Entry entry = imports.ceilingEntry(systemAdjustedMaxId); - Map.Entry entry = imports.tailMap(systemAdjustedMaxId).entrySet().iterator().next(); - Integer previousMaxId = lowerKey(systemAdjustedMaxId); - if (previousMaxId == null) { - previousMaxId = systemMaxId; - } - return new ImportLocation(entry.getValue().getName(), sid - previousMaxId); + private int maxSymbolId() { + return symbols.size() + imports.getMaxId(); } /** @@ -832,10 +754,10 @@ private ImportLocation getImportLocation(int sid) { * @return a String. */ private String getSymbol(int sid) { - if (sid >= symbols.size()) { + if (sid > maxSymbolId()) { throw new IonException("Symbol ID exceeds the max ID of the symbol table."); } - return symbols.get(sid); + return getSymbolString(sid, imports, symbols); } /** @@ -844,25 +766,26 @@ private String getSymbol(int sid) { * @return a SymbolToken. */ private SymbolToken getSymbolToken(int sid) { + int symbolTableSize = maxSymbolId() + 1; if (symbolTokensById == null) { - symbolTokensById = new ArrayList(symbols.size()); + symbolTokensById = new ArrayList(symbolTableSize); } - if (symbolTokensById.size() < symbols.size()) { - for (int i = symbolTokensById.size(); i < symbols.size(); i++) { + if (symbolTokensById.size() < symbolTableSize) { + for (int i = symbolTokensById.size(); i < symbolTableSize; i++) { symbolTokensById.add(null); } } - if (sid >= symbols.size()) { + if (sid >= symbolTableSize) { throw new IonException("Symbol ID exceeds the max ID of the symbol table."); } SymbolToken token = symbolTokensById.get(sid); if (token == null) { - String text = symbols.get(sid); + String text = getSymbolString(sid, imports, symbols); ImportLocation importLocation = null; if (text == null) { // Note: this will never be a system symbol. - if (sid > 0 && sid <= importMaxId) { - importLocation = getImportLocation(sid); + if (sid > 0 && sid <= imports.getMaxId()) { + importLocation = imports.getImportLocation(sid); } else { // All symbols with unknown text in the local symbol range are equivalent to symbol zero. sid = 0; @@ -874,38 +797,6 @@ private SymbolToken getSymbolToken(int sid) { return token; } - /** - * Adds the symbols declared in the given shared symbol table to the reader's current local symbol table, stopping - * once the given max_id has been reached. - * @param shared a shared symbol table to import. - * @param maxId the maximum symbol ID to import from the shared symbol table. - */ - private void addSymbolsFromImport(SymbolTable shared, int maxId) { - // The system symbol table is never included in the imports map, so the local SID must be adjusted lower - // by the max ID of the system symbol table. - int systemMaxId = SharedSymbolTable.getSystemSymbolTable(majorVersion).getMaxId(); - Integer previousMaxId = lowerKey(maxId); - if (previousMaxId == null) { - previousMaxId = systemMaxId; - } - int adjustedMaxId = maxId - previousMaxId; - int id = 1; - Iterator importedSymbols = shared.iterateDeclaredSymbolNames(); - while (importedSymbols.hasNext() && id <= adjustedMaxId) { - symbols.add(importedSymbols.next()); - id++; - } - } - - /** - * Adds the symbols from all of the imported shared symbol tables to the reader's current local symbol table. - */ - private void addSymbolsFromImports() { - for (Map.Entry entry : imports.entrySet()) { - addSymbolsFromImport(entry.getValue(), entry.getKey()); - } - } - /** * Reads a local symbol table from the buffer. * @param marker marker for the start and end positions of the local symbol table in the buffer. @@ -917,6 +808,7 @@ private void readSymbolTable(IonReaderLookaheadBuffer.SymbolTableMarker marker) boolean hasSeenSymbols = false; int symbolsPosition = -1; int symbolsEndPosition = -1; + List newImports; while (peekIndex < marker.endIndex) { fieldNameSid = readVarUInt(); IonTypeID typeID = readTypeId(); @@ -931,6 +823,8 @@ private void readSymbolTable(IonReaderLookaheadBuffer.SymbolTableMarker marker) peekIndex = currentValueEndPosition; } else if (typeID.type == IonType.LIST) { resetImports(); + newImports = new ArrayList(3); + newImports.add(getSystemSymbolTable()); stepIn(); IonType type = next(); while (type != null) { @@ -959,15 +853,15 @@ private void readSymbolTable(IonReaderLookaheadBuffer.SymbolTableMarker marker) } stepOut(); } - addImport(name, version, maxId); + newImports.add(createImport(name, version, maxId)); type = next(); } stepOut(); + imports = new LocalSymbolTableImports(newImports); } if (!isAppend) { // Clear the existing symbols before adding the new imported symbols. resetSymbolTable(); - addSymbolsFromImports(); } hasSeenImports = true; } else if (fieldNameSid == SystemSymbolIDs.SYMBOLS_ID) { @@ -1242,8 +1136,8 @@ public int getDepth() { @Override public SymbolTable getSymbolTable() { if (cachedReadOnlySymbolTable == null) { - if (symbols.size() == SYSTEM_SYMBOLS_1_0_SIZE) { - cachedReadOnlySymbolTable = SharedSymbolTable.getSystemSymbolTable(majorVersion); + if (symbols.size() == 0 && imports == ION_1_0_IMPORTS) { + cachedReadOnlySymbolTable = imports.getSystemSymbolTable(); } else { cachedReadOnlySymbolTable = new LocalSymbolTableSnapshot(); } diff --git a/src/com/amazon/ion/impl/LocalSymbolTableImports.java b/src/com/amazon/ion/impl/LocalSymbolTableImports.java index 3d9bfe7e58..b4b7b5e131 100644 --- a/src/com/amazon/ion/impl/LocalSymbolTableImports.java +++ b/src/com/amazon/ion/impl/LocalSymbolTableImports.java @@ -278,6 +278,21 @@ SymbolTable[] getImportedTablesNoCopy() return myImports; } + /** + * Gets the {@link ImportLocation} for the given symbol ID. + * @param sid the symbol ID. + * @return an ImportLocation, or null if the given symbol ID is not present in the imports. + */ + ImportLocation getImportLocation(int sid) { + // TODO once ion-java moves to Java 1.6+, the search for the import could use Arrays.binarySearch. + for (int i = myBaseSids.length - 1; i >= 0; i--) { + if (myBaseSids[i] < sid) { + return new ImportLocation(myImports[i].getName(), sid - myBaseSids[i]); + } + } + return null; + } + @Override public String toString() { diff --git a/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java b/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java index 5d114b19a4..a00c3c2c6d 100644 --- a/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java +++ b/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java @@ -1675,7 +1675,7 @@ public void write(_Private_IonRawWriter writer, ByteArrayOutputStream out) throw private static void assertSymbolEquals( String expectedText, - IonReaderBinaryIncremental.ImportLocation expectedImportLocation, + ImportLocation expectedImportLocation, SymbolToken actual ) { assertEquals(expectedText, actual.getText()); @@ -2162,11 +2162,11 @@ public void write(_Private_IonRawWriter writer, ByteArrayOutputStream out) throw IonReaderBinaryIncremental.SymbolTokenImpl symbolValue = (IonReaderBinaryIncremental.SymbolTokenImpl) reader.symbolValue(); assertNull(symbolValue.getText()); - assertEquals(new IonReaderBinaryIncremental.ImportLocation("foo", 3), symbolValue.getImportLocation()); + assertEquals(new ImportLocation("foo", 3), symbolValue.getImportLocation()); assertEquals(IonType.SYMBOL, reader.next()); symbolValue = (IonReaderBinaryIncremental.SymbolTokenImpl) reader.symbolValue(); assertNull(symbolValue.getText()); - assertEquals(new IonReaderBinaryIncremental.ImportLocation("foo", 4), symbolValue.getImportLocation()); + assertEquals(new ImportLocation("foo", 4), symbolValue.getImportLocation()); assertEquals(IonType.SYMBOL, reader.next()); assertEquals("123", reader.stringValue()); assertEquals(IonType.SYMBOL, reader.next()); @@ -2175,7 +2175,7 @@ public void write(_Private_IonRawWriter writer, ByteArrayOutputStream out) throw new IonReaderBinaryIncremental.SymbolTokenImpl( null, 15, - new IonReaderBinaryIncremental.ImportLocation("baz", 1) + new ImportLocation("baz", 1) ), symbolValue ); @@ -2185,7 +2185,7 @@ public void write(_Private_IonRawWriter writer, ByteArrayOutputStream out) throw new IonReaderBinaryIncremental.SymbolTokenImpl( null, 16, - new IonReaderBinaryIncremental.ImportLocation("baz", 2) + new ImportLocation("baz", 2) ), symbolValue ); From 4266318af556322c00f7178336ad571f3636df7f Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Wed, 26 Jan 2022 17:47:48 -0800 Subject: [PATCH 152/490] Enables the incremental reader to iterate annotations lazily. --- .../ion/impl/IonReaderBinaryIncremental.java | 139 ++++++++++------- .../ion/impl/IonReaderLookaheadBuffer.java | 141 +++++++++++------- src/com/amazon/ion/impl/bin/IntList.java | 10 ++ .../impl/IonReaderLookaheadBufferTest.java | 84 ++++++++--- 4 files changed, 245 insertions(+), 129 deletions(-) diff --git a/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java b/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java index 7ff310702d..22d774173b 100644 --- a/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java +++ b/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java @@ -15,6 +15,7 @@ import com.amazon.ion.Timestamp; import com.amazon.ion.UnknownSymbolException; import com.amazon.ion.ValueFactory; +import com.amazon.ion.impl.bin.IntList; import com.amazon.ion.system.IonReaderBuilder; import com.amazon.ion.impl.bin.utf8.Utf8StringDecoder; import com.amazon.ion.impl.bin.utf8.Utf8StringDecoderPool; @@ -27,7 +28,6 @@ import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.Iterator; @@ -216,7 +216,7 @@ private static class SystemSymbolIDs { private final Utf8StringDecoder utf8Decoder = Utf8StringDecoderPool.getInstance().getOrCreate(); // The symbol IDs for the annotations on the current value. - private final List annotationSids; + private final IntList annotationSids; // True if the annotation iterator will be reused across values; otherwise, false. private final boolean isAnnotationIteratorReuseEnabled; @@ -285,8 +285,8 @@ private static class SystemSymbolIDs { // The buffer position of the first byte of the annotation wrapper for the current value. private int annotationStartPosition = -1; - // The number of bytes occupied by the annotation SIDs in the annotation wrapper for the current value. - private int annotationsLength = -1; + // The buffer position of the byte after the last byte in the annotation wrapper for the current value. + private int annotationEndPosition = -1; // The index of the next byte to peek from the underlying buffer. private int peekIndex = -1; @@ -318,7 +318,7 @@ private static class SystemSymbolIDs { CONTAINER_STACK_INITIAL_CAPACITY, CONTAINER_INFO_FACTORY ); - annotationSids = new ArrayList(ANNOTATIONS_LIST_INITIAL_CAPACITY); + annotationSids = new IntList(ANNOTATIONS_LIST_INITIAL_CAPACITY); symbols = new ArrayList(SYMBOLS_LIST_INITIAL_CAPACITY); scalarConverter = new _Private_ScalarConversions.ValueVariant(); resetImports(); @@ -329,24 +329,25 @@ private static class SystemSymbolIDs { */ private class AnnotationIterator implements Iterator { - // All of the annotation SIDs on the current value. - protected List annotationSids = Collections.emptyList(); - // The index into `annotationSids` containing the next annotation to be returned. - protected int index = 0; + // The byte position of the annotation to return from the next call to next(). + private int nextAnnotationPeekIndex; @Override public boolean hasNext() { - return index < annotationSids.size(); + return nextAnnotationPeekIndex < annotationEndPosition; } @Override public String next() { - int sid = annotationSids.get(index); + int savedPeekIndex = peekIndex; + peekIndex = nextAnnotationPeekIndex; + int sid = readVarUInt(); + nextAnnotationPeekIndex = peekIndex; + peekIndex = savedPeekIndex; String annotation = getSymbol(sid); if (annotation == null) { throw new UnknownSymbolException(sid); } - index++; return annotation; } @@ -356,11 +357,18 @@ public void remove() { } /** - * Reset the iterator so that it may be reused. + * Prepare the iterator to iterate over the annotations on the current value. + */ + void ready() { + nextAnnotationPeekIndex = annotationStartPosition; + } + + /** + * Invalidate the iterator so that all future calls to {@link #hasNext()} will return false until the + * next call to {@link #ready()}. */ - public void reset() { - index = 0; - annotationSids = getAnnotationSids(); + void invalidate() { + nextAnnotationPeekIndex = Integer.MAX_VALUE; } } @@ -368,16 +376,36 @@ public void reset() { * Non-reusable iterator over the annotations on the current value. May be iterated even if the reader advances * past the current value. */ - private class SingleUseAnnotationIterator extends AnnotationIterator { + private class SingleUseAnnotationIterator implements Iterator { + + // All of the annotation SIDs on the current value. + private final IntList annotationSids; + // The index into `annotationSids` containing the next annotation to be returned. + private int index = 0; SingleUseAnnotationIterator() { - index = 0; - annotationSids = new ArrayList(getAnnotationSids()); + annotationSids = new IntList(getAnnotationSids()); } @Override - public void reset() { - throw new IllegalStateException("Single-use annotation iterators cannot be reset."); + public boolean hasNext() { + return index < annotationSids.size(); + } + + @Override + public String next() { + int sid = annotationSids.get(index); + String annotation = getSymbol(sid); + if (annotation == null) { + throw new UnknownSymbolException(sid); + } + index++; + return annotation; + } + + @Override + public void remove() { + throw new UnsupportedOperationException("This iterator does not support element removal."); } } @@ -694,6 +722,16 @@ private void resetSymbolTable() { } } + /** + * Resets the value's annotations. + */ + private void resetAnnotations() { + hasAnnotations = false; + if (isAnnotationIteratorReuseEnabled) { + annotationIterator.invalidate(); + } + } + /** * Clear the list of imported shared symbol tables. */ @@ -801,7 +839,7 @@ private SymbolToken getSymbolToken(int sid) { * Reads a local symbol table from the buffer. * @param marker marker for the start and end positions of the local symbol table in the buffer. */ - private void readSymbolTable(IonReaderLookaheadBuffer.SymbolTableMarker marker) { + private void readSymbolTable(IonReaderLookaheadBuffer.Marker marker) { peekIndex = marker.startIndex; boolean isAppend = false; boolean hasSeenImports = false; @@ -937,22 +975,24 @@ private void nextAtTopLevel() { // not represent valid binary Ion, a quick failure is necessary. throw new IonException("Binary Ion must start with an Ion version marker."); } - List symbolTableMarkers = lookahead.getSymbolTableMarkers(); + List symbolTableMarkers = lookahead.getSymbolTableMarkers(); if (!symbolTableMarkers.isEmpty()) { // The cached SymbolTable (if any) is a snapshot in time, so it must be cleared whenever a new symbol // table is read regardless of whether the new LST is an append or a reset. cachedReadOnlySymbolTable = null; - for (IonReaderLookaheadBuffer.SymbolTableMarker symbolTableMarker : symbolTableMarkers) { + for (IonReaderLookaheadBuffer.Marker symbolTableMarker : symbolTableMarkers) { readSymbolTable(symbolTableMarker); } lookahead.resetSymbolTableMarkers(); } peekIndex = lookahead.getValueStart(); - if (lookahead.getAnnotationSids().isEmpty()) { - valueTypeID = lookahead.getValueTid(); - valueType = valueTypeID.type; - } else { - hasAnnotations = true; + hasAnnotations = lookahead.hasAnnotations(); + if (hasAnnotations) { + annotationSids.clear(); + IonReaderLookaheadBuffer.Marker annotationSidsMarker = lookahead.getAnnotationSidsMarker(); + annotationStartPosition = annotationSidsMarker.startIndex; + annotationEndPosition = annotationSidsMarker.endIndex; + peekIndex = annotationEndPosition; valueTypeID = IonTypeID.TYPE_IDS[buffer.peek(peekIndex++)]; int wrappedValueLength = valueTypeID.length; if (valueTypeID.variableLength) { @@ -965,6 +1005,9 @@ private void nextAtTopLevel() { if (peekIndex + wrappedValueLength != lookahead.getValueEnd()) { throw new IonException("Mismatched annotation wrapper length."); } + } else { + valueTypeID = lookahead.getValueTid(); + valueType = valueTypeID.type; } valueStartPosition = peekIndex; valueEndPosition = lookahead.getValueEnd(); @@ -1005,7 +1048,7 @@ private void endContainer() { valueType = null; valueTypeID = null; annotationStartPosition = -1; - annotationsLength = -1; + annotationEndPosition = -1; hasAnnotations = false; } @@ -1040,9 +1083,10 @@ private void nextBelowTopLevel() { if (valueType == IonTypeID.ION_TYPE_ANNOTATION_WRAPPER) { hasAnnotations = true; annotationSids.clear(); - annotationsLength = readVarUInt(); + int annotationsLength = readVarUInt(); annotationStartPosition = peekIndex; - peekIndex = annotationStartPosition + annotationsLength; + annotationEndPosition = annotationStartPosition + annotationsLength; + peekIndex = annotationEndPosition; typeID = readTypeId(); if (typeID.isNopPad) { throw new IonException( @@ -1061,7 +1105,7 @@ private void nextBelowTopLevel() { } } else { annotationStartPosition = -1; - annotationsLength = -1; + annotationEndPosition = -1; hasAnnotations = false; if (valueEndPosition > containerStack.peek().endPosition) { throw new IonException("Value overflowed its container."); @@ -1079,7 +1123,7 @@ public IonType next() { fieldNameSid = -1; lobBytesRead = 0; valueStartPosition = -1; - hasAnnotations = false; + resetAnnotations(); if (containerStack.isEmpty()) { nextAtTopLevel(); } else { @@ -1701,27 +1745,22 @@ public Timestamp timestampValue() { * Gets the annotation symbol IDs for the current value, reading them from the buffer first if necessary. * @return the annotation symbol IDs, or an empty list if the current value is not annotated. */ - private List getAnnotationSids() { - if (containerStack.isEmpty()) { - return lookahead.getAnnotationSids(); - } else { - if (annotationSids.isEmpty()) { - int savedPeekIndex = peekIndex; - peekIndex = annotationStartPosition; - long annotationsEndPosition = peekIndex + annotationsLength; - while (peekIndex < annotationsEndPosition) { - annotationSids.add(readVarUInt()); - } - peekIndex = savedPeekIndex; + private IntList getAnnotationSids() { + if (annotationSids.isEmpty()) { + int savedPeekIndex = peekIndex; + peekIndex = annotationStartPosition; + while (peekIndex < annotationEndPosition) { + annotationSids.add(readVarUInt()); } - return annotationSids; + peekIndex = savedPeekIndex; } + return annotationSids; } @Override public String[] getTypeAnnotations() { if (hasAnnotations) { - List annotationSids = getAnnotationSids(); + IntList annotationSids = getAnnotationSids(); String[] annotationArray = new String[annotationSids.size()]; for (int i = 0; i < annotationArray.length; i++) { String symbol = getSymbol(annotationSids.get(i)); @@ -1738,7 +1777,7 @@ public String[] getTypeAnnotations() { @Override public SymbolToken[] getTypeAnnotationSymbols() { if (hasAnnotations) { - List annotationSids = getAnnotationSids(); + IntList annotationSids = getAnnotationSids(); SymbolToken[] annotationArray = new SymbolToken[annotationSids.size()]; for (int i = 0; i < annotationArray.length; i++) { annotationArray[i] = getSymbolToken(annotationSids.get(i)); @@ -1770,7 +1809,7 @@ public void remove() { public Iterator iterateTypeAnnotations() { if (hasAnnotations) { if (isAnnotationIteratorReuseEnabled) { - annotationIterator.reset(); + annotationIterator.ready(); return annotationIterator; } else { return new SingleUseAnnotationIterator(); diff --git a/src/com/amazon/ion/impl/IonReaderLookaheadBuffer.java b/src/com/amazon/ion/impl/IonReaderLookaheadBuffer.java index 325daa9990..c510b3af11 100644 --- a/src/com/amazon/ion/impl/IonReaderLookaheadBuffer.java +++ b/src/com/amazon/ion/impl/IonReaderLookaheadBuffer.java @@ -142,24 +142,24 @@ private enum State { } /** - * Holds the start and end indices of a buffered symbol table. + * Holds the start and end indices of a slice of the buffer. */ - static class SymbolTableMarker { + static class Marker { /** - * Index of the first byte of the symbol table struct's contents. + * Index of the first byte in the slice. */ int startIndex; /** - * Index of the first byte after the end of the symbol table. + * Index of the first byte after the end of the slice. */ int endIndex; /** - * @param startIndex index of the first byte of the symbol table struct's contents. - * @param length the number of bytes that remain in the symbol table struct after 'startIndex'. + * @param startIndex index of the first byte in the slice. + * @param length the number of bytes in the slice. */ - private SymbolTableMarker(final int startIndex, final int length) { + private Marker(final int startIndex, final int length) { this.startIndex = startIndex; this.endIndex = startIndex + length; } @@ -178,12 +178,13 @@ private SymbolTableMarker(final int startIndex, final int length) { /** * Markers for any symbol tables that occurred in the stream between the last value and the current value. */ - private final List symbolTableMarkers = new ArrayList(2); + private final List symbolTableMarkers = new ArrayList(2); /** - * The symbol IDs of any annotations on the current value. + * Marker for the sequence of annotation symbol IDs on the current value. If there are no annotations on the + * current value, the startIndex will be negative. */ - private final List annotationSids = new ArrayList(3); + private final Marker annotationSidsMarker = new Marker(-1, 0); /** * The handler that will be notified when a symbol table exceeds the maximum buffer size. @@ -204,12 +205,6 @@ private SymbolTableMarker(final int startIndex, final int length) { */ private boolean isSystemValue; - /** - * True if the current value has an annotation wrapper whose first annotation is `$ion_symbol_table`. - * The value will be deemed a system value if it is later determined to be a struct. - */ - private boolean isSymbolTableAnnotationFirst; - /** * The number of bytes of annotation SIDs left to read from the value's annotation wrapper. */ @@ -284,14 +279,13 @@ private SymbolTableMarker(final int startIndex, final int length) { private void reset() { additionalBytesNeeded = 0; isSystemValue = false; - isSymbolTableAnnotationFirst = false; numberOfAnnotationSidBytesRemaining = 0; currentNumberOfAnnotations = 0; valuePreHeaderIndex = -1; valuePostHeaderIndex = -1; valueTid = null; valueEndIndex = -1; - annotationSids.clear(); + annotationSidsMarker.startIndex = -1; valueStartAvailable = pipe.available(); startNewValue(); } @@ -490,29 +484,21 @@ private void readHeader() throws Exception { additionalBytesNeeded -= inProgressVarUInt.numberOfBytesRead; numberOfAnnotationSidBytesRemaining = inProgressVarUInt.value; initializeVarUInt(VarUInt.Location.ANNOTATION_WRAPPER_SID); + annotationSidsMarker.startIndex = peekIndex; + annotationSidsMarker.endIndex = annotationSidsMarker.startIndex + (int) numberOfAnnotationSidBytesRemaining; } if (inProgressVarUInt.location == VarUInt.Location.ANNOTATION_WRAPPER_SID) { - while (true) { - readVarUInt(); - if (inProgressVarUInt.isComplete) { - currentNumberOfAnnotations++; - if (currentNumberOfAnnotations == 1 && inProgressVarUInt.value == ION_SYMBOL_TABLE_SID) { - isSymbolTableAnnotationFirst = true; - } - annotationSids.add((int) inProgressVarUInt.value); - numberOfAnnotationSidBytesRemaining -= inProgressVarUInt.numberOfBytesRead; - additionalBytesNeeded -= inProgressVarUInt.numberOfBytesRead; - if (numberOfAnnotationSidBytesRemaining <= 0) { - state = State.SKIPPING_VALUE; - } else { - initializeVarUInt(VarUInt.Location.ANNOTATION_WRAPPER_SID); - continue; - } - if (isSymbolTableAnnotationFirst) { - state = State.READING_VALUE_WITH_SYMBOL_TABLE_ANNOTATION; - } + // Read the first annotation SID, which is all that is required to determine whether the value is a + // symbol table. + readVarUInt(); + if (inProgressVarUInt.isComplete) { + numberOfAnnotationSidBytesRemaining -= inProgressVarUInt.numberOfBytesRead; + additionalBytesNeeded -= inProgressVarUInt.numberOfBytesRead; + if (inProgressVarUInt.value == ION_SYMBOL_TABLE_SID) { + state = State.READING_VALUE_WITH_SYMBOL_TABLE_ANNOTATION; + } else { + state = State.SKIPPING_VALUE; } - break; } } } @@ -528,12 +514,16 @@ private void shiftIndicesLeft(int afterIndex, int shiftAmount) { peekIndex = Math.max(peekIndex - shiftAmount, 0); valuePreHeaderIndex -= shiftAmount; valuePostHeaderIndex -= shiftAmount; - for (SymbolTableMarker symbolTableMarker : symbolTableMarkers) { + for (Marker symbolTableMarker : symbolTableMarkers) { if (symbolTableMarker.startIndex > afterIndex) { symbolTableMarker.startIndex -= shiftAmount; symbolTableMarker.endIndex -= shiftAmount; } } + if (annotationSidsMarker.startIndex > afterIndex) { + annotationSidsMarker.startIndex -= shiftAmount; + annotationSidsMarker.endIndex -= shiftAmount; + } if (ivmSecondByteIndex > afterIndex) { ivmSecondByteIndex -= shiftAmount; } @@ -660,6 +650,27 @@ private int fillOrSkip() throws Exception { return bytesFilled; } + /** + * Attempts to skip the requested number of bytes. + * @param numberOfBytesToSkip the number of bytes to attempt to skip. + * @return the number of bytes actually skipped. + * @throws Exception if thrown by the event handler. + */ + private long skip(long numberOfBytesToSkip) throws Exception { + int numberOfBytesSkipped; + if (pipe.availableBeyondBoundary() >= numberOfBytesToSkip) { + numberOfBytesSkipped = (int) numberOfBytesToSkip; + pipe.extendBoundary(numberOfBytesSkipped); + peekIndex += numberOfBytesSkipped; + } else { + numberOfBytesSkipped = fillOrSkip(); + } + if (numberOfBytesSkipped > 0) { + dataHandler.onData(numberOfBytesSkipped); + } + return numberOfBytesSkipped; + } + /* * The state transitions of the fillInput() method are summarized by the following diagram. * @@ -729,6 +740,15 @@ protected void fillInputHelper() throws Exception { valuePostHeaderIndex = peekIndex; } if (state == State.READING_VALUE_WITH_SYMBOL_TABLE_ANNOTATION) { + // Skip annotations until positioned on the value's type ID. + while (numberOfAnnotationSidBytesRemaining > 0) { + long numberOfBytesSkipped = skip(numberOfAnnotationSidBytesRemaining); + if (numberOfBytesSkipped < 1) { + return; + } + numberOfAnnotationSidBytesRemaining -= numberOfBytesSkipped; + additionalBytesNeeded -= numberOfBytesSkipped; + } ReadTypeIdResult result = readTypeID(false); if (result == ReadTypeIdResult.NO_DATA) { return; @@ -750,7 +770,7 @@ protected void fillInputHelper() throws Exception { } additionalBytesNeeded = inProgressVarUInt.value; } - symbolTableMarkers.add(new SymbolTableMarker(peekIndex, (int) additionalBytesNeeded)); + symbolTableMarkers.add(new Marker(peekIndex, (int) additionalBytesNeeded)); state = State.SKIPPING_VALUE; } if (state == State.SKIPPING_VALUE) { @@ -768,19 +788,11 @@ protected void fillInputHelper() throws Exception { // large enough that it doesn't fit within the buffer's configured maximum size. } while (additionalBytesNeeded > 0) { - int numberOfBytesToRead; - if (pipe.availableBeyondBoundary() >= additionalBytesNeeded) { - numberOfBytesToRead = (int) additionalBytesNeeded; - pipe.extendBoundary(numberOfBytesToRead); - peekIndex += numberOfBytesToRead; - } else { - numberOfBytesToRead = fillOrSkip(); - if (numberOfBytesToRead < 1) { - return; - } + long numberOfBytesSkipped = skip(additionalBytesNeeded); + if (numberOfBytesSkipped < 1) { + return; } - dataHandler.onData(numberOfBytesToRead); - additionalBytesNeeded -= numberOfBytesToRead; + additionalBytesNeeded -= numberOfBytesSkipped; } state = State.BEFORE_TYPE_ID; } @@ -880,6 +892,9 @@ void resetNopPadIndex() { * @return the index of the first byte of the value representation (past the type ID and the optional length field). */ int getValueStart() { + if (hasAnnotations()) { + return annotationSidsMarker.endIndex; + } return valuePostHeaderIndex; } @@ -898,9 +913,12 @@ int getValueEnd() { } /** - * @return markers for any symbol tables that occurred in the stream between the last value and the current value. + * Returns markers for any symbol tables that occurred in the stream between the last value and the current value. + * The startIndex of the returned markers is the index of the first byte of the symbol table struct's contents. + * The endIndex of the returned markers is the index of the first byte after the end of the symbol table. + * @return the markers. */ - List getSymbolTableMarkers() { + List getSymbolTableMarkers() { return symbolTableMarkers; } @@ -912,10 +930,19 @@ void resetSymbolTableMarkers() { } /** - * @return the symbol IDs of any annotations on the current value. + * @return true if the current value has annotations; otherwise, false. + */ + boolean hasAnnotations() { + return annotationSidsMarker.startIndex >= 0; + } + /** + * Returns the marker for the sequence of annotation symbol IDs on the current value. The startIndex of the + * returned marker is the index of the first byte of the first annotation symbol ID in the sequence. The endIndex + * of the returned marker is the index of the type ID byte of the value to which the annotations are applied. + * @return the marker. */ - List getAnnotationSids() { - return annotationSids; + Marker getAnnotationSidsMarker() { + return annotationSidsMarker; } } diff --git a/src/com/amazon/ion/impl/bin/IntList.java b/src/com/amazon/ion/impl/bin/IntList.java index 8c0ba5c214..3ff039cc44 100644 --- a/src/com/amazon/ion/impl/bin/IntList.java +++ b/src/com/amazon/ion/impl/bin/IntList.java @@ -29,6 +29,16 @@ public IntList(final int initialCapacity) { numberOfValues = 0; } + /** + * Constructs a new IntList that contains all the elements of the given IntList. + * @param other the IntList to copy. + */ + public IntList(final IntList other) { + this.numberOfValues = other.numberOfValues; + this.data = new int[other.data.length]; + System.arraycopy(other.data, 0, this.data, 0, numberOfValues); + } + /** * Accessor. * @return The number of ints currently stored in the list. diff --git a/test/com/amazon/ion/impl/IonReaderLookaheadBufferTest.java b/test/com/amazon/ion/impl/IonReaderLookaheadBufferTest.java index d560275075..bac28697eb 100644 --- a/test/com/amazon/ion/impl/IonReaderLookaheadBufferTest.java +++ b/test/com/amazon/ion/impl/IonReaderLookaheadBufferTest.java @@ -1000,7 +1000,7 @@ public void valueMarkersAreSet() throws Exception { lookahead.fillInput(); assertEquals(1, lookahead.getIvmIndex()); - assertTrue(lookahead.getAnnotationSids().isEmpty()); + assertFalse(lookahead.hasAnnotations()); assertTrue(lookahead.getSymbolTableMarkers().isEmpty()); assertEquals(5, lookahead.getValueStart()); assertIonTypeId(IonType.INT, 0x1, lookahead.getValueTid()); @@ -1014,7 +1014,7 @@ public void valueMarkersAreSet() throws Exception { assertEquals(indexForInitialBufferSize(1, 1, 7), lookahead.getIvmIndex()); lookahead.resetIvmIndex(); assertEquals(-1, lookahead.getIvmIndex()); - assertTrue(lookahead.getAnnotationSids().isEmpty()); + assertFalse(lookahead.hasAnnotations()); assertTrue(lookahead.getSymbolTableMarkers().isEmpty()); assertEquals(indexForInitialBufferSize(6, 6, 12), lookahead.getValueStart()); assertIonTypeId(IonType.STRUCT, 0x1, lookahead.getValueTid()); @@ -1023,7 +1023,9 @@ public void valueMarkersAreSet() throws Exception { lookahead.fillInput(); assertEquals(-1, lookahead.getIvmIndex()); assertTrue(lookahead.getSymbolTableMarkers().isEmpty()); - assertEquals(Arrays.asList(4, 5), lookahead.getAnnotationSids()); + assertTrue(lookahead.hasAnnotations()); + assertEquals(indexForInitialBufferSize(10, 10, 16), lookahead.getAnnotationSidsMarker().startIndex); + assertEquals(indexForInitialBufferSize(12, 12, 18), lookahead.getAnnotationSidsMarker().endIndex); assertEquals(indexForInitialBufferSize(12, 12, 18), lookahead.getValueStart()); assertIonTypeId(IonTypeID.ION_TYPE_ANNOTATION_WRAPPER, 0x5, lookahead.getValueTid()); assertEquals(indexForInitialBufferSize(14, 14, 20), lookahead.getValueEnd()); @@ -1031,7 +1033,7 @@ public void valueMarkersAreSet() throws Exception { lookahead.fillInput(); assertEquals(-1, lookahead.getIvmIndex()); assertTrue(lookahead.getSymbolTableMarkers().isEmpty()); - assertTrue(lookahead.getAnnotationSids().isEmpty()); + assertFalse(lookahead.hasAnnotations()); assertEquals(indexForInitialBufferSize(15, 15, 21), lookahead.getValueStart()); assertIonTypeId(IonType.INT, 0x0, lookahead.getValueTid()); assertEquals(indexForInitialBufferSize(15, 15, 21), lookahead.getValueEnd()); @@ -1060,8 +1062,8 @@ public void symbolTableMarkersAreSet() throws Exception { builder.withInitialBufferSize(initialBufferSize); } lookahead = bufferFor( - // Symbol table declaring symbol 'x'. - 0xE7, 0x81, 0x83, 0xD4, 0x87, 0xB2, 0x81, 'x', + // Symbol table with extra annotation 'name' declaring symbol 'x'. + 0xE8, 0x82, 0x83, 0x84, 0xD4, 0x87, 0xB2, 0x81, 'x', // Symbol value with SID 10 ('x'). 0x71, 0x0A, // The value will not be consumed, so the indices continue. @@ -1087,25 +1089,25 @@ public void symbolTableMarkersAreSet() throws Exception { ); lookahead.fillInput(); assertEquals(1, lookahead.getIvmIndex()); - assertTrue(lookahead.getAnnotationSids().isEmpty()); - List symbolTableMarkers = lookahead.getSymbolTableMarkers(); + assertFalse(lookahead.hasAnnotations()); + List symbolTableMarkers = lookahead.getSymbolTableMarkers(); assertEquals(1, symbolTableMarkers.size()); - assertEquals(8, symbolTableMarkers.get(0).startIndex); - assertEquals(12, symbolTableMarkers.get(0).endIndex); - assertEquals(13, lookahead.getValueStart()); + assertEquals(9, symbolTableMarkers.get(0).startIndex); + assertEquals(13, symbolTableMarkers.get(0).endIndex); + assertEquals(14, lookahead.getValueStart()); assertIonTypeId(IonType.SYMBOL, 0x1, lookahead.getValueTid()); - assertEquals(14, lookahead.getValueEnd()); + assertEquals(15, lookahead.getValueEnd()); lookahead.resetSymbolTableMarkers(); assertTrue(lookahead.getSymbolTableMarkers().isEmpty()); lookahead.fillInput(); symbolTableMarkers = lookahead.getSymbolTableMarkers(); assertEquals(1, symbolTableMarkers.size()); - assertEquals(18, symbolTableMarkers.get(0).startIndex); - assertEquals(25, symbolTableMarkers.get(0).endIndex); - assertEquals(26, lookahead.getValueStart()); + assertEquals(19, symbolTableMarkers.get(0).startIndex); + assertEquals(26, symbolTableMarkers.get(0).endIndex); + assertEquals(27, lookahead.getValueStart()); assertIonTypeId(IonType.SYMBOL, 0x1, lookahead.getValueTid()); - assertEquals(27, lookahead.getValueEnd()); + assertEquals(28, lookahead.getValueEnd()); IonReader reader = lookahead.newIonReader(IonReaderBuilder.standard()); assertEquals(IonType.SYMBOL, reader.next()); @@ -1116,13 +1118,13 @@ public void symbolTableMarkersAreSet() throws Exception { lookahead.fillInput(); symbolTableMarkers = lookahead.getSymbolTableMarkers(); assertEquals(2, symbolTableMarkers.size()); - assertEquals(indexForInitialBufferSize(28, 28, 55), symbolTableMarkers.get(0).startIndex); - assertEquals(indexForInitialBufferSize(32, 32, 59), symbolTableMarkers.get(0).endIndex); - assertEquals(indexForInitialBufferSize(36, 36, 63), symbolTableMarkers.get(1).startIndex); - assertEquals(indexForInitialBufferSize(40, 40, 67), symbolTableMarkers.get(1).endIndex); - assertEquals(indexForInitialBufferSize(41, 41, 68), lookahead.getValueStart()); + assertEquals(indexForInitialBufferSize(28, 28, 56), symbolTableMarkers.get(0).startIndex); + assertEquals(indexForInitialBufferSize(32, 32, 60), symbolTableMarkers.get(0).endIndex); + assertEquals(indexForInitialBufferSize(36, 36, 64), symbolTableMarkers.get(1).startIndex); + assertEquals(indexForInitialBufferSize(40, 40, 68), symbolTableMarkers.get(1).endIndex); + assertEquals(indexForInitialBufferSize(41, 41, 69), lookahead.getValueStart()); assertIonTypeId(IonType.SYMBOL, 0x1, lookahead.getValueTid()); - assertEquals(indexForInitialBufferSize(42, 42, 69), lookahead.getValueEnd()); + assertEquals(indexForInitialBufferSize(42, 42, 70), lookahead.getValueEnd()); assertEquals(IonType.SYMBOL, reader.next()); assertEquals("d", reader.stringValue()); @@ -1194,4 +1196,42 @@ public void nopPadsInterspersedWithSystemValuesDoNotCauseOversizedErrors() throw assertNull(reader.next()); reader.close(); } + + @Test + public void annotationMarkersAreCorrectlyShifted() throws Exception { + if (initialBufferSize == null || initialBufferSize != 1) { + return; + } + // Set the maximum size at 1 IVM (4 bytes) + the symbol table (12 bytes) + the value (2 bytes) + part of the + // next value (3 bytes). + builder = IonBufferConfiguration.Builder.standard() + .withInitialBufferSize(21) + .withMaximumBufferSize(21); + createCountingEventHandler(builder, new AtomicLong()); + lookahead = bufferFor( + // Symbol table with the symbol 'hello'. + 0xEB, 0x81, 0x83, 0xD8, 0x87, 0xB6, 0x85, 'h', 'e', 'l', 'l', 'o', + // Symbol 10 (hello) + 0x71, 0x0A, + // name::hello + 0xE4, 0x81, 0x84, 0x71, 0x0A + ); + + lookahead.fillInput(); + assertFalse(lookahead.moreDataRequired()); + IonReader reader = lookahead.newIonReader(IonReaderBuilder.standard()); + assertEquals(IonType.SYMBOL, reader.next()); + assertFalse(lookahead.hasAnnotations()); + // The buffer will reach its maximum size after recording the annotation marker. The marker's indices will + // be shifted left when space is reclaimed to fit the value. + lookahead.fillInput(); + assertFalse(lookahead.moreDataRequired()); + assertEquals("hello", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertTrue(lookahead.hasAnnotations()); + assertEquals(2, lookahead.getAnnotationSidsMarker().startIndex); + assertEquals(3, lookahead.getAnnotationSidsMarker().endIndex); + assertNull(reader.next()); + reader.close(); + } } From 01f813a54c57fd03a892990ea11d2c7c6a920d1f Mon Sep 17 00:00:00 2001 From: Matthew Pope <81593196+popematt@users.noreply.github.com> Date: Mon, 31 Jan 2022 16:55:47 -0800 Subject: [PATCH 153/490] Adds CodeQL analysis --- .github/workflows/codeql-analysis.yml | 66 +++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000000..6364966701 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,66 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +name: "CodeQL" + +on: + push: + branches: [ master ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] + schedule: + - cron: '0 22 * * 3' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'java' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # â„šī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # âœī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From dcaa4a36bca914f329b4d8d568d7f93b7c0740e0 Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Tue, 1 Feb 2022 13:29:23 -0800 Subject: [PATCH 154/490] Adds explicit narrowing casts in Timestamp.apply_offset --- src/com/amazon/ion/Timestamp.java | 32 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/com/amazon/ion/Timestamp.java b/src/com/amazon/ion/Timestamp.java index 591d865222..0480090eae 100644 --- a/src/com/amazon/ion/Timestamp.java +++ b/src/com/amazon/ion/Timestamp.java @@ -216,7 +216,7 @@ private boolean alwaysUnknownOffset() private static final int[] LEAP_DAYS_IN_MONTH = { 0, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; private static final int[] NORMAL_DAYS_IN_MONTH = { 0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; - private static int last_day_in_month(int year, int month) { + private static byte last_day_in_month(int year, int month) { boolean is_leap; if ((year % 4) == 0) { // divisible by 4 (lower 2 bits are zero) - may be a leap year @@ -237,7 +237,7 @@ private static int last_day_in_month(int year, int month) { else { is_leap = false; } - return is_leap ? LEAP_DAYS_IN_MONTH[month] : NORMAL_DAYS_IN_MONTH[month]; + return (byte) (is_leap ? LEAP_DAYS_IN_MONTH[month] : NORMAL_DAYS_IN_MONTH[month]); } /** @@ -256,19 +256,19 @@ private void apply_offset(int offset) } // To convert _to_ UTC you must SUBTRACT the local offset offset = -offset; - int hour_offset = offset / 60; - int min_offset = offset - (hour_offset * 60); + byte hour_offset = (byte) (offset / 60); + byte min_offset = (byte) (offset - (hour_offset * 60)); if (offset < 0) { _minute += min_offset; // lower the minute value by adding a negative offset _hour += hour_offset; if (_minute < 0) { - _minute += 60; + _minute += (byte) 60; _hour -= 1; } if (_hour >= 0) return; // hour is 0-23 - _hour += 24; - _day -= 1; + _hour += (byte) 24; + _day -= (byte) 1; if (_day >= 1) return; // day is 1-31 // we can't do this until we've figured out the month and year: _day += last_day_in_month(_year, _month); _month -= 1; @@ -277,8 +277,8 @@ private void apply_offset(int offset) assert(_day == last_day_in_month(_year, _month)); return; // 1-12 } - _month += 12; - _year -= 1; + _month += (byte) 12; + _year -= (short) 1; if (_year < 1) throw new IllegalArgumentException("year is less than 1"); _day += last_day_in_month(_year, _month); // and now we know, even if the year did change assert(_day == last_day_in_month(_year, _month)); @@ -287,21 +287,21 @@ private void apply_offset(int offset) _minute += min_offset; // lower the minute value by adding a negative offset _hour += hour_offset; if (_minute > 59) { - _minute -= 60; - _hour += 1; + _minute -= (byte) 60; + _hour += (byte) 1; } if (_hour < 24) return; // hour is 0-23 - _hour -= 24; - _day += 1; + _hour -= (byte) 24; + _day += (byte) 1; if (_day <= last_day_in_month(_year, _month)) return; // day is 1-31 // we can't do this until we figure out the final month and year: _day -= last_day_in_month(_year, _month); _day = 1; // this is always the case - _month += 1; + _month += (byte) 1; if (_month <= 12) { return; // 1-12 } - _month -= 12; - _year += 1; + _month -= (byte) 12; + _year += (short) 1; if (_year > 9999) throw new IllegalArgumentException("year exceeds 9999"); } } From e7efcbf1652681a40d14f0f725d055e043aed654 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Mon, 31 Jan 2022 18:41:39 -0800 Subject: [PATCH 155/490] Properly sizes the buffer when a fixed-size stream is provided to the incremental reader with default configuration. --- .../amazon/ion/IonBufferConfiguration.java | 15 +++++ .../ion/impl/IonReaderBinaryIncremental.java | 61 +++++++++++++++++-- 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/src/com/amazon/ion/IonBufferConfiguration.java b/src/com/amazon/ion/IonBufferConfiguration.java index 960ff4a074..9845b9d501 100644 --- a/src/com/amazon/ion/IonBufferConfiguration.java +++ b/src/com/amazon/ion/IonBufferConfiguration.java @@ -89,6 +89,21 @@ public static Builder standard() { return new Builder(); } + /** + * Provides a new builder that would build IonBufferConfiguration instances with configuration identical to + * the given configuration. + * @param existingConfiguration an existing configuration. + * @return a new mutable builder. + */ + public static Builder from(IonBufferConfiguration existingConfiguration) { + return IonBufferConfiguration.Builder.standard() + .onData(existingConfiguration.getDataHandler()) + .onOversizedValue(existingConfiguration.getOversizedValueHandler()) + .onOversizedSymbolTable(existingConfiguration.getOversizedSymbolTableHandler()) + .withInitialBufferSize(existingConfiguration.getInitialBufferSize()) + .withMaximumBufferSize(existingConfiguration.getMaximumBufferSize()); + } + /** * Sets the handler that will be notified when oversized symbol tables are encountered. If the maximum buffer * size is finite (see {@link #withMaximumBufferSize(int)}, this handler is required to be non-null. diff --git a/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java b/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java index 22d774173b..9cf786a823 100644 --- a/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java +++ b/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java @@ -21,6 +21,7 @@ import com.amazon.ion.impl.bin.utf8.Utf8StringDecoderPool; import com.amazon.ion.system.SimpleCatalog; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.math.BigDecimal; @@ -140,6 +141,36 @@ private static class SystemSymbolIDs { private static final int MAX_ID_ID = 8; } + /** + * @param value a non-negative number. + * @return the exponent of the next power of two greater than the given number. + */ + private static int logBase2(int value) { + return 32 - Integer.numberOfLeadingZeros(value == 0 ? 0 : value - 1); + } + + /** + * Cache of configurations for fixed-sized streams. FIXED_SIZE_CONFIGURATIONS[i] returns a configuration with + * buffer size max(8, 2^i). Retrieve a configuration large enough for a given size using + * FIXED_SIZE_CONFIGURATIONS(logBase2(size)). Only supports sizes less than or equal to + * STANDARD_BUFFER_CONFIGURATION.getInitialBufferSize(). + */ + private static final IonBufferConfiguration[] FIXED_SIZE_CONFIGURATIONS; + + static { + int maxBufferSizeExponent = logBase2(STANDARD_BUFFER_CONFIGURATION.getInitialBufferSize()); + FIXED_SIZE_CONFIGURATIONS = new IonBufferConfiguration[maxBufferSizeExponent + 1]; + for (int i = 0; i <= maxBufferSizeExponent; i++) { + // Create a buffer configuration for buffers of size 2^i. The minimum size is 8: the smallest power of two + // larger than the minimum buffer size allowed. + int size = Math.max(8, (int) Math.pow(2, i)); + FIXED_SIZE_CONFIGURATIONS[i] = IonBufferConfiguration.Builder.from(STANDARD_BUFFER_CONFIGURATION) + .withInitialBufferSize(size) + .withMaximumBufferSize(size) + .build(); + } + } + // The final byte of the binary IVM. private static final int IVM_FINAL_BYTE = 0xEA; @@ -308,11 +339,27 @@ private static class SystemSymbolIDs { isAnnotationIteratorReuseEnabled = false; annotationIterator = null; } - if (builder.getBufferConfiguration() == null) { - lookahead = new IonReaderLookaheadBuffer(STANDARD_BUFFER_CONFIGURATION, inputStream); - } else { - lookahead = new IonReaderLookaheadBuffer(builder.getBufferConfiguration(), inputStream); + IonBufferConfiguration configuration = builder.getBufferConfiguration(); + if (configuration == null) { + configuration = STANDARD_BUFFER_CONFIGURATION; + if (inputStream instanceof ByteArrayInputStream) { + // ByteArrayInputStreams are fixed-size streams. Clamp the reader's internal buffer size at the size of + // the stream to avoid wastefully allocating extra space that will never be needed. It is still + // preferable for the user to manually specify the buffer size if it's less than the default, as doing + // so allows this branch to be skipped. + int fixedBufferSize; + try { + fixedBufferSize = inputStream.available(); + } catch (IOException e) { + // ByteArrayInputStream.available() does not throw. + throw new IllegalStateException(e); + } + if (configuration.getInitialBufferSize() > fixedBufferSize) { + configuration = FIXED_SIZE_CONFIGURATIONS[logBase2(fixedBufferSize)]; + } + } } + lookahead = new IonReaderLookaheadBuffer(configuration, inputStream); buffer = (ResizingPipedInputStream) lookahead.getPipe(); containerStack = new _Private_RecyclingStack( CONTAINER_STACK_INITIAL_CAPACITY, @@ -988,6 +1035,9 @@ private void nextAtTopLevel() { peekIndex = lookahead.getValueStart(); hasAnnotations = lookahead.hasAnnotations(); if (hasAnnotations) { + if (peekIndex >= lookahead.getValueEnd()) { + throw new IonException("Annotation wrappers without values are invalid."); + } annotationSids.clear(); IonReaderLookaheadBuffer.Marker annotationSidsMarker = lookahead.getAnnotationSidsMarker(); annotationStartPosition = annotationSidsMarker.startIndex; @@ -1020,6 +1070,9 @@ private void nextAtTopLevel() { */ private IonTypeID readTypeId() { valueTypeID = IonTypeID.TYPE_IDS[buffer.peek(peekIndex++)]; + if (!valueTypeID.isValid) { + throw new IonException("Invalid type ID."); + } valueType = valueTypeID.type; return valueTypeID; } From 521961bca3f80c00b81bf8ad27fd25a1685ff35e Mon Sep 17 00:00:00 2001 From: jobarr-amzn <70981087+jobarr-amzn@users.noreply.github.com> Date: Tue, 15 Feb 2022 15:00:21 -0800 Subject: [PATCH 156/490] Prepares version 1.9.1 for release. (#413) --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c5a1c7a6b9..e025fbfd54 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ dependency into your project's `pom.xml`: com.amazon.ion ion-java - 1.9.0 + 1.9.1 ``` diff --git a/pom.xml b/pom.xml index 978df8b635..4b208eefe5 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.9.1-SNAPSHOT + 1.9.1 bundle ${project.groupId}:${project.artifactId} From 3efa93a44479890e14e48501160a69b53a872af4 Mon Sep 17 00:00:00 2001 From: jobarr-amzn <70981087+jobarr-amzn@users.noreply.github.com> Date: Wed, 16 Feb 2022 08:18:18 -0800 Subject: [PATCH 157/490] Bumps version to 1.9.2-SNAPSHOT (#414) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4b208eefe5..cddd77cb0c 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.9.1 + 1.9.2-SNAPSHOT bundle ${project.groupId}:${project.artifactId} From af464e44f3b4828cd57c062d9f423e1aa58c5425 Mon Sep 17 00:00:00 2001 From: Zack Slayton Date: Thu, 3 Mar 2022 10:25:07 -0500 Subject: [PATCH 158/490] Fixes a bug in WriteBuffer#shiftBytesLeft (#416) This patch fixes a bug in the `WriteBuffer` that caused the `shiftBytesLeft` method to corrupt data if it was called such that the final `Block` in the list was emptied AND the second-to-last block reclaimed some capacity. This would only occur if the writer was configured to use length preallocation greater than 0 and if they were writing more than 32KB of Ion (the default block size) between flushes. This change accounts for that case and, if the last `Block` is emptied, returns it to the pool for reuse. --- src/com/amazon/ion/impl/bin/WriteBuffer.java | 121 ++++++++++-------- .../amazon/ion/impl/bin/WriteBufferTest.java | 114 +++++++++++++++++ 2 files changed, 180 insertions(+), 55 deletions(-) diff --git a/src/com/amazon/ion/impl/bin/WriteBuffer.java b/src/com/amazon/ion/impl/bin/WriteBuffer.java index 0df5ba6895..c4550dc398 100644 --- a/src/com/amazon/ion/impl/bin/WriteBuffer.java +++ b/src/com/amazon/ion/impl/bin/WriteBuffer.java @@ -180,7 +180,7 @@ public void writeBytes(final byte[] bytes, final int off, final int len) * @param shiftBy The number of bytes to the left that we'll be shifting. */ public void shiftBytesLeft(int length, int shiftBy) { - if (shiftBy == 0) { + if (length == 0 || shiftBy == 0) { // Nothing to do. return; } @@ -227,63 +227,74 @@ private void shiftBytesLeftWithinASingleBlock(int length, int shiftBy) { * @param shiftBy The number of bytes to the left that we'll be shifting. */ private void shiftBytesLeftAcrossBlocks(int length, int shiftBy) { - // Our starting position is the first byte that we plan to shift backwards. The `bufferOffset` is the cursor's - // current position in the stream; we can use this to derive which block it's in as well as its position within - // that block. - long bufferOffset = position() - length; - - Block block = null; - while (length > 0) - { - // Using the `bufferOffset`, determine which block we're in... - int blockIndex = index(bufferOffset); - block = blocks.get(blockIndex); - // ...and our offset within that block. - int blockOffset = offset(bufferOffset); - - // If the block offset is within `shiftBy` bytes of beginning of the block, some bytes we're shifting - // will end up in the previous block. Here's an illustrated example: - // - // shiftBy = 2, blockIndex = 1, blockOffset = 1 - // v---- Cursor is here - // Before: [A B C D E] [F G H I J] - // After : [A B C D G] [F G H I J] - // Now this is G --^ ^-- And the cursor is here, ready to continue copy the rest. - if (blockOffset < shiftBy) { - Block previousBlock = blocks.get(blockIndex - 1); - int numberOfBytesToShift = Math.min(length, shiftBy) - blockOffset; - System.arraycopy(block.data, blockOffset, previousBlock.data, previousBlock.data.length - numberOfBytesToShift, numberOfBytesToShift); - - // Now that we've shifted some bytes, update our position within the buffer. - bufferOffset += numberOfBytesToShift; - length -= numberOfBytesToShift; - - // If there are no more bytes to shift... - if (length == 0) { - // ...lower the `limit` because we've reclaimed some bytes in this block... - block.limit -= numberOfBytesToShift; - // ...and early return. - return; - } - // Otherwise, use our new buffer offset to recalculate our block-specific position. - blockIndex = index(bufferOffset); - block = blocks.get(blockIndex); - blockOffset = offset(bufferOffset); - } - - // At this point, the block offset is at least `shiftBy` bytes away from the beginning of the block, so - // we can do a memcpy to shift the rest of the bytes in the block. - int numberOfBytesToShift = Math.min(length, block.data.length - blockOffset); - System.arraycopy(block.data, blockOffset, block.data, blockOffset - shiftBy, numberOfBytesToShift); - - // Update our counters and see if there are any more bytes to shift. - bufferOffset += numberOfBytesToShift; + // In this method, "buffer offsets" are absolute indexes into the WriteBuffer and + // "block offsets" are indexes that are relative to the beginning of the current Block. + + // The first buffer offset that does not yet contain data. + long position = position(); + // This is the buffer offset of the first byte that we will be shifting backwards. + long sourceBufferOffset = position - length; + // When we're done, this will be the first offset in the buffer that does not contain data. + long writeBufferLimit = position - shiftBy; + + while (length > 0) { + // Convert the source buffer offset into a (Block, block offset) pair. + int sourceBlockIndex = index(sourceBufferOffset); + Block sourceBlock = blocks.get(sourceBlockIndex); + int sourceBlockOffset = offset(sourceBufferOffset); + + // Convert the destination buffer offset into a (Block, block offset) pair. + // Because buffer offsets are absolute, the `destinationBufferOffset` in each loop iteration is + // `shiftBy` positions behind the `sourceBufferOffset`. + long destinationBufferOffset = sourceBufferOffset - shiftBy; + int destinationBlockIndex = index(destinationBufferOffset); + Block destinationBlock = blocks.get(destinationBlockIndex); + int destinationBlockOffset = offset(destinationBufferOffset); + + // Determine how many bytes are left in the source and destination blocks following their respective + // block offsets. + int bytesLeftInSourceBlock = sourceBlock.limit - sourceBlockOffset; + int bytesLeftInDestinationBlock = destinationBlock.limit - destinationBlockOffset; + // Whichever block has fewer bytes remaining will determine how many bytes we consider to be + // available for shifting in this pass. + int bytesAvailableToCopy = Math.min(bytesLeftInSourceBlock, bytesLeftInDestinationBlock); + + // If there are more bytes available than we need to finish the shifting operation, take `length` instead. + int numberOfBytesToShift = Math.min(length, bytesAvailableToCopy); + + // Copy the bytes from the source to the destination. + System.arraycopy( + sourceBlock.data, + sourceBlockOffset, + destinationBlock.data, + destinationBlockOffset, + numberOfBytesToShift + ); + + // Update our record of how many bytes to shift remain... length -= numberOfBytesToShift; + // ...and from which point we should resume in the next iteration. + sourceBufferOffset += numberOfBytesToShift; } - if (block != null) { - // We've reclaimed some space in this block; lower the `limit` accordingly. - block.limit -= shiftBy; + + // At this point, the shifting is complete. However, we have reclaimed some amount of space in the WriteBuffer. + // Using the `writeBufferLimit` we calculated at the beginning of the method, find the last Block that still + // contains data. + int lastBlockIndex = index(writeBufferLimit); + Block lastBlock = blocks.get(lastBlockIndex); + int lastBlockOffset = offset(writeBufferLimit); + + // Update that Block's limit... + lastBlock.limit = lastBlockOffset; + // ...and return any empty blocks at the tail of the `blocks` list to the pool. + for (int m = blocks.size() - 1; m > lastBlockIndex; m--) { + Block emptyBlock = blocks.remove(m); + emptyBlock.close(); } + + // Update the WriteBuffer's member fields to reflect the changes we've made. + current = lastBlock; + index = lastBlockIndex; } /** Writes an array of bytes to the buffer expanding if necessary, defaulting to the entire array. */ diff --git a/test/com/amazon/ion/impl/bin/WriteBufferTest.java b/test/com/amazon/ion/impl/bin/WriteBufferTest.java index 24822820a2..babf113abc 100644 --- a/test/com/amazon/ion/impl/bin/WriteBufferTest.java +++ b/test/com/amazon/ion/impl/bin/WriteBufferTest.java @@ -959,4 +959,118 @@ public void shiftBytesLeftAcrossBufferBlocksExclusively() throws IOException { buf.shiftBytesLeft(5, 5); assertBuffer("012345BCDEF".getBytes()); } + + @Test + public void shiftBytesLeftAcrossBufferBlocksEmptyingLastBlock() throws IOException { + assertEquals(11, ALLOCATOR.getBlockSize()); + // The "B" is the first and only byte in the second block. + // Shifting 1 byte left by one empties the last block. + buf.writeBytes("0123456789AB".getBytes()); + buf.shiftBytesLeft(1, 1); + assertBuffer("0123456789B".getBytes()); + } + + @Test + public void shiftEntireBlock() throws IOException { + assertEquals(11, ALLOCATOR.getBlockSize()); + // The buffer contains two full blocks + buf.writeBytes("0123456789|ABCDEFGHIJ|".getBytes()); + // We shift an entire block left by the block size + buf.shiftBytesLeft(11, 11); + assertBuffer("ABCDEFGHIJ|".getBytes()); + } + + @Test + public void shiftBytesLeftByMoreThanTheBlockSize() { + assertEquals(11, ALLOCATOR.getBlockSize()); + // We have 5 blocks' worth of data + buf.writeBytes("0123456789|0123456789|0123456789|0123456789|0123456789|".getBytes()); + // We can shift left amounts greater than the block size + buf.shiftBytesLeft(24, 24); + assertBuffer("01234569|0123456789|0123456789|".getBytes()); + } + + @Test + public void shiftBytesLeftAcrossBufferBlocksShorteningNextToLastBlock() throws IOException { + assertEquals(11, ALLOCATOR.getBlockSize()); + // The "B" is the first and only byte in the second block. + // Shifting 2 bytes left by two empties the last block and shortens the next-to-last block by 1 byte. + // Following this operation, the next-to-last block becomes the last block. + buf.writeBytes("0123456789AB".getBytes()); + buf.shiftBytesLeft(2, 2); + assertBuffer("01234567AB".getBytes()); + } + + @Test + public void writingAfterLastBlockChanges() throws IOException { + assertEquals(11, ALLOCATOR.getBlockSize()); + // The "B" is the first and only byte in the second block. + // Shifting 2 bytes left by two empties the last block and shortens the next-to-last block by 1 byte. + // Following this operation, the next-to-last block becomes the last block. + buf.writeBytes("0123456789AB".getBytes()); + buf.shiftBytesLeft(2, 2); + assertBuffer("01234567AB".getBytes()); + // After shifting and changing the last block, we can still append data without issue. + buf.writeBytes("CDE".getBytes()); + assertBuffer("01234567ABCDE".getBytes()); + } + + @Test + public void updateLastBlock() throws IOException { + assertEquals(11, ALLOCATOR.getBlockSize()); + // The "B" is the first and only byte in the second block. + // Shifting 2 bytes left by two empties the last block and shortens the next-to-last block by 1 byte. + // Following this operation, the next-to-last block becomes the last block. + buf.writeBytes("0123456789AB".getBytes()); + buf.shiftBytesLeft(2, 2); + assertBuffer("01234567AB".getBytes()); + // We write some more data to the buffer. If the last block has been discarded correctly, we can do another + // shift operation without getting corrupt data. + buf.writeBytes("CDEFGH".getBytes()); + assertBuffer("01234567ABCDEFGH".getBytes()); + buf.shiftBytesLeft(4, 2); + assertBuffer("01234567ABEFGH".getBytes()); + } + + @Test + public void shiftNBytesLeftByMoreThanNBytes() { + assertEquals(11, ALLOCATOR.getBlockSize()); + buf.writeBytes("0123456789AB".getBytes()); + // Shift left by more bytes than we're shifting (shiftBy > length) + buf.shiftBytesLeft(2, 5); + assertBuffer("01234AB".getBytes()); + } + + @Test + public void shiftLeftToBeginning() { + assertEquals(11, ALLOCATOR.getBlockSize()); + buf.writeBytes("01234567AB".getBytes()); + buf.shiftBytesLeft(2, 8); + assertBuffer("AB".getBytes()); + } + + @Test + public void shiftLeftToBeginningAcrossBlocks() { + assertEquals(11, ALLOCATOR.getBlockSize()); + buf.writeBytes("0123456789AB".getBytes()); + buf.shiftBytesLeft(2, 10); + assertBuffer("AB".getBytes()); + } + + @Test + public void shiftBytesLeftAcrossBufferBlocksToStartThenWritePastBlockBoundary() throws IOException { + assertEquals(11, ALLOCATOR.getBlockSize()); + // The "B" is the first and only byte in the second block. + // Shifting 2 bytes left by 10 empties the last block and positions the shifted bytes at the start of the first + // block. + buf.writeBytes("0123456789AB".getBytes()); + buf.shiftBytesLeft(2, 10); + assertBuffer("AB".getBytes()); + // Write enough bytes to expand back into the next block. + buf.writeBytes("CDE0123456789AB".getBytes()); + assertBuffer("ABCDE0123456789AB".getBytes()); + // Shift one byte left past the block boundary. This should once again empty the last block. + buf.shiftBytesLeft(1, 6); + assertBuffer("ABCDE01234B".getBytes()); + } } From ee0832d20906d4a63ebb598d709e7c5b1f0300a2 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Thu, 3 Mar 2022 21:28:47 -0800 Subject: [PATCH 159/490] Increases the non-incremental binary reader's container length limit from 2GB to 72PB. --- .../amazon/ion/impl/IonReaderBinaryRawX.java | 134 +++++--- .../ion/impl/IonReaderBinarySystemX.java | 14 +- .../ion/impl/IonReaderLookaheadBuffer.java | 38 ++- .../IonReaderBinaryRawLargeStreamTest.java | 315 +++++++++++++++++- .../amazon/ion/util/RepeatInputStream.java | 2 +- 5 files changed, 434 insertions(+), 69 deletions(-) diff --git a/src/com/amazon/ion/impl/IonReaderBinaryRawX.java b/src/com/amazon/ion/impl/IonReaderBinaryRawX.java index 3c3494c678..a1585d6ed5 100644 --- a/src/com/amazon/ion/impl/IonReaderBinaryRawX.java +++ b/src/com/amazon/ion/impl/IonReaderBinaryRawX.java @@ -69,7 +69,7 @@ protected enum State { } State _state; UnifiedInputStreamX _input; - int _local_remaining; + long _local_remaining; boolean _eof; boolean _has_next_needed; ValueVariant _v; @@ -83,7 +83,7 @@ protected enum State { */ int _value_field_id; int _value_tid; - int _value_len; + long _value_len; long _value_start; int _value_lob_remaining; boolean _value_lob_is_ready; @@ -169,11 +169,14 @@ public void close() static private final int POS_OFFSET = 0; static private final int TYPE_LIMIT_OFFSET = 1; - static private final long TYPE_MASK = 0xffffffff; - static private final int LIMIT_SHIFT = 32; + // All type IDs can fit in one byte. + static private final long TYPE_MASK = 0xff; + // The limit is restricted to 7 bytes (see readVarUIntOrEOF(boolean longAllowed)), leaving space for one byte + // to hold the type ID. Shift the limit 8 bits left to make room for the type ID byte. + static private final int LIMIT_SHIFT = 8; static private final int POS_STACK_STEP = 2; - private final void push(int type, long position, int local_remaining) + private final void push(int type, long position, long local_remaining) { int oldlen = _container_stack.length; if ((_container_top + POS_STACK_STEP) >= oldlen) { @@ -203,11 +206,10 @@ private final int get_top_type() { } return type; } - private final int get_top_local_remaining() { + private final long get_top_local_remaining() { assert(_container_top > 0); long type_limit = _container_stack[_container_top - POS_STACK_STEP + TYPE_LIMIT_OFFSET]; - int local_remaining = (int)((type_limit >> LIMIT_SHIFT) & TYPE_MASK); - return local_remaining; + return type_limit >> LIMIT_SHIFT; } private final void pop() { assert(_container_top > 0); @@ -404,7 +406,7 @@ protected final int load_annotations() { case S_BEFORE_VALUE: case S_AFTER_VALUE: if (_annotations.isDefined()) { - int local_remaining_save = _local_remaining; + long local_remaining_save = _local_remaining; _input._save_points.savePointPushActive(_annotations, getPosition(), 0); _local_remaining = NO_LIMIT; // limit will be handled by the save point _annotation_count = 0; @@ -475,7 +477,7 @@ private final int read_type_id() throws IOException return UnifiedInputStreamX.EOF; } int tid = _Private_IonConstants.getTypeCode(td); - int len = _Private_IonConstants.getLowNibble(td); + long len = _Private_IonConstants.getLowNibble(td); // NOP Padding if (tid == _Private_IonConstants.tidNull && len != _Private_IonConstants.lnIsNull) { @@ -486,7 +488,16 @@ private final int read_type_id() throws IOException tid = _Private_IonConstants.tidNopPad; // override typeId to use Pad marker } else if (len == _Private_IonConstants.lnIsVarLen) { - len = readVarUInt(); + // For now, only allow *container* lengths to exceed Integer.MAX_VALUE. + // Note: for annotated scalars that exceed the limit, this method will be called again for the wrapped + // value's type ID, and will fail. + boolean isLongLengthAllowed = ( + tid == _Private_IonConstants.tidTypedecl // Annotation wrapper + || tid == _Private_IonConstants.tidList + || tid == _Private_IonConstants.tidSexp + || tid == _Private_IonConstants.tidStruct + ); + len = readVarUInt(isLongLengthAllowed); start_of_value = _input.getPosition(); } else if (tid == _Private_IonConstants.tidNull) { @@ -500,7 +511,7 @@ else if (len == _Private_IonConstants.lnIsNull) { _state = State.S_AFTER_VALUE; } else if (tid == _Private_IonConstants.tidBoolean) { - switch (len) { + switch ((int) len) { case _Private_IonConstants.lnBooleanFalse: _value_is_true = false; break; @@ -518,7 +529,7 @@ else if (tid == _Private_IonConstants.tidStruct) { if ((_struct_is_ordered = (len == 1))) { // special case of an ordered struct, it gets the // otherwise impossible to have length of 1 - len = readVarUInt(); + len = readVarUInt(true); if (len == 0) { throwErrorAt("Structs flagged as having ordered keys must contain at least one key/value pair."); } @@ -621,7 +632,7 @@ public void stepIn() // value processing when we step out long curr_position = getPosition(); long next_position = curr_position + _value_len; - int next_remaining = _local_remaining; + long next_remaining = _local_remaining; if (next_remaining != NO_LIMIT) { next_remaining -= _value_len; if (next_remaining < 0) { @@ -644,7 +655,7 @@ public void stepOut() // first we get the top values, then we // pop them all off in one fell swoop. long next_position = get_top_position(); - int local_remaining = get_top_local_remaining(); + long local_remaining = get_top_local_remaining(); int parent_tid = get_top_type(); pop(); _eof = false; @@ -672,8 +683,7 @@ public void stepOut() distance -= max_skip; } if (distance > 0) { - assert( distance < Integer.MAX_VALUE ); - skip((int)distance); + skip(distance); } } catch (IOException e) { @@ -705,7 +715,7 @@ public int byteSize() len = 0; } else { - len = _value_len; + len = (int) _value_len; } _value_lob_remaining = len; _value_lob_is_ready = true; @@ -808,7 +818,7 @@ private final int read(byte[] dst, int start, int len) throws IOException if (_local_remaining < 1) { throwUnexpectedEOFException(); } - len = _local_remaining; + len = (int) _local_remaining; } read = _input.read(dst, start, len); _local_remaining -= read; @@ -848,7 +858,7 @@ private final long getPosition() { long pos = _input.getPosition(); return pos; } - private final void skip(int len) throws IOException + private final void skip(long len) throws IOException { if (len < 0) { // no need to test this start >= dst.length || @@ -856,7 +866,11 @@ private final void skip(int len) throws IOException throw new IllegalArgumentException(); } if (_local_remaining == NO_LIMIT) { - _input.skip(len); + while (len > 0) { + int toSkip = (int) Math.min(Integer.MAX_VALUE, len); + _input.skip(toSkip); + len -= toSkip; + } } else { if (len > _local_remaining) { @@ -865,8 +879,12 @@ private final void skip(int len) throws IOException } len = _local_remaining; } - _input.skip(len); _local_remaining -= len; + while (len > 0) { + int toSkip = (int) Math.min(Integer.MAX_VALUE, len); + _input.skip(toSkip); + len -= toSkip; + } } return; } @@ -986,7 +1004,7 @@ private int readVarInt(int firstByte) throws IOException { if ((b & 0x80) != 0) break; // Don't support anything above a 5-byte VarInt for now, see https://github.com/amzn/ion-java/issues/146 - throwVarIntOverflowException(); + throwVarIntOverflowException(5); } if (isNegative) { @@ -995,13 +1013,24 @@ private int readVarInt(int firstByte) throws IOException { int retValueAsInt = (int) retValue; if (retValue != ((long) retValueAsInt)) { - throwVarIntOverflowException(); + throwVarIntOverflowException(4); } return retValueAsInt; } protected final int readVarUIntOrEOF() throws IOException + { + return (int) readVarUIntOrEOF(false); + } + + /** + * Attempts to read a VarUInt. + * @param longAllowed true if values over Integer.MAX_VALUE are allowed. If false, such values will raise an error. + * @return the value of the VarUInt, or -1 if EOF has been reached. + * @throws IOException if thrown when reading from the stream. + */ + protected final long readVarUIntOrEOF(boolean longAllowed) throws IOException { // VarUInt uses the high-order bit of the last octet as a marker; some (but not all) 5-byte VarUInt can fit // into a Java int. @@ -1034,26 +1063,49 @@ protected final int readVarUIntOrEOF() throws IOException retvalue = (retvalue << 7) | (b & 0x7F); if ((b & 0x80) != 0) break; - // Don't support anything above a 5-byte VarUInt for now, see https://github.com/amzn/ion-java/issues/146 - throwVarIntOverflowException(); - } + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break; + + if ((b = read()) < 0) throwUnexpectedEOFException(); + retvalue = (retvalue << 7) | (b & 0x7F); + if ((b & 0x80) != 0) break; - int retValueAsInt = (int) retvalue; - if (retvalue != ((long) retValueAsInt)) { - throwVarIntOverflowException(); + // Don't support anything above a 7-byte VarUInt for now, see https://github.com/amzn/ion-java/issues/146 + throwVarIntOverflowException(7); } - return retValueAsInt; + if (!longAllowed) { + int retValueAsInt = (int) retvalue; + if (retvalue != ((long) retValueAsInt)) { + throwVarIntOverflowException(4); + } + + return retValueAsInt; + } + return retvalue; } protected final int readVarUInt() throws IOException { - int varUInt = readVarUIntOrEOF(); + return (int) readVarUInt(false); + } + + /** + * Attempts to read a VarUInt, raising an error if EOF is encountered. + * @param longAllowed true if values over Integer.MAX_VALUE are allowed. If false, such values will raise an error. + * @return the value of the VarUInt. + * @throws IOException if thrown when reading from the stream. + */ + protected final long readVarUInt(boolean longAllowed) throws IOException + { + long varUInt = readVarUIntOrEOF(longAllowed); if (varUInt == UnifiedInputStreamX.EOF) { throwUnexpectedEOFException(); } return varUInt; } + protected final double readFloat(int len) throws IOException { if (len == 0) @@ -1083,7 +1135,7 @@ protected final Decimal readDecimal(int len) throws IOException } else { // otherwise we to it the hard way .... - int save_limit = NO_LIMIT; + long save_limit = NO_LIMIT; if (_local_remaining != NO_LIMIT) { save_limit = _local_remaining - len; } @@ -1093,8 +1145,8 @@ protected final Decimal readDecimal(int len) throws IOException int signum; if (_local_remaining > 0) { - byte[] bits = new byte[_local_remaining]; - readAll(bits, 0, _local_remaining); + byte[] bits = new byte[(int) _local_remaining]; + readAll(bits, 0, (int) _local_remaining); signum = 1; if (bits[0] < 0) { @@ -1136,7 +1188,7 @@ protected final Timestamp readTimestamp(int len) throws IOException int year = 0, month = 0, day = 0, hour = 0, minute = 0, second = 0; BigDecimal frac = null; - int save_limit = NO_LIMIT; + long save_limit = NO_LIMIT; if (_local_remaining != NO_LIMIT) { save_limit = _local_remaining - len; } @@ -1172,7 +1224,7 @@ protected final Timestamp readTimestamp(int len) throws IOException p = Precision.SECOND; if (_local_remaining > 0) { // now we read in our actual "milliseconds since the epoch" - frac = readDecimal(_local_remaining); + frac = readDecimal((int) _local_remaining); if (frac.compareTo(BigDecimal.ZERO) < 0 || frac.compareTo(BigDecimal.ONE) >= 0) { throwErrorAt( "The fractional seconds value in a timestamp must be greater than or " @@ -1211,7 +1263,7 @@ protected final String readString(int numberOfBytes) throws IOException utf8Decoder.prepareDecode(numberOfBytes); - int save_limit = NO_LIMIT; + long save_limit = NO_LIMIT; if (_local_remaining != NO_LIMIT) { save_limit = _local_remaining - numberOfBytes; } @@ -1267,7 +1319,7 @@ protected final String readString(int numberOfBytes) throws IOException } private String readStringWithReusableBuffer(int numberOfBytes, ByteBuffer utf8InputBuffer) throws IOException { - int save_limit = NO_LIMIT; + long save_limit = NO_LIMIT; if (_local_remaining != NO_LIMIT) { save_limit = _local_remaining - numberOfBytes; } @@ -1284,8 +1336,8 @@ private String readStringWithReusableBuffer(int numberOfBytes, ByteBuffer utf8In private final void throwUnexpectedEOFException() throws IOException { throwErrorAt("unexpected EOF in value"); } - private final void throwVarIntOverflowException() throws IOException { - throwErrorAt("int in stream is too long for a Java int 32"); + private final void throwVarIntOverflowException(int byteLimit) throws IOException { + throwErrorAt("int in stream is too long to fit in " + byteLimit + " bytes."); } protected IonException newErrorAt(String msg) { diff --git a/src/com/amazon/ion/impl/IonReaderBinarySystemX.java b/src/com/amazon/ion/impl/IonReaderBinarySystemX.java index 840d8336a3..b18c819b2b 100644 --- a/src/com/amazon/ion/impl/IonReaderBinarySystemX.java +++ b/src/com/amazon/ion/impl/IonReaderBinarySystemX.java @@ -189,7 +189,7 @@ private final void load_scalar_value() throws IOException _v.setAuthoritativeType(AS_TYPE.int_value); } else if (_value_len <= MAX_BINARY_LENGTH_LONG) { - long v = readULong(_value_len); + long v = readULong((int) _value_len); if (v < 0) { // we probably can't fit this magnitude properly into a Java long @@ -222,7 +222,7 @@ else if (_value_len <= MAX_BINARY_LENGTH_LONG) { } } else { - BigInteger v = readBigInteger(_value_len, is_negative); + BigInteger v = readBigInteger((int) _value_len, is_negative); _v.setValue(v); _v.setAuthoritativeType(AS_TYPE.bigInteger_value); } @@ -233,24 +233,24 @@ else if (_value_len <= MAX_BINARY_LENGTH_LONG) { d = 0.0; } else { - d = readFloat(_value_len); + d = readFloat((int) _value_len); } _v.setValue(d); _v.setAuthoritativeType(AS_TYPE.double_value); break; case DECIMAL: - Decimal dec = readDecimal(_value_len); + Decimal dec = readDecimal((int) _value_len); _v.setValue(dec); _v.setAuthoritativeType(AS_TYPE.decimal_value); break; case TIMESTAMP: // TODO: it looks like a 0 length return a null timestamp - is that right? - Timestamp t = readTimestamp(_value_len); + Timestamp t = readTimestamp((int) _value_len); _v.setValue(t); _v.setAuthoritativeType(AS_TYPE.timestamp_value); break; case SYMBOL: - long sid = readULong(_value_len); + long sid = readULong((int) _value_len); if (sid < 0 || sid > Integer.MAX_VALUE) { String message = "symbol id [" + sid @@ -265,7 +265,7 @@ else if (_value_len <= MAX_BINARY_LENGTH_LONG) { _v.setAuthoritativeType(AS_TYPE.int_value); break; case STRING: - String s = readString(_value_len); + String s = readString((int) _value_len); _v.setValue(s); _v.setAuthoritativeType(AS_TYPE.string_value); break; diff --git a/src/com/amazon/ion/impl/IonReaderLookaheadBuffer.java b/src/com/amazon/ion/impl/IonReaderLookaheadBuffer.java index c510b3af11..174045c30e 100644 --- a/src/com/amazon/ion/impl/IonReaderLookaheadBuffer.java +++ b/src/com/amazon/ion/impl/IonReaderLookaheadBuffer.java @@ -29,6 +29,8 @@ public final class IonReaderLookaheadBuffer extends ReaderLookaheadBufferBase { private static final int IVM_START_BYTE = 0xE0; private static final int IVM_REMAINING_LENGTH = 3; // Length of the IVM after the first byte. private static final int ION_SYMBOL_TABLE_SID = 3; + // The following is a limitation imposed by this implementation, not the Ion specification. + private static final long MAXIMUM_VALUE_SIZE = Integer.MAX_VALUE; /** * Represents a VarUInt that may be read in multiple steps. @@ -544,9 +546,9 @@ private void reclaimNopPadding() { * @return the number of bytes actually skipped. * @throws IOException if thrown by the underlying InputStream. */ - private int skipBytesFromInput(int numberOfBytesToSkip) throws IOException { + private long skipBytesFromInput(long numberOfBytesToSkip) throws IOException { try { - return (int) getInput().skip(numberOfBytesToSkip); + return getInput().skip(numberOfBytesToSkip); } catch (EOFException e) { // Certain InputStream implementations (e.g. GZIPInputStream) throw EOFException if more bytes are requested // to skip than are currently available (e.g. if a header or trailer is incomplete). @@ -584,7 +586,7 @@ private int fillPage(int numberOfBytesRequested) throws Exception { if (isSkippingCurrentValue()) { if (state == State.SKIPPING_VALUE) { // This is a seek operation, meaning that the bytes don't need to be interpreted. - received = skipBytesFromInput(amountToFill); + received = (int) skipBytesFromInput(amountToFill); } else { // The bytes need to be interpreted, so they cannot be skipped. The caller must retrieve them from // the input. @@ -621,15 +623,18 @@ private void notifyHandlerOfOversizedValue() throws Exception { * @return the number of bytes filled or skipped. * @throws Exception if thrown by the event handler. */ - private int fillOrSkip() throws Exception { + private long fillOrSkip() throws Exception { // Clamping at the number of buffered bytes available guarantees that the buffer // will never grow beyond its initial size. - int bytesRequested = (int) additionalBytesNeeded - pipe.availableBeyondBoundary(); - int bytesFilled; + long bytesRequested = additionalBytesNeeded - pipe.availableBeyondBoundary(); + long bytesFilled; if (isSkippingCurrentValue()) { bytesFilled = skipBytesFromInput(bytesRequested); } else { - bytesFilled = fillPage(bytesRequested); + if (additionalBytesNeeded > MAXIMUM_VALUE_SIZE) { + throw new IonException("The size of the value exceeds the limits of the implementation."); + } + bytesFilled = fillPage((int) bytesRequested); } if (bytesFilled < 1) { return 0; @@ -643,9 +648,9 @@ private int fillOrSkip() throws Exception { // buffered. bytesFilled = bytesFilled + ((int) additionalBytesNeeded - bytesRequested); } else { - bytesFilled = (int) Math.min(additionalBytesNeeded, bytesFilled); - pipe.extendBoundary(bytesFilled); - peekIndex += bytesFilled; + bytesFilled = Math.min(additionalBytesNeeded, bytesFilled); + pipe.extendBoundary((int) bytesFilled); + peekIndex += (int) bytesFilled; } return bytesFilled; } @@ -657,16 +662,21 @@ private int fillOrSkip() throws Exception { * @throws Exception if thrown by the event handler. */ private long skip(long numberOfBytesToSkip) throws Exception { - int numberOfBytesSkipped; + long numberOfBytesSkipped; if (pipe.availableBeyondBoundary() >= numberOfBytesToSkip) { numberOfBytesSkipped = (int) numberOfBytesToSkip; - pipe.extendBoundary(numberOfBytesSkipped); - peekIndex += numberOfBytesSkipped; + pipe.extendBoundary((int) numberOfBytesSkipped); + peekIndex += (int) numberOfBytesSkipped; } else { numberOfBytesSkipped = fillOrSkip(); } if (numberOfBytesSkipped > 0) { - dataHandler.onData(numberOfBytesSkipped); + long numberOfBytesToReport = numberOfBytesSkipped; + while (numberOfBytesToReport > 0) { + int numberOfBytesToReportThisIteration = (int) Math.min(Integer.MAX_VALUE, numberOfBytesToReport); + dataHandler.onData(numberOfBytesToReportThisIteration); + numberOfBytesToReport -= numberOfBytesToReportThisIteration; + } } return numberOfBytesSkipped; } diff --git a/test/com/amazon/ion/impl/IonReaderBinaryRawLargeStreamTest.java b/test/com/amazon/ion/impl/IonReaderBinaryRawLargeStreamTest.java index 9ed965105a..cd94276dc9 100644 --- a/test/com/amazon/ion/impl/IonReaderBinaryRawLargeStreamTest.java +++ b/test/com/amazon/ion/impl/IonReaderBinaryRawLargeStreamTest.java @@ -1,5 +1,6 @@ package com.amazon.ion.impl; +import com.amazon.ion.IonException; import com.amazon.ion.IonReader; import com.amazon.ion.IonType; import com.amazon.ion.IonWriter; @@ -7,25 +8,27 @@ import com.amazon.ion.system.IonBinaryWriterBuilder; import com.amazon.ion.system.IonReaderBuilder; import com.amazon.ion.util.RepeatInputStream; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.InputStream; import java.io.SequenceInputStream; import java.math.BigDecimal; import static com.amazon.ion.impl._Private_IonConstants.BINARY_VERSION_MARKER_1_0; +import static junit.framework.TestCase.assertNull; import static org.junit.Assert.assertEquals; +// NOTE: these tests each take several seconds to complete. public class IonReaderBinaryRawLargeStreamTest { - // NOTE: this test takes several seconds to complete. - @Test - public void testReadLargeScalarStream() throws Exception { + private byte[] testData(Timestamp timestamp) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); IonWriter writer = IonBinaryWriterBuilder.standard().build(out); - final Timestamp timestamp = Timestamp.forDay(2000, 1, 1); writer.writeString("foo"); writer.writeDecimal(BigDecimal.TEN); writer.writeTimestamp(timestamp); @@ -34,6 +37,12 @@ public void testReadLargeScalarStream() throws Exception { // Strip the IVM, as this needs to be one continuous stream to avoid resetting the reader's internals. byte[] data = new byte[dataWithIvm.length - BINARY_VERSION_MARKER_1_0.length]; System.arraycopy(dataWithIvm, BINARY_VERSION_MARKER_1_0.length, data, 0, data.length); + return data; + } + + public void readLargeScalarStream(IonReaderBuilder readerBuilder) throws Exception { + final Timestamp timestamp = Timestamp.forDay(2000, 1, 1); + byte[] data = testData(timestamp); // The binary reader uses Integer.MIN_VALUE to mean NO_LIMIT for its _local_remaining value, which keeps track // of the remaining number of bytes in the current value. Between values at the top level, this should always be // NO_LIMIT. No arithmetic should ever be performed on the value when it is set to NO_LIMIT. If bugs exist that @@ -49,12 +58,12 @@ public void testReadLargeScalarStream() throws Exception { // the stream reached Integer.MAX_VALUE in length. // Repeat the batch a sufficient number of times to exceed a total stream length of Integer.MAX_VALUE, plus // a few more to make sure batches continue to be read correctly. - final int totalNumberOfBatches = (Integer.MAX_VALUE / data.length) + 7; + final int totalNumberOfBatches = (Integer.MAX_VALUE / data.length) + 7; // 7 makes the value exceed Integer.MAX_VALUE by an arbitrary amount. InputStream inputStream = new SequenceInputStream( new ByteArrayInputStream(BINARY_VERSION_MARKER_1_0), new RepeatInputStream(data, totalNumberOfBatches - 1) // This will provide the data 'totalNumberOfBatches' times ); - IonReader reader = IonReaderBuilder.standard().build(inputStream); + IonReader reader = readerBuilder.build(inputStream); reader.next(); assertEquals("foo", reader.stringValue()); reader.next(); @@ -70,4 +79,298 @@ public void testReadLargeScalarStream() throws Exception { } assertEquals(totalNumberOfBatches, batchesRead); } + + @Test + public void readLargeScalarStreamNonIncremental() throws Exception { + readLargeScalarStream(IonReaderBuilder.standard()); + } + + @Test + public void readLargeScalarStreamIncremental() throws Exception { + readLargeScalarStream(IonReaderBuilder.standard().withIncrementalReadingEnabled(true)); + } + + @Test + public void readLargeContainer() throws Exception { + final Timestamp timestamp = Timestamp.forDay(2000, 1, 1); + + byte[] data = testData(timestamp); + + final int totalNumberOfBatches = (Integer.MAX_VALUE / data.length) + 32768; // 32768 makes the value exceed Integer.MAX_VALUE by an arbitrary amount. + ByteArrayOutputStream header = new ByteArrayOutputStream(); + header.write(BINARY_VERSION_MARKER_1_0); + header.write(0xBE); // List with length subfield. + IonBinary.writeVarUInt(header, (long) data.length * totalNumberOfBatches); // Length + InputStream inputStream = new SequenceInputStream( + new ByteArrayInputStream(header.toByteArray()), + new RepeatInputStream(data, totalNumberOfBatches - 1) // This will provide the data 'totalNumberOfBatches' times + ); + + IonReader reader = IonReaderBuilder.standard().build(inputStream); + assertEquals(IonType.LIST, reader.next()); + reader.stepIn(); + int batchesRead = 0; + while (reader.next() != null) { + // Materializing on every iteration makes the tests take too long. Do it on every 100. + boolean materializeValues = (batchesRead % 100) == 0; + assertEquals(IonType.STRING, reader.getType()); + if (materializeValues) { + assertEquals("foo", reader.stringValue()); + } + assertEquals(IonType.DECIMAL, reader.next()); + if (materializeValues) { + assertEquals(BigDecimal.TEN, reader.decimalValue()); + } + assertEquals(IonType.TIMESTAMP, reader.next()); + if (materializeValues) { + assertEquals(timestamp, reader.timestampValue()); + } + batchesRead++; + } + assertNull(reader.next()); + reader.stepOut(); + assertNull(reader.next()); + assertEquals(totalNumberOfBatches, batchesRead); + } + + @Test + public void skipLargeContainer() throws Exception { + final Timestamp timestamp = Timestamp.forDay(2000, 1, 1); + + byte[] data = testData(timestamp); + + final int totalNumberOfBatches = (Integer.MAX_VALUE / data.length) + 17; // 17 makes the value exceed Integer.MAX_VALUE by an arbitrary amount. + ByteArrayOutputStream header = new ByteArrayOutputStream(); + header.write(BINARY_VERSION_MARKER_1_0); + header.write(0xCE); // S-exp with length subfield. + IonBinary.writeVarUInt(header, (long) data.length * totalNumberOfBatches); // Length + InputStream inputStream = new SequenceInputStream( + new ByteArrayInputStream(header.toByteArray()), + new SequenceInputStream( + new RepeatInputStream(data, totalNumberOfBatches - 1), // This will provide the data 'totalNumberOfBatches' times + new ByteArrayInputStream(new byte[]{(byte) 0x83, 'b', 'a', 'r'}) // The string "bar" + ) + ); + + IonReader reader = IonReaderBuilder.standard().build(inputStream); + assertEquals(IonType.SEXP, reader.next()); + assertEquals(IonType.STRING, reader.next()); + assertEquals("bar", reader.stringValue()); + assertNull(reader.next()); + } + + @Test + public void skipLargeNestedContainer() throws Exception { + final Timestamp timestamp = Timestamp.forDay(2000, 1, 1); + + byte[] data = testData(timestamp); + + final int totalNumberOfBatches = (Integer.MAX_VALUE / data.length) + 512; // 512 makes the value exceed Integer.MAX_VALUE by an arbitrary amount. + final long nestedDataLength = (long) data.length * totalNumberOfBatches; + ByteArrayOutputStream header = new ByteArrayOutputStream(); + header.write(BINARY_VERSION_MARKER_1_0); + header.write(0xCE); // S-exp with length subfield. + IonBinary.writeVarUInt(header, 1 + IonBinary.lenVarUInt(nestedDataLength) + nestedDataLength); // Length + header.write(0xBE); // List with length subfield. + IonBinary.writeVarUInt(header, nestedDataLength); + InputStream inputStream = new SequenceInputStream( + new ByteArrayInputStream(header.toByteArray()), + new SequenceInputStream( + new RepeatInputStream(data, totalNumberOfBatches - 1), // This will provide the data 'totalNumberOfBatches' times + new ByteArrayInputStream(new byte[]{(byte) 0x83, 'b', 'a', 'r'}) // The string "bar" + ) + ); + + IonReader reader = IonReaderBuilder.standard().build(inputStream); + assertEquals(IonType.SEXP, reader.next()); + reader.stepIn(); + assertEquals(IonType.LIST, reader.next()); + assertNull(reader.next()); + reader.stepOut(); + assertEquals(IonType.STRING, reader.next()); + assertEquals("bar", reader.stringValue()); + assertNull(reader.next()); + } + + @Test + public void readLargeAnnotatedContainer() throws Exception { + final Timestamp timestamp = Timestamp.forDay(2000, 1, 1); + byte[] raw = testData(timestamp); + ByteArrayOutputStream dataBuilder = new ByteArrayOutputStream(); + dataBuilder.write(0x85); // Field name. Conveniently use SID 5 ("version"), which is in the system symbol table. + dataBuilder.write(0xBE); // List with length subfield. + IonBinary.writeVarUInt(dataBuilder, raw.length); + dataBuilder.write(raw); + byte[] data = dataBuilder.toByteArray(); + + final int totalNumberOfBatches = (Integer.MAX_VALUE / data.length) + 100000; // 100000 makes the value exceed Integer.MAX_VALUE by an arbitrary amount. + ByteArrayOutputStream header = new ByteArrayOutputStream(); + header.write(BINARY_VERSION_MARKER_1_0); + long containerLength = (long) data.length * totalNumberOfBatches; + header.write(0xEE); // Annotation wrapper with length subfield. + IonBinary.writeVarUInt(header, containerLength + 3 + IonBinary.lenVarUInt(containerLength)); + header.write(0x81); // One byte of annotations. + header.write(0x84); // Conveniently use SID 4 ("name"), which is in the system symbol table. + header.write(0xDE); // Struct with length subfield. + IonBinary.writeVarUInt(header, containerLength); // Length + InputStream inputStream = new SequenceInputStream( + new ByteArrayInputStream(header.toByteArray()), + new RepeatInputStream(data, totalNumberOfBatches - 1) // This will provide the data 'totalNumberOfBatches' times + ); + IonReader reader = IonReaderBuilder.standard().build(inputStream); + assertEquals(IonType.STRUCT, reader.next()); + String[] annotations = reader.getTypeAnnotations(); + assertEquals(1, annotations.length); + assertEquals("name", annotations[0]); + reader.stepIn(); + int batchesRead = 0; + while (reader.next() != null) { + assertEquals(IonType.LIST, reader.getType()); + // Materializing on every iteration makes the tests take too long. Do it on every 100. + boolean materializeValues = (batchesRead % 100) == 0; + if (materializeValues) { + assertEquals("version", reader.getFieldName()); + reader.stepIn(); + assertEquals(IonType.STRING, reader.next()); + assertEquals("foo", reader.stringValue()); + assertEquals(IonType.DECIMAL, reader.next()); + assertEquals(BigDecimal.TEN, reader.decimalValue()); + assertEquals(IonType.TIMESTAMP, reader.next()); + assertEquals(timestamp, reader.timestampValue()); + assertNull(reader.next()); + reader.stepOut(); + } + batchesRead++; + } + assertNull(reader.next()); + reader.stepOut(); + assertNull(reader.next()); + assertEquals(totalNumberOfBatches, batchesRead); + } + + @Test + public void skipLargeAnnotatedContainer() throws Exception { + final Timestamp timestamp = Timestamp.forDay(2000, 1, 1); + byte[] raw = testData(timestamp); + ByteArrayOutputStream dataBuilder = new ByteArrayOutputStream(); + dataBuilder.write(0xEE); // Annotation wrapper with length subfield. + IonBinary.writeVarUInt(dataBuilder, 3 + IonBinary.lenVarUInt(raw.length) + raw.length); + dataBuilder.write(0x81); // One byte of annotations. + dataBuilder.write(0x85); // Annotation. Conveniently use SID 5 ("version"), which is in the system symbol table. + dataBuilder.write(0xCE); // S-exp with length subfield. + IonBinary.writeVarUInt(dataBuilder, raw.length); + dataBuilder.write(raw); + byte[] data = dataBuilder.toByteArray(); + + final int totalNumberOfBatches = (Integer.MAX_VALUE / data.length) + 1000000; // 1000000 makes the value exceed Integer.MAX_VALUE by an arbitrary amount. + ByteArrayOutputStream header = new ByteArrayOutputStream(); + header.write(BINARY_VERSION_MARKER_1_0); + long containerLength = (long) data.length * totalNumberOfBatches; + header.write(0xEE); // Annotation wrapper with length subfield. + IonBinary.writeVarUInt(header, containerLength + 3 + IonBinary.lenVarUInt(containerLength)); + header.write(0x81); // One byte of annotations. + header.write(0x84); // Conveniently use SID 4 ("name"), which is in the system symbol table. + header.write(0xBE); // List with length subfield. + IonBinary.writeVarUInt(header, containerLength); // Length + InputStream inputStream = new SequenceInputStream( + new ByteArrayInputStream(header.toByteArray()), + new RepeatInputStream(data, totalNumberOfBatches - 1) // This will provide the data 'totalNumberOfBatches' times + ); + IonReader reader = IonReaderBuilder.standard().build(inputStream); + assertEquals(IonType.LIST, reader.next()); + String[] annotations = reader.getTypeAnnotations(); + assertEquals(1, annotations.length); + assertEquals("name", annotations[0]); + assertNull(reader.next()); + } + + // Note: the objective of the following tests is not to assert that large scalars *should* fail, but rather that + // when they *do* fail due to limitations of the current implementation, they fail by throwing an IonException + // and not something unexpected and ugly. + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + private void cleanlyFailsOnLargeScalar(IonReaderBuilder readerBuilder) throws Exception { + byte[] data = "foobarbaz".getBytes("UTF-8"); + final int totalNumberOfBatches = (Integer.MAX_VALUE / data.length) + 123; // 123 makes the value exceed Integer.MAX_VALUE by an arbitrary amount. + ByteArrayOutputStream header = new ByteArrayOutputStream(); + header.write(BINARY_VERSION_MARKER_1_0); + header.write(0x8E); // String with length subfield. + IonBinary.writeVarUInt(header, (long) totalNumberOfBatches * data.length); + InputStream inputStream = new SequenceInputStream( + new ByteArrayInputStream(header.toByteArray()), + new RepeatInputStream(data, totalNumberOfBatches - 1) // This will provide the data 'totalNumberOfBatches' times + ); + IonReader reader = readerBuilder.build(inputStream); + // If support for large scalars is added, the following line will be deleted and the rest of the test + // completed to assert the correctness of the value. + thrown.expect(IonException.class); + reader.next(); + } + + @Test + public void cleanlyFailsOnLargeScalarNonIncremental() throws Exception { + cleanlyFailsOnLargeScalar(IonReaderBuilder.standard()); + } + + @Test + public void cleanlyFailsOnLargeScalarIncremental() throws Exception { + cleanlyFailsOnLargeScalar(IonReaderBuilder.standard().withIncrementalReadingEnabled(true)); + } + + private void cleanlyFailsOnLargeAnnotatedScalar(IonReaderBuilder readerBuilder) throws Exception { + byte[] data = "foobarbaz".getBytes("UTF-8"); + final int totalNumberOfBatches = (Integer.MAX_VALUE / data.length) + 9999; // 9999 makes the value exceed Integer.MAX_VALUE by an arbitrary amount. + final long stringLength = (long) totalNumberOfBatches * data.length; + ByteArrayOutputStream header = new ByteArrayOutputStream(); + header.write(BINARY_VERSION_MARKER_1_0); + header.write(0xEE); // Annotation wrapper with length subfield. + IonBinary.writeVarUInt(header, 3 + IonBinary.lenVarUInt(stringLength) + stringLength); + header.write(0x81); // One byte of annotations. + header.write(0x84); // Conveniently use SID 4 ("name"), which is in the system symbol table. + header.write(0x8E); // String with length subfield. + IonBinary.writeVarUInt(header, stringLength); + InputStream inputStream = new SequenceInputStream( + new ByteArrayInputStream(header.toByteArray()), + new RepeatInputStream(data, totalNumberOfBatches - 1) // This will provide the data 'totalNumberOfBatches' times + ); + IonReader reader = readerBuilder.build(inputStream); + // If support for large scalars is added, the following line will be deleted and the rest of the test + // completed to assert the correctness of the value. + thrown.expect(IonException.class); + reader.next(); + } + + @Test + public void cleanlyFailsOnLargeAnnotatedScalarNonIncremental() throws Exception { + cleanlyFailsOnLargeAnnotatedScalar(IonReaderBuilder.standard()); + } + + @Test + public void cleanlyFailsOnLargeAnnotatedScalarIncremental() throws Exception { + cleanlyFailsOnLargeAnnotatedScalar(IonReaderBuilder.standard().withIncrementalReadingEnabled(true)); + } + + @Test + public void cleanlyFailsOnLargeContainerIncremental() throws Exception { + final Timestamp timestamp = Timestamp.forDay(2000, 1, 1); + + byte[] data = testData(timestamp); + + final int totalNumberOfBatches = (Integer.MAX_VALUE / data.length) + 42; // 42 makes the value exceed Integer.MAX_VALUE by an arbitrary amount. + ByteArrayOutputStream header = new ByteArrayOutputStream(); + header.write(BINARY_VERSION_MARKER_1_0); + header.write(0xCE); // S-exp with length subfield. + IonBinary.writeVarUInt(header, (long) data.length * totalNumberOfBatches); // Length + InputStream inputStream = new SequenceInputStream( + new ByteArrayInputStream(header.toByteArray()), + new RepeatInputStream(data, totalNumberOfBatches - 1) // This will provide the data 'totalNumberOfBatches' times + ); + + IonReader reader = IonReaderBuilder.standard().withIncrementalReadingEnabled(true).build(inputStream); + thrown.expect(IonException.class); + reader.next(); + } + } diff --git a/test/com/amazon/ion/util/RepeatInputStream.java b/test/com/amazon/ion/util/RepeatInputStream.java index c9e254af42..fb58fd2135 100644 --- a/test/com/amazon/ion/util/RepeatInputStream.java +++ b/test/com/amazon/ion/util/RepeatInputStream.java @@ -77,7 +77,7 @@ public int read(byte[] b, int off, int len) throws IOException return -1; } - int rem = len - off; + int rem = len; int consumed = 0; while (rem > 0 && !isDone()) { From 8356c932326e6091d65b20e43c344bf90475f80f Mon Sep 17 00:00:00 2001 From: jobarr-amzn <70981087+jobarr-amzn@users.noreply.github.com> Date: Mon, 7 Mar 2022 12:13:11 -0800 Subject: [PATCH 160/490] Bumps version to 1.9.3-SNAPSHOT (#419) Looks like we dropped this in the blur of the 1.9.2 release --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index cddd77cb0c..dbada93957 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.9.2-SNAPSHOT + 1.9.3-SNAPSHOT bundle ${project.groupId}:${project.artifactId} From 939db897d4cccc3bad2eab633031f3cd65e34e73 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Mon, 7 Mar 2022 17:55:22 -0800 Subject: [PATCH 161/490] Fixes a bug in header compaction for empty containers; adds testing for all writer option combinations over all good ion-tests files. --- src/com/amazon/ion/impl/bin/WriteBuffer.java | 2 +- test/AllTests.java | 4 +- .../bin/IonManagedBinaryWriterGoodTest.java | 56 ++++++ .../impl/bin/IonManagedBinaryWriterTest.java | 169 +++--------------- .../bin/IonManagedBinaryWriterTestCase.java | 152 ++++++++++++++++ .../amazon/ion/impl/bin/WriteBufferTest.java | 20 +++ 6 files changed, 256 insertions(+), 147 deletions(-) create mode 100644 test/com/amazon/ion/impl/bin/IonManagedBinaryWriterGoodTest.java create mode 100644 test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTestCase.java diff --git a/src/com/amazon/ion/impl/bin/WriteBuffer.java b/src/com/amazon/ion/impl/bin/WriteBuffer.java index c4550dc398..88e5db1749 100644 --- a/src/com/amazon/ion/impl/bin/WriteBuffer.java +++ b/src/com/amazon/ion/impl/bin/WriteBuffer.java @@ -180,7 +180,7 @@ public void writeBytes(final byte[] bytes, final int off, final int len) * @param shiftBy The number of bytes to the left that we'll be shifting. */ public void shiftBytesLeft(int length, int shiftBy) { - if (length == 0 || shiftBy == 0) { + if (shiftBy == 0) { // Nothing to do. return; } diff --git a/test/AllTests.java b/test/AllTests.java index 6817e13d54..8ed4056d4b 100644 --- a/test/AllTests.java +++ b/test/AllTests.java @@ -77,6 +77,7 @@ import com.amazon.ion.impl.SharedSymbolTableTest; import com.amazon.ion.impl.SymbolTableTest; import com.amazon.ion.impl.TreeReaderTest; +import com.amazon.ion.impl.bin.IonManagedBinaryWriterGoodTest; import com.amazon.ion.impl.bin.IonManagedBinaryWriterTest; import com.amazon.ion.impl.bin.IonRawBinaryWriterTest; import com.amazon.ion.impl.bin.PooledBlockAllocatorProviderTest; @@ -217,11 +218,12 @@ ResizingPipedInputStreamTest.class, IonReaderLookaheadBufferTest.class, - // experimental binary writer tests + // binary writer tests PooledBlockAllocatorProviderTest.class, WriteBufferTest.class, IonRawBinaryWriterTest.class, IonManagedBinaryWriterTest.class, + IonManagedBinaryWriterGoodTest.class, // Hash code tests HashCodeCorrectnessTest.class, diff --git a/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterGoodTest.java b/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterGoodTest.java new file mode 100644 index 0000000000..db5c997d81 --- /dev/null +++ b/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterGoodTest.java @@ -0,0 +1,56 @@ +package com.amazon.ion.impl.bin; + +import com.amazon.ion.IonException; +import com.amazon.ion.IonReader; +import com.amazon.ion.IonValue; +import com.amazon.ion.impl._Private_Utils; +import com.amazon.ion.junit.Injected; +import com.amazon.ion.system.IonReaderBuilder; +import org.junit.Test; + +import java.io.File; + +import static com.amazon.ion.TestUtils.GLOBAL_SKIP_LIST; +import static com.amazon.ion.TestUtils.GOOD_IONTESTS_FILES; +import static com.amazon.ion.TestUtils.hexDump; +import static com.amazon.ion.TestUtils.testdataFiles; + +/** + * Re-writes and verifies all "good" ion-tests files using all combinations of IonManagedBinaryWriter options. + */ +public class IonManagedBinaryWriterGoodTest extends IonManagedBinaryWriterTestCase { + + @Injected.Inject("testFile") + public static final File[] FILES = + testdataFiles(GLOBAL_SKIP_LIST, + GOOD_IONTESTS_FILES); + + + private File myTestFile; + + public void setTestFile(File file) + { + myTestFile = file; + } + + @Test + public void allGoodFiles() throws Exception { + byte[] testData = _Private_Utils.loadFileBytes(myTestFile); + IonReader reader = IonReaderBuilder.standard().build(testData); + writer.writeValues(reader); + reader.close(); + + writer.finish(); + final byte[] data = writer.getBytes(); + final IonValue actual; + try { + actual = system().getLoader().load(data); + } catch (final Exception e) { + throw new IonException("Bad generated data:\n" + hexDump(data), e); + } + final IonValue expected = system().getLoader().load(testData); + assertEquals(expected, actual); + + additionalValueAssertions(actual); + } +} diff --git a/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java b/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java index 853136c59f..c808a8458a 100644 --- a/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java +++ b/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTest.java @@ -15,165 +15,21 @@ package com.amazon.ion.impl.bin; -import static java.util.Arrays.asList; -import static java.util.Collections.unmodifiableList; -import static java.util.Collections.unmodifiableMap; - -import com.amazon.ion.IonContainer; import com.amazon.ion.IonDatagram; import com.amazon.ion.IonInt; -import com.amazon.ion.IonMutableCatalog; import com.amazon.ion.IonReader; import com.amazon.ion.IonStruct; import com.amazon.ion.IonSymbol; import com.amazon.ion.IonType; -import com.amazon.ion.IonValue; import com.amazon.ion.IonWriter; -import com.amazon.ion.SymbolTable; -import com.amazon.ion.SymbolToken; -import com.amazon.ion.SystemSymbols; -import com.amazon.ion.impl.bin.IonManagedBinaryWriter.ImportedSymbolResolverMode; -import com.amazon.ion.impl.bin._Private_IonManagedBinaryWriterBuilder.AllocatorMode; -import com.amazon.ion.junit.Injected.Inject; import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import org.junit.Test; @SuppressWarnings("deprecation") -public class IonManagedBinaryWriterTest extends IonRawBinaryWriterTest +public class IonManagedBinaryWriterTest extends IonManagedBinaryWriterTestCase { - @SuppressWarnings("unchecked") - private static final List> SHARED_SYMBOLS = unmodifiableList(asList( - unmodifiableList(asList( - "a", - "b", - "c" - )), - unmodifiableList(asList( - "d", - "e" - )) - )); - - private static final Map SHARED_SYMBOL_LOCAL_SIDS ; - static - { - final Map sidMap = new HashMap(); - - for (final SymbolToken token : Symbols.systemSymbols()) - { - sidMap.put(token.getText(), token.getSid()); - } - int sid = SystemSymbols.ION_1_0_MAX_ID + 1; - for (final List symbolList : SHARED_SYMBOLS) - { - for (final String symbol : symbolList) { - sidMap.put(symbol, sid); - sid++; - } - } - SHARED_SYMBOL_LOCAL_SIDS = unmodifiableMap(sidMap); - } - - private enum LSTAppendMode - { - LST_APPEND_DISABLED, - LST_APPEND_ENABLED; - public boolean isEnabled() { return this == LST_APPEND_ENABLED; } - } - - @Inject("lstAppendMode") - public static final LSTAppendMode[] LST_APPEND_ENABLED_DIMENSIONS = LSTAppendMode.values(); - private LSTAppendMode lstAppendMode; - public void setLstAppendMode(final LSTAppendMode mode) - { - this.lstAppendMode = mode; - } - - private void checkSymbolTokenAgainstImport(final SymbolToken token) - { - final Integer sid = SHARED_SYMBOL_LOCAL_SIDS.get(token.getText()); - if (sid != null) - { - assertEquals(sid.intValue(), token.getSid()); - } - } - - @Override - protected void additionalValueAssertions(final IonValue value) - { - for (final SymbolToken token : value.getTypeAnnotationSymbols()) { - checkSymbolTokenAgainstImport(token); - } - final IonType type = value.getType(); - if (type == IonType.SYMBOL && !value.isNullValue()) - { - checkSymbolTokenAgainstImport(((IonSymbol) value).symbolValue()); - } - else if (IonType.isContainer(type)) - { - for (final IonValue child : ((IonContainer) value)) - { - additionalValueAssertions(child); - } - } - } - - @Override - public int ivmLength() { - return 4; - } - - - @Inject("importedSymbolResolverMode") - public static final ImportedSymbolResolverMode[] RESOLVER_DIMENSIONS = ImportedSymbolResolverMode.values(); - - private ImportedSymbolResolverMode importedSymbolResolverMode; - - public void setImportedSymbolResolverMode(final ImportedSymbolResolverMode mode) - { - importedSymbolResolverMode = mode; - } - - @Override - protected IonWriter createWriter(final OutputStream out) throws IOException - { - final IonMutableCatalog catalog = ((IonMutableCatalog) system().getCatalog()); - - final List symbolTables = new ArrayList(); - int i = 1; - for (final List symbols : SHARED_SYMBOLS) { - final SymbolTable table = system().newSharedSymbolTable("test_" + (i++), 1, symbols.iterator()); - symbolTables.add(table); - catalog.putTable(table); - } - - final _Private_IonManagedBinaryWriterBuilder builder = _Private_IonManagedBinaryWriterBuilder - .create(AllocatorMode.POOLED) - .withImports(importedSymbolResolverMode, symbolTables) - .withPreallocationMode(preallocationMode) - .withFloatBinary32Enabled(); - - if (lstAppendMode.isEnabled()) { - builder.withLocalSymbolTableAppendEnabled(); - } else { - builder.withLocalSymbolTableAppendDisabled(); - } - - final IonWriter writer = builder.newWriter(out); - - final SymbolTable locals = writer.getSymbolTable(); - assertEquals(14, locals.getImportedMaxId()); - - return writer; - } @Test public void testSetStringAnnotations() throws Exception @@ -374,4 +230,27 @@ public void testSymbolTableExport() throws Exception { bos.toByteArray(); } + + @Test + public void testNestedEmptyContainer() throws Exception + { + writer.stepIn(IonType.STRUCT); + writer.setFieldName("bar"); + writer.stepIn(IonType.LIST); + writer.stepOut(); + writer.stepOut(); + assertValue("{bar: []}"); + } + + @Test + public void testNestedEmptyAnnotatedContainer() throws Exception + { + writer.stepIn(IonType.STRUCT); + writer.setFieldName("bar"); + writer.addTypeAnnotation("foo"); + writer.stepIn(IonType.LIST); + writer.stepOut(); + writer.stepOut(); + assertValue("{bar: foo::[]}"); + } } diff --git a/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTestCase.java b/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTestCase.java new file mode 100644 index 0000000000..fcc28498a8 --- /dev/null +++ b/test/com/amazon/ion/impl/bin/IonManagedBinaryWriterTestCase.java @@ -0,0 +1,152 @@ +package com.amazon.ion.impl.bin; + +import com.amazon.ion.IonContainer; +import com.amazon.ion.IonMutableCatalog; +import com.amazon.ion.IonSymbol; +import com.amazon.ion.IonType; +import com.amazon.ion.IonValue; +import com.amazon.ion.IonWriter; +import com.amazon.ion.SymbolTable; +import com.amazon.ion.SymbolToken; +import com.amazon.ion.SystemSymbols; +import com.amazon.ion.junit.Injected; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.util.Arrays.asList; +import static java.util.Collections.unmodifiableList; +import static java.util.Collections.unmodifiableMap; + +public class IonManagedBinaryWriterTestCase extends IonRawBinaryWriterTest { + + @SuppressWarnings("unchecked") + private static final List> SHARED_SYMBOLS = unmodifiableList(asList( + unmodifiableList(asList( + "a", + "b", + "c" + )), + unmodifiableList(asList( + "d", + "e" + )) + )); + + private static final Map SHARED_SYMBOL_LOCAL_SIDS ; + static + { + final Map sidMap = new HashMap(); + + for (final SymbolToken token : Symbols.systemSymbols()) + { + sidMap.put(token.getText(), token.getSid()); + } + int sid = SystemSymbols.ION_1_0_MAX_ID + 1; + for (final List symbolList : SHARED_SYMBOLS) + { + for (final String symbol : symbolList) { + sidMap.put(symbol, sid); + sid++; + } + } + SHARED_SYMBOL_LOCAL_SIDS = unmodifiableMap(sidMap); + } + + protected enum LSTAppendMode + { + LST_APPEND_DISABLED, + LST_APPEND_ENABLED; + public boolean isEnabled() { return this == LST_APPEND_ENABLED; } + } + + @Injected.Inject("lstAppendMode") + public static final LSTAppendMode[] LST_APPEND_ENABLED_DIMENSIONS = LSTAppendMode.values(); + protected LSTAppendMode lstAppendMode; + public void setLstAppendMode(final LSTAppendMode mode) + { + this.lstAppendMode = mode; + } + + private void checkSymbolTokenAgainstImport(final SymbolToken token) + { + final Integer sid = SHARED_SYMBOL_LOCAL_SIDS.get(token.getText()); + if (sid != null) + { + assertEquals(sid.intValue(), token.getSid()); + } + } + + @Override + protected void additionalValueAssertions(final IonValue value) + { + for (final SymbolToken token : value.getTypeAnnotationSymbols()) { + checkSymbolTokenAgainstImport(token); + } + final IonType type = value.getType(); + if (type == IonType.SYMBOL && !value.isNullValue()) + { + checkSymbolTokenAgainstImport(((IonSymbol) value).symbolValue()); + } + else if (IonType.isContainer(type)) + { + for (final IonValue child : ((IonContainer) value)) + { + additionalValueAssertions(child); + } + } + } + + @Override + public int ivmLength() { + return 4; + } + + + @Injected.Inject("importedSymbolResolverMode") + public static final IonManagedBinaryWriter.ImportedSymbolResolverMode[] RESOLVER_DIMENSIONS = IonManagedBinaryWriter.ImportedSymbolResolverMode.values(); + + private IonManagedBinaryWriter.ImportedSymbolResolverMode importedSymbolResolverMode; + + public void setImportedSymbolResolverMode(final IonManagedBinaryWriter.ImportedSymbolResolverMode mode) + { + importedSymbolResolverMode = mode; + } + + @Override + protected IonWriter createWriter(final OutputStream out) throws IOException + { + final IonMutableCatalog catalog = ((IonMutableCatalog) system().getCatalog()); + + final List symbolTables = new ArrayList(); + int i = 1; + for (final List symbols : SHARED_SYMBOLS) { + final SymbolTable table = system().newSharedSymbolTable("test_" + (i++), 1, symbols.iterator()); + symbolTables.add(table); + catalog.putTable(table); + } + + final _Private_IonManagedBinaryWriterBuilder builder = _Private_IonManagedBinaryWriterBuilder + .create(_Private_IonManagedBinaryWriterBuilder.AllocatorMode.POOLED) + .withImports(importedSymbolResolverMode, symbolTables) + .withPreallocationMode(preallocationMode) + .withFloatBinary32Enabled(); + + if (lstAppendMode.isEnabled()) { + builder.withLocalSymbolTableAppendEnabled(); + } else { + builder.withLocalSymbolTableAppendDisabled(); + } + + final IonWriter writer = builder.newWriter(out); + + final SymbolTable locals = writer.getSymbolTable(); + assertEquals(14, locals.getImportedMaxId()); + + return writer; + } +} diff --git a/test/com/amazon/ion/impl/bin/WriteBufferTest.java b/test/com/amazon/ion/impl/bin/WriteBufferTest.java index babf113abc..27b566d162 100644 --- a/test/com/amazon/ion/impl/bin/WriteBufferTest.java +++ b/test/com/amazon/ion/impl/bin/WriteBufferTest.java @@ -1073,4 +1073,24 @@ public void shiftBytesLeftAcrossBufferBlocksToStartThenWritePastBlockBoundary() buf.shiftBytesLeft(1, 6); assertBuffer("ABCDE01234B".getBytes()); } + + @Test + public void shiftBytesLeftWithLengthZero() { + assertEquals(11, ALLOCATOR.getBlockSize()); + buf.writeBytes("012345".getBytes()); + // Shift left by two, retaining zero bytes (i.e. truncate). + // In Ion, this situation occurs when a length bytes are preallocated for containers that end up being empty. + buf.shiftBytesLeft(0, 2); + assertBuffer("0123".getBytes()); + } + + @Test + public void shiftBytesLeftWithLengthZeroAcrossBlocks() { + assertEquals(11, ALLOCATOR.getBlockSize()); + buf.writeBytes("0123456789|0".getBytes()); + // Shift left by two, retaining zero bytes (i.e. truncate). + // In Ion, this situation occurs when a length bytes are preallocated for containers that end up being empty. + buf.shiftBytesLeft(0, 2); + assertBuffer("0123456789".getBytes()); + } } From 400eebf8c00852cb3c6f2c7a5eef22863b0fbfb7 Mon Sep 17 00:00:00 2001 From: jobarr-amzn <70981087+jobarr-amzn@users.noreply.github.com> Date: Wed, 23 Mar 2022 13:05:12 -0700 Subject: [PATCH 162/490] Release 1.9.3 (#423) --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e025fbfd54..79505eb868 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ dependency into your project's `pom.xml`: com.amazon.ion ion-java - 1.9.1 + 1.9.3 ``` diff --git a/pom.xml b/pom.xml index dbada93957..e4843549ad 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.9.3-SNAPSHOT + 1.9.3 bundle ${project.groupId}:${project.artifactId} From 9d1b3659a97810fe840bc41a4e87f13096167c08 Mon Sep 17 00:00:00 2001 From: jobarr-amzn <70981087+jobarr-amzn@users.noreply.github.com> Date: Wed, 23 Mar 2022 17:12:42 -0700 Subject: [PATCH 163/490] Bump version to 1.9.4-SNAPSHOT (#424) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e4843549ad..1813fe61f4 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.9.3 + 1.9.4-SNAPSHOT bundle ${project.groupId}:${project.artifactId} From 05f4b1df23380d6f95d463b21ffd8daed5142181 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Wed, 30 Mar 2022 13:53:48 -0700 Subject: [PATCH 164/490] Fixes a bug that prevented the incremental reader from recovering from an oversized value that occurred mid-stream. --- .../ion/impl/IonReaderLookaheadBuffer.java | 1 + .../impl/IonReaderBinaryIncrementalTest.java | 124 ++++++++++++++++++ 2 files changed, 125 insertions(+) diff --git a/src/com/amazon/ion/impl/IonReaderLookaheadBuffer.java b/src/com/amazon/ion/impl/IonReaderLookaheadBuffer.java index 174045c30e..7012cdfab3 100644 --- a/src/com/amazon/ion/impl/IonReaderLookaheadBuffer.java +++ b/src/com/amazon/ion/impl/IonReaderLookaheadBuffer.java @@ -516,6 +516,7 @@ private void shiftIndicesLeft(int afterIndex, int shiftAmount) { peekIndex = Math.max(peekIndex - shiftAmount, 0); valuePreHeaderIndex -= shiftAmount; valuePostHeaderIndex -= shiftAmount; + valueStartWriteIndex -= shiftAmount; for (Marker symbolTableMarker : symbolTableMarkers) { if (symbolTableMarker.startIndex > afterIndex) { symbolTableMarker.startIndex -= shiftAmount; diff --git a/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java b/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java index a00c3c2c6d..a99654fc09 100644 --- a/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java +++ b/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java @@ -2671,6 +2671,130 @@ public void onData(int numberOfBytes) { assertEquals(out.size(), byteCounter.get()); } + /** + * Calls next() on the given reader and returns the result. + * @param reader an Ion reader. + * @param pipe the stream from which the reader pulls data. If null, all data is available up front. + * @param source the source of data to be fed into the pipe. Only used if pipe is not null. + * @return the result of the first non-null call to reader.next(), or null if the source data is exhausted before + * reader.next() returns non-null. + */ + private static IonType ionReaderNext(IonReader reader, ResizingPipedInputStream pipe, ByteArrayInputStream source) { + if (pipe == null) { + return reader.next(); + } + while (source.available() > 0) { + pipe.receive(source.read()); + if (reader.next() != null) { + return reader.getType(); + } + } + return null; + } + + /** + * Verifies that oversized value handling works properly when the second value is oversized. + * @param withSymbolTable true if the first value should be preceded by a symbol table. + * @param withThirdValue true if the second (oversized) value should be followed by a third value that fits. + * @param byteByByte true if bytes should be fed to the reader one at a time. + * @throws Exception if thrown unexpectedly. + */ + private static void oversizedSecondValue( + boolean withSymbolTable, + boolean withThirdValue, + boolean byteByByte + ) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IonWriter writer = IonBinaryWriterBuilder.standard().build(out); + String firstValue = "12345678"; + if (withSymbolTable) { + writer.writeSymbol(firstValue); + } else { + writer.writeString(firstValue); + } + writer.writeString("abcdefghijklmnopqrstuvwxyz"); + String thirdValue = "abc"; + if (withThirdValue) { + writer.writeString(thirdValue); + } + writer.close(); + + final AtomicInteger oversizedCounter = new AtomicInteger(); + final AtomicInteger byteCounter = new AtomicInteger(); + UnifiedTestHandler handler = new UnifiedTestHandler() { + @Override + public void onOversizedSymbolTable() { + throw new IllegalStateException("not expected"); + } + + @Override + public void onOversizedValue() { + oversizedCounter.incrementAndGet(); + } + + @Override + public void onData(int numberOfBytes) { + byteCounter.addAndGet(numberOfBytes); + } + }; + // Greater than the first value (and symbol table, if any), less than the second and third values. + int maximumBufferSize = 25; + IonReaderBuilder builder = IonReaderBuilder.standard().withBufferConfiguration( + IonBufferConfiguration.Builder.standard() + .withInitialBufferSize(maximumBufferSize) + .withMaximumBufferSize(maximumBufferSize) + .onOversizedValue(handler) + .onOversizedSymbolTable(handler) + .onData(handler) + .build() + ); + ResizingPipedInputStream pipe = null; + ByteArrayInputStream source = new ByteArrayInputStream(out.toByteArray()); + InputStream in; + if (byteByByte) { + pipe = new ResizingPipedInputStream(out.size()); + in = pipe; + } else { + in = source; + } + IonReaderBinaryIncremental reader = new IonReaderBinaryIncremental(builder, in); + assertEquals(withSymbolTable ? IonType.SYMBOL : IonType.STRING, ionReaderNext(reader, pipe, source)); + assertEquals(0, oversizedCounter.get()); + assertEquals(firstValue, reader.stringValue()); + if (withThirdValue) { + assertEquals(IonType.STRING, ionReaderNext(reader, pipe, source)); + assertEquals(thirdValue, reader.stringValue()); + } + assertNull(ionReaderNext(reader, pipe, source)); + reader.close(); + assertEquals(1, oversizedCounter.get()); + assertEquals(out.size(), byteCounter.get()); + } + + @Test + public void oversizedSecondValueWithoutSymbolTable() throws Exception { + oversizedSecondValue(false, false, false); + oversizedSecondValue(false, true, false); + } + + @Test + public void oversizedSecondValueWithoutSymbolTableIncremental() throws Exception { + oversizedSecondValue(false, false, true); + oversizedSecondValue(false, true, true); + } + + @Test + public void oversizedSecondValueWithSymbolTable() throws Exception { + oversizedSecondValue(true, false, false); + oversizedSecondValue(true, true, false); + } + + @Test + public void oversizedSecondValueWithSymbolTableIncremental() throws Exception { + oversizedSecondValue(true, false, true); + oversizedSecondValue(true, true, true); + } + private void oversizeSymbolTableDetectedInHeader(int maximumBufferSize) throws Exception { // The system values require ~40 bytes (4 IVM, 5 symtab struct header, 1 'symbols' sid, 2 list header, 2 + 26 // for symbol 10. From fb9ddf792ac50f6494715c794d2a98ee29702e0f Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Fri, 1 Apr 2022 12:32:31 -0700 Subject: [PATCH 165/490] Adds missing validation for the length of symbol table child values. --- .../ion/impl/IonReaderBinaryIncremental.java | 3 +++ .../impl/IonReaderBinaryIncrementalTest.java | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java b/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java index 9cf786a823..43e67d52e1 100644 --- a/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java +++ b/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java @@ -962,6 +962,9 @@ private void readSymbolTable(IonReaderLookaheadBuffer.Marker marker) { } peekIndex = currentValueEndPosition; } + if (peekIndex > marker.endIndex) { + throw new IonException("Malformed symbol table. Child values exceeded the length declared in the header."); + } if (!hasSeenImports) { resetSymbolTable(); resetImports(); diff --git a/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java b/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java index a99654fc09..60e4d8fa64 100644 --- a/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java +++ b/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java @@ -3388,4 +3388,22 @@ public void annotationIteratorReuseEnabled() throws Exception { public void annotationIteratorReuseDisabled() throws Exception { annotationIteratorReuse(false); } + + @Test + public void failsOnMalformedSymbolTable() { + byte[] data = bytes( + 0xE0, 0x01, 0x00, 0xEA, // Binary IVM + 0xE6, // 6-byte annotation wrapper + 0x81, // 1 byte of annotation SIDs + 0x83, // SID 3 ($ion_symbol_table) + 0xD3, // 3-byte struct + 0x84, // Field name SID 4 (name) + 0xE7, // 7-byte annotation wrapper (error: there should only be two bytes remaining). + 0x81, // Junk byte to fill the 6 bytes of the annotation wrapper and 3 bytes of the struct. + 0x20 // Next top-level value (int 0). + ); + IonReader reader = newBoundedIncrementalReader(data, 1024); + thrown.expect(IonException.class); + reader.next(); + } } From 90569aefb6fb4cf3f3830fa8bcd255fd48635044 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Tue, 5 Apr 2022 15:49:00 -0700 Subject: [PATCH 166/490] Fixes an incorrect comment in IonReaderBinaryIncrementalTest. --- test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java b/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java index 60e4d8fa64..534cffdbdb 100644 --- a/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java +++ b/test/com/amazon/ion/impl/IonReaderBinaryIncrementalTest.java @@ -2737,7 +2737,7 @@ public void onData(int numberOfBytes) { byteCounter.addAndGet(numberOfBytes); } }; - // Greater than the first value (and symbol table, if any), less than the second and third values. + // Greater than the first value (and symbol table, if any) and third value, less than the second value. int maximumBufferSize = 25; IonReaderBuilder builder = IonReaderBuilder.standard().withBufferConfiguration( IonBufferConfiguration.Builder.standard() From 9f97a6d0e20a414d010087701d37f6506e22460f Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Tue, 5 Apr 2022 16:15:58 -0700 Subject: [PATCH 167/490] Prepares version 1.9.4 for release. --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 79505eb868..04add8c946 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ dependency into your project's `pom.xml`: com.amazon.ion ion-java - 1.9.3 + 1.9.4 ``` diff --git a/pom.xml b/pom.xml index 1813fe61f4..24253ca539 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.9.4-SNAPSHOT + 1.9.4 bundle ${project.groupId}:${project.artifactId} From ab41cd18e6727ad53ca354752cdefcaa374a503c Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Wed, 6 Apr 2022 10:52:19 -0700 Subject: [PATCH 168/490] Bumps version to 1.9.5-SNAPSHOT (#431) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 24253ca539..4a3d696a2c 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.9.4 + 1.9.5-SNAPSHOT bundle ${project.groupId}:${project.artifactId} From ee2a306d3ecb509973b708afcbf0ddab9dd31887 Mon Sep 17 00:00:00 2001 From: jobarr-amzn <70981087+jobarr-amzn@users.noreply.github.com> Date: Wed, 11 May 2022 12:57:37 -0700 Subject: [PATCH 169/490] Jar info changes (#433) * adds changes for JarInfo file * adds build.properties * modfies JarInfo build logic * removes manifest file logic * modifies unit test for JarInfo * Use filtering to get properties interpolation Also clean up pom.xml * Look for the correct .properties file, add tests Co-authored-by: Khushboo Desai --- pom.xml | 74 ++++++++------- src/com/amazon/ion/util/JarInfo.java | 108 +++++++++------------- src/main/resources/build.properties | 4 + test/com/amazon/ion/util/JarInfoTest.java | 98 ++++---------------- 4 files changed, 102 insertions(+), 182 deletions(-) create mode 100644 src/main/resources/build.properties diff --git a/pom.xml b/pom.xml index 4a3d696a2c..781b0d20f2 100644 --- a/pom.xml +++ b/pom.xml @@ -65,7 +65,17 @@ UTF-8 yyyy ${maven.build.timestamp} - 1.6 + + yyyy-MM-dd'T'HH:mm:ssXXX + ${maven.build.timestamp} + + 6 + 6 @@ -76,6 +86,13 @@ 4.13.1 test + + + org.hamcrest + hamcrest + 2.2 + test + pl.pragmatists JUnitParams @@ -85,43 +102,29 @@ + + + + src/main/resources + true + + src test + org.apache.maven.plugins maven-compiler-plugin - 3.8.0 + 3.8.1 - ${jdkVersion} - ${jdkVersion} + ${jdk.source.version} + ${jdk.target.version} - - - org.apache.maven.plugins - maven-antrun-plugin - 1.8 - - - initialize - - run - - - true - - - - - - - - - + org.apache.maven.plugins @@ -134,6 +137,7 @@ -ea -Xmx3000M ${argLine} + org.apache.felix maven-bundle-plugin @@ -148,13 +152,11 @@ com.amazon.ion.impl._Private_CommandLine - - ${build.time} - ${project.version} com.amazon.ion + org.jacoco @@ -271,11 +273,9 @@ release - - 1.5 - + org.apache.maven.plugins @@ -290,6 +290,7 @@ + org.apache.maven.plugins @@ -318,6 +319,7 @@ + org.apache.maven.plugins @@ -333,6 +335,7 @@ + org.sonatype.plugins @@ -345,6 +348,7 @@ true + diff --git a/src/com/amazon/ion/util/JarInfo.java b/src/com/amazon/ion/util/JarInfo.java index 1f0ba4807f..061ee2340c 100644 --- a/src/com/amazon/ion/util/JarInfo.java +++ b/src/com/amazon/ion/util/JarInfo.java @@ -16,10 +16,12 @@ package com.amazon.ion.util; import java.io.IOException; +import java.io.InputStream; import java.net.URL; import java.util.ArrayList; import java.util.Enumeration; import java.util.List; +import java.util.Properties; import java.util.jar.Attributes; import java.util.jar.Manifest; import com.amazon.ion.IonException; @@ -31,11 +33,6 @@ */ public final class JarInfo { - - private static final String MANIFEST_FILE = "META-INF/MANIFEST.MF"; - private static final String BUILD_TIME_ATTRIBUTE = "Ion-Java-Build-Time"; - private static final String PROJECT_VERSION_ATTRIBUTE = "Ion-Java-Project-Version"; - private String ourProjectVersion; private Timestamp ourBuildTime; @@ -49,34 +46,9 @@ public final class JarInfo */ public JarInfo() throws IonException { - Enumeration manifestUrls; - try - { - manifestUrls = getClass().getClassLoader().getResources(MANIFEST_FILE); - } - catch (IOException e) - { - throw new IonException("Unable to load manifests.", e); - } - List manifests = new ArrayList(); - while (manifestUrls.hasMoreElements()) - { - try - { - manifests.add(new Manifest(manifestUrls.nextElement().openStream())); - } - catch (IOException e) - { - continue; // try the next manifest - } - } - loadBuildProperties(manifests); + loadBuildProperties(); } - JarInfo(List manifests) - { - loadBuildProperties(manifests); - } /** * Gets the ion-java project version of this build. @@ -101,50 +73,54 @@ public Timestamp getBuildTime() // TODO writeTo(IonWriter) // ======================================================================== - - private void loadBuildProperties(List manifests) throws IonException + /** + * @return null but not empty string + */ + private static String nonEmptyProperty(Properties props, String name) { - boolean propertiesLoaded = false; - for(Manifest manifest : manifests) - { - boolean success = tryLoadBuildProperties(manifest); - if(success && propertiesLoaded) - { - // In the event of conflicting manifests, fail instead of risking returning incorrect version info. - throw new IonException("Found multiple manifests with ion-java version info on the classpath."); - } - propertiesLoaded |= success; - } - if (!propertiesLoaded) - { - throw new IonException("Unable to locate manifest with ion-java version info on the classpath."); - } + String value = props.getProperty(name, ""); + if (value.length() == 0) value = null; + return value; } - /* - * Returns true if the properties were loaded, otherwise false. - */ - private boolean tryLoadBuildProperties(Manifest manifest) + private void loadBuildProperties() + throws IonException { - Attributes mainAttributes = manifest.getMainAttributes(); - String projectVersion = mainAttributes.getValue(PROJECT_VERSION_ATTRIBUTE); - String time = mainAttributes.getValue(BUILD_TIME_ATTRIBUTE); - - if (projectVersion == null || time == null) + String file = "/build.properties"; + try { - return false; - } + Properties props = new Properties(); - ourProjectVersion = projectVersion; + InputStream in = getClass().getResourceAsStream(file); + if (in != null) + { + try + { + props.load(in); + } + finally + { + in.close(); + } + } - try - { - ourBuildTime = Timestamp.valueOf(time); + ourProjectVersion = nonEmptyProperty(props, "build.version"); + + String time = nonEmptyProperty(props, "build.time"); + if (time != null) + { + try { + ourBuildTime = Timestamp.valueOf(time); + } + catch (IllegalArgumentException e) + { + // Badly formatted timestamp. Ignore it. + } + } } - catch (IllegalArgumentException e) + catch (IOException e) { - // Badly formatted timestamp. Ignore it. + throw new IonException("Unable to load " + file, e); } - return true; } } \ No newline at end of file diff --git a/src/main/resources/build.properties b/src/main/resources/build.properties new file mode 100644 index 0000000000..3894cedaa4 --- /dev/null +++ b/src/main/resources/build.properties @@ -0,0 +1,4 @@ +# These values are defined in pom.xml +build.version=${pom.version} +build.time=${build.time} +build.name=${pom.name} diff --git a/test/com/amazon/ion/util/JarInfoTest.java b/test/com/amazon/ion/util/JarInfoTest.java index 3994ed4e15..f51e17892d 100644 --- a/test/com/amazon/ion/util/JarInfoTest.java +++ b/test/com/amazon/ion/util/JarInfoTest.java @@ -15,95 +15,31 @@ package com.amazon.ion.util; -import com.amazon.ion.IonException; import com.amazon.ion.Timestamp; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.jar.Manifest; - -import static org.junit.Assert.*; - -public class JarInfoTest -{ - @Rule - public ExpectedException thrown = ExpectedException.none(); - - private static String getAttributes(String manifestVersion) - { - return "Manifest-Version: " + manifestVersion + "\n"; - } - - private static String getIonJavaAttributes(String buildTime, String version) - { - return getAttributes("1.0") - + "Ion-Java-Build-Time: " + buildTime + "\n" - + "Ion-Java-Project-Version: " + version + "\n"; - } - - private static Manifest getManifest(String attributes) throws IOException - { - return new Manifest(new ByteArrayInputStream(attributes.getBytes("UTF-8"))); - } +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.matchesPattern; +public class JarInfoTest { @Test - public void testSingleManifest() throws Exception - { - String expectedBuildTime = "1984T"; - String expectedVersion = "42.0"; - - Manifest manifest = getManifest(getIonJavaAttributes(expectedBuildTime, expectedVersion)); - JarInfo info = new JarInfo(Collections.singletonList(manifest)); - - assertEquals(expectedVersion, info.getProjectVersion()); - assertEquals(Timestamp.valueOf(expectedBuildTime), info.getBuildTime()); + public void getBuildTime() { + JarInfo info = new JarInfo(); + Timestamp timestamp = info.getBuildTime(); + // This test was written at the timestamp below, the loaded time should always be greater + Timestamp original = Timestamp.valueOf("2022-05-10T22:56:48Z"); + assertThat(timestamp, greaterThan(original)); } @Test - public void testMultipleManifests() throws Exception - { - String expectedBuildTime = "1984T"; - String expectedVersion = "42.0"; - - List manifests = new ArrayList(); - manifests.add(getManifest(getAttributes("1.0"))); - manifests.add(getManifest(getIonJavaAttributes(expectedBuildTime, expectedVersion))); - manifests.add(getManifest(getAttributes("2.0"))); - - JarInfo info = new JarInfo(manifests); - - assertEquals(expectedVersion, info.getProjectVersion()); - assertEquals(Timestamp.valueOf(expectedBuildTime), info.getBuildTime()); - } - - @Test - public void testNoMatchingManifests() throws Exception - { - List manifests = new ArrayList(); - manifests.add(getManifest(getAttributes("1.0"))); - manifests.add(getManifest(getAttributes("2.0"))); - manifests.add(getManifest(getAttributes("3.0"))); - - thrown.expect(IonException.class); - new JarInfo(manifests); - } - - @Test - public void testConflictingManifests() throws Exception - { - List manifests = new ArrayList(); - manifests.add(getManifest(getIonJavaAttributes("1984T", "42.0"))); - manifests.add(getManifest(getAttributes("1.0"))); - manifests.add(getManifest(getIonJavaAttributes("1985T", "43.0"))); - - thrown.expect(IonException.class); - new JarInfo(manifests); + public void getProjectVersion() { + JarInfo info = new JarInfo(); + String projectVersion = info.getProjectVersion(); + // Semantic version MAJOR.MINOR.PATCH with optional -SNAPSHOT suffix + // Should always match our project version + String projectVersionPattern = "\\d+\\.\\d+\\.\\d+(-SNAPSHOT)?"; + assertThat(projectVersion, matchesPattern(projectVersionPattern)); } } \ No newline at end of file From 53394b6c5ab92280f6f727d33acde93125087b8f Mon Sep 17 00:00:00 2001 From: linlin-s Date: Tue, 9 Aug 2022 15:14:55 -0700 Subject: [PATCH 170/490] Updates ion-java-performance-regression-detector workflow. (#435) --- .../ion-java-performance-regression-detector.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ion-java-performance-regression-detector.yml b/.github/workflows/ion-java-performance-regression-detector.yml index abf209448c..216331a1c5 100644 --- a/.github/workflows/ion-java-performance-regression-detector.yml +++ b/.github/workflows/ion-java-performance-regression-detector.yml @@ -43,9 +43,9 @@ jobs: - name: Generate test Ion Data run: | mkdir -p testData - java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar generate -S 50000 --input-ion-schema ion-java-benchmark-cli/tst/com/amazon/ion/benchmark/testStruct.isl testData/testStruct.10n - java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar generate -S 50000 --input-ion-schema ion-java-benchmark-cli/tst/com/amazon/ion/benchmark/testList.isl testData/testList.10n - java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar generate -S 50000 --input-ion-schema ion-java-benchmark-cli/tst/com/amazon/ion/benchmark/testNestedStruct.isl testData/testNestedStruct.10n + java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar generate -S 50000 --input-ion-schema ion-java-benchmark-cli/tst/com/amazon/ion/workflow/nestedStruct.isl testData/testStructs.10n + java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar generate -S 50000 --input-ion-schema ion-java-benchmark-cli/tst/com/amazon/ion/workflow/nestedList.isl testData/testLists.10n + java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar generate -S 50000 --input-ion-schema ion-java-benchmark-cli/tst/com/amazon/ion/workflow/sexp.isl testData/testSexps.10n - name: Upload test Ion Data to artifacts uses: actions/upload-artifact@v2 @@ -106,9 +106,9 @@ jobs: id: regression_result run: | result=true - cd benchmarkResults && for FILE in *; do message=$(java -jar /home/runner/work/ion-java/ion-java/ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous $FILE/previous.ion --benchmark-result-new $FILE/new.ion $FILE/report.ion | tee /dev/stderr) && if [ "$message" != "" ]; then result=false; fi; done + cd benchmarkResults && for FILE in *; do message=$(java -jar /home/runner/work/ion-java/ion-java/ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous $FILE/previous.ion --benchmark-result-new $FILE/new.ion $FILE/report.ion | tee /dev/stderr) && if [ "$message" != "no regression detected" ]; then result=false; fi; done echo "::set-output name=regression-result::$result" - echo $result + if [ "$result" = "true" ]; then echo "No regression detected!" >> $GITHUB_STEP_SUMMARY; fi - name: Upload comparison reports to the benchmark results directory uses: actions/upload-artifact@v2 @@ -120,4 +120,6 @@ jobs: env: regression_detect: ${{steps.regression_result.outputs.regression-result}} if: ${{ env.regression_detect == 'false' }} - run: exit 1 + run: | + cd benchmarkResults && echo "| Benchmark command | GC Allocation Rate | Heap Usage | Speed |" >> $GITHUB_STEP_SUMMARY && echo "| ----------- | ----------- |----------- | ----------- |" >> $GITHUB_STEP_SUMMARY && for FILE in *; do regressionDetection=$(java -jar /home/runner/work/ion-java/ion-java/ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous $FILE/previous.ion --benchmark-result-new $FILE/new.ion $FILE/report.ion) && if [ "$regressionDetection" != "no regression detected" ]; then command=$(echo $FILE | sed "s/_/ /g") && read gc heapUsage speed <<< $( echo ${regressionDetection} | awk -F", " '{print $1" "$2" "$3}' ) && echo "|$command|$gc|$heapUsage|$speed|" >> $GITHUB_STEP_SUMMARY; fi; done + exit 1 From e90a6361dd7fdd28b569b3a16405e6708f19f1fd Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Tue, 16 Aug 2022 17:53:25 -0700 Subject: [PATCH 171/490] Fixes a bug in the incremental reader that caused certain Decimal, BigDecimal, and BigInteger values to be read incorrectly in multi-reader contexts. --- src/com/amazon/ion/impl/IonReaderBinaryIncremental.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java b/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java index 43e67d52e1..f5487b7dbc 100644 --- a/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java +++ b/src/com/amazon/ion/impl/IonReaderBinaryIncremental.java @@ -1374,7 +1374,7 @@ private int readVarInt() { } // Scratch space for various byte sizes. Only for use while computing a single value. - private static final byte[][] SCRATCH_FOR_SIZE = new byte[][] { + private final byte[][] scratchForSize = new byte[][] { new byte[0], new byte[1], new byte[2], @@ -1400,8 +1400,8 @@ private byte[] copyBytesToScratch(int startIndex, int length) { // Note: using reusable scratch buffers makes reading ints and decimals 1-5% faster and causes much less // GC churn. byte[] bytes = null; - if (length < SCRATCH_FOR_SIZE.length) { - bytes = SCRATCH_FOR_SIZE[length]; + if (length < scratchForSize.length) { + bytes = scratchForSize[length]; } if (bytes == null) { bytes = new byte[length]; From a90e7ca511e77ee23da63781587f0b44459f5164 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Tue, 16 Aug 2022 21:34:25 -0700 Subject: [PATCH 172/490] Prepares version 1.9.5 for release. --- README.md | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 04add8c946..a77b9d7215 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ dependency into your project's `pom.xml`: com.amazon.ion ion-java - 1.9.4 + 1.9.5 ``` diff --git a/pom.xml b/pom.xml index 781b0d20f2..e6822cef39 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.9.5-SNAPSHOT + 1.9.5 bundle ${project.groupId}:${project.artifactId} From b885cbc6334c8ec85928f78dbfdad15ef1b42cb3 Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Tue, 16 Aug 2022 23:21:20 -0700 Subject: [PATCH 173/490] Bumps version to 1.9.6-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e6822cef39..1d8e61fb7f 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 4.0.0 com.amazon.ion ion-java - 1.9.5 + 1.9.6-SNAPSHOT bundle ${project.groupId}:${project.artifactId} From d8deff1c38b01616cf5ad27bf93b3f2ef8bc6241 Mon Sep 17 00:00:00 2001 From: Rob Marrowstone <99764876+rmarrowstone@users.noreply.github.com> Date: Wed, 7 Sep 2022 16:27:38 -0700 Subject: [PATCH 174/490] Fix Annotated Dot in Sexp Parser Error (#440) This fixes GH 434 by adding the proper transition to the state machine. There are two trivial code cleanups in related code. One drops duplicate branch logic in the switch to the next state. The other removes a boolean comparison by negating the variable instead. I am adding a regression test to the ion-tests in a seperate PR. --- src/com/amazon/ion/impl/IonReaderTextRawTokensX.java | 5 +---- src/com/amazon/ion/impl/IonReaderTextRawX.java | 1 + src/com/amazon/ion/impl/IonReaderTextUserX.java | 2 +- test/com/amazon/ion/SexpTest.java | 11 +++++++++++ 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/com/amazon/ion/impl/IonReaderTextRawTokensX.java b/src/com/amazon/ion/impl/IonReaderTextRawTokensX.java index bdf3462fb4..c74bb36406 100644 --- a/src/com/amazon/ion/impl/IonReaderTextRawTokensX.java +++ b/src/com/amazon/ion/impl/IonReaderTextRawTokensX.java @@ -616,9 +616,6 @@ public final int nextToken() throws IOException switch (c) { case -1: return next_token_finish(IonTokenConstsX.TOKEN_EOF, true); - case '/': - unread_char(c); - return next_token_finish(IonTokenConstsX.TOKEN_SYMBOL_OPERATOR, true); case ':': c2 = read_char(); if (c2 != ':') { @@ -670,7 +667,7 @@ public final int nextToken() throws IOException } unread_char(c); return next_token_finish(IonTokenConstsX.TOKEN_SYMBOL_OPERATOR, true); - case '#': + case '#': case '/': case '<': case '>': case '*': case '=': case '^': case '&': case '|': case '~': case ';': case '!': case '?': case '@': case '%': case '`': unread_char(c); diff --git a/src/com/amazon/ion/impl/IonReaderTextRawX.java b/src/com/amazon/ion/impl/IonReaderTextRawX.java index 437094c920..7729340369 100644 --- a/src/com/amazon/ion/impl/IonReaderTextRawX.java +++ b/src/com/amazon/ion/impl/IonReaderTextRawX.java @@ -202,6 +202,7 @@ static final int[][] makeTransitionActionArray() actions[STATE_BEFORE_VALUE_CONTENT_SEXP][IonTokenConstsX.TOKEN_SYMBOL_IDENTIFIER]= ACTION_LOAD_SCALAR; actions[STATE_BEFORE_VALUE_CONTENT_SEXP][IonTokenConstsX.TOKEN_SYMBOL_QUOTED] = ACTION_LOAD_SCALAR; actions[STATE_BEFORE_VALUE_CONTENT_SEXP][IonTokenConstsX.TOKEN_SYMBOL_OPERATOR] = ACTION_LOAD_SCALAR; + actions[STATE_BEFORE_VALUE_CONTENT_SEXP][IonTokenConstsX.TOKEN_DOT] = ACTION_LOAD_SCALAR; actions[STATE_BEFORE_FIELD_NAME][IonTokenConstsX.TOKEN_EOF] = 0; actions[STATE_BEFORE_FIELD_NAME][IonTokenConstsX.TOKEN_SYMBOL_IDENTIFIER] = ACTION_LOAD_FIELD_NAME; diff --git a/src/com/amazon/ion/impl/IonReaderTextUserX.java b/src/com/amazon/ion/impl/IonReaderTextUserX.java index 1b0d0dcd7d..61e09041ef 100644 --- a/src/com/amazon/ion/impl/IonReaderTextUserX.java +++ b/src/com/amazon/ion/impl/IonReaderTextUserX.java @@ -167,7 +167,7 @@ private final boolean has_next_user_value() } } } - return (_eof != true); + return (!_eof); } private static boolean isIonVersionMarker(String text) diff --git a/test/com/amazon/ion/SexpTest.java b/test/com/amazon/ion/SexpTest.java index 18ceca29e6..aa5dc90994 100644 --- a/test/com/amazon/ion/SexpTest.java +++ b/test/com/amazon/ion/SexpTest.java @@ -149,6 +149,17 @@ public void testTrickyParsing() assertEquals(3, value.size()); } + @Test + public void testAnnotatedDotInSexp() + throws Exception + { + IonSexp value = (IonSexp) oneValue("(a::.)"); + assertEquals(1, value.size()); + IonSymbol item = (IonSymbol) value.get(0); + checkSymbol(".", item); + checkAnnotation("a", item); + } + @Test public void testNegativeNumbersInSexp() throws Exception From c939aed27e4d3596ff48c3196a8766c5c77aa976 Mon Sep 17 00:00:00 2001 From: linlin-s Date: Mon, 12 Sep 2022 13:35:15 -0700 Subject: [PATCH 175/490] Adds real-world-data schema to generate more test data. (#446) --- .github/workflows/ion-java-performance-regression-detector.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ion-java-performance-regression-detector.yml b/.github/workflows/ion-java-performance-regression-detector.yml index 216331a1c5..e222ec9e0e 100644 --- a/.github/workflows/ion-java-performance-regression-detector.yml +++ b/.github/workflows/ion-java-performance-regression-detector.yml @@ -46,6 +46,9 @@ jobs: java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar generate -S 50000 --input-ion-schema ion-java-benchmark-cli/tst/com/amazon/ion/workflow/nestedStruct.isl testData/testStructs.10n java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar generate -S 50000 --input-ion-schema ion-java-benchmark-cli/tst/com/amazon/ion/workflow/nestedList.isl testData/testLists.10n java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar generate -S 50000 --input-ion-schema ion-java-benchmark-cli/tst/com/amazon/ion/workflow/sexp.isl testData/testSexps.10n + java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar generate -S 50000 --input-ion-schema ion-java-benchmark-cli/tst/com/amazon/ion/workflow/realWorldDataSchema01.isl testData/realWorldData01.10n + java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar generate -S 50000 --input-ion-schema ion-java-benchmark-cli/tst/com/amazon/ion/workflow/realWorldDataSchema02.isl testData/realWorldData02.10n + java -jar ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar generate -S 50000 --input-ion-schema ion-java-benchmark-cli/tst/com/amazon/ion/workflow/realWorldDataSchema03.isl testData/realWorldData03.10n - name: Upload test Ion Data to artifacts uses: actions/upload-artifact@v2 From 3fa498079b28ec51b1def16478a6f43f4f47741f Mon Sep 17 00:00:00 2001 From: linlin-s Date: Wed, 14 Sep 2022 11:06:22 -0700 Subject: [PATCH 176/490] Update ion-java-regression-detection workflow. (#449) --- ...on-java-performance-regression-detector.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ion-java-performance-regression-detector.yml b/.github/workflows/ion-java-performance-regression-detector.yml index e222ec9e0e..0473069243 100644 --- a/.github/workflows/ion-java-performance-regression-detector.yml +++ b/.github/workflows/ion-java-performance-regression-detector.yml @@ -20,12 +20,11 @@ jobs: - name: Checkout ion-java from the new commit. uses: actions/checkout@v2 with: - fetch-depth: 2 - submodules: recursive - path: ion-java + ref: ${{ github.event.pull_request.head.sha }} + path: ion-java-new - name: Build ion-java from the new commit - run: cd ion-java && mvn clean install + run: cd ion-java-new && git submodule init && git submodule update && mvn clean install - name: Checkout ion-java-benchmark-cli uses: actions/checkout@v2 @@ -70,8 +69,15 @@ jobs: - name: Clean maven dependencies repository run : rm -r /home/runner/.m2 + - name: Checkout the current commit + uses: actions/checkout@v2 + with: + repository: amzn/ion-java + ref: master + path: ion-java + - name: Build ion-java from the previous commit - run: cd ion-java && git checkout HEAD^ && git submodule update --recursive && mvn clean install + run: cd ion-java && git submodule init && git submodule update && mvn clean install - name: Build ion-java-benchmark-cli run: cd ion-java-benchmark-cli && mvn clean install @@ -125,4 +131,4 @@ jobs: if: ${{ env.regression_detect == 'false' }} run: | cd benchmarkResults && echo "| Benchmark command | GC Allocation Rate | Heap Usage | Speed |" >> $GITHUB_STEP_SUMMARY && echo "| ----------- | ----------- |----------- | ----------- |" >> $GITHUB_STEP_SUMMARY && for FILE in *; do regressionDetection=$(java -jar /home/runner/work/ion-java/ion-java/ion-java-benchmark-cli/target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous $FILE/previous.ion --benchmark-result-new $FILE/new.ion $FILE/report.ion) && if [ "$regressionDetection" != "no regression detected" ]; then command=$(echo $FILE | sed "s/_/ /g") && read gc heapUsage speed <<< $( echo ${regressionDetection} | awk -F", " '{print $1" "$2" "$3}' ) && echo "|$command|$gc|$heapUsage|$speed|" >> $GITHUB_STEP_SUMMARY; fi; done - exit 1 + exit 1 \ No newline at end of file From 57bd49bca6ec6d8b1f66f5648f1846ed48f18a4d Mon Sep 17 00:00:00 2001 From: linlin-s Date: Tue, 18 Oct 2022 17:33:54 -0700 Subject: [PATCH 177/490] Adds workflows to create ion-java prerelease on GitHub. (#452) --- ...ub-prerelease-after-fast-forward-check.yml | 74 +++++++++++++++ .../version-bump-and-fast-forward-check.yml | 91 +++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 .github/workflows/create-gitHub-prerelease-after-fast-forward-check.yml create mode 100644 .github/workflows/version-bump-and-fast-forward-check.yml diff --git a/.github/workflows/create-gitHub-prerelease-after-fast-forward-check.yml b/.github/workflows/create-gitHub-prerelease-after-fast-forward-check.yml new file mode 100644 index 0000000000..d28a958a74 --- /dev/null +++ b/.github/workflows/create-gitHub-prerelease-after-fast-forward-check.yml @@ -0,0 +1,74 @@ +name: Create GitHub prerelease after fast-forward check +on: + pull_request: + types: + - closed + +jobs: + check-version-change: + if: github.event.pull_request.merged == true + name: check-version-change + runs-on: ubuntu-latest + outputs: + output1: ${{ steps.check-version-change.outputs.result }} + steps: + - name: Checkout the current repository master branch + uses: actions/checkout@v3 + with: + repository: amzn/ion-java + ref: master + fetch-depth: 2 + path: ion-java-current + + - name: Check whether the pull request contains version number bump + id: check-version-change + run: | + cd ion-java-current + versionNew=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + git reset --hard HEAD^ + versionPrevious=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + if [[ $versionNew != $versionPrevious ]]; then echo "::set-output name=result::pass"; else echo "::set-output name=result::failed"; fi + + prerelease: + name: prerelease + needs: check-version-change + if: ${{ needs.check-version-change.outputs.output1 == 'pass' }} + runs-on: ubuntu-latest + steps: + - name: Access to the incoming release + uses: actions/checkout@v3 + with: + repository: amzn/ion-java + path: current_release + + - name: Build ion-java + run: | + cd current_release + mvn clean install -DskipTests + + - name: Get the parameters of executable jar + run: | + cd current_release/target + echo "path=$(readlink -f $(ls *.jar))" >> $GITHUB_ENV + echo "file_name=$(ls *.jar)" >> $GITHUB_ENV + cd .. && echo "version_number=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)" >> $GITHUB_ENV + + - name: Create new release + id: create_new_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: v${{ env.version_number }} + release_name: v${{ env.version_number }} + prerelease: true + + - name: Upload release asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_new_release.outputs.upload_url }} + asset_path: ${{ env.path }} + asset_name: ${{ env.file_name }} + asset_content_type: application/zip \ No newline at end of file diff --git a/.github/workflows/version-bump-and-fast-forward-check.yml b/.github/workflows/version-bump-and-fast-forward-check.yml new file mode 100644 index 0000000000..d355144565 --- /dev/null +++ b/.github/workflows/version-bump-and-fast-forward-check.yml @@ -0,0 +1,91 @@ +name: Version-bump-and-fast-forward-check +on: [pull_request] + +jobs: + check-version-bump: + name: check-version-bump + runs-on: ubuntu-latest + outputs: + output1: ${{ steps.check-version-change.outputs.result }} + steps: + - name: Switch to the branch of the pull request + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + path: ion-java-new + + - name: Checkout the current repository master branch + uses: actions/checkout@v3 + with: + repository: amzn/ion-java + ref: master + path: ion-java-current + + - name: Check whether the pull request contains version number bump + id: check-version-change + run: | + cd /home/runner/work/ion-java/ion-java/ion-java-new + versionNew=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + commitID=$(git rev-parse HEAD) + cd /home/runner/work/ion-java/ion-java/ion-java-current + versionCurrent=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + if [[ $versionNew != $versionCurrent ]]; then echo "::set-output name=result::pass"; else echo "::set-output name=result::failed"; fi + + fast-forward-check: + needs: check-version-bump + name: fast-forward-check + if: ${{ needs.check-version-bump.outputs.output1 == 'pass' }} + runs-on: ubuntu-latest + steps: + - name: Access to the last release + uses: oprypin/find-latest-tag@v1 + id: check_last_release + with: + repository: amzn/ion-java + releases-only: true + + - name: Checkout the last release + uses: actions/checkout@v3 + with: + repository: amzn/ion-java + ref: ${{ steps.check_last_release.outputs.tag }} + path: last-release + + - name: Get the commit id of the last release + run: | + cd last-release + echo "last_release_commit_id=$(git rev-parse HEAD)" >> $GITHUB_ENV + + - name: Access to the incoming release + uses: actions/checkout@v3 + with: + repository: amzn/ion-java + ref: ${{ github.event.pull_request.head.sha }} + path: ion-java-new + + - name: Get the commit id of the incoming release + run: | + cd ion-java-new + echo "current_release_commit_id=$(git rev-parse HEAD)" >> $GITHUB_ENV + echo $(git rev-parse HEAD) + + - name: Check whether the incoming release is fast-forward compared to the last release + run: | + cd ion-java-new + if git merge-base --is-ancestor ${{env.last_release_commit_id}} ${{env.current_release_commit_id}}; then echo "Trigger github release"; else exit 1; fi + + - name: comment + if: ${{ success() }} + uses: peter-evans/create-or-update-comment@v2 + with: + issue-number: ${{ github.event.pull_request.number }} + body: | + This PR passes version bump check and fast-forward check, please double check the version number and merge this PR. After merging this PR, "Create gitHub prerelease after fast-forward check" workflow will be automatically triggered to create prerelease. + + - name: comment + if: ${{ failure() }} + uses: peter-evans/create-or-update-comment@v2 + with: + issue-number: ${{ github.event.pull_request.number }} + body: | + This PR is not fast-forward compared to the last release, please check. From 2e673e31bc470d7b5fd14a3554c196e83c29fe32 Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Thu, 10 Nov 2022 10:12:43 -0800 Subject: [PATCH 178/490] Use Gradle build system --- ...ub-prerelease-after-fast-forward-check.yml | 8 +- ...n-java-performance-regression-detector.yml | 4 +- .github/workflows/main.yml | 20 +- .../version-bump-and-fast-forward-check.yml | 4 +- README.md | 11 +- build.gradle.kts | 160 ++++++++ gradle.properties | 6 + gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 60756 bytes gradle/wrapper/gradle-wrapper.properties | 5 + gradlew | 240 ++++++++++++ gradlew.bat | 91 +++++ ion-java-cli/README.md | 9 +- ion-java-cli/build.gradle.kts | 28 ++ ion-java-cli/pom.xml | 121 ------ ion-test-driver-run | 2 + ion-test-driver-setup | 2 + pom.xml | 357 ------------------ project.version | 1 + settings.gradle.kts | 2 + src/com/amazon/ion/util/JarInfo.java | 2 +- src/main/resources/build.properties | 4 - 21 files changed, 567 insertions(+), 510 deletions(-) create mode 100644 build.gradle.kts create mode 100644 gradle.properties create mode 100644 gradle/wrapper/gradle-wrapper.jar create mode 100644 gradle/wrapper/gradle-wrapper.properties create mode 100755 gradlew create mode 100644 gradlew.bat create mode 100644 ion-java-cli/build.gradle.kts delete mode 100644 ion-java-cli/pom.xml create mode 100755 ion-test-driver-run create mode 100755 ion-test-driver-setup delete mode 100644 pom.xml create mode 100644 project.version create mode 100644 settings.gradle.kts delete mode 100644 src/main/resources/build.properties diff --git a/.github/workflows/create-gitHub-prerelease-after-fast-forward-check.yml b/.github/workflows/create-gitHub-prerelease-after-fast-forward-check.yml index d28a958a74..aa54308bd3 100644 --- a/.github/workflows/create-gitHub-prerelease-after-fast-forward-check.yml +++ b/.github/workflows/create-gitHub-prerelease-after-fast-forward-check.yml @@ -24,9 +24,9 @@ jobs: id: check-version-change run: | cd ion-java-current - versionNew=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + versionNew="$(> $GITHUB_ENV echo "file_name=$(ls *.jar)" >> $GITHUB_ENV - cd .. && echo "version_number=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)" >> $GITHUB_ENV + cd .. && echo "version_number=$(> $GITHUB_ENV - name: Create new release id: create_new_release diff --git a/.github/workflows/ion-java-performance-regression-detector.yml b/.github/workflows/ion-java-performance-regression-detector.yml index 0473069243..277ccff505 100644 --- a/.github/workflows/ion-java-performance-regression-detector.yml +++ b/.github/workflows/ion-java-performance-regression-detector.yml @@ -24,7 +24,7 @@ jobs: path: ion-java-new - name: Build ion-java from the new commit - run: cd ion-java-new && git submodule init && git submodule update && mvn clean install + run: cd ion-java-new && git submodule init && git submodule update && ./gradlew clean publishToMavenLocal - name: Checkout ion-java-benchmark-cli uses: actions/checkout@v2 @@ -77,7 +77,7 @@ jobs: path: ion-java - name: Build ion-java from the previous commit - run: cd ion-java && git submodule init && git submodule update && mvn clean install + run: cd ion-java && git submodule init && git submodule update && ./gradlew clean publishToMavenLocal - name: Build ion-java-benchmark-cli run: cd ion-java-benchmark-cli && mvn clean install diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9d242e2d16..d5ed52e144 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,9 +22,10 @@ jobs: with: distribution: 'zulu' java-version: ${{ matrix.java }} - - run: mvn test - - run: mvn package -f ion-java-cli/pom.xml - - run: java -jar ion-java-cli/target/ion-java-cli-1.0.jar version + - uses: gradle/gradle-build-action@v2 + with: + arguments: build + - run: ./ion-test-driver-run version report-generation: runs-on: ubuntu-latest @@ -36,17 +37,12 @@ jobs: uses: actions/setup-java@v1 with: java-version: 11 - - run: mvn test site - - uses: codecov/codecov-action@v1 + - uses: gradle/gradle-build-action@v2 with: - file: target/site/jacoco/jacoco.xml - - name: deploy gh-pages - if: ${{ github.ref == 'refs/heads/master' && github.event_name == 'push' }} - uses: JamesIves/github-pages-deploy-action@3.7.1 + arguments: test + - uses: codecov/codecov-action@v1 with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BRANCH: gh-pages - FOLDER: "./target/site" + file: build/reports/jacoco/test/jacocoTestReport.xml diff --git a/.github/workflows/version-bump-and-fast-forward-check.yml b/.github/workflows/version-bump-and-fast-forward-check.yml index d355144565..1d27f5acd5 100644 --- a/.github/workflows/version-bump-and-fast-forward-check.yml +++ b/.github/workflows/version-bump-and-fast-forward-check.yml @@ -25,10 +25,10 @@ jobs: id: check-version-change run: | cd /home/runner/work/ion-java/ion-java/ion-java-new - versionNew=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + versionNew="$( com.amazon.ion ion-java @@ -64,6 +64,13 @@ dependency into your project's `pom.xml`: ``` +For Gradle (Kotlin DSL), use: +```kotlin +dependencies { + implementation("com.amazon.ion:ion-java:1.9.5") +} +``` + #### Legacy group id Originally ion-java was published using the group id `software.amazon.ion`. Since 1.4.0 the diff --git a/build.gradle.kts b/build.gradle.kts new file mode 100644 index 0000000000..d877473aed --- /dev/null +++ b/build.gradle.kts @@ -0,0 +1,160 @@ +import java.net.URI +import java.time.Instant +import java.util.Properties + +plugins { + java + `maven-publish` + jacoco + signing + // TODO: static analysis. E.g.: + // id("com.diffplug.spotless") version "6.11.0" + // id("com.github.spotbugs") version "4.8.0" +} + +repositories { + mavenCentral() +} + +dependencies { + testImplementation("junit:junit:4.13.1") + testImplementation("org.hamcrest:hamcrest:2.2") + testImplementation("pl.pragmatists:JUnitParams:1.1.1") +} + +group = "com.amazon.ion" +// The version is stored in a separate file so that we can easily create CI workflows that run when the version changes +// and so that any tool can access the version without having to do any special parsing. +version = File(project.rootDir.path + "/project.version").readLines().single() +description = "A Java implementation of the Amazon Ion data notation." +// Can't target 6 from 8 +// https://stackoverflow.com/questions/48146930/is-it-possible-to-compile-project-on-java-8-for-target-java-6 +java.sourceCompatibility = JavaVersion.VERSION_1_6 + +val isReleaseVersion: Boolean = !version.toString().endsWith("SNAPSHOT") +val generatedJarInfoDir = "${buildDir}/generated/jar-info" +lateinit var sourcesArtifact: PublishArtifact +lateinit var javadocArtifact: PublishArtifact + +sourceSets { + main { + java.srcDir("src") + resources.srcDir(generatedJarInfoDir) + } + test { + java.srcDir("test") + } +} + +tasks { + withType { options.encoding = "UTF-8" } + + javadoc { + // Suppressing Javadoc warnings is clunky, but there doesn't seem to be any nicer way to do it. + // https://stackoverflow.com/questions/52205209/configure-gradle-build-to-suppress-javadoc-console-warnings + options { + (this as CoreJavadocOptions).addStringOption("Xdoclint:none", "-quiet") + } + } + + create("sourcesJar") sourcesJar@ { + archiveClassifier.set("sources") + from(sourceSets["main"].java.srcDirs) + artifacts { sourcesArtifact = archives(this@sourcesJar) } + } + + create("javadocJar") javadocJar@ { + archiveClassifier.set("javadoc") + from(javadoc) + artifacts { javadocArtifact = archives(this@javadocJar) } + } + + /** + * This task creates a properties file that will be included in the compiled jar. It contains information about + * the build/version of the library used in [com.amazon.ion.util.JarInfo]. See https://github.com/amzn/ion-java/pull/433 + * for why this is done with a properties file rather than the Jar manifest. + */ + val generateJarInfo by creating { + doLast { + val propertiesFile = File("$generatedJarInfoDir/${project.name}.properties") + propertiesFile.parentFile.mkdirs() + val properties = Properties() + properties.setProperty("build.version", version.toString()) + properties.setProperty("build.time", Instant.now().toString()) + properties.store(propertiesFile.writer(), null) + } + outputs.dir(generatedJarInfoDir) + } + + processResources { dependsOn(generateJarInfo) } + + jacocoTestReport { + dependsOn(test) + reports { + xml.isEnabled = true + html.isEnabled = true + } + doLast { + logger.quiet("Coverage report written to file://${reports.html.destination}/index.html") + } + } + + test { + // report is always generated after tests run + finalizedBy(jacocoTestReport) + } + + withType { + setOnlyIf { isReleaseVersion && gradle.taskGraph.hasTask("publish") } + } +} + +publishing { + publications.create("IonJava") { + from(components["java"]) + artifact(sourcesArtifact) + artifact(javadocArtifact) + + pom { + name.set("Ion Java") + description.set(project.description) + url.set("https://github.com/amzn/ion-java/") + + licenses { + license { + name.set("The Apache License, Version 2.0") + url.set("http://www.apache.org/licenses/LICENSE-2.0.txt") + } + } + developers { + developer { + name.set("Amazon Ion Team") + email.set("ion-team@amazon.com") + organization.set("Amazon Ion") + organizationUrl.set("https://github.com/amazon-ion") + } + } + scm { + connection.set("scm:git:git@github.com:amzn/ion-java.git") + developerConnection.set("scm:git:git@github.com:amzn/ion-java.git") + url.set("git@github.com:amzn/ion-java.git") + } + } + } + repositories.mavenCentral { + credentials { + username = properties["ossrhUsername"].toString() + password = properties["ossrhPassword"].toString() + } + url = URI.create("https://aws.oss.sonatype.org/service/local/staging/deploy/maven2") + } +} + +signing { + // Allow publishing to maven local even if we don't have the signing keys + // This works because when not required, the signing task will be skipped + // if signing.keyId, signing.password, signing.secretKeyRingFile, etc are + // not present in gradle.properties. + isRequired = isReleaseVersion + sign(publishing.publications["IonJava"]) +} diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 0000000000..1808671847 --- /dev/null +++ b/gradle.properties @@ -0,0 +1,6 @@ +signing.keyId=EMPTY +signing.password=EMPTY +signing.secretKeyRingFile=EMPTY + +ossrhUsername=EMPTY +ossrhPassword=EMPTY diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..249e5832f090a2944b7473328c07c9755baa3196 GIT binary patch literal 60756 zcmb5WV{~QRw(p$^Dz@00IL3?^hro$gg*4VI_WAaTyVM5Foj~O|-84 z$;06hMwt*rV;^8iB z1~&0XWpYJmG?Ts^K9PC62H*`G}xom%S%yq|xvG~FIfP=9*f zZoDRJBm*Y0aId=qJ?7dyb)6)JGWGwe)MHeNSzhi)Ko6J<-m@v=a%NsP537lHe0R* z`If4$aaBA#S=w!2z&m>{lpTy^Lm^mg*3?M&7HFv}7K6x*cukLIGX;bQG|QWdn{%_6 zHnwBKr84#B7Z+AnBXa16a?or^R?+>$4`}{*a_>IhbjvyTtWkHw)|ay)ahWUd-qq$~ zMbh6roVsj;_qnC-R{G+Cy6bApVOinSU-;(DxUEl!i2)1EeQ9`hrfqj(nKI7?Z>Xur zoJz-a`PxkYit1HEbv|jy%~DO^13J-ut986EEG=66S}D3!L}Efp;Bez~7tNq{QsUMm zh9~(HYg1pA*=37C0}n4g&bFbQ+?-h-W}onYeE{q;cIy%eZK9wZjSwGvT+&Cgv z?~{9p(;bY_1+k|wkt_|N!@J~aoY@|U_RGoWX<;p{Nu*D*&_phw`8jYkMNpRTWx1H* z>J-Mi_!`M468#5Aix$$u1M@rJEIOc?k^QBc?T(#=n&*5eS#u*Y)?L8Ha$9wRWdH^3D4|Ps)Y?m0q~SiKiSfEkJ!=^`lJ(%W3o|CZ zSrZL-Xxc{OrmsQD&s~zPfNJOpSZUl%V8tdG%ei}lQkM+z@-4etFPR>GOH9+Y_F<3=~SXln9Kb-o~f>2a6Xz@AS3cn^;c_>lUwlK(n>z?A>NbC z`Ud8^aQy>wy=$)w;JZzA)_*Y$Z5hU=KAG&htLw1Uh00yE!|Nu{EZkch zY9O6x7Y??>!7pUNME*d!=R#s)ghr|R#41l!c?~=3CS8&zr6*aA7n9*)*PWBV2w+&I zpW1-9fr3j{VTcls1>ua}F*bbju_Xq%^v;-W~paSqlf zolj*dt`BBjHI)H9{zrkBo=B%>8}4jeBO~kWqO!~Thi!I1H(in=n^fS%nuL=X2+s!p}HfTU#NBGiwEBF^^tKU zbhhv+0dE-sbK$>J#t-J!B$TMgN@Wh5wTtK2BG}4BGfsZOoRUS#G8Cxv|6EI*n&Xxq zt{&OxCC+BNqz$9b0WM7_PyBJEVObHFh%%`~!@MNZlo*oXDCwDcFwT~Rls!aApL<)^ zbBftGKKBRhB!{?fX@l2_y~%ygNFfF(XJzHh#?`WlSL{1lKT*gJM zs>bd^H9NCxqxn(IOky5k-wALFowQr(gw%|`0991u#9jXQh?4l|l>pd6a&rx|v=fPJ z1mutj{YzpJ_gsClbWFk(G}bSlFi-6@mwoQh-XeD*j@~huW4(8ub%^I|azA)h2t#yG z7e_V_<4jlM3D(I+qX}yEtqj)cpzN*oCdYHa!nm%0t^wHm)EmFP*|FMw!tb@&`G-u~ zK)=Sf6z+BiTAI}}i{*_Ac$ffr*Wrv$F7_0gJkjx;@)XjYSh`RjAgrCck`x!zP>Ifu z&%he4P|S)H*(9oB4uvH67^0}I-_ye_!w)u3v2+EY>eD3#8QR24<;7?*hj8k~rS)~7 zSXs5ww)T(0eHSp$hEIBnW|Iun<_i`}VE0Nc$|-R}wlSIs5pV{g_Dar(Zz<4X3`W?K z6&CAIl4U(Qk-tTcK{|zYF6QG5ArrEB!;5s?tW7 zrE3hcFY&k)+)e{+YOJ0X2uDE_hd2{|m_dC}kgEKqiE9Q^A-+>2UonB+L@v3$9?AYw zVQv?X*pK;X4Ovc6Ev5Gbg{{Eu*7{N3#0@9oMI~}KnObQE#Y{&3mM4`w%wN+xrKYgD zB-ay0Q}m{QI;iY`s1Z^NqIkjrTlf`B)B#MajZ#9u41oRBC1oM1vq0i|F59> z#StM@bHt|#`2)cpl_rWB($DNJ3Lap}QM-+A$3pe}NyP(@+i1>o^fe-oxX#Bt`mcQc zb?pD4W%#ep|3%CHAYnr*^M6Czg>~L4?l16H1OozM{P*en298b+`i4$|w$|4AHbzqB zHpYUsHZET$Z0ztC;U+0*+amF!@PI%^oUIZy{`L{%O^i{Xk}X0&nl)n~tVEpcAJSJ} zverw15zP1P-O8h9nd!&hj$zuwjg?DoxYIw{jWM zW5_pj+wFy8Tsa9g<7Qa21WaV&;ejoYflRKcz?#fSH_)@*QVlN2l4(QNk| z4aPnv&mrS&0|6NHq05XQw$J^RR9T{3SOcMKCXIR1iSf+xJ0E_Wv?jEc*I#ZPzyJN2 zUG0UOXHl+PikM*&g$U@g+KbG-RY>uaIl&DEtw_Q=FYq?etc!;hEC_}UX{eyh%dw2V zTTSlap&5>PY{6I#(6`j-9`D&I#|YPP8a;(sOzgeKDWsLa!i-$frD>zr-oid!Hf&yS z!i^cr&7tN}OOGmX2)`8k?Tn!!4=tz~3hCTq_9CdiV!NIblUDxHh(FJ$zs)B2(t5@u z-`^RA1ShrLCkg0)OhfoM;4Z{&oZmAec$qV@ zGQ(7(!CBk<5;Ar%DLJ0p0!ResC#U<+3i<|vib1?{5gCebG7$F7URKZXuX-2WgF>YJ^i zMhHDBsh9PDU8dlZ$yJKtc6JA#y!y$57%sE>4Nt+wF1lfNIWyA`=hF=9Gj%sRwi@vd z%2eVV3y&dvAgyuJ=eNJR+*080dbO_t@BFJO<@&#yqTK&+xc|FRR;p;KVk@J3$S{p` zGaMj6isho#%m)?pOG^G0mzOAw0z?!AEMsv=0T>WWcE>??WS=fII$t$(^PDPMU(P>o z_*0s^W#|x)%tx8jIgZY~A2yG;US0m2ZOQt6yJqW@XNY_>_R7(Nxb8Ged6BdYW6{prd!|zuX$@Q2o6Ona8zzYC1u!+2!Y$Jc9a;wy+pXt}o6~Bu1oF1c zp7Y|SBTNi@=I(K%A60PMjM#sfH$y*c{xUgeSpi#HB`?|`!Tb&-qJ3;vxS!TIzuTZs-&%#bAkAyw9m4PJgvey zM5?up*b}eDEY+#@tKec)-c(#QF0P?MRlD1+7%Yk*jW;)`f;0a-ZJ6CQA?E%>i2Dt7T9?s|9ZF|KP4;CNWvaVKZ+Qeut;Jith_y{v*Ny6Co6!8MZx;Wgo z=qAi%&S;8J{iyD&>3CLCQdTX*$+Rx1AwA*D_J^0>suTgBMBb=*hefV+Ars#mmr+YsI3#!F@Xc1t4F-gB@6aoyT+5O(qMz*zG<9Qq*f0w^V!03rpr*-WLH}; zfM{xSPJeu6D(%8HU%0GEa%waFHE$G?FH^kMS-&I3)ycx|iv{T6Wx}9$$D&6{%1N_8 z_CLw)_9+O4&u94##vI9b-HHm_95m)fa??q07`DniVjAy`t7;)4NpeyAY(aAk(+T_O z1om+b5K2g_B&b2DCTK<>SE$Ode1DopAi)xaJjU>**AJK3hZrnhEQ9E`2=|HHe<^tv z63e(bn#fMWuz>4erc47}!J>U58%<&N<6AOAewyzNTqi7hJc|X{782&cM zHZYclNbBwU6673=!ClmxMfkC$(CykGR@10F!zN1Se83LR&a~$Ht&>~43OX22mt7tcZUpa;9@q}KDX3O&Ugp6< zLZLfIMO5;pTee1vNyVC$FGxzK2f>0Z-6hM82zKg44nWo|n}$Zk6&;5ry3`(JFEX$q zK&KivAe${e^5ZGc3a9hOt|!UOE&OocpVryE$Y4sPcs4rJ>>Kbi2_subQ9($2VN(3o zb~tEzMsHaBmBtaHAyES+d3A(qURgiskSSwUc9CfJ@99&MKp2sooSYZu+-0t0+L*!I zYagjOlPgx|lep9tiU%ts&McF6b0VE57%E0Ho%2oi?=Ks+5%aj#au^OBwNwhec zta6QAeQI^V!dF1C)>RHAmB`HnxyqWx?td@4sd15zPd*Fc9hpDXP23kbBenBxGeD$k z;%0VBQEJ-C)&dTAw_yW@k0u?IUk*NrkJ)(XEeI z9Y>6Vel>#s_v@=@0<{4A{pl=9cQ&Iah0iD0H`q)7NeCIRz8zx;! z^OO;1+IqoQNak&pV`qKW+K0^Hqp!~gSohcyS)?^P`JNZXw@gc6{A3OLZ?@1Uc^I2v z+X!^R*HCm3{7JPq{8*Tn>5;B|X7n4QQ0Bs79uTU%nbqOJh`nX(BVj!#f;#J+WZxx4 z_yM&1Y`2XzhfqkIMO7tB3raJKQS+H5F%o83bM+hxbQ zeeJm=Dvix$2j|b4?mDacb67v-1^lTp${z=jc1=j~QD>7c*@+1?py>%Kj%Ejp7Y-!? z8iYRUlGVrQPandAaxFfks53@2EC#0)%mrnmGRn&>=$H$S8q|kE_iWko4`^vCS2aWg z#!`RHUGyOt*k?bBYu3*j3u0gB#v(3tsije zgIuNNWNtrOkx@Pzs;A9un+2LX!zw+p3_NX^Sh09HZAf>m8l@O*rXy_82aWT$Q>iyy zqO7Of)D=wcSn!0+467&!Hl))eff=$aneB?R!YykdKW@k^_uR!+Q1tR)+IJb`-6=jj zymzA>Sv4>Z&g&WWu#|~GcP7qP&m*w-S$)7Xr;(duqCTe7p8H3k5>Y-n8438+%^9~K z3r^LIT_K{i7DgEJjIocw_6d0!<;wKT`X;&vv+&msmhAAnIe!OTdybPctzcEzBy88_ zWO{6i4YT%e4^WQZB)KHCvA(0tS zHu_Bg+6Ko%a9~$EjRB90`P(2~6uI@SFibxct{H#o&y40MdiXblu@VFXbhz>Nko;7R z70Ntmm-FePqhb%9gL+7U8@(ch|JfH5Fm)5${8|`Lef>LttM_iww6LW2X61ldBmG0z zax3y)njFe>j*T{i0s8D4=L>X^j0)({R5lMGVS#7(2C9@AxL&C-lZQx~czI7Iv+{%1 z2hEG>RzX4S8x3v#9sgGAnPzptM)g&LB}@%E>fy0vGSa(&q0ch|=ncKjNrK z`jA~jObJhrJ^ri|-)J^HUyeZXz~XkBp$VhcTEcTdc#a2EUOGVX?@mYx#Vy*!qO$Jv zQ4rgOJ~M*o-_Wptam=~krnmG*p^j!JAqoQ%+YsDFW7Cc9M%YPiBOrVcD^RY>m9Pd< zu}#9M?K{+;UIO!D9qOpq9yxUquQRmQNMo0pT`@$pVt=rMvyX)ph(-CCJLvUJy71DI zBk7oc7)-%ngdj~s@76Yse3L^gV0 z2==qfp&Q~L(+%RHP0n}+xH#k(hPRx(!AdBM$JCfJ5*C=K3ts>P?@@SZ_+{U2qFZb>4kZ{Go37{# zSQc+-dq*a-Vy4?taS&{Ht|MLRiS)Sn14JOONyXqPNnpq&2y~)6wEG0oNy>qvod$FF z`9o&?&6uZjhZ4_*5qWVrEfu(>_n2Xi2{@Gz9MZ8!YmjYvIMasE9yVQL10NBrTCczq zcTY1q^PF2l!Eraguf{+PtHV3=2A?Cu&NN&a8V(y;q(^_mFc6)%Yfn&X&~Pq zU1?qCj^LF(EQB1F`8NxNjyV%fde}dEa(Hx=r7$~ts2dzDwyi6ByBAIx$NllB4%K=O z$AHz1<2bTUb>(MCVPpK(E9wlLElo(aSd(Os)^Raum`d(g9Vd_+Bf&V;l=@mM=cC>) z)9b0enb)u_7V!!E_bl>u5nf&Rl|2r=2F3rHMdb7y9E}}F82^$Rf+P8%dKnOeKh1vs zhH^P*4Ydr^$)$h@4KVzxrHyy#cKmWEa9P5DJ|- zG;!Qi35Tp7XNj60=$!S6U#!(${6hyh7d4q=pF{`0t|N^|L^d8pD{O9@tF~W;#Je*P z&ah%W!KOIN;SyAEhAeTafJ4uEL`(RtnovM+cb(O#>xQnk?dzAjG^~4$dFn^<@-Na3 z395;wBnS{t*H;Jef2eE!2}u5Ns{AHj>WYZDgQJt8v%x?9{MXqJsGP|l%OiZqQ1aB! z%E=*Ig`(!tHh>}4_z5IMpg{49UvD*Pp9!pxt_gdAW%sIf3k6CTycOT1McPl=_#0?8 zVjz8Hj*Vy9c5-krd-{BQ{6Xy|P$6LJvMuX$* zA+@I_66_ET5l2&gk9n4$1M3LN8(yEViRx&mtd#LD}AqEs?RW=xKC(OCWH;~>(X6h!uDxXIPH06xh z*`F4cVlbDP`A)-fzf>MuScYsmq&1LUMGaQ3bRm6i7OsJ|%uhTDT zlvZA1M}nz*SalJWNT|`dBm1$xlaA>CCiQ zK`xD-RuEn>-`Z?M{1%@wewf#8?F|(@1e0+T4>nmlSRrNK5f)BJ2H*$q(H>zGD0>eL zQ!tl_Wk)k*e6v^m*{~A;@6+JGeWU-q9>?+L_#UNT%G?4&BnOgvm9@o7l?ov~XL+et zbGT)|G7)KAeqb=wHSPk+J1bdg7N3$vp(ekjI1D9V$G5Cj!=R2w=3*4!z*J-r-cyeb zd(i2KmX!|Lhey!snRw z?#$Gu%S^SQEKt&kep)up#j&9}e+3=JJBS(s>MH+|=R(`8xK{mmndWo_r`-w1#SeRD&YtAJ#GiVI*TkQZ}&aq<+bU2+coU3!jCI6E+Ad_xFW*ghnZ$q zAoF*i&3n1j#?B8x;kjSJD${1jdRB;)R*)Ao!9bd|C7{;iqDo|T&>KSh6*hCD!rwv= zyK#F@2+cv3=|S1Kef(E6Niv8kyLVLX&e=U;{0x{$tDfShqkjUME>f8d(5nzSkY6@! z^-0>DM)wa&%m#UF1F?zR`8Y3X#tA!*7Q$P3lZJ%*KNlrk_uaPkxw~ zxZ1qlE;Zo;nb@!SMazSjM>;34ROOoygo%SF);LL>rRonWwR>bmSd1XD^~sGSu$Gg# zFZ`|yKU0%!v07dz^v(tY%;So(e`o{ZYTX`hm;@b0%8|H>VW`*cr8R%3n|ehw2`(9B+V72`>SY}9^8oh$En80mZK9T4abVG*to;E z1_S6bgDOW?!Oy1LwYy=w3q~KKdbNtyH#d24PFjX)KYMY93{3-mPP-H>@M-_>N~DDu zENh~reh?JBAK=TFN-SfDfT^=+{w4ea2KNWXq2Y<;?(gf(FgVp8Zp-oEjKzB%2Iqj;48GmY3h=bcdYJ}~&4tS`Q1sb=^emaW$IC$|R+r-8V- zf0$gGE(CS_n4s>oicVk)MfvVg#I>iDvf~Ov8bk}sSxluG!6#^Z_zhB&U^`eIi1@j( z^CK$z^stBHtaDDHxn+R;3u+>Lil^}fj?7eaGB z&5nl^STqcaBxI@v>%zG|j))G(rVa4aY=B@^2{TFkW~YP!8!9TG#(-nOf^^X-%m9{Z zCC?iC`G-^RcBSCuk=Z`(FaUUe?hf3{0C>>$?Vs z`2Uud9M+T&KB6o4o9kvdi^Q=Bw!asPdxbe#W-Oaa#_NP(qpyF@bVxv5D5))srkU#m zj_KA+#7sqDn*Ipf!F5Byco4HOSd!Ui$l94|IbW%Ny(s1>f4|Mv^#NfB31N~kya9!k zWCGL-$0ZQztBate^fd>R!hXY_N9ZjYp3V~4_V z#eB)Kjr8yW=+oG)BuNdZG?jaZlw+l_ma8aET(s+-x+=F-t#Qoiuu1i`^x8Sj>b^U} zs^z<()YMFP7CmjUC@M=&lA5W7t&cxTlzJAts*%PBDAPuqcV5o7HEnqjif_7xGt)F% zGx2b4w{@!tE)$p=l3&?Bf#`+!-RLOleeRk3 z7#pF|w@6_sBmn1nECqdunmG^}pr5(ZJQVvAt$6p3H(16~;vO>?sTE`Y+mq5YP&PBo zvq!7#W$Gewy`;%6o^!Dtjz~x)T}Bdk*BS#=EY=ODD&B=V6TD2z^hj1m5^d6s)D*wk zu$z~D7QuZ2b?5`p)E8e2_L38v3WE{V`bVk;6fl#o2`) z99JsWhh?$oVRn@$S#)uK&8DL8>An0&S<%V8hnGD7Z^;Y(%6;^9!7kDQ5bjR_V+~wp zfx4m3z6CWmmZ<8gDGUyg3>t8wgJ5NkkiEm^(sedCicP^&3D%}6LtIUq>mXCAt{9eF zNXL$kGcoUTf_Lhm`t;hD-SE)m=iBnxRU(NyL}f6~1uH)`K!hmYZjLI%H}AmEF5RZt z06$wn63GHnApHXZZJ}s^s)j9(BM6e*7IBK6Bq(!)d~zR#rbxK9NVIlgquoMq z=eGZ9NR!SEqP6=9UQg#@!rtbbSBUM#ynF);zKX+|!Zm}*{H z+j=d?aZ2!?@EL7C~%B?6ouCKLnO$uWn;Y6Xz zX8dSwj732u(o*U3F$F=7xwxm>E-B+SVZH;O-4XPuPkLSt_?S0)lb7EEg)Mglk0#eS z9@jl(OnH4juMxY+*r03VDfPx_IM!Lmc(5hOI;`?d37f>jPP$?9jQQIQU@i4vuG6MagEoJrQ=RD7xt@8E;c zeGV*+Pt+t$@pt!|McETOE$9k=_C!70uhwRS9X#b%ZK z%q(TIUXSS^F0`4Cx?Rk07C6wI4!UVPeI~-fxY6`YH$kABdOuiRtl73MqG|~AzZ@iL&^s?24iS;RK_pdlWkhcF z@Wv-Om(Aealfg)D^adlXh9Nvf~Uf@y;g3Y)i(YP zEXDnb1V}1pJT5ZWyw=1i+0fni9yINurD=EqH^ciOwLUGi)C%Da)tyt=zq2P7pV5-G zR7!oq28-Fgn5pW|nlu^b!S1Z#r7!Wtr{5J5PQ>pd+2P7RSD?>(U7-|Y z7ZQ5lhYIl_IF<9?T9^IPK<(Hp;l5bl5tF9>X-zG14_7PfsA>6<$~A338iYRT{a@r_ zuXBaT=`T5x3=s&3=RYx6NgG>No4?5KFBVjE(swfcivcIpPQFx5l+O;fiGsOrl5teR z_Cm+;PW}O0Dwe_(4Z@XZ)O0W-v2X><&L*<~*q3dg;bQW3g7)a#3KiQP>+qj|qo*Hk z?57>f2?f@`=Fj^nkDKeRkN2d$Z@2eNKpHo}ksj-$`QKb6n?*$^*%Fb3_Kbf1(*W9K>{L$mud2WHJ=j0^=g30Xhg8$#g^?36`p1fm;;1@0Lrx+8t`?vN0ZorM zSW?rhjCE8$C|@p^sXdx z|NOHHg+fL;HIlqyLp~SSdIF`TnSHehNCU9t89yr@)FY<~hu+X`tjg(aSVae$wDG*C zq$nY(Y494R)hD!i1|IIyP*&PD_c2FPgeY)&mX1qujB1VHPG9`yFQpLFVQ0>EKS@Bp zAfP5`C(sWGLI?AC{XEjLKR4FVNw(4+9b?kba95ukgR1H?w<8F7)G+6&(zUhIE5Ef% z=fFkL3QKA~M@h{nzjRq!Y_t!%U66#L8!(2-GgFxkD1=JRRqk=n%G(yHKn%^&$dW>; zSjAcjETMz1%205se$iH_)ZCpfg_LwvnsZQAUCS#^FExp8O4CrJb6>JquNV@qPq~3A zZ<6dOU#6|8+fcgiA#~MDmcpIEaUO02L5#T$HV0$EMD94HT_eXLZ2Zi&(! z&5E>%&|FZ`)CN10tM%tLSPD*~r#--K(H-CZqIOb99_;m|D5wdgJ<1iOJz@h2Zkq?} z%8_KXb&hf=2Wza(Wgc;3v3TN*;HTU*q2?#z&tLn_U0Nt!y>Oo>+2T)He6%XuP;fgn z-G!#h$Y2`9>Jtf}hbVrm6D70|ERzLAU>3zoWhJmjWfgM^))T+2u$~5>HF9jQDkrXR z=IzX36)V75PrFjkQ%TO+iqKGCQ-DDXbaE;C#}!-CoWQx&v*vHfyI>$HNRbpvm<`O( zlx9NBWD6_e&J%Ous4yp~s6)Ghni!I6)0W;9(9$y1wWu`$gs<$9Mcf$L*piP zPR0Av*2%ul`W;?-1_-5Zy0~}?`e@Y5A&0H!^ApyVTT}BiOm4GeFo$_oPlDEyeGBbh z1h3q&Dx~GmUS|3@4V36&$2uO8!Yp&^pD7J5&TN{?xphf*-js1fP?B|`>p_K>lh{ij zP(?H%e}AIP?_i^f&Li=FDSQ`2_NWxL+BB=nQr=$ zHojMlXNGauvvwPU>ZLq!`bX-5F4jBJ&So{kE5+ms9UEYD{66!|k~3vsP+mE}x!>%P za98bAU0!h0&ka4EoiDvBM#CP#dRNdXJcb*(%=<(g+M@<)DZ!@v1V>;54En?igcHR2 zhubQMq}VSOK)onqHfczM7YA@s=9*ow;k;8)&?J3@0JiGcP! zP#00KZ1t)GyZeRJ=f0^gc+58lc4Qh*S7RqPIC6GugG1gXe$LIQMRCo8cHf^qXgAa2 z`}t>u2Cq1CbSEpLr~E=c7~=Qkc9-vLE%(v9N*&HF`(d~(0`iukl5aQ9u4rUvc8%m) zr2GwZN4!s;{SB87lJB;veebPmqE}tSpT>+`t?<457Q9iV$th%i__Z1kOMAswFldD6 ztbOvO337S5o#ZZgN2G99_AVqPv!?Gmt3pzgD+Hp3QPQ`9qJ(g=kjvD+fUSS3upJn! zqoG7acIKEFRX~S}3|{EWT$kdz#zrDlJU(rPkxjws_iyLKU8+v|*oS_W*-guAb&Pj1 z35Z`3z<&Jb@2Mwz=KXucNYdY#SNO$tcVFr9KdKm|%^e-TXzs6M`PBper%ajkrIyUe zp$vVxVs9*>Vp4_1NC~Zg)WOCPmOxI1V34QlG4!aSFOH{QqSVq1^1)- z0P!Z?tT&E-ll(pwf0?=F=yOzik=@nh1Clxr9}Vij89z)ePDSCYAqw?lVI?v?+&*zH z)p$CScFI8rrwId~`}9YWPFu0cW1Sf@vRELs&cbntRU6QfPK-SO*mqu|u~}8AJ!Q$z znzu}50O=YbjwKCuSVBs6&CZR#0FTu)3{}qJJYX(>QPr4$RqWiwX3NT~;>cLn*_&1H zaKpIW)JVJ>b{uo2oq>oQt3y=zJjb%fU@wLqM{SyaC6x2snMx-}ivfU<1- znu1Lh;i$3Tf$Kh5Uk))G!D1UhE8pvx&nO~w^fG)BC&L!_hQk%^p`Kp@F{cz>80W&T ziOK=Sq3fdRu*V0=S53rcIfWFazI}Twj63CG(jOB;$*b`*#B9uEnBM`hDk*EwSRdwP8?5T?xGUKs=5N83XsR*)a4|ijz|c{4tIU+4j^A5C<#5 z*$c_d=5ml~%pGxw#?*q9N7aRwPux5EyqHVkdJO=5J>84!X6P>DS8PTTz>7C#FO?k#edkntG+fJk8ZMn?pmJSO@`x-QHq;7^h6GEXLXo1TCNhH z8ZDH{*NLAjo3WM`xeb=X{((uv3H(8&r8fJJg_uSs_%hOH%JDD?hu*2NvWGYD+j)&` zz#_1%O1wF^o5ryt?O0n;`lHbzp0wQ?rcbW(F1+h7_EZZ9{>rePvLAPVZ_R|n@;b$;UchU=0j<6k8G9QuQf@76oiE*4 zXOLQ&n3$NR#p4<5NJMVC*S);5x2)eRbaAM%VxWu9ohlT;pGEk7;002enCbQ>2r-us z3#bpXP9g|mE`65VrN`+3mC)M(eMj~~eOf)do<@l+fMiTR)XO}422*1SL{wyY(%oMpBgJagtiDf zz>O6(m;};>Hi=t8o{DVC@YigqS(Qh+ix3Rwa9aliH}a}IlOCW1@?%h_bRbq-W{KHF z%Vo?-j@{Xi@=~Lz5uZP27==UGE15|g^0gzD|3x)SCEXrx`*MP^FDLl%pOi~~Il;dc z^hrwp9sYeT7iZ)-ajKy@{a`kr0-5*_!XfBpXwEcFGJ;%kV$0Nx;apKrur zJN2J~CAv{Zjj%FolyurtW8RaFmpn&zKJWL>(0;;+q(%(Hx!GMW4AcfP0YJ*Vz!F4g z!ZhMyj$BdXL@MlF%KeInmPCt~9&A!;cRw)W!Hi@0DY(GD_f?jeV{=s=cJ6e}JktJw zQORnxxj3mBxfrH=x{`_^Z1ddDh}L#V7i}$njUFRVwOX?qOTKjfPMBO4y(WiU<)epb zvB9L=%jW#*SL|Nd_G?E*_h1^M-$PG6Pc_&QqF0O-FIOpa4)PAEPsyvB)GKasmBoEt z?_Q2~QCYGH+hW31x-B=@5_AN870vY#KB~3a*&{I=f);3Kv7q4Q7s)0)gVYx2#Iz9g(F2;=+Iy4 z6KI^8GJ6D@%tpS^8boU}zpi=+(5GfIR)35PzrbuXeL1Y1N%JK7PG|^2k3qIqHfX;G zQ}~JZ-UWx|60P5?d1e;AHx!_;#PG%d=^X(AR%i`l0jSpYOpXoKFW~7ip7|xvN;2^? zsYC9fanpO7rO=V7+KXqVc;Q5z%Bj})xHVrgoR04sA2 zl~DAwv=!(()DvH*=lyhIlU^hBkA0$e*7&fJpB0|oB7)rqGK#5##2T`@_I^|O2x4GO z;xh6ROcV<9>?e0)MI(y++$-ksV;G;Xe`lh76T#Htuia+(UrIXrf9?

    L(tZ$0BqX1>24?V$S+&kLZ`AodQ4_)P#Q3*4xg8}lMV-FLwC*cN$< zt65Rf%7z41u^i=P*qO8>JqXPrinQFapR7qHAtp~&RZ85$>ob|Js;GS^y;S{XnGiBc zGa4IGvDl?x%gY`vNhv8wgZnP#UYI-w*^4YCZnxkF85@ldepk$&$#3EAhrJY0U)lR{F6sM3SONV^+$;Zx8BD&Eku3K zKNLZyBni3)pGzU0;n(X@1fX8wYGKYMpLmCu{N5-}epPDxClPFK#A@02WM3!myN%bkF z|GJ4GZ}3sL{3{qXemy+#Uk{4>Kf8v11;f8I&c76+B&AQ8udd<8gU7+BeWC`akUU~U zgXoxie>MS@rBoyY8O8Tc&8id!w+_ooxcr!1?#rc$-|SBBtH6S?)1e#P#S?jFZ8u-Bs&k`yLqW|{j+%c#A4AQ>+tj$Y z^CZajspu$F%73E68Lw5q7IVREED9r1Ijsg#@DzH>wKseye>hjsk^{n0g?3+gs@7`i zHx+-!sjLx^fS;fY!ERBU+Q zVJ!e0hJH%P)z!y%1^ZyG0>PN@5W~SV%f>}c?$H8r;Sy-ui>aruVTY=bHe}$e zi&Q4&XK!qT7-XjCrDaufT@>ieQ&4G(SShUob0Q>Gznep9fR783jGuUynAqc6$pYX; z7*O@@JW>O6lKIk0G00xsm|=*UVTQBB`u1f=6wGAj%nHK_;Aqmfa!eAykDmi-@u%6~ z;*c!pS1@V8r@IX9j&rW&d*}wpNs96O2Ute>%yt{yv>k!6zfT6pru{F1M3P z2WN1JDYqoTB#(`kE{H676QOoX`cnqHl1Yaru)>8Ky~VU{)r#{&s86Vz5X)v15ULHA zAZDb{99+s~qI6;-dQ5DBjHJP@GYTwn;Dv&9kE<0R!d z8tf1oq$kO`_sV(NHOSbMwr=To4r^X$`sBW4$gWUov|WY?xccQJN}1DOL|GEaD_!@& z15p?Pj+>7d`@LvNIu9*^hPN)pwcv|akvYYq)ks%`G>!+!pW{-iXPZsRp8 z35LR;DhseQKWYSD`%gO&k$Dj6_6q#vjWA}rZcWtQr=Xn*)kJ9kacA=esi*I<)1>w^ zO_+E>QvjP)qiSZg9M|GNeLtO2D7xT6vsj`88sd!94j^AqxFLi}@w9!Y*?nwWARE0P znuI_7A-saQ+%?MFA$gttMV-NAR^#tjl_e{R$N8t2NbOlX373>e7Ox=l=;y#;M7asp zRCz*CLnrm$esvSb5{T<$6CjY zmZ(i{Rs_<#pWW>(HPaaYj`%YqBra=Ey3R21O7vUbzOkJJO?V`4-D*u4$Me0Bx$K(lYo`JO}gnC zx`V}a7m-hLU9Xvb@K2ymioF)vj12<*^oAqRuG_4u%(ah?+go%$kOpfb`T96P+L$4> zQ#S+sA%VbH&mD1k5Ak7^^dZoC>`1L%i>ZXmooA!%GI)b+$D&ziKrb)a=-ds9xk#~& z7)3iem6I|r5+ZrTRe_W861x8JpD`DDIYZNm{$baw+$)X^Jtjnl0xlBgdnNY}x%5za zkQ8E6T<^$sKBPtL4(1zi_Rd(tVth*3Xs!ulflX+70?gb&jRTnI8l+*Aj9{|d%qLZ+ z>~V9Z;)`8-lds*Zgs~z1?Fg?Po7|FDl(Ce<*c^2=lFQ~ahwh6rqSjtM5+$GT>3WZW zj;u~w9xwAhOc<kF}~`CJ68 z?(S5vNJa;kriPlim33{N5`C{9?NWhzsna_~^|K2k4xz1`xcui*LXL-1#Y}Hi9`Oo!zQ>x-kgAX4LrPz63uZ+?uG*84@PKq-KgQlMNRwz=6Yes) zY}>YN+qP}nwr$(CZQFjUOI=-6J$2^XGvC~EZ+vrqWaOXB$k?%Suf5k=4>AveC1aJ! ziaW4IS%F$_Babi)kA8Y&u4F7E%99OPtm=vzw$$ zEz#9rvn`Iot_z-r3MtV>k)YvErZ<^Oa${`2>MYYODSr6?QZu+be-~MBjwPGdMvGd!b!elsdi4% z`37W*8+OGulab8YM?`KjJ8e+jM(tqLKSS@=jimq3)Ea2EB%88L8CaM+aG7;27b?5` z4zuUWBr)f)k2o&xg{iZ$IQkJ+SK>lpq4GEacu~eOW4yNFLU!Kgc{w4&D$4ecm0f}~ zTTzquRW@`f0}|IILl`!1P+;69g^upiPA6F{)U8)muWHzexRenBU$E^9X-uIY2%&1w z_=#5*(nmxJ9zF%styBwivi)?#KMG96-H@hD-H_&EZiRNsfk7mjBq{L%!E;Sqn!mVX*}kXhwH6eh;b42eD!*~upVG@ z#smUqz$ICm!Y8wY53gJeS|Iuard0=;k5i5Z_hSIs6tr)R4n*r*rE`>38Pw&lkv{_r!jNN=;#?WbMj|l>cU(9trCq; z%nN~r^y7!kH^GPOf3R}?dDhO=v^3BeP5hF|%4GNQYBSwz;x({21i4OQY->1G=KFyu z&6d`f2tT9Yl_Z8YACZaJ#v#-(gcyeqXMhYGXb=t>)M@fFa8tHp2x;ODX=Ap@a5I=U z0G80^$N0G4=U(>W%mrrThl0DjyQ-_I>+1Tdd_AuB3qpYAqY54upwa3}owa|x5iQ^1 zEf|iTZxKNGRpI>34EwkIQ2zHDEZ=(J@lRaOH>F|2Z%V_t56Km$PUYu^xA5#5Uj4I4RGqHD56xT%H{+P8Ag>e_3pN$4m8n>i%OyJFPNWaEnJ4McUZPa1QmOh?t8~n& z&RulPCors8wUaqMHECG=IhB(-tU2XvHP6#NrLVyKG%Ee*mQ5Ps%wW?mcnriTVRc4J`2YVM>$ixSF2Xi+Wn(RUZnV?mJ?GRdw%lhZ+t&3s7g!~g{%m&i<6 z5{ib-<==DYG93I(yhyv4jp*y3#*WNuDUf6`vTM%c&hiayf(%=x@4$kJ!W4MtYcE#1 zHM?3xw63;L%x3drtd?jot!8u3qeqctceX3m;tWetK+>~q7Be$h>n6riK(5@ujLgRS zvOym)k+VAtyV^mF)$29Y`nw&ijdg~jYpkx%*^ z8dz`C*g=I?;clyi5|!27e2AuSa$&%UyR(J3W!A=ZgHF9OuKA34I-1U~pyD!KuRkjA zbkN!?MfQOeN>DUPBxoy5IX}@vw`EEB->q!)8fRl_mqUVuRu|C@KD-;yl=yKc=ZT0% zB$fMwcC|HE*0f8+PVlWHi>M`zfsA(NQFET?LrM^pPcw`cK+Mo0%8*x8@65=CS_^$cG{GZQ#xv($7J z??R$P)nPLodI;P!IC3eEYEHh7TV@opr#*)6A-;EU2XuogHvC;;k1aI8asq7ovoP!* z?x%UoPrZjj<&&aWpsbr>J$Er-7!E(BmOyEv!-mbGQGeJm-U2J>74>o5x`1l;)+P&~ z>}f^=Rx(ZQ2bm+YE0u=ZYrAV@apyt=v1wb?R@`i_g64YyAwcOUl=C!i>=Lzb$`tjv zOO-P#A+)t-JbbotGMT}arNhJmmGl-lyUpMn=2UacVZxmiG!s!6H39@~&uVokS zG=5qWhfW-WOI9g4!R$n7!|ViL!|v3G?GN6HR0Pt_L5*>D#FEj5wM1DScz4Jv@Sxnl zB@MPPmdI{(2D?;*wd>3#tjAirmUnQoZrVv`xM3hARuJksF(Q)wd4P$88fGYOT1p6U z`AHSN!`St}}UMBT9o7i|G`r$ zrB=s$qV3d6$W9@?L!pl0lf%)xs%1ko^=QY$ty-57=55PvP(^6E7cc zGJ*>m2=;fOj?F~yBf@K@9qwX0hA803Xw+b0m}+#a(>RyR8}*Y<4b+kpp|OS+!whP( zH`v{%s>jsQI9rd$*vm)EkwOm#W_-rLTHcZRek)>AtF+~<(did)*oR1|&~1|e36d-d zgtm5cv1O0oqgWC%Et@P4Vhm}Ndl(Y#C^MD03g#PH-TFy+7!Osv1z^UWS9@%JhswEq~6kSr2DITo59+; ze=ZC}i2Q?CJ~Iyu?vn|=9iKV>4j8KbxhE4&!@SQ^dVa-gK@YfS9xT(0kpW*EDjYUkoj! zE49{7H&E}k%5(>sM4uGY)Q*&3>{aitqdNnRJkbOmD5Mp5rv-hxzOn80QsG=HJ_atI-EaP69cacR)Uvh{G5dTpYG7d zbtmRMq@Sexey)||UpnZ?;g_KMZq4IDCy5}@u!5&B^-=6yyY{}e4Hh3ee!ZWtL*s?G zxG(A!<9o!CL+q?u_utltPMk+hn?N2@?}xU0KlYg?Jco{Yf@|mSGC<(Zj^yHCvhmyx z?OxOYoxbptDK()tsJ42VzXdINAMWL$0Gcw?G(g8TMB)Khw_|v9`_ql#pRd2i*?CZl z7k1b!jQB=9-V@h%;Cnl7EKi;Y^&NhU0mWEcj8B|3L30Ku#-9389Q+(Yet0r$F=+3p z6AKOMAIi|OHyzlHZtOm73}|ntKtFaXF2Fy|M!gOh^L4^62kGUoWS1i{9gsds_GWBc zLw|TaLP64z3z9?=R2|T6Xh2W4_F*$cq>MtXMOy&=IPIJ`;!Tw?PqvI2b*U1)25^<2 zU_ZPoxg_V0tngA0J+mm?3;OYw{i2Zb4x}NedZug!>EoN3DC{1i)Z{Z4m*(y{ov2%- zk(w>+scOO}MN!exSc`TN)!B=NUX`zThWO~M*ohqq;J2hx9h9}|s#?@eR!=F{QTrq~ zTcY|>azkCe$|Q0XFUdpFT=lTcyW##i;-e{}ORB4D?t@SfqGo_cS z->?^rh$<&n9DL!CF+h?LMZRi)qju!meugvxX*&jfD!^1XB3?E?HnwHP8$;uX{Rvp# zh|)hM>XDv$ZGg=$1{+_bA~u-vXqlw6NH=nkpyWE0u}LQjF-3NhATL@9rRxMnpO%f7 z)EhZf{PF|mKIMFxnC?*78(}{Y)}iztV12}_OXffJ;ta!fcFIVjdchyHxH=t%ci`Xd zX2AUB?%?poD6Zv*&BA!6c5S#|xn~DK01#XvjT!w!;&`lDXSJT4_j$}!qSPrb37vc{ z9^NfC%QvPu@vlxaZ;mIbn-VHA6miwi8qJ~V;pTZkKqqOii<1Cs}0i?uUIss;hM4dKq^1O35y?Yp=l4i zf{M!@QHH~rJ&X~8uATV><23zZUbs-J^3}$IvV_ANLS08>k`Td7aU_S1sLsfi*C-m1 z-e#S%UGs4E!;CeBT@9}aaI)qR-6NU@kvS#0r`g&UWg?fC7|b^_HyCE!8}nyh^~o@< zpm7PDFs9yxp+byMS(JWm$NeL?DNrMCNE!I^ko-*csB+dsf4GAq{=6sfyf4wb>?v1v zmb`F*bN1KUx-`ra1+TJ37bXNP%`-Fd`vVQFTwWpX@;s(%nDQa#oWhgk#mYlY*!d>( zE&!|ySF!mIyfING+#%RDY3IBH_fW$}6~1%!G`suHub1kP@&DoAd5~7J55;5_noPI6eLf{t;@9Kf<{aO0`1WNKd?<)C-|?C?)3s z>wEq@8=I$Wc~Mt$o;g++5qR+(6wt9GI~pyrDJ%c?gPZe)owvy^J2S=+M^ z&WhIE`g;;J^xQLVeCtf7b%Dg#Z2gq9hp_%g)-%_`y*zb; zn9`f`mUPN-Ts&fFo(aNTsXPA|J!TJ{0hZp0^;MYHLOcD=r_~~^ymS8KLCSeU3;^QzJNqS z5{5rEAv#l(X?bvwxpU;2%pQftF`YFgrD1jt2^~Mt^~G>T*}A$yZc@(k9orlCGv&|1 zWWvVgiJsCAtamuAYT~nzs?TQFt<1LSEx!@e0~@yd6$b5!Zm(FpBl;(Cn>2vF?k zOm#TTjFwd2D-CyA!mqR^?#Uwm{NBemP>(pHmM}9;;8`c&+_o3#E5m)JzfwN?(f-a4 zyd%xZc^oQx3XT?vcCqCX&Qrk~nu;fxs@JUoyVoi5fqpi&bUhQ2y!Ok2pzsFR(M(|U zw3E+kH_zmTRQ9dUMZWRE%Zakiwc+lgv7Z%|YO9YxAy`y28`Aw;WU6HXBgU7fl@dnt z-fFBV)}H-gqP!1;V@Je$WcbYre|dRdp{xt!7sL3Eoa%IA`5CAA%;Wq8PktwPdULo! z8!sB}Qt8#jH9Sh}QiUtEPZ6H0b*7qEKGJ%ITZ|vH)5Q^2m<7o3#Z>AKc%z7_u`rXA zqrCy{-{8;9>dfllLu$^M5L z-hXs))h*qz%~ActwkIA(qOVBZl2v4lwbM>9l70Y`+T*elINFqt#>OaVWoja8RMsep z6Or3f=oBnA3vDbn*+HNZP?8LsH2MY)x%c13@(XfuGR}R?Nu<|07{$+Lc3$Uv^I!MQ z>6qWgd-=aG2Y^24g4{Bw9ueOR)(9h`scImD=86dD+MnSN4$6 z^U*o_mE-6Rk~Dp!ANp#5RE9n*LG(Vg`1)g6!(XtDzsov$Dvz|Gv1WU68J$CkshQhS zCrc|cdkW~UK}5NeaWj^F4MSgFM+@fJd{|LLM)}_O<{rj z+?*Lm?owq?IzC%U%9EBga~h-cJbIu=#C}XuWN>OLrc%M@Gu~kFEYUi4EC6l#PR2JS zQUkGKrrS#6H7}2l0F@S11DP`@pih0WRkRJl#F;u{c&ZC{^$Z+_*lB)r)-bPgRFE;* zl)@hK4`tEP=P=il02x7-C7p%l=B`vkYjw?YhdJU9!P!jcmY$OtC^12w?vy3<<=tlY zUwHJ_0lgWN9vf>1%WACBD{UT)1qHQSE2%z|JHvP{#INr13jM}oYv_5#xsnv9`)UAO zuwgyV4YZ;O)eSc3(mka6=aRohi!HH@I#xq7kng?Acdg7S4vDJb6cI5fw?2z%3yR+| zU5v@Hm}vy;${cBp&@D=HQ9j7NcFaOYL zj-wV=eYF{|XTkFNM2uz&T8uH~;)^Zo!=KP)EVyH6s9l1~4m}N%XzPpduPg|h-&lL` zAXspR0YMOKd2yO)eMFFJ4?sQ&!`dF&!|niH*!^*Ml##o0M(0*uK9&yzekFi$+mP9s z>W9d%Jb)PtVi&-Ha!o~Iyh@KRuKpQ@)I~L*d`{O8!kRObjO7=n+Gp36fe!66neh+7 zW*l^0tTKjLLzr`x4`_8&on?mjW-PzheTNox8Hg7Nt@*SbE-%kP2hWYmHu#Fn@Q^J(SsPUz*|EgOoZ6byg3ew88UGdZ>9B2Tq=jF72ZaR=4u%1A6Vm{O#?@dD!(#tmR;eP(Fu z{$0O%=Vmua7=Gjr8nY%>ul?w=FJ76O2js&17W_iq2*tb!i{pt#`qZB#im9Rl>?t?0c zicIC}et_4d+CpVPx)i4~$u6N-QX3H77ez z?ZdvXifFk|*F8~L(W$OWM~r`pSk5}#F?j_5u$Obu9lDWIknO^AGu+Blk7!9Sb;NjS zncZA?qtASdNtzQ>z7N871IsPAk^CC?iIL}+{K|F@BuG2>qQ;_RUYV#>hHO(HUPpk@ z(bn~4|F_jiZi}Sad;_7`#4}EmD<1EiIxa48QjUuR?rC}^HRocq`OQPM@aHVKP9E#q zy%6bmHygCpIddPjE}q_DPC`VH_2m;Eey&ZH)E6xGeStOK7H)#+9y!%-Hm|QF6w#A( zIC0Yw%9j$s-#odxG~C*^MZ?M<+&WJ+@?B_QPUyTg9DJGtQN#NIC&-XddRsf3n^AL6 zT@P|H;PvN;ZpL0iv$bRb7|J{0o!Hq+S>_NrH4@coZtBJu#g8#CbR7|#?6uxi8d+$g z87apN>EciJZ`%Zv2**_uiET9Vk{pny&My;+WfGDw4EVL#B!Wiw&M|A8f1A@ z(yFQS6jfbH{b8Z-S7D2?Ixl`j0{+ZnpT=;KzVMLW{B$`N?Gw^Fl0H6lT61%T2AU**!sX0u?|I(yoy&Xveg7XBL&+>n6jd1##6d>TxE*Vj=8lWiG$4=u{1UbAa5QD>5_ z;Te^42v7K6Mmu4IWT6Rnm>oxrl~b<~^e3vbj-GCdHLIB_>59}Ya+~OF68NiH=?}2o zP(X7EN=quQn&)fK>M&kqF|<_*H`}c zk=+x)GU>{Af#vx&s?`UKUsz})g^Pc&?Ka@t5$n$bqf6{r1>#mWx6Ep>9|A}VmWRnowVo`OyCr^fHsf# zQjQ3Ttp7y#iQY8l`zEUW)(@gGQdt(~rkxlkefskT(t%@i8=|p1Y9Dc5bc+z#n$s13 zGJk|V0+&Ekh(F};PJzQKKo+FG@KV8a<$gmNSD;7rd_nRdc%?9)p!|B-@P~kxQG}~B zi|{0}@}zKC(rlFUYp*dO1RuvPC^DQOkX4<+EwvBAC{IZQdYxoq1Za!MW7%p7gGr=j zzWnAq%)^O2$eItftC#TTSArUyL$U54-O7e|)4_7%Q^2tZ^0-d&3J1}qCzR4dWX!)4 zzIEKjgnYgMus^>6uw4Jm8ga6>GBtMjpNRJ6CP~W=37~||gMo_p@GA@#-3)+cVYnU> zE5=Y4kzl+EbEh%dhQokB{gqNDqx%5*qBusWV%!iprn$S!;oN_6E3?0+umADVs4ako z?P+t?m?};gev9JXQ#Q&KBpzkHPde_CGu-y z<{}RRAx=xlv#mVi+Ibrgx~ujW$h{?zPfhz)Kp7kmYS&_|97b&H&1;J-mzrBWAvY} zh8-I8hl_RK2+nnf&}!W0P+>5?#?7>npshe<1~&l_xqKd0_>dl_^RMRq@-Myz&|TKZBj1=Q()) zF{dBjv5)h=&Z)Aevx}+i|7=R9rG^Di!sa)sZCl&ctX4&LScQ-kMncgO(9o6W6)yd< z@Rk!vkja*X_N3H=BavGoR0@u0<}m-7|2v!0+2h~S2Q&a=lTH91OJsvms2MT~ zY=c@LO5i`mLpBd(vh|)I&^A3TQLtr>w=zoyzTd=^f@TPu&+*2MtqE$Avf>l>}V|3-8Fp2hzo3y<)hr_|NO(&oSD z!vEjTWBxbKTiShVl-U{n*B3#)3a8$`{~Pk}J@elZ=>Pqp|MQ}jrGv7KrNcjW%TN_< zZz8kG{#}XoeWf7qY?D)L)8?Q-b@Na&>i=)(@uNo zr;cH98T3$Iau8Hn*@vXi{A@YehxDE2zX~o+RY`)6-X{8~hMpc#C`|8y> zU8Mnv5A0dNCf{Ims*|l-^ z(MRp{qoGohB34|ggDI*p!Aw|MFyJ|v+<+E3brfrI)|+l3W~CQLPbnF@G0)P~Ly!1TJLp}xh8uW`Q+RB-v`MRYZ9Gam3cM%{ zb4Cb*f)0deR~wtNb*8w-LlIF>kc7DAv>T0D(a3@l`k4TFnrO+g9XH7;nYOHxjc4lq zMmaW6qpgAgy)MckYMhl?>sq;-1E)-1llUneeA!ya9KM$)DaNGu57Z5aE>=VST$#vb zFo=uRHr$0M{-ha>h(D_boS4zId;3B|Tpqo|?B?Z@I?G(?&Iei+-{9L_A9=h=Qfn-U z1wIUnQe9!z%_j$F_{rf&`ZFSott09gY~qrf@g3O=Y>vzAnXCyL!@(BqWa)Zqt!#_k zfZHuwS52|&&)aK;CHq9V-t9qt0au{$#6c*R#e5n3rje0hic7c7m{kW$p(_`wB=Gw7 z4k`1Hi;Mc@yA7dp@r~?@rfw)TkjAW++|pkfOG}0N|2guek}j8Zen(!+@7?qt_7ndX zB=BG6WJ31#F3#Vk3=aQr8T)3`{=p9nBHlKzE0I@v`{vJ}h8pd6vby&VgFhzH|q;=aonunAXL6G2y(X^CtAhWr*jI zGjpY@raZDQkg*aMq}Ni6cRF z{oWv}5`nhSAv>usX}m^GHt`f(t8@zHc?K|y5Zi=4G*UG1Sza{$Dpj%X8 zzEXaKT5N6F5j4J|w#qlZP!zS7BT)9b+!ZSJdToqJts1c!)fwih4d31vfb{}W)EgcA zH2pZ^8_k$9+WD2n`6q5XbOy8>3pcYH9 z07eUB+p}YD@AH!}p!iKv><2QF-Y^&xx^PAc1F13A{nUeCDg&{hnix#FiO!fe(^&%Qcux!h znu*S!s$&nnkeotYsDthh1dq(iQrE|#f_=xVgfiiL&-5eAcC-> z5L0l|DVEM$#ulf{bj+Y~7iD)j<~O8CYM8GW)dQGq)!mck)FqoL^X zwNdZb3->hFrbHFm?hLvut-*uK?zXn3q1z|UX{RZ;-WiLoOjnle!xs+W0-8D)kjU#R z+S|A^HkRg$Ij%N4v~k`jyHffKaC~=wg=9)V5h=|kLQ@;^W!o2^K+xG&2n`XCd>OY5Ydi= zgHH=lgy++erK8&+YeTl7VNyVm9-GfONlSlVb3)V9NW5tT!cJ8d7X)!b-$fb!s76{t z@d=Vg-5K_sqHA@Zx-L_}wVnc@L@GL9_K~Zl(h5@AR#FAiKad8~KeWCo@mgXIQ#~u{ zgYFwNz}2b6Vu@CP0XoqJ+dm8px(5W5-Jpis97F`+KM)TuP*X8H@zwiVKDKGVp59pI zifNHZr|B+PG|7|Y<*tqap0CvG7tbR1R>jn70t1X`XJixiMVcHf%Ez*=xm1(CrTSDt z0cle!+{8*Ja&EOZ4@$qhBuKQ$U95Q%rc7tg$VRhk?3=pE&n+T3upZg^ZJc9~c2es% zh7>+|mrmA-p&v}|OtxqmHIBgUxL~^0+cpfkSK2mhh+4b=^F1Xgd2)}U*Yp+H?ls#z zrLxWg_hm}AfK2XYWr!rzW4g;+^^&bW%LmbtRai9f3PjU${r@n`JThy-cphbcwn)rq9{A$Ht`lmYKxOacy z6v2R(?gHhD5@&kB-Eg?4!hAoD7~(h>(R!s1c1Hx#s9vGPePUR|of32bS`J5U5w{F) z>0<^ktO2UHg<0{oxkdOQ;}coZDQph8p6ruj*_?uqURCMTac;>T#v+l1Tc~%^k-Vd@ zkc5y35jVNc49vZpZx;gG$h{%yslDI%Lqga1&&;mN{Ush1c7p>7e-(zp}6E7f-XmJb4nhk zb8zS+{IVbL$QVF8pf8}~kQ|dHJAEATmmnrb_wLG}-yHe>W|A&Y|;muy-d^t^<&)g5SJfaTH@P1%euONny=mxo+C z4N&w#biWY41r8k~468tvuYVh&XN&d#%QtIf9;iVXfWY)#j=l`&B~lqDT@28+Y!0E+MkfC}}H*#(WKKdJJq=O$vNYCb(ZG@p{fJgu;h z21oHQ(14?LeT>n5)s;uD@5&ohU!@wX8w*lB6i@GEH0pM>YTG+RAIWZD;4#F1&F%Jp zXZUml2sH0!lYJT?&sA!qwez6cXzJEd(1ZC~kT5kZSp7(@=H2$Azb_*W&6aA|9iwCL zdX7Q=42;@dspHDwYE?miGX#L^3xD&%BI&fN9^;`v4OjQXPBaBmOF1;#C)8XA(WFlH zycro;DS2?(G&6wkr6rqC>rqDv3nfGw3hmN_9Al>TgvmGsL8_hXx09};l9Ow@)F5@y z#VH5WigLDwZE4nh^7&@g{1FV^UZ%_LJ-s<{HN*2R$OPg@R~Z`c-ET*2}XB@9xvAjrK&hS=f|R8Gr9 zr|0TGOsI7RD+4+2{ZiwdVD@2zmg~g@^D--YL;6UYGSM8i$NbQr4!c7T9rg!8;TM0E zT#@?&S=t>GQm)*ua|?TLT2ktj#`|R<_*FAkOu2Pz$wEc%-=Y9V*$&dg+wIei3b*O8 z2|m$!jJG!J!ZGbbIa!(Af~oSyZV+~M1qGvelMzPNE_%5?c2>;MeeG2^N?JDKjFYCy z7SbPWH-$cWF9~fX%9~v99L!G(wi!PFp>rB!9xj7=Cv|F+7CsGNwY0Q_J%FID%C^CBZQfJ9K(HK%k31j~e#&?hQ zNuD6gRkVckU)v+53-fc} z7ZCzYN-5RG4H7;>>Hg?LU9&5_aua?A0)0dpew1#MMlu)LHe(M;OHjHIUl7|%%)YPo z0cBk;AOY00%Fe6heoN*$(b<)Cd#^8Iu;-2v@>cE-OB$icUF9EEoaC&q8z9}jMTT2I z8`9;jT%z0;dy4!8U;GW{i`)3!c6&oWY`J3669C!tM<5nQFFrFRglU8f)5Op$GtR-3 zn!+SPCw|04sv?%YZ(a7#L?vsdr7ss@WKAw&A*}-1S|9~cL%uA+E~>N6QklFE>8W|% zyX-qAUGTY1hQ-+um`2|&ji0cY*(qN!zp{YpDO-r>jPk*yuVSay<)cUt`t@&FPF_&$ zcHwu1(SQ`I-l8~vYyUxm@D1UEdFJ$f5Sw^HPH7b!9 zzYT3gKMF((N(v0#4f_jPfVZ=ApN^jQJe-X$`A?X+vWjLn_%31KXE*}5_}d8 zw_B1+a#6T1?>M{ronLbHIlEsMf93muJ7AH5h%;i99<~JX^;EAgEB1uHralD*!aJ@F zV2ruuFe9i2Q1C?^^kmVy921eb=tLDD43@-AgL^rQ3IO9%+vi_&R2^dpr}x{bCVPej z7G0-0o64uyWNtr*loIvslyo0%)KSDDKjfThe0hcqs)(C-MH1>bNGBDRTW~scy_{w} zp^aq8Qb!h9Lwielq%C1b8=?Z=&U)ST&PHbS)8Xzjh2DF?d{iAv)Eh)wsUnf>UtXN( zL7=$%YrZ#|^c{MYmhn!zV#t*(jdmYdCpwqpZ{v&L8KIuKn`@IIZfp!uo}c;7J57N` zAxyZ-uA4=Gzl~Ovycz%MW9ZL7N+nRo&1cfNn9(1H5eM;V_4Z_qVann7F>5f>%{rf= zPBZFaV@_Sobl?Fy&KXyzFDV*FIdhS5`Uc~S^Gjo)aiTHgn#<0C=9o-a-}@}xDor;D zZyZ|fvf;+=3MZd>SR1F^F`RJEZo+|MdyJYQAEauKu%WDol~ayrGU3zzbHKsnHKZ*z zFiwUkL@DZ>!*x05ql&EBq@_Vqv83&?@~q5?lVmffQZ+V-=qL+!u4Xs2Z2zdCQ3U7B&QR9_Iggy} z(om{Y9eU;IPe`+p1ifLx-XWh?wI)xU9ik+m#g&pGdB5Bi<`PR*?92lE0+TkRuXI)z z5LP!N2+tTc%cB6B1F-!fj#}>S!vnpgVU~3!*U1ej^)vjUH4s-bd^%B=ItQqDCGbrEzNQi(dJ`J}-U=2{7-d zK8k^Rlq2N#0G?9&1?HSle2vlkj^KWSBYTwx`2?9TU_DX#J+f+qLiZCqY1TXHFxXZqYMuD@RU$TgcnCC{_(vwZ-*uX)~go#%PK z@}2Km_5aQ~(<3cXeJN6|F8X_1@L%@xTzs}$_*E|a^_URF_qcF;Pfhoe?FTFwvjm1o z8onf@OY@jC2tVcMaZS;|T!Ks(wOgPpRzRnFS-^RZ4E!9dsnj9sFt609a|jJbb1Dt@ z<=Gal2jDEupxUSwWu6zp<<&RnAA;d&4gKVG0iu6g(DsST(4)z6R)zDpfaQ}v{5ARt zyhwvMtF%b-YazR5XLz+oh=mn;y-Mf2a8>7?2v8qX;19y?b>Z5laGHvzH;Nu9S`B8} zI)qN$GbXIQ1VL3lnof^6TS~rvPVg4V?Dl2Bb*K2z4E{5vy<(@@K_cN@U>R!>aUIRnb zL*)=787*cs#zb31zBC49x$`=fkQbMAef)L2$dR{)6BAz!t5U_B#1zZG`^neKSS22oJ#5B=gl%U=WeqL9REF2g zZnfCb0?quf?Ztj$VXvDSWoK`0L=Zxem2q}!XWLoT-kYMOx)!7fcgT35uC~0pySEme z`{wGWTkGr7>+Kb^n;W?BZH6ZP(9tQX%-7zF>vc2}LuWDI(9kh1G#7B99r4x6;_-V+k&c{nPUrR zAXJGRiMe~aup{0qzmLNjS_BC4cB#sXjckx{%_c&^xy{M61xEb>KW_AG5VFXUOjAG4 z^>Qlm9A#1N{4snY=(AmWzatb!ngqiqPbBZ7>Uhb3)dTkSGcL#&SH>iMO-IJBPua`u zo)LWZ>=NZLr758j{%(|uQuZ)pXq_4c!!>s|aDM9#`~1bzK3J1^^D#<2bNCccH7~-X}Ggi!pIIF>uFx%aPARGQsnC8ZQc8lrQ5o~smqOg>Ti^GNme94*w z)JZy{_{#$jxGQ&`M z!OMvZMHR>8*^>eS%o*6hJwn!l8VOOjZQJvh)@tnHVW&*GYPuxqXw}%M!(f-SQf`=L z5;=5w2;%82VMH6Xi&-K3W)o&K^+vJCepWZ-rW%+Dc6X3(){z$@4zjYxQ|}8UIojeC zYZpQ1dU{fy=oTr<4VX?$q)LP}IUmpiez^O&N3E_qPpchGTi5ZM6-2ScWlQq%V&R2Euz zO|Q0Hx>lY1Q1cW5xHv5!0OGU~PVEqSuy#fD72d#O`N!C;o=m+YioGu-wH2k6!t<~K zSr`E=W9)!g==~x9VV~-8{4ZN9{~-A9zJpRe%NGg$+MDuI-dH|b@BD)~>pPCGUNNzY zMDg||0@XGQgw`YCt5C&A{_+J}mvV9Wg{6V%2n#YSRN{AP#PY?1FF1#|vO_%e+#`|2*~wGAJaeRX6=IzFNeWhz6gJc8+(03Ph4y6ELAm=AkN7TOgMUEw*N{= z_)EIDQx5q22oUR+_b*tazu9+pX|n1c*IB-}{DqIj z-?E|ks{o3AGRNb;+iKcHkZvYJvFsW&83RAPs1Oh@IWy%l#5x2oUP6ZCtv+b|q>jsf zZ_9XO;V!>n`UxH1LvH8)L4?8raIvasEhkpQoJ`%!5rBs!0Tu(s_D{`4opB;57)pkX z4$A^8CsD3U5*!|bHIEqsn~{q+Ddj$ME@Gq4JXtgVz&7l{Ok!@?EA{B3P~NAqb9)4? zkQo30A^EbHfQ@87G5&EQTd`frrwL)&Yw?%-W@uy^Gn23%j?Y!Iea2xw<-f;esq zf%w5WN@E1}zyXtYv}}`U^B>W`>XPmdLj%4{P298|SisrE;7HvXX;A}Ffi8B#3Lr;1 zHt6zVb`8{#+e$*k?w8|O{Uh|&AG}|DG1PFo1i?Y*cQm$ZwtGcVgMwtBUDa{~L1KT-{jET4w60>{KZ27vXrHJ;fW{6| z=|Y4!&UX020wU1>1iRgB@Q#m~1^Z^9CG1LqDhYBrnx%IEdIty z!46iOoKlKs)c}newDG)rWUikD%j`)p z_w9Ph&e40=(2eBy;T!}*1p1f1SAUDP9iWy^u^Ubdj21Kn{46;GR+hwLO=4D11@c~V zI8x&(D({K~Df2E)Nx_yQvYfh4;MbMJ@Z}=Dt3_>iim~QZ*hZIlEs0mEb z_54+&*?wMD`2#vsQRN3KvoT>hWofI_Vf(^C1ff-Ike@h@saEf7g}<9T`W;HAne-Nd z>RR+&SP35w)xKn8^U$7))PsM!jKwYZ*RzEcG-OlTrX3}9a{q%#Un5E5W{{hp>w~;` zGky+3(vJvQyGwBo`tCpmo0mo((?nM8vf9aXrrY1Ve}~TuVkB(zeds^jEfI}xGBCM2 zL1|#tycSaWCurP+0MiActG3LCas@_@tao@(R1ANlwB$4K53egNE_;!&(%@Qo$>h`^1S_!hN6 z)vZtG$8fN!|BXBJ=SI>e(LAU(y(i*PHvgQ2llulxS8>qsimv7yL}0q_E5WiAz7)(f zC(ahFvG8&HN9+6^jGyLHM~$)7auppeWh_^zKk&C_MQ~8;N??OlyH~azgz5fe^>~7F zl3HnPN3z-kN)I$4@`CLCMQx3sG~V8hPS^}XDXZrQA>}mQPw%7&!sd(Pp^P=tgp-s^ zjl}1-KRPNWXgV_K^HkP__SR`S-|OF0bR-N5>I%ODj&1JUeAQ3$9i;B~$S6}*^tK?= z**%aCiH7y?xdY?{LgVP}S0HOh%0%LI$wRx;$T|~Y8R)Vdwa}kGWv8?SJVm^>r6+%I z#lj1aR94{@MP;t-scEYQWc#xFA30^}?|BeX*W#9OL;Q9#WqaaM546j5j29((^_8Nu z4uq}ESLr~r*O7E7$D{!k9W>`!SLoyA53i9QwRB{!pHe8um|aDE`Cg0O*{jmor)^t)3`>V>SWN-2VJcFmj^1?~tT=JrP`fVh*t zXHarp=8HEcR#vFe+1a%XXuK+)oFs`GDD}#Z+TJ}Ri`FvKO@ek2ayn}yaOi%(8p%2$ zpEu)v0Jym@f}U|-;}CbR=9{#<^z28PzkkTNvyKvJDZe+^VS2bES3N@Jq!-*}{oQlz z@8bgC_KnDnT4}d#&Cpr!%Yb?E!brx0!eVOw~;lLwUoz#Np%d$o%9scc3&zPm`%G((Le|6o1 zM(VhOw)!f84zG^)tZ1?Egv)d8cdNi+T${=5kV+j;Wf%2{3g@FHp^Gf*qO0q!u$=m9 zCaY`4mRqJ;FTH5`a$affE5dJrk~k`HTP_7nGTY@B9o9vvnbytaID;^b=Tzp7Q#DmD zC(XEN)Ktn39z5|G!wsVNnHi) z%^q94!lL|hF`IijA^9NR0F$@h7k5R^ljOW(;Td9grRN0Mb)l_l7##{2nPQ@?;VjXv zaLZG}yuf$r$<79rVPpXg?6iiieX|r#&`p#Con2i%S8*8F}(E) zI5E6c3tG*<;m~6>!&H!GJ6zEuhH7mkAzovdhLy;)q z{H2*8I^Pb}xC4s^6Y}6bJvMu=8>g&I)7!N!5QG$xseeU#CC?ZM-TbjsHwHgDGrsD= z{%f;@Sod+Ch66Ko2WF~;Ty)v>&x^aovCbCbD7>qF*!?BXmOV3(s|nxsb*Lx_2lpB7 zokUnzrk;P=T-&kUHO}td+Zdj!3n&NR?K~cRU zAXU!DCp?51{J4w^`cV#ye}(`SQhGQkkMu}O3M*BWt4UsC^jCFUy;wTINYmhD$AT;4 z?Xd{HaJjP`raZ39qAm;%beDbrLpbRf(mkKbANan7XsL>_pE2oo^$TgdidjRP!5-`% zv0d!|iKN$c0(T|L0C~XD0aS8t{*&#LnhE;1Kb<9&=c2B+9JeLvJr*AyyRh%@jHej=AetOMSlz^=!kxX>>B{2B1uIrQyfd8KjJ+DBy!h)~*(!|&L4^Q_07SQ~E zcemVP`{9CwFvPFu7pyVGCLhH?LhEVb2{7U+Z_>o25#+3<|8%1T^5dh}*4(kfJGry} zm%r#hU+__Z;;*4fMrX=Bkc@7|v^*B;HAl0((IBPPii%X9+u3DDF6%bI&6?Eu$8&aWVqHIM7mK6?Uvq$1|(-T|)IV<>e?!(rY zqkmO1MRaLeTR=)io(0GVtQT@s6rN%C6;nS3@eu;P#ry4q;^O@1ZKCJyp_Jo)Ty^QW z+vweTx_DLm{P-XSBj~Sl<%_b^$=}odJ!S2wAcxenmzFGX1t&Qp8Vxz2VT`uQsQYtdn&_0xVivIcxZ_hnrRtwq4cZSj1c-SG9 z7vHBCA=fd0O1<4*=lu$6pn~_pVKyL@ztw1swbZi0B?spLo56ZKu5;7ZeUml1Ws1?u zqMf1p{5myAzeX$lAi{jIUqo1g4!zWLMm9cfWcnw`k6*BR^?$2(&yW?>w;G$EmTA@a z6?y#K$C~ZT8+v{87n5Dm&H6Pb_EQ@V0IWmG9cG=O;(;5aMWWrIPzz4Q`mhK;qQp~a z+BbQrEQ+w{SeiuG-~Po5f=^EvlouB@_|4xQXH@A~KgpFHrwu%dwuCR)=B&C(y6J4J zvoGk9;lLs9%iA-IJGU#RgnZZR+@{5lYl8(e1h6&>Vc_mvg0d@);X zji4T|n#lB!>pfL|8tQYkw?U2bD`W{na&;*|znjmalA&f;*U++_aBYerq;&C8Kw7mI z7tsG*?7*5j&dU)Lje;^{D_h`%(dK|pB*A*1(Jj)w^mZ9HB|vGLkF1GEFhu&rH=r=8 zMxO42e{Si6$m+Zj`_mXb&w5Q(i|Yxyg?juUrY}78uo@~3v84|8dfgbPd0iQJRdMj< zncCNGdMEcsxu#o#B5+XD{tsg*;j-eF8`mp~K8O1J!Z0+>0=7O=4M}E?)H)ENE;P*F z$Ox?ril_^p0g7xhDUf(q652l|562VFlC8^r8?lQv;TMvn+*8I}&+hIQYh2 z1}uQQaag&!-+DZ@|C+C$bN6W;S-Z@)d1|en+XGvjbOxCa-qAF*LA=6s(Jg+g;82f$ z(Vb)8I)AH@cdjGFAR5Rqd0wiNCu!xtqWbcTx&5kslzTb^7A78~Xzw1($UV6S^VWiP zFd{Rimd-0CZC_Bu(WxBFW7+k{cOW7DxBBkJdJ;VsJ4Z@lERQr%3eVv&$%)b%<~ zCl^Y4NgO}js@u{|o~KTgH}>!* z_iDNqX2(As7T0xivMH|3SC1ivm8Q}6Ffcd7owUKN5lHAtzMM4<0v+ykUT!QiowO;`@%JGv+K$bBx@*S7C8GJVqQ_K>12}M`f_Ys=S zKFh}HM9#6Izb$Y{wYzItTy+l5U2oL%boCJn?R3?jP@n$zSIwlmyGq30Cw4QBO|14` zW5c);AN*J3&eMFAk$SR~2k|&+&Bc$e>s%c{`?d~85S-UWjA>DS5+;UKZ}5oVa5O(N zqqc@>)nee)+4MUjH?FGv%hm2{IlIF-QX}ym-7ok4Z9{V+ZHVZQl$A*x!(q%<2~iVv znUa+BX35&lCb#9VE-~Y^W_f;Xhl%vgjwdjzMy$FsSIj&ok}L+X`4>J=9BkN&nu^E*gbhj3(+D>C4E z@Fwq_=N)^bKFSHTzZk?-gNU$@l}r}dwGyh_fNi=9b|n}J>&;G!lzilbWF4B}BBq4f zYIOl?b)PSh#XTPp4IS5ZR_2C!E)Z`zH0OW%4;&~z7UAyA-X|sh9@~>cQW^COA9hV4 zXcA6qUo9P{bW1_2`eo6%hgbN%(G-F1xTvq!sc?4wN6Q4`e9Hku zFwvlAcRY?6h^Fj$R8zCNEDq8`=uZB8D-xn)tA<^bFFy}4$vA}Xq0jAsv1&5!h!yRA zU()KLJya5MQ`q&LKdH#fwq&(bNFS{sKlEh_{N%{XCGO+po#(+WCLmKW6&5iOHny>g z3*VFN?mx!16V5{zyuMWDVP8U*|BGT$(%IO|)?EF|OI*sq&RovH!N%=>i_c?K*A>>k zyg1+~++zY4Q)J;VWN0axhoIKx;l&G$gvj(#go^pZskEVj8^}is3Jw26LzYYVos0HX zRPvmK$dVxM8(Tc?pHFe0Z3uq){{#OK3i-ra#@+;*=ui8)y6hsRv z4Fxx1c1+fr!VI{L3DFMwXKrfl#Q8hfP@ajgEau&QMCxd{g#!T^;ATXW)nUg&$-n25 zruy3V!!;{?OTobo|0GAxe`Acn3GV@W=&n;~&9 zQM>NWW~R@OYORkJAo+eq1!4vzmf9K%plR4(tB@TR&FSbDoRgJ8qVcH#;7lQub*nq&?Z>7WM=oeEVjkaG zT#f)=o!M2DO5hLR+op>t0CixJCIeXH*+z{-XS|%jx)y(j&}Wo|3!l7{o)HU3m7LYyhv*xF&tq z%IN7N;D4raue&&hm0xM=`qv`+TK@;_xAcGKuK(2|75~ar2Yw)geNLSmVxV@x89bQu zpViVKKnlkwjS&&c|-X6`~xdnh}Ps)Hs z4VbUL^{XNLf7_|Oi>tA%?SG5zax}esF*FH3d(JH^Gvr7Rp*n=t7frH!U;!y1gJB^i zY_M$KL_}mW&XKaDEi9K-wZR|q*L32&m+2n_8lq$xRznJ7p8}V>w+d@?uB!eS3#u<} zIaqi!b!w}a2;_BfUUhGMy#4dPx>)_>yZ`ai?Rk`}d0>~ce-PfY-b?Csd(28yX22L% zI7XI>OjIHYTk_@Xk;Gu^F52^Gn6E1&+?4MxDS2G_#PQ&yXPXP^<-p|2nLTb@AAQEY zI*UQ9Pmm{Kat}wuazpjSyXCdnrD&|C1c5DIb1TnzF}f4KIV6D)CJ!?&l&{T)e4U%3HTSYqsQ zo@zWB1o}ceQSV)<4G<)jM|@@YpL+XHuWsr5AYh^Q{K=wSV99D~4RRU52FufmMBMmd z_H}L#qe(}|I9ZyPRD6kT>Ivj&2Y?qVZq<4bG_co_DP`sE*_Xw8D;+7QR$Uq(rr+u> z8bHUWbV19i#)@@G4bCco@Xb<8u~wVDz9S`#k@ciJtlu@uP1U0X?yov8v9U3VOig2t zL9?n$P3=1U_Emi$#slR>N5wH-=J&T=EdUHA}_Z zZIl3nvMP*AZS9{cDqFanrA~S5BqxtNm9tlu;^`)3X&V4tMAkJ4gEIPl= zoV!Gyx0N{3DpD@)pv^iS*dl2FwANu;1;%EDl}JQ7MbxLMAp>)UwNwe{=V}O-5C*>F zu?Ny+F64jZn<+fKjF01}8h5H_3pey|;%bI;SFg$w8;IC<8l|3#Lz2;mNNik6sVTG3 z+Su^rIE#40C4a-587$U~%KedEEw1%r6wdvoMwpmlXH$xPnNQN#f%Z7|p)nC>WsuO= z4zyqapLS<8(UJ~Qi9d|dQijb_xhA2)v>la)<1md5s^R1N&PiuA$^k|A<+2C?OiHbj z>Bn$~t)>Y(Zb`8hW7q9xQ=s>Rv81V+UiuZJc<23HplI88isqRCId89fb`Kt|CxVIg znWcwprwXnotO>3s&Oypkte^9yJjlUVVxSe%_xlzmje|mYOVPH^vjA=?6xd0vaj0Oz zwJ4OJNiFdnHJX3rw&inskjryukl`*fRQ#SMod5J|KroJRsVXa5_$q7whSQ{gOi*s0 z1LeCy|JBWRsDPn7jCb4s(p|JZiZ8+*ExC@Vj)MF|*Vp{B(ziccSn`G1Br9bV(v!C2 z6#?eqpJBc9o@lJ#^p-`-=`4i&wFe>2)nlPK1p9yPFzJCzBQbpkcR>={YtamIw)3nt z(QEF;+)4`>8^_LU)_Q3 zC5_7lgi_6y>U%m)m@}Ku4C}=l^J=<<7c;99ec3p{aR+v=diuJR7uZi%aQv$oP?dn?@6Yu_+*^>T0ptf(oobdL;6)N-I!TO`zg^Xbv3#L0I~sn@WGk-^SmPh5>W+LB<+1PU}AKa?FCWF|qMNELOgdxR{ zbqE7@jVe+FklzdcD$!(A$&}}H*HQFTJ+AOrJYnhh}Yvta(B zQ_bW4Rr;R~&6PAKwgLWXS{Bnln(vUI+~g#kl{r+_zbngT`Y3`^Qf=!PxN4IYX#iW4 zucW7@LLJA9Zh3(rj~&SyN_pjO8H&)|(v%!BnMWySBJV=eSkB3YSTCyIeJ{i;(oc%_hk{$_l;v>nWSB)oVeg+blh=HB5JSlG_r7@P z3q;aFoZjD_qS@zygYqCn=;Zxjo!?NK!%J$ z52lOP`8G3feEj+HTp@Tnn9X~nG=;tS+z}u{mQX_J0kxtr)O30YD%oo)L@wy`jpQYM z@M>Me=95k1p*FW~rHiV1CIfVc{K8r|#Kt(ApkXKsDG$_>76UGNhHExFCw#Ky9*B-z zNq2ga*xax!HMf_|Vp-86r{;~YgQKqu7%szk8$hpvi_2I`OVbG1doP(`gn}=W<8%Gn z%81#&WjkH4GV;4u43EtSW>K_Ta3Zj!XF?;SO3V#q=<=>Tc^@?A`i;&`-cYj|;^ zEo#Jl5zSr~_V-4}y8pnufXLa80vZY4z2ko7fj>DR)#z=wWuS1$$W!L?(y}YC+yQ|G z@L&`2upy3f>~*IquAjkVNU>}c10(fq#HdbK$~Q3l6|=@-eBbo>B9(6xV`*)sae58*f zym~RRVx;xoCG3`JV`xo z!lFw)=t2Hy)e!IFs?0~7osWk(d%^wxq&>_XD4+U#y&-VF%4z?XH^i4w`TxpF{`XhZ z%G}iEzf!T(l>g;W9<~K+)$g!{UvhW{E0Lis(S^%I8OF&%kr!gJ&fMOpM=&=Aj@wuL zBX?*6i51Qb$uhkwkFYkaD_UDE+)rh1c;(&Y=B$3)J&iJfQSx!1NGgPtK!$c9OtJuu zX(pV$bfuJpRR|K(dp@^j}i&HeJOh@|7lWo8^$*o~Xqo z5Sb+!EtJ&e@6F+h&+_1ETbg7LfP5GZjvIUIN3ibCOldAv z)>YdO|NH$x7AC8dr=<2ekiY1%fN*r~e5h6Yaw<{XIErujKV~tiyrvV_DV0AzEknC- zR^xKM3i<1UkvqBj3C{wDvytOd+YtDSGu!gEMg+!&|8BQrT*|p)(dwQLEy+ zMtMzij3zo40)CA!BKZF~yWg?#lWhqD3@qR)gh~D{uZaJO;{OWV8XZ_)J@r3=)T|kt zUS1pXr6-`!Z}w2QR7nP%d?ecf90;K_7C3d!UZ`N(TZoWNN^Q~RjVhQG{Y<%E1PpV^4 z-m-K+$A~-+VDABs^Q@U*)YvhY4Znn2^w>732H?NRK(5QSS$V@D7yz2BVX4)f5A04~$WbxGOam22>t&uD)JB8-~yiQW6ik;FGblY_I>SvB_z2?PS z*Qm&qbKI{H1V@YGWzpx`!v)WeLT02};JJo*#f$a*FH?IIad-^(;9XC#YTWN6;Z6+S zm4O1KH=#V@FJw7Pha0!9Vb%ZIM$)a`VRMoiN&C|$YA3~ZC*8ayZRY^fyuP6$n%2IU z$#XceYZeqLTXw(m$_z|33I$B4k~NZO>pP6)H_}R{E$i%USGy{l{-jOE;%CloYPEU+ zRFxOn4;7lIOh!7abb23YKD+_-?O z0FP9otcAh+oSj;=f#$&*ExUHpd&e#bSF%#8*&ItcL2H$Sa)?pt0Xtf+t)z$_u^wZi z44oE}r4kIZGy3!Mc8q$B&6JqtnHZ>Znn!Zh@6rgIu|yU+zG8q`q9%B18|T|oN3zMq z`l&D;U!OL~%>vo&q0>Y==~zLiCZk4v%s_7!9DxQ~id1LLE93gf*gg&2$|hB#j8;?3 z5v4S;oM6rT{Y;I+#FdmNw z){d%tNM<<#GN%n9ox7B=3#;u7unZ~tLB_vRZ52a&2=IM)2VkXm=L+Iqq~uk#Dug|x z>S84e+A7EiOY5lj*!q?6HDkNh~0g;0Jy(al!ZHHDtur9T$y-~)94HelX1NHjXWIM7UAe}$?jiz z9?P4`I0JM=G5K{3_%2jPLC^_Mlw?-kYYgb7`qGa3@dn|^1fRMwiyM@Ch z;CB&o7&&?c5e>h`IM;Wnha0QKnEp=$hA8TJgR-07N~U5(>9vJzeoFsSRBkDq=x(YgEMpb=l4TDD`2 zwVJpWGTA_u7}?ecW7s6%rUs&NXD3+n;jB86`X?8(l3MBo6)PdakI6V6a}22{)8ilT zM~T*mU}__xSy|6XSrJ^%lDAR3Lft%+yxC|ZUvSO_nqMX!_ul3;R#*{~4DA=h$bP)%8Yv9X zyp><|e8=_ttI}ZAwOd#dlnSjck#6%273{E$kJuCGu=I@O)&6ID{nWF5@gLb16sj|&Sb~+du4e4O_%_o`Ix4NRrAsyr1_}MuP94s>de8cH-OUkVPk3+K z&jW)It9QiU-ti~AuJkL`XMca8Oh4$SyJ=`-5WU<{cIh+XVH#e4d&zive_UHC!pN>W z3TB;Mn5i)9Qn)#6@lo4QpI3jFYc0~+jS)4AFz8fVC;lD^+idw^S~Qhq>Tg(!3$yLD zzktzoFrU@6s4wwCMz}edpF5i5Q1IMmEJQHzp(LAt)pgN3&O!&d?3W@6U4)I^2V{;- z6A(?zd93hS*uQmnh4T)nHnE{wVhh(=MMD(h(P4+^p83Om6t<*cUW>l(qJzr%5vp@K zN27ka(L{JX=1~e2^)F^i=TYj&;<7jyUUR2Bek^A8+3Up*&Xwc{)1nRR5CT8vG>ExV zHnF3UqXJOAno_?bnhCX-&kwI~Ti8t4`n0%Up>!U`ZvK^w2+0Cs-b9%w%4`$+To|k= zKtgc&l}P`*8IS>8DOe?EB84^kx4BQp3<7P{Pq}&p%xF_81pg!l2|u=&I{AuUgmF5n zJQCTLv}%}xbFGYtKfbba{CBo)lWW%Z>i(_NvLhoQZ*5-@2l&x>e+I~0Nld3UI9tdL zRzu8}i;X!h8LHVvN?C+|M81e>Jr38%&*9LYQec9Ax>?NN+9(_>XSRv&6hlCYB`>Qm z1&ygi{Y()OU4@D_jd_-7vDILR{>o|7-k)Sjdxkjgvi{@S>6GqiF|o`*Otr;P)kLHN zZkpts;0zw_6;?f(@4S1FN=m!4^mv~W+lJA`&7RH%2$)49z0A+8@0BCHtj|yH--AEL z0tW6G%X-+J+5a{5*WKaM0QDznf;V?L5&uQw+yegDNDP`hA;0XPYc6e0;Xv6|i|^F2WB)Z$LR|HR4 zTQsRAby9(^Z@yATyOgcfQw7cKyr^3Tz7lc7+JEwwzA7)|2x+PtEb>nD(tpxJQm)Kn zW9K_*r!L%~N*vS8<5T=iv|o!zTe9k_2jC_j*7ik^M_ zaf%k{WX{-;0*`t`G!&`eW;gChVXnJ-Rn)To8vW-?>>a%QU1v`ZC=U)f8iA@%JG0mZ zDqH;~mgBnrCP~1II<=V9;EBL)J+xzCoiRBaeH&J6rL!{4zIY8tZka?_FBeQeNO3q6 zyG_alW54Ba&wQf{&F1v-r1R6ID)PTsqjIBc+5MHkcW5Fnvi~{-FjKe)t1bl}Y;z@< z=!%zvpRua>>t_x}^}z0<7MI!H2v6|XAyR9!t50q-A)xk0nflgF4*OQlCGK==4S|wc zRMsSscNhRzHMBU8TdcHN!q^I}x0iXJ%uehac|Zs_B$p@CnF)HeXPpB_Za}F{<@6-4 zl%kml@}kHQ(ypD8FsPJ2=14xXJE|b20RUIgs!2|R3>LUMGF6X*B_I|$`Qg=;zm7C z{mEDy9dTmPbued7mlO@phdmAmJ7p@GR1bjCkMw6*G7#4+`k>fk1czdJUB!e@Q(~6# zwo%@p@V5RL0ABU2LH7Asq^quDUho@H>eTZH9f*no9fY0T zD_-9px3e}A!>>kv5wk91%C9R1J_Nh!*&Kk$J3KNxC}c_@zlgpJZ+5L)Nw|^p=2ue}CJtm;uj*Iqr)K})kA$xtNUEvX;4!Px*^&9T_`IN{D z{6~QY=Nau6EzpvufB^hflc#XIsSq0Y9(nf$d~6ZwK}fal92)fr%T3=q{0mP-EyP_G z)UR5h@IX}3Qll2b0oCAcBF>b*@Etu*aTLPU<%C>KoOrk=x?pN!#f_Og-w+;xbFgjQ zXp`et%lDBBh~OcFnMKMUoox0YwBNy`N0q~bSPh@+enQ=4RUw1) zpovN`QoV>vZ#5LvC;cl|6jPr}O5tu!Ipoyib8iXqy}TeJ;4+_7r<1kV0v5?Kv>fYp zg>9L`;XwXa&W7-jf|9~uP2iyF5`5AJ`Q~p4eBU$MCC00`rcSF>`&0fbd^_eqR+}mK z4n*PMMa&FOcc)vTUR zlDUAn-mh`ahi_`f`=39JYTNVjsTa_Y3b1GOIi)6dY)D}xeshB0T8Eov5%UhWd1)u}kjEQ|LDo{tqKKrYIfVz~@dp!! zMOnah@vp)%_-jDTUG09l+;{CkDCH|Q{NqX*uHa1YxFShy*1+;J`gywKaz|2Q{lG8x zP?KBur`}r`!WLKXY_K;C8$EWG>jY3UIh{+BLv0=2)KH%P}6xE2kg)%(-uA6lC?u8}{K(#P*c zE9C8t*u%j2r_{;Rpe1A{9nNXU;b_N0vNgyK!EZVut~}+R2rcbsHilqsOviYh-pYX= zHw@53nlmwYI5W5KP>&`dBZe0Jn?nAdC^HY1wlR6$u^PbpB#AS&5L6zqrXN&7*N2Q` z+Rae1EwS)H=aVSIkr8Ek^1jy2iS2o7mqm~Mr&g5=jjt7VxwglQ^`h#Mx+x2v|9ZAwE$i_9918MjJxTMr?n!bZ6n$}y11u8I9COTU`Z$Fi z!AeAQLMw^gp_{+0QTEJrhL424pVDp%wpku~XRlD3iv{vQ!lAf!_jyqd_h}+Tr1XG| z`*FT*NbPqvHCUsYAkFnM`@l4u_QH&bszpUK#M~XLJt{%?00GXY?u_{gj3Hvs!=N(I z(=AuWPijyoU!r?aFTsa8pLB&cx}$*%;K$e*XqF{~*rA-qn)h^!(-;e}O#B$|S~c+U zN4vyOK0vmtx$5K!?g*+J@G1NmlEI=pyZXZ69tAv=@`t%ag_Hk{LP~OH9iE)I= zaJ69b4kuCkV0V zo(M0#>phpQ_)@j;h%m{-a*LGi(72TP)ws2w*@4|C-3+;=5DmC4s7Lp95%n%@Ko zfdr3-a7m*dys9iIci$A=4NPJ`HfJ;hujLgU)ZRuJI`n;Pw|yksu!#LQnJ#dJysgNb z@@qwR^wrk(jbq4H?d!lNyy72~Dnn87KxsgQ!)|*m(DRM+eC$wh7KnS-mho3|KE)7h zK3k;qZ;K1Lj6uEXLYUYi)1FN}F@-xJ z@@3Hb84sl|j{4$3J}aTY@cbX@pzB_qM~APljrjju6P0tY{C@ zpUCOz_NFmALMv1*blCcwUD3?U6tYs+N%cmJ98D%3)%)Xu^uvzF zS5O!sc#X6?EwsYkvPo6A%O8&y8sCCQH<%f2togVwW&{M;PR!a(ZT_A+jVAbf{@5kL zB@Z(hb$3U{T_}SKA_CoQVU-;j>2J=L#lZ~aQCFg-d<9rzs$_gO&d5N6eFSc z1ml8)P*FSi+k@!^M9nDWR5e@ATD8oxtDu=36Iv2!;dZzidIS(PCtEuXAtlBb1;H%Z zwnC^Ek*D)EX4#Q>R$$WA2sxC_t(!!6Tr?C#@{3}n{<^o;9id1RA&-Pig1e-2B1XpG zliNjgmd3c&%A}s>qf{_j#!Z`fu0xIwm4L0)OF=u(OEmp;bLCIaZX$&J_^Z%4Sq4GZ zPn6sV_#+6pJmDN_lx@1;Zw6Md_p0w9h6mHtzpuIEwNn>OnuRSC2=>fP^Hqgc)xu^4 z<3!s`cORHJh#?!nKI`Et7{3C27+EuH)Gw1f)aoP|B3y?fuVfvpYYmmukx0ya-)TQX zR{ggy5cNf4X|g)nl#jC9p>7|09_S7>1D2GTRBUTW zAkQ=JMRogZqG#v;^=11O6@rPPwvJkr{bW-Qg8`q8GoD#K`&Y+S#%&B>SGRL>;ZunM@49!}Uy zN|bBCJ%sO;@3wl0>0gbl3L@1^O60ONObz8ZI7nder>(udj-jt`;yj^nTQ$L9`OU9W zX4alF#$|GiR47%x@s&LV>2Sz2R6?;2R~5k6V>)nz!o_*1Y!$p>BC5&?hJg_MiE6UBy>RkVZj`9UWbRkN-Hk!S`=BS3t3uyX6)7SF#)71*}`~Ogz z1rap5H6~dhBJ83;q-Y<5V35C2&F^JI-it(=5D#v!fAi9p#UwV~2tZQI+W(Dv?1t9? zfh*xpxxO{-(VGB>!Q&0%^YW_F!@aZS#ucP|YaD#>wd1Fv&Z*SR&mc;asi}1G) z_H>`!akh-Zxq9#io(7%;a$)w+{QH)Y$?UK1Dt^4)up!Szcxnu}kn$0afcfJL#IL+S z5gF_Y30j;{lNrG6m~$Ay?)*V9fZuU@3=kd40=LhazjFrau>(Y>SJNtOz>8x_X-BlA zIpl{i>OarVGj1v(4?^1`R}aQB&WCRQzS~;7R{tDZG=HhgrW@B`W|#cdyj%YBky)P= zpxuOZkW>S6%q7U{VsB#G(^FMsH5QuGXhb(sY+!-R8Bmv6Sx3WzSW<1MPPN1!&PurYky(@`bP9tz z52}LH9Q?+FF5jR6-;|+GVdRA!qtd;}*-h&iIw3Tq3qF9sDIb1FFxGbo&fbG5n8$3F zyY&PWL{ys^dTO}oZ#@sIX^BKW*bon=;te9j5k+T%wJ zNJtoN1~YVj4~YRrlZl)b&kJqp+Z`DqT!la$x&&IxgOQw#yZd-nBP3!7FijBXD|IsU8Zl^ zc6?MKpJQ+7ka|tZQLfchD$PD|;K(9FiLE|eUZX#EZxhG!S-63C$jWX1Yd!6-Yxi-u zjULIr|0-Q%D9jz}IF~S%>0(jOqZ(Ln<$9PxiySr&2Oic7vb<8q=46)Ln%Z|<*z5&> z3f~Zw@m;vR(bESB<=Jqkxn(=#hQw42l(7)h`vMQQTttz9XW6^|^8EK7qhju4r_c*b zJIi`)MB$w@9epwdIfnEBR+?~);yd6C(LeMC& zn&&N*?-g&BBJcV;8&UoZi4Lmxcj16ojlxR~zMrf=O_^i1wGb9X-0@6_rpjPYemIin zmJb+;lHe;Yp=8G)Q(L1bzH*}I>}uAqhj4;g)PlvD9_e_ScR{Ipq|$8NvAvLD8MYr}xl=bU~)f%B3E>r3Bu9_t|ThF3C5~BdOve zEbk^r&r#PT&?^V1cb{72yEWH}TXEE}w>t!cY~rA+hNOTK8FAtIEoszp!qqptS&;r$ zaYV-NX96-h$6aR@1xz6_E0^N49mU)-v#bwtGJm)ibygzJ8!7|WIrcb`$XH~^!a#s& z{Db-0IOTFq#9!^j!n_F}#Z_nX{YzBK8XLPVmc&X`fT7!@$U-@2KM9soGbmOSAmqV z{nr$L^MBo_u^Joyf0E^=eo{Rt0{{e$IFA(#*kP@SQd6lWT2-#>` zP1)7_@IO!9lk>Zt?#CU?cuhiLF&)+XEM9B)cS(gvQT!X3`wL*{fArTS;Ak`J<84du zALKPz4}3nlG8Fo^MH0L|oK2-4xIY!~Oux~1sw!+It)&D3p;+N8AgqKI`ld6v71wy8I!eP0o~=RVcFQR2Gr(eP_JbSytoQ$Yt}l*4r@A8Me94y z8cTDWhqlq^qoAhbOzGBXv^Wa4vUz$(7B!mX`T=x_ueKRRDfg&Uc-e1+z4x$jyW_Pm zp?U;-R#xt^Z8Ev~`m`iL4*c#65Nn)q#=Y0l1AuD&+{|8-Gsij3LUZXpM0Bx0u7WWm zH|%yE@-#XEph2}-$-thl+S;__ciBxSSzHveP%~v}5I%u!z_l_KoW{KRx2=eB33umE zIYFtu^5=wGU`Jab8#}cnYry@9p5UE#U|VVvx_4l49JQ;jQdp(uw=$^A$EA$LM%vmE zvdEOaIcp5qX8wX{mYf0;#51~imYYPn4=k&#DsKTxo{_Mg*;S495?OBY?#gv=edYC* z^O@-sd-qa+U24xvcbL0@C7_6o!$`)sVr-jSJE4XQUQ$?L7}2(}Eixqv;L8AdJAVqc zq}RPgpnDb@E_;?6K58r3h4-!4rT4Ab#rLHLX?eMOfluJk=3i1@Gt1i#iA=O`M0@x! z(HtJP9BMHXEzuD93m|B&woj0g6T?f#^)>J>|I4C5?Gam>n9!8CT%~aT;=oco5d6U8 zMXl(=W;$ND_8+DD*?|5bJ!;8ebESXMUKBAf7YBwNVJibGaJ*(2G`F%wx)grqVPjudiaq^Kl&g$8A2 zWMxMr@_$c}d+;_B`#kUX-t|4VKH&_f^^EP0&=DPLW)H)UzBG%%Tra*5 z%$kyZe3I&S#gfie^z5)!twG={3Cuh)FdeA!Kj<-9** zvT*5%Tb`|QbE!iW-XcOuy39>D3oe6x{>&<#E$o8Ac|j)wq#kQzz|ATd=Z0K!p2$QE zPu?jL8Lb^y3_CQE{*}sTDe!2!dtlFjq&YLY@2#4>XS`}v#PLrpvc4*@q^O{mmnr5D zmyJq~t?8>FWU5vZdE(%4cuZuao0GNjp3~Dt*SLaxI#g_u>hu@k&9Ho*#CZP~lFJHj z(e!SYlLigyc?&5-YxlE{uuk$9b&l6d`uIlpg_z15dPo*iU&|Khx2*A5Fp;8iK_bdP z?T6|^7@lcx2j0T@x>X7|kuuBSB7<^zeY~R~4McconTxA2flHC0_jFxmSTv-~?zVT| zG_|yDqa9lkF*B6_{j=T>=M8r<0s;@z#h)3BQ4NLl@`Xr__o7;~M&dL3J8fP&zLfDfy z);ckcTev{@OUlZ`bCo(-3? z1u1xD`PKgSg?RqeVVsF<1SLF;XYA@Bsa&cY!I48ZJn1V<3d!?s=St?TLo zC0cNr`qD*M#s6f~X>SCNVkva^9A2ZP>CoJ9bvgXe_c}WdX-)pHM5m7O zrHt#g$F0AO+nGA;7dSJ?)|Mo~cf{z2L)Rz!`fpi73Zv)H=a5K)*$5sf_IZypi($P5 zsPwUc4~P-J1@^3C6-r9{V-u0Z&Sl7vNfmuMY4yy*cL>_)BmQF!8Om9Dej%cHxbIzA zhtV0d{=%cr?;bpBPjt@4w=#<>k5ee=TiWAXM2~tUGfm z$s&!Dm0R^V$}fOR*B^kGaipi~rx~A2cS0;t&khV1a4u38*XRUP~f za!rZMtay8bsLt6yFYl@>-y^31(*P!L^^s@mslZy(SMsv9bVoX`O#yBgEcjCmGpyc* zeH$Dw6vB5P*;jor+JOX@;6K#+xc)Z9B8M=x2a@Wx-{snPGpRmOC$zpsqW*JCh@M2Y z#K+M(>=#d^>Of9C`))h<=Bsy)6zaMJ&x-t%&+UcpLjV`jo4R2025 zXaG8EA!0lQa)|dx-@{O)qP6`$rhCkoQqZ`^SW8g-kOwrwsK8 z3ms*AIcyj}-1x&A&vSq{r=QMyp3CHdWH35!sad#!Sm>^|-|afB+Q;|Iq@LFgqIp#Z zD1%H+3I?6RGnk&IFo|u+E0dCxXz4yI^1i!QTu7uvIEH>i3rR{srcST`LIRwdV1P;W z+%AN1NIf@xxvVLiSX`8ILA8MzNqE&7>%jMzGt9wm78bo9<;h*W84i29^w!>V>{N+S zd`5Zmz^G;f=icvoOZfK5#1ctx*~UwD=ab4DGQXehQ!XYnak*dee%YN$_ZPL%KZuz$ zD;$PpT;HM^$KwtQm@7uvT`i6>Hae1CoRVM2)NL<2-k2PiX=eAx+-6j#JI?M}(tuBW zkF%jjLR)O`gI2fcPBxF^HeI|DWwQWHVR!;;{BXXHskxh8F@BMDn`oEi-NHt;CLymW z=KSv5)3dyzec0T5B*`g-MQ<;gz=nIWKUi9ko<|4I(-E0k$QncH>E4l z**1w&#={&zv4Tvhgz#c29`m|;lU-jmaXFMC11 z*dlXDMEOG>VoLMc>!rApwOu2prKSi*!w%`yzGmS+k(zm*CsLK*wv{S_0WX^8A-rKy zbk^Gf_92^7iB_uUF)EE+ET4d|X|>d&mdN?x@vxKAQk`O+r4Qdu>XGy(a(19g;=jU} zFX{O*_NG>!$@jh!U369Lnc+D~qch3uT+_Amyi}*k#LAAwh}k8IPK5a-WZ81ufD>l> z$4cF}GSz>ce`3FAic}6W4Z7m9KGO?(eWqi@L|5Hq0@L|&2flN1PVl}XgQ2q*_n2s3 zt5KtowNkTYB5b;SVuoXA@i5irXO)A&%7?V`1@HGCB&)Wgk+l|^XXChq;u(nyPB}b3 zY>m5jkxpZgi)zfbgv&ec4Zqdvm+D<?Im*mXweS9H+V>)zF#Zp3)bhl$PbISY{5=_z!8&*Jv~NYtI-g!>fDs zmvL5O^U%!^VaKA9gvKw|5?-jk>~%CVGvctKmP$kpnpfN{D8@X*Aazi$txfa%vd-|E z>kYmV66W!lNekJPom29LdZ%(I+ZLZYTXzTg*to~m?7vp%{V<~>H+2}PQ?PPAq`36R z<%wR8v6UkS>Wt#hzGk#44W<%9S=nBfB);6clKwnxY}T*w21Qc3_?IJ@4gYzC7s;WP zVQNI(M=S=JT#xsZy7G`cR(BP9*je0bfeN8JN5~zY(DDs0t{LpHOIbN);?T-69Pf3R zSNe*&p2%AwXHL>__g+xd4Hlc_vu<25H?(`nafS%)3UPP7_4;gk-9ckt8SJRTv5v0M z_Hww`qPudL?ajIR&X*;$y-`<)6dxx1U~5eGS13CB!lX;3w7n&lDDiArbAhSycd}+b zya_3p@A`$kQy;|NJZ~s44Hqo7Hwt}X86NK=(ey>lgWTtGL6k@Gy;PbO!M%1~Wcn2k zUFP|*5d>t-X*RU8g%>|(wwj*~#l4z^Aatf^DWd1Wj#Q*AY0D^V@sC`M zjJc6qXu0I7Y*2;;gGu!plAFzG=J;1%eIOdn zQA>J&e05UN*7I5@yRhK|lbBSfJ+5Uq;!&HV@xfPZrgD}kE*1DSq^=%{o%|LChhl#0 zlMb<^a6ixzpd{kNZr|3jTGeEzuo}-eLT-)Q$#b{!vKx8Tg}swCni>{#%vDY$Ww$84 zew3c9BBovqb}_&BRo#^!G(1Eg((BScRZ}C)Oz?y`T5wOrv);)b^4XR8 zhJo7+<^7)qB>I;46!GySzdneZ>n_E1oWZY;kf94#)s)kWjuJN1c+wbVoNQcmnv}{> zN0pF+Sl3E}UQ$}slSZeLJrwT>Sr}#V(dVaezCQl2|4LN`7L7v&siYR|r7M(*JYfR$ zst3=YaDw$FSc{g}KHO&QiKxuhEzF{f%RJLKe3p*7=oo`WNP)M(9X1zIQPP0XHhY3c znrP{$4#Ol$A0s|4S7Gx2L23dv*Gv2o;h((XVn+9+$qvm}s%zi6nI-_s6?mG! zj{DV;qesJb&owKeEK?=J>UcAlYckA7Sl+I&IN=yasrZOkejir*kE@SN`fk<8Fgx*$ zy&fE6?}G)d_N`){P~U@1jRVA|2*69)KSe_}!~?+`Yb{Y=O~_+@!j<&oVQQMnhoIRU zA0CyF1OFfkK44n*JD~!2!SCPM;PRSk%1XL=0&rz00wxPs&-_eapJy#$h!eqY%nS0{ z!aGg58JIJPF3_ci%n)QSVpa2H`vIe$RD43;#IRfDV&Ibit z+?>HW4{2wOfC6Fw)}4x}i1maDxcE1qi@BS*qcxD2gE@h3#4cgU*D-&3z7D|tVZWt= z-Cy2+*Cm@P4GN_TPUtaVyVesbVDazF@)j8VJ4>XZv!f%}&eO1SvIgr}4`A*3#vat< z_MoByL(qW6L7SFZ#|Gc1fFN)L2PxY+{B8tJp+pxRyz*87)vXR}*=&ahXjBlQKguuf zX6x<<6fQulE^C*KH8~W%ptpaC0l?b=_{~*U4?5Vt;dgM4t_{&UZ1C2j?b>b+5}{IF_CUyvz-@QZPMlJ)r_tS$9kH%RPv#2_nMb zRLj5;chJ72*U`Z@Dqt4$@_+k$%|8m(HqLG!qT4P^DdfvGf&){gKnGCX#H0!;W=AGP zbA&Z`-__a)VTS}kKFjWGk z%|>yE?t*EJ!qeQ%dPk$;xIQ+P0;()PCBDgjJm6Buj{f^awNoVx+9<|lg3%-$G(*f) zll6oOkN|yamn1uyl2*N-lnqRI1cvs_JxLTeahEK=THV$Sz*gQhKNb*p0fNoda#-&F zB-qJgW^g}!TtM|0bS2QZekW7_tKu%GcJ!4?lObt0z_$mZ4rbQ0o=^curCs3bJK6sq z9fu-aW-l#>z~ca(B;4yv;2RZ?tGYAU)^)Kz{L|4oPj zdOf_?de|#yS)p2v8-N||+XL=O*%3+y)oI(HbM)Ds?q8~HPzIP(vs*G`iddbWq}! z(2!VjP&{Z1w+%eUq^ '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit + +APP_NAME="Gradle" +APP_BASE_NAME=${0##*/} + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000000..53a6b238d4 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,91 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/ion-java-cli/README.md b/ion-java-cli/README.md index 6bc6d9b84e..1d7006d05a 100644 --- a/ion-java-cli/README.md +++ b/ion-java-cli/README.md @@ -4,15 +4,14 @@ A Java implementation of CLI where its design document is located in [here](http The package is stored under `ion-java/ion-java-cli`. ## Setup -Build ion-java-cli. Note that using -f option for ion-java-cli's `pom.xml`. +Build ion-java-cli. ``` -$ mvn -f ion-java-cli/pom.xml install +./gradle ion-java-cli:build ``` ## Getting Started -Invoking `ion-java-cli-x.y.jar` under `ion-java-cli/target/` directory.
    +Running the test driver CLI. -For example: ``` -java -jar ion-java-cli/target/ion-java-cli-1.0.jar process test_file.ion -f pretty -o output.ion +./gradlew ion-java-cli:run -q --args="process test_file.ion -f pretty -o output.ion" ``` diff --git a/ion-java-cli/build.gradle.kts b/ion-java-cli/build.gradle.kts new file mode 100644 index 0000000000..288e8b3697 --- /dev/null +++ b/ion-java-cli/build.gradle.kts @@ -0,0 +1,28 @@ +plugins { + java + application +} + +description = "A CLI that implements the standard interface defined by ion-test-driver." +version = 1.0 +java.sourceCompatibility = JavaVersion.VERSION_1_8 +java.targetCompatibility = JavaVersion.VERSION_1_8 + +repositories { + mavenCentral() +} + +dependencies { + implementation("args4j:args4j:2.33") + implementation(rootProject) +} + +sourceSets { + main { + java.srcDir("src") + } +} + +application { + mainClass.set("com.amazon.tools.cli.IonJavaCli") +} diff --git a/ion-java-cli/pom.xml b/ion-java-cli/pom.xml deleted file mode 100644 index 17dd85d335..0000000000 --- a/ion-java-cli/pom.xml +++ /dev/null @@ -1,121 +0,0 @@ - - - - 4.0.0 - com.amazon.ion - ion-java-cli - 1.0 - jar - - ${project.groupId}:${project.artifactId} - - A CLI that implements the standard interface defined by ion-test-driver. - - https://github.com/amzn/ion-java/tree/master/ion-java-cli - - - - The Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - - - - - - Amazon Ion Team - ion-team@amazon.com - Amazon - https://github.com/amzn - - - - - scm:git:git@github.com:amzn/ion-java.git - scm:git:git@github.com:amzn/ion-java.git - git@github.com:amzn/ion-java.git - - - - UTF-8 - yyyy - ${maven.build.timestamp} - 1.8 - - - - - - args4j - args4j - 2.33 - - - com.amazon.ion - ion-java - [1.7.0,) - - - - - src - - - - org.apache.maven.plugins - maven-jar-plugin - 3.2.0 - - ${project.build.outputDirectory} - - - com.amazon.tools.cli.IonJavaCli - false - true - lib/ - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - copy-dependencies - package - - copy-dependencies - - - jar - jar - ${project.build.directory}/lib - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.0 - - ${jdkVersion} - ${jdkVersion} - - - - - diff --git a/ion-test-driver-run b/ion-test-driver-run new file mode 100755 index 0000000000..7cf11bed30 --- /dev/null +++ b/ion-test-driver-run @@ -0,0 +1,2 @@ +#!/usr/bin/env sh +./gradlew ion-java-cli:run -q --args="$*" \ No newline at end of file diff --git a/ion-test-driver-setup b/ion-test-driver-setup new file mode 100755 index 0000000000..efef295ca1 --- /dev/null +++ b/ion-test-driver-setup @@ -0,0 +1,2 @@ +#!/usr/bin/env sh +./gradlew ion-java-cli:build \ No newline at end of file diff --git a/pom.xml b/pom.xml deleted file mode 100644 index 1d8e61fb7f..0000000000 --- a/pom.xml +++ /dev/null @@ -1,357 +0,0 @@ - - - - - 4.0.0 - com.amazon.ion - ion-java - 1.9.6-SNAPSHOT - bundle - - ${project.groupId}:${project.artifactId} - - A Java implementation of the Amazon Ion data notation. - - https://github.com/amzn/ion-java/ - - - - The Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - - - - - - Amazon Ion Team - ion-team@amazon.com - Amazon - https://github.com/amzn - - - - - scm:git:git@github.com:amzn/ion-java.git - scm:git:git@github.com:amzn/ion-java.git - git@github.com:amzn/ion-java.git - - - - - - ossrh - https://aws.oss.sonatype.org/content/repositories/snapshots - - - ossrh - https://aws.oss.sonatype.org/service/local/staging/deploy/maven2/ - - - - - UTF-8 - yyyy - ${maven.build.timestamp} - - yyyy-MM-dd'T'HH:mm:ssXXX - ${maven.build.timestamp} - - 6 - 6 - - - - - - junit - junit - 4.13.1 - test - - - - org.hamcrest - hamcrest - 2.2 - test - - - pl.pragmatists - JUnitParams - 1.1.1 - test - - - - - - - - src/main/resources - true - - - src - test - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - ${jdk.source.version} - ${jdk.target.version} - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.19.1 - - - **/*TestCase.java - - -ea -Xmx3000M ${argLine} - - - - - org.apache.felix - maven-bundle-plugin - 4.1.0 - true - - - - !com.amazon.ion.apps.*, - !com.amazon.ion.impl.*, - com.amazon.ion.*, - - - com.amazon.ion.impl._Private_CommandLine - com.amazon.ion - - - - - - - org.jacoco - jacoco-maven-plugin - 0.8.2 - - - default-prepare-agent - - prepare-agent - - - - default-report - test - - report - - - - default-check - - check - - - - - - - - - - - org.apache.maven.plugins - maven-site-plugin - 3.7.1 - - - - - - - - - org.jacoco - jacoco-maven-plugin - 0.8.2 - - - - - report - - - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 2.10.3 - - 8 - - - - - com.amazon.ion.apps:com.amazon.ion.impl - - **/*Private* - - ${project.basedir}/src/com/amazon/ion/overview.html - Amazon Ion Java ${project.version} API Reference - Ion Java ${project.version} -

    Amazon Ion Java ${project.version} API Reference
    - UTF-8 - Copyright © 2007–${build.year} Amazon.com. All Rights Reserved.]]> - Ion Java Javadoc - -Xdoclint:none -
    - - javadoc-no-fork - - - - - - - org.codehaus.mojo - findbugs-maven-plugin - 3.0.5 - - - - org.apache.maven.plugins - maven-pmd-plugin - 3.11.0 - - true - utf-8 - 100 - - - - - org.apache.maven.plugins - maven-jxr-plugin - 3.0.0 - - - - - - - release - - - - - - org.apache.maven.plugins - maven-source-plugin - 2.2.1 - - - attach-sources - - jar-no-fork - - - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 2.10.3 - - com.amazon.ion.apps:com.amazon.ion.impl - - **/*Private* - - ${project.basedir}/src/com/amazon/ion/overview.html - Amazon Ion Java ${project.version} API Reference - Ion Java ${project.version} -
    Amazon Ion Java ${project.version} API Reference
    - UTF-8 - Copyright © 2007–${build.year} Amazon.com. All Rights Reserved.]]> - Ion Java Javadoc - -Xdoclint:none -
    - - - attach-javadocs - - jar - - - -
    - - - - org.apache.maven.plugins - maven-gpg-plugin - 1.5 - - - sign-artifacts - verify - - sign - - - - - - - - org.sonatype.plugins - nexus-staging-maven-plugin - 1.6.3 - true - - ossrh - https://aws.oss.sonatype.org/ - true - - - -
    -
    -
    -
    - - diff --git a/project.version b/project.version new file mode 100644 index 0000000000..47144dcbf3 --- /dev/null +++ b/project.version @@ -0,0 +1 @@ +1.9.6-SNAPSHOT \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts new file mode 100644 index 0000000000..06a2b7f50d --- /dev/null +++ b/settings.gradle.kts @@ -0,0 +1,2 @@ +rootProject.name = "ion-java" +include("ion-java-cli") diff --git a/src/com/amazon/ion/util/JarInfo.java b/src/com/amazon/ion/util/JarInfo.java index 061ee2340c..d7e0f3abea 100644 --- a/src/com/amazon/ion/util/JarInfo.java +++ b/src/com/amazon/ion/util/JarInfo.java @@ -86,7 +86,7 @@ private static String nonEmptyProperty(Properties props, String name) private void loadBuildProperties() throws IonException { - String file = "/build.properties"; + String file = "/ion-java.properties"; try { Properties props = new Properties(); diff --git a/src/main/resources/build.properties b/src/main/resources/build.properties deleted file mode 100644 index 3894cedaa4..0000000000 --- a/src/main/resources/build.properties +++ /dev/null @@ -1,4 +0,0 @@ -# These values are defined in pom.xml -build.version=${pom.version} -build.time=${build.time} -build.name=${pom.name} From ff91d4c3fdd529c03b419b3813912bc5f1006716 Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Fri, 11 Nov 2022 12:01:25 -0800 Subject: [PATCH 179/490] General improvements to CI workflows --- .github/actions/inspect-version/action.yml | 99 +++++++++++++++++++ ...ub-prerelease-after-fast-forward-check.yml | 74 -------------- .github/workflows/main.yml | 45 +++++---- .github/workflows/prepare-release.yml | 75 ++++++++++++++ .../workflows/publish-release-artifacts.yml | 61 ++++++++++++ .../workflows/test-inspect-version-action.yml | 90 +++++++++++++++++ .../version-bump-and-fast-forward-check.yml | 91 ----------------- 7 files changed, 350 insertions(+), 185 deletions(-) create mode 100644 .github/actions/inspect-version/action.yml delete mode 100644 .github/workflows/create-gitHub-prerelease-after-fast-forward-check.yml create mode 100644 .github/workflows/prepare-release.yml create mode 100644 .github/workflows/publish-release-artifacts.yml create mode 100644 .github/workflows/test-inspect-version-action.yml delete mode 100644 .github/workflows/version-bump-and-fast-forward-check.yml diff --git a/.github/actions/inspect-version/action.yml b/.github/actions/inspect-version/action.yml new file mode 100644 index 0000000000..dd34cd7d8b --- /dev/null +++ b/.github/actions/inspect-version/action.yml @@ -0,0 +1,99 @@ +# With a bit of work, this reusable composite action could be used by ANY of our repos +# that use semantic versioned releases. In order to make it reusable in other repos, we +# need to move it to a separate repo so that it can be versioned independently of ion-java. +# +# See https://docs.github.com/en/actions/creating-actions/creating-a-composite-action +# +# However, the more maintainable long-term solution is to rewrite this action as a +# Javascript-based action. +# See https://docs.github.com/en/actions/creating-actions/creating-a-javascript-action + +name: Inspect Version + +description: > + Inspects whether the version is a release version (i.e. has no "-SNAPSHOT" or "-beta" + suffixes) and whether the release version is greater than the most recent release on GitHub. + The action can optionally have an outcome of 'failure' if the version is a release version + that is not valid to release (i.e. not greater than latest existing release). + + When using this action, you should provide a github API auth token in the environment to + prevent being throttled by the GitHub API and to ensure that the action can read the + latest release of the repository, if the repository is private. + + Example usage: + ```yaml + - uses: ./.github/actions/inspect-version + env: + GITHUB_TOKEN: $\{{ secrets.GITHUB_TOKEN }} + with: + project_version: $\{{ env.PROJECT_VERSION }} + repo: amzn/ion-java + fail_if_invalid: true + ``` + +inputs: + project_version: + description: The semantic version of the project. + required: true + repo: + description: The source repo in 'org/repo-name' form + required: true + fail_if_invalid: + description: Whether the action should fail if the conditions are not met. + required: false + default: 'false' + +outputs: + is_newest: + description: Whether or not the project version is greater than the most recent existing version + value: ${{ steps.inspection.outputs.is_newest }} + is_release: + description: Whether or not the version is a release version + value: ${{ steps.inspection.outputs.is_release }} + is_valid_to_release: + description: Whether or not the version is valid to release + value: ${{ steps.inspection.outputs.is_release == 'true' && steps.inspection.outputs.is_newest == 'true' }} + +runs: + using: composite + steps: + - name: Get latest release version + # TODO: Consider handling cases where most recent release is not the "latest" + # This step assumes that the most recent release will have the highest version number. + # That is usually true, but it is NOT guaranteed. + shell: bash + run: | + echo "LATEST_RELEASE_VERSION=$(gh release view -R "${{ inputs.repo }}" --json tagName --jq '.tagName')" >> $GITHUB_ENV + - name: Compare the library version with the latest release version + # TODO: Replace this with a stricter check that the library version is one of the three + # possible subsequent versions. E.g. if latest version is 1.2.3, then library version + # should be one of 1.2.4, 1.3.0, or 2.0.0 + id: is-greater-version + uses: popematt/compare-semvers@v1 + with: + first: ${{ inputs.project_version }} + second: ${{ env.LATEST_RELEASE_VERSION }} + - name: Inspect Version + id: inspection + shell: bash + run: | + if [[ "${{ steps.is-greater-version.outputs.result }}" == 1 ]]; then + IS_NEWEST=true; + else + IS_NEWEST=false; + fi + echo "Is Newest? $IS_NEWEST"; + echo "is_newest=$IS_NEWEST" >> $GITHUB_OUTPUT; + + pattern='^v?[0-9]+\.[0-9]+\.[0-9]+$' + if [[ "${{ inputs.project_version }}" =~ $pattern ]]; then + IS_RELEASE=true; + else + IS_RELEASE=false; + fi + echo "Is Release? $IS_RELEASE"; + echo "is_release=$IS_RELEASE" >> $GITHUB_OUTPUT; + - name: Exit with Error + if: ${{ inputs.fail_if_invalid == 'true' && steps.inspection.outputs.is_newest != 'true' && steps.inspection.outputs.is_release == 'true' }} + shell: bash + run: exit 1 diff --git a/.github/workflows/create-gitHub-prerelease-after-fast-forward-check.yml b/.github/workflows/create-gitHub-prerelease-after-fast-forward-check.yml deleted file mode 100644 index aa54308bd3..0000000000 --- a/.github/workflows/create-gitHub-prerelease-after-fast-forward-check.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Create GitHub prerelease after fast-forward check -on: - pull_request: - types: - - closed - -jobs: - check-version-change: - if: github.event.pull_request.merged == true - name: check-version-change - runs-on: ubuntu-latest - outputs: - output1: ${{ steps.check-version-change.outputs.result }} - steps: - - name: Checkout the current repository master branch - uses: actions/checkout@v3 - with: - repository: amzn/ion-java - ref: master - fetch-depth: 2 - path: ion-java-current - - - name: Check whether the pull request contains version number bump - id: check-version-change - run: | - cd ion-java-current - versionNew="$(> $GITHUB_ENV - echo "file_name=$(ls *.jar)" >> $GITHUB_ENV - cd .. && echo "version_number=$(> $GITHUB_ENV - - - name: Create new release - id: create_new_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: v${{ env.version_number }} - release_name: v${{ env.version_number }} - prerelease: true - - - name: Upload release asset - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_new_release.outputs.upload_url }} - asset_path: ${{ env.path }} - asset_name: ${{ env.file_name }} - asset_content_type: application/zip \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d5ed52e144..9a6613b2a1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,38 +12,43 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - java: [8, 9, 10, 11] + include: + - java: 8 + upload_reports: true + - java: 11 steps: - uses: actions/checkout@v2 with: submodules: recursive - name: Use java ${{ matrix.java }} - uses: actions/setup-java@v2 + uses: actions/setup-java@v3 with: - distribution: 'zulu' + distribution: 'corretto' java-version: ${{ matrix.java }} - uses: gradle/gradle-build-action@v2 with: arguments: build - run: ./ion-test-driver-run version + - if: ${{ matrix.upload_reports }} + uses: codecov/codecov-action@v3 + with: + files: build/reports/jacoco/test/jacocoTestReport.xml - report-generation: + check-version: + # Ensures that the version is not a release (i.e. -SNAPSHOT) or if it is a release version, + # ensures that the version is a later version number than the existing releases. + # See limitations at: + # https://github.com/amzn/ion-java/blob/master/.github/actions/inspect-version/action.yml runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - with: - submodules: recursive - - name: Use java 11 - uses: actions/setup-java@v1 - with: - java-version: 11 - - uses: gradle/gradle-build-action@v2 + - uses: actions/checkout@v3 + - name: Get Project Version + run: | + echo "PROJECT_VERSION=v$(> $GITHUB_ENV + - uses: ./.github/actions/inspect-version + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - arguments: test - - uses: codecov/codecov-action@v1 - with: - file: build/reports/jacoco/test/jacocoTestReport.xml - - - - + repo: amzn/ion-java + project_version: ${{ env.PROJECT_VERSION }} + fail_if_invalid: true diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml new file mode 100644 index 0000000000..a98b44604f --- /dev/null +++ b/.github/workflows/prepare-release.yml @@ -0,0 +1,75 @@ +# This workflow runs when a commit is pushed to 'master' that has a change to the +# 'project.version' file. Since we have a branch protection rule for 'master', a push +# event is functionally the same as a merged PR event. +# +# If the project/library version is not a snapshot version, and it is greater than the +# latest release in GitHub, then it creates a draft PR for the new library version. +# This workflow always compares against the releases in amzn/ion-java, but will create +# a draft release in a forked repository if the workflow is triggered in a fork. +# See limitations at: +# https://github.com/amzn/ion-java/blob/master/.github/actions/inspect-version/action.yml +# +# This workflow is almost a reusable workflow that can be used by any of our repos that +# use semantic versioned releases. In order to make it reusable, we need to move it to +# a separate repo, update the workflow trigger ('on') to be 'workflow_call', and define +# inputs for any context that needs to be passed in. Any repo that uses this must either +# pass in the library version as a workflow argument, or it will need to have the +# `project.version` file. Also, there are some references to the 'master' branch that +# will need to be factored out (since some of our repos use 'main' as the default branch). +# Finally, there are references to "amzn/ion-java" should come from an input variable +# or be retrieved from the GitHub Actions context. +# See https://docs.github.com/en/actions/using-workflows/reusing-workflows +# +# This could also be bundled into a composite custom action instead of a workflow. +# See https://docs.github.com/en/actions/creating-actions/creating-a-composite-action + +name: Prepare Release +on: + # We have a branch protection rule for master, so all commits to master require + # an approved PR. That means this workflow trigger is effectively equivalent to + # "on: PR that is merged into master" + push: + branches: + - 'master' + paths: + - 'project.version' + # Allow it to be manually triggered if necessary + workflow_dispatch: + +jobs: + check-version: + name: Check Project Version + runs-on: ubuntu-latest + outputs: + should_create_draft: ${{ steps.inspect.outputs.is_valid_to_release }} + steps: + - uses: actions/checkout@v3 + - name: Get project version + run: | + echo "PROJECT_VERSION=$(> $GITHUB_ENV + - uses: ./.github/actions/inspect-version + id: inspect + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + repo: amzn/ion-java + project_version: ${{ env.PROJECT_VERSION }} + - run: | + echo "Result: ${{ toJSON(steps.inspect.outputs) }}" + create-draft: + name: Create Draft + runs-on: ubuntu-latest + needs: check-version + if: ${{ needs.check-version.outputs.should_create_draft == 'true' }} + steps: + - uses: actions/checkout@v3 + - name: Create a draft release + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "valid to release: ${{ needs.check-version.steps.inspect.outputs.is_valid_to_release }}" + RELEASE_TAG="v$( + ${{ + matrix.expected.is_newest == steps.inspect.outputs.is_newest + && matrix.expected.is_release == steps.inspect.outputs.is_release + && matrix.expected.is_valid_to_release == steps.inspect.outputs.is_valid_to_release + }} + run: | + echo "Pass? $PASS" + echo "expected: ${{ toJSON(matrix.expected) }}" + echo "was: ${{ toJSON(steps.inspect.outputs) }}" + if [[ $PASS == 'false' ]]; then exit 1; fi diff --git a/.github/workflows/version-bump-and-fast-forward-check.yml b/.github/workflows/version-bump-and-fast-forward-check.yml deleted file mode 100644 index 1d27f5acd5..0000000000 --- a/.github/workflows/version-bump-and-fast-forward-check.yml +++ /dev/null @@ -1,91 +0,0 @@ -name: Version-bump-and-fast-forward-check -on: [pull_request] - -jobs: - check-version-bump: - name: check-version-bump - runs-on: ubuntu-latest - outputs: - output1: ${{ steps.check-version-change.outputs.result }} - steps: - - name: Switch to the branch of the pull request - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - path: ion-java-new - - - name: Checkout the current repository master branch - uses: actions/checkout@v3 - with: - repository: amzn/ion-java - ref: master - path: ion-java-current - - - name: Check whether the pull request contains version number bump - id: check-version-change - run: | - cd /home/runner/work/ion-java/ion-java/ion-java-new - versionNew="$(> $GITHUB_ENV - - - name: Access to the incoming release - uses: actions/checkout@v3 - with: - repository: amzn/ion-java - ref: ${{ github.event.pull_request.head.sha }} - path: ion-java-new - - - name: Get the commit id of the incoming release - run: | - cd ion-java-new - echo "current_release_commit_id=$(git rev-parse HEAD)" >> $GITHUB_ENV - echo $(git rev-parse HEAD) - - - name: Check whether the incoming release is fast-forward compared to the last release - run: | - cd ion-java-new - if git merge-base --is-ancestor ${{env.last_release_commit_id}} ${{env.current_release_commit_id}}; then echo "Trigger github release"; else exit 1; fi - - - name: comment - if: ${{ success() }} - uses: peter-evans/create-or-update-comment@v2 - with: - issue-number: ${{ github.event.pull_request.number }} - body: | - This PR passes version bump check and fast-forward check, please double check the version number and merge this PR. After merging this PR, "Create gitHub prerelease after fast-forward check" workflow will be automatically triggered to create prerelease. - - - name: comment - if: ${{ failure() }} - uses: peter-evans/create-or-update-comment@v2 - with: - issue-number: ${{ github.event.pull_request.number }} - body: | - This PR is not fast-forward compared to the last release, please check. From 9531842e5b93daf2daf33ebc2720bd77b9e304bd Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Fri, 18 Nov 2022 08:29:55 -0800 Subject: [PATCH 180/490] Adds Software BOM to releases Signed-off-by: Matthew Pope --- .github/workflows/publish-release-artifacts.yml | 3 ++- build.gradle.kts | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-release-artifacts.yml b/.github/workflows/publish-release-artifacts.yml index d1dc91a0f3..a708b057fd 100644 --- a/.github/workflows/publish-release-artifacts.yml +++ b/.github/workflows/publish-release-artifacts.yml @@ -48,7 +48,7 @@ jobs: submodules: recursive - uses: gradle/gradle-build-action@v2 with: - arguments: build + arguments: build cyclonedxBom - name: Upload Jar to GitHub release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -58,4 +58,5 @@ jobs: # It may also need to be able to upload more than one file. run: | gh release upload "v$( { setOnlyIf { isReleaseVersion && gradle.taskGraph.hasTask("publish") } } + + cyclonedxBom { + setIncludeConfigs(listOf("runtimeClasspath")) + setSkipConfigs(listOf("compileClasspath", "testCompileClasspath")) + } } publishing { From f16ab292c70f620dc7d6399570b4cf631ad639da Mon Sep 17 00:00:00 2001 From: Dhiresh A Jain Date: Tue, 10 Jan 2023 02:38:02 +0530 Subject: [PATCH 181/490] Fixing Ion Cookbook url (#464) Updated to working cookbook url https://amazon-ion.github.io/ion-docs/guides/cookbook.html --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c715520b82..8f19b1c470 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,6 @@ source libraries. We still maintain the legacy group id but strongly encourage u to the official one. ## Using the Library -A great way to get started is to use the [Ion cookbook](http://amzn.github.io/ion-docs/cookbook.html). +A great way to get started is to use the [Ion cookbook](https://amazon-ion.github.io/ion-docs/guides/cookbook.html). The [API documentation](http://www.javadoc.io/doc/com.amazon.ion/ion-java) will give a lot of detailed information about how to use the library. From 59e2a84bed64b519fa06024d241ad8250b1d4e8c Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Wed, 18 Jan 2023 16:32:11 -0800 Subject: [PATCH 182/490] Update all references to the GitHub organization: amzn -> amazon-ion --- .github/actions/inspect-version/action.yml | 2 +- ...on-java-performance-regression-detector.yml | 4 ++-- .github/workflows/ion-test-driver.yml | 6 +++--- .github/workflows/main.yml | 4 ++-- .github/workflows/prepare-release.yml | 8 ++++---- .../workflows/test-inspect-version-action.yml | 2 +- .gitmodules | 2 +- CONTRIBUTING.md | 6 +++--- README.md | 6 +++--- build.gradle.kts | 10 +++++----- ion-java-cli/README.md | 2 +- src/com/amazon/ion/IonCatalog.java | 2 +- src/com/amazon/ion/IonDatagram.java | 16 ++++++++-------- src/com/amazon/ion/IonReader.java | 4 ++-- src/com/amazon/ion/IonSystem.java | 2 +- src/com/amazon/ion/IonTimestamp.java | 2 +- .../ion/SubstituteSymbolTableException.java | 2 +- src/com/amazon/ion/SymbolTable.java | 2 +- src/com/amazon/ion/Timestamp.java | 2 +- src/com/amazon/ion/impl/ImportLocation.java | 2 +- src/com/amazon/ion/impl/IonBinary.java | 2 +- .../amazon/ion/impl/IonReaderBinaryRawX.java | 10 +++++----- .../ion/impl/IonReaderBinarySystemX.java | 2 +- .../amazon/ion/impl/IonReaderBinaryUserX.java | 8 ++++---- src/com/amazon/ion/impl/IonReaderTextRawX.java | 2 +- .../amazon/ion/impl/IonReaderTextSystemX.java | 2 +- src/com/amazon/ion/impl/IonTypeID.java | 2 +- src/com/amazon/ion/impl/IonWriterSystem.java | 12 ++++++------ .../amazon/ion/impl/IonWriterSystemText.java | 2 +- .../amazon/ion/impl/IonWriterUserBinary.java | 2 +- src/com/amazon/ion/impl/LocalSymbolTable.java | 4 ++-- .../ion/impl/LocalSymbolTableImports.java | 2 +- src/com/amazon/ion/impl/SharedSymbolTable.java | 8 ++++---- src/com/amazon/ion/impl/SymbolTokenImpl.java | 2 +- .../impl/_Private_IonBinaryWriterBuilder.java | 6 +++--- .../amazon/ion/impl/_Private_IonConstants.java | 2 +- .../amazon/ion/impl/_Private_IonSystem.java | 4 ++-- src/com/amazon/ion/impl/_Private_Utils.java | 8 ++++---- .../ion/impl/bin/IonRawBinaryWriter.java | 2 +- .../amazon/ion/impl/lite/IonContainerLite.java | 2 +- .../amazon/ion/impl/lite/IonDatagramLite.java | 2 +- .../amazon/ion/impl/lite/IonStructLite.java | 2 +- .../amazon/ion/impl/lite/IonSymbolLite.java | 10 +++++----- .../amazon/ion/impl/lite/IonSystemLite.java | 2 +- src/com/amazon/ion/impl/lite/IonValueLite.java | 10 +++++----- .../ion/impl/lite/ReverseBinaryEncoder.java | 6 +++--- .../amazon/ion/impl/lite/TopLevelContext.java | 2 +- src/com/amazon/ion/overview.html | 2 +- .../ion/system/IonTextWriterBuilder.java | 2 +- src/com/amazon/ion/util/Equivalence.java | 14 +++++++------- test/com/amazon/ion/CloneTest.java | 6 +++--- test/com/amazon/ion/ContainerTestCase.java | 4 ++-- test/com/amazon/ion/DatagramTest.java | 12 ++++++------ test/com/amazon/ion/FloatTest.java | 2 +- test/com/amazon/ion/IonTestCase.java | 2 +- test/com/amazon/ion/IonValueTest.java | 4 ++-- test/com/amazon/ion/NopPaddingTest.java | 2 +- test/com/amazon/ion/RoundTripTest.java | 4 ++-- test/com/amazon/ion/SequenceTestCase.java | 2 +- .../amazon/ion/SystemProcessingTestCase.java | 4 ++-- test/com/amazon/ion/TestUtils.java | 18 +++++++++--------- test/com/amazon/ion/TimestampTest.java | 6 +++--- test/com/amazon/ion/TrBwBrProcessingTest.java | 2 +- test/com/amazon/ion/TrueSequenceTestCase.java | 16 ++++++++-------- test/com/amazon/ion/impl/BinaryWriterTest.java | 6 +++--- .../amazon/ion/impl/IonMarkupWriterTest.java | 2 +- .../com/amazon/ion/impl/IonWriterTestCase.java | 14 +++++++------- .../OptimizedBinaryWriterSymbolTableTest.java | 4 ++-- .../impl/OptimizedBinaryWriterTestCase.java | 4 ++-- .../ion/impl/OutputStreamWriterTestCase.java | 4 ++-- test/com/amazon/ion/impl/SymbolTableTest.java | 4 ++-- test/com/amazon/ion/impl/TextWriterTest.java | 2 +- test/com/amazon/ion/impl/ValueWriterTest.java | 6 +++--- test/com/amazon/ion/junit/IonAssert.java | 2 +- .../ion/streaming/BinaryStreamingTest.java | 2 +- .../ion/streaming/ReaderFacetTestCase.java | 2 +- test/com/amazon/ion/streaming/ReaderTest.java | 4 ++-- .../ion/streaming/RoundTripStreamingTest.java | 2 +- test/com/amazon/ion/util/EquivalenceTest.java | 4 ++-- 79 files changed, 184 insertions(+), 184 deletions(-) diff --git a/.github/actions/inspect-version/action.yml b/.github/actions/inspect-version/action.yml index dd34cd7d8b..77e3f53370 100644 --- a/.github/actions/inspect-version/action.yml +++ b/.github/actions/inspect-version/action.yml @@ -27,7 +27,7 @@ description: > GITHUB_TOKEN: $\{{ secrets.GITHUB_TOKEN }} with: project_version: $\{{ env.PROJECT_VERSION }} - repo: amzn/ion-java + repo: amazon-ion/ion-java fail_if_invalid: true ``` diff --git a/.github/workflows/ion-java-performance-regression-detector.yml b/.github/workflows/ion-java-performance-regression-detector.yml index 277ccff505..ae6e3b668b 100644 --- a/.github/workflows/ion-java-performance-regression-detector.yml +++ b/.github/workflows/ion-java-performance-regression-detector.yml @@ -29,7 +29,7 @@ jobs: - name: Checkout ion-java-benchmark-cli uses: actions/checkout@v2 with: - repository: amzn/ion-java-benchmark-cli + repository: amazon-ion/ion-java-benchmark-cli ref: master path: ion-java-benchmark-cli @@ -72,7 +72,7 @@ jobs: - name: Checkout the current commit uses: actions/checkout@v2 with: - repository: amzn/ion-java + repository: amazon-ion/ion-java ref: master path: ion-java diff --git a/.github/workflows/ion-test-driver.yml b/.github/workflows/ion-test-driver.yml index e39846aed3..c7b166b8e0 100644 --- a/.github/workflows/ion-test-driver.yml +++ b/.github/workflows/ion-test-driver.yml @@ -9,14 +9,14 @@ jobs: - name: Checkout ion-java uses: actions/checkout@master with: - repository: amzn/ion-java + repository: amazon-ion/ion-java ref: master path: ion-java - name: Checkout ion-test-driver uses: actions/checkout@master with: - repository: amzn/ion-test-driver + repository: amazon-ion/ion-test-driver ref: master path: ion-test-driver @@ -36,7 +36,7 @@ jobs: - name: Run ion-test-driver run: python3 ion-test-driver/amazon/iontest/ion_test_driver.py -o output -i ion-java,${{ github.event.pull_request.head.repo.html_url }},${{ github.event.pull_request.head.sha }} - --replace ion-java,https://github.com/amzn/ion-java.git,$main + --replace ion-java,https://github.com/amazon-ion/ion-java.git,$main - name: Upload result uses: actions/upload-artifact@v2 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9a6613b2a1..e0bbcde442 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -38,7 +38,7 @@ jobs: # Ensures that the version is not a release (i.e. -SNAPSHOT) or if it is a release version, # ensures that the version is a later version number than the existing releases. # See limitations at: - # https://github.com/amzn/ion-java/blob/master/.github/actions/inspect-version/action.yml + # https://github.com/amazon-ion/ion-java/blob/master/.github/actions/inspect-version/action.yml runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -49,6 +49,6 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - repo: amzn/ion-java + repo: amazon-ion/ion-java project_version: ${{ env.PROJECT_VERSION }} fail_if_invalid: true diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index a98b44604f..beff803d54 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -4,10 +4,10 @@ # # If the project/library version is not a snapshot version, and it is greater than the # latest release in GitHub, then it creates a draft PR for the new library version. -# This workflow always compares against the releases in amzn/ion-java, but will create +# This workflow always compares against the releases in amazon-ion/ion-java, but will create # a draft release in a forked repository if the workflow is triggered in a fork. # See limitations at: -# https://github.com/amzn/ion-java/blob/master/.github/actions/inspect-version/action.yml +# https://github.com/amazon-ion/ion-java/blob/master/.github/actions/inspect-version/action.yml # # This workflow is almost a reusable workflow that can be used by any of our repos that # use semantic versioned releases. In order to make it reusable, we need to move it to @@ -16,7 +16,7 @@ # pass in the library version as a workflow argument, or it will need to have the # `project.version` file. Also, there are some references to the 'master' branch that # will need to be factored out (since some of our repos use 'main' as the default branch). -# Finally, there are references to "amzn/ion-java" should come from an input variable +# Finally, there are references to "amazon-ion/ion-java" should come from an input variable # or be retrieved from the GitHub Actions context. # See https://docs.github.com/en/actions/using-workflows/reusing-workflows # @@ -52,7 +52,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - repo: amzn/ion-java + repo: amazon-ion/ion-java project_version: ${{ env.PROJECT_VERSION }} - run: | echo "Result: ${{ toJSON(steps.inspect.outputs) }}" diff --git a/.github/workflows/test-inspect-version-action.yml b/.github/workflows/test-inspect-version-action.yml index 0191fdc6b6..e7d5bf4bcf 100644 --- a/.github/workflows/test-inspect-version-action.yml +++ b/.github/workflows/test-inspect-version-action.yml @@ -60,7 +60,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - repo: amzn/ion-java + repo: amazon-ion/ion-java project_version: ${{ matrix.version }} fail_if_invalid: ${{ matrix.fail_if_invalid }} diff --git a/.gitmodules b/.gitmodules index 38171e6698..23a0a7946d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "ion-tests"] path = ion-tests - url = https://github.com/amzn/ion-tests.git + url = https://github.com/amazon-ion/ion-tests.git diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f226f24d2d..b81cdc1215 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,7 +11,7 @@ information to effectively respond to your bug report or contribution. We welcome you to use the GitHub issue tracker to report bugs or suggest features. -When filing an issue, please check [existing open](https://github.com/amzn/ion-java/issues), or [recently closed](https://github.com/amzn/ion-java/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already +When filing an issue, please check [existing open](https://github.com/amazon-ion/ion-java/issues), or [recently closed](https://github.com/amazon-ion/ion-java/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: * A reproducible test case or series of steps @@ -41,7 +41,7 @@ GitHub provides additional document on [forking a repository](https://help.githu ## Finding contributions to work on -Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/amzn/ion-java/labels/help%20wanted) issues is a great place to start. +Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/amazon-ion/ion-java/labels/help%20wanted) issues is a great place to start. ## Code of Conduct @@ -56,6 +56,6 @@ If you discover a potential security issue in this project we ask that you notif ## Licensing -See the [LICENSE](https://github.com/amzn/ion-java/blob/master/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. +See the [LICENSE](https://github.com/amazon-ion/ion-java/blob/master/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. \ No newline at end of file diff --git a/README.md b/README.md index 8f19b1c470..23038b4814 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Amazon Ion Java -A Java implementation of the [Ion data notation](http://amzn.github.io/ion-docs). +A Java implementation of the [Ion data notation](https://amazon-ion.github.io/ion-docs). -[![Build Status](https://travis-ci.org/amzn/ion-java.svg?branch=master)](https://travis-ci.org/amzn/ion-java) +[![Build Status](https://travis-ci.org/amazon-ion/ion-java.svg?branch=master)](https://travis-ci.org/amazon-ion/ion-java) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.amazon.ion/ion-java/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.amazon.ion/ion-java) [![Javadoc](https://javadoc-badge.appspot.com/com.amazon.ion/ion-java.svg?label=javadoc)](http://www.javadoc.io/doc/com.amazon.ion/ion-java) @@ -13,7 +13,7 @@ The easiest way to clone the `ion-java` repository and initialize its `ion-tests submodule is to run the following command. ``` -$ git clone --recursive https://github.com/amzn/ion-java.git ion-java +$ git clone --recursive https://github.com/amazon-ion/ion-java.git ion-java ``` Alternatively, the submodule may be initialized independently from the clone diff --git a/build.gradle.kts b/build.gradle.kts index dbb6456154..f0ae76c691 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -72,7 +72,7 @@ tasks { /** * This task creates a properties file that will be included in the compiled jar. It contains information about - * the build/version of the library used in [com.amazon.ion.util.JarInfo]. See https://github.com/amzn/ion-java/pull/433 + * the build/version of the library used in [com.amazon.ion.util.JarInfo]. See https://github.com/amazon-ion/ion-java/pull/433 * for why this is done with a properties file rather than the Jar manifest. */ val generateJarInfo by creating { @@ -124,7 +124,7 @@ publishing { pom { name.set("Ion Java") description.set(project.description) - url.set("https://github.com/amzn/ion-java/") + url.set("https://github.com/amazon-ion/ion-java/") licenses { license { @@ -141,9 +141,9 @@ publishing { } } scm { - connection.set("scm:git:git@github.com:amzn/ion-java.git") - developerConnection.set("scm:git:git@github.com:amzn/ion-java.git") - url.set("git@github.com:amzn/ion-java.git") + connection.set("scm:git:git@github.com:amazon-ion/ion-java.git") + developerConnection.set("scm:git:git@github.com:amazon-ion/ion-java.git") + url.set("git@github.com:amazon-ion/ion-java.git") } } } diff --git a/ion-java-cli/README.md b/ion-java-cli/README.md index 1d7006d05a..6507f1ee1f 100644 --- a/ion-java-cli/README.md +++ b/ion-java-cli/README.md @@ -1,5 +1,5 @@ # Amazon Ion Java CLI -A Java implementation of CLI where its design document is located in [here](https://github.com/amzn/ion-test-driver#design). +A Java implementation of CLI where its design document is located in [here](https://github.com/amazon-ion/ion-test-driver#design). The package is stored under `ion-java/ion-java-cli`. diff --git a/src/com/amazon/ion/IonCatalog.java b/src/com/amazon/ion/IonCatalog.java index cf79390d92..943a4c33ac 100644 --- a/src/com/amazon/ion/IonCatalog.java +++ b/src/com/amazon/ion/IonCatalog.java @@ -72,7 +72,7 @@ * Binary decoding prefers an exact match, and in a couple edge cases, * requires it. Therefore a single "get latest version" method is insufficient. * See the - * Ion Symbols page + * Ion Symbols page * for more details on this topic. *

    * It's expected that many if not most applications will implement a dynamic diff --git a/src/com/amazon/ion/IonDatagram.java b/src/com/amazon/ion/IonDatagram.java index 72394b35d3..bbb442124d 100644 --- a/src/com/amazon/ion/IonDatagram.java +++ b/src/com/amazon/ion/IonDatagram.java @@ -42,11 +42,11 @@ public interface IonDatagram /** * This inherited method is not yet supported by datagrams. *

    - * Vote for issue amzn/ion-java/issues/48 if you need this. + * Vote for issue amazon-ion/ion-java/issues/48 if you need this. * * @throws UnsupportedOperationException at every call. * - * @see amzn/ion-java/issues/48 + * @see amazon-ion/ion-java/issues/48 */ public void add(int index, IonValue element) throws ContainedValueException, NullPointerException; @@ -54,11 +54,11 @@ public void add(int index, IonValue element) /** * This inherited method is not yet supported by datagrams. *

    - * Vote for issue amzn/ion-java/issues/48 if you need this. + * Vote for issue amazon-ion/ion-java/issues/48 if you need this. * * @throws UnsupportedOperationException at every call. * - * @see amzn/ion-java/issues/48 + * @see amazon-ion/ion-java/issues/48 */ public ValueFactory add(int index) throws ContainedValueException, NullPointerException; @@ -66,22 +66,22 @@ public ValueFactory add(int index) /** * This inherited method is not yet supported by datagrams. *

    - * Vote for issue amzn/ion-java/issues/47 if you need this. + * Vote for issue amazon-ion/ion-java/issues/47 if you need this. * * @throws UnsupportedOperationException at every call. * - * @see amzn/ion-java/issues/47 + * @see amazon-ion/ion-java/issues/47 */ public boolean addAll(int index, Collection c); /** * This inherited method is not yet supported by datagrams. *

    - * Vote for issue amzn/ion-java/issues/50 if you need this. + * Vote for issue amazon-ion/ion-java/issues/50 if you need this. * * @throws UnsupportedOperationException at every call. * - * @see amzn/ion-java/issues/50 + * @see amazon-ion/ion-java/issues/50 */ public IonValue set(int index, IonValue element); diff --git a/src/com/amazon/ion/IonReader.java b/src/com/amazon/ion/IonReader.java index 9066c47326..f8e678737a 100644 --- a/src/com/amazon/ion/IonReader.java +++ b/src/com/amazon/ion/IonReader.java @@ -41,7 +41,7 @@ * WARNING: This interface should not be implemented or extended by * code outside of this library. * We still have some work to do before this interface is stable. - * See issue amzn/ion-java/issues/11 + * See issue amazon-ion/ion-java/issues/11 *

    * An {@code IonReader} has a "cursor" tracking the current value on * which the reader is positioned. Generally, newly created readers are not @@ -88,7 +88,7 @@ *

    The {@link SeekableReader} Facet

    * This facet is available on all readers except those created from * an {@link java.io.InputStream InputStream}. - * (See issue amzn/ion-java/issues/17.) + * (See issue amazon-ion/ion-java/issues/17.) * It allows the user to reposition the reader to a {@link Span} over the * same reader instance or another reader with the same source. * diff --git a/src/com/amazon/ion/IonSystem.java b/src/com/amazon/ion/IonSystem.java index 8dd489562a..aa1b4b93f5 100644 --- a/src/com/amazon/ion/IonSystem.java +++ b/src/com/amazon/ion/IonSystem.java @@ -94,7 +94,7 @@ public SymbolTable getSystemSymbolTable(String ionVersionId) * or if any but the first is a system table. * @throws NullPointerException if any import is null. */ - // TODO amzn/ion-java/issues/38 Should we allow substituted imports as valid args? + // TODO amazon-ion/ion-java/issues/38 Should we allow substituted imports as valid args? public SymbolTable newLocalSymbolTable(SymbolTable... imports); diff --git a/src/com/amazon/ion/IonTimestamp.java b/src/com/amazon/ion/IonTimestamp.java index 63bb78c5bc..04c2006f71 100644 --- a/src/com/amazon/ion/IonTimestamp.java +++ b/src/com/amazon/ion/IonTimestamp.java @@ -29,7 +29,7 @@ public interface IonTimestamp extends IonValue { - // TODO amzn/ion-java/issues/223 Deprecate setters and getters + // TODO amazon-ion/ion-java/issues/223 Deprecate setters and getters /** * Gets the value of this timestamp in a form suitable for diff --git a/src/com/amazon/ion/SubstituteSymbolTableException.java b/src/com/amazon/ion/SubstituteSymbolTableException.java index 7472ec8657..9f291f3858 100644 --- a/src/com/amazon/ion/SubstituteSymbolTableException.java +++ b/src/com/amazon/ion/SubstituteSymbolTableException.java @@ -21,7 +21,7 @@ * * @see SymbolTable#isSubstitute() */ -// TODO amzn/ion-java/issues/40 Provide some useful info to assist callers with handling this +// TODO amazon-ion/ion-java/issues/40 Provide some useful info to assist callers with handling this // exception. E.g. reference to the substitute import in violation. public class SubstituteSymbolTableException extends IonException diff --git a/src/com/amazon/ion/SymbolTable.java b/src/com/amazon/ion/SymbolTable.java index e73e1da738..2a7d2a0b55 100644 --- a/src/com/amazon/ion/SymbolTable.java +++ b/src/com/amazon/ion/SymbolTable.java @@ -43,7 +43,7 @@ * Implementations of this interface are safe for use by multiple * threads. * - * @see Ion Symbols page + * @see Ion Symbols page */ public interface SymbolTable { diff --git a/src/com/amazon/ion/Timestamp.java b/src/com/amazon/ion/Timestamp.java index 0480090eae..6c8955a8a5 100644 --- a/src/com/amazon/ion/Timestamp.java +++ b/src/com/amazon/ion/Timestamp.java @@ -932,7 +932,7 @@ private static IllegalArgumentException fail(CharSequence input) * @return * {@code null} if the {@code CharSequence} is "null.timestamp" * - * @see Ion Timestamp Page + * @see Ion Timestamp Page * @see W3C Note on Date and Time Formats */ public static Timestamp valueOf(CharSequence ionFormattedTimestamp) diff --git a/src/com/amazon/ion/impl/ImportLocation.java b/src/com/amazon/ion/impl/ImportLocation.java index acd5873887..0fa859e9a9 100644 --- a/src/com/amazon/ion/impl/ImportLocation.java +++ b/src/com/amazon/ion/impl/ImportLocation.java @@ -5,7 +5,7 @@ * in a shared symbol table. * NOTE: this is currently not publicly accessible, but it is an important step toward being able to correctly * round-trip symbols with unknown text from shared symbol tables in different symbol table contexts. See - * https://github.com/amzn/ion-java/issues/126 . Support is added now to avoid risking the appearance of performance + * https://github.com/amazon-ion/ion-java/issues/126 . Support is added now to avoid risking the appearance of performance * degradation if ImportLocation support were added after initial release of the incremental IonReader. */ class ImportLocation { diff --git a/src/com/amazon/ion/impl/IonBinary.java b/src/com/amazon/ion/impl/IonBinary.java index 956d49212c..a846239c51 100644 --- a/src/com/amazon/ion/impl/IonBinary.java +++ b/src/com/amazon/ion/impl/IonBinary.java @@ -800,7 +800,7 @@ public static int lenAnnotationListWithLen(String[] annotations, // add up the length of the encoded symbols for (int ii=0; ii 0; // TODO amzn/ion-java/issues/12 + assert symid > 0; // TODO amazon-ion/ion-java/issues/12 annotationLen += IonBinary.lenVarUInt(symid); } diff --git a/src/com/amazon/ion/impl/IonReaderBinaryRawX.java b/src/com/amazon/ion/impl/IonReaderBinaryRawX.java index a1585d6ed5..e1304035be 100644 --- a/src/com/amazon/ion/impl/IonReaderBinaryRawX.java +++ b/src/com/amazon/ion/impl/IonReaderBinaryRawX.java @@ -546,7 +546,7 @@ else if (tid == _Private_IonConstants.tidStruct) { // the value BEFORE a *Value() method is even called. Once that is fixed, // _value_start can be removed, and _input._pos can be used to find the // start of the value at the current valid position. - // amzn/ion-java/issues/88 tracks the fix for bringing IVM handling up to + // amazon-ion/ion-java/issues/88 tracks the fix for bringing IVM handling up to // spec. _value_start = start_of_value; _position_len = len + (start_of_value - start_of_tid); @@ -968,7 +968,7 @@ private int readVarInt(int firstByte) throws IOException { // into a Java int. // To validate overflows we accumulate the VarInt in a long and then check if it can be represented by an int // - // see http://amzn.github.io/ion-docs/docs/binary.html#varuint-and-varint-fields + // see https://amazon-ion.github.io/ion-docs/docs/binary.html#varuint-and-varint-fields long retValue = 0; int b = firstByte; @@ -1003,7 +1003,7 @@ private int readVarInt(int firstByte) throws IOException { retValue = (retValue << 7) | (b & 0x7F); if ((b & 0x80) != 0) break; - // Don't support anything above a 5-byte VarInt for now, see https://github.com/amzn/ion-java/issues/146 + // Don't support anything above a 5-byte VarInt for now, see https://github.com/amazon-ion/ion-java/issues/146 throwVarIntOverflowException(5); } @@ -1036,7 +1036,7 @@ protected final long readVarUIntOrEOF(boolean longAllowed) throws IOException // into a Java int. // To validate overflows we accumulate the VarInt in a long and then check if it can be represented by an int // - // see http://amzn.github.io/ion-docs/docs/binary.html#varuint-and-varint-fields + // see https://amazon-ion.github.io/ion-docs/docs/binary.html#varuint-and-varint-fields long retvalue = 0; int b; @@ -1071,7 +1071,7 @@ protected final long readVarUIntOrEOF(boolean longAllowed) throws IOException retvalue = (retvalue << 7) | (b & 0x7F); if ((b & 0x80) != 0) break; - // Don't support anything above a 7-byte VarUInt for now, see https://github.com/amzn/ion-java/issues/146 + // Don't support anything above a 7-byte VarUInt for now, see https://github.com/amazon-ion/ion-java/issues/146 throwVarIntOverflowException(7); } diff --git a/src/com/amazon/ion/impl/IonReaderBinarySystemX.java b/src/com/amazon/ion/impl/IonReaderBinarySystemX.java index b18c819b2b..f0f73723af 100644 --- a/src/com/amazon/ion/impl/IonReaderBinarySystemX.java +++ b/src/com/amazon/ion/impl/IonReaderBinarySystemX.java @@ -50,7 +50,7 @@ class IonReaderBinarySystemX { super(); init_raw(in); - // TODO check IVM to determine version: amzn/ion-java#19, amzn/ion-java#24 + // TODO check IVM to determine version: amazon-ion/ion-java#19, amazon-ion/ion-java#24 _symbols = SharedSymbolTable.getSystemSymbolTable(1); } diff --git a/src/com/amazon/ion/impl/IonReaderBinaryUserX.java b/src/com/amazon/ion/impl/IonReaderBinaryUserX.java index 31d3d775be..a9ea04080f 100644 --- a/src/com/amazon/ion/impl/IonReaderBinaryUserX.java +++ b/src/com/amazon/ion/impl/IonReaderBinaryUserX.java @@ -97,7 +97,7 @@ public IonReaderBinaryUserX(IonCatalog catalog, //FIXME: PERF_TEST was :private final void init_user(IonCatalog catalog) { - // TODO check IVM to determine version: amzn/ion-java#19, amzn/ion-java#24 + // TODO check IVM to determine version: amazon-ion/ion-java#19, amazon-ion/ion-java#24 _symbols = SharedSymbolTable.getSystemSymbolTable(1); _catalog = catalog; } @@ -343,7 +343,7 @@ public T asFacet(Class facetType) return facetType.cast(new SpanProviderFacet()); } - // TODO amzn/ion-java/issues/17 support seeking over InputStream + // TODO amazon-ion/ion-java/issues/17 support seeking over InputStream if (_input instanceof FromByteArray) { if (facetType == SeekableReader.class) @@ -362,7 +362,7 @@ public T asFacet(Class facetType) // of the facet depends upon the current state of this subject, // and that can change over time. - // TODO amzn/ion-java/issues/16 Our {@link #transferCurrentValue} doesn't handle + // TODO amazon-ion/ion-java/issues/16 Our {@link #transferCurrentValue} doesn't handle // field names and annotations. // Ensure there's a contiguous buffer we can copy. @@ -431,7 +431,7 @@ public void transferCurrentValue(_Private_ByteTransferSink sink) throw new UnsupportedOperationException(); } - // TODO amzn/ion-java/issues/16 wrong if current value has a field name or + // TODO amazon-ion/ion-java/issues/16 wrong if current value has a field name or // annotations since the position is in the wrong place. // TODO when implementing that, be careful to handle the case where // the writer already holds a pending field name or annotations! diff --git a/src/com/amazon/ion/impl/IonReaderTextRawX.java b/src/com/amazon/ion/impl/IonReaderTextRawX.java index 7729340369..4bfb34b31a 100644 --- a/src/com/amazon/ion/impl/IonReaderTextRawX.java +++ b/src/com/amazon/ion/impl/IonReaderTextRawX.java @@ -50,7 +50,7 @@ * * This reader scan skip values and in doing so it does not * materialize the contents and it does not validate the contents. - * TODO amzn/ion-java/issues/7 We may want to make validation on skip optional. + * TODO amazon-ion/ion-java/issues/7 We may want to make validation on skip optional. * * This manages the value buffer (_v ValueVariant) and the lob * content (_lob_*) which is cached in some cases. It's main diff --git a/src/com/amazon/ion/impl/IonReaderTextSystemX.java b/src/com/amazon/ion/impl/IonReaderTextSystemX.java index 62b743a7b2..661fbd5c19 100644 --- a/src/com/amazon/ion/impl/IonReaderTextSystemX.java +++ b/src/com/amazon/ion/impl/IonReaderTextSystemX.java @@ -64,7 +64,7 @@ class IonReaderTextSystemX protected IonReaderTextSystemX(UnifiedInputStreamX iis) { - _system_symtab = _Private_Utils.systemSymtab(1); // TODO check IVM to determine version: amzn/ion-java/issues/19 + _system_symtab = _Private_Utils.systemSymtab(1); // TODO check IVM to determine version: amazon-ion/ion-java/issues/19 init_once(); init(iis, IonType.DATAGRAM); } diff --git a/src/com/amazon/ion/impl/IonTypeID.java b/src/com/amazon/ion/impl/IonTypeID.java index d94cce0d6a..95b5299750 100644 --- a/src/com/amazon/ion/impl/IonTypeID.java +++ b/src/com/amazon/ion/impl/IonTypeID.java @@ -23,7 +23,7 @@ final class IonTypeID { // does not have a type ID, so we will use it to mean 'annotation wrapper' instead. static final IonType ION_TYPE_ANNOTATION_WRAPPER = IonType.DATAGRAM; - // Lookup table from type ID to IonType. See https://amzn.github.io/ion-docs/docs/binary.html#typed-value-formats + // Lookup table from type ID to IonType. See https://amazon-ion.github.io/ion-docs/docs/binary.html#typed-value-formats static final IonType[] ION_TYPES = new IonType[] { IonType.NULL, IonType.BOOL, diff --git a/src/com/amazon/ion/impl/IonWriterSystem.java b/src/com/amazon/ion/impl/IonWriterSystem.java index 02890c0a85..be0c5d245e 100644 --- a/src/com/amazon/ion/impl/IonWriterSystem.java +++ b/src/com/amazon/ion/impl/IonWriterSystem.java @@ -139,20 +139,20 @@ boolean shouldWriteIvm() } if (_initial_ivm_handling == InitialIvmHandling.SUPPRESS) { - // TODO amzn/ion-java/issues/24 Must write IVM if given system != 1.0 + // TODO amazon-ion/ion-java/issues/24 Must write IVM if given system != 1.0 return false; } - // TODO amzn/ion-java/issues/24 Add SUPPRESS_ALL to suppress non 1.0 IVMs + // TODO amazon-ion/ion-java/issues/24 Add SUPPRESS_ALL to suppress non 1.0 IVMs if (_ivm_minimizing == IvmMinimizing.ADJACENT) { - // TODO amzn/ion-java/issues/24 Write IVM if current system version != given system + // TODO amazon-ion/ion-java/issues/24 Write IVM if current system version != given system // For now we assume that it's the same since we only support 1.0 return ! _previous_value_was_ivm; } if (_ivm_minimizing == IvmMinimizing.DISTANT) { - // TODO amzn/ion-java/issues/24 Write IVM if current system version != given system + // TODO amazon-ion/ion-java/issues/24 Write IVM if current system version != given system // For now we assume that it's the same since we only support 1.0 return ! _anything_written; } @@ -291,7 +291,7 @@ final void writeSymbol(int symbolId) throws IOException && getDepth() == 0 && _annotation_count == 0) { // $ion_1_0 is written as an IVM only if it is not annotated - // TODO amzn/ion-java/issues/24 Make sure to get the right symtab, default may differ. + // TODO amazon-ion/ion-java/issues/24 Make sure to get the right symtab, default may differ. writeIonVersionMarker(); } else @@ -306,7 +306,7 @@ public final void writeSymbol(String value) throws IOException && getDepth() == 0 && _annotation_count == 0) { // $ion_1_0 is written as an IVM only if it is not annotated - // TODO amzn/ion-java/issues/24 Make sure to get the right symtab, default may differ. + // TODO amazon-ion/ion-java/issues/24 Make sure to get the right symtab, default may differ. writeIonVersionMarker(); } else { diff --git a/src/com/amazon/ion/impl/IonWriterSystemText.java b/src/com/amazon/ion/impl/IonWriterSystemText.java index 4c7b2c2192..8e1ea9b4a9 100644 --- a/src/com/amazon/ion/impl/IonWriterSystemText.java +++ b/src/com/amazon/ion/impl/IonWriterSystemText.java @@ -660,7 +660,7 @@ public void writeString(String value) && ! _following_long_string && _long_string_threshold < value.length()) { - // TODO amzn/ion-java/issues/57 This can lead to mixed newlines in the output. + // TODO amazon-ion/ion-java/issues/57 This can lead to mixed newlines in the output. // It assumes NL line separators, but _options could use CR+NL _output.printLongString(value); diff --git a/src/com/amazon/ion/impl/IonWriterUserBinary.java b/src/com/amazon/ion/impl/IonWriterUserBinary.java index ae1c072f3b..7db4c8fcc2 100644 --- a/src/com/amazon/ion/impl/IonWriterUserBinary.java +++ b/src/com/amazon/ion/impl/IonWriterUserBinary.java @@ -97,7 +97,7 @@ public void writeValue(IonReader reader) mySymtabExtendsCache.symtabsCompat(getSymbolTable(), reader.getSymbolTable()))) { - // TODO amzn/ion-java/issues/16 Doesn't copy annotations or field names. + // TODO amazon-ion/ion-java/issues/16 Doesn't copy annotations or field names. transfer.transferCurrentValue(myCopySink); return; } diff --git a/src/com/amazon/ion/impl/LocalSymbolTable.java b/src/com/amazon/ion/impl/LocalSymbolTable.java index b5990d5cce..56198844c1 100644 --- a/src/com/amazon/ion/impl/LocalSymbolTable.java +++ b/src/com/amazon/ion/impl/LocalSymbolTable.java @@ -257,7 +257,7 @@ protected static LocalSymbolTableImports readLocalSymbolTable(IonReader reader, sid = getSidForSymbolTableField(fieldName); } - // TODO amzn/ion-java/issues/36 Switching over SIDs doesn't cover the case + // TODO amazon-ion/ion-java/issues/36 Switching over SIDs doesn't cover the case // where the relevant field names are defined by a prev LST; // the prev LST could have 'symbols' defined locally with a // different SID! @@ -839,7 +839,7 @@ boolean symtabExtends(SymbolTable other) // Superset must have same/more known symbols than subset. if (getMaxId() < subset.getMaxId()) return false; - // TODO amzn/ion-java/issues/18 Currently, we check imports by their refs. which + // TODO amazon-ion/ion-java/issues/18 Currently, we check imports by their refs. which // might be overly strict; imports which are not the same ref. // but have the same semantic states fails the extension check. if (! myImportsList.equalImports(subset.myImportsList)) diff --git a/src/com/amazon/ion/impl/LocalSymbolTableImports.java b/src/com/amazon/ion/impl/LocalSymbolTableImports.java index b4b7b5e131..eb03bf0af9 100644 --- a/src/com/amazon/ion/impl/LocalSymbolTableImports.java +++ b/src/com/amazon/ion/impl/LocalSymbolTableImports.java @@ -29,7 +29,7 @@ *

    * This class is immutable, and hence safe for use by multiple threads. */ -// TODO amzn/ion-java/issues/37 Create specialized class to handle the common case where +// TODO amazon-ion/ion-java/issues/37 Create specialized class to handle the common case where // there are zero or one imported non-system shared symtab(s). final class LocalSymbolTableImports { diff --git a/src/com/amazon/ion/impl/SharedSymbolTable.java b/src/com/amazon/ion/impl/SharedSymbolTable.java index dedb85985f..02051143a5 100644 --- a/src/com/amazon/ion/impl/SharedSymbolTable.java +++ b/src/com/amazon/ion/impl/SharedSymbolTable.java @@ -69,7 +69,7 @@ final class SharedSymbolTable /** * The singleton instance of Ion 1.0 system symbol table. *

    - * TODO amzn/ion-java/issues/34 Optimize system symtabs by using our custom backing impl. + * TODO amazon-ion/ion-java/issues/34 Optimize system symtabs by using our custom backing impl. */ private static final SymbolTable ION_1_0_SYSTEM_SYMTAB; static @@ -238,9 +238,9 @@ static SymbolTable newSharedSymbolTable(IonReader reader, sid = getSidForSymbolTableField(fieldName); } - // TODO amzn/ion-java/issues/35 If there's more than one 'symbols' or 'imports' + // TODO amazon-ion/ion-java/issues/35 If there's more than one 'symbols' or 'imports' // field, they will be merged together. - // TODO amzn/ion-java/issues/36 Switching over SIDs doesn't cover the case + // TODO amazon-ion/ion-java/issues/36 Switching over SIDs doesn't cover the case // where the relevant field names are defined by a prev LST; // the prev LST could have 'symbols' defined locally with a // different SID! @@ -398,7 +398,7 @@ private static void putToMapIfNotThere(Map symbolsMap, while (symbols.hasNext()) { String text = symbols.next(); - // TODO amzn/ion-java/issues/12 What about empty symbols? + // TODO amazon-ion/ion-java/issues/12 What about empty symbols? if (symbolsMap.get(text) == null) { putToMapIfNotThere(symbolsMap, text, sid); diff --git a/src/com/amazon/ion/impl/SymbolTokenImpl.java b/src/com/amazon/ion/impl/SymbolTokenImpl.java index b464e1aef0..1df8a9c6d3 100644 --- a/src/com/amazon/ion/impl/SymbolTokenImpl.java +++ b/src/com/amazon/ion/impl/SymbolTokenImpl.java @@ -65,7 +65,7 @@ public String toString() } /* - * TODO amzn/ion-java#126 + * TODO amazon-ion/ion-java#126 *Equals and hashCode must be symmetric. *Two symboltokens are only equal when text1 equals text2 (including null == null) *This is an incomplete solution, needs to be updated as symboltokens are fleshed out. diff --git a/src/com/amazon/ion/impl/_Private_IonBinaryWriterBuilder.java b/src/com/amazon/ion/impl/_Private_IonBinaryWriterBuilder.java index 3465bd474d..85e4714fef 100644 --- a/src/com/amazon/ion/impl/_Private_IonBinaryWriterBuilder.java +++ b/src/com/amazon/ion/impl/_Private_IonBinaryWriterBuilder.java @@ -40,7 +40,7 @@ public class _Private_IonBinaryWriterBuilder extends IonBinaryWriterBuilder { - // amzn/ion-java/issues/59 expose configuration points properly and figure out deprecation path for the old writer. + // amazon-ion/ion-java/issues/59 expose configuration points properly and figure out deprecation path for the old writer. private final _Private_IonManagedBinaryWriterBuilder myBinaryWriterBuilder; private ValueFactory mySymtabValueFactory; @@ -298,7 +298,7 @@ private _Private_IonBinaryWriterBuilder fillDefaults() */ private _Private_IonBinaryWriterBuilder fillLegacyDefaults() { - // amzn/ion-java/issues/59 Fix this to use the new writer or eliminate it + // amazon-ion/ion-java/issues/59 Fix this to use the new writer or eliminate it // Ensure that we don't modify the user's builder. _Private_IonBinaryWriterBuilder b = copy(); @@ -375,7 +375,7 @@ public final IonWriter build(OutputStream out) @Deprecated public final IonBinaryWriter buildLegacy() { - // amzn/ion-java/issues/59 Fix this to use the new writer or eliminate it + // amazon-ion/ion-java/issues/59 Fix this to use the new writer or eliminate it _Private_IonBinaryWriterBuilder b = fillLegacyDefaults(); IonWriterSystemBinary systemWriter = diff --git a/src/com/amazon/ion/impl/_Private_IonConstants.java b/src/com/amazon/ion/impl/_Private_IonConstants.java index 38b9b1ee97..156e8600e6 100644 --- a/src/com/amazon/ion/impl/_Private_IonConstants.java +++ b/src/com/amazon/ion/impl/_Private_IonConstants.java @@ -291,7 +291,7 @@ public static final int getLowNibble(int td) * * *

    - * TODO amzn/ion-java/issues/23 However, there is still a potential failure if one of the + * TODO amazon-ion/ion-java/issues/23 However, there is still a potential failure if one of the * IonStruct's nested value has a field name with text * {@code " -- UNKNOWN SYMBOL TEXT -- $123"}, and that another nested value * of an IonStruct has a field name with unknown text and sid 123, these diff --git a/src/com/amazon/ion/impl/_Private_IonSystem.java b/src/com/amazon/ion/impl/_Private_IonSystem.java index d9024761f8..38fd3d81d0 100644 --- a/src/com/amazon/ion/impl/_Private_IonSystem.java +++ b/src/com/amazon/ion/impl/_Private_IonSystem.java @@ -36,12 +36,12 @@ public interface _Private_IonSystem public SymbolTable newSharedSymbolTable(IonStruct ionRep); /** - * TODO Must correct amzn/ion-java/issues/63 before exposing this or using from public API. + * TODO Must correct amazon-ion/ion-java/issues/63 before exposing this or using from public API. */ public Iterator systemIterate(String ionText); /** - * TODO Must correct amzn/ion-java/issues/63 before exposing this or using from public API. + * TODO Must correct amazon-ion/ion-java/issues/63 before exposing this or using from public API. */ public Iterator systemIterate(Reader ionText); diff --git a/src/com/amazon/ion/impl/_Private_Utils.java b/src/com/amazon/ion/impl/_Private_Utils.java index 09fff660d7..be806aeb72 100644 --- a/src/com/amazon/ion/impl/_Private_Utils.java +++ b/src/com/amazon/ion/impl/_Private_Utils.java @@ -208,7 +208,7 @@ public static SymbolTokenImpl newSymbolToken(int sid) public static SymbolToken newSymbolToken(SymbolTable symtab, String text) { - // TODO amzn/ion-java/issues/21 symtab should not be null + // TODO amazon-ion/ion-java/issues/21 symtab should not be null text.getClass(); // quick null check SymbolToken tok = (symtab == null ? null : symtab.find(text)); @@ -227,7 +227,7 @@ public static SymbolToken newSymbolToken(SymbolTable symtab, { if (sid < 1) throw new IllegalArgumentException(); - // TODO amzn/ion-java/issues/21 symtab should not be null + // TODO amazon-ion/ion-java/issues/21 symtab should not be null String text = (symtab == null ? null : symtab.findKnownSymbol(sid)); return new SymbolTokenImpl(text, sid); } @@ -288,7 +288,7 @@ public static SymbolToken localize(SymbolTable symtab, String text = sym.getText(); int sid = sym.getSid(); - if (symtab != null) // TODO amzn/ion-java/issues/21 require symtab + if (symtab != null) // TODO amazon-ion/ion-java/issues/21 require symtab { if (text == null) { @@ -975,7 +975,7 @@ public static boolean symtabExtends(SymbolTable superset, SymbolTable subset) // If the subset's symtab is a system symtab, the superset's is always // an extension of the subset's as system symtab-ness is irrelevant to // the conditions for copy opt. to be safe. - // TODO amzn/ion-java/issues/24 System symtab-ness ARE relevant if there's multiple + // TODO amazon-ion/ion-java/issues/24 System symtab-ness ARE relevant if there's multiple // versions. if (subset.isSystemTable()) return true; diff --git a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java index 8bc415829a..2a1923afcf 100644 --- a/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java +++ b/src/com/amazon/ion/impl/bin/IonRawBinaryWriter.java @@ -1360,7 +1360,7 @@ boolean isIVM(int sid) // When SID 2 occurs at the top level with no annotations, it has the // special properties of an IVM. Otherwise, it's considered a normal // symbol value. - // TODO amzn/ion-java/issues/88 requires this behavior to be changed, + // TODO amazon-ion/ion-java/issues/88 requires this behavior to be changed, // such that top-level SID 2 is treated as a symbol value, not an IVM. return depth == 0 && sid == ION_1_0_SID && !hasAnnotations(); } diff --git a/src/com/amazon/ion/impl/lite/IonContainerLite.java b/src/com/amazon/ion/impl/lite/IonContainerLite.java index e0276329a7..1cc365311c 100644 --- a/src/com/amazon/ion/impl/lite/IonContainerLite.java +++ b/src/com/amazon/ion/impl/lite/IonContainerLite.java @@ -78,7 +78,7 @@ protected IonContainerLite(ContainerlessContext context, boolean isNull) // when name is null it could be a sid 0 so we need to perform the full symbol token lookup. // this is expensive so only do it when necessary // TODO profile `getKnownFieldNameSymbol` to see if we can improve its performance so branching - // is not necessary. https://github.com/amzn/ion-java/issues/140 + // is not necessary. https://github.com/amazon-ion/ion-java/issues/140 copy.setFieldNameSymbol(child.getKnownFieldNameSymbol()); } else { diff --git a/src/com/amazon/ion/impl/lite/IonDatagramLite.java b/src/com/amazon/ion/impl/lite/IonDatagramLite.java index 9bb1a3d037..a9eebcd082 100644 --- a/src/com/amazon/ion/impl/lite/IonDatagramLite.java +++ b/src/com/amazon/ion/impl/lite/IonDatagramLite.java @@ -417,7 +417,7 @@ public final void writeTo(IonWriter writer) // refer to slots in shared symbol table imports declared by the discarded table, an // error will be raised unnecessarily. To avoid that, only write an IVM when the writer's // symbol table is already the system symbol table. - // TODO evaluate whether an IVM should ever be written here. amzn/ion-java#200 + // TODO evaluate whether an IVM should ever be written here. amazon-ion/ion-java#200 try { writer.writeSymbol(SystemSymbols.ION_1_0); } catch (IOException ioe) { diff --git a/src/com/amazon/ion/impl/lite/IonStructLite.java b/src/com/amazon/ion/impl/lite/IonStructLite.java index e8864163ba..e6884bd67d 100644 --- a/src/com/amazon/ion/impl/lite/IonStructLite.java +++ b/src/com/amazon/ion/impl/lite/IonStructLite.java @@ -44,7 +44,7 @@ final class IonStructLite { private static final int HASH_SIGNATURE = IonType.STRUCT.toString().hashCode(); - // TODO amzn/ion-java/issues/41: add support for _isOrdered + // TODO amazon-ion/ion-java/issues/41: add support for _isOrdered IonStructLite(ContainerlessContext context, boolean isNull) { diff --git a/src/com/amazon/ion/impl/lite/IonSymbolLite.java b/src/com/amazon/ion/impl/lite/IonSymbolLite.java index 01bf587afb..fe9b9351e8 100644 --- a/src/com/amazon/ion/impl/lite/IonSymbolLite.java +++ b/src/com/amazon/ion/impl/lite/IonSymbolLite.java @@ -65,11 +65,11 @@ final class IonSymbolLite if (text != null) { super.setValue(text); - // TODO [amzn/ion-java/issues/27] - needs consistent handling, when to retain SID's vs ignore + // TODO [amazon-ion/ion-java/issues/27] - needs consistent handling, when to retain SID's vs ignore } else { - // TODO [amzn/ion-java/issues/223] - needs consistent handling, resolution against context symbol table + // TODO [amazon-ion/ion-java/issues/223] - needs consistent handling, resolution against context symbol table _sid = sid; // there *is* an encoding present so we must update _isSymbolIdPresent(true); @@ -156,7 +156,7 @@ private int getSymbolId(SymbolTableProvider symbolTableProvider) assert(symtab != null); String name = _get_value(); - // TODO [amzn/ion-java/issues/27] - needs consistent handling, when to retain SID's vs ignore (here memoizing SID on read) + // TODO [amazon-ion/ion-java/issues/27] - needs consistent handling, when to retain SID's vs ignore (here memoizing SID on read) if (!symtab.isLocalTable()) { setSID(symtab.findSymbol(name)); @@ -262,7 +262,7 @@ else if (_stringValue() != null) } else { - // TODO [amzn/ion-java/issues/223] - needs consistent handling, resolution against context symbol table + // TODO [amazon-ion/ion-java/issues/223] - needs consistent handling, resolution against context symbol table // there is not text, so we can't clear the SID. allSymbolIDsCleared = false; } @@ -286,7 +286,7 @@ protected void setIsIonVersionMarker(boolean isIVM) final void writeBodyTo(IonWriter writer, SymbolTableProvider symbolTableProvider) throws IOException { - // TODO amzn/ion-java/issues/27 Fix symbol handling + // TODO amazon-ion/ion-java/issues/27 Fix symbol handling // A million-dollar question is - if text is missing, do // we throw (cannot serialize) or do we pass the sid thru??? diff --git a/src/com/amazon/ion/impl/lite/IonSystemLite.java b/src/com/amazon/ion/impl/lite/IonSystemLite.java index 415bb3933a..2bc8cec3bd 100644 --- a/src/com/amazon/ion/impl/lite/IonSystemLite.java +++ b/src/com/amazon/ion/impl/lite/IonSystemLite.java @@ -342,7 +342,7 @@ private IonValueLite load_value_helper(IonReader reader, boolean isTopLevel) v = newBool(reader.booleanValue()); break; case INT: - // TODO amzn/ion-java/issues/9 Inefficient since we can't determine the size + // TODO amazon-ion/ion-java/issues/9 Inefficient since we can't determine the size // of the integer in order to avoid making BigIntegers. v = newInt(reader.bigIntegerValue()); break; diff --git a/src/com/amazon/ion/impl/lite/IonValueLite.java b/src/com/amazon/ion/impl/lite/IonValueLite.java index 598c4a0a6a..203ecdf43d 100644 --- a/src/com/amazon/ion/impl/lite/IonValueLite.java +++ b/src/com/amazon/ion/impl/lite/IonValueLite.java @@ -310,7 +310,7 @@ public SymbolTable getSymbolTable() this._annotations[i] = _Private_Utils.newSymbolToken(text, UNKNOWN_SYMBOL_ID); } else { - // TODO - amzn/ion-java/issues/223 needs consistent handling, should attempt to resolve and if it cant; fail + // TODO - amazon-ion/ion-java/issues/223 needs consistent handling, should attempt to resolve and if it cant; fail this._annotations[i] = existing._annotations[i]; hasSIDsRetained |= this._annotations[i].getSid() > UNKNOWN_SYMBOL_ID; } @@ -325,7 +325,7 @@ public SymbolTable getSymbolTable() // existing 'read only' flag - we force the deep-copy back to being mutable clear_flag(IS_LOCKED); // whilst the clone *should* guarantee symbol context is purged, the annotation behavior existing above - // under the TO DO for amzn/ion-java/issues/223 does mean that SID context can be propogated through a clone, therefore + // under the TO DO for amazon-ion/ion-java/issues/223 does mean that SID context can be propogated through a clone, therefore // the encoding flag has to reflect this reality _isSymbolIdPresent(hasSIDsRetained); } @@ -467,7 +467,7 @@ public final int getFieldId() public SymbolToken getFieldNameSymbol() { - // TODO amzn/ion-java/issues/27 We should memoize the results of symtab lookups. + // TODO amazon-ion/ion-java/issues/27 We should memoize the results of symtab lookups. // BUT: that could cause thread-safety problems for read-only values. // I think makeReadOnly should populate the tokens fully // so that we only need to lookup from mutable instances. @@ -613,7 +613,7 @@ public final String getFieldName() if (_fieldName != null) return _fieldName; if (_fieldId <= 0) return null; - // TODO amzn/ion-java/issues/27 why no symtab lookup, like getFieldNameSymbol()? + // TODO amazon-ion/ion-java/issues/27 why no symtab lookup, like getFieldNameSymbol()? throw new UnknownSymbolException(_fieldId); } @@ -683,7 +683,7 @@ public final SymbolToken[] getTypeAnnotationSymbols(SymbolTableProvider symbolTa String text = token.getText(); if (text != null && token.getSid() == UNKNOWN_SYMBOL_ID) { - // TODO amzn/ion-java/issues/27 We should memoize the result of symtab lookups + // TODO amazon-ion/ion-java/issues/27 We should memoize the result of symtab lookups // into _annotations. // See getFieldNameSymbol() for challenges doing so. diff --git a/src/com/amazon/ion/impl/lite/ReverseBinaryEncoder.java b/src/com/amazon/ion/impl/lite/ReverseBinaryEncoder.java index 250ca1e0f9..8db8187856 100644 --- a/src/com/amazon/ion/impl/lite/ReverseBinaryEncoder.java +++ b/src/com/amazon/ion/impl/lite/ReverseBinaryEncoder.java @@ -1161,7 +1161,7 @@ private void writeIonStructContent(IonStruct val) { final int originalOffset = myBuffer.length - myOffset; - // TODO amzn/ion-java/issues/31 should not preserve the ordering of fields + // TODO amazon-ion/ion-java/issues/31 should not preserve the ordering of fields ArrayList values = new ArrayList(); // Fill ArrayList with IonValues, the add() just copies the @@ -1182,7 +1182,7 @@ private void writeIonStructContent(IonStruct val) writeVarUInt(sid); } - // TODO amzn/ion-java/issues/41 Detect if the struct fields are sorted in ascending + // TODO amazon-ion/ion-java/issues/41 Detect if the struct fields are sorted in ascending // order of Sids. If so, 1 should be written into 'length' field. // Note that this 'length' field is not the same as the four-bit // length L in the type descriptor octet. @@ -1273,7 +1273,7 @@ private int findSid(SymbolToken symToken) * have different Ion versions. * * - * TODO amzn/ion-java/issues/25 Currently, {@link IonDatagram#systemIterator()} doesn't + * TODO amazon-ion/ion-java/issues/25 Currently, {@link IonDatagram#systemIterator()} doesn't * retain information about interspersed IVMs within the IonDatagram. * As such, we cannot obtain the location of interspersed IVMs, if any. * diff --git a/src/com/amazon/ion/impl/lite/TopLevelContext.java b/src/com/amazon/ion/impl/lite/TopLevelContext.java index 797ff6c9cc..a3b88b7776 100644 --- a/src/com/amazon/ion/impl/lite/TopLevelContext.java +++ b/src/com/amazon/ion/impl/lite/TopLevelContext.java @@ -40,7 +40,7 @@ final class TopLevelContext * symbol table and following the parent/owning_context * chain will lead to a system object. *

    - * TODO amzn/ion-java/issues/19 we cannot assume that the IonSystem knows the proper IVM + * TODO amazon-ion/ion-java/issues/19 we cannot assume that the IonSystem knows the proper IVM * in this context */ private final SymbolTable _symbols; diff --git a/src/com/amazon/ion/overview.html b/src/com/amazon/ion/overview.html index cf81a4bda3..7d993c1bdd 100644 --- a/src/com/amazon/ion/overview.html +++ b/src/com/amazon/ion/overview.html @@ -16,7 +16,7 @@ ion-java is the reference implementation of the -Ion data notation for the +Ion data notation for the JavaTM 2 Platform Standard Edition 5.0 and above.

    diff --git a/src/com/amazon/ion/system/IonTextWriterBuilder.java b/src/com/amazon/ion/system/IonTextWriterBuilder.java index 3f201c6137..b88fb4bca0 100644 --- a/src/com/amazon/ion/system/IonTextWriterBuilder.java +++ b/src/com/amazon/ion/system/IonTextWriterBuilder.java @@ -66,7 +66,7 @@ *

    * Currently, there is no configuration point available to disable the * auto-flushing mechanism. Please vote on - * issue amzn/ion-java/issues/32 + * issue amazon-ion/ion-java/issues/32 * if you require it. * */ diff --git a/src/com/amazon/ion/util/Equivalence.java b/src/com/amazon/ion/util/Equivalence.java index 70759b884f..371cb4c8b0 100644 --- a/src/com/amazon/ion/util/Equivalence.java +++ b/src/com/amazon/ion/util/Equivalence.java @@ -123,7 +123,7 @@ *

    Terminology

    * Within this class, strict equivalence refers to Ion data model * equivalence as defined above and by the - * Ion + * Ion * Specification. Structural or non-strict equivalence * follows the same rules as strict equivalence, except that *